diff --git a/.bumpversion.cfg b/.bumpversion.cfg --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.15.2 +current_version = 4.16.0 message = release: Bump version {current_version} to {new_version} [bumpversion:file:rhodecode/VERSION] diff --git a/.release.cfg b/.release.cfg --- a/.release.cfg +++ b/.release.cfg @@ -5,25 +5,20 @@ done = false done = true [task:rc_tools_pinned] -done = true [task:fixes_on_stable] -done = true [task:pip2nix_generated] -done = true [task:changelog_updated] -done = true [task:generate_api_docs] -done = true + +[task:updated_translation] [release] -state = prepared -version = 4.15.2 - -[task:updated_translation] +state = in_progress +version = 4.16.0 [task:generate_js_routes] diff --git a/LICENSE.txt b/LICENSE.txt --- a/LICENSE.txt +++ b/LICENSE.txt @@ -12,8 +12,6 @@ permission notice: file:licenses/msgpack_license.txt Copyright (c) 2009 - tornado file:licenses/tornado_license.txt - Copyright (c) 2015 - pygments-markdown-lexer - file:licenses/pygments_markdown_lexer_license.txt Copyright 2006 - diff_match_patch file:licenses/diff_match_patch_license.txt diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -.PHONY: clean docs docs-clean docs-cleanup test test-clean test-only test-only-postgres test-only-mysql web-build +.PHONY: clean docs docs-clean docs-cleanup test test-clean test-only test-only-postgres test-only-mysql web-build generate-pkgs NODE_PATH=./node_modules WEBPACK=./node_binaries/webpack @@ -8,7 +8,7 @@ GRUNT=./node_binaries/grunt clean: make test-clean - find . -type f \( -iname '*.c' -o -iname '*.pyc' -o -iname '*.so' \) -exec rm '{}' ';' + find . -type f \( -iname '*.c' -o -iname '*.pyc' -o -iname '*.so' -o -iname '*.orig' \) -exec rm '{}' ';' test: make test-clean @@ -51,3 +51,5 @@ docs-cleanup: web-build: NODE_PATH=$(NODE_PATH) $(GRUNT) +generate-pkgs: + nix-shell pkgs/shell-generate.nix --command "pip2nix generate --licenses" diff --git a/configs/development.ini b/configs/development.ini --- a/configs/development.ini +++ b/configs/development.ini @@ -53,7 +53,7 @@ asyncore_use_poll = true ## run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini #use = egg:gunicorn#main -## Sets the number of process workers. More workers means more concurent connections +## Sets the number of process workers. More workers means more concurrent connections ## RhodeCode can handle at the same time. Each additional worker also it increases ## memory usage as each has it's own set of caches. ## Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more @@ -133,10 +133,10 @@ rhodecode.api.url = /_admin/api ## `SignatureVerificationError` in case of wrong key, or damaged encryption data. #rhodecode.encrypted_values.strict = false -## return gzipped responses from Rhodecode (static files/application) +## return gzipped responses from RhodeCode (static files/application) gzip_responses = false -## autogenerate javascript routes file on startup +## auto-generate javascript routes file on startup generate_js_files = false ## System global default language. @@ -153,7 +153,7 @@ startup.import_repos = false ## the repository. #archive_cache_dir = /tmp/tarballcache -## URL at which the application is running. This is used for bootstraping +## URL at which the application is running. This is used for Bootstrapping ## requests in context when no web request is available. Used in ishell, or ## SSH calls. Set this for events to receive proper url for SSH calls. app.base_url = http://rhodecode.local @@ -203,7 +203,7 @@ gist_alias_url = ## used for access. ## Adding ?auth_token=TOKEN_HASH to the url authenticates this request as if it ## came from the the logged in user who own this authentication token. -## Additionally @TOKEN syntaxt can be used to bound the view to specific +## Additionally @TOKEN syntax can be used to bound the view to specific ## authentication token. Such view would be only accessible when used together ## with this authentication token ## @@ -227,14 +227,14 @@ default_encoding = UTF-8 ## instance-id prefix ## a prefix key for this instance used for cache invalidation when running -## multiple instances of rhodecode, make sure it's globally unique for -## all running rhodecode instances. Leave empty if you don't use it +## multiple instances of RhodeCode, make sure it's globally unique for +## all running RhodeCode instances. Leave empty if you don't use it instance_id = ## Fallback authentication plugin. Set this to a plugin ID to force the usage ## of an authentication plugin also if it is disabled by it's settings. ## This could be useful if you are unable to log in to the system due to broken -## authentication settings. Then you can enable e.g. the internal rhodecode auth +## authentication settings. Then you can enable e.g. the internal RhodeCode auth ## module to log in again and fix the settings. ## ## Available builtin plugin IDs (hash is part of the ID): @@ -250,7 +250,7 @@ instance_id = ## response is 401 HTTPUnauthorized. Currently HG clients have troubles with ## handling that causing a series of failed authentication calls. ## Set this variable to 403 to return HTTPForbidden, or any other HTTP code -## This will be served instead of default 401 on bad authnetication +## This will be served instead of default 401 on bad authentication auth_ret_code = ## use special detection method when serving auth_ret_code, instead of serving @@ -284,6 +284,13 @@ labs_settings_active = true ## This is used to store exception from RhodeCode in shared directory #exception_tracker.store_path = +## File store configuration. This is used to store and serve uploaded files +file_store.enabled = true +## Storage backend, available options are: local +file_store.backend = local +## path to store the uploaded binaries +file_store.storage_path = %(here)s/data/file_store + #################################### ### CELERY CONFIG #### @@ -325,6 +332,7 @@ rc_cache.cache_perms.expiration_time = 3 #rc_cache.cache_perms.arguments.host = localhost #rc_cache.cache_perms.arguments.port = 6379 #rc_cache.cache_perms.arguments.db = 0 +## more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends #rc_cache.cache_perms.arguments.distributed_lock = true ## `cache_repo` cache settings for FileTree, Readme, RSS FEEDS @@ -340,6 +348,7 @@ rc_cache.cache_repo.expiration_time = 25 #rc_cache.cache_repo.arguments.host = localhost #rc_cache.cache_repo.arguments.port = 6379 #rc_cache.cache_repo.arguments.db = 1 +## more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends #rc_cache.cache_repo.arguments.distributed_lock = true ## cache settings for SQL queries, this needs to use memory type backend @@ -424,7 +433,7 @@ channelstream.server = 127.0.0.1:9800 ## location of the channelstream server from outside world ## use ws:// for http or wss:// for https. This address needs to be handled ## by external HTTP server such as Nginx or Apache -## see nginx/apache configuration examples in our docs +## see Nginx/Apache configuration examples in our docs channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream channelstream.secret = secret channelstream.history.location = %(here)s/channelstream_history @@ -441,14 +450,14 @@ channelstream.proxy_path = /_channelstre ## Appenlight is tailored to work with RhodeCode, see ## http://appenlight.com for details how to obtain an account -## appenlight integration enabled +## Appenlight integration enabled appenlight = false appenlight.server_url = https://api.appenlight.com appenlight.api_key = YOUR_API_KEY #appenlight.transport_config = https://api.appenlight.com?threaded=1&timeout=5 -# used for JS client +## used for JS client appenlight.api_public_key = YOUR_API_PUBLIC_KEY ## TWEAK AMOUNT OF INFO SENT HERE @@ -473,7 +482,7 @@ appenlight.logging.level = WARNING ## (saves API quota for intensive logging) appenlight.logging_on_error = false -## list of additonal keywords that should be grabbed from environ object +## list of additional keywords that should be grabbed from environ object ## can be string with comma separated list of words in lowercase ## (by default client will always send following info: ## 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that @@ -533,7 +542,7 @@ sqlalchemy.db1.convert_unicode = true vcs.server.enable = true vcs.server = localhost:9900 -## Web server connectivity protocol, responsible for web based VCS operatations +## Web server connectivity protocol, responsible for web based VCS operations ## Available protocols are: ## `http` - use http-rpc backend (default) vcs.server.protocol = http @@ -582,7 +591,8 @@ svn.proxy.config_file_path = %(here)s/mo ## In most cases it should be set to `/`. svn.proxy.location_root = / ## Command to reload the mod dav svn configuration on change. -## Example: `/etc/init.d/apache2 reload` +## Example: `/etc/init.d/apache2 reload` or /home/USER/apache_reload.sh +## Make sure user who runs RhodeCode process is allowed to reload Apache #svn.proxy.reload_cmd = /etc/init.d/apache2 reload ## If the timeout expires before the reload command finishes, the command will ## be killed. Setting it to zero means no timeout. Defaults to 10 seconds. @@ -593,7 +603,7 @@ svn.proxy.location_root = / ############################################################ ## Defines if a custom authorized_keys file should be created and written on -## any change user ssh keys. Setting this to false also disables posibility +## any change user ssh keys. Setting this to false also disables possibility ## of adding SSH keys by users from web interface. Super admins can still ## manage SSH Keys. ssh.generate_authorized_keyfile = false @@ -601,13 +611,13 @@ ssh.generate_authorized_keyfile = false ## Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding` # ssh.authorized_keys_ssh_opts = -## Path to the authrozied_keys file where the generate entries are placed. +## Path to the authorized_keys file where the generate entries are placed. ## It is possible to have multiple key files specified in `sshd_config` e.g. ## AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode ssh.authorized_keys_file_path = ~/.ssh/authorized_keys_rhodecode ## Command to execute the SSH wrapper. The binary is available in the -## rhodecode installation directory. +## RhodeCode installation directory. ## e.g ~/.rccontrol/community-1/profile/bin/rc-ssh-wrapper ssh.wrapper_cmd = ~/.rccontrol/community-1/rc-ssh-wrapper @@ -615,7 +625,7 @@ ssh.wrapper_cmd = ~/.rccontrol/community ssh.wrapper_cmd_allow_shell = false ## Enables logging, and detailed output send back to the client during SSH -## operations. Usefull for debugging, shouldn't be used in production. +## operations. Useful for debugging, shouldn't be used in production. ssh.enable_debug_logging = true ## Paths to binary executable, by default they are the names, but we can @@ -624,6 +634,10 @@ ssh.executable.hg = ~/.rccontrol/vcsserv ssh.executable.git = ~/.rccontrol/vcsserver-1/profile/bin/git ssh.executable.svn = ~/.rccontrol/vcsserver-1/profile/bin/svnserve +## Enables SSH key generator web interface. Disabling this still allows users +## to add their own keys. +ssh.enable_ui_key_generator = true + ## Dummy marker to add new entries after. ## Add any custom entries below. Please don't remove. diff --git a/configs/production.ini b/configs/production.ini --- a/configs/production.ini +++ b/configs/production.ini @@ -53,7 +53,7 @@ port = 5000 ## run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini use = egg:gunicorn#main -## Sets the number of process workers. More workers means more concurent connections +## Sets the number of process workers. More workers means more concurrent connections ## RhodeCode can handle at the same time. Each additional worker also it increases ## memory usage as each has it's own set of caches. ## Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more @@ -108,10 +108,10 @@ use = egg:rhodecode-enterprise-ce ## `SignatureVerificationError` in case of wrong key, or damaged encryption data. #rhodecode.encrypted_values.strict = false -## return gzipped responses from Rhodecode (static files/application) +## return gzipped responses from RhodeCode (static files/application) gzip_responses = false -## autogenerate javascript routes file on startup +## auto-generate javascript routes file on startup generate_js_files = false ## System global default language. @@ -128,7 +128,7 @@ startup.import_repos = false ## the repository. #archive_cache_dir = /tmp/tarballcache -## URL at which the application is running. This is used for bootstraping +## URL at which the application is running. This is used for Bootstrapping ## requests in context when no web request is available. Used in ishell, or ## SSH calls. Set this for events to receive proper url for SSH calls. app.base_url = http://rhodecode.local @@ -178,7 +178,7 @@ gist_alias_url = ## used for access. ## Adding ?auth_token=TOKEN_HASH to the url authenticates this request as if it ## came from the the logged in user who own this authentication token. -## Additionally @TOKEN syntaxt can be used to bound the view to specific +## Additionally @TOKEN syntax can be used to bound the view to specific ## authentication token. Such view would be only accessible when used together ## with this authentication token ## @@ -202,14 +202,14 @@ default_encoding = UTF-8 ## instance-id prefix ## a prefix key for this instance used for cache invalidation when running -## multiple instances of rhodecode, make sure it's globally unique for -## all running rhodecode instances. Leave empty if you don't use it +## multiple instances of RhodeCode, make sure it's globally unique for +## all running RhodeCode instances. Leave empty if you don't use it instance_id = ## Fallback authentication plugin. Set this to a plugin ID to force the usage ## of an authentication plugin also if it is disabled by it's settings. ## This could be useful if you are unable to log in to the system due to broken -## authentication settings. Then you can enable e.g. the internal rhodecode auth +## authentication settings. Then you can enable e.g. the internal RhodeCode auth ## module to log in again and fix the settings. ## ## Available builtin plugin IDs (hash is part of the ID): @@ -225,7 +225,7 @@ instance_id = ## response is 401 HTTPUnauthorized. Currently HG clients have troubles with ## handling that causing a series of failed authentication calls. ## Set this variable to 403 to return HTTPForbidden, or any other HTTP code -## This will be served instead of default 401 on bad authnetication +## This will be served instead of default 401 on bad authentication auth_ret_code = ## use special detection method when serving auth_ret_code, instead of serving @@ -259,6 +259,13 @@ labs_settings_active = true ## This is used to store exception from RhodeCode in shared directory #exception_tracker.store_path = +## File store configuration. This is used to store and serve uploaded files +file_store.enabled = true +## Storage backend, available options are: local +file_store.backend = local +## path to store the uploaded binaries +file_store.storage_path = %(here)s/data/file_store + #################################### ### CELERY CONFIG #### @@ -300,6 +307,7 @@ rc_cache.cache_perms.expiration_time = 3 #rc_cache.cache_perms.arguments.host = localhost #rc_cache.cache_perms.arguments.port = 6379 #rc_cache.cache_perms.arguments.db = 0 +## more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends #rc_cache.cache_perms.arguments.distributed_lock = true ## `cache_repo` cache settings for FileTree, Readme, RSS FEEDS @@ -315,6 +323,7 @@ rc_cache.cache_repo.expiration_time = 25 #rc_cache.cache_repo.arguments.host = localhost #rc_cache.cache_repo.arguments.port = 6379 #rc_cache.cache_repo.arguments.db = 1 +## more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends #rc_cache.cache_repo.arguments.distributed_lock = true ## cache settings for SQL queries, this needs to use memory type backend @@ -399,7 +408,7 @@ channelstream.server = 127.0.0.1:9800 ## location of the channelstream server from outside world ## use ws:// for http or wss:// for https. This address needs to be handled ## by external HTTP server such as Nginx or Apache -## see nginx/apache configuration examples in our docs +## see Nginx/Apache configuration examples in our docs channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream channelstream.secret = secret channelstream.history.location = %(here)s/channelstream_history @@ -416,14 +425,14 @@ channelstream.proxy_path = /_channelstre ## Appenlight is tailored to work with RhodeCode, see ## http://appenlight.com for details how to obtain an account -## appenlight integration enabled +## Appenlight integration enabled appenlight = false appenlight.server_url = https://api.appenlight.com appenlight.api_key = YOUR_API_KEY #appenlight.transport_config = https://api.appenlight.com?threaded=1&timeout=5 -# used for JS client +## used for JS client appenlight.api_public_key = YOUR_API_PUBLIC_KEY ## TWEAK AMOUNT OF INFO SENT HERE @@ -448,7 +457,7 @@ appenlight.logging.level = WARNING ## (saves API quota for intensive logging) appenlight.logging_on_error = false -## list of additonal keywords that should be grabbed from environ object +## list of additional keywords that should be grabbed from environ object ## can be string with comma separated list of words in lowercase ## (by default client will always send following info: ## 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that @@ -506,7 +515,7 @@ sqlalchemy.db1.convert_unicode = true vcs.server.enable = true vcs.server = localhost:9900 -## Web server connectivity protocol, responsible for web based VCS operatations +## Web server connectivity protocol, responsible for web based VCS operations ## Available protocols are: ## `http` - use http-rpc backend (default) vcs.server.protocol = http @@ -555,7 +564,8 @@ svn.proxy.config_file_path = %(here)s/mo ## In most cases it should be set to `/`. svn.proxy.location_root = / ## Command to reload the mod dav svn configuration on change. -## Example: `/etc/init.d/apache2 reload` +## Example: `/etc/init.d/apache2 reload` or /home/USER/apache_reload.sh +## Make sure user who runs RhodeCode process is allowed to reload Apache #svn.proxy.reload_cmd = /etc/init.d/apache2 reload ## If the timeout expires before the reload command finishes, the command will ## be killed. Setting it to zero means no timeout. Defaults to 10 seconds. @@ -566,7 +576,7 @@ svn.proxy.location_root = / ############################################################ ## Defines if a custom authorized_keys file should be created and written on -## any change user ssh keys. Setting this to false also disables posibility +## any change user ssh keys. Setting this to false also disables possibility ## of adding SSH keys by users from web interface. Super admins can still ## manage SSH Keys. ssh.generate_authorized_keyfile = false @@ -574,13 +584,13 @@ ssh.generate_authorized_keyfile = false ## Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding` # ssh.authorized_keys_ssh_opts = -## Path to the authrozied_keys file where the generate entries are placed. +## Path to the authorized_keys file where the generate entries are placed. ## It is possible to have multiple key files specified in `sshd_config` e.g. ## AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode ssh.authorized_keys_file_path = ~/.ssh/authorized_keys_rhodecode ## Command to execute the SSH wrapper. The binary is available in the -## rhodecode installation directory. +## RhodeCode installation directory. ## e.g ~/.rccontrol/community-1/profile/bin/rc-ssh-wrapper ssh.wrapper_cmd = ~/.rccontrol/community-1/rc-ssh-wrapper @@ -588,7 +598,7 @@ ssh.wrapper_cmd = ~/.rccontrol/community ssh.wrapper_cmd_allow_shell = false ## Enables logging, and detailed output send back to the client during SSH -## operations. Usefull for debugging, shouldn't be used in production. +## operations. Useful for debugging, shouldn't be used in production. ssh.enable_debug_logging = false ## Paths to binary executable, by default they are the names, but we can @@ -597,6 +607,10 @@ ssh.executable.hg = ~/.rccontrol/vcsserv ssh.executable.git = ~/.rccontrol/vcsserver-1/profile/bin/git ssh.executable.svn = ~/.rccontrol/vcsserver-1/profile/bin/svnserve +## Enables SSH key generator web interface. Disabling this still allows users +## to add their own keys. +ssh.enable_ui_key_generator = true + ## Dummy marker to add new entries after. ## Add any custom entries below. Please don't remove. diff --git a/default.nix b/default.nix --- a/default.nix +++ b/default.nix @@ -19,14 +19,15 @@ # } args@ -{ pythonPackages ? "python27Packages" +{ system ? builtins.currentSystem +, pythonPackages ? "python27Packages" , pythonExternalOverrides ? self: super: {} , doCheck ? false , ... }: let - pkgs_ = (import {}); + pkgs_ = args.pkgs or (import { inherit system; }); in let @@ -80,7 +81,8 @@ let nodeEnv = import ./pkgs/node-default.nix { inherit - pkgs; + pkgs + system; }; nodeDependencies = nodeEnv.shell.nodeDependencies; diff --git a/docs/admin/backup-restore.rst b/docs/admin/backup-restore.rst --- a/docs/admin/backup-restore.rst +++ b/docs/admin/backup-restore.rst @@ -116,7 +116,7 @@ Full-text Search Backup You may also have full text search set up, but the index can be rebuild from re-imported |repos| if necessary. You will most likely want to backup your -:file:`mapping.ini` file if you've configured that. For more information, see +:file:`search_mapping.ini` file if you've configured that. For more information, see the :ref:`indexing-ref` section. Restoration Steps @@ -140,7 +140,7 @@ Post Restoration Steps Once you have restored your |RCE| instance to basic functionality, you can then work on restoring any specific setup changes you had made. -* To recreate the |RCE| index, use the backed up :file:`mapping.ini` file if +* To recreate the |RCE| index, use the backed up :file:`search_mapping.ini` file if you had made changes and rerun the indexer. See the :ref:`indexing-ref` section for details. * To reconfigure any extensions, copy the backed up extensions into the diff --git a/docs/admin/config-files-overview.rst b/docs/admin/config-files-overview.rst --- a/docs/admin/config-files-overview.rst +++ b/docs/admin/config-files-overview.rst @@ -23,9 +23,9 @@ sections. * :ref:`increase-gunicorn` * :ref:`x-frame` - \- **mapping.ini** + \- **search_mapping.ini** Default location: - :file:`/home/{user}/.rccontrol/{instance-id}/mapping.ini` + :file:`/home/{user}/.rccontrol/{instance-id}/search_mapping.ini` This file is used to control the |RCE| indexer. It comes configured to index your instance. To change the default configuration, see diff --git a/docs/admin/indexing.rst b/docs/admin/indexing.rst --- a/docs/admin/indexing.rst +++ b/docs/admin/indexing.rst @@ -3,35 +3,41 @@ Full-text Search ---------------- -By default RhodeCode is configured to use `Whoosh`_ to index |repos| and -provide full-text search. +RhodeCode provides a full text search capabilities to search inside file content, +commit message, and file paths. Indexing is not enabled by default and to use +full text search building an index is a pre-requisite. -|RCE| also provides support for `Elasticsearch`_ as a backend for scalable -search. See :ref:`enable-elasticsearch` for details. +By default RhodeCode is configured to use `Whoosh`_ to index |repos| and +provide full-text search. `Whoosh`_ works well for a small amount of data and +shouldn't be used in case of large code-bases and lots of repositories. + +|RCE| also provides support for `ElasticSearch 6`_ as a backend more for advanced +and scalable search. See :ref:`enable-elasticsearch` for details. Indexing ^^^^^^^^ -To run the indexer you need to use an |authtoken| with admin rights to all -|repos|. +To run the indexer you need to have an |authtoken| with admin rights to all |repos|. -To index new content added, you have the option to set the indexer up in a +To index repositories stored in RhodeCode, you have the option to set the indexer up in a number of ways, for example: -* Call the indexer via a cron job. We recommend running this nightly, - unless you need everything indexed immediately. -* Set the indexer to infinitely loop and reindex as soon as it has run its - cycle. +* Call the indexer via a cron job. We recommend running this once at night. + In case you need everything indexed immediately it's possible to index few + times during the day. Indexer has a special locking mechanism that won't allow + two instances of indexer running at once. It's safe to run it even every 1hr. +* Set the indexer to infinitely loop and reindex as soon as it has run its previous cycle. * Hook the indexer up with your CI server to reindex after each push. -The indexer works by indexing new commits added since the last run. If you -wish to build a brand new index from scratch each time, -use the ``force`` option in the configuration file. +The indexer works by indexing new commits added since the last run, and comparing +file changes to index only new or modified files. +If you wish to build a brand new index from scratch each time, use the ``force`` +option in the configuration file, or run it with --force flag. .. important:: You need to have |RCT| installed, see :ref:`install-tools`. Since |RCE| - 3.5.0 they are installed by default. + 3.5.0 they are installed by default and available with community/enterprise installations. To set up indexing, use the following steps: @@ -45,6 +51,13 @@ 4. :ref:`advanced-indexing` Configure the ``.rhoderc`` File ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. note:: + + Optionally it's possible to use indexer without the ``.rhoderc``. Simply instead of + executing with `--instance-name=enterprise-1` execute providing the host and token + directly: `--api-host=http://127.0.0.1:10000 --api-key=` + + |RCT| uses the :file:`/home/{user}/.rhoderc` file for connection details to |RCE| instances. If this file is not automatically created, you can configure it using the following example. You need to configure the @@ -56,11 +69,11 @@ details for each instance you want to in # of the instance you want to index $ rccontrol status - - NAME: enterprise-1 - - STATUS: RUNNING - - TYPE: Momentum - - VERSION: 1.5.0 - - URL: http://127.0.0.1:10000 + - NAME: enterprise-1 + - STATUS: RUNNING + - TYPE: Enterprise + - VERSION: 4.1.0 + - URL: http://127.0.0.1:10003 To get your API Token, on the |RCE| interface go to :menuselection:`username --> My Account --> Auth tokens` @@ -72,29 +85,35 @@ To get your API Token, on the |RCE| inte [instance:enterprise-1] api_host = http://127.0.0.1:10000 api_key = - repo_dir = /home//repos + .. _run-index: Run the Indexer ^^^^^^^^^^^^^^^ -Run the indexer using the following command, and specify the instance you -want to index: +Run the indexer using the following command, and specify the instance you want to index: .. code-block:: bash - # From inside a virtualevv - (venv)$ rhodecode-index --instance-name=enterprise-1 - - # Using default installation + # Using default simples indexing of all repositories $ /home/user/.rccontrol/enterprise-1/profile/bin/rhodecode-index \ --instance-name=enterprise-1 - # Using a custom mapping file + # Using a custom mapping file with indexing rules, and using elasticsearch 6 backend $ /home/user/.rccontrol/enterprise-1/profile/bin/rhodecode-index \ --instance-name=enterprise-1 \ - --mapping=/home/user/.rccontrol/enterprise-1/mapping.ini + --mapping=/home/user/.rccontrol/enterprise-1/search_mapping.ini \ + --es-version=6 --engine-location=http://elasticsearch-host:9200 + + # Using a custom mapping file and invocation without ``.rhoderc`` + $ /home/user/.rccontrol/enterprise-1/profile/bin/rhodecode-index \ + --api-host=http://rhodecodecode.myserver.com --api-key=xxxxx \ + --mapping=/home/user/.rccontrol/enterprise-1/search_mapping.ini + + # From inside a virtualev on your local machine or CI server. + (venv)$ rhodecode-index --instance-name=enterprise-1 + .. note:: @@ -136,119 +155,173 @@ 3. Save the file. # using a specially configured mapping file */15 * * * * ~/.rccontrol/enterprise-4/profile/bin/rhodecode-index \ --instance-name=enterprise-4 \ - --mapping=/home/user/.rccontrol/enterprise-4/mapping.ini + --mapping=/home/user/.rccontrol/enterprise-4/search_mapping.ini .. _advanced-indexing: Advanced Indexing ^^^^^^^^^^^^^^^^^ -|RCT| indexes based on the :file:`mapping.ini` file. To configure your index, -you can specify different options in this file. The default location is: + +Force Re-Indexing single repository ++++++++++++++++++++++++++++++++++++ + +Often it's required to re-index whole repository because of some repository changes, +or to remove some indexed secrets, or files. There's a special `--repo-name=` flag +for the indexer that limits execution to a single repository. For example to force-reindex +single repository such call can be made:: + + rhodecode-index --instance-name=enterprise-1 --force --repo-name=rhodecode-vcsserver + + +Removing repositories from index +++++++++++++++++++++++++++++++++ -* :file:`/home/{user}/.rccontrol/{instance-id}/mapping.ini`, using default - |RCT|. +The indexer automatically removes renamed repositories and builds index for new names. +In the same way if a listed repository in mapping.ini is not reported existing by the +server it's removed from the index. +In case that you wish to remove indexed repository manually such call would allow that:: + + rhodecode-index --instance-name=enterprise-1 --remove-only --repo-name=rhodecode-vcsserver + + +Using search_mapping.ini file for advanced index rules +++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +By default rhodecode-index runs for all repositories, all files with parsing limits +defined by the CLI default arguments. You can change those limits by calling with +different flags such as `--max-filesize=2048kb` or `--repo-limit=10` + +For more advanced execution logic it's possible to use a configuration file that +would define detailed rules which repositories and how should be indexed. + +|RCT| provides an example index configuration file called :file:`search_mapping.ini`. +This file is created by default during installation and is located at: + +* :file:`/home/{user}/.rccontrol/{instance-id}/search_mapping.ini`, using default |RCT|. * :file:`~/venv/lib/python2.7/site-packages/rhodecode_tools/templates/mapping.ini`, when using ``virtualenv``. .. note:: - If you need to create the :file:`mapping.ini` file, use the |RCT| - ``rhodecode-index --create-mapping path/to/file`` API call. For details, - see the :ref:`tools-cli` section. - -The indexer runs in a random order to prevent a failing |repo| from stopping -a build. To configure different indexing scenarios, set the following options -inside the :file:`mapping.ini` and specify the altered file using the -``--mapping`` option. + If you need to create the :file:`search_mapping.ini` file manually, use the |RCT| + ``rhodecode-index --create-mapping path/to/search_mapping.ini`` API call. + For details, see the :ref:`tools-cli` section. -* ``index_files`` : Index the specified file types. -* ``skip_files`` : Do not index the specified file types. -* ``index_files_content`` : Index the content of the specified file types. -* ``skip_files_content`` : Do not index the content of the specified files. -* ``force`` : Create a fresh index on each run. -* ``max_filesize`` : Files larger than the set size will not be indexed. -* ``commit_parse_limit`` : Set the batch size when indexing commit messages. - Set to a lower number to lessen memory load. -* ``repo_limit`` : Set the maximum number or |repos| indexed per run. -* ``[INCLUDE]`` : Set |repos| you want indexed. This takes precedent over - ``[EXCLUDE]``. -* ``[EXCLUDE]`` : Set |repos| you do not want indexed. Exclude can be used to - not index branches, forks, or log |repos|. +To Run the indexer with mapping file provide it using `--mapping` flag:: -At the end of the file you can specify conditions for specific |repos| that -will override the default values. To configure your indexer, -use the following example :file:`mapping.ini` file. + rhodecode-index --instance-name=enterprise-1 --mapping=/my/path/search_mapping.ini + + +Here's a detailed example of using :file:`search_mapping.ini` file. .. code-block:: ini [__DEFAULT__] - # default patterns for indexing files and content of files. - # Binary files are skipped by default. + ; Create index on commits data, and files data in this order. Available options + ; are `commits`, `files` + index_types = commits,files + + ; Commit fetch limit. In what amount of chunks commits should be fetched + ; via api and parsed. This allows server to transfer smaller chunks and be less loaded + commit_fetch_limit = 1000 - # Index python and markdown files - index_files = *.py, *.md + ; Commit process limit. Limit the number of commits indexer should fetch, and + ; store inside the full text search index. eg. if repo has 2000 commits, and + ; limit is 1000, on the first run it will process commits 0-1000 and on the + ; second 1000-2000 commits. Help reduce memory usage, default is 50000 + ; (set -1 for unlimited) + commit_process_limit = 20000 - # Do not index these file types - skip_files = *.svg, *.log, *.dump, *.txt + ; Limit of how many repositories each run can process, default is -1 (unlimited) + ; in case of 1000s of repositories it's better to execute in chunks to not overload + ; the server. + repo_limit = -1 - # Index both file types and their content - index_files_content = *.cpp, *.ini, *.py + ; Default patterns for indexing files and content of files. Binary files + ; are skipped by default. + + ; Add to index those comma separated files; globs syntax + ; e.g index_files = *.py, *.c, *.h, *.js + index_files = *, + + ; Do not add to index those comma separated files, this excludes + ; both search by name and content; globs syntax + ; e.g index_files = *.key, *.sql, *.xml, *.pem, *.crt + skip_files = , - # Index file names, but not file content - skip_files_content = *.svg, + ; Add to index content of those comma separated files; globs syntax + ; e.g index_files = *.h, *.obj + index_files_content = *, - # Force rebuilding an index from scratch. Each repository will be rebuild - # from scratch with a global flag. Use local flag to rebuild single repos + ; Do not add to index content of those comma separated files; globs syntax + ; Binary files are not indexed by default. + ; e.g index_files = *.min.js, *.xml, *.dump, *.log, *.dump + skip_files_content = , + + ; Force rebuilding an index from scratch. Each repository will be rebuild from + ; scratch with a global flag. Use --repo-name=NAME --force to rebuild single repo force = false - # Do not index files larger than 385KB - max_filesize = 385KB + ; maximum file size that indexer will use, files above that limit are not going + ; to have they content indexed. + ; Possible options are KB (kilobytes), MB (megabytes), eg 1MB or 1024KB + max_filesize = 10MB - # Limit commit indexing to 500 per batch - commit_parse_limit = 500 - - # Limit each index run to 25 repos - repo_limit = 25 - # __INCLUDE__ is more important that __EXCLUDE__. - - [__INCLUDE__] - # Include all repos with these names + [__INDEX_RULES__] + ; Ordered match rules for repositories. A list of all repositories will be fetched + ; using API and this list will be filtered using those rules. + ; Syntax for entry: `glob_pattern_OR_full_repo_name = 0 OR 1` where 0=exclude, 1=include + ; When this ordered list is traversed first match will return the include/exclude marker + ; For example: + ; upstream/binary_repo = 0 + ; upstream/subrepo/xml_files = 0 + ; upstream/* = 1 + ; special-repo = 1 + ; * = 0 + ; This will index all repositories under upstream/*, but skip upstream/binary_repo + ; and upstream/sub_repo/xml_files, last * = 0 means skip all other matches - docs/* = 1 - lib/* = 1 - - [__EXCLUDE__] - # Do not include the following repo in index - dev-docs/* = 1 - legacy-repos/* = 1 - *-dev/* = 1 - - # Each repo that needs special indexing is a separate section below. - # In each section set the options to override the global configuration - # parameters above. - # If special settings are not configured, the global configuration values - # above are inherited. If no special repositories are - # defined here RhodeCode will use the API to ask for all repositories + ; == EXPLICIT REPOSITORY INDEXING == + ; If defined this will skip using __INDEX_RULES__, and will not use API to fetch + ; list of repositories, it will explicitly take names defined with [NAME] format and + ; try to build the index, to build index just for repo_name_1 and special-repo use: + ; [repo_name_1] + ; [special-repo] - # For this repo use different settings - [special-repo] - commit_parse_limit = 20, - skip_files = *.idea, *.xml, + ; == PER REPOSITORY CONFIGURATION == + ; This allows overriding the global configuration per repository. + ; example to set specific file limit, and skip certain files for repository special-repo + ; the CLI flags doesn't override the conf settings. + ; [conf:special-repo] + ; max_filesize = 5mb + ; skip_files = *.xml, *.sql + - # For another repo use different settings - [another-special-repo] - index_files = *, - max_filesize = 800MB - commit_parse_limit = 20000 + +In case of 1000s of repositories it can be tricky to write the include/exclude rules at first. +There's a special flag to test the mapping file rules and list repositories that would +be indexed. Run the indexer with `--show-matched-repos` to list only the +match repositories defined in .ini file rules:: + + rhodecode-index --instance-name=enterprise-1 --show-matched-repos --mapping=/my/path/search_mapping.ini + .. _enable-elasticsearch: -Enabling Elasticsearch +Enabling ElasticSearch ^^^^^^^^^^^^^^^^^^^^^^ +ElasticSearch is available in EE edition only. It provides much scalable and more advanced +search capabilities. While Whoosh is fine for upto 1-2GB of data, beyond that amount it +starts slowing down, and can cause other problems. +New ElasticSearch 6 also provides much more advanced query language. +It allows advanced filtering by file paths, extensions, use OR statements, ranges etc. +Please check query language examples in the search field for some advanced query language usage. + + 1. Open the :file:`rhodecode.ini` file for the instance you wish to edit. The default location is :file:`home/{user}/.rccontrol/{instance-id}/rhodecode.ini` @@ -268,9 +341,19 @@ and change it to: .. code-block:: ini search.module = rc_elasticsearch - search.location = http://localhost:9200/ + search.location = http://localhost:9200 + ## specify Elastic Search version, 6 for latest or 2 for legacy + search.es_version = 6 + +where ``search.location`` points to the ElasticSearch server +by default running on port 9200. -where ``search.location`` points to the elasticsearch server. +Index invocation also needs change. Please provide --es-version= and +--engine-location= parameters to define ElasticSearch server location and it's version. +For example:: + + rhodecode-index --instace-name=enterprise-1 --es-version=6 --engine-location=http://localhost:9200 + .. _Whoosh: https://pypi.python.org/pypi/Whoosh/ -.. _Elasticsearch: https://www.elastic.co/ \ No newline at end of file +.. _ElasticSearch 6: https://www.elastic.co/ diff --git a/docs/admin/nginx-config-example.rst b/docs/admin/nginx-config-example.rst --- a/docs/admin/nginx-config-example.rst +++ b/docs/admin/nginx-config-example.rst @@ -9,6 +9,9 @@ Use the following example to configure N ## Rate limiter for certain pages to prevent brute force attacks limit_req_zone $binary_remote_addr zone=req_limit:10m rate=1r/s; + ## cache zone + proxy_cache_path /etc/nginx/nginx_cache levels=1:2 use_temp_path=off keys_zone=cache_zone:10m inactive=720h max_size=10g; + ## Custom log format log_format log_custom '$remote_addr - $remote_user [$time_local] ' '"$request" $status $body_bytes_sent ' @@ -141,6 +144,34 @@ Use the following example to configure N try_files $uri @rhodecode_http; } + ## Special Cache for file store, make sure you enable this intentionally as + ## it could bypass upload files permissions + # location /_file_store/download { + # + # proxy_cache cache_zone; + # # ignore Set-Cookie + # proxy_ignore_headers Set-Cookie; + # proxy_ignore_headers Cookie; + # + # proxy_cache_key $host$uri$is_args$args; + # proxy_cache_methods GET; + # + # proxy_cache_bypass $http_cache_control; + # proxy_cache_valid 200 302 720h; + # + # proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504; + # + # # returns cache status in headers + # add_header X-Proxy-Cache $upstream_cache_status; + # add_header Cache-Control "public"; + # + # proxy_cache_lock on; + # proxy_cache_lock_age 5m; + # + # proxy_pass http://rc; + # + # } + location / { try_files $uri @rhodecode_http; } diff --git a/docs/admin/repo-perm-steps.rst b/docs/admin/repo-perm-steps.rst --- a/docs/admin/repo-perm-steps.rst +++ b/docs/admin/repo-perm-steps.rst @@ -35,7 +35,7 @@ 2. On the |repo| group settings page you * :guilabel:`Owner`: Lets you change the group owner. Useful when users are moving roles within an organisation. -* :guilabel:`Group parent`: Lets you add the |repo| group as a sub-group +* :guilabel:`Repository group`: Lets you add the |repo| group as a sub-group of a larger group, i.e. :guilabel:`QA-Repos >> QA-Repos-Berlin` * :guilabel:`Enable automatic locking`: For more information, see :ref:`repo-locking` diff --git a/docs/admin/restore-deleted-repositories.rst b/docs/admin/restore-deleted-repositories.rst new file mode 100644 --- /dev/null +++ b/docs/admin/restore-deleted-repositories.rst @@ -0,0 +1,42 @@ +.. _restore-deleted-repositories: + +Restoring Deleted Repositories +============================== + +By default when repository or whole repository group is deleted an archived copy +of filesystem repositories are kept. You can see them as special entries in the +repository storage such as:: + + drwxrwxr-x 3 rcdev rcdev 4096 Dec 4 2017 rm__20171204_105727_400795__ce-import + drwxrwxr-x 6 rcdev rcdev 4096 Nov 21 2017 rm__20180221_152430_675047__svn-repo + drwxr-xr-x 7 rcdev rcdev 4096 Mar 28 2018 rm__20180328_143124_617576__test-git + drwxr-xr-x 7 rcdev rcdev 4096 Mar 28 2018 rm__20180328_144954_317729__test-git-install-hooks + + +Data from those repositories can be restored by simply removing the +`rm_YYYYDDMM_HHMMSS_DDDDDD__` prefix and additionally only in case of Mercurial +repositories remove the `.hg` store prefix.:: + + rm__.hg => .hg + + +For Git or SVN repositories this operation is not required. + +After removing the prefix repository can be brought by opening +:menuselection:`Admin --> Settings --> Remap and Rescan` and running `Rescan Filesystem` + +This will create a new DB entry restoring the data previously removed. +To restore OLD database entries this should be done by restoring from a Database backup. + +RhodeCode also keeps the repository group structure, this is marked by entries that +in addition have GROUP in the prefix, eg:: + + drwxr-xr-x 2 rcdev rcdev 4096 Jan 18 16:13 rm__20181130_120650_977082_GROUP_Test1 + drwxr-xr-x 2 rcdev rcdev 4096 Jan 18 16:13 rm__20181130_120659_922952_GROUP_Docs + + + +.. note:: + + RhodeCode Tools have a special cleanup tool for the archived repositories. Please + see :ref:`clean-up-cmds` diff --git a/docs/admin/system-admin.rst b/docs/admin/system-admin.rst --- a/docs/admin/system-admin.rst +++ b/docs/admin/system-admin.rst @@ -30,3 +30,5 @@ The following are the most common system enable-debug admin-tricks cleanup-cmds + restore-deleted-repositories + diff --git a/docs/admin/system-overview.rst b/docs/admin/system-overview.rst --- a/docs/admin/system-overview.rst +++ b/docs/admin/system-overview.rst @@ -78,7 +78,7 @@ Configuration Files ------------------- * :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` -* :file:`/home/{user}/.rccontrol/{instance-id}/mapping.ini` +* :file:`/home/{user}/.rccontrol/{instance-id}/search_mapping.ini` * :file:`/home/{user}/.rccontrol/{vcsserver-id}/vcsserver.ini` * :file:`/home/{user}/.rccontrol/supervisor/supervisord.ini` * :file:`/home/{user}/.rccontrol.ini` diff --git a/docs/api/api.rst b/docs/api/api.rst --- a/docs/api/api.rst +++ b/docs/api/api.rst @@ -196,7 +196,8 @@ are not required in args. .. --- API DEFS MARKER --- .. toctree:: - methods/views + methods/repo-methods + methods/store-methods methods/license-methods methods/deprecated-methods methods/gist-methods diff --git a/docs/api/methods/pull-request-methods.rst b/docs/api/methods/pull-request-methods.rst --- a/docs/api/methods/pull-request-methods.rst +++ b/docs/api/methods/pull-request-methods.rst @@ -83,7 +83,7 @@ comment_pull_request create_pull_request ------------------- -.. py:function:: create_pull_request(apiuser, source_repo, target_repo, source_ref, target_ref, title=, description=, description_renderer=, reviewers=) +.. py:function:: create_pull_request(apiuser, source_repo, target_repo, source_ref, target_ref, owner=>, title=, description=, description_renderer=, reviewers=) Creates a new pull request. @@ -104,6 +104,8 @@ create_pull_request :type source_ref: str :param target_ref: Set the target ref name. :type target_ref: str + :param owner: user_id or username + :type owner: Optional(str) :param title: Optionally Set the pull request title, it's generated otherwise :type title: str :param description: Set the pull request description. @@ -248,7 +250,7 @@ get_pull_request_comments get_pull_requests ----------------- -.. py:function:: get_pull_requests(apiuser, repoid, status=) +.. py:function:: get_pull_requests(apiuser, repoid, status=, merge_state=) Get all pull requests from the repository specified in `repoid`. @@ -262,6 +264,9 @@ get_pull_requests * ``open`` * ``closed`` :type status: str + :param merge_state: Optional calculate merge state for each repository. + This could result in longer time to fetch the data + :type merge_state: bool Example output: @@ -356,10 +361,11 @@ merge_pull_request "id": , "result": { - "executed": "", - "failure_reason": "", - "merge_commit_id": "", - "possible": "", + "executed": "", + "failure_reason": "", + "merge_status_message": "", + "merge_commit_id": "", + "possible": "", "merge_ref": { "commit_id": "", "type": "", diff --git a/docs/api/methods/repo-methods.rst b/docs/api/methods/repo-methods.rst --- a/docs/api/methods/repo-methods.rst +++ b/docs/api/methods/repo-methods.rst @@ -394,15 +394,54 @@ get_repo_changesets of changed files. -get_repo_nodes --------------- +get_repo_comments +----------------- + +.. py:function:: get_repo_comments(apiuser, repoid, commit_id=, comment_type=, userid=) + + Get all comments for a repository -.. py:function:: get_repo_nodes(apiuser, repoid, revision, root_path, ret_type=, details=, max_file_bytes=) + :param apiuser: This is filled automatically from the |authtoken|. + :type apiuser: AuthUser + :param repoid: Set the repository name or repository ID. + :type repoid: str or int + :param commit_id: Optionally filter the comments by the commit_id + :type commit_id: Optional(str), default: None + :param comment_type: Optionally filter the comments by the comment_type + one of: 'note', 'todo' + :type comment_type: Optional(str), default: None + :param userid: Optionally filter the comments by the author of comment + :type userid: Optional(str or int), Default: None + + Example error output: + + .. code-block:: bash - Returns a list of nodes and children in a flat list for a given - path at given revision. + { + "id" : , + "result" : [ + { + "comment_author": , + "comment_created_on": "2017-02-01T14:38:16.309", + "comment_f_path": "file.txt", + "comment_id": 282, + "comment_lineno": "n1", + "comment_resolved_by": null, + "comment_status": [], + "comment_text": "This file needs a header", + "comment_type": "todo" + } + ], + "error" : null + } - It's possible to specify ret_type to show only `files` or `dirs`. + +get_repo_file +------------- + +.. py:function:: get_repo_file(apiuser, repoid, commit_id, file_path, max_file_bytes=, details=, cache=) + + Returns a single file from repository at given revision. This command can only be run using an |authtoken| with admin rights, or users with at least read rights to |repos|. @@ -411,37 +450,104 @@ get_repo_nodes :type apiuser: AuthUser :param repoid: The repository name or repository ID. :type repoid: str or int - :param revision: The revision for which listing should be done. - :type revision: str - :param root_path: The path from which to start displaying. - :type root_path: str - :param ret_type: Set the return type. Valid options are - ``all`` (default), ``files`` and ``dirs``. - :type ret_type: Optional(str) - :param details: Returns extended information about nodes, such as - md5, binary, and or content. The valid options are ``basic`` and - ``full``. + :param commit_id: The revision for which listing should be done. + :type commit_id: str + :param file_path: The path from which to start displaying. + :type file_path: str + :param details: Returns different set of information about nodes. + The valid options are ``minimal`` ``basic`` and ``full``. :type details: Optional(str) :param max_file_bytes: Only return file content under this file size bytes - :type details: Optional(int) - + :type max_file_bytes: Optional(int) + :param cache: Use internal caches for fetching files. If disabled fetching + files is slower but more memory efficient + :type cache: Optional(bool) Example output: .. code-block:: bash id : - result: [ - { - "name" : "" - "type" : "", - "binary": "" (only in extended mode) - "md5" : "" (only in extended mode) - }, - ... - ] + result: { + "binary": false, + "extension": "py", + "lines": 35, + "content": "....", + "md5": "76318336366b0f17ee249e11b0c99c41", + "mimetype": "text/x-python", + "name": "python.py", + "size": 817, + "type": "file", + } error: null +get_repo_fts_tree +----------------- + +.. py:function:: get_repo_fts_tree(apiuser, repoid, commit_id, root_path) + + Returns a list of tree nodes for path at given revision. This api is built + strictly for usage in full text search building, and shouldn't be consumed + + This command can only be run using an |authtoken| with admin rights, + or users with at least read rights to |repos|. + + +get_repo_nodes +-------------- + +.. py:function:: get_repo_nodes(apiuser, repoid, revision, root_path, ret_type=, details=, max_file_bytes=) + + Returns a list of nodes and children in a flat list for a given + path at given revision. + + It's possible to specify ret_type to show only `files` or `dirs`. + + This command can only be run using an |authtoken| with admin rights, + or users with at least read rights to |repos|. + + :param apiuser: This is filled automatically from the |authtoken|. + :type apiuser: AuthUser + :param repoid: The repository name or repository ID. + :type repoid: str or int + :param revision: The revision for which listing should be done. + :type revision: str + :param root_path: The path from which to start displaying. + :type root_path: str + :param ret_type: Set the return type. Valid options are + ``all`` (default), ``files`` and ``dirs``. + :type ret_type: Optional(str) + :param details: Returns extended information about nodes, such as + md5, binary, and or content. + The valid options are ``basic`` and ``full``. + :type details: Optional(str) + :param max_file_bytes: Only return file content under this file size bytes + :type details: Optional(int) + + Example output: + + .. code-block:: bash + + id : + result: [ + { + "binary": false, + "content": "File line + Line2 + ", + "extension": "md", + "lines": 2, + "md5": "059fa5d29b19c0657e384749480f6422", + "mimetype": "text/x-minidsrc", + "name": "file.md", + "size": 580, + "type": "file" + }, + ... + ] + error: null + + get_repo_refs ------------- diff --git a/docs/api/methods/server-methods.rst b/docs/api/methods/server-methods.rst --- a/docs/api/methods/server-methods.rst +++ b/docs/api/methods/server-methods.rst @@ -103,7 +103,7 @@ get_method :param apiuser: This is filled automatically from the |authtoken|. :type apiuser: AuthUser :param pattern: pattern to match method names against - :type older_then: Optional("*") + :type pattern: Optional("*") Example output: @@ -232,3 +232,37 @@ rescan_repos } +store_exception +--------------- + +.. py:function:: store_exception(apiuser, exc_data_json, prefix=) + + Stores sent exception inside the built-in exception tracker in |RCE| server. + + This command can only be run using an |authtoken| with admin rights to + the specified repository. + + This command takes the following options: + + :param apiuser: This is filled automatically from the |authtoken|. + :type apiuser: AuthUser + + :param exc_data_json: JSON data with exception e.g + {"exc_traceback": "Value `1` is not allowed", "exc_type_name": "ValueError"} + :type exc_data_json: JSON data + + :param prefix: prefix for error type, e.g 'rhodecode', 'vcsserver', 'rhodecode-tools' + :type prefix: Optional("rhodecode") + + Example output: + + .. code-block:: bash + + id : + "result": { + "exc_id": 139718459226384, + "exc_url": "http://localhost:8080/_admin/settings/exceptions/139718459226384" + } + error : null + + diff --git a/docs/api/methods/store-methods.rst b/docs/api/methods/store-methods.rst new file mode 100644 --- /dev/null +++ b/docs/api/methods/store-methods.rst @@ -0,0 +1,37 @@ +.. _store-methods-ref: + +store methods +============= + +file_store_add (EE only) +------------------------ + +.. py:function:: file_store_add(apiuser, filename, content) + + Upload API for the file_store + + Example usage from CLI:: + rhodecode-api --instance-name=enterprise-1 upload_file "{"content": "$(cat image.jpg | base64)", "filename":"image.jpg"}" + + This command takes the following options: + + :param apiuser: This is filled automatically from the |authtoken|. + :type apiuser: AuthUser + :param filename: name of the file uploaded + :type filename: str + :param content: base64 encoded content of the uploaded file + :type content: str + + Example output: + + .. code-block:: bash + + id : + result: { + "access_path": "/_file_store/download/84d156f7-8323-4ad3-9fce-4a8e88e1deaf-0.jpg", + "access_path_fqn": "http://server.domain.com/_file_store/download/84d156f7-8323-4ad3-9fce-4a8e88e1deaf-0.jpg", + "store_fid": "84d156f7-8323-4ad3-9fce-4a8e88e1deaf-0.jpg" + } + error : null + + diff --git a/docs/api/methods/views.rst b/docs/api/methods/views.rst deleted file mode 100644 --- a/docs/api/methods/views.rst +++ /dev/null @@ -1,48 +0,0 @@ -.. _views-ref: - -views -===== - -push (EE only) --------------- - -.. py:function:: push(apiuser, repoid, remote_uri=) - - Triggers a push on the given repository from a remote location. You - can use this to keep remote repositories up-to-date. - - This command can only be run using an |authtoken| with admin - rights to the specified repository. For more information, - see :ref:`config-token-ref`. - - This command takes the following options: - - :param apiuser: This is filled automatically from the |authtoken|. - :type apiuser: AuthUser - :param repoid: The repository name or repository ID. - :type repoid: str or int - :param remote_uri: Optional remote URI to pass in for push - :type remote_uri: str - - Example output: - - .. code-block:: bash - - id : - result : { - "msg": "Pushed to url `` on repo ``" - "repository": "" - } - error : null - - Example error output: - - .. code-block:: bash - - id : - result : null - error : { - "Unable to push changes to ``" - } - - diff --git a/docs/auth/auth-saml-bulk-enroll-users.rst b/docs/auth/auth-saml-bulk-enroll-users.rst new file mode 100644 --- /dev/null +++ b/docs/auth/auth-saml-bulk-enroll-users.rst @@ -0,0 +1,88 @@ +.. _auth-saml-bulk-enroll-users-ref: + + +Bulk enroll multiple existing users +----------------------------------- + + +RhodeCode Supports standard SAML 2.0 SSO for the web-application part. +Below is an example how to enroll list of all or some users to use SAML authentication. +This method simply enables SAML authentication for many users at once. + + +From the server RhodeCode Enterprise is running run ishell on the instance which we +want to apply the SAML migration:: + + rccontrol ishell enterprise-1 + +Follow these steps to enable SAML authentication for multiple users. + + +1) Create a user_id => attribute mapping + + +`saml2user` is a mapping of external ID from SAML provider such as OneLogin, DuoSecurity, Google. +This mapping consists of local rhodecode user_id mapped to set of required attributes needed to bind SAML +account to internal rhodecode user. +For example, 123 is local rhodecode user_id, and '48253211' is OneLogin ID. +For other providers you'd have to figure out what would be the user-id, sometimes it's the email, i.e for Google +The most important this id needs to be unique for each user. + +.. code-block:: python + + In [1]: saml2user = { + ...: # OneLogin, uses externalID available to read from in the UI + ...: 123: {'id: '48253211'}, + ...: # for Google/DuoSecurity email is also an option for unique ID + ...: 124: {'id: 'email@domain.com'}, + ...: } + + +2) Import the plugin you want to run migration for. + +From available options pick only one and run the `import` statement + +.. code-block:: python + + # for Duo Security + In [2]: from rc_auth_plugins.auth_duo_security import RhodeCodeAuthPlugin + # for OneLogin + In [2]: from rc_auth_plugins.auth_onelogin import RhodeCodeAuthPlugin + # generic SAML plugin + In [2]: from rc_auth_plugins.auth_saml import RhodeCodeAuthPlugin + +3) Run the migration based on saml2user mapping. + +Enter in the ishell prompt + +.. code-block:: python + + In [3]: for user in User.get_all(): + ...: existing_identity = ExternalIdentity().query().filter(ExternalIdentity.local_user_id == user.user_id).scalar() + ...: attrs = saml2user.get(user.user_id) + ...: provider = RhodeCodeAuthPlugin.uid + ...: if existing_identity: + ...: print('Identity for user `{}` already exists, skipping'.format(user.username)) + ...: continue + ...: if attrs: + ...: external_id = attrs['id'] + ...: new_external_identity = ExternalIdentity() + ...: new_external_identity.external_id = external_id + ...: new_external_identity.external_username = '{}-saml-{}'.format(user.username, user.user_id) + ...: new_external_identity.provider_name = provider + ...: new_external_identity.local_user_id = user_id + ...: new_external_identity.access_token = '' + ...: new_external_identity.token_secret = '' + ...: new_external_identity.alt_token = '' + ...: Session().add(ex_identity) + ...: Session().commit() + ...: print('Set user `{}` external identity bound to ExternalID:{}'.format(user.username, external_id)) + +.. note:: + + saml2user can be really big and hard to maintain in ishell. It's also possible + to load it as a JSON file prepared before and stored on disk. To do so run:: + + import json + saml2user = json.loads(open('/path/to/saml2user.json','rb').read()) + diff --git a/docs/auth/auth-saml-generic.rst b/docs/auth/auth-saml-generic.rst --- a/docs/auth/auth-saml-generic.rst +++ b/docs/auth/auth-saml-generic.rst @@ -15,4 +15,5 @@ Please check for reference two example p auth-saml-duosecurity auth-saml-onelogin + auth-saml-bulk-enroll-users diff --git a/docs/auth/ssh-connection.rst b/docs/auth/ssh-connection.rst --- a/docs/auth/ssh-connection.rst +++ b/docs/auth/ssh-connection.rst @@ -73,6 +73,10 @@ 2. Enable the SSH module on instance. ssh.executable.git = ~/.rccontrol/vcsserver-1/profile/bin/git ssh.executable.svn = ~/.rccontrol/vcsserver-1/profile/bin/svnserve + ## Enables SSH key generator web interface. Disabling this still allows users + ## to add their own keys. + ssh.enable_ui_key_generator = true + 3. Set base_url for instance to enable proper event handling (Optional): diff --git a/docs/contributing/dev-setup.rst b/docs/contributing/dev-setup.rst --- a/docs/contributing/dev-setup.rst +++ b/docs/contributing/dev-setup.rst @@ -52,6 +52,7 @@ run:: Followed by:: nix-channel --update + nix-env -i nix-2.0.4 Install required binaries @@ -65,6 +66,18 @@ run:: nix-env -i nix-prefetch-git +Speed up JS build by installing PhantomJS +----------------------------------------- + +PhantomJS will be downloaded each time nix-shell is invoked. To speed this by +setting already downloaded version do this:: + + nix-env -i phantomjs-2.1.1 + + # and set nix bin path + export PATH=$PATH:~/.nix-profile/bin + + Clone the required repositories ------------------------------- @@ -76,8 +89,8 @@ you have it installed before continuing. To obtain the required sources, use the following commands:: mkdir rhodecode-develop && cd rhodecode-develop - hg clone https://code.rhodecode.com/rhodecode-enterprise-ce - hg clone https://code.rhodecode.com/rhodecode-vcsserver + hg clone -u default https://code.rhodecode.com/rhodecode-enterprise-ce + hg clone -u default https://code.rhodecode.com/rhodecode-vcsserver .. note:: @@ -93,11 +106,15 @@ need to install the following. required libraries:: + # svn related sudo apt-get install libapr1-dev libaprutil1-dev sudo apt-get install libsvn-dev + # libcurl required too + sudo apt-get install libcurl4-openssl-dev + # mysql/pg server for development, optional sudo apt-get install mysql-server libmysqlclient-dev sudo apt-get install postgresql postgresql-contrib libpq-dev - sudo apt-get install libcurl4-openssl-dev + Enter the Development Shell @@ -106,7 +123,7 @@ Enter the Development Shell The final step is to start the development shells. To do this, run the following command from inside the cloned repository:: - #first, the vcsserver + # first, the vcsserver cd ~/rhodecode-vcsserver nix-shell @@ -182,7 +199,7 @@ To use the application's frontend and pr you will need to compile the CSS and JavaScript with Grunt. This is easily done from within the nix-shell using the following command:: - grunt + make web-build When developing new features you will need to recompile following any changes made to the CSS or JavaScript files when developing the code:: diff --git a/docs/integrations/integrations-rcextensions.rst b/docs/integrations/integrations-rcextensions.rst --- a/docs/integrations/integrations-rcextensions.rst +++ b/docs/integrations/integrations-rcextensions.rst @@ -18,11 +18,11 @@ Activating rcextensions To activate rcextensions simply copy or rename the created template rcextensions into the path where the rhodecode.ini file is located:: - pushd ~/rccontrol/enterprise-1/ + pushd ~/.rccontrol/enterprise-1/ or - pushd ~/rccontrol/community-1/ + pushd ~/.rccontrol/community-1/ - mv etc/rcextensions.tmpl rcextensions + mv profile/etc/rcextensions.tmpl rcextensions rcextensions are loaded when |RCE| starts. So a restart is required after activation or diff --git a/docs/release-notes/release-notes-4.14.0.rst b/docs/release-notes/release-notes-4.14.0.rst --- a/docs/release-notes/release-notes-4.14.0.rst +++ b/docs/release-notes/release-notes-4.14.0.rst @@ -104,7 +104,7 @@ Upgrade notes - In this release, we're shipping a new `rcextensions`. The changes made are backward incompatible. An update of `rcextensions` is required prior to using them again. Please check the new `rcextensions.tmpl` directory - located in `etc/rcextensions.tmpl` in your instance installation path. + located in `profile/etc/rcextensions.tmpl` in your instance installation path. Old code should be 100% portable by just copy&paste to the right function. - Mailing: We introduced a new mailing library. The older options should be compatible and diff --git a/docs/release-notes/release-notes-4.16.0.rst b/docs/release-notes/release-notes-4.16.0.rst new file mode 100644 --- /dev/null +++ b/docs/release-notes/release-notes-4.16.0.rst @@ -0,0 +1,148 @@ +|RCE| 4.16.0 |RNS| +------------------ + +Release Date +^^^^^^^^^^^^ + +- 2019-02-15 + + +New Features +^^^^^^^^^^^^ + + +- Full-text search: added support for ElasticSearch 6.X (ES6) +- Full-text search: Expose a quick way to search within repository groups using ES6. +- Full-text search: Add quick links to broaden/narrow search scope to repositories or + repository groups from global search. +- Full-text search: ES6 backend adds new highlighter, and search markers for better UX when searching. +- Full-text search: ES6 backend has enabled advanced `query string syntax` + adding more search and filtering capabilities. +- Full-text search: ES6 engine will now show added information where available such as line numbers file size. +- Files: added option to use highlight marker to show keywords inside file source. This + is used now for ES6 backend extended highlighting capabilities +- Artifacts (beta): EE edition exposes new feature called storage_api this allows storing + binary files outside of Version Control System, but in the scope of a repository or group. + This will soon become an Artifacts functionality available in EE edition. +- Authentication: introduced `User restriction` and `Scope restriction` for RhodeCode authentication plugins. + Admins can limit usage of RhodeCode plugins to super-admins user types, and usage in Web, or VCS protocol only. + This is mostly to help to migrate users to SAML, keeping the super-admins to manage instances via local-logins, + and secondly to force usage of AuthenticationTokens instead of re-using same credentials for + WEB and VCS authentication. +- API: added basic upload API for the storage_api. It's possible to store files using internal + API. This is a start for attachments upload in RhodeCode. +- API: added store_exception_api for remote exception storage. This is used by a new + indexer that will report any problems back into the RhodeCode instance in case of indexing problems. +- API: added function to fetch comments for a repository. +- Quick search: improve the styling of search input and results. +- Pull requests: allowed to select all forks and parent forks of target repository in creation UI. + This is a common workflow supported by GitHub etc. + + +General +^^^^^^^ + +- Users/Repositories/Repository groups: expose IDs of those objects in advanced views. + Useful for API calls or usage in ishell. +- UI: moved repo group select next to the name as it's very relevant to each other. +- Pull requests: increase the stability of concurrent pull requests created. +- Pull requests: introduced operation state for pull requests to prevent from + locks during merge/update operations in concurrent busy environments. +- Pull requests: ensure that merge response provide more details about failed operations. +- UI / Files: expose downloads options onto files view similar as in summary page. +- Repositories: show hooks version and update link in the advanced section of repository page. +- Events: trigger 'review_status_change' in all cases when reviewers are changed + influencing review status. +- Files: display submodules in a sorted way, equal to how Directories are sorted. +- API: fetching all pull-requests now sorts the results and exposed a flag to show/hide + the merge result state for faster result fetching. +- API: merge_pull_request expose detailed merge message in the merge operation + next to numeric merge response code. +- API: added possibility to specify owner to create_pull_request API. +- SSH: Added ability to disable server-side SSH key generation to enforce users + generated SSH keys only outside of the server. +- Integrations: allow PUT method for WebHook integration. +- Dependencies: bumped git to 2.19.2 release. +- Dependencies: dropped pygments-markdown-lexer as it's natively supported by pygments now. +- Dependencies: bumped pyramid to 1.10.1 +- Dependencies: bumped pastedeploy to 2.0.1 +- Dependencies: bumped pastescript to 3.0.0 +- Dependencies: bumped pathlib2 to 2.3.3 +- Dependencies: bumped webob to 1.8.4 +- Dependencies: bumped iso8601 to 0.1.12 +- Dependencies: bumped more-itertools to 5.0.0 +- Dependencies: bumped psutil to 5.4.8 +- Dependencies: bumped pyasn1 to 0.4.5 +- Dependencies: bumped pygments to 2.3.1 +- Dependencies: bumped pyramid-debugtoolbar to 4.5.0 +- Dependencies: bumped subprocess32 to 3.5.3 +- Dependencies: bumped supervisor to 3.3.5 +- Dependencies: bumped dogpile.cache to 0.7.1 +- Dependencies: bumped simplejson to 3.16.0 +- Dependencies: bumped gevent to 1.4.0 +- Dependencies: bumped configparser to 3.5.1 + + +Security +^^^^^^^^ + +- Fork page: don't expose fork origin link if we don't have permission to access this repository. + Additionally don't pre-select such repository in pull request ref selector. +- Security: fix possible XSS in the issue tracker URL. +- Security: sanitize plaintext renderer with bleach, preventing XSS in rendered html. +- Audit logs: added audit logs for API permission calls. + + +Performance +^^^^^^^^^^^ + +- Summary page: don't load repo size when showing expanded information about repository. + Size calculation needs to be triggered manually. +- Git: use rev-list for fetching last commit data in case of single commit history. + In some cases, it is much faster than previously used git log command. + + +Fixes +^^^^^ + +- Installer: fixed 32bit package builds broken in previous releases. +- Git: use iterative fetch to prevent errors about too many arguments on + synchronizing very large repositories. +- Git: pass in the SSL dir that is exposed from wire for remote GIT commands. +- LDAP+Groups: improve logging, and fix the case when extracting group name from LDAP + returned nothing. We should warn about that, but not FAIL on login. +- Default reviewers: fixed submodule support in picking reviewers from annotation for files. +- Hooks: handle non-ascii characters in hooks new pull-requests open template. +- Diffs: fixed missing limited diff container display on over-size limit diffs. +- Diffs: fixed 500 error in case of some very uncommon diffs containing only Unicode characters. +- Repositories: handle VCS backend unavailable correctly in advanced settings for the repository. +- Remap & rescan: prevent empty/damaged repositories to break the remap operation. +- Visual: fixed show revision/commit length settings. +- Mercurial submodules: only show submodule in the path that it belongs too. + Before even submodules from root node were shown in subdirectories. +- UI/Files: fixed icons in file tree search. +- WebHook integration: quote URL variables to prevent URL errors with special chars + like # in the title. +- API: pull-requests, fixed invocation of merge as another user. +- VCS: limit fd leaks on subprocessio calls. +- VCS: expose SSL certificate path over the wire to the vcsserver, this solves some + remote SSL import problems reported. + + +Upgrade notes +^^^^^^^^^^^^^ + +This release brings the new Full-text search capabilities using ElasticSearch 6. +If you use Elastic Search backend a backward compatibility mode is enabled and +ElasticSearch backend defaults to previously used ElasticSearch 2. + +To use new features a full index rebuild is required, in addition ```--es-version=6``` flag +needs to be used with indexer and ```search.es_version = 6``` should be set in rhodecode.ini + +Additionally new mapping format is available for the indexer that has additional capabilities +for include/exclude rules. Old format should work as well, but we encourage to +generate a new mapping.ini file using rhodecode-index command, and migrate your repositories +to the new format. + +Please refer to the :ref:`indexing-ref` documentation for more details. + diff --git a/docs/release-notes/release-notes.rst b/docs/release-notes/release-notes.rst --- a/docs/release-notes/release-notes.rst +++ b/docs/release-notes/release-notes.rst @@ -9,6 +9,7 @@ Release Notes .. toctree:: :maxdepth: 1 + release-notes-4.16.0.rst release-notes-4.15.2.rst release-notes-4.15.1.rst release-notes-4.15.0.rst diff --git a/docs/tools/tools-cli.rst b/docs/tools/tools-cli.rst --- a/docs/tools/tools-cli.rst +++ b/docs/tools/tools-cli.rst @@ -516,7 +516,7 @@ Example usage: $ ~/.rccontrol/enterprise-4/profile/bin/rhodecode-index \ --instance-name=enterprise-4 - # Run indexer based on mapping.ini file + # Run indexer based on search_mapping.ini file # This is using pre-350 virtualenv (venv)$ rhodecode-index --instance-name=enterprise-1 @@ -527,7 +527,7 @@ Example usage: # Create the indexing mapping file $ ~/.rccontrol/enterprise-4/profile/bin/rhodecode-index \ - --create-mapping mapping.ini --instance-name=enterprise-4 + --create-mapping search_mapping.ini --instance-name=enterprise-4 .. _tools-rhodecode-list-instance: diff --git a/grunt_config.json b/grunt_config.json --- a/grunt_config.json +++ b/grunt_config.json @@ -52,7 +52,7 @@ "<%= dirs.js.src %>/plugins/jquery.auto-grow-input.js", "<%= dirs.js.src %>/plugins/jquery.autocomplete.js", "<%= dirs.js.src %>/plugins/jquery.debounce.js", - "<%= dirs.js.src %>/plugins/jquery.mark.js", + "<%= dirs.js.node_modules %>/mark.js/dist/jquery.mark.min.js", "<%= dirs.js.src %>/plugins/jquery.timeago.js", "<%= dirs.js.src %>/plugins/jquery.timeago-extension.js", "<%= dirs.js.src %>/select2/select2.js", diff --git a/package.json b/package.json --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "grunt-contrib-watch": "^0.6.1", "grunt-webpack": "^3.1.3", "jquery": "1.11.3", + "mark.js": "8.11.1", "jshint": "^2.9.1-rc3", "moment": "^2.18.1", "mousetrap": "^1.6.1", diff --git a/pkgs/node-packages.nix b/pkgs/node-packages.nix --- a/pkgs/node-packages.nix +++ b/pkgs/node-packages.nix @@ -409,13 +409,13 @@ let sha512 = "mJ3QKWtCchL1vhU/kZlJnLPuQZnlDOdZsyP0bbLWPGdYsQDnSBvyTLhzwBA3QAMlzEL9V4JHygEmK6/OTEyytA=="; }; }; - "@webcomponents/shadycss-1.6.0" = { + "@webcomponents/shadycss-1.7.1" = { name = "_at_webcomponents_slash_shadycss"; packageName = "@webcomponents/shadycss"; - version = "1.6.0"; - src = fetchurl { - url = "https://registry.npmjs.org/@webcomponents/shadycss/-/shadycss-1.6.0.tgz"; - sha512 = "iURGZZU6BaiRJtGgjMn208QxPkY11QwT/VmuHNa4Yb+kJxU/WODe4C8b0LDOtnk4KJzJg50hCfwvPRAjePEzbA=="; + version = "1.7.1"; + src = fetchurl { + url = "https://registry.npmjs.org/@webcomponents/shadycss/-/shadycss-1.7.1.tgz"; + sha512 = "6SZqLajRPWL0rrKDZOGF8PCBq5B9JqgFmE5rX5psk6i8WrqiMkSCuO8+rnirzViTsU5CqnjQPFC3OvG4YJdMrQ=="; }; }; "@webcomponents/webcomponentsjs-2.2.1" = { @@ -499,13 +499,13 @@ let sha1 = "82ffb02b29e662ae53bdc20af15947706739c536"; }; }; - "ajv-6.6.1" = { + "ajv-6.6.2" = { name = "ajv"; packageName = "ajv"; - version = "6.6.1"; - src = fetchurl { - url = "https://registry.npmjs.org/ajv/-/ajv-6.6.1.tgz"; - sha512 = "ZoJjft5B+EJBjUyu9C9Hc0OZyPZSSlOF+plzouTrg6UlA8f+e/n8NIgBFG/9tppJtpPWfthHakK7juJdNDODww=="; + version = "6.6.2"; + src = fetchurl { + url = "https://registry.npmjs.org/ajv/-/ajv-6.6.2.tgz"; + sha512 = "FBHEW6Jf5TB9MGBgUUA9XHkTbjXYfAUjY43ACMfmdMRHniyoMHjHjzD50OK8LGDWQwp4rWEsIq5kEqq7rvIM1g=="; }; }; "ajv-keywords-3.2.0" = { @@ -815,15 +815,6 @@ let sha1 = "b6bbe0b0674b9d719708ca38de8c237cb526c3d1"; }; }; - "async-1.0.0" = { - name = "async"; - packageName = "async"; - version = "1.0.0"; - src = fetchurl { - url = "https://registry.npmjs.org/async/-/async-1.0.0.tgz"; - sha1 = "f8fc04ca3a13784ade9e1641af98578cfbd647a9"; - }; - }; "async-2.6.1" = { name = "async"; packageName = "async"; @@ -1445,6 +1436,15 @@ let sha512 = "+hN/Zh2D08Mx65pZ/4g5bsmNiZUuChDiQfTUQ7qJr4/kuopCr88xZsAXv6mBoZEsUI4OuGHlX59qE94K2mMW8Q=="; }; }; + "big.js-5.2.2" = { + name = "big.js"; + packageName = "big.js"; + version = "5.2.2"; + src = fetchurl { + url = "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz"; + sha512 = "vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ=="; + }; + }; "binary-extensions-1.12.0" = { name = "binary-extensions"; packageName = "binary-extensions"; @@ -1679,22 +1679,22 @@ let sha1 = "b534e7c734c4f81ec5fbe8aca2ad24354b962c6c"; }; }; - "caniuse-db-1.0.30000912" = { + "caniuse-db-1.0.30000927" = { name = "caniuse-db"; packageName = "caniuse-db"; - version = "1.0.30000912"; - src = fetchurl { - url = "https://registry.npmjs.org/caniuse-db/-/caniuse-db-1.0.30000912.tgz"; - sha512 = "uiepPdHcJ06Na9t15L5l+pp3NWQU4IETbmleghD6tqCqbIYqhHSu7nVfbK2gqPjfy+9jl/wHF1UQlyTszh9tJQ=="; - }; - }; - "caniuse-lite-1.0.30000912" = { + version = "1.0.30000927"; + src = fetchurl { + url = "https://registry.npmjs.org/caniuse-db/-/caniuse-db-1.0.30000927.tgz"; + sha512 = "CX/QvLA8oh7kQ9cHCCzFm0UZW4KwSyQSRJ5A1XtH42HaMJQ0yh+9fEVWagMqv9I1vSCtaqA5Mb8k0uKfv7jhDw=="; + }; + }; + "caniuse-lite-1.0.30000927" = { name = "caniuse-lite"; packageName = "caniuse-lite"; - version = "1.0.30000912"; - src = fetchurl { - url = "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30000912.tgz"; - sha512 = "M3zAtV36U+xw5mMROlTXpAHClmPAor6GPKAMD5Yi7glCB5sbMPFtnQ3rGpk4XqPdUrrTIaVYSJZxREZWNy8QJg=="; + version = "1.0.30000927"; + src = fetchurl { + url = "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30000927.tgz"; + sha512 = "ogq4NbUWf1uG/j66k0AmiO3GjqJAlQyF8n4w8a954cbCyFKmYGvRtgz6qkq2fWuduTXHibX7GyYL5Pg58Aks2g=="; }; }; "caseless-0.12.0" = { @@ -1733,13 +1733,13 @@ let sha1 = "a8115c55e4a702fe4d150abd3872822a7e09fc98"; }; }; - "chalk-2.4.1" = { + "chalk-2.4.2" = { name = "chalk"; packageName = "chalk"; - version = "2.4.1"; - src = fetchurl { - url = "https://registry.npmjs.org/chalk/-/chalk-2.4.1.tgz"; - sha512 = "ObN6h1v2fTJSmUXoS3nMQ92LbDK9be4TV+6G+omQlGJFdcUX5heKi1LZ1YnRMIgwTLEj3E24bT6tYni50rlCfQ=="; + version = "2.4.2"; + src = fetchurl { + url = "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz"; + sha512 = "Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ=="; }; }; "chokidar-2.0.4" = { @@ -1958,15 +1958,6 @@ let sha1 = "2423fe6678ac0c5dae8852e5d0e5be08c997abcc"; }; }; - "colors-1.0.3" = { - name = "colors"; - packageName = "colors"; - version = "1.0.3"; - src = fetchurl { - url = "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz"; - sha1 = "0433f44d809680fdeb60ed260f1b0c262e82a40b"; - }; - }; "colors-1.1.2" = { name = "colors"; packageName = "colors"; @@ -1976,13 +1967,13 @@ let sha1 = "168a4701756b6a7f51a12ce0c97bfa28c084ed63"; }; }; - "colors-1.3.2" = { + "colors-1.3.3" = { name = "colors"; packageName = "colors"; - version = "1.3.2"; - src = fetchurl { - url = "https://registry.npmjs.org/colors/-/colors-1.3.2.tgz"; - sha512 = "rhP0JSBGYvpcNQj4s5AdShMeE5ahMop96cTeDl/v9qQQm2fYClE2QXZRi8wLzc+GmXSxdIqqbOIAhyObEXDbfQ=="; + version = "1.3.3"; + src = fetchurl { + url = "https://registry.npmjs.org/colors/-/colors-1.3.3.tgz"; + sha512 = "mmGt/1pZqYRjMxB1axhTo16/snVZ5krrKkcmMeVKxzECMMXoCgnvTPp10QgHfcbQZw8Dq2jMNG6je4JlWU0gWg=="; }; }; "combined-stream-1.0.7" = { @@ -2102,13 +2093,13 @@ let sha512 = "Y+SQCF+0NoWQryez2zXn5J5knmr9z/9qSQt7fbL78u83rxmigOy8X5+BFn8CFSuX+nKT8gpYwJX68ekqtQt6ZA=="; }; }; - "core-js-2.5.7" = { + "core-js-2.6.1" = { name = "core-js"; packageName = "core-js"; - version = "2.5.7"; - src = fetchurl { - url = "https://registry.npmjs.org/core-js/-/core-js-2.5.7.tgz"; - sha512 = "RszJCAxg/PP6uzXVXL6BsxSXx/B05oJAQ2vkJRjyjrEcNVycaqOmNb5OTxZPE3xa5gwZduqza6L9JOCenh/Ecw=="; + version = "2.6.1"; + src = fetchurl { + url = "https://registry.npmjs.org/core-js/-/core-js-2.6.1.tgz"; + sha512 = "L72mmmEayPJBejKIWe2pYtGis5r0tQ5NaJekdhyXgeMQTpJoBsH0NL4ElY2LfSoV15xeQWKQ+XTTOZdyero5Xg=="; }; }; "core-util-is-1.0.2" = { @@ -2246,15 +2237,6 @@ let sha1 = "ddd52c587033f49e94b71fc55569f252e8ff5f85"; }; }; - "cycle-1.0.3" = { - name = "cycle"; - packageName = "cycle"; - version = "1.0.3"; - src = fetchurl { - url = "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz"; - sha1 = "21e80b2be8580f98b468f379430662b046c34ad2"; - }; - }; "cyclist-0.2.2" = { name = "cyclist"; packageName = "cyclist"; @@ -2489,13 +2471,13 @@ let sha1 = "bd28773e2642881aec51544924299c5cd822185b"; }; }; - "domelementtype-1.3.0" = { + "domelementtype-1.3.1" = { name = "domelementtype"; packageName = "domelementtype"; - version = "1.3.0"; - src = fetchurl { - url = "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.0.tgz"; - sha1 = "b17aed82e8ab59e52dd9c19b1756e0fc187204c2"; + version = "1.3.1"; + src = fetchurl { + url = "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz"; + sha512 = "BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w=="; }; }; "domhandler-2.1.0" = { @@ -2552,13 +2534,13 @@ let sha1 = "3a83a904e54353287874c564b7549386849a98c9"; }; }; - "electron-to-chromium-1.3.85" = { + "electron-to-chromium-1.3.98" = { name = "electron-to-chromium"; packageName = "electron-to-chromium"; - version = "1.3.85"; - src = fetchurl { - url = "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.85.tgz"; - sha512 = "kWSDVVF9t3mft2OHVZy4K85X2beP6c6mFm3teFS/mLSDJpQwuFIWHrULCX+w6H1E55ZYmFRlT+ATAFRwhrYzsw=="; + version = "1.3.98"; + src = fetchurl { + url = "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.98.tgz"; + sha512 = "WIZdNuvE3dFr6kkPgv4d/cfswNZD6XbeLBM8baOIQTsnbf4xWrVEaLvp7oNnbnMWWXDqq7Tbv+H5JfciLTJm4Q=="; }; }; "elliptic-6.4.1" = { @@ -2633,13 +2615,13 @@ let sha512 = "MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg=="; }; }; - "es-abstract-1.12.0" = { + "es-abstract-1.13.0" = { name = "es-abstract"; packageName = "es-abstract"; - version = "1.12.0"; - src = fetchurl { - url = "https://registry.npmjs.org/es-abstract/-/es-abstract-1.12.0.tgz"; - sha512 = "C8Fx/0jFmV5IPoMOFPA9P9G5NtqW+4cOPit3MIuvR2t7Ag2K15EJTpxnHAYTzL+aYQJIESYeXZmDBfOBE1HcpA=="; + version = "1.13.0"; + src = fetchurl { + url = "https://registry.npmjs.org/es-abstract/-/es-abstract-1.13.0.tgz"; + sha512 = "vDZfg/ykNxQVwup/8E1BZhVzFfBxs9NqMzGcvIJrqg5k2/5Za2bWo40dK2J1pgLngZ7c+Shh8lwYtLGyrwPutg=="; }; }; "es-to-primitive-1.2.0" = { @@ -2651,15 +2633,6 @@ let sha512 = "qZryBOJjV//LaxLTV6UC//WewneB3LcXOL9NP++ozKVXsIIIpm/2c13UDiD9Jp2eThsecw9m3jPqDwTyobcdbg=="; }; }; - "es6-promise-4.2.5" = { - name = "es6-promise"; - packageName = "es6-promise"; - version = "4.2.5"; - src = fetchurl { - url = "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.5.tgz"; - sha512 = "n6wvpdE43VFtJq+lUDYDBFUwV8TZbuGXLV4D6wKafg13ldznKsyEvatubnmUe31zcvelSzOHF+XbaT+Bl9ObDg=="; - }; - }; "es6-templates-0.2.3" = { name = "es6-templates"; packageName = "es6-templates"; @@ -2777,13 +2750,13 @@ let sha512 = "/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA=="; }; }; - "execa-0.10.0" = { + "execa-1.0.0" = { name = "execa"; packageName = "execa"; - version = "0.10.0"; - src = fetchurl { - url = "https://registry.npmjs.org/execa/-/execa-0.10.0.tgz"; - sha512 = "7XOMnz8Ynx1gGo/3hyV9loYNPWM94jG3+3T3Y8tsfSstFmETmENCMU/A/zj8Lyaj1lkgEepKepvd6240tBRvlw=="; + version = "1.0.0"; + src = fetchurl { + url = "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz"; + sha512 = "adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA=="; }; }; "exit-0.1.2" = { @@ -2858,15 +2831,6 @@ let sha512 = "Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw=="; }; }; - "extract-zip-1.6.7" = { - name = "extract-zip"; - packageName = "extract-zip"; - version = "1.6.7"; - src = fetchurl { - url = "https://registry.npmjs.org/extract-zip/-/extract-zip-1.6.7.tgz"; - sha1 = "a840b4b8af6403264c8db57f4f1a74333ef81fe9"; - }; - }; "extsprintf-1.3.0" = { name = "extsprintf"; packageName = "extsprintf"; @@ -2876,15 +2840,6 @@ let sha1 = "96918440e3041a7a414f8c52e3c574eb3c3e1e05"; }; }; - "eyes-0.1.8" = { - name = "eyes"; - packageName = "eyes"; - version = "0.1.8"; - src = fetchurl { - url = "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz"; - sha1 = "62cf120234c683785d902348a800ef3e0cc20bc0"; - }; - }; "fast-deep-equal-2.0.1" = { name = "fast-deep-equal"; packageName = "fast-deep-equal"; @@ -2930,15 +2885,6 @@ let sha1 = "c14c5b3bf14d7417ffbfd990c0a7495cd9f337bc"; }; }; - "fd-slicer-1.0.1" = { - name = "fd-slicer"; - packageName = "fd-slicer"; - version = "1.0.1"; - src = fetchurl { - url = "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz"; - sha1 = "8b5bcbd9ec327c5041bf9ab023fd6750f1177e65"; - }; - }; "file-sync-cmp-0.1.1" = { name = "file-sync-cmp"; packageName = "file-sync-cmp"; @@ -3002,22 +2948,22 @@ let sha1 = "9326b1488c22d1a6088650a86901b2d9a90a2cbc"; }; }; - "fined-1.1.0" = { + "fined-1.1.1" = { name = "fined"; packageName = "fined"; - version = "1.1.0"; - src = fetchurl { - url = "https://registry.npmjs.org/fined/-/fined-1.1.0.tgz"; - sha1 = "b37dc844b76a2f5e7081e884f7c0ae344f153476"; - }; - }; - "flagged-respawn-1.0.0" = { + version = "1.1.1"; + src = fetchurl { + url = "https://registry.npmjs.org/fined/-/fined-1.1.1.tgz"; + sha512 = "jQp949ZmEbiYHk3gkbdtpJ0G1+kgtLQBNdP5edFP7Fh+WAYceLQz6yO1SBj72Xkg8GVyTB3bBzAYrHJVh5Xd5g=="; + }; + }; + "flagged-respawn-1.0.1" = { name = "flagged-respawn"; packageName = "flagged-respawn"; - version = "1.0.0"; - src = fetchurl { - url = "https://registry.npmjs.org/flagged-respawn/-/flagged-respawn-1.0.0.tgz"; - sha1 = "4e79ae9b2eb38bf86b3bb56bf3e0a56aa5fcabd7"; + version = "1.0.1"; + src = fetchurl { + url = "https://registry.npmjs.org/flagged-respawn/-/flagged-respawn-1.0.1.tgz"; + sha512 = "lNaHNVymajmk0OJMBn8fVUAU1BtDeKIqKoVhk4xAALB57aALg6b4W0MfJ/cUE0g9YBXy5XhSlPIpYIJ7HaY/3Q=="; }; }; "flatten-1.0.2" = { @@ -3092,15 +3038,6 @@ let sha1 = "8bfb5502bde4a4d36cfdeea007fcca21d7e382af"; }; }; - "fs-extra-1.0.0" = { - name = "fs-extra"; - packageName = "fs-extra"; - version = "1.0.0"; - src = fetchurl { - url = "https://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz"; - sha1 = "cd3ce5f7e7cb6145883fcae3191e9877f8587950"; - }; - }; "fs-write-stream-atomic-1.0.10" = { name = "fs-write-stream-atomic"; packageName = "fs-write-stream-atomic"; @@ -3155,13 +3092,13 @@ let sha512 = "3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w=="; }; }; - "get-stream-3.0.0" = { + "get-stream-4.1.0" = { name = "get-stream"; packageName = "get-stream"; - version = "3.0.0"; - src = fetchurl { - url = "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz"; - sha1 = "8e943d1358dc37555054ecbe2edb05aa174ede14"; + version = "4.1.0"; + src = fetchurl { + url = "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz"; + sha512 = "GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w=="; }; }; "get-value-2.0.6" = { @@ -3236,13 +3173,13 @@ let sha512 = "sKzpEkf11GpOFuw0Zzjzmt4B4UZwjOcG757PPvrfhxcLFbq0wpsgpOqxpxtxFiCG4DtG93M6XRVbF2oGdev7bg=="; }; }; - "global-modules-path-2.3.0" = { + "global-modules-path-2.3.1" = { name = "global-modules-path"; packageName = "global-modules-path"; - version = "2.3.0"; - src = fetchurl { - url = "https://registry.npmjs.org/global-modules-path/-/global-modules-path-2.3.0.tgz"; - sha512 = "HchvMJNYh9dGSCy8pOQ2O8u/hoXaL+0XhnrwH0RyLiSXMMTl9W3N6KUU73+JFOg5PGjtzl6VZzUQsnrpm7Szag=="; + version = "2.3.1"; + src = fetchurl { + url = "https://registry.npmjs.org/global-modules-path/-/global-modules-path-2.3.1.tgz"; + sha512 = "y+shkf4InI7mPRHSo2b/k6ix6+NLDtyccYv86whhxrSGX9wjPX1VMITmrDbE1eh7zkzhiWtW2sHklJYoQ62Cxg=="; }; }; "global-prefix-1.0.2" = { @@ -3533,22 +3470,13 @@ let sha1 = "5fc8686847ecd73499403319a6b0a3f3f6ae4918"; }; }; - "hash.js-1.1.5" = { + "hash.js-1.1.7" = { name = "hash.js"; packageName = "hash.js"; - version = "1.1.5"; - src = fetchurl { - url = "https://registry.npmjs.org/hash.js/-/hash.js-1.1.5.tgz"; - sha512 = "eWI5HG9Np+eHV1KQhisXWwM+4EPPYe5dFX1UZZH7k/E3JzDEazVH+VGlZi6R94ZqImq+A3D1mCEtrFIfg/E7sA=="; - }; - }; - "hasha-2.2.0" = { - name = "hasha"; - packageName = "hasha"; - version = "2.2.0"; - src = fetchurl { - url = "https://registry.npmjs.org/hasha/-/hasha-2.2.0.tgz"; - sha1 = "78d7cbfc1e6d66303fe79837365984517b2f6ee1"; + version = "1.1.7"; + src = fetchurl { + url = "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz"; + sha512 = "taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA=="; }; }; "hawk-3.1.3" = { @@ -4217,13 +4145,13 @@ let sha1 = "dd8b74278b27102d29df63eae28308a8cfa1b583"; }; }; - "js-base64-2.4.9" = { + "js-base64-2.5.0" = { name = "js-base64"; packageName = "js-base64"; - version = "2.4.9"; - src = fetchurl { - url = "https://registry.npmjs.org/js-base64/-/js-base64-2.4.9.tgz"; - sha512 = "xcinL3AuDJk7VSzsHgb9DvvIXayBbadtMZ4HFPx8rUszbW1MuNMlwYVC4zzCZ6e1sqZpnNS5ZFYOhXqA39T7LQ=="; + version = "2.5.0"; + src = fetchurl { + url = "https://registry.npmjs.org/js-base64/-/js-base64-2.5.0.tgz"; + sha512 = "wlEBIZ5LP8usDylWbDNhKPEFVFdI5hCHpnVoT/Ysvoi/PRhJENm/Rlh9TvjYB38HFfKZN7OzEbRjmjvLkFw11g=="; }; }; "js-tokens-3.0.2" = { @@ -4280,13 +4208,13 @@ let sha1 = "46c3fec8c1892b12b0833db9bc7622176dbab34b"; }; }; - "jshint-2.9.6" = { + "jshint-2.9.7" = { name = "jshint"; packageName = "jshint"; - version = "2.9.6"; - src = fetchurl { - url = "https://registry.npmjs.org/jshint/-/jshint-2.9.6.tgz"; - sha512 = "KO9SIAKTlJQOM4lE64GQUtGBRpTOuvbrRrSZw3AhUxMNG266nX9hK2cKA4SBhXOj0irJGyNyGSLT62HGOVDEOA=="; + version = "2.9.7"; + src = fetchurl { + url = "https://registry.npmjs.org/jshint/-/jshint-2.9.7.tgz"; + sha512 = "Q8XN38hGsVQhdlM+4gd1Xl7OB1VieSuCJf+fEJjpo59JH99bVJhXRXAh26qQ15wfdd1VPMuDWNeSWoNl53T4YA=="; }; }; "json-parse-better-errors-1.0.2" = { @@ -4343,13 +4271,13 @@ let sha1 = "1eade7acc012034ad84e2396767ead9fa5495821"; }; }; - "jsonfile-2.4.0" = { - name = "jsonfile"; - packageName = "jsonfile"; - version = "2.4.0"; - src = fetchurl { - url = "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz"; - sha1 = "3736a2b428b87bbda0cc83b53fa3d633a35c2ae8"; + "json5-1.0.1" = { + name = "json5"; + packageName = "json5"; + version = "1.0.1"; + src = fetchurl { + url = "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz"; + sha512 = "aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow=="; }; }; "jsonify-0.0.0" = { @@ -4370,15 +4298,6 @@ let sha1 = "313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2"; }; }; - "kew-0.7.0" = { - name = "kew"; - packageName = "kew"; - version = "0.7.0"; - src = fetchurl { - url = "https://registry.npmjs.org/kew/-/kew-0.7.0.tgz"; - sha1 = "79d93d2d33363d6fdd2970b335d9141ad591d79b"; - }; - }; "kind-of-3.2.2" = { name = "kind-of"; packageName = "kind-of"; @@ -4415,15 +4334,6 @@ let sha512 = "s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA=="; }; }; - "klaw-1.3.1" = { - name = "klaw"; - packageName = "klaw"; - version = "1.3.1"; - src = fetchurl { - url = "https://registry.npmjs.org/klaw/-/klaw-1.3.1.tgz"; - sha1 = "4088433b46b3b1ba259d78785d8e96f73ba02439"; - }; - }; "lazy-cache-1.0.4" = { name = "lazy-cache"; packageName = "lazy-cache"; @@ -4478,13 +4388,13 @@ let sha1 = "f86e6374d43205a6e6c60e9196f17c0299bfb348"; }; }; - "loader-utils-1.1.0" = { + "loader-utils-1.2.3" = { name = "loader-utils"; packageName = "loader-utils"; - version = "1.1.0"; - src = fetchurl { - url = "https://registry.npmjs.org/loader-utils/-/loader-utils-1.1.0.tgz"; - sha1 = "c98aef488bcceda2ffb5e2de646d6a754429f5cd"; + version = "1.2.3"; + src = fetchurl { + url = "https://registry.npmjs.org/loader-utils/-/loader-utils-1.2.3.tgz"; + sha512 = "fkpz8ejdnEMG3s37wGL07iSBDg99O9D5yflE9RGNH3hRdx9SOwYfnGYdZOUIZitN8E+E2vkq3MUMYMvPYl5ZZA=="; }; }; "locate-path-2.0.0" = { @@ -4676,6 +4586,15 @@ let sha1 = "ecdca8f13144e660f1b5bd41f12f3479d98dfb8f"; }; }; + "mark.js-8.11.1" = { + name = "mark.js"; + packageName = "mark.js"; + version = "8.11.1"; + src = fetchurl { + url = "https://registry.npmjs.org/mark.js/-/mark.js-8.11.1.tgz"; + sha1 = "180f1f9ebef8b0e638e4166ad52db879beb2ffc5"; + }; + }; "math-expression-evaluator-1.2.17" = { name = "math-expression-evaluator"; packageName = "math-expression-evaluator"; @@ -4820,6 +4739,15 @@ let sha1 = "857fcabfc3397d2625b8228262e86aa7a011b05d"; }; }; + "minimist-1.2.0" = { + name = "minimist"; + packageName = "minimist"; + version = "1.2.0"; + src = fetchurl { + url = "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz"; + sha1 = "a35008b20f41383eec1fb914f4cd5df79a264284"; + }; + }; "mississippi-2.0.0" = { name = "mississippi"; packageName = "mississippi"; @@ -4847,13 +4775,13 @@ let sha1 = "30057438eac6cf7f8c4767f38648d6697d75c903"; }; }; - "moment-2.22.2" = { + "moment-2.23.0" = { name = "moment"; packageName = "moment"; - version = "2.22.2"; - src = fetchurl { - url = "https://registry.npmjs.org/moment/-/moment-2.22.2.tgz"; - sha1 = "3c257f9839fc0e93ff53149632239eb90783ff66"; + version = "2.23.0"; + src = fetchurl { + url = "https://registry.npmjs.org/moment/-/moment-2.23.0.tgz"; + sha512 = "3IE39bHVqFbWWaPOMHZF98Q9c3LDKGTmypMiTM2QygGXXElkFWIH7GxfmlwmY2vwa+wmNsoYZmG2iusf1ZjJoA=="; }; }; "mousetrap-1.6.2" = { @@ -4883,13 +4811,13 @@ let sha1 = "5608aeadfc00be6c2901df5f9861788de0d597c8"; }; }; - "nan-2.11.1" = { + "nan-2.12.1" = { name = "nan"; packageName = "nan"; - version = "2.11.1"; - src = fetchurl { - url = "https://registry.npmjs.org/nan/-/nan-2.11.1.tgz"; - sha512 = "iji6k87OSXa0CcrLl9z+ZiYSuR2o+c0bGuNmXdrhTQTakxytAFsC56SArGYoiHlJlFoHSnvmhpceZJaXkVuOtA=="; + version = "2.12.1"; + src = fetchurl { + url = "https://registry.npmjs.org/nan/-/nan-2.12.1.tgz"; + sha512 = "JY7V6lRkStKcKTvHO5NVSQRv+RV+FIL5pvDoLiAtSL9pKlC5x9PKQcZDsq7m4FO4d57mkhC6Z+QhAh3Jdk5JFw=="; }; }; "nanomatch-1.2.13" = { @@ -5144,13 +5072,13 @@ let sha1 = "ffbc4988336e0e833de0c168c7ef152121aa7fb3"; }; }; - "os-locale-3.0.1" = { + "os-locale-3.1.0" = { name = "os-locale"; packageName = "os-locale"; - version = "3.0.1"; - src = fetchurl { - url = "https://registry.npmjs.org/os-locale/-/os-locale-3.0.1.tgz"; - sha512 = "7g5e7dmXPtzcP4bgsZ8ixDVqA7oWYuEz4lOSujeWyliPai4gfVDiFIcwBg3aGCPnmSGfzOKTK3ccPn0CKv3DBw=="; + version = "3.1.0"; + src = fetchurl { + url = "https://registry.npmjs.org/os-locale/-/os-locale-3.1.0.tgz"; + sha512 = "Z8l3R4wYWM40/52Z+S265okfFj8Kt2cC2MKY+xNi3kFs+XGI7WXu/I309QQQYbRW4ijiZ+yxs9pqEhJh0DqW3Q=="; }; }; "os-tmpdir-1.0.2" = { @@ -5207,13 +5135,13 @@ let sha512 = "vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q=="; }; }; - "p-limit-2.0.0" = { + "p-limit-2.1.0" = { name = "p-limit"; packageName = "p-limit"; - version = "2.0.0"; - src = fetchurl { - url = "https://registry.npmjs.org/p-limit/-/p-limit-2.0.0.tgz"; - sha512 = "fl5s52lI5ahKCernzzIyAP0QAZbGIovtVHGwpcu1Jr/EpzLVDI2myISHwGqK7m8uQFugVWSrbxH7XnhGtvEc+A=="; + version = "2.1.0"; + src = fetchurl { + url = "https://registry.npmjs.org/p-limit/-/p-limit-2.1.0.tgz"; + sha512 = "NhURkNcrVB+8hNfLuysU8enY5xn2KXphsHBaC2YmRNTZRc7RWusw6apSpdEj3jo4CMb6W9nrF6tTnsJsJeyu6g=="; }; }; "p-locate-2.0.0" = { @@ -5432,15 +5360,6 @@ let sha512 = "U/il5MsrZp7mGg3mSQfn742na2T+1/vHDCG5/iTI3X9MKUuYUZVLQhyRsg06mCgDBTd57TxzgZt7P+fYfjRLtA=="; }; }; - "pend-1.2.0" = { - name = "pend"; - packageName = "pend"; - version = "1.2.0"; - src = fetchurl { - url = "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz"; - sha1 = "7a57eb550a6783f9115331fcf4663d5c8e007a50"; - }; - }; "performance-now-0.2.0" = { name = "performance-now"; packageName = "performance-now"; @@ -5450,24 +5369,6 @@ let sha1 = "33ef30c5c77d4ea21c5a53869d91b56d8f2555e5"; }; }; - "phantom-4.0.12" = { - name = "phantom"; - packageName = "phantom"; - version = "4.0.12"; - src = fetchurl { - url = "https://registry.npmjs.org/phantom/-/phantom-4.0.12.tgz"; - sha512 = "Tz82XhtPmwCk1FFPmecy7yRGZG2btpzY2KI9fcoPT7zT9det0CcMyfBFPp1S8DqzsnQnm8ZYEfdy528mwVtksA=="; - }; - }; - "phantomjs-prebuilt-2.1.16" = { - name = "phantomjs-prebuilt"; - packageName = "phantomjs-prebuilt"; - version = "2.1.16"; - src = fetchurl { - url = "https://registry.npmjs.org/phantomjs-prebuilt/-/phantomjs-prebuilt-2.1.16.tgz"; - sha1 = "efd212a4a3966d3647684ea8ba788549be2aefef"; - }; - }; "pify-3.0.0" = { name = "pify"; packageName = "pify"; @@ -5477,24 +5378,6 @@ let sha1 = "e5a4acd2c101fdf3d9a4d07f0dbc4db49dd28176"; }; }; - "pinkie-2.0.4" = { - name = "pinkie"; - packageName = "pinkie"; - version = "2.0.4"; - src = fetchurl { - url = "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz"; - sha1 = "72556b80cfa0d48a974e80e77248e80ed4f7f870"; - }; - }; - "pinkie-promise-2.0.1" = { - name = "pinkie-promise"; - packageName = "pinkie-promise"; - version = "2.0.1"; - src = fetchurl { - url = "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz"; - sha1 = "2135d6dfa7a358c069ac9b178776288228450ffa"; - }; - }; "pkg-dir-2.0.0" = { name = "pkg-dir"; packageName = "pkg-dir"; @@ -5882,15 +5765,6 @@ let sha512 = "MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw=="; }; }; - "progress-1.1.8" = { - name = "progress"; - packageName = "progress"; - version = "1.1.8"; - src = fetchurl { - url = "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz"; - sha1 = "e260c78f6161cdd9b0e56cc3e0a85de17c7a57be"; - }; - }; "promise-7.3.1" = { name = "promise"; packageName = "promise"; @@ -5945,6 +5819,15 @@ let sha512 = "ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA=="; }; }; + "pump-3.0.0" = { + name = "pump"; + packageName = "pump"; + version = "3.0.0"; + src = fetchurl { + url = "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz"; + sha512 = "LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww=="; + }; + }; "pumpify-1.5.1" = { name = "pumpify"; packageName = "pumpify"; @@ -6278,15 +6161,6 @@ let sha1 = "c6928946a0e06c5f8d6f8a9333469ffda46298a0"; }; }; - "request-progress-2.0.1" = { - name = "request-progress"; - packageName = "request-progress"; - version = "2.0.1"; - src = fetchurl { - url = "https://registry.npmjs.org/request-progress/-/request-progress-2.0.1.tgz"; - sha1 = "5d36bb57961c673aa5b788dbc8141fdf23b44e08"; - }; - }; "require-directory-2.1.1" = { name = "require-directory"; packageName = "require-directory"; @@ -6305,13 +6179,13 @@ let sha1 = "97f717b69d48784f5f526a6c5aa8ffdda055a4d1"; }; }; - "resolve-1.8.1" = { + "resolve-1.9.0" = { name = "resolve"; packageName = "resolve"; - version = "1.8.1"; - src = fetchurl { - url = "https://registry.npmjs.org/resolve/-/resolve-1.8.1.tgz"; - sha512 = "AicPrAC7Qu1JxPCZ9ZgCZlY35QgFnNqc+0LtbRNxnVw4TXvjQ72wnuL9JQcEBgXkI9JM8MsT9kaQoHcpCRJOYA=="; + version = "1.9.0"; + src = fetchurl { + url = "https://registry.npmjs.org/resolve/-/resolve-1.9.0.tgz"; + sha512 = "TZNye00tI67lwYvzxCxHGjwTNlUV70io54/Ed4j6PscB8xVfuBJpRenI/o6dVk0cY0PYTY27AgCoGGxRnYuItQ=="; }; }; "resolve-cwd-2.0.0" = { @@ -6377,13 +6251,13 @@ let sha1 = "e439be2aaee327321952730f99a8929e4fc50582"; }; }; - "rimraf-2.6.2" = { + "rimraf-2.6.3" = { name = "rimraf"; packageName = "rimraf"; - version = "2.6.2"; - src = fetchurl { - url = "https://registry.npmjs.org/rimraf/-/rimraf-2.6.2.tgz"; - sha512 = "lreewLK/BlghmxtfH36YYVg1i8IAce4TI7oao75I1g245+6BctqTVQiBP3YUJ9C6DQOXJmkYR9X9fCLtCOJc5w=="; + version = "2.6.3"; + src = fetchurl { + url = "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz"; + sha512 = "mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA=="; }; }; "ripemd160-2.0.2" = { @@ -6467,13 +6341,13 @@ let sha512 = "RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg=="; }; }; - "serialize-javascript-1.5.0" = { + "serialize-javascript-1.6.1" = { name = "serialize-javascript"; packageName = "serialize-javascript"; - version = "1.5.0"; - src = fetchurl { - url = "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-1.5.0.tgz"; - sha512 = "Ga8c8NjAAp46Br4+0oZ2WxJCwIzwP60Gq1YPgU+39PiTVxyed/iKE/zyZI6+UlVYH5Q4PaQdHhcegIFPZTUfoQ=="; + version = "1.6.1"; + src = fetchurl { + url = "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-1.6.1.tgz"; + sha512 = "A5MOagrPFga4YaKQSWHryl7AXvbQkEqpw4NNYMTNYUNV51bA8ABHgYFpqKx+YFFrw59xMV1qGH1R4AgoNIVgCw=="; }; }; "set-blocking-2.0.0" = { @@ -6701,15 +6575,6 @@ let sha1 = "3e935d7ddd73631b97659956d55128e87b5084a3"; }; }; - "split-1.0.1" = { - name = "split"; - packageName = "split"; - version = "1.0.1"; - src = fetchurl { - url = "https://registry.npmjs.org/split/-/split-1.0.1.tgz"; - sha512 = "mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg=="; - }; - }; "split-string-3.1.0" = { name = "split-string"; packageName = "split-string"; @@ -6728,13 +6593,13 @@ let sha1 = "04e6926f662895354f3dd015203633b857297e2c"; }; }; - "sshpk-1.15.2" = { + "sshpk-1.16.0" = { name = "sshpk"; packageName = "sshpk"; - version = "1.15.2"; - src = fetchurl { - url = "https://registry.npmjs.org/sshpk/-/sshpk-1.15.2.tgz"; - sha512 = "Ra/OXQtuh0/enyl4ETZAfTaeksa6BXks5ZcjpSUNrjBr0DvrJKX+1fsKDPpT9TBXgHAFsa4510aNVgI8g/+SzA=="; + version = "1.16.0"; + src = fetchurl { + url = "https://registry.npmjs.org/sshpk/-/sshpk-1.16.0.tgz"; + sha512 = "Zhev35/y7hRMcID/upReIvRse+I9SVhyVre/KTJSJQWMz3C3+G+HpO7m1wK/yckEtujKZ7dS4hkVxAnmHaIGVQ=="; }; }; "ssri-5.3.0" = { @@ -6746,15 +6611,6 @@ let sha512 = "XRSIPqLij52MtgoQavH/x/dU1qVKtWUAAZeOHsR9c2Ddi4XerFy3mc1alf+dLJKl9EUIm/Ht+EowFkTUOA6GAQ=="; }; }; - "stack-trace-0.0.10" = { - name = "stack-trace"; - packageName = "stack-trace"; - version = "0.0.10"; - src = fetchurl { - url = "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz"; - sha1 = "547c70b347e8d32b4e108ea1a2a159e5fdde19c0"; - }; - }; "static-extend-0.1.2" = { name = "static-extend"; packageName = "static-extend"; @@ -6989,15 +6845,6 @@ let sha512 = "9I2ydhj8Z9veORCw5PRm4u9uebCn0mcCa6scWoNcbZ6dAtoo2618u9UUzxgmsCOreJpqDDuv61LvwofW7hLcBA=="; }; }; - "throttleit-1.0.0" = { - name = "throttleit"; - packageName = "throttleit"; - version = "1.0.0"; - src = fetchurl { - url = "https://registry.npmjs.org/throttleit/-/throttleit-1.0.0.tgz"; - sha1 = "9e785836daf46743145a5984b6268d828528ac6c"; - }; - }; "through-2.3.8" = { name = "through"; packageName = "through"; @@ -7259,15 +7106,6 @@ let sha1 = "8cdd8fbac4e2d2ea1e7e2e8097c42f442280f85b"; }; }; - "unicode-5.2.0-0.7.5" = { - name = "unicode-5.2.0"; - packageName = "unicode-5.2.0"; - version = "0.7.5"; - src = fetchurl { - url = "https://registry.npmjs.org/unicode-5.2.0/-/unicode-5.2.0-0.7.5.tgz"; - sha512 = "KVGLW1Bri30x00yv4HNM8kBxoqFXr0Sbo55735nvrlsx4PYBZol3UtoWgO492fSwmsetzPEZzy73rbU8OGXJcA=="; - }; - }; "union-value-1.0.0" = { name = "union-value"; packageName = "union-value"; @@ -7439,13 +7277,13 @@ let sha512 = "1wFuMUIM16MDJRCrpbpuEPTUGmM5QMUg0cr3KFwra2XgOgFcPGDQHDh3CszSCD2Zewc/dh/pamNEW8CbfDebUw=="; }; }; - "v8flags-3.1.1" = { + "v8flags-3.1.2" = { name = "v8flags"; packageName = "v8flags"; - version = "3.1.1"; - src = fetchurl { - url = "https://registry.npmjs.org/v8flags/-/v8flags-3.1.1.tgz"; - sha512 = "iw/1ViSEaff8NJ3HLyEjawk/8hjJib3E7pvG4pddVXfUg1983s3VGsiClDjhK64MQVDGqc1Q8r18S4VKQZS9EQ=="; + version = "3.1.2"; + src = fetchurl { + url = "https://registry.npmjs.org/v8flags/-/v8flags-3.1.2.tgz"; + sha512 = "MtivA7GF24yMPte9Rp/BWGCYQNaUj86zeYxV/x2RRJMKagImbbv3u8iJC57lNhWLPcGLJmHcHmFWkNsplbbLWw=="; }; }; "vendors-1.0.2" = { @@ -7583,15 +7421,6 @@ let sha1 = "5438cd2ea93b202efa3a19fe8887aee7c94f9c9d"; }; }; - "winston-2.4.4" = { - name = "winston"; - packageName = "winston"; - version = "2.4.4"; - src = fetchurl { - url = "https://registry.npmjs.org/winston/-/winston-2.4.4.tgz"; - sha512 = "NBo2Pepn4hK4V01UfcWcDlmiVTs7VTB1h7bgnB0rgP146bYhMxX0ypCz3lBOfNxCO4Zuek7yeT+y/zM1OfMw4Q=="; - }; - }; "wordwrap-0.0.2" = { name = "wordwrap"; packageName = "wordwrap"; @@ -7682,15 +7511,6 @@ let sha512 = "C6kB/WJDiaxONLJQnF8ccx9SEeoTTLek8RVbaOIsrAUS8VrBEXfmeSnCZxygc+XC2sNMBIwOOnfcxiynjHsVSQ=="; }; }; - "yauzl-2.4.1" = { - name = "yauzl"; - packageName = "yauzl"; - version = "2.4.1"; - src = fetchurl { - url = "https://registry.npmjs.org/yauzl/-/yauzl-2.4.1.tgz"; - sha1 = "9528f442dab1b2284e58b4379bb194e22e0c4005"; - }; - }; }; args = { name = "rhodecode-enterprise"; @@ -7743,7 +7563,7 @@ let sources."@webassemblyjs/wasm-parser-1.7.10" sources."@webassemblyjs/wast-parser-1.7.10" sources."@webassemblyjs/wast-printer-1.7.10" - sources."@webcomponents/shadycss-1.6.0" + sources."@webcomponents/shadycss-1.7.1" sources."@webcomponents/webcomponentsjs-2.2.1" sources."@xtuc/ieee754-1.2.0" sources."@xtuc/long-4.2.1" @@ -7818,6 +7638,7 @@ let }) (sources."babel-core-6.26.3" // { dependencies = [ + sources."json5-0.5.1" sources."lodash-4.17.11" sources."minimatch-3.0.4" ]; @@ -7916,7 +7737,7 @@ let }) sources."base64-js-1.3.0" sources."bcrypt-pbkdf-1.0.2" - sources."big.js-3.2.0" + sources."big.js-5.2.2" sources."binary-extensions-1.12.0" sources."bluebird-3.5.3" sources."bn.js-4.11.8" @@ -7946,7 +7767,7 @@ let sources."graceful-fs-4.1.15" sources."lru-cache-4.1.5" sources."minimatch-3.0.4" - sources."rimraf-2.6.2" + sources."rimraf-2.6.3" ]; }) sources."cache-base-1.0.1" @@ -7957,8 +7778,8 @@ let sources."browserslist-1.7.7" ]; }) - sources."caniuse-db-1.0.30000912" - sources."caniuse-lite-1.0.30000912" + sources."caniuse-db-1.0.30000927" + sources."caniuse-lite-1.0.30000927" sources."caseless-0.12.0" sources."center-align-0.1.3" sources."chalk-0.5.1" @@ -8045,7 +7866,7 @@ let dependencies = [ sources."glob-7.1.3" sources."minimatch-3.0.4" - sources."rimraf-2.6.2" + sources."rimraf-2.6.3" ]; }) sources."copy-descriptor-0.1.1" @@ -8055,7 +7876,7 @@ let sources."minimatch-3.0.4" ]; }) - sources."core-js-2.5.7" + sources."core-js-2.6.1" sources."core-util-is-1.0.2" sources."create-ecdh-4.0.3" sources."create-hash-1.2.0" @@ -8079,7 +7900,6 @@ let sources."cssesc-0.1.0" sources."cssnano-3.10.0" sources."csso-2.3.2" - sources."cycle-1.0.3" sources."cyclist-0.2.2" (sources."dashdash-1.14.1" // { dependencies = [ @@ -8116,7 +7936,7 @@ let ]; }) sources."domain-browser-1.2.0" - sources."domelementtype-1.3.0" + sources."domelementtype-1.3.1" sources."domhandler-2.3.0" sources."domutils-1.5.1" (sources."duplexify-3.6.1" // { @@ -8126,7 +7946,7 @@ let ]; }) sources."ecc-jsbn-0.1.2" - sources."electron-to-chromium-1.3.85" + sources."electron-to-chromium-1.3.98" sources."elliptic-6.4.1" sources."emojis-list-2.1.0" sources."end-of-stream-1.4.1" @@ -8137,9 +7957,8 @@ let }) sources."entities-1.0.0" sources."errno-0.1.7" - sources."es-abstract-1.12.0" + sources."es-abstract-1.13.0" sources."es-to-primitive-1.2.0" - sources."es6-promise-4.2.5" sources."es6-templates-0.2.3" sources."escape-string-regexp-1.0.5" sources."eslint-scope-4.0.0" @@ -8151,7 +7970,7 @@ let sources."eventemitter2-0.4.14" sources."events-1.1.1" sources."evp_bytestokey-1.0.3" - sources."execa-0.10.0" + sources."execa-1.0.0" sources."exit-0.1.2" (sources."expand-brackets-2.1.4" // { dependencies = [ @@ -8185,15 +8004,12 @@ let sources."extend-shallow-2.0.1" ]; }) - sources."extract-zip-1.6.7" sources."extsprintf-1.3.0" - sources."eyes-0.1.8" sources."fast-deep-equal-2.0.1" sources."fast-json-stable-stringify-2.0.0" sources."fastparse-1.1.2" sources."favico.js-0.3.10" sources."faye-websocket-0.4.4" - sources."fd-slicer-1.0.1" sources."file-sync-cmp-0.1.1" (sources."fill-range-4.0.0" // { dependencies = [ @@ -8209,8 +8025,8 @@ let sources."minimatch-0.3.0" ]; }) - sources."fined-1.1.0" - sources."flagged-respawn-1.0.0" + sources."fined-1.1.1" + sources."flagged-respawn-1.0.1" sources."flatten-1.0.2" (sources."flush-write-stream-1.0.3" // { dependencies = [ @@ -8229,11 +8045,6 @@ let sources."string_decoder-1.1.1" ]; }) - (sources."fs-extra-1.0.0" // { - dependencies = [ - sources."graceful-fs-4.1.15" - ]; - }) (sources."fs-write-stream-atomic-1.0.10" // { dependencies = [ sources."graceful-fs-4.1.15" @@ -8244,7 +8055,11 @@ let sources."function-bind-1.1.1" sources."gaze-0.5.2" sources."get-caller-file-1.0.3" - sources."get-stream-3.0.0" + (sources."get-stream-4.1.0" // { + dependencies = [ + sources."pump-3.0.0" + ]; + }) sources."get-value-2.0.6" sources."getobject-0.1.0" (sources."getpass-0.1.7" // { @@ -8259,7 +8074,7 @@ let }) sources."glob-parent-3.1.0" sources."global-modules-1.0.0" - sources."global-modules-path-2.3.0" + sources."global-modules-path-2.3.1" (sources."global-prefix-1.0.2" // { dependencies = [ sources."which-1.3.1" @@ -8351,8 +8166,7 @@ let ]; }) sources."hash-base-3.0.4" - sources."hash.js-1.1.5" - sources."hasha-2.2.0" + sources."hash.js-1.1.7" sources."hawk-3.1.3" sources."he-1.2.0" sources."hmac-drbg-1.0.1" @@ -8369,6 +8183,8 @@ let }) (sources."html-webpack-plugin-3.2.0" // { dependencies = [ + sources."big.js-3.2.0" + sources."json5-0.5.1" sources."loader-utils-0.2.17" sources."lodash-4.17.11" ]; @@ -8381,7 +8197,7 @@ let (sources."icss-utils-2.1.0" // { dependencies = [ sources."ansi-styles-3.2.1" - sources."chalk-2.4.1" + sources."chalk-2.4.2" sources."postcss-6.0.23" sources."source-map-0.6.1" sources."supports-color-5.5.0" @@ -8395,7 +8211,7 @@ let dependencies = [ sources."find-up-3.0.0" sources."locate-path-3.0.0" - sources."p-limit-2.0.0" + sources."p-limit-2.1.0" sources."p-locate-3.0.0" sources."p-try-2.0.0" sources."pkg-dir-3.0.0" @@ -8445,12 +8261,12 @@ let sources."isobject-3.0.1" sources."isstream-0.1.2" sources."jquery-1.11.3" - sources."js-base64-2.4.9" + sources."js-base64-2.5.0" sources."js-tokens-3.0.2" sources."js-yaml-2.0.5" sources."jsbn-0.1.1" sources."jsesc-1.3.0" - (sources."jshint-2.9.6" // { + (sources."jshint-2.9.7" // { dependencies = [ sources."lodash-4.17.11" sources."minimatch-3.0.4" @@ -8461,25 +8277,14 @@ let sources."json-schema-traverse-0.4.1" sources."json-stable-stringify-1.0.1" sources."json-stringify-safe-5.0.1" - sources."json5-0.5.1" - (sources."jsonfile-2.4.0" // { - dependencies = [ - sources."graceful-fs-4.1.15" - ]; - }) + sources."json5-1.0.1" sources."jsonify-0.0.0" (sources."jsprim-1.4.1" // { dependencies = [ sources."assert-plus-1.0.0" ]; }) - sources."kew-0.7.0" sources."kind-of-6.0.2" - (sources."klaw-1.3.1" // { - dependencies = [ - sources."graceful-fs-4.1.15" - ]; - }) sources."lazy-cache-1.0.4" sources."lcid-2.0.0" (sources."less-2.7.3" // { @@ -8493,7 +8298,7 @@ let ]; }) sources."loader-runner-2.3.1" - sources."loader-utils-1.1.0" + sources."loader-utils-1.2.3" sources."locate-path-2.0.0" sources."lodash-0.9.2" sources."lodash.camelcase-4.3.0" @@ -8510,6 +8315,7 @@ let sources."map-age-cleaner-0.1.3" sources."map-cache-0.2.2" sources."map-visit-1.0.0" + sources."mark.js-8.11.1" sources."math-expression-evaluator-1.2.17" sources."md5.js-1.3.5" sources."mem-4.0.0" @@ -8528,25 +8334,29 @@ let sources."minimalistic-assert-1.0.1" sources."minimalistic-crypto-utils-1.0.1" sources."minimatch-0.2.14" - sources."minimist-0.0.8" + sources."minimist-1.2.0" sources."mississippi-2.0.0" (sources."mixin-deep-1.3.1" // { dependencies = [ sources."is-extendable-1.0.1" ]; }) - sources."mkdirp-0.5.1" - sources."moment-2.22.2" + (sources."mkdirp-0.5.1" // { + dependencies = [ + sources."minimist-0.0.8" + ]; + }) + sources."moment-2.23.0" sources."mousetrap-1.6.2" (sources."move-concurrently-1.0.1" // { dependencies = [ sources."glob-7.1.3" sources."minimatch-3.0.4" - sources."rimraf-2.6.2" + sources."rimraf-2.6.3" ]; }) sources."ms-2.0.0" - sources."nan-2.11.1" + sources."nan-2.12.1" sources."nanomatch-1.2.13" sources."neo-async-2.6.0" sources."nice-try-1.0.5" @@ -8598,7 +8408,7 @@ let sources."once-1.4.0" sources."os-browserify-0.3.0" sources."os-homedir-1.0.2" - sources."os-locale-3.0.1" + sources."os-locale-3.1.0" sources."os-tmpdir-1.0.2" sources."osenv-0.1.5" sources."p-defer-1.0.0" @@ -8635,22 +8445,13 @@ let sources."path-root-regex-0.1.2" sources."path-type-3.0.0" sources."pbkdf2-3.0.17" - sources."pend-1.2.0" sources."performance-now-0.2.0" - sources."phantom-4.0.12" - (sources."phantomjs-prebuilt-2.1.16" // { - dependencies = [ - sources."which-1.3.1" - ]; - }) sources."pify-3.0.0" - sources."pinkie-2.0.4" - sources."pinkie-promise-2.0.1" sources."pkg-dir-2.0.0" (sources."polymer-webpack-loader-2.0.3" // { dependencies = [ sources."ansi-styles-3.2.1" - sources."chalk-2.4.1" + sources."chalk-2.4.2" sources."html-loader-0.5.5" (sources."postcss-6.0.23" // { dependencies = [ @@ -8700,7 +8501,7 @@ let (sources."postcss-modules-extract-imports-1.2.1" // { dependencies = [ sources."ansi-styles-3.2.1" - sources."chalk-2.4.1" + sources."chalk-2.4.2" sources."postcss-6.0.23" sources."source-map-0.6.1" sources."supports-color-5.5.0" @@ -8709,7 +8510,7 @@ let (sources."postcss-modules-local-by-default-1.2.0" // { dependencies = [ sources."ansi-styles-3.2.1" - sources."chalk-2.4.1" + sources."chalk-2.4.2" sources."postcss-6.0.23" sources."source-map-0.6.1" sources."supports-color-5.5.0" @@ -8718,7 +8519,7 @@ let (sources."postcss-modules-scope-1.1.0" // { dependencies = [ sources."ansi-styles-3.2.1" - sources."chalk-2.4.1" + sources."chalk-2.4.2" sources."postcss-6.0.23" sources."source-map-0.6.1" sources."supports-color-5.5.0" @@ -8727,7 +8528,7 @@ let (sources."postcss-modules-values-1.3.0" // { dependencies = [ sources."ansi-styles-3.2.1" - sources."chalk-2.4.1" + sources."chalk-2.4.2" sources."postcss-6.0.23" sources."source-map-0.6.1" sources."supports-color-5.5.0" @@ -8749,7 +8550,6 @@ let sources."private-0.1.8" sources."process-0.11.10" sources."process-nextick-args-2.0.0" - sources."progress-1.1.8" sources."promise-7.3.1" sources."promise-inflight-1.0.1" sources."prr-1.0.1" @@ -8823,10 +8623,9 @@ let sources."repeat-string-1.6.1" sources."repeating-2.0.1" sources."request-2.81.0" - sources."request-progress-2.0.1" sources."require-directory-2.1.1" sources."require-main-filename-1.0.1" - sources."resolve-1.8.1" + sources."resolve-1.9.0" sources."resolve-cwd-2.0.0" sources."resolve-dir-1.0.1" sources."resolve-from-3.0.0" @@ -8842,12 +8641,12 @@ let sources."sax-1.2.4" (sources."schema-utils-0.4.7" // { dependencies = [ - sources."ajv-6.6.1" + sources."ajv-6.6.2" ]; }) sources."select-1.1.2" sources."semver-5.6.0" - sources."serialize-javascript-1.5.0" + sources."serialize-javascript-1.6.1" sources."set-blocking-2.0.0" (sources."set-value-2.0.0" // { dependencies = [ @@ -8897,16 +8696,14 @@ let sources."source-map-resolve-0.5.2" sources."source-map-support-0.4.18" sources."source-map-url-0.4.0" - sources."split-1.0.1" sources."split-string-3.1.0" sources."sprintf-js-1.0.3" - (sources."sshpk-1.15.2" // { + (sources."sshpk-1.16.0" // { dependencies = [ sources."assert-plus-1.0.0" ]; }) sources."ssri-5.3.0" - sources."stack-trace-0.0.10" (sources."static-extend-0.1.2" // { dependencies = [ sources."define-property-0.2.5" @@ -8963,7 +8760,6 @@ let ]; }) sources."tapable-1.1.1" - sources."throttleit-1.0.0" sources."through-2.3.8" (sources."through2-2.0.5" // { dependencies = [ @@ -8993,9 +8789,11 @@ let sources."trim-right-1.0.1" (sources."ts-loader-1.3.3" // { dependencies = [ - sources."colors-1.3.2" + sources."big.js-3.2.0" + sources."colors-1.3.3" sources."enhanced-resolve-3.4.1" sources."graceful-fs-4.1.15" + sources."json5-0.5.1" sources."loader-utils-0.2.17" sources."tapable-0.2.9" ]; @@ -9025,7 +8823,6 @@ let sources."unc-path-regex-0.1.2" sources."underscore-1.7.0" sources."underscore.string-2.2.1" - sources."unicode-5.2.0-0.7.5" (sources."union-value-1.0.0" // { dependencies = [ sources."extend-shallow-2.0.1" @@ -9066,7 +8863,7 @@ let sources."utila-0.4.0" sources."uuid-3.3.2" sources."v8-compile-cache-2.0.2" - sources."v8flags-3.1.1" + sources."v8flags-3.1.2" sources."vendors-1.0.2" (sources."verror-1.10.0" // { dependencies = [ @@ -9082,13 +8879,13 @@ let sources."waypoints-4.0.1" (sources."webpack-4.23.1" // { dependencies = [ - sources."ajv-6.6.1" + sources."ajv-6.6.2" ]; }) (sources."webpack-cli-3.1.2" // { dependencies = [ sources."ansi-styles-3.2.1" - sources."chalk-2.4.1" + sources."chalk-2.4.2" sources."supports-color-5.5.0" ]; }) @@ -9121,12 +8918,6 @@ let sources."which-1.0.9" sources."which-module-2.0.0" sources."window-size-0.1.0" - (sources."winston-2.4.4" // { - dependencies = [ - sources."async-1.0.0" - sources."colors-1.0.3" - ]; - }) sources."wordwrap-0.0.2" sources."worker-farm-1.6.0" (sources."wrap-ansi-2.1.0" // { @@ -9144,13 +8935,12 @@ let dependencies = [ sources."find-up-3.0.0" sources."locate-path-3.0.0" - sources."p-limit-2.0.0" + sources."p-limit-2.1.0" sources."p-locate-3.0.0" sources."p-try-2.0.0" ]; }) sources."yargs-parser-11.1.1" - sources."yauzl-2.4.1" ]; buildInputs = globalBuildInputs; meta = { diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix --- a/pkgs/python-packages.nix +++ b/pkgs/python-packages.nix @@ -219,11 +219,11 @@ self: super: { }; }; "click" = super.buildPythonPackage { - name = "click-6.6"; + name = "click-7.0"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/7a/00/c14926d8232b36b08218067bcd5853caefb4737cda3f0a47437151344792/click-6.6.tar.gz"; - sha256 = "1sggipyz52crrybwbr9xvwxd4aqigvplf53k9w3ygxmzivd1jsnc"; + url = "https://files.pythonhosted.org/packages/f8/5c/f60e9d8a1e77005f664b76ff8aeaee5bc05d0a91798afd7f53fc998dbc47/Click-7.0.tar.gz"; + sha256 = "1mzjixd4vjbjvzb6vylki9w1556a9qmdh35kzmq6cign46av952v"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; @@ -260,11 +260,11 @@ self: super: { }; }; "configparser" = super.buildPythonPackage { - name = "configparser-3.5.0"; + name = "configparser-3.7.1"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/7c/69/c2ce7e91c89dc073eb1aa74c0621c3eefbffe8216b3f9af9d3885265c01c/configparser-3.5.0.tar.gz"; - sha256 = "0fi7vf09vi1588jd8f16a021m5y6ih2hy7rpbjb408xw45qb822k"; + url = "https://files.pythonhosted.org/packages/b6/a6/eceea7c5a5dbcf56815bed411c38cabd8a879386be10717b160e7362b5a2/configparser-3.7.1.tar.gz"; + sha256 = "0cnz213il9lhgda6x70fw7mfqr8da43s3wm343lwzhqx94mgmmav"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -374,11 +374,14 @@ self: super: { }; }; "dogpile.cache" = super.buildPythonPackage { - name = "dogpile.cache-0.6.7"; + name = "dogpile.cache-0.7.1"; doCheck = false; + propagatedBuildInputs = [ + self."decorator" + ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/ee/bd/440da735a11c6087eed7cc8747fc4b995cbac2464168682f8ee1c8e43844/dogpile.cache-0.6.7.tar.gz"; - sha256 = "1aw8rx8vhb75y7zc6gi67g21sw057jdx7i8m3jq7kf3nqavxx9zw"; + url = "https://files.pythonhosted.org/packages/84/3e/dbf1cfc5228f1d3dca80ef714db2c5aaec5cd9efaf54d7e3daef6bc48b19/dogpile.cache-0.7.1.tar.gz"; + sha256 = "0caazmrzhnfqb5yrp8myhw61ny637jj69wcngrpbvi31jlcpy6v9"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; @@ -407,30 +410,75 @@ self: super: { }; }; "elasticsearch" = super.buildPythonPackage { - name = "elasticsearch-2.3.0"; + name = "elasticsearch-6.3.1"; doCheck = false; propagatedBuildInputs = [ self."urllib3" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/10/35/5fd52c5f0b0ee405ed4b5195e8bce44c5e041787680dc7b94b8071cac600/elasticsearch-2.3.0.tar.gz"; - sha256 = "10ad2dk73xsys9vajwsncibs69asa63w1hgwz6lz1prjpyi80c5y"; + url = "https://files.pythonhosted.org/packages/9d/ce/c4664e8380e379a9402ecfbaf158e56396da90d520daba21cfa840e0eb71/elasticsearch-6.3.1.tar.gz"; + sha256 = "12y93v0yn7a4xmf969239g8gb3l4cdkclfpbk1qc8hx5qkymrnma"; }; meta = { license = [ pkgs.lib.licenses.asl20 ]; }; }; "elasticsearch-dsl" = super.buildPythonPackage { - name = "elasticsearch-dsl-2.2.0"; + name = "elasticsearch-dsl-6.3.1"; doCheck = false; propagatedBuildInputs = [ self."six" self."python-dateutil" self."elasticsearch" + self."ipaddress" + ]; + src = fetchurl { + url = "https://files.pythonhosted.org/packages/4c/0d/1549f50c591db6bb4e66cbcc8d34a6e537c3d89aa426b167c244fd46420a/elasticsearch-dsl-6.3.1.tar.gz"; + sha256 = "1gh8a0shqi105k325hgwb9avrpdjh0mc6mxwfg9ba7g6lssb702z"; + }; + meta = { + license = [ pkgs.lib.licenses.asl20 ]; + }; + }; + "elasticsearch1" = super.buildPythonPackage { + name = "elasticsearch1-1.10.0"; + doCheck = false; + propagatedBuildInputs = [ + self."urllib3" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/66/2f/52a086968788e58461641570f45c3207a52d46ebbe9b77dc22b6a8ffda66/elasticsearch-dsl-2.2.0.tar.gz"; - sha256 = "1g4kxzxsdwlsl2a9kscmx11pafgimhj7y8wrfksv8pgvpkfb9fwr"; + url = "https://files.pythonhosted.org/packages/a6/eb/73e75f9681fa71e3157b8ee878534235d57f24ee64f0e77f8d995fb57076/elasticsearch1-1.10.0.tar.gz"; + sha256 = "0g89444kd5zwql4vbvyrmi2m6l6dcj6ga98j4hqxyyyz6z20aki2"; + }; + meta = { + license = [ pkgs.lib.licenses.asl20 ]; + }; + }; + "elasticsearch1-dsl" = super.buildPythonPackage { + name = "elasticsearch1-dsl-0.0.12"; + doCheck = false; + propagatedBuildInputs = [ + self."six" + self."python-dateutil" + self."elasticsearch1" + ]; + src = fetchurl { + url = "https://files.pythonhosted.org/packages/eb/9d/785342775cb10eddc9b8d7457d618a423b4f0b89d8b2b2d1bc27190d71db/elasticsearch1-dsl-0.0.12.tar.gz"; + sha256 = "0ig1ly39v93hba0z975wnhbmzwj28w6w1sqlr2g7cn5spp732bhk"; + }; + meta = { + license = [ pkgs.lib.licenses.asl20 ]; + }; + }; + "elasticsearch2" = super.buildPythonPackage { + name = "elasticsearch2-2.5.0"; + doCheck = false; + propagatedBuildInputs = [ + self."urllib3" + ]; + src = fetchurl { + url = "https://files.pythonhosted.org/packages/84/77/63cf63d4ba11d913b5278406f2a37b0712bec6fc85edfb6151a33eaeba25/elasticsearch2-2.5.0.tar.gz"; + sha256 = "0ky0q16lbvz022yv6q3pix7aamf026p1y994537ccjf0p0dxnbxr"; }; meta = { license = [ pkgs.lib.licenses.asl20 ]; @@ -517,14 +565,14 @@ self: super: { }; }; "gevent" = super.buildPythonPackage { - name = "gevent-1.3.7"; + name = "gevent-1.4.0"; doCheck = false; propagatedBuildInputs = [ self."greenlet" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/10/c1/9499b146bfa43aa4f1e0ed1bab1bd3209a4861d25650c11725036c731cf5/gevent-1.3.7.tar.gz"; - sha256 = "0b0fr04qdk1p4sniv87fh8z5psac60x01pv054kpgi94520g81iz"; + url = "https://files.pythonhosted.org/packages/ed/27/6c49b70808f569b66ec7fac2e78f076e9b204db9cf5768740cff3d5a07ae/gevent-1.4.0.tar.gz"; + sha256 = "1lchr4akw2jkm5v4kz7bdm4wv3knkfhbfn9vkkz4s5yrkcxzmdqy"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -673,11 +721,11 @@ self: super: { }; }; "iso8601" = super.buildPythonPackage { - name = "iso8601-0.1.11"; + name = "iso8601-0.1.12"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/c0/75/c9209ee4d1b5975eb8c2cba4428bde6b61bd55664a98290dd015cdb18e98/iso8601-0.1.11.tar.gz"; - sha256 = "0c7gh3lsdjds262h0v1sqc66l7hqgfwbakn96qrhdbl0i3vm5yz8"; + url = "https://files.pythonhosted.org/packages/45/13/3db24895497345fb44c4248c08b16da34a9eb02643cea2754b21b5ed08b0/iso8601-0.1.12.tar.gz"; + sha256 = "10nyvvnrhw2w3p09v1ica4lgj6f4g9j3kkfx17qmraiq3w7b5i29"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -768,14 +816,14 @@ self: super: { }; }; "kombu" = super.buildPythonPackage { - name = "kombu-4.2.0"; + name = "kombu-4.2.1"; doCheck = false; propagatedBuildInputs = [ self."amqp" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/ab/b1/46a7a8babf5e60f3b2ca081a100af8edfcf132078a726375f52a054e70cf/kombu-4.2.0.tar.gz"; - sha256 = "1yz19qlqf0inl1mnwlpq9j6kj9r67clpy0xg99phyg4329rw80fn"; + url = "https://files.pythonhosted.org/packages/39/9f/556b988833abede4a80dbd18b2bdf4e8ff4486dd482ed45da961347e8ed2/kombu-4.2.1.tar.gz"; + sha256 = "10lh3hncvw67fz0k5vgbx3yh9gjfpqdlia1f13i28cgnc1nfrbc6"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; @@ -818,11 +866,11 @@ self: super: { }; }; "markupsafe" = super.buildPythonPackage { - name = "markupsafe-1.0"; + name = "markupsafe-1.1.0"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/4d/de/32d741db316d8fdb7680822dd37001ef7a448255de9699ab4bfcbdf4172b/MarkupSafe-1.0.tar.gz"; - sha256 = "0rdn1s8x9ni7ss8rfiacj7x1085lx8mh2zdwqslnw8xc3l4nkgm6"; + url = "https://files.pythonhosted.org/packages/ac/7e/1b4c2e05809a4414ebce0892fe1e32c14ace86ca7d50c70f00979ca9b3a3/MarkupSafe-1.1.0.tar.gz"; + sha256 = "1lxirjypbdd3l9jl4vliilhfnhy7c7f2vlldqg1b0i74khn375sf"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; @@ -862,14 +910,14 @@ self: super: { }; }; "more-itertools" = super.buildPythonPackage { - name = "more-itertools-4.3.0"; + name = "more-itertools-5.0.0"; doCheck = false; propagatedBuildInputs = [ self."six" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/88/ff/6d485d7362f39880810278bdc906c13300db05485d9c65971dec1142da6a/more-itertools-4.3.0.tar.gz"; - sha256 = "17h3na0rdh8xq30w4b9pizgkdxmm51896bxw600x84jflg9vaxn4"; + url = "https://files.pythonhosted.org/packages/dd/26/30fc0d541d9fdf55faf5ba4b0fd68f81d5bd2447579224820ad525934178/more-itertools-5.0.0.tar.gz"; + sha256 = "1r12cm6mcdwdzz7d47a6g4l437xsvapdlgyhqay3i2nrlv03da9q"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -960,32 +1008,32 @@ self: super: { }; }; "paste" = super.buildPythonPackage { - name = "paste-2.0.3"; + name = "paste-3.0.5"; doCheck = false; propagatedBuildInputs = [ self."six" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/30/c3/5c2f7c7a02e4f58d4454353fa1c32c94f79fa4e36d07a67c0ac295ea369e/Paste-2.0.3.tar.gz"; - sha256 = "062jk0nlxf6lb2wwj6zc20rlvrwsnikpkh90y0dn8cjch93s6ii3"; + url = "https://files.pythonhosted.org/packages/d4/41/91bde422400786b1b06357c1e6e3a5379f54dc3002aeb337cb767233304e/Paste-3.0.5.tar.gz"; + sha256 = "1a6i8fh1fg8r4x800fvy9r82m15clwjim6yf2g9r4dff0y40dchv"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; "pastedeploy" = super.buildPythonPackage { - name = "pastedeploy-1.5.2"; + name = "pastedeploy-2.0.1"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/0f/90/8e20cdae206c543ea10793cbf4136eb9a8b3f417e04e40a29d72d9922cbd/PasteDeploy-1.5.2.tar.gz"; - sha256 = "1jz3m4hq8v6hyhfjz9425nd3nvn52cvbfipdcd72krjmla4qz1fm"; + url = "https://files.pythonhosted.org/packages/19/a0/5623701df7e2478a68a1b685d1a84518024eef994cde7e4da8449a31616f/PasteDeploy-2.0.1.tar.gz"; + sha256 = "02imfbbx1mi2h546f3sr37m47dk9qizaqhzzlhx8bkzxa6fzn8yl"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; "pastescript" = super.buildPythonPackage { - name = "pastescript-2.0.2"; + name = "pastescript-3.0.0"; doCheck = false; propagatedBuildInputs = [ self."paste" @@ -993,23 +1041,23 @@ self: super: { self."six" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/e5/f0/78e766c3dcc61a4f3a6f71dd8c95168ae9c7a31722b5663d19c1fdf62cb6/PasteScript-2.0.2.tar.gz"; - sha256 = "1h3nnhn45kf4pbcv669ik4faw04j58k8vbj1hwrc532k0nc28gy0"; + url = "https://files.pythonhosted.org/packages/08/2a/3797377a884ab9a064ad4d564ed612e54d26d7997caa8229c9c9df4eac31/PasteScript-3.0.0.tar.gz"; + sha256 = "1hvmyz1sbn7ws1syw567ph7km9fi0wi75r3vlyzx6sk0z26xkm6r"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; "pathlib2" = super.buildPythonPackage { - name = "pathlib2-2.3.2"; + name = "pathlib2-2.3.3"; doCheck = false; propagatedBuildInputs = [ self."six" self."scandir" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/db/a8/7d6439c1aec525ed70810abee5b7d7f3aa35347f59bc28343e8f62019aa2/pathlib2-2.3.2.tar.gz"; - sha256 = "10yb0iv5x2hs631rcppkhbddx799d3h8pcwmkbh2a66ns3w71ccf"; + url = "https://files.pythonhosted.org/packages/bf/d7/a2568f4596b75d2c6e2b4094a7e64f620decc7887f69a1f2811931ea15b9/pathlib2-2.3.3.tar.gz"; + sha256 = "0hpp92vqqgcd8h92msm9slv161b1q160igjwnkf2ag6cx0c96695"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -1084,11 +1132,11 @@ self: super: { }; }; "pluggy" = super.buildPythonPackage { - name = "pluggy-0.8.0"; + name = "pluggy-0.8.1"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/65/25/81d0de17cd00f8ca994a4e74e3c4baf7cd25072c0b831dad5c7d9d6138f8/pluggy-0.8.0.tar.gz"; - sha256 = "1580p47l2zqzsza8jcnw1h2wh3vvmygk6ly8bvi4w0g8j14sjys4"; + url = "https://files.pythonhosted.org/packages/38/e1/83b10c17688af7b2998fa5342fec58ecbd2a5a7499f31e606ae6640b71ac/pluggy-0.8.1.tar.gz"; + sha256 = "05l6g42p9ilmabw0hlbiyxy6gyzjri41m5l11a8dzgvi77q35p4d"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -1110,11 +1158,11 @@ self: super: { }; }; "psutil" = super.buildPythonPackage { - name = "psutil-5.4.7"; + name = "psutil-5.4.8"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/7d/9a/1e93d41708f8ed2b564395edfa3389f0fd6d567597401c2e5e2775118d8b/psutil-5.4.7.tar.gz"; - sha256 = "0fsgmvzwbdbszkwfnqhib8jcxm4w6zyhvlxlcda0rfm5cyqj4qsv"; + url = "https://files.pythonhosted.org/packages/e3/58/0eae6e4466e5abf779d7e2b71fac7fba5f59e00ea36ddb3ed690419ccb0f/psutil-5.4.8.tar.gz"; + sha256 = "1hyna338sml2cl1mfb2gs89np18z27mvyhmq4ifh22x07n7mq9kf"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; @@ -1180,25 +1228,25 @@ self: super: { }; }; "pyasn1" = super.buildPythonPackage { - name = "pyasn1-0.4.4"; + name = "pyasn1-0.4.5"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/10/46/059775dc8e50f722d205452bced4b3cc965d27e8c3389156acd3b1123ae3/pyasn1-0.4.4.tar.gz"; - sha256 = "0drilmx5j25aplfr5wrml0030cs5fgxp9yp94fhllxgx28yjm3zm"; + url = "https://files.pythonhosted.org/packages/46/60/b7e32f6ff481b8a1f6c8f02b0fd9b693d1c92ddd2efb038ec050d99a7245/pyasn1-0.4.5.tar.gz"; + sha256 = "1xqh3jh2nfi2bflk5a0vn59y3pp1vn54f3ksx652sid92gz2096s"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; }; }; "pyasn1-modules" = super.buildPythonPackage { - name = "pyasn1-modules-0.2.2"; + name = "pyasn1-modules-0.2.4"; doCheck = false; propagatedBuildInputs = [ self."pyasn1" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/37/33/74ebdc52be534e683dc91faf263931bc00ae05c6073909fde53999088541/pyasn1-modules-0.2.2.tar.gz"; - sha256 = "0ivm850yi7ajjbi8j115qpsj95bgxdsx48nbjzg0zip788c3xkx0"; + url = "https://files.pythonhosted.org/packages/bd/a5/ef7bf693e8a8f015386c9167483199f54f8a8ec01d1c737e05524f16e792/pyasn1-modules-0.2.4.tar.gz"; + sha256 = "0z3w5dqrrvdplg9ma45j8n23xvyrj9ki8mg4ibqbn7l4qpl90855"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; @@ -1238,30 +1286,16 @@ self: super: { }; }; "pygments" = super.buildPythonPackage { - name = "pygments-2.3.0"; + name = "pygments-2.3.1"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/63/a2/91c31c4831853dedca2a08a0f94d788fc26a48f7281c99a303769ad2721b/Pygments-2.3.0.tar.gz"; - sha256 = "1z34ms51dh4jq4h3cizp7vd1dmsxcbvffkjsd2xxfav22nn6lrl2"; + url = "https://files.pythonhosted.org/packages/64/69/413708eaf3a64a6abb8972644e0f20891a55e621c6759e2c3f3891e05d63/Pygments-2.3.1.tar.gz"; + sha256 = "0ji87g09jph8jqcvclgb02qvxasdnr9pzvk90rl66d90yqcxmyjz"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; }; }; - "pygments-markdown-lexer" = super.buildPythonPackage { - name = "pygments-markdown-lexer-0.1.0.dev39"; - doCheck = false; - propagatedBuildInputs = [ - self."pygments" - ]; - src = fetchurl { - url = "https://files.pythonhosted.org/packages/c3/12/674cdee66635d638cedb2c5d9c85ce507b7b2f91bdba29e482f1b1160ff6/pygments-markdown-lexer-0.1.0.dev39.zip"; - sha256 = "1pzb5wy23q3fhs0rqzasjnw6hdzwjngpakb73i98cn0b8lk8q4jc"; - }; - meta = { - license = [ pkgs.lib.licenses.asl20 ]; - }; - }; "pymysql" = super.buildPythonPackage { name = "pymysql-0.8.1"; doCheck = false; @@ -1285,35 +1319,34 @@ self: super: { }; }; "pyparsing" = super.buildPythonPackage { - name = "pyparsing-1.5.7"; + name = "pyparsing-2.3.0"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/6f/2c/47457771c02a8ff0f302b695e094ec309e30452232bd79198ee94fda689f/pyparsing-1.5.7.tar.gz"; - sha256 = "17z7ws076z977sclj628fvwrp8y9j2rvdjcsq42v129n1gwi8vk4"; + url = "https://files.pythonhosted.org/packages/d0/09/3e6a5eeb6e04467b737d55f8bba15247ac0876f98fae659e58cd744430c6/pyparsing-2.3.0.tar.gz"; + sha256 = "14k5v7n3xqw8kzf42x06bzp184spnlkya2dpjyflax6l3yrallzk"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; "pyramid" = super.buildPythonPackage { - name = "pyramid-1.9.2"; + name = "pyramid-1.10.1"; doCheck = false; propagatedBuildInputs = [ - self."setuptools" - self."webob" - self."repoze.lru" - self."zope.interface" - self."zope.deprecation" - self."venusian" - self."translationstring" - self."pastedeploy" + self."hupper" self."plaster" self."plaster-pastedeploy" - self."hupper" + self."setuptools" + self."translationstring" + self."venusian" + self."webob" + self."zope.deprecation" + self."zope.interface" + self."repoze.lru" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/a0/c1/b321d07cfc4870541989ad131c86a1d593bfe802af0eca9718a0dadfb97a/pyramid-1.9.2.tar.gz"; - sha256 = "09drsl0346nchgxp2j7sa5hlk7mkhfld9wvbd0wicacrp26a92fg"; + url = "https://files.pythonhosted.org/packages/0a/3e/22e3ac9be1b70a01139adba8906ee4b8f628bb469fea3c52f6c97b73063c/pyramid-1.10.1.tar.gz"; + sha256 = "1h5105nfh6rsrfjiyw20aavyibj36la3hajy6vh1fa77xb4y3hrp"; }; meta = { license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ]; @@ -1335,7 +1368,7 @@ self: super: { }; }; "pyramid-debugtoolbar" = super.buildPythonPackage { - name = "pyramid-debugtoolbar-4.4"; + name = "pyramid-debugtoolbar-4.5"; doCheck = false; propagatedBuildInputs = [ self."pyramid" @@ -1345,8 +1378,8 @@ self: super: { self."ipaddress" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/00/6f/c04eb4e715a7a5a4b24079ab7ffd1dceb1f70b2e24fc17686a2922dbac0a/pyramid_debugtoolbar-4.4.tar.gz"; - sha256 = "17p7nxvapvy2hab1rah3ndq2kbs4v83pixj8x2n4m7008ai9lxsz"; + url = "https://files.pythonhosted.org/packages/14/28/1f240239af340d19ee271ac62958158c79edb01a44ad8c9885508dd003d2/pyramid_debugtoolbar-4.5.tar.gz"; + sha256 = "0x2p3409pnx66n6dx5vc0mk2r1cp1ydr8mp120w44r9pwcngbibl"; }; meta = { license = [ { fullName = "Repoze Public License"; } pkgs.lib.licenses.bsdOriginal ]; @@ -1519,11 +1552,11 @@ self: super: { }; }; "python-editor" = super.buildPythonPackage { - name = "python-editor-1.0.3"; + name = "python-editor-1.0.4"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/65/1e/adf6e000ea5dc909aa420352d6ba37f16434c8a3c2fa030445411a1ed545/python-editor-1.0.3.tar.gz"; - sha256 = "0rf5xz8vw93v7mhdcvind7fkykipzga430wkcd7wk892xsn6dh53"; + url = "https://files.pythonhosted.org/packages/0a/85/78f4a216d28343a67b7397c99825cff336330893f00601443f7c7b2f2234/python-editor-1.0.4.tar.gz"; + sha256 = "0yrjh8w72ivqxi4i7xsg5b1vz15x8fg51xra7c3bgfyxqnyadzai"; }; meta = { license = [ pkgs.lib.licenses.asl20 { fullName = "Apache"; } ]; @@ -1657,7 +1690,7 @@ self: super: { }; }; "rhodecode-enterprise-ce" = super.buildPythonPackage { - name = "rhodecode-enterprise-ce-4.15.2"; + name = "rhodecode-enterprise-ce-4.16.0"; buildInputs = [ self."pytest" self."py" @@ -1668,9 +1701,10 @@ self: super: { self."pytest-timeout" self."gprof2dot" self."mock" - self."webtest" self."cov-core" self."coverage" + self."webtest" + self."beautifulsoup4" self."configobj" ]; doCheck = true; @@ -1723,7 +1757,6 @@ self: super: { self."pycrypto" self."pycurl" self."pyflakes" - self."pygments-markdown-lexer" self."pygments" self."pyparsing" self."pyramid-beaker" @@ -1794,9 +1827,10 @@ self: super: { self."pytest-timeout" self."gprof2dot" self."mock" - self."webtest" self."cov-core" self."coverage" + self."webtest" + self."beautifulsoup4" ]; src = ./.; meta = { @@ -1804,7 +1838,7 @@ self: super: { }; }; "rhodecode-tools" = super.buildPythonPackage { - name = "rhodecode-tools-1.0.1"; + name = "rhodecode-tools-1.2.1"; doCheck = false; propagatedBuildInputs = [ self."click" @@ -1813,14 +1847,16 @@ self: super: { self."mako" self."markupsafe" self."requests" - self."elasticsearch" - self."elasticsearch-dsl" self."urllib3" self."whoosh" + self."elasticsearch" + self."elasticsearch-dsl" + self."elasticsearch2" + self."elasticsearch1-dsl" ]; src = fetchurl { - url = "https://code.rhodecode.com/rhodecode-tools-ce/archive/v1.0.1.tar.gz?md5=ffb5d6bcb855305b93cfe23ad42e500b"; - sha256 = "0nr300s4sg685qs4wgbwlplwriawrwi6jq79z37frcnpyc89gpvm"; + url = "https://code.rhodecode.com/rhodecode-tools-ce/archive/v1.2.1.tar.gz?md5=25bc2f7de1da318e547236d3fb463d28"; + sha256 = "1k8l3s4mvshza1zay6dfxprq54fyb5dc85dqdva9wa3f466y0adk"; }; meta = { license = [ { fullName = "Apache 2.0 and Proprietary"; } ]; @@ -1864,11 +1900,11 @@ self: super: { }; }; "setuptools" = super.buildPythonPackage { - name = "setuptools-40.6.2"; + name = "setuptools-40.8.0"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip"; - sha256 = "0r2c5hapirlzm34h7pl1lgkm6gk7bcrlrdj28qgsvaqg3f74vfw6"; + url = "https://files.pythonhosted.org/packages/c2/f7/c7b501b783e5a74cf1768bc174ee4fb0a8a6ee5af6afa92274ff964703e0/setuptools-40.8.0.zip"; + sha256 = "0k9hifpgahnw2a26w3cr346iy733k6d3nwh3f7g9m13y6f8fqkkf"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -1897,11 +1933,11 @@ self: super: { }; }; "simplejson" = super.buildPythonPackage { - name = "simplejson-3.11.1"; + name = "simplejson-3.16.0"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/08/48/c97b668d6da7d7bebe7ea1817a6f76394b0ec959cb04214ca833c34359df/simplejson-3.11.1.tar.gz"; - sha256 = "1rr58dppsq73p0qcd9bsw066cdd3v63sqv7j6sqni8frvm4jv8h1"; + url = "https://files.pythonhosted.org/packages/e3/24/c35fb1c1c315fc0fffe61ea00d3f88e85469004713dab488dee4f35b0aff/simplejson-3.16.0.tar.gz"; + sha256 = "19cws1syk8jzq2pw43878dv6fjkb0ifvjpx0i9aajix6kc9jkwxi"; }; meta = { license = [ { fullName = "Academic Free License (AFL)"; } pkgs.lib.licenses.mit ]; @@ -1945,25 +1981,25 @@ self: super: { }; }; "subprocess32" = super.buildPythonPackage { - name = "subprocess32-3.5.2"; + name = "subprocess32-3.5.3"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/c3/5f/7117737fc7114061837a4f51670d863dd7f7f9c762a6546fa8a0dcfe61c8/subprocess32-3.5.2.tar.gz"; - sha256 = "11v62shwmdys48g7ncs3a8jwwnkcl8d4zcwy6dk73z1zy2f9hazb"; + url = "https://files.pythonhosted.org/packages/be/2b/beeba583e9877e64db10b52a96915afc0feabf7144dcbf2a0d0ea68bf73d/subprocess32-3.5.3.tar.gz"; + sha256 = "1hr5fan8i719hmlmz73hf8rhq74014w07d8ryg7krvvf6692kj3b"; }; meta = { license = [ pkgs.lib.licenses.psfl ]; }; }; "supervisor" = super.buildPythonPackage { - name = "supervisor-3.3.4"; + name = "supervisor-3.3.5"; doCheck = false; propagatedBuildInputs = [ self."meld3" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/44/60/698e54b4a4a9b956b2d709b4b7b676119c833d811d53ee2500f1b5e96dc3/supervisor-3.3.4.tar.gz"; - sha256 = "0wp62z9xprvz2krg02xnbwcnq6pxfq3byd8cxx8c2d8xznih28i1"; + url = "https://files.pythonhosted.org/packages/ba/65/92575a8757ed576beaee59251f64a3287bde82bdc03964b89df9e1d29e1b/supervisor-3.3.5.tar.gz"; + sha256 = "1w3ahridzbc6rxfpbyx8lij6pjlcgf2ymzyg53llkjqxalp6sk8v"; }; meta = { license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ]; @@ -2059,11 +2095,11 @@ self: super: { }; }; "urllib3" = super.buildPythonPackage { - name = "urllib3-1.21"; + name = "urllib3-1.24.1"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/34/95/7b28259d0006ed681c424cd71a668363265eac92b67dddd018eb9a22bff8/urllib3-1.21.tar.gz"; - sha256 = "0irnj4wvh2y36s4q3l2vas9qr9m766w6w418nb490j3mf8a8zw6h"; + url = "https://files.pythonhosted.org/packages/b1/53/37d82ab391393565f2f831b8eedbffd57db5a718216f82f1a8b4d381a1c1/urllib3-1.24.1.tar.gz"; + sha256 = "08lwd9f3hqznyf32vnzwvp87pchx062nkbgyrf67rwlkgj0jk5fy"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -2081,22 +2117,22 @@ self: super: { }; }; "venusian" = super.buildPythonPackage { - name = "venusian-1.1.0"; + name = "venusian-1.2.0"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/38/24/b4b470ab9e0a2e2e9b9030c7735828c8934b4c6b45befd1bb713ec2aeb2d/venusian-1.1.0.tar.gz"; - sha256 = "0zapz131686qm0gazwy8bh11vr57pr89jbwbl50s528sqy9f80lr"; + url = "https://files.pythonhosted.org/packages/7e/6f/40a9d43ac77cb51cb62be5b5662d170f43f8037bdc4eab56336c4ca92bb7/venusian-1.2.0.tar.gz"; + sha256 = "0ghyx66g8ikx9nx1mnwqvdcqm11i1vlq0hnvwl50s48bp22q5v34"; }; meta = { license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ]; }; }; "vine" = super.buildPythonPackage { - name = "vine-1.1.4"; + name = "vine-1.2.0"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/32/23/36284986e011f3c130d802c3c66abd8f1aef371eae110ddf80c5ae22e1ff/vine-1.1.4.tar.gz"; - sha256 = "0wkskb2hb494v9gixqnf4bl972p4ibcmxdykzpwjlfa5picns4aj"; + url = "https://files.pythonhosted.org/packages/46/1a/c94317efa98040c5d50fe3cf9080cafb0372ff5afb0283dc018c751c6746/vine-1.2.0.tar.gz"; + sha256 = "0xjz2sjbr5jrpjk411b7alkghdskhphgsqqrbi7abqfh2pli6j7f"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; @@ -2182,18 +2218,18 @@ self: super: { }; }; "webob" = super.buildPythonPackage { - name = "webob-1.7.4"; + name = "webob-1.8.4"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/75/34/731e23f52371852dfe7490a61644826ba7fe70fd52a377aaca0f4956ba7f/WebOb-1.7.4.tar.gz"; - sha256 = "1na01ljg04z40il7vcrn8g29vaw7nvg1xvhk64cr4jys5wcay44d"; + url = "https://files.pythonhosted.org/packages/e4/6c/99e322c3d4cc11d9060a67a9bf2f7c9c581f40988c11fffe89bb8c36bc5e/WebOb-1.8.4.tar.gz"; + sha256 = "16cfg5y4n6sihz59vsmns2yqbfm0gfsn3l5xgz2g0pdhilaib0x4"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; "webtest" = super.buildPythonPackage { - name = "webtest-2.0.29"; + name = "webtest-2.0.32"; doCheck = false; propagatedBuildInputs = [ self."six" @@ -2202,8 +2238,8 @@ self: super: { self."beautifulsoup4" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/94/de/8f94738be649997da99c47b104aa3c3984ecec51a1d8153ed09638253d56/WebTest-2.0.29.tar.gz"; - sha256 = "0bcj1ica5lnmj5zbvk46x28kgphcsgh7sfnwjmn0cr94mhawrg6v"; + url = "https://files.pythonhosted.org/packages/27/9f/9e74449d272ffbef4fb3012e6dbc53c0b24822d545e7a33a342f80131e59/WebTest-2.0.32.tar.gz"; + sha256 = "0qp0nnbazzm4ibjiyqfcn6f230svk09i4g58zg2i9x1ga06h48a2"; }; meta = { license = [ pkgs.lib.licenses.mit ]; diff --git a/release.nix b/release.nix --- a/release.nix +++ b/release.nix @@ -1,12 +1,15 @@ # This file defines how to "build" for packaging. -{ doCheck ? false +{ pkgs ? import {} +, system ? builtins.currentSystem +, doCheck ? false }: let enterprise_ce = import ./default.nix { inherit - doCheck; + doCheck + system; # disable checkPhase for build checkPhase = '' diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ bleach==3.0.2 celery==4.1.1 chameleon==2.24 channelstream==0.5.2 -click==6.6 +click==7.0 colander==1.5.1 # our custom configobj https://code.rhodecode.com/upstream/configobj/archive/a11ff0a0bd4fbda9e3a91267e720f88329efb4a6.tar.gz?md5=9916c524ea11a6c418217af6b28d4b3c#egg=configobj==5.0.6 @@ -20,7 +20,7 @@ cssselect==1.0.3 decorator==4.1.2 deform==2.0.7 docutils==0.14.0 -dogpile.cache==0.6.7 +dogpile.cache==0.7.1 dogpile.core==0.4.1 ecdsa==0.13 formencode==1.2.4 @@ -28,42 +28,41 @@ future==0.14.3 futures==3.0.2 gnureadline==6.3.8 infrae.cache==1.0.1 -iso8601==0.1.11 +iso8601==0.1.12 itsdangerous==0.24 jinja2==2.9.6 billiard==3.5.0.3 -kombu==4.2.0 +kombu==4.2.1 lxml==4.2.5 mako==1.0.7 markdown==2.6.11 -markupsafe==1.0.0 +markupsafe==1.1.0 msgpack-python==0.5.6 pyotp==2.2.7 packaging==15.2 -paste==2.0.3 -pastedeploy==1.5.2 -pastescript==2.0.2 -pathlib2==2.3.2 +paste==3.0.5 +pastedeploy==2.0.1 +pastescript==3.0.0 +pathlib2==2.3.3 peppercorn==0.6 -psutil==5.4.7 +psutil==5.4.8 py-bcrypt==0.4 pycrypto==2.6.1 pycurl==7.43.0.2 pyflakes==0.8.1 -pygments-markdown-lexer==0.1.0.dev39 -pygments==2.3.0 -pyparsing==1.5.7 +pygments==2.3.1 +pyparsing==2.3.0 pyramid-beaker==0.8 -pyramid-debugtoolbar==4.4.0 +pyramid-debugtoolbar==4.5.0 pyramid-jinja2==2.7 pyramid-mako==1.0.2 -pyramid==1.9.2 +pyramid==1.10.1 pyramid_mailer==0.15.1 python-dateutil python-ldap==3.1.0 python-memcached==1.59 python-pam==1.8.4 -python-saml +python-saml==2.4.2 pytz==2018.4 tzlocal==1.5.1 pyzmq==14.6.0 @@ -72,21 +71,21 @@ redis==2.10.6 repoze.lru==0.7 requests==2.9.1 routes==2.4.1 -simplejson==3.11.1 +simplejson==3.16.0 six==1.11.0 sqlalchemy==1.1.18 sshpubkeys==2.2.0 -subprocess32==3.5.2 -supervisor==3.3.4 +subprocess32==3.5.3 +supervisor==3.3.5 tempita==0.5.2 translationstring==1.3 -urllib3==1.21 +urllib3==1.24.1 urlobject==2.4.3 -venusian==1.1.0 +venusian==1.2.0 weberror==0.10.3 webhelpers2==2.0 webhelpers==1.3 -webob==1.7.4 +webob==1.8.4 whoosh==2.7.4 wsgiref==0.1.2 zope.cachedescriptors==4.3.1 @@ -113,7 +112,7 @@ invoke==0.13.0 bumpversion==0.5.3 ## http servers -gevent==1.3.7 +gevent==1.4.0 greenlet==0.4.15 gunicorn==19.9.0 waitress==1.1.0 @@ -124,7 +123,7 @@ ipdb==0.11.0 ipython==5.1.0 ## rhodecode-tools, special case -https://code.rhodecode.com/rhodecode-tools-ce/archive/v1.0.1.tar.gz?md5=ffb5d6bcb855305b93cfe23ad42e500b#egg=rhodecode-tools==1.0.1 +https://code.rhodecode.com/rhodecode-tools-ce/archive/v1.2.1.tar.gz?md5=25bc2f7de1da318e547236d3fb463d28#egg=rhodecode-tools==1.2.1 ## appenlight appenlight-client==0.6.26 diff --git a/requirements_test.txt b/requirements_test.txt --- a/requirements_test.txt +++ b/requirements_test.txt @@ -9,6 +9,8 @@ pytest-timeout==1.3.2 gprof2dot==2017.9.19 mock==1.0.1 -webtest==2.0.29 cov-core==1.15.0 coverage==4.5.1 + +webtest==2.0.32 +beautifulsoup4==4.6.3 diff --git a/rhodecode/VERSION b/rhodecode/VERSION --- a/rhodecode/VERSION +++ b/rhodecode/VERSION @@ -1,1 +1,1 @@ -4.15.2 \ No newline at end of file +4.16.0 \ No newline at end of file diff --git a/rhodecode/__init__.py b/rhodecode/__init__.py --- a/rhodecode/__init__.py +++ b/rhodecode/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -18,12 +18,6 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ -""" - -RhodeCode, a web based repository management software -versioning implementation: http://www.python.org/dev/peps/pep-0386/ -""" - import os import sys import platform @@ -51,7 +45,7 @@ PYRAMID_SETTINGS = {} EXTENSIONS = {} __version__ = ('.'.join((str(each) for each in VERSION[:3]))) -__dbversion__ = 91 # defines current db version for migrations +__dbversion__ = 95 # defines current db version for migrations __platform__ = platform.system() __license__ = 'AGPLv3, and Commercial License' __author__ = 'RhodeCode GmbH' diff --git a/rhodecode/api/__init__.py b/rhodecode/api/__init__.py --- a/rhodecode/api/__init__.py +++ b/rhodecode/api/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -21,6 +21,7 @@ import inspect import itertools import logging +import sys import types import fnmatch @@ -38,6 +39,7 @@ from rhodecode.api.exc import ( from rhodecode.apps._base import TemplateArgs from rhodecode.lib.auth import AuthUser from rhodecode.lib.base import get_ip_addr, attach_context_attributes +from rhodecode.lib.exc_tracking import store_exception from rhodecode.lib.ext_json import json from rhodecode.lib.utils2 import safe_str from rhodecode.lib.plugins.utils import get_plugin_settings @@ -140,15 +142,14 @@ def jsonrpc_error(request, message, reti def exception_view(exc, request): rpc_id = getattr(request, 'rpc_id', None) - fault_message = 'undefined error' if isinstance(exc, JSONRPCError): - fault_message = exc.message + fault_message = safe_str(exc.message) log.debug('json-rpc error rpc_id:%s "%s"', rpc_id, fault_message) elif isinstance(exc, JSONRPCValidationError): colander_exc = exc.colander_exception # TODO(marcink): think maybe of nicer way to serialize errors ? fault_message = colander_exc.asdict() - log.debug('json-rpc error rpc_id:%s "%s"', rpc_id, fault_message) + log.debug('json-rpc colander error rpc_id:%s "%s"', rpc_id, fault_message) elif isinstance(exc, JSONRPCForbidden): fault_message = 'Access was denied to this resource.' log.warning('json-rpc forbidden call rpc_id:%s "%s"', rpc_id, fault_message) @@ -170,6 +171,10 @@ def exception_view(exc, request): fault_message = "No such method: {}. Similar methods: {}".format( method, similar) + else: + fault_message = 'undefined error' + exc_info = exc.exc_info() + store_exception(id(exc_info), exc_info, prefix='rhodecode-api') return jsonrpc_error(request, fault_message, rpc_id) @@ -292,8 +297,10 @@ def request_view(request): raise except Exception: log.exception('Unhandled exception occurred on api call: %s', func) - return jsonrpc_error(request, retid=request.rpc_id, - message='Internal server error') + exc_info = sys.exc_info() + store_exception(id(exc_info), exc_info, prefix='rhodecode-api') + return jsonrpc_error( + request, retid=request.rpc_id, message='Internal server error') def setup_request(request): @@ -414,8 +421,7 @@ def add_jsonrpc_method(config, view, **k if method is None: raise ConfigurationError( - 'Cannot register a JSON-RPC method without specifying the ' - '"method"') + 'Cannot register a JSON-RPC method without specifying the "method"') # we define custom predicate, to enable to detect conflicting methods, # those predicates are kind of "translation" from the decorator variables @@ -524,6 +530,7 @@ def includeme(config): # match filter by given method only config.add_view_predicate('jsonrpc_method', MethodPredicate) + config.add_view_predicate('jsonrpc_method_not_found', NotFoundPredicate) config.add_renderer(DEFAULT_RENDERER, ExtJsonRenderer( serializer=json.dumps, indent=4)) @@ -538,5 +545,4 @@ def includeme(config): config.scan(plugin_module, ignore='rhodecode.api.tests') # register some exception handling view config.add_view(exception_view, context=JSONRPCBaseError) - config.add_view_predicate('jsonrpc_method_not_found', NotFoundPredicate) config.add_notfound_view(exception_view, jsonrpc_method_not_found=True) diff --git a/rhodecode/api/exc.py b/rhodecode/api/exc.py --- a/rhodecode/api/exc.py +++ b/rhodecode/api/exc.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/__init__.py b/rhodecode/api/tests/__init__.py --- a/rhodecode/api/tests/__init__.py +++ b/rhodecode/api/tests/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/conftest.py b/rhodecode/api/tests/conftest.py --- a/rhodecode/api/tests/conftest.py +++ b/rhodecode/api/tests/conftest.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_add_field_to_repo.py b/rhodecode/api/tests/test_add_field_to_repo.py --- a/rhodecode/api/tests/test_add_field_to_repo.py +++ b/rhodecode/api/tests/test_add_field_to_repo.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_add_user_to_user_group.py b/rhodecode/api/tests/test_add_user_to_user_group.py --- a/rhodecode/api/tests/test_add_user_to_user_group.py +++ b/rhodecode/api/tests/test_add_user_to_user_group.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_api.py b/rhodecode/api/tests/test_api.py --- a/rhodecode/api/tests/test_api.py +++ b/rhodecode/api/tests/test_api.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -86,7 +86,9 @@ class TestApi(object): def test_api_non_existing_method_have_similar(self, request): id_, params = build_data(self.apikey, 'comment', args='xx') response = api_call(self.app, params) - expected = 'No such method: comment. Similar methods: changeset_comment, comment_pull_request, get_pull_request_comments, comment_commit' + expected = 'No such method: comment. ' \ + 'Similar methods: changeset_comment, comment_pull_request, ' \ + 'get_pull_request_comments, comment_commit, get_repo_comments' assert_error(id_, expected, given=response.body) def test_api_disabled_user(self, request): diff --git a/rhodecode/api/tests/test_cleanup_sessions.py b/rhodecode/api/tests/test_cleanup_sessions.py --- a/rhodecode/api/tests/test_cleanup_sessions.py +++ b/rhodecode/api/tests/test_cleanup_sessions.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2017-2018 RhodeCode GmbH +# Copyright (C) 2017-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_close_pull_request.py b/rhodecode/api/tests/test_close_pull_request.py --- a/rhodecode/api/tests/test_close_pull_request.py +++ b/rhodecode/api/tests/test_close_pull_request.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_comment_commit.py b/rhodecode/api/tests/test_comment_commit.py --- a/rhodecode/api/tests/test_comment_commit.py +++ b/rhodecode/api/tests/test_comment_commit.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_comment_pull_request.py b/rhodecode/api/tests/test_comment_pull_request.py --- a/rhodecode/api/tests/test_comment_pull_request.py +++ b/rhodecode/api/tests/test_comment_pull_request.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_create_gist.py b/rhodecode/api/tests/test_create_gist.py --- a/rhodecode/api/tests/test_create_gist.py +++ b/rhodecode/api/tests/test_create_gist.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_create_pull_request.py b/rhodecode/api/tests/test_create_pull_request.py --- a/rhodecode/api/tests/test_create_pull_request.py +++ b/rhodecode/api/tests/test_create_pull_request.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_create_repo.py b/rhodecode/api/tests/test_create_repo.py --- a/rhodecode/api/tests/test_create_repo.py +++ b/rhodecode/api/tests/test_create_repo.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_create_repo_group.py b/rhodecode/api/tests/test_create_repo_group.py --- a/rhodecode/api/tests/test_create_repo_group.py +++ b/rhodecode/api/tests/test_create_repo_group.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_create_user.py b/rhodecode/api/tests/test_create_user.py --- a/rhodecode/api/tests/test_create_user.py +++ b/rhodecode/api/tests/test_create_user.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_create_user_group.py b/rhodecode/api/tests/test_create_user_group.py --- a/rhodecode/api/tests/test_create_user_group.py +++ b/rhodecode/api/tests/test_create_user_group.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_delete_gist.py b/rhodecode/api/tests/test_delete_gist.py --- a/rhodecode/api/tests/test_delete_gist.py +++ b/rhodecode/api/tests/test_delete_gist.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_delete_repo.py b/rhodecode/api/tests/test_delete_repo.py --- a/rhodecode/api/tests/test_delete_repo.py +++ b/rhodecode/api/tests/test_delete_repo.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_delete_repo_group.py b/rhodecode/api/tests/test_delete_repo_group.py --- a/rhodecode/api/tests/test_delete_repo_group.py +++ b/rhodecode/api/tests/test_delete_repo_group.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_delete_user.py b/rhodecode/api/tests/test_delete_user.py --- a/rhodecode/api/tests/test_delete_user.py +++ b/rhodecode/api/tests/test_delete_user.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_delete_user_group.py b/rhodecode/api/tests/test_delete_user_group.py --- a/rhodecode/api/tests/test_delete_user_group.py +++ b/rhodecode/api/tests/test_delete_user_group.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_deprecated_api.py b/rhodecode/api/tests/test_deprecated_api.py --- a/rhodecode/api/tests/test_deprecated_api.py +++ b/rhodecode/api/tests/test_deprecated_api.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_fork_repo.py b/rhodecode/api/tests/test_fork_repo.py --- a/rhodecode/api/tests/test_fork_repo.py +++ b/rhodecode/api/tests/test_fork_repo.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_get_gist.py b/rhodecode/api/tests/test_get_gist.py --- a/rhodecode/api/tests/test_get_gist.py +++ b/rhodecode/api/tests/test_get_gist.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_get_gists.py b/rhodecode/api/tests/test_get_gists.py --- a/rhodecode/api/tests/test_get_gists.py +++ b/rhodecode/api/tests/test_get_gists.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_get_ip.py b/rhodecode/api/tests/test_get_ip.py --- a/rhodecode/api/tests/test_get_ip.py +++ b/rhodecode/api/tests/test_get_ip.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_get_locks.py b/rhodecode/api/tests/test_get_locks.py --- a/rhodecode/api/tests/test_get_locks.py +++ b/rhodecode/api/tests/test_get_locks.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_get_method.py b/rhodecode/api/tests/test_get_method.py --- a/rhodecode/api/tests/test_get_method.py +++ b/rhodecode/api/tests/test_get_method.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -38,7 +38,7 @@ class TestGetMethod(object): response = api_call(self.app, params) expected = ['changeset_comment', 'comment_pull_request', - 'get_pull_request_comments', 'comment_commit'] + 'get_pull_request_comments', 'comment_commit', 'get_repo_comments'] assert_ok(id_, expected, given=response.body) def test_get_methods_on_single_match(self): diff --git a/rhodecode/api/tests/test_get_pull_request.py b/rhodecode/api/tests/test_get_pull_request.py --- a/rhodecode/api/tests/test_get_pull_request.py +++ b/rhodecode/api/tests/test_get_pull_request.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -65,6 +65,7 @@ class TestGetPullRequest(object): 'title': pull_request.title, 'description': pull_request.description, 'status': pull_request.status, + 'state': pull_request.pull_request_state, 'created_on': pull_request.created_on, 'updated_on': pull_request.updated_on, 'commit_ids': pull_request.revisions, diff --git a/rhodecode/api/tests/test_get_pull_request_comments.py b/rhodecode/api/tests/test_get_pull_request_comments.py --- a/rhodecode/api/tests/test_get_pull_request_comments.py +++ b/rhodecode/api/tests/test_get_pull_request_comments.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -59,6 +59,7 @@ class TestGetPullRequestComments(object) 'status_lbl': 'Under Review'}, 'comment_text': 'Auto status change to |new_status|\n\n.. |new_status| replace:: *"Under Review"*', 'comment_type': 'note', + 'comment_resolved_by': None, 'pull_request_version': None} ] assert_ok(id_, expected, response.body) diff --git a/rhodecode/api/tests/test_get_pull_requests.py b/rhodecode/api/tests/test_get_pull_requests.py --- a/rhodecode/api/tests/test_get_pull_requests.py +++ b/rhodecode/api/tests/test_get_pull_requests.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_get_repo.py b/rhodecode/api/tests/test_get_repo.py --- a/rhodecode/api/tests/test_get_repo.py +++ b/rhodecode/api/tests/test_get_repo.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_get_repo_changeset.py b/rhodecode/api/tests/test_get_repo_changeset.py --- a/rhodecode/api/tests/test_get_repo_changeset.py +++ b/rhodecode/api/tests/test_get_repo_changeset.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_get_repo_comments.py b/rhodecode/api/tests/test_get_repo_comments.py new file mode 100644 --- /dev/null +++ b/rhodecode/api/tests/test_get_repo_comments.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2019 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + + +import pytest + +from rhodecode.model.db import User, ChangesetComment +from rhodecode.model.meta import Session +from rhodecode.model.comment import CommentsModel +from rhodecode.api.tests.utils import ( + build_data, api_call, assert_error, assert_call_ok) + + +@pytest.fixture() +def make_repo_comments_factory(request): + + def maker(repo): + user = User.get_first_super_admin() + commit = repo.scm_instance()[0] + + commit_id = commit.raw_id + file_0 = commit.affected_files[0] + comments = [] + + # general + CommentsModel().create( + text='General Comment', repo=repo, user=user, commit_id=commit_id, + comment_type=ChangesetComment.COMMENT_TYPE_NOTE, send_email=False) + + # inline + CommentsModel().create( + text='Inline Comment', repo=repo, user=user, commit_id=commit_id, + f_path=file_0, line_no='n1', + comment_type=ChangesetComment.COMMENT_TYPE_NOTE, send_email=False) + + # todo + CommentsModel().create( + text='INLINE TODO Comment', repo=repo, user=user, commit_id=commit_id, + f_path=file_0, line_no='n1', + comment_type=ChangesetComment.COMMENT_TYPE_TODO, send_email=False) + + @request.addfinalizer + def cleanup(): + for comment in comments: + Session().delete(comment) + return maker + + +@pytest.mark.usefixtures("testuser_api", "app") +class TestGetRepo(object): + + @pytest.mark.parametrize('filters, expected_count', [ + ({}, 3), + ({'comment_type': ChangesetComment.COMMENT_TYPE_NOTE}, 2), + ({'comment_type': ChangesetComment.COMMENT_TYPE_TODO}, 1), + ({'commit_id': 'FILLED DYNAMIC'}, 3), + ]) + def test_api_get_repo_comments(self, backend, user_util, + make_repo_comments_factory, filters, expected_count): + commits = [{'message': 'A'}, {'message': 'B'}] + repo = backend.create_repo(commits=commits) + make_repo_comments_factory(repo) + + api_call_params = {'repoid': repo.repo_name,} + api_call_params.update(filters) + + if 'commit_id' in api_call_params: + commit = repo.scm_instance()[0] + commit_id = commit.raw_id + api_call_params['commit_id'] = commit_id + + id_, params = build_data(self.apikey, 'get_repo_comments', **api_call_params) + response = api_call(self.app, params) + result = assert_call_ok(id_, given=response.body) + + assert len(result) == expected_count + + def test_api_get_repo_comments_wrong_comment_typ(self, backend_hg): + + repo = backend_hg.create_repo() + make_repo_comments_factory(repo) + + api_call_params = {'repoid': repo.repo_name,} + api_call_params.update({'comment_type': 'bogus'}) + + expected = 'comment_type must be one of `{}` got {}'.format( + ChangesetComment.COMMENT_TYPES, 'bogus') + id_, params = build_data(self.apikey, 'get_repo_comments', **api_call_params) + response = api_call(self.app, params) + assert_error(id_, expected, given=response.body) diff --git a/rhodecode/api/tests/test_get_repo_group.py b/rhodecode/api/tests/test_get_repo_group.py --- a/rhodecode/api/tests/test_get_repo_group.py +++ b/rhodecode/api/tests/test_get_repo_group.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_get_repo_groups.py b/rhodecode/api/tests/test_get_repo_groups.py --- a/rhodecode/api/tests/test_get_repo_groups.py +++ b/rhodecode/api/tests/test_get_repo_groups.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_get_repo_nodes.py b/rhodecode/api/tests/test_get_repo_nodes.py --- a/rhodecode/api/tests/test_get_repo_nodes.py +++ b/rhodecode/api/tests/test_get_repo_nodes.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_get_repo_refs.py b/rhodecode/api/tests/test_get_repo_refs.py --- a/rhodecode/api/tests/test_get_repo_refs.py +++ b/rhodecode/api/tests/test_get_repo_refs.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_get_repos.py b/rhodecode/api/tests/test_get_repos.py --- a/rhodecode/api/tests/test_get_repos.py +++ b/rhodecode/api/tests/test_get_repos.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_get_server_info.py b/rhodecode/api/tests/test_get_server_info.py --- a/rhodecode/api/tests/test_get_server_info.py +++ b/rhodecode/api/tests/test_get_server_info.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_get_user.py b/rhodecode/api/tests/test_get_user.py --- a/rhodecode/api/tests/test_get_user.py +++ b/rhodecode/api/tests/test_get_user.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_get_user_group.py b/rhodecode/api/tests/test_get_user_group.py --- a/rhodecode/api/tests/test_get_user_group.py +++ b/rhodecode/api/tests/test_get_user_group.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_get_user_groups.py b/rhodecode/api/tests/test_get_user_groups.py --- a/rhodecode/api/tests/test_get_user_groups.py +++ b/rhodecode/api/tests/test_get_user_groups.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_get_users.py b/rhodecode/api/tests/test_get_users.py --- a/rhodecode/api/tests/test_get_users.py +++ b/rhodecode/api/tests/test_get_users.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_grant_user_group_permission.py b/rhodecode/api/tests/test_grant_user_group_permission.py --- a/rhodecode/api/tests/test_grant_user_group_permission.py +++ b/rhodecode/api/tests/test_grant_user_group_permission.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_grant_user_group_permission_to_repo_group.py b/rhodecode/api/tests/test_grant_user_group_permission_to_repo_group.py --- a/rhodecode/api/tests/test_grant_user_group_permission_to_repo_group.py +++ b/rhodecode/api/tests/test_grant_user_group_permission_to_repo_group.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_grant_user_group_permission_to_user_group.py b/rhodecode/api/tests/test_grant_user_group_permission_to_user_group.py --- a/rhodecode/api/tests/test_grant_user_group_permission_to_user_group.py +++ b/rhodecode/api/tests/test_grant_user_group_permission_to_user_group.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_grant_user_permission.py b/rhodecode/api/tests/test_grant_user_permission.py --- a/rhodecode/api/tests/test_grant_user_permission.py +++ b/rhodecode/api/tests/test_grant_user_permission.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_grant_user_permission_to_repo_group.py b/rhodecode/api/tests/test_grant_user_permission_to_repo_group.py --- a/rhodecode/api/tests/test_grant_user_permission_to_repo_group.py +++ b/rhodecode/api/tests/test_grant_user_permission_to_repo_group.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_grant_user_permission_to_user_group.py b/rhodecode/api/tests/test_grant_user_permission_to_user_group.py --- a/rhodecode/api/tests/test_grant_user_permission_to_user_group.py +++ b/rhodecode/api/tests/test_grant_user_permission_to_user_group.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_invalidate_cache.py b/rhodecode/api/tests/test_invalidate_cache.py --- a/rhodecode/api/tests/test_invalidate_cache.py +++ b/rhodecode/api/tests/test_invalidate_cache.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_merge_pull_request.py b/rhodecode/api/tests/test_merge_pull_request.py --- a/rhodecode/api/tests/test_merge_pull_request.py +++ b/rhodecode/api/tests/test_merge_pull_request.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -29,11 +29,10 @@ from rhodecode.api.tests.utils import ( @pytest.mark.usefixtures("testuser_api", "app") class TestMergePullRequest(object): + @pytest.mark.backends("git", "hg") def test_api_merge_pull_request_merge_failed(self, pr_util, no_notifications): pull_request = pr_util.create_pull_request(mergeable=True) - author = pull_request.user_id - repo = pull_request.target_repo.repo_id pull_request_id = pull_request.pull_request_id pull_request_repo = pull_request.target_repo.repo_name @@ -46,8 +45,7 @@ class TestMergePullRequest(object): # The above api call detaches the pull request DB object from the # session because of an unconditional transaction rollback in our - # middleware. Therefore we need to add it back here if we want to use - # it. + # middleware. Therefore we need to add it back here if we want to use it. Session().add(pull_request) expected = 'merge not possible for following reasons: ' \ @@ -55,6 +53,29 @@ class TestMergePullRequest(object): assert_error(id_, expected, given=response.body) @pytest.mark.backends("git", "hg") + def test_api_merge_pull_request_merge_failed_disallowed_state( + self, pr_util, no_notifications): + pull_request = pr_util.create_pull_request(mergeable=True, approved=True) + pull_request_id = pull_request.pull_request_id + pull_request_repo = pull_request.target_repo.repo_name + + pr = PullRequest.get(pull_request_id) + pr.pull_request_state = pull_request.STATE_UPDATING + Session().add(pr) + Session().commit() + + id_, params = build_data( + self.apikey, 'merge_pull_request', + repoid=pull_request_repo, + pullrequestid=pull_request_id) + + response = api_call(self.app, params) + expected = 'Operation forbidden because pull request is in state {}, '\ + 'only state {} is allowed.'.format(PullRequest.STATE_UPDATING, + PullRequest.STATE_CREATED) + assert_error(id_, expected, given=response.body) + + @pytest.mark.backends("git", "hg") def test_api_merge_pull_request(self, pr_util, no_notifications): pull_request = pr_util.create_pull_request(mergeable=True, approved=True) author = pull_request.user_id @@ -88,6 +109,7 @@ class TestMergePullRequest(object): expected = { 'executed': True, 'failure_reason': 0, + 'merge_status_message': 'This pull request can be automatically merged.', 'possible': True, 'merge_commit_id': pull_request.shadow_merge_ref.commit_id, 'merge_ref': pull_request.shadow_merge_ref._asdict() @@ -112,6 +134,107 @@ class TestMergePullRequest(object): assert_error(id_, expected, given=response.body) @pytest.mark.backends("git", "hg") + def test_api_merge_pull_request_as_another_user_no_perms_to_merge( + self, pr_util, no_notifications, user_util): + merge_user = user_util.create_user() + merge_user_id = merge_user.user_id + merge_user_username = merge_user.username + + pull_request = pr_util.create_pull_request(mergeable=True, approved=True) + + pull_request_id = pull_request.pull_request_id + pull_request_repo = pull_request.target_repo.repo_name + + id_, params = build_data( + self.apikey, 'comment_pull_request', + repoid=pull_request_repo, + pullrequestid=pull_request_id, + status='approved') + + response = api_call(self.app, params) + expected = { + 'comment_id': response.json.get('result', {}).get('comment_id'), + 'pull_request_id': pull_request_id, + 'status': {'given': 'approved', 'was_changed': True} + } + assert_ok(id_, expected, given=response.body) + id_, params = build_data( + self.apikey, 'merge_pull_request', + repoid=pull_request_repo, + pullrequestid=pull_request_id, + userid=merge_user_id + ) + + response = api_call(self.app, params) + expected = 'merge not possible for following reasons: User `{}` ' \ + 'not allowed to perform merge.'.format(merge_user_username) + assert_error(id_, expected, response.body) + + @pytest.mark.backends("git", "hg") + def test_api_merge_pull_request_as_another_user(self, pr_util, no_notifications, user_util): + merge_user = user_util.create_user() + merge_user_id = merge_user.user_id + pull_request = pr_util.create_pull_request(mergeable=True, approved=True) + user_util.grant_user_permission_to_repo( + pull_request.target_repo, merge_user, 'repository.write') + author = pull_request.user_id + repo = pull_request.target_repo.repo_id + pull_request_id = pull_request.pull_request_id + pull_request_repo = pull_request.target_repo.repo_name + + id_, params = build_data( + self.apikey, 'comment_pull_request', + repoid=pull_request_repo, + pullrequestid=pull_request_id, + status='approved') + + response = api_call(self.app, params) + expected = { + 'comment_id': response.json.get('result', {}).get('comment_id'), + 'pull_request_id': pull_request_id, + 'status': {'given': 'approved', 'was_changed': True} + } + assert_ok(id_, expected, given=response.body) + + id_, params = build_data( + self.apikey, 'merge_pull_request', + repoid=pull_request_repo, + pullrequestid=pull_request_id, + userid=merge_user_id + ) + + response = api_call(self.app, params) + + pull_request = PullRequest.get(pull_request_id) + + expected = { + 'executed': True, + 'failure_reason': 0, + 'merge_status_message': 'This pull request can be automatically merged.', + 'possible': True, + 'merge_commit_id': pull_request.shadow_merge_ref.commit_id, + 'merge_ref': pull_request.shadow_merge_ref._asdict() + } + + assert_ok(id_, expected, response.body) + + journal = UserLog.query() \ + .filter(UserLog.user_id == merge_user_id) \ + .filter(UserLog.repository_id == repo) \ + .order_by('user_log_id') \ + .all() + assert journal[-2].action == 'repo.pull_request.merge' + assert journal[-1].action == 'repo.pull_request.close' + + id_, params = build_data( + self.apikey, 'merge_pull_request', + repoid=pull_request_repo, pullrequestid=pull_request_id, userid=merge_user_id) + response = api_call(self.app, params) + + expected = 'merge not possible for following reasons: This pull request is closed.' + assert_error(id_, expected, given=response.body) + + @pytest.mark.backends("git", "hg") def test_api_merge_pull_request_repo_error(self, pr_util): pull_request = pr_util.create_pull_request() id_, params = build_data( @@ -123,8 +246,7 @@ class TestMergePullRequest(object): assert_error(id_, expected, given=response.body) @pytest.mark.backends("git", "hg") - def test_api_merge_pull_request_non_admin_with_userid_error(self, - pr_util): + def test_api_merge_pull_request_non_admin_with_userid_error(self, pr_util): pull_request = pr_util.create_pull_request(mergeable=True) id_, params = build_data( self.apikey_regular, 'merge_pull_request', diff --git a/rhodecode/api/tests/test_pull.py b/rhodecode/api/tests/test_pull.py --- a/rhodecode/api/tests/test_pull.py +++ b/rhodecode/api/tests/test_pull.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_remove_field_from_repo.py b/rhodecode/api/tests/test_remove_field_from_repo.py --- a/rhodecode/api/tests/test_remove_field_from_repo.py +++ b/rhodecode/api/tests/test_remove_field_from_repo.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_remove_user_from_user_group.py b/rhodecode/api/tests/test_remove_user_from_user_group.py --- a/rhodecode/api/tests/test_remove_user_from_user_group.py +++ b/rhodecode/api/tests/test_remove_user_from_user_group.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_repo_locking.py b/rhodecode/api/tests/test_repo_locking.py --- a/rhodecode/api/tests/test_repo_locking.py +++ b/rhodecode/api/tests/test_repo_locking.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_rescan_repos.py b/rhodecode/api/tests/test_rescan_repos.py --- a/rhodecode/api/tests/test_rescan_repos.py +++ b/rhodecode/api/tests/test_rescan_repos.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_revoke_user_group_permission.py b/rhodecode/api/tests/test_revoke_user_group_permission.py --- a/rhodecode/api/tests/test_revoke_user_group_permission.py +++ b/rhodecode/api/tests/test_revoke_user_group_permission.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_revoke_user_group_permission_from_repo_group.py b/rhodecode/api/tests/test_revoke_user_group_permission_from_repo_group.py --- a/rhodecode/api/tests/test_revoke_user_group_permission_from_repo_group.py +++ b/rhodecode/api/tests/test_revoke_user_group_permission_from_repo_group.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_revoke_user_group_permission_from_user_group.py b/rhodecode/api/tests/test_revoke_user_group_permission_from_user_group.py --- a/rhodecode/api/tests/test_revoke_user_group_permission_from_user_group.py +++ b/rhodecode/api/tests/test_revoke_user_group_permission_from_user_group.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_revoke_user_permission.py b/rhodecode/api/tests/test_revoke_user_permission.py --- a/rhodecode/api/tests/test_revoke_user_permission.py +++ b/rhodecode/api/tests/test_revoke_user_permission.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_revoke_user_permission_from_repo_group.py b/rhodecode/api/tests/test_revoke_user_permission_from_repo_group.py --- a/rhodecode/api/tests/test_revoke_user_permission_from_repo_group.py +++ b/rhodecode/api/tests/test_revoke_user_permission_from_repo_group.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_revoke_user_permission_from_user_group.py b/rhodecode/api/tests/test_revoke_user_permission_from_user_group.py --- a/rhodecode/api/tests/test_revoke_user_permission_from_user_group.py +++ b/rhodecode/api/tests/test_revoke_user_permission_from_user_group.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_store_exception.py b/rhodecode/api/tests/test_store_exception.py new file mode 100644 --- /dev/null +++ b/rhodecode/api/tests/test_store_exception.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2019 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + + +import pytest + +from rhodecode.api.tests.utils import build_data, api_call, assert_ok, assert_error + + +@pytest.mark.usefixtures("testuser_api", "app") +class TestStoreException(object): + + def test_store_exception_invalid_json(self): + id_, params = build_data(self.apikey, 'store_exception', + exc_data_json='XXX,{') + response = api_call(self.app, params) + + expected = 'Failed to parse JSON data from exc_data_json field. ' \ + 'Please make sure it contains a valid JSON.' + assert_error(id_, expected, given=response.body) + + def test_store_exception_missing_json_params_json(self): + id_, params = build_data(self.apikey, 'store_exception', + exc_data_json='{"foo":"bar"}') + response = api_call(self.app, params) + + expected = "Missing exc_traceback, or exc_type_name in " \ + "exc_data_json field. Missing: 'exc_traceback'" + assert_error(id_, expected, given=response.body) + + def test_store_exception(self): + id_, params = build_data( + self.apikey, 'store_exception', + exc_data_json='{"exc_traceback": "invalid", "exc_type_name":"ValueError"}') + response = api_call(self.app, params) + exc_id = response.json['result']['exc_id'] + + expected = { + 'exc_id': exc_id, + 'exc_url': 'http://example.com/_admin/settings/exceptions/{}'.format(exc_id) + } + assert_ok(id_, expected, given=response.body) diff --git a/rhodecode/api/tests/test_update_pull_request.py b/rhodecode/api/tests/test_update_pull_request.py --- a/rhodecode/api/tests/test_update_pull_request.py +++ b/rhodecode/api/tests/test_update_pull_request.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_update_repo.py b/rhodecode/api/tests/test_update_repo.py --- a/rhodecode/api/tests/test_update_repo.py +++ b/rhodecode/api/tests/test_update_repo.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_update_repo_group.py b/rhodecode/api/tests/test_update_repo_group.py --- a/rhodecode/api/tests/test_update_repo_group.py +++ b/rhodecode/api/tests/test_update_repo_group.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_update_user.py b/rhodecode/api/tests/test_update_user.py --- a/rhodecode/api/tests/test_update_user.py +++ b/rhodecode/api/tests/test_update_user.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_update_user_group.py b/rhodecode/api/tests/test_update_user_group.py --- a/rhodecode/api/tests/test_update_user_group.py +++ b/rhodecode/api/tests/test_update_user_group.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/test_utils.py b/rhodecode/api/tests/test_utils.py --- a/rhodecode/api/tests/test_utils.py +++ b/rhodecode/api/tests/test_utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/tests/utils.py b/rhodecode/api/tests/utils.py --- a/rhodecode/api/tests/utils.py +++ b/rhodecode/api/tests/utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -28,6 +28,19 @@ from rhodecode.lib.ext_json import json API_URL = '/_admin/api' +def assert_call_ok(id_, given): + expected = jsonify({ + 'id': id_, + 'error': None, + 'result': None + }) + given = json.loads(given) + + assert expected['id'] == given['id'] + assert expected['error'] == given['error'] + return given['result'] + + def assert_ok(id_, expected, given): expected = jsonify({ 'id': id_, @@ -55,8 +68,6 @@ def jsonify(obj): def build_data(apikey, method, **kw): """ Builds API data with given random ID - - :param random_id: """ random_id = random.randrange(1, 9999) return random_id, json.dumps({ diff --git a/rhodecode/api/utils.py b/rhodecode/api/utils.py --- a/rhodecode/api/utils.py +++ b/rhodecode/api/utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -30,7 +30,7 @@ from rhodecode.lib.auth import ( HasPermissionAnyApi, HasRepoPermissionAnyApi, HasRepoGroupPermissionAnyApi) from rhodecode.lib.utils import safe_unicode from rhodecode.lib.vcs.exceptions import RepositoryError -from rhodecode.controllers.utils import get_commit_from_ref_name +from rhodecode.lib.view_utils import get_commit_from_ref_name from rhodecode.lib.utils2 import str2bool log = logging.getLogger(__name__) diff --git a/rhodecode/api/views/__init__.py b/rhodecode/api/views/__init__.py --- a/rhodecode/api/views/__init__.py +++ b/rhodecode/api/views/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2015-2018 RhodeCode GmbH +# Copyright (C) 2015-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/views/deprecated_api.py b/rhodecode/api/views/deprecated_api.py --- a/rhodecode/api/views/deprecated_api.py +++ b/rhodecode/api/views/deprecated_api.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/views/gist_api.py b/rhodecode/api/views/gist_api.py --- a/rhodecode/api/views/gist_api.py +++ b/rhodecode/api/views/gist_api.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/views/pull_request_api.py b/rhodecode/api/views/pull_request_api.py --- a/rhodecode/api/views/pull_request_api.py +++ b/rhodecode/api/views/pull_request_api.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -26,13 +26,13 @@ from rhodecode.api import jsonrpc_method from rhodecode.api.utils import ( has_superadmin_permission, Optional, OAttr, get_repo_or_error, get_pull_request_or_error, get_commit_or_error, get_user_or_error, - validate_repo_permissions, resolve_ref_or_error) + validate_repo_permissions, resolve_ref_or_error, validate_set_owner_permissions) from rhodecode.lib.auth import (HasRepoPermissionAnyApi) from rhodecode.lib.base import vcs_operation_context from rhodecode.lib.utils2 import str2bool from rhodecode.model.changeset_status import ChangesetStatusModel from rhodecode.model.comment import CommentsModel -from rhodecode.model.db import Session, ChangesetStatus, ChangesetComment +from rhodecode.model.db import Session, ChangesetStatus, ChangesetComment, PullRequest from rhodecode.model.pull_request import PullRequestModel, MergeCheck from rhodecode.model.settings import SettingsModel from rhodecode.model.validation_schema import Invalid @@ -128,16 +128,21 @@ def get_pull_request(request, apiuser, p else: repo = pull_request.target_repo - if not PullRequestModel().check_user_read( - pull_request, apiuser, api=True): + if not PullRequestModel().check_user_read(pull_request, apiuser, api=True): raise JSONRPCError('repository `%s` or pull request `%s` ' 'does not exist' % (repoid, pullrequestid)) - data = pull_request.get_api_data() + + # NOTE(marcink): only calculate and return merge state if the pr state is 'created' + # otherwise we can lock the repo on calculation of merge state while update/merge + # is happening. + merge_state = pull_request.pull_request_state == pull_request.STATE_CREATED + data = pull_request.get_api_data(with_merge_state=merge_state) return data @jsonrpc_method() -def get_pull_requests(request, apiuser, repoid, status=Optional('new')): +def get_pull_requests(request, apiuser, repoid, status=Optional('new'), + merge_state=Optional(True)): """ Get all pull requests from the repository specified in `repoid`. @@ -151,6 +156,9 @@ def get_pull_requests(request, apiuser, * ``open`` * ``closed`` :type status: str + :param merge_state: Optional calculate merge state for each repository. + This could result in longer time to fetch the data + :type merge_state: bool Example output: @@ -228,8 +236,10 @@ def get_pull_requests(request, apiuser, validate_repo_permissions(apiuser, repoid, repo, _perms) status = Optional.extract(status) - pull_requests = PullRequestModel().get_all(repo, statuses=[status]) - data = [pr.get_api_data() for pr in pull_requests] + merge_state = Optional.extract(merge_state, binary=True) + pull_requests = PullRequestModel().get_all(repo, statuses=[status], + order_by='id', order_dir='desc') + data = [pr.get_api_data(with_merge_state=merge_state) for pr in pull_requests] return data @@ -257,10 +267,11 @@ def merge_pull_request( "id": , "result": { - "executed": "", - "failure_reason": "", - "merge_commit_id": "", - "possible": "", + "executed": "", + "failure_reason": "", + "merge_status_message": "", + "merge_commit_id": "", + "possible": "", "merge_ref": { "commit_id": "", "type": "", @@ -274,17 +285,25 @@ def merge_pull_request( repo = get_repo_or_error(repoid) else: repo = pull_request.target_repo - + auth_user = apiuser if not isinstance(userid, Optional): if (has_superadmin_permission(apiuser) or HasRepoPermissionAnyApi('repository.admin')( user=apiuser, repo_name=repo.repo_name)): apiuser = get_user_or_error(userid) + auth_user = apiuser.AuthUser() else: raise JSONRPCError('userid is not the same as your user') - check = MergeCheck.validate( - pull_request, auth_user=apiuser, translator=request.translate) + if pull_request.pull_request_state != PullRequest.STATE_CREATED: + raise JSONRPCError( + 'Operation forbidden because pull request is in state {}, ' + 'only state {} is allowed.'.format( + pull_request.pull_request_state, PullRequest.STATE_CREATED)) + + with pull_request.set_state(PullRequest.STATE_UPDATING): + check = MergeCheck.validate(pull_request, auth_user=auth_user, + translator=request.translate) merge_possible = not check.failed if not merge_possible: @@ -300,20 +319,20 @@ def merge_pull_request( target_repo = pull_request.target_repo extras = vcs_operation_context( request.environ, repo_name=target_repo.repo_name, - username=apiuser.username, action='push', + username=auth_user.username, action='push', scm=target_repo.repo_type) - merge_response = PullRequestModel().merge_repo( - pull_request, apiuser, extras=extras) + with pull_request.set_state(PullRequest.STATE_UPDATING): + merge_response = PullRequestModel().merge_repo( + pull_request, apiuser, extras=extras) if merge_response.executed: - PullRequestModel().close_pull_request( - pull_request.pull_request_id, apiuser) + PullRequestModel().close_pull_request(pull_request.pull_request_id, auth_user) Session().commit() # In previous versions the merge response directly contained the merge # commit id. It is now contained in the merge reference object. To be # backwards compatible we have to extract it again. - merge_response = merge_response._asdict() + merge_response = merge_response.asdict() merge_response['merge_commit_id'] = merge_response['merge_ref'].commit_id return merge_response @@ -474,14 +493,21 @@ def comment_pull_request( else: repo = pull_request.target_repo + auth_user = apiuser if not isinstance(userid, Optional): if (has_superadmin_permission(apiuser) or HasRepoPermissionAnyApi('repository.admin')( user=apiuser, repo_name=repo.repo_name)): apiuser = get_user_or_error(userid) + auth_user = apiuser.AuthUser() else: raise JSONRPCError('userid is not the same as your user') + if pull_request.is_closed(): + raise JSONRPCError( + 'pull request `%s` comment failed, pull request is closed' % ( + pullrequestid,)) + if not PullRequestModel().check_user_read( pull_request, apiuser, api=True): raise JSONRPCError('repository `%s` does not exist' % (repoid,)) @@ -549,7 +575,7 @@ def comment_pull_request( renderer=renderer, comment_type=comment_type, resolves_comment_id=resolves_comment_id, - auth_user=apiuser + auth_user=auth_user ) if allowed_to_change_status and status: @@ -589,8 +615,8 @@ def comment_pull_request( @jsonrpc_method() def create_pull_request( request, apiuser, source_repo, target_repo, source_ref, target_ref, - title=Optional(''), description=Optional(''), description_renderer=Optional(''), - reviewers=Optional(None)): + owner=Optional(OAttr('apiuser')), title=Optional(''), description=Optional(''), + description_renderer=Optional(''), reviewers=Optional(None)): """ Creates a new pull request. @@ -611,6 +637,8 @@ def create_pull_request( :type source_ref: str :param target_ref: Set the target ref name. :type target_ref: str + :param owner: user_id or username + :type owner: Optional(str) :param title: Optionally Set the pull request title, it's generated otherwise :type title: str :param description: Set the pull request description. @@ -634,6 +662,8 @@ def create_pull_request( _perms = ('repository.admin', 'repository.write', 'repository.read',) validate_repo_permissions(apiuser, source_repo, source_db_repo, _perms) + owner = validate_set_owner_permissions(apiuser, owner) + full_source_ref = resolve_ref_or_error(source_ref, source_db_repo) full_target_ref = resolve_ref_or_error(target_ref, target_db_repo) @@ -679,7 +709,7 @@ def create_pull_request( # recalculate reviewers logic, to make sure we can validate this reviewer_rules = get_default_reviewers_data( - apiuser.get_instance(), source_db_repo, + owner, source_db_repo, source_commit, target_db_repo, target_commit) # now MERGE our given with the calculated @@ -706,7 +736,7 @@ def create_pull_request( description_renderer = Optional.extract(description_renderer) or default_system_renderer pull_request = PullRequestModel().create( - created_by=apiuser.user_id, + created_by=owner.user_id, source_repo=source_repo, source_ref=full_source_ref, target_repo=target_repo, @@ -844,11 +874,18 @@ def update_pull_request( commit_changes = {"added": [], "common": [], "removed": []} if str2bool(Optional.extract(update_commits)): - if PullRequestModel().has_valid_update_type(pull_request): - update_response = PullRequestModel().update_commits( - pull_request) - commit_changes = update_response.changes or commit_changes - Session().commit() + + if pull_request.pull_request_state != PullRequest.STATE_CREATED: + raise JSONRPCError( + 'Operation forbidden because pull request is in state {}, ' + 'only state {} is allowed.'.format( + pull_request.pull_request_state, PullRequest.STATE_CREATED)) + + with pull_request.set_state(PullRequest.STATE_UPDATING): + if PullRequestModel().has_valid_update_type(pull_request): + update_response = PullRequestModel().update_commits(pull_request) + commit_changes = update_response.changes or commit_changes + Session().commit() reviewers_changes = {"added": [], "removed": []} if reviewers: diff --git a/rhodecode/api/views/repo_api.py b/rhodecode/api/views/repo_api.py --- a/rhodecode/api/views/repo_api.py +++ b/rhodecode/api/views/repo_api.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -29,13 +29,15 @@ from rhodecode.api.utils import ( get_user_group_or_error, get_user_or_error, validate_repo_permissions, get_perm_or_error, parse_args, get_origin, build_commit_data, validate_set_owner_permissions) -from rhodecode.lib import audit_logger +from rhodecode.lib import audit_logger, rc_cache from rhodecode.lib import repo_maintenance from rhodecode.lib.auth import HasPermissionAnyApi, HasUserGroupPermissionAnyApi from rhodecode.lib.celerylib.utils import get_task_id -from rhodecode.lib.utils2 import str2bool, time_to_datetime, safe_str +from rhodecode.lib.utils2 import str2bool, time_to_datetime, safe_str, safe_int from rhodecode.lib.ext_json import json from rhodecode.lib.exceptions import StatusChangeOnClosedPullRequestError +from rhodecode.lib.vcs import RepositoryError +from rhodecode.lib.vcs.exceptions import NodeDoesNotExistError from rhodecode.model.changeset_status import ChangesetStatusModel from rhodecode.model.comment import CommentsModel from rhodecode.model.db import ( @@ -380,7 +382,7 @@ def get_repo_changesets(request, apiuser try: commits = vcs_repo.get_commits( - start_id=start_rev, pre_load=pre_load) + start_id=start_rev, pre_load=pre_load, translate_tags=False) except TypeError as e: raise JSONRPCError(safe_str(e)) except Exception: @@ -428,8 +430,8 @@ def get_repo_nodes(request, apiuser, rep ``all`` (default), ``files`` and ``dirs``. :type ret_type: Optional(str) :param details: Returns extended information about nodes, such as - md5, binary, and or content. The valid options are ``basic`` and - ``full``. + md5, binary, and or content. + The valid options are ``basic`` and ``full``. :type details: Optional(str) :param max_file_bytes: Only return file content under this file size bytes :type details: Optional(int) @@ -440,12 +442,17 @@ def get_repo_nodes(request, apiuser, rep id : result: [ - { - "name" : "" - "type" : "", - "binary": "" (only in extended mode) - "md5" : "" (only in extended mode) - }, + { + "binary": false, + "content": "File line\nLine2\n", + "extension": "md", + "lines": 2, + "md5": "059fa5d29b19c0657e384749480f6422", + "mimetype": "text/x-minidsrc", + "name": "file.md", + "size": 580, + "type": "file" + }, ... ] error: null @@ -453,16 +460,14 @@ def get_repo_nodes(request, apiuser, rep repo = get_repo_or_error(repoid) if not has_superadmin_permission(apiuser): - _perms = ( - 'repository.admin', 'repository.write', 'repository.read',) + _perms = ('repository.admin', 'repository.write', 'repository.read',) validate_repo_permissions(apiuser, repoid, repo, _perms) ret_type = Optional.extract(ret_type) details = Optional.extract(details) _extended_types = ['basic', 'full'] if details not in _extended_types: - raise JSONRPCError( - 'ret_type must be one of %s' % (','.join(_extended_types))) + raise JSONRPCError('ret_type must be one of %s' % (','.join(_extended_types))) extended_info = False content = False if details == 'basic': @@ -499,6 +504,149 @@ def get_repo_nodes(request, apiuser, rep @jsonrpc_method() +def get_repo_file(request, apiuser, repoid, commit_id, file_path, + max_file_bytes=Optional(None), details=Optional('basic'), + cache=Optional(True)): + """ + Returns a single file from repository at given revision. + + This command can only be run using an |authtoken| with admin rights, + or users with at least read rights to |repos|. + + :param apiuser: This is filled automatically from the |authtoken|. + :type apiuser: AuthUser + :param repoid: The repository name or repository ID. + :type repoid: str or int + :param commit_id: The revision for which listing should be done. + :type commit_id: str + :param file_path: The path from which to start displaying. + :type file_path: str + :param details: Returns different set of information about nodes. + The valid options are ``minimal`` ``basic`` and ``full``. + :type details: Optional(str) + :param max_file_bytes: Only return file content under this file size bytes + :type max_file_bytes: Optional(int) + :param cache: Use internal caches for fetching files. If disabled fetching + files is slower but more memory efficient + :type cache: Optional(bool) + Example output: + + .. code-block:: bash + + id : + result: { + "binary": false, + "extension": "py", + "lines": 35, + "content": "....", + "md5": "76318336366b0f17ee249e11b0c99c41", + "mimetype": "text/x-python", + "name": "python.py", + "size": 817, + "type": "file", + } + error: null + """ + + repo = get_repo_or_error(repoid) + if not has_superadmin_permission(apiuser): + _perms = ('repository.admin', 'repository.write', 'repository.read',) + validate_repo_permissions(apiuser, repoid, repo, _perms) + + cache = Optional.extract(cache, binary=True) + details = Optional.extract(details) + _extended_types = ['minimal', 'minimal+search', 'basic', 'full'] + if details not in _extended_types: + raise JSONRPCError( + 'ret_type must be one of %s, got %s' % (','.join(_extended_types)), details) + extended_info = False + content = False + + if details == 'minimal': + extended_info = False + + elif details == 'basic': + extended_info = True + + elif details == 'full': + extended_info = content = True + + try: + # check if repo is not empty by any chance, skip quicker if it is. + _scm = repo.scm_instance() + if _scm.is_empty(): + return None + + node = ScmModel().get_node( + repo, commit_id, file_path, extended_info=extended_info, + content=content, max_file_bytes=max_file_bytes, cache=cache) + except NodeDoesNotExistError: + raise JSONRPCError('There is no file in repo: `{}` at path `{}` for commit: `{}`'.format( + repo.repo_name, file_path, commit_id)) + except Exception: + log.exception("Exception occurred while trying to get repo %s file", + repo.repo_name) + raise JSONRPCError('failed to get repo: `{}` file at path {}'.format( + repo.repo_name, file_path)) + + return node + + +@jsonrpc_method() +def get_repo_fts_tree(request, apiuser, repoid, commit_id, root_path): + """ + Returns a list of tree nodes for path at given revision. This api is built + strictly for usage in full text search building, and shouldn't be consumed + + This command can only be run using an |authtoken| with admin rights, + or users with at least read rights to |repos|. + + """ + + repo = get_repo_or_error(repoid) + if not has_superadmin_permission(apiuser): + _perms = ('repository.admin', 'repository.write', 'repository.read',) + validate_repo_permissions(apiuser, repoid, repo, _perms) + + repo_id = repo.repo_id + cache_seconds = safe_int(rhodecode.CONFIG.get('rc_cache.cache_repo.expiration_time')) + cache_on = cache_seconds > 0 + + cache_namespace_uid = 'cache_repo.{}'.format(repo_id) + region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid) + + @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, + condition=cache_on) + def compute_fts_tree(repo_id, commit_id, root_path, cache_ver): + return ScmModel().get_fts_data(repo_id, commit_id, root_path) + + try: + # check if repo is not empty by any chance, skip quicker if it is. + _scm = repo.scm_instance() + if _scm.is_empty(): + return [] + except RepositoryError: + log.exception("Exception occurred while trying to get repo nodes") + raise JSONRPCError('failed to get repo: `%s` nodes' % repo.repo_name) + + try: + # we need to resolve commit_id to a FULL sha for cache to work correctly. + # sending 'master' is a pointer that needs to be translated to current commit. + commit_id = _scm.get_commit(commit_id=commit_id).raw_id + log.debug( + 'Computing FTS REPO TREE for repo_id %s commit_id `%s` ' + 'with caching: %s[TTL: %ss]' % ( + repo_id, commit_id, cache_on, cache_seconds or 0)) + + tree_files = compute_fts_tree(repo_id, commit_id, root_path, 'v1') + return tree_files + + except Exception: + log.exception("Exception occurred while trying to get repo nodes") + raise JSONRPCError('failed to get repo: `%s` nodes' % repo.repo_name) + + +@jsonrpc_method() def get_repo_refs(request, apiuser, repoid): """ Returns a dictionary of current references. It returns @@ -1506,6 +1654,73 @@ def comment_commit( @jsonrpc_method() +def get_repo_comments(request, apiuser, repoid, + commit_id=Optional(None), comment_type=Optional(None), + userid=Optional(None)): + """ + Get all comments for a repository + + :param apiuser: This is filled automatically from the |authtoken|. + :type apiuser: AuthUser + :param repoid: Set the repository name or repository ID. + :type repoid: str or int + :param commit_id: Optionally filter the comments by the commit_id + :type commit_id: Optional(str), default: None + :param comment_type: Optionally filter the comments by the comment_type + one of: 'note', 'todo' + :type comment_type: Optional(str), default: None + :param userid: Optionally filter the comments by the author of comment + :type userid: Optional(str or int), Default: None + + Example error output: + + .. code-block:: bash + + { + "id" : , + "result" : [ + { + "comment_author": , + "comment_created_on": "2017-02-01T14:38:16.309", + "comment_f_path": "file.txt", + "comment_id": 282, + "comment_lineno": "n1", + "comment_resolved_by": null, + "comment_status": [], + "comment_text": "This file needs a header", + "comment_type": "todo" + } + ], + "error" : null + } + + """ + repo = get_repo_or_error(repoid) + if not has_superadmin_permission(apiuser): + _perms = ('repository.read', 'repository.write', 'repository.admin') + validate_repo_permissions(apiuser, repoid, repo, _perms) + + commit_id = Optional.extract(commit_id) + + userid = Optional.extract(userid) + if userid: + user = get_user_or_error(userid) + else: + user = None + + comment_type = Optional.extract(comment_type) + if comment_type and comment_type not in ChangesetComment.COMMENT_TYPES: + raise JSONRPCError( + 'comment_type must be one of `{}` got {}'.format( + ChangesetComment.COMMENT_TYPES, comment_type) + ) + + comments = CommentsModel().get_repository_comments( + repo=repo, comment_type=comment_type, user=user, commit_id=commit_id) + return comments + + +@jsonrpc_method() def grant_user_permission(request, apiuser, repoid, userid, perm): """ Grant permissions for the specified user on the given repository, @@ -1543,9 +1758,18 @@ def grant_user_permission(request, apius _perms = ('repository.admin',) validate_repo_permissions(apiuser, repoid, repo, _perms) + perm_additions = [[user.user_id, perm.permission_name, "user"]] try: + changes = RepoModel().update_permissions( + repo=repo, perm_additions=perm_additions, cur_user=apiuser) - RepoModel().grant_user_permission(repo=repo, user=user, perm=perm) + action_data = { + 'added': changes['added'], + 'updated': changes['updated'], + 'deleted': changes['deleted'], + } + audit_logger.store_api( + 'repo.edit.permissions', action_data=action_data, user=apiuser, repo=repo) Session().commit() return { @@ -1555,8 +1779,7 @@ def grant_user_permission(request, apius 'success': True } except Exception: - log.exception( - "Exception occurred while trying edit permissions for repo") + log.exception("Exception occurred while trying edit permissions for repo") raise JSONRPCError( 'failed to edit permission for user: `%s` in repo: `%s`' % ( userid, repoid @@ -1597,8 +1820,19 @@ def revoke_user_permission(request, apiu _perms = ('repository.admin',) validate_repo_permissions(apiuser, repoid, repo, _perms) + perm_deletions = [[user.user_id, None, "user"]] try: - RepoModel().revoke_user_permission(repo=repo, user=user) + changes = RepoModel().update_permissions( + repo=repo, perm_deletions=perm_deletions, cur_user=user) + + action_data = { + 'added': changes['added'], + 'updated': changes['updated'], + 'deleted': changes['deleted'], + } + audit_logger.store_api( + 'repo.edit.permissions', action_data=action_data, user=apiuser, repo=repo) + Session().commit() return { 'msg': 'Revoked perm for user: `%s` in repo: `%s`' % ( @@ -1607,8 +1841,7 @@ def revoke_user_permission(request, apiu 'success': True } except Exception: - log.exception( - "Exception occurred while trying revoke permissions to repo") + log.exception("Exception occurred while trying revoke permissions to repo") raise JSONRPCError( 'failed to edit permission for user: `%s` in repo: `%s`' % ( userid, repoid @@ -1674,9 +1907,17 @@ def grant_user_group_permission(request, raise JSONRPCError( 'user group `%s` does not exist' % (usergroupid,)) + perm_additions = [[user_group.users_group_id, perm.permission_name, "user_group"]] try: - RepoModel().grant_user_group_permission( - repo=repo, group_name=user_group, perm=perm) + changes = RepoModel().update_permissions( + repo=repo, perm_additions=perm_additions, cur_user=apiuser) + action_data = { + 'added': changes['added'], + 'updated': changes['updated'], + 'deleted': changes['deleted'], + } + audit_logger.store_api( + 'repo.edit.permissions', action_data=action_data, user=apiuser, repo=repo) Session().commit() return { @@ -1739,9 +1980,17 @@ def revoke_user_group_permission(request raise JSONRPCError( 'user group `%s` does not exist' % (usergroupid,)) + perm_deletions = [[user_group.users_group_id, None, "user_group"]] try: - RepoModel().revoke_user_group_permission( - repo=repo, group_name=user_group) + changes = RepoModel().update_permissions( + repo=repo, perm_deletions=perm_deletions, cur_user=apiuser) + action_data = { + 'added': changes['added'], + 'updated': changes['updated'], + 'deleted': changes['deleted'], + } + audit_logger.store_api( + 'repo.edit.permissions', action_data=action_data, user=apiuser, repo=repo) Session().commit() return { diff --git a/rhodecode/api/views/repo_group_api.py b/rhodecode/api/views/repo_group_api.py --- a/rhodecode/api/views/repo_group_api.py +++ b/rhodecode/api/views/repo_group_api.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -453,10 +453,19 @@ def grant_user_permission_to_repo_group( perm_additions = [[user.user_id, perm.permission_name, "user"]] try: - RepoGroupModel().update_permissions(repo_group=repo_group, - perm_additions=perm_additions, - recursive=apply_to_children, - cur_user=apiuser) + changes = RepoGroupModel().update_permissions( + repo_group=repo_group, perm_additions=perm_additions, + recursive=apply_to_children, cur_user=apiuser) + + action_data = { + 'added': changes['added'], + 'updated': changes['updated'], + 'deleted': changes['deleted'], + } + audit_logger.store_api( + 'repo_group.edit.permissions', action_data=action_data, + user=apiuser) + Session().commit() return { 'msg': 'Granted perm: `%s` (recursive:%s) for user: ' @@ -527,10 +536,19 @@ def revoke_user_permission_from_repo_gro perm_deletions = [[user.user_id, None, "user"]] try: - RepoGroupModel().update_permissions(repo_group=repo_group, - perm_deletions=perm_deletions, - recursive=apply_to_children, - cur_user=apiuser) + changes = RepoGroupModel().update_permissions( + repo_group=repo_group, perm_deletions=perm_deletions, + recursive=apply_to_children, cur_user=apiuser) + + action_data = { + 'added': changes['added'], + 'updated': changes['updated'], + 'deleted': changes['deleted'], + } + audit_logger.store_api( + 'repo_group.edit.permissions', action_data=action_data, + user=apiuser) + Session().commit() return { 'msg': 'Revoked perm (recursive:%s) for user: ' @@ -611,10 +629,19 @@ def grant_user_group_permission_to_repo_ perm_additions = [[user_group.users_group_id, perm.permission_name, "user_group"]] try: - RepoGroupModel().update_permissions(repo_group=repo_group, - perm_additions=perm_additions, - recursive=apply_to_children, - cur_user=apiuser) + changes = RepoGroupModel().update_permissions( + repo_group=repo_group, perm_additions=perm_additions, + recursive=apply_to_children, cur_user=apiuser) + + action_data = { + 'added': changes['added'], + 'updated': changes['updated'], + 'deleted': changes['deleted'], + } + audit_logger.store_api( + 'repo_group.edit.permissions', action_data=action_data, + user=apiuser) + Session().commit() return { 'msg': 'Granted perm: `%s` (recursive:%s) ' @@ -694,10 +721,19 @@ def revoke_user_group_permission_from_re perm_deletions = [[user_group.users_group_id, None, "user_group"]] try: - RepoGroupModel().update_permissions(repo_group=repo_group, - perm_deletions=perm_deletions, - recursive=apply_to_children, - cur_user=apiuser) + changes = RepoGroupModel().update_permissions( + repo_group=repo_group, perm_deletions=perm_deletions, + recursive=apply_to_children, cur_user=apiuser) + + action_data = { + 'added': changes['added'], + 'updated': changes['updated'], + 'deleted': changes['deleted'], + } + audit_logger.store_api( + 'repo_group.edit.permissions', action_data=action_data, + user=apiuser) + Session().commit() return { 'msg': 'Revoked perm (recursive:%s) for user group: ' @@ -716,4 +752,3 @@ def revoke_user_group_permission_from_re user_group.users_group_name, repo_group.name ) ) - diff --git a/rhodecode/api/views/server_api.py b/rhodecode/api/views/server_api.py --- a/rhodecode/api/views/server_api.py +++ b/rhodecode/api/views/server_api.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -21,6 +21,9 @@ import inspect import logging import itertools +import base64 + +from pyramid import compat from rhodecode.api import ( jsonrpc_method, JSONRPCError, JSONRPCForbidden, find_methods) @@ -30,10 +33,15 @@ from rhodecode.api.utils import ( from rhodecode.lib.utils import repo2db_mapper from rhodecode.lib import system_info from rhodecode.lib import user_sessions +from rhodecode.lib import exc_tracking +from rhodecode.lib.ext_json import json from rhodecode.lib.utils2 import safe_int from rhodecode.model.db import UserIpMap from rhodecode.model.scm import ScmModel from rhodecode.model.settings import VcsSettingsModel +from rhodecode.apps.file_store import utils +from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \ + FileOverSizeException log = logging.getLogger(__name__) @@ -293,7 +301,7 @@ def get_method(request, apiuser, pattern :param apiuser: This is filled automatically from the |authtoken|. :type apiuser: AuthUser :param pattern: pattern to match method names against - :type older_then: Optional("*") + :type pattern: Optional("*") Example output: @@ -349,3 +357,63 @@ def get_method(request, apiuser, pattern args_desc.append(func_kwargs) return matches.keys() + args_desc + + +@jsonrpc_method() +def store_exception(request, apiuser, exc_data_json, prefix=Optional('rhodecode')): + """ + Stores sent exception inside the built-in exception tracker in |RCE| server. + + This command can only be run using an |authtoken| with admin rights to + the specified repository. + + This command takes the following options: + + :param apiuser: This is filled automatically from the |authtoken|. + :type apiuser: AuthUser + + :param exc_data_json: JSON data with exception e.g + {"exc_traceback": "Value `1` is not allowed", "exc_type_name": "ValueError"} + :type exc_data_json: JSON data + + :param prefix: prefix for error type, e.g 'rhodecode', 'vcsserver', 'rhodecode-tools' + :type prefix: Optional("rhodecode") + + Example output: + + .. code-block:: bash + + id : + "result": { + "exc_id": 139718459226384, + "exc_url": "http://localhost:8080/_admin/settings/exceptions/139718459226384" + } + error : null + """ + if not has_superadmin_permission(apiuser): + raise JSONRPCForbidden() + + prefix = Optional.extract(prefix) + exc_id = exc_tracking.generate_id() + + try: + exc_data = json.loads(exc_data_json) + except Exception: + log.error('Failed to parse JSON: %r', exc_data_json) + raise JSONRPCError('Failed to parse JSON data from exc_data_json field. ' + 'Please make sure it contains a valid JSON.') + + try: + exc_traceback = exc_data['exc_traceback'] + exc_type_name = exc_data['exc_type_name'] + except KeyError as err: + raise JSONRPCError('Missing exc_traceback, or exc_type_name ' + 'in exc_data_json field. Missing: {}'.format(err)) + + exc_tracking._store_exception( + exc_id=exc_id, exc_traceback=exc_traceback, + exc_type_name=exc_type_name, prefix=prefix) + + exc_url = request.route_url( + 'admin_settings_exception_tracker_show', exception_id=exc_id) + return {'exc_id': exc_id, 'exc_url': exc_url} diff --git a/rhodecode/api/views/testing_api.py b/rhodecode/api/views/testing_api.py --- a/rhodecode/api/views/testing_api.py +++ b/rhodecode/api/views/testing_api.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/api/views/user_api.py b/rhodecode/api/views/user_api.py --- a/rhodecode/api/views/user_api.py +++ b/rhodecode/api/views/user_api.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -19,6 +19,7 @@ # and proprietary license terms, please see https://rhodecode.com/licenses/ import logging +from pyramid import compat from rhodecode.api import ( jsonrpc_method, JSONRPCError, JSONRPCForbidden, JSONRPCValidationError) @@ -241,7 +242,7 @@ def create_user(request, apiuser, userna # generate temporary password if user is external password = PasswordGenerator().gen_password(length=16) create_repo_group = Optional.extract(create_personal_repo_group) - if isinstance(create_repo_group, basestring): + if isinstance(create_repo_group, compat.string_types): create_repo_group = str2bool(create_repo_group) username = Optional.extract(username) diff --git a/rhodecode/api/views/user_group_api.py b/rhodecode/api/views/user_group_api.py --- a/rhodecode/api/views/user_group_api.py +++ b/rhodecode/api/views/user_group_api.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/__init__.py b/rhodecode/apps/__init__.py --- a/rhodecode/apps/__init__.py +++ b/rhodecode/apps/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/_base/__init__.py b/rhodecode/apps/_base/__init__.py --- a/rhodecode/apps/_base/__init__.py +++ b/rhodecode/apps/_base/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -22,11 +22,12 @@ import time import logging import operator +from pyramid import compat from pyramid.httpexceptions import HTTPFound, HTTPForbidden, HTTPBadRequest from rhodecode.lib import helpers as h, diffs from rhodecode.lib.utils2 import ( - StrictAttributeDict, safe_int, datetime_to_time, safe_unicode) + StrictAttributeDict, str2bool, safe_int, datetime_to_time, safe_unicode) from rhodecode.lib.vcs.exceptions import RepositoryRequirementError from rhodecode.model import repo from rhodecode.model import repo_group @@ -249,6 +250,12 @@ class RepoAppView(BaseAppView): else: # redirect if we don't show missing requirements raise HTTPFound(h.route_path('home')) + c.has_origin_repo_read_perm = False + if self.db_repo.fork: + c.has_origin_repo_read_perm = h.HasRepoPermissionAny( + 'repository.write', 'repository.read', 'repository.admin')( + self.db_repo.fork.repo_name, 'summary fork link') + return c def _get_f_path_unchecked(self, matchdict, default=None): @@ -271,6 +278,13 @@ class RepoAppView(BaseAppView): settings = settings_model.get_general_settings() return settings.get(settings_key, default) + def get_recache_flag(self): + for flag_name in ['force_recache', 'force-recache', 'no-cache']: + flag_val = self.request.GET.get(flag_name) + if str2bool(flag_val): + return True + return False + class PathFilter(object): @@ -327,6 +341,13 @@ class RepoGroupAppView(BaseAppView): self.db_repo_group = request.db_repo_group self.db_repo_group_name = self.db_repo_group.group_name + def _get_local_tmpl_context(self, include_app_defaults=True): + _ = self.request.translate + c = super(RepoGroupAppView, self)._get_local_tmpl_context( + include_app_defaults=include_app_defaults) + c.repo_group = self.db_repo_group + return c + def _revoke_perms_on_yourself(self, form_result): _updates = filter(lambda u: self._rhodecode_user.user_id == int(u[0]), form_result['perm_updates']) @@ -389,7 +410,7 @@ class DataGridAppView(object): return draw, start, length def _get_order_col(self, order_by, model): - if isinstance(order_by, basestring): + if isinstance(order_by, compat.string_types): try: return operator.attrgetter(order_by)(model) except AttributeError: diff --git a/rhodecode/apps/_base/interfaces.py b/rhodecode/apps/_base/interfaces.py --- a/rhodecode/apps/_base/interfaces.py +++ b/rhodecode/apps/_base/interfaces.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/_base/navigation.py b/rhodecode/apps/_base/navigation.py --- a/rhodecode/apps/_base/navigation.py +++ b/rhodecode/apps/_base/navigation.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/_base/subscribers.py b/rhodecode/apps/_base/subscribers.py --- a/rhodecode/apps/_base/subscribers.py +++ b/rhodecode/apps/_base/subscribers.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/__init__.py b/rhodecode/apps/admin/__init__.py --- a/rhodecode/apps/admin/__init__.py +++ b/rhodecode/apps/admin/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -280,8 +280,12 @@ def admin_routes(config): pattern='/users/{user_id:\d+}/delete', user_route=True) config.add_route( - name='user_force_password_reset', - pattern='/users/{user_id:\d+}/password_reset', + name='user_enable_force_password_reset', + pattern='/users/{user_id:\d+}/password_reset_enable', + user_route=True) + config.add_route( + name='user_disable_force_password_reset', + pattern='/users/{user_id:\d+}/password_reset_disable', user_route=True) config.add_route( name='user_create_personal_repo_group', diff --git a/rhodecode/apps/admin/tests/test_admin_audit_logs.py b/rhodecode/apps/admin/tests/test_admin_audit_logs.py --- a/rhodecode/apps/admin/tests/test_admin_audit_logs.py +++ b/rhodecode/apps/admin/tests/test_admin_audit_logs.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/tests/test_admin_auth_settings.py b/rhodecode/apps/admin/tests/test_admin_auth_settings.py --- a/rhodecode/apps/admin/tests/test_admin_auth_settings.py +++ b/rhodecode/apps/admin/tests/test_admin_auth_settings.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/tests/test_admin_defaults.py b/rhodecode/apps/admin/tests/test_admin_defaults.py --- a/rhodecode/apps/admin/tests/test_admin_defaults.py +++ b/rhodecode/apps/admin/tests/test_admin_defaults.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/tests/test_admin_main_views.py b/rhodecode/apps/admin/tests/test_admin_main_views.py --- a/rhodecode/apps/admin/tests/test_admin_main_views.py +++ b/rhodecode/apps/admin/tests/test_admin_main_views.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/tests/test_admin_permissions.py b/rhodecode/apps/admin/tests/test_admin_permissions.py --- a/rhodecode/apps/admin/tests/test_admin_permissions.py +++ b/rhodecode/apps/admin/tests/test_admin_permissions.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/tests/test_admin_repos.py b/rhodecode/apps/admin/tests/test_admin_repos.py --- a/rhodecode/apps/admin/tests/test_admin_repos.py +++ b/rhodecode/apps/admin/tests/test_admin_repos.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/tests/test_admin_repository_groups.py b/rhodecode/apps/admin/tests/test_admin_repository_groups.py --- a/rhodecode/apps/admin/tests/test_admin_repository_groups.py +++ b/rhodecode/apps/admin/tests/test_admin_repository_groups.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/tests/test_admin_settings.py b/rhodecode/apps/admin/tests/test_admin_settings.py --- a/rhodecode/apps/admin/tests/test_admin_settings.py +++ b/rhodecode/apps/admin/tests/test_admin_settings.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/tests/test_admin_user_groups.py b/rhodecode/apps/admin/tests/test_admin_user_groups.py --- a/rhodecode/apps/admin/tests/test_admin_user_groups.py +++ b/rhodecode/apps/admin/tests/test_admin_user_groups.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/tests/test_admin_users.py b/rhodecode/apps/admin/tests/test_admin_users.py --- a/rhodecode/apps/admin/tests/test_admin_users.py +++ b/rhodecode/apps/admin/tests/test_admin_users.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -59,8 +59,6 @@ def route_path(name, params=None, **kwar ADMIN_PREFIX + '/users/{user_id}/update', 'user_delete': ADMIN_PREFIX + '/users/{user_id}/delete', - 'user_force_password_reset': - ADMIN_PREFIX + '/users/{user_id}/password_reset', 'user_create_personal_repo_group': ADMIN_PREFIX + '/users/{user_id}/create_repo_group', diff --git a/rhodecode/apps/admin/tests/test_admin_users_ssh_keys.py b/rhodecode/apps/admin/tests/test_admin_users_ssh_keys.py --- a/rhodecode/apps/admin/tests/test_admin_users_ssh_keys.py +++ b/rhodecode/apps/admin/tests/test_admin_users_ssh_keys.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/views/__init__.py b/rhodecode/apps/admin/views/__init__.py --- a/rhodecode/apps/admin/views/__init__.py +++ b/rhodecode/apps/admin/views/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/views/audit_logs.py b/rhodecode/apps/admin/views/audit_logs.py --- a/rhodecode/apps/admin/views/audit_logs.py +++ b/rhodecode/apps/admin/views/audit_logs.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/views/defaults.py b/rhodecode/apps/admin/views/defaults.py --- a/rhodecode/apps/admin/views/defaults.py +++ b/rhodecode/apps/admin/views/defaults.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/views/exception_tracker.py b/rhodecode/apps/admin/views/exception_tracker.py --- a/rhodecode/apps/admin/views/exception_tracker.py +++ b/rhodecode/apps/admin/views/exception_tracker.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2018-2018 RhodeCode GmbH +# Copyright (C) 2018-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/views/main_views.py b/rhodecode/apps/admin/views/main_views.py --- a/rhodecode/apps/admin/views/main_views.py +++ b/rhodecode/apps/admin/views/main_views.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/views/open_source_licenses.py b/rhodecode/apps/admin/views/open_source_licenses.py --- a/rhodecode/apps/admin/views/open_source_licenses.py +++ b/rhodecode/apps/admin/views/open_source_licenses.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/views/permissions.py b/rhodecode/apps/admin/views/permissions.py --- a/rhodecode/apps/admin/views/permissions.py +++ b/rhodecode/apps/admin/views/permissions.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/views/process_management.py b/rhodecode/apps/admin/views/process_management.py --- a/rhodecode/apps/admin/views/process_management.py +++ b/rhodecode/apps/admin/views/process_management.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -119,8 +119,7 @@ class AdminProcessManagementView(BaseApp result = [] def on_terminate(proc): - msg = "process `PID:{}` terminated with exit code {}".format( - proc.pid, proc.returncode or 0) + msg = "terminated" result.append(msg) procs = [] diff --git a/rhodecode/apps/admin/views/repo_groups.py b/rhodecode/apps/admin/views/repo_groups.py --- a/rhodecode/apps/admin/views/repo_groups.py +++ b/rhodecode/apps/admin/views/repo_groups.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/views/repositories.py b/rhodecode/apps/admin/views/repositories.py --- a/rhodecode/apps/admin/views/repositories.py +++ b/rhodecode/apps/admin/views/repositories.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/views/sessions.py b/rhodecode/apps/admin/views/sessions.py --- a/rhodecode/apps/admin/views/sessions.py +++ b/rhodecode/apps/admin/views/sessions.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/views/settings.py b/rhodecode/apps/admin/views/settings.py --- a/rhodecode/apps/admin/views/settings.py +++ b/rhodecode/apps/admin/views/settings.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -666,8 +666,8 @@ class AdminSettingsView(BaseAppView): c = self.load_default_context() c.active = 'search' - searcher = searcher_from_config(self.request.registry.settings) - c.statistics = searcher.statistics(self.request.translate) + c.searcher = searcher_from_config(self.request.registry.settings) + c.statistics = c.searcher.statistics(self.request.translate) return self._get_template_context(c) diff --git a/rhodecode/apps/admin/views/svn_config.py b/rhodecode/apps/admin/views/svn_config.py --- a/rhodecode/apps/admin/views/svn_config.py +++ b/rhodecode/apps/admin/views/svn_config.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/views/system_info.py b/rhodecode/apps/admin/views/system_info.py --- a/rhodecode/apps/admin/views/system_info.py +++ b/rhodecode/apps/admin/views/system_info.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/views/user_groups.py b/rhodecode/apps/admin/views/user_groups.py --- a/rhodecode/apps/admin/views/user_groups.py +++ b/rhodecode/apps/admin/views/user_groups.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/admin/views/users.py b/rhodecode/apps/admin/views/users.py --- a/rhodecode/apps/admin/views/users.py +++ b/rhodecode/apps/admin/views/users.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -604,12 +604,9 @@ class UsersView(UserAppView): @HasPermissionAllDecorator('hg.admin') @CSRFRequired() @view_config( - route_name='user_force_password_reset', request_method='POST', + route_name='user_enable_force_password_reset', request_method='POST', renderer='rhodecode:templates/admin/users/user_edit.mako') - def user_force_password_reset(self): - """ - toggle reset password flag for this user - """ + def user_enable_force_password_reset(self): _ = self.request.translate c = self.load_default_context() @@ -617,19 +614,41 @@ class UsersView(UserAppView): c.user = self.db_user try: - old_value = c.user.user_data.get('force_password_change') - c.user.update_userdata(force_password_change=not old_value) + c.user.update_userdata(force_password_change=True) + + msg = _('Force password change enabled for user') + audit_logger.store_web('user.edit.password_reset.enabled', + user=c.rhodecode_user) + + Session().commit() + h.flash(msg, category='success') + except Exception: + log.exception("Exception during password reset for user") + h.flash(_('An error occurred during password reset for user'), + category='error') + + raise HTTPFound(h.route_path('user_edit_advanced', user_id=user_id)) - if old_value: - msg = _('Force password change disabled for user') - audit_logger.store_web( - 'user.edit.password_reset.disabled', - user=c.rhodecode_user) - else: - msg = _('Force password change enabled for user') - audit_logger.store_web( - 'user.edit.password_reset.enabled', - user=c.rhodecode_user) + @LoginRequired() + @HasPermissionAllDecorator('hg.admin') + @CSRFRequired() + @view_config( + route_name='user_disable_force_password_reset', request_method='POST', + renderer='rhodecode:templates/admin/users/user_edit.mako') + def user_disable_force_password_reset(self): + _ = self.request.translate + c = self.load_default_context() + + user_id = self.db_user_id + c.user = self.db_user + + try: + c.user.update_userdata(force_password_change=False) + + msg = _('Force password change disabled for user') + audit_logger.store_web( + 'user.edit.password_reset.disabled', + user=c.rhodecode_user) Session().commit() h.flash(msg, category='success') diff --git a/rhodecode/apps/channelstream/__init__.py b/rhodecode/apps/channelstream/__init__.py --- a/rhodecode/apps/channelstream/__init__.py +++ b/rhodecode/apps/channelstream/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/channelstream/views.py b/rhodecode/apps/channelstream/views.py --- a/rhodecode/apps/channelstream/views.py +++ b/rhodecode/apps/channelstream/views.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/debug_style/__init__.py b/rhodecode/apps/debug_style/__init__.py --- a/rhodecode/apps/debug_style/__init__.py +++ b/rhodecode/apps/debug_style/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/debug_style/views.py b/rhodecode/apps/debug_style/views.py --- a/rhodecode/apps/debug_style/views.py +++ b/rhodecode/apps/debug_style/views.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/file_store/__init__.py b/rhodecode/apps/file_store/__init__.py new file mode 100755 --- /dev/null +++ b/rhodecode/apps/file_store/__init__.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2016-2019 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ +import os +from rhodecode.apps.file_store import config_keys +from rhodecode.config.middleware import _bool_setting, _string_setting + + +def _sanitize_settings_and_apply_defaults(settings): + """ + Set defaults, convert to python types and validate settings. + """ + _bool_setting(settings, config_keys.enabled, 'true') + + _string_setting(settings, config_keys.backend, 'local') + + default_store = os.path.join(os.path.dirname(settings['__file__']), 'upload_store') + _string_setting(settings, config_keys.store_path, default_store) + + +def includeme(config): + settings = config.registry.settings + _sanitize_settings_and_apply_defaults(settings) + + config.add_route( + name='upload_file', + pattern='/_file_store/upload') + config.add_route( + name='download_file', + pattern='/_file_store/download/{fid}') + + # Scan module for configuration decorators. + config.scan('.views', ignore='.tests') diff --git a/rhodecode/apps/file_store/config_keys.py b/rhodecode/apps/file_store/config_keys.py new file mode 100644 --- /dev/null +++ b/rhodecode/apps/file_store/config_keys.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2016-2019 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + + +# Definition of setting keys used to configure this module. Defined here to +# avoid repetition of keys throughout the module. + +enabled = 'file_store.enabled' +backend = 'file_store.backend' +store_path = 'file_store.storage_path' diff --git a/rhodecode/apps/file_store/exceptions.py b/rhodecode/apps/file_store/exceptions.py new file mode 100755 --- /dev/null +++ b/rhodecode/apps/file_store/exceptions.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2016-2019 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + + +class FileNotAllowedException(Exception): + """ + Thrown if file does not have an allowed extension. + """ + + +class FileOverSizeException(Exception): + """ + Thrown if file is over the set limit. + """ diff --git a/rhodecode/apps/file_store/extensions.py b/rhodecode/apps/file_store/extensions.py new file mode 100644 --- /dev/null +++ b/rhodecode/apps/file_store/extensions.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2016-2019 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + + +ANY = [] +TEXT_EXT = ['txt', 'md', 'rst', 'log'] +DOCUMENTS_EXT = ['pdf', 'rtf', 'odf', 'ods', 'gnumeric', 'abw', 'doc', 'docx', 'xls', 'xlsx'] +IMAGES_EXT = ['jpg', 'jpe', 'jpeg', 'png', 'gif', 'svg', 'bmp', 'tiff'] +AUDIO_EXT = ['wav', 'mp3', 'aac', 'ogg', 'oga', 'flac'] +VIDEO_EXT = ['mpeg', '3gp', 'avi', 'divx', 'dvr', 'flv', 'mp4', 'wmv'] +DATA_EXT = ['csv', 'ini', 'json', 'plist', 'xml', 'yaml', 'yml'] +SCRIPTS_EXT = ['js', 'php', 'pl', 'py', 'rb', 'sh', 'go', 'c', 'h'] +ARCHIVES_EXT = ['gz', 'bz2', 'zip', 'tar', 'tgz', 'txz', '7z'] +EXECUTABLES_EXT = ['so', 'exe', 'dll'] + + +DEFAULT = DOCUMENTS_EXT + TEXT_EXT + IMAGES_EXT + DATA_EXT + +GROUPS = dict(( + ('any', ANY), + ('text', TEXT_EXT), + ('documents', DOCUMENTS_EXT), + ('images', IMAGES_EXT), + ('audio', AUDIO_EXT), + ('video', VIDEO_EXT), + ('data', DATA_EXT), + ('scripts', SCRIPTS_EXT), + ('archives', ARCHIVES_EXT), + ('executables', EXECUTABLES_EXT), + ('default', DEFAULT), +)) + + +def resolve_extensions(extensions, groups=None): + """ + Calculate allowed extensions based on a list of extensions provided, and optional + groups of extensions from the available lists. + + :param extensions: a list of extensions e.g ['py', 'txt'] + :param groups: additionally groups to extend the extensions. + """ + groups = groups or [] + valid_exts = set([x.lower() for x in extensions]) + + for group in groups: + if group in GROUPS: + valid_exts.update(GROUPS[group]) + + return valid_exts diff --git a/rhodecode/apps/file_store/local_store.py b/rhodecode/apps/file_store/local_store.py new file mode 100755 --- /dev/null +++ b/rhodecode/apps/file_store/local_store.py @@ -0,0 +1,211 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2016-2019 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +import os +import time +import shutil +import hashlib + +from rhodecode.lib.ext_json import json +from rhodecode.apps.file_store import utils +from rhodecode.apps.file_store.extensions import resolve_extensions +from rhodecode.apps.file_store.exceptions import FileNotAllowedException + +METADATA_VER = 'v1' + + +class LocalFileStorage(object): + + @classmethod + def resolve_name(cls, name, directory): + """ + Resolves a unique name and the correct path. If a filename + for that path already exists then a numeric prefix with values > 0 will be + added, for example test.jpg -> test-1.jpg etc. initially file would have 0 prefix. + + :param name: base name of file + :param directory: absolute directory path + """ + + basename, ext = os.path.splitext(name) + counter = 0 + while True: + name = '%s-%d%s' % (basename, counter, ext) + + # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file + sub_store = cls._sub_store_from_filename(basename) + sub_store_path = os.path.join(directory, sub_store) + if not os.path.exists(sub_store_path): + os.makedirs(sub_store_path) + + path = os.path.join(sub_store_path, name) + if not os.path.exists(path): + return name, path + counter += 1 + + @classmethod + def _sub_store_from_filename(cls, filename): + return filename[:2] + + @classmethod + def calculate_path_hash(cls, file_path): + """ + Efficient calculation of file_path sha256 sum + + :param file_path: + :return: sha256sum + """ + digest = hashlib.sha256() + with open(file_path, 'rb') as f: + for chunk in iter(lambda: f.read(1024 * 100), b""): + digest.update(chunk) + + return digest.hexdigest() + + def __init__(self, base_path, extension_groups=None): + + """ + Local file storage + + :param base_path: the absolute base path where uploads are stored + :param extension_groups: extensions string + """ + + extension_groups = extension_groups or ['any'] + self.base_path = base_path + self.extensions = resolve_extensions([], groups=extension_groups) + + def store_path(self, filename): + """ + Returns absolute file path of the filename, joined to the + base_path. + + :param filename: base name of file + """ + sub_store = self._sub_store_from_filename(filename) + return os.path.join(self.base_path, sub_store, filename) + + def delete(self, filename): + """ + Deletes the filename. Filename is resolved with the + absolute path based on base_path. If file does not exist, + returns **False**, otherwise **True** + + :param filename: base name of file + """ + if self.exists(filename): + os.remove(self.store_path(filename)) + return True + return False + + def exists(self, filename): + """ + Checks if file exists. Resolves filename's absolute + path based on base_path. + + :param filename: base name of file + """ + return os.path.exists(self.store_path(filename)) + + def filename_allowed(self, filename, extensions=None): + """Checks if a filename has an allowed extension + + :param filename: base name of file + :param extensions: iterable of extensions (or self.extensions) + """ + _, ext = os.path.splitext(filename) + return self.extension_allowed(ext, extensions) + + def extension_allowed(self, ext, extensions=None): + """ + Checks if an extension is permitted. Both e.g. ".jpg" and + "jpg" can be passed in. Extension lookup is case-insensitive. + + :param ext: extension to check + :param extensions: iterable of extensions to validate against (or self.extensions) + """ + + extensions = extensions or self.extensions + if not extensions: + return True + if ext.startswith('.'): + ext = ext[1:] + return ext.lower() in extensions + + def save_file(self, file_obj, filename, directory=None, extensions=None, + extra_metadata=None, **kwargs): + """ + Saves a file object to the uploads location. + Returns the resolved filename, i.e. the directory + + the (randomized/incremented) base name. + + :param file_obj: **cgi.FieldStorage** object (or similar) + :param filename: original filename + :param directory: relative path of sub-directory + :param extensions: iterable of allowed extensions, if not default + :param extra_metadata: extra JSON metadata to store next to the file with .meta suffix + """ + + extensions = extensions or self.extensions + + if not self.filename_allowed(filename, extensions): + raise FileNotAllowedException() + + if directory: + dest_directory = os.path.join(self.base_path, directory) + else: + dest_directory = self.base_path + + if not os.path.exists(dest_directory): + os.makedirs(dest_directory) + + filename = utils.uid_filename(filename) + + # resolve also produces special sub-dir for file optimized store + filename, path = self.resolve_name(filename, dest_directory) + stored_file_dir = os.path.dirname(path) + + file_obj.seek(0) + + with open(path, "wb") as dest: + shutil.copyfileobj(file_obj, dest) + + metadata = {} + if extra_metadata: + metadata = extra_metadata + + size = os.stat(path).st_size + file_hash = self.calculate_path_hash(path) + + metadata.update( + {"filename": filename, + "size": size, + "time": time.time(), + "sha256": file_hash, + "meta_ver": METADATA_VER}) + + filename_meta = filename + '.meta' + with open(os.path.join(stored_file_dir, filename_meta), "wb") as dest_meta: + dest_meta.write(json.dumps(metadata)) + + if directory: + filename = os.path.join(directory, filename) + + return filename, metadata diff --git a/rhodecode/apps/file_store/tests/__init__.py b/rhodecode/apps/file_store/tests/__init__.py new file mode 100644 --- /dev/null +++ b/rhodecode/apps/file_store/tests/__init__.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2016-2019 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + diff --git a/rhodecode/apps/file_store/tests/test_upload_file.py b/rhodecode/apps/file_store/tests/test_upload_file.py new file mode 100644 --- /dev/null +++ b/rhodecode/apps/file_store/tests/test_upload_file.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2019 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ +import os +import pytest + +from rhodecode.lib.ext_json import json +from rhodecode.tests import TestController +from rhodecode.apps.file_store import utils, config_keys + + +def route_path(name, params=None, **kwargs): + import urllib + + base_url = { + 'upload_file': '/_file_store/upload', + 'download_file': '/_file_store/download/{fid}', + + }[name].format(**kwargs) + + if params: + base_url = '{}?{}'.format(base_url, urllib.urlencode(params)) + return base_url + + +class TestFileStoreViews(TestController): + + @pytest.mark.parametrize("fid, content, exists", [ + ('abcde-0.jpg', "xxxxx", True), + ('abcde-0.exe', "1234567", True), + ('abcde-0.jpg', "xxxxx", False), + ]) + def test_get_files_from_store(self, fid, content, exists, tmpdir): + self.log_user() + store_path = self.app._pyramid_settings[config_keys.store_path] + + if exists: + status = 200 + store = utils.get_file_storage({config_keys.store_path: store_path}) + filesystem_file = os.path.join(str(tmpdir), fid) + with open(filesystem_file, 'wb') as f: + f.write(content) + + with open(filesystem_file, 'rb') as f: + fid, metadata = store.save_file(f, fid, extra_metadata={'filename': fid}) + + else: + status = 404 + + response = self.app.get(route_path('download_file', fid=fid), status=status) + + if exists: + assert response.text == content + file_store_path = os.path.dirname(store.resolve_name(fid, store_path)[1]) + metadata_file = os.path.join(file_store_path, fid + '.meta') + assert os.path.exists(metadata_file) + with open(metadata_file, 'rb') as f: + json_data = json.loads(f.read()) + + assert json_data + assert 'size' in json_data + + def test_upload_files_without_content_to_store(self): + self.log_user() + response = self.app.post( + route_path('upload_file'), + params={'csrf_token': self.csrf_token}, + status=200) + + assert response.json == { + u'error': u'store_file data field is missing', + u'access_path': None, + u'store_fid': None} + + def test_upload_files_bogus_content_to_store(self): + self.log_user() + response = self.app.post( + route_path('upload_file'), + params={'csrf_token': self.csrf_token, 'store_file': 'bogus'}, + status=200) + + assert response.json == { + u'error': u'filename cannot be read from the data field', + u'access_path': None, + u'store_fid': None} + + def test_upload_content_to_store(self): + self.log_user() + response = self.app.post( + route_path('upload_file'), + upload_files=[('store_file', 'myfile.txt', 'SOME CONTENT')], + params={'csrf_token': self.csrf_token}, + status=200) + + assert response.json['store_fid'] diff --git a/rhodecode/apps/file_store/utils.py b/rhodecode/apps/file_store/utils.py new file mode 100755 --- /dev/null +++ b/rhodecode/apps/file_store/utils.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2016-2019 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + + +import os +import uuid + + +def get_file_storage(settings): + from rhodecode.apps.file_store.local_store import LocalFileStorage + from rhodecode.apps.file_store import config_keys + store_path = settings.get(config_keys.store_path) + return LocalFileStorage(base_path=store_path) + + +def uid_filename(filename, randomized=True): + """ + Generates a randomized or stable (uuid) filename, + preserving the original extension. + + :param filename: the original filename + :param randomized: define if filename should be stable (sha1 based) or randomized + """ + _, ext = os.path.splitext(filename) + if randomized: + uid = uuid.uuid4() + else: + hash_key = '{}.{}'.format(filename, 'store') + uid = uuid.uuid5(uuid.NAMESPACE_URL, hash_key) + return str(uid) + ext.lower() diff --git a/rhodecode/apps/file_store/views.py b/rhodecode/apps/file_store/views.py new file mode 100644 --- /dev/null +++ b/rhodecode/apps/file_store/views.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2016-2019 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ +import logging + +from pyramid.view import view_config +from pyramid.response import FileResponse +from pyramid.httpexceptions import HTTPFound, HTTPNotFound + +from rhodecode.apps._base import BaseAppView +from rhodecode.apps.file_store import utils +from rhodecode.apps.file_store.exceptions import ( + FileNotAllowedException, FileOverSizeException) + +from rhodecode.lib import helpers as h +from rhodecode.lib import audit_logger +from rhodecode.lib.auth import (CSRFRequired, NotAnonymous) +from rhodecode.model.db import Session, FileStore + +log = logging.getLogger(__name__) + + +class FileStoreView(BaseAppView): + upload_key = 'store_file' + + def load_default_context(self): + c = self._get_local_tmpl_context() + self.storage = utils.get_file_storage(self.request.registry.settings) + return c + + @NotAnonymous() + @CSRFRequired() + @view_config(route_name='upload_file', request_method='POST', renderer='json_ext') + def upload_file(self): + self.load_default_context() + file_obj = self.request.POST.get(self.upload_key) + + if file_obj is None: + return {'store_fid': None, + 'access_path': None, + 'error': '{} data field is missing'.format(self.upload_key)} + + if not hasattr(file_obj, 'filename'): + return {'store_fid': None, + 'access_path': None, + 'error': 'filename cannot be read from the data field'} + + filename = file_obj.filename + + metadata = { + 'user_uploaded': {'username': self._rhodecode_user.username, + 'user_id': self._rhodecode_user.user_id, + 'ip': self._rhodecode_user.ip_addr}} + try: + store_fid, metadata = self.storage.save_file( + file_obj.file, filename, extra_metadata=metadata) + except FileNotAllowedException: + return {'store_fid': None, + 'access_path': None, + 'error': 'File {} is not allowed.'.format(filename)} + + except FileOverSizeException: + return {'store_fid': None, + 'access_path': None, + 'error': 'File {} is exceeding allowed limit.'.format(filename)} + + try: + entry = FileStore.create( + file_uid=store_fid, filename=metadata["filename"], + file_hash=metadata["sha256"], file_size=metadata["size"], + file_description='upload attachment', + check_acl=False, user_id=self._rhodecode_user.user_id + ) + Session().add(entry) + Session().commit() + log.debug('Stored upload in DB as %s', entry) + except Exception: + log.exception('Failed to store file %s', filename) + return {'store_fid': None, + 'access_path': None, + 'error': 'File {} failed to store in DB.'.format(filename)} + + return {'store_fid': store_fid, + 'access_path': h.route_path('download_file', fid=store_fid)} + + @view_config(route_name='download_file') + def download_file(self): + self.load_default_context() + file_uid = self.request.matchdict['fid'] + log.debug('Requesting FID:%s from store %s', file_uid, self.storage) + + if not self.storage.exists(file_uid): + log.debug('File with FID:%s not found in the store', file_uid) + raise HTTPNotFound() + + FileStore.bump_access_counter(file_uid) + + file_path = self.storage.store_path(file_uid) + return FileResponse(file_path) diff --git a/rhodecode/apps/gist/__init__.py b/rhodecode/apps/gist/__init__.py --- a/rhodecode/apps/gist/__init__.py +++ b/rhodecode/apps/gist/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/gist/tests/__init__.py b/rhodecode/apps/gist/tests/__init__.py --- a/rhodecode/apps/gist/tests/__init__.py +++ b/rhodecode/apps/gist/tests/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/gist/tests/test_admin_gists.py b/rhodecode/apps/gist/tests/test_admin_gists.py --- a/rhodecode/apps/gist/tests/test_admin_gists.py +++ b/rhodecode/apps/gist/tests/test_admin_gists.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/gist/views.py b/rhodecode/apps/gist/views.py --- a/rhodecode/apps/gist/views.py +++ b/rhodecode/apps/gist/views.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2013-2018 RhodeCode GmbH +# Copyright (C) 2013-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/home/__init__.py b/rhodecode/apps/home/__init__.py --- a/rhodecode/apps/home/__init__.py +++ b/rhodecode/apps/home/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -56,6 +56,10 @@ def includeme(config): pattern='/_repos') config.add_route( + name='repo_group_list_data', + pattern='/_repo_groups') + + config.add_route( name='goto_switcher_data', pattern='/_goto_data') diff --git a/rhodecode/apps/home/tests/__init__.py b/rhodecode/apps/home/tests/__init__.py --- a/rhodecode/apps/home/tests/__init__.py +++ b/rhodecode/apps/home/tests/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -31,7 +31,14 @@ def assert_and_get_main_filter_content(r assert data_item['url'] if data_item['type'] == 'search': - assert data_item['value_display'].startswith('Full text search for:') + display_val = data_item['value_display'] + if data_item['id'] == -1: + assert 'File search for:' in display_val, display_val + elif data_item['id'] == -2: + assert 'Commit search for:' in display_val, display_val + else: + assert False, 'No Proper ID returned {}'.format(data_item['id']) + elif data_item['type'] == 'repo': repos.append(data_item) elif data_item['type'] == 'repo_group': diff --git a/rhodecode/apps/home/tests/test_get_goto_switched_data.py b/rhodecode/apps/home/tests/test_get_goto_switched_data.py --- a/rhodecode/apps/home/tests/test_get_goto_switched_data.py +++ b/rhodecode/apps/home/tests/test_get_goto_switched_data.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/home/tests/test_get_repo_list_data.py b/rhodecode/apps/home/tests/test_get_repo_list_data.py --- a/rhodecode/apps/home/tests/test_get_repo_list_data.py +++ b/rhodecode/apps/home/tests/test_get_repo_list_data.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/home/tests/test_get_user_data.py b/rhodecode/apps/home/tests/test_get_user_data.py --- a/rhodecode/apps/home/tests/test_get_user_data.py +++ b/rhodecode/apps/home/tests/test_get_user_data.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/home/tests/test_get_user_group_data.py b/rhodecode/apps/home/tests/test_get_user_group_data.py --- a/rhodecode/apps/home/tests/test_get_user_group_data.py +++ b/rhodecode/apps/home/tests/test_get_user_group_data.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -19,7 +19,7 @@ # and proprietary license terms, please see https://rhodecode.com/licenses/ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/home/tests/test_home.py b/rhodecode/apps/home/tests/test_home.py --- a/rhodecode/apps/home/tests/test_home.py +++ b/rhodecode/apps/home/tests/test_home.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/home/views.py b/rhodecode/apps/home/views.py --- a/rhodecode/apps/home/views.py +++ b/rhodecode/apps/home/views.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -172,7 +172,9 @@ class HomeView(BaseAppView): 'id': obj.group_name, 'value': org_query, 'value_display': obj.group_name, + 'text': obj.group_name, 'type': 'repo_group', + 'repo_group_id': obj.group_id, 'url': h.route_path( 'repo_group_home', repo_group_name=obj.group_name) } @@ -246,9 +248,9 @@ class HomeView(BaseAppView): } for obj in acl_iter] - def _get_hash_commit_list(self, auth_user, query): + def _get_hash_commit_list(self, auth_user, searcher, query): org_query = query - if not query or len(query) < 3: + if not query or len(query) < 3 or not searcher: return [] commit_hashes = re.compile('(?:commit:)([0-9a-f]{2,40})').findall(query) @@ -257,24 +259,34 @@ class HomeView(BaseAppView): return [] commit_hash = commit_hashes[0] - searcher = searcher_from_config(self.request.registry.settings) result = searcher.search( - 'commit_id:%s*' % commit_hash, 'commit', auth_user, + 'commit_id:{}*'.format(commit_hash), 'commit', auth_user, raise_on_exc=False) - return [ - { + commits = [] + for entry in result['results']: + repo_data = { + 'repository_id': entry.get('repository_id'), + 'repository_type': entry.get('repo_type'), + 'repository_name': entry.get('repository'), + } + + commit_entry = { 'id': entry['commit_id'], 'value': org_query, - 'value_display': 'repo `{}` commit: {}'.format( + 'value_display': '`{}` commit: {}'.format( entry['repository'], entry['commit_id']), 'type': 'commit', 'repo': entry['repository'], + 'repo_data': repo_data, + 'url': h.route_path( 'repo_commit', repo_name=entry['repository'], commit_id=entry['commit_id']) } - for entry in result['results']] + + commits.append(commit_entry) + return commits @LoginRequired() @view_config( @@ -305,6 +317,144 @@ class HomeView(BaseAppView): @LoginRequired() @view_config( + route_name='repo_group_list_data', request_method='GET', + renderer='json_ext', xhr=True) + def repo_group_list_data(self): + _ = self.request.translate + self.load_default_context() + + query = self.request.GET.get('query') + + log.debug('generating repo group list, query:%s', + query) + + res = [] + repo_groups = self._get_repo_group_list(query) + if repo_groups: + res.append({ + 'text': _('Repository Groups'), + 'children': repo_groups + }) + + data = { + 'more': False, + 'results': res + } + return data + + def _get_default_search_queries(self, search_context, searcher, query): + if not searcher: + return [] + + is_es_6 = searcher.is_es_6 + + queries = [] + repo_group_name, repo_name, repo_context = None, None, None + + # repo group context + if search_context.get('search_context[repo_group_name]'): + repo_group_name = search_context.get('search_context[repo_group_name]') + if search_context.get('search_context[repo_name]'): + repo_name = search_context.get('search_context[repo_name]') + repo_context = search_context.get('search_context[repo_view_type]') + + if is_es_6 and repo_name: + # files + def query_modifier(): + qry = query + return {'q': qry, 'type': 'content'} + label = u'File search for `{}` in this repository.'.format(query) + queries.append( + { + 'id': -10, + 'value': query, + 'value_display': label, + 'type': 'search', + 'url': h.route_path('search_repo', + repo_name=repo_name, + _query=query_modifier()) + } + ) + + # commits + def query_modifier(): + qry = query + return {'q': qry, 'type': 'commit'} + + label = u'Commit search for `{}` in this repository.'.format(query) + queries.append( + { + 'id': -20, + 'value': query, + 'value_display': label, + 'type': 'search', + 'url': h.route_path('search_repo', + repo_name=repo_name, + _query=query_modifier()) + } + ) + + elif is_es_6 and repo_group_name: + # files + def query_modifier(): + qry = query + return {'q': qry, 'type': 'content'} + + label = u'File search for `{}` in this repository group'.format(query) + queries.append( + { + 'id': -30, + 'value': query, + 'value_display': label, + 'type': 'search', + 'url': h.route_path('search_repo_group', + repo_group_name=repo_group_name, + _query=query_modifier()) + } + ) + + # commits + def query_modifier(): + qry = query + return {'q': qry, 'type': 'commit'} + + label = u'Commit search for `{}` in this repository group'.format(query) + queries.append( + { + 'id': -40, + 'value': query, + 'value_display': label, + 'type': 'search', + 'url': h.route_path('search_repo_group', + repo_group_name=repo_group_name, + _query=query_modifier()) + } + ) + + if not queries: + queries.append( + { + 'id': -1, + 'value': query, + 'value_display': u'File search for: `{}`'.format(query), + 'type': 'search', + 'url': h.route_path('search', + _query={'q': query, 'type': 'content'}) + }) + queries.append( + { + 'id': -2, + 'value': query, + 'value_display': u'Commit search for: `{}`'.format(query), + 'type': 'search', + 'url': h.route_path('search', + _query={'q': query, 'type': 'commit'}) + }) + + return queries + + @LoginRequired() + @view_config( route_name='goto_switcher_data', request_method='GET', renderer='json_ext', xhr=True) def goto_switcher_data(self): @@ -315,26 +465,21 @@ class HomeView(BaseAppView): query = self.request.GET.get('query') log.debug('generating main filter data, query %s', query) - default_search_val = u'Full text search for: `{}`'.format(query) res = [] if not query: return {'suggestions': res} - res.append({ - 'id': -1, - 'value': query, - 'value_display': default_search_val, - 'type': 'search', - 'url': h.route_path( - 'search', _query={'q': query}) - }) - repo_group_id = safe_int(self.request.GET.get('repo_group_id')) + searcher = searcher_from_config(self.request.registry.settings) + for _q in self._get_default_search_queries(self.request.GET, searcher, query): + res.append(_q) + + repo_group_id = safe_int(self.request.GET.get('search_context[repo_group_id]')) if repo_group_id: repo_group = RepoGroup.get(repo_group_id) composed_hint = '{}/{}'.format(repo_group.group_name, query) show_hint = not query.startswith(repo_group.group_name) if repo_group and show_hint: - hint = u'Group search: `{}`'.format(composed_hint) + hint = u'Repository search inside: `{}`'.format(composed_hint) res.append({ 'id': -1, 'value': composed_hint, @@ -351,7 +496,7 @@ class HomeView(BaseAppView): for serialized_repo in repos: res.append(serialized_repo) - # TODO(marcink): permissions for that ? + # TODO(marcink): should all logged in users be allowed to search others? allowed_user_search = self._rhodecode_user.username != User.DEFAULT_USER if allowed_user_search: users = self._get_user_list(query) @@ -362,7 +507,7 @@ class HomeView(BaseAppView): for serialized_user_group in user_groups: res.append(serialized_user_group) - commits = self._get_hash_commit_list(c.auth_user, query) + commits = self._get_hash_commit_list(c.auth_user, searcher, query) if commits: unique_repos = collections.OrderedDict() for commit in commits: diff --git a/rhodecode/apps/journal/__init__.py b/rhodecode/apps/journal/__init__.py --- a/rhodecode/apps/journal/__init__.py +++ b/rhodecode/apps/journal/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/journal/tests/__init__.py b/rhodecode/apps/journal/tests/__init__.py --- a/rhodecode/apps/journal/tests/__init__.py +++ b/rhodecode/apps/journal/tests/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/journal/tests/test_journal.py b/rhodecode/apps/journal/tests/test_journal.py --- a/rhodecode/apps/journal/tests/test_journal.py +++ b/rhodecode/apps/journal/tests/test_journal.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/journal/views.py b/rhodecode/apps/journal/views.py --- a/rhodecode/apps/journal/views.py +++ b/rhodecode/apps/journal/views.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/login/__init__.py b/rhodecode/apps/login/__init__.py --- a/rhodecode/apps/login/__init__.py +++ b/rhodecode/apps/login/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/login/tests/test_login.py b/rhodecode/apps/login/tests/test_login.py --- a/rhodecode/apps/login/tests/test_login.py +++ b/rhodecode/apps/login/tests/test_login.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -107,6 +107,26 @@ class TestLoginController(object): response.mustcontain('/%s' % HG_REPO) + def test_login_regular_forbidden_when_super_admin_restriction(self): + from rhodecode.authentication.plugins.auth_rhodecode import RhodeCodeAuthPlugin + with fixture.auth_restriction(RhodeCodeAuthPlugin.AUTH_RESTRICTION_SUPER_ADMIN): + response = self.app.post(route_path('login'), + {'username': 'test_regular', + 'password': 'test12'}) + + response.mustcontain('invalid user name') + response.mustcontain('invalid password') + + def test_login_regular_forbidden_when_scope_restriction(self): + from rhodecode.authentication.plugins.auth_rhodecode import RhodeCodeAuthPlugin + with fixture.scope_restriction(RhodeCodeAuthPlugin.AUTH_RESTRICTION_SCOPE_VCS): + response = self.app.post(route_path('login'), + {'username': 'test_regular', + 'password': 'test12'}) + + response.mustcontain('invalid user name') + response.mustcontain('invalid password') + def test_login_ok_came_from(self): test_came_from = '/_admin/users?branch=stable' _url = '{}?came_from={}'.format(route_path('login'), test_came_from) diff --git a/rhodecode/apps/login/tests/test_password_reset.py b/rhodecode/apps/login/tests/test_password_reset.py --- a/rhodecode/apps/login/tests/test_password_reset.py +++ b/rhodecode/apps/login/tests/test_password_reset.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -74,20 +74,17 @@ class TestPasswordReset(TestController): 'default_password_reset': pwd_reset_setting, 'default_extern_activate': 'hg.extern_activate.auto', } - resp = self.app.post(route_path('admin_permissions_application_update'), params=params) + resp = self.app.post( + route_path('admin_permissions_application_update'), params=params) self.logout_user() login_page = self.app.get(route_path('login')) asr_login = AssertResponse(login_page) - index_page = self.app.get(h.route_path('home')) - asr_index = AssertResponse(index_page) if show_link: asr_login.one_element_exists('a.pwd_reset') - asr_index.one_element_exists('a.pwd_reset') else: asr_login.no_element_exists('a.pwd_reset') - asr_index.no_element_exists('a.pwd_reset') response = self.app.get(route_path('reset_password')) diff --git a/rhodecode/apps/login/tests/test_register_captcha.py b/rhodecode/apps/login/tests/test_register_captcha.py --- a/rhodecode/apps/login/tests/test_register_captcha.py +++ b/rhodecode/apps/login/tests/test_register_captcha.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/login/views.py b/rhodecode/apps/login/views.py --- a/rhodecode/apps/login/views.py +++ b/rhodecode/apps/login/views.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/my_account/__init__.py b/rhodecode/apps/my_account/__init__.py --- a/rhodecode/apps/my_account/__init__.py +++ b/rhodecode/apps/my_account/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -95,6 +95,18 @@ def includeme(config): pattern=ADMIN_PREFIX + '/my_account/watched') config.add_route( + name='my_account_bookmarks', + pattern=ADMIN_PREFIX + '/my_account/bookmarks') + + config.add_route( + name='my_account_bookmarks_update', + pattern=ADMIN_PREFIX + '/my_account/bookmarks/update') + + config.add_route( + name='my_account_goto_bookmark', + pattern=ADMIN_PREFIX + '/my_account/bookmark/{bookmark_id}') + + config.add_route( name='my_account_perms', pattern=ADMIN_PREFIX + '/my_account/perms') diff --git a/rhodecode/apps/my_account/tests/__init__.py b/rhodecode/apps/my_account/tests/__init__.py --- a/rhodecode/apps/my_account/tests/__init__.py +++ b/rhodecode/apps/my_account/tests/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/my_account/tests/test_my_account_auth_tokens.py b/rhodecode/apps/my_account/tests/test_my_account_auth_tokens.py --- a/rhodecode/apps/my_account/tests/test_my_account_auth_tokens.py +++ b/rhodecode/apps/my_account/tests/test_my_account_auth_tokens.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/my_account/tests/test_my_account_edit.py b/rhodecode/apps/my_account/tests/test_my_account_edit.py --- a/rhodecode/apps/my_account/tests/test_my_account_edit.py +++ b/rhodecode/apps/my_account/tests/test_my_account_edit.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -19,7 +19,7 @@ # and proprietary license terms, please see https://rhodecode.com/licenses/ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/my_account/tests/test_my_account_emails.py b/rhodecode/apps/my_account/tests/test_my_account_emails.py --- a/rhodecode/apps/my_account/tests/test_my_account_emails.py +++ b/rhodecode/apps/my_account/tests/test_my_account_emails.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/my_account/tests/test_my_account_notifications.py b/rhodecode/apps/my_account/tests/test_my_account_notifications.py --- a/rhodecode/apps/my_account/tests/test_my_account_notifications.py +++ b/rhodecode/apps/my_account/tests/test_my_account_notifications.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/my_account/tests/test_my_account_password.py b/rhodecode/apps/my_account/tests/test_my_account_password.py --- a/rhodecode/apps/my_account/tests/test_my_account_password.py +++ b/rhodecode/apps/my_account/tests/test_my_account_password.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/my_account/tests/test_my_account_profile.py b/rhodecode/apps/my_account/tests/test_my_account_profile.py --- a/rhodecode/apps/my_account/tests/test_my_account_profile.py +++ b/rhodecode/apps/my_account/tests/test_my_account_profile.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/my_account/tests/test_my_account_simple_views.py b/rhodecode/apps/my_account/tests/test_my_account_simple_views.py --- a/rhodecode/apps/my_account/tests/test_my_account_simple_views.py +++ b/rhodecode/apps/my_account/tests/test_my_account_simple_views.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/my_account/tests/test_my_account_ssh_keys.py b/rhodecode/apps/my_account/tests/test_my_account_ssh_keys.py --- a/rhodecode/apps/my_account/tests/test_my_account_ssh_keys.py +++ b/rhodecode/apps/my_account/tests/test_my_account_ssh_keys.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/my_account/views/__init__.py b/rhodecode/apps/my_account/views/__init__.py --- a/rhodecode/apps/my_account/views/__init__.py +++ b/rhodecode/apps/my_account/views/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/my_account/views/my_account.py b/rhodecode/apps/my_account/views/my_account.py --- a/rhodecode/apps/my_account/views/my_account.py +++ b/rhodecode/apps/my_account/views/my_account.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -20,29 +20,30 @@ import logging import datetime +import string import formencode import formencode.htmlfill +import peppercorn from pyramid.httpexceptions import HTTPFound from pyramid.view import view_config -from pyramid.renderers import render -from pyramid.response import Response from rhodecode.apps._base import BaseAppView, DataGridAppView from rhodecode import forms from rhodecode.lib import helpers as h from rhodecode.lib import audit_logger from rhodecode.lib.ext_json import json -from rhodecode.lib.auth import LoginRequired, NotAnonymous, CSRFRequired +from rhodecode.lib.auth import LoginRequired, NotAnonymous, CSRFRequired, \ + HasRepoPermissionAny, HasRepoGroupPermissionAny from rhodecode.lib.channelstream import ( channelstream_request, ChannelstreamException) from rhodecode.lib.utils2 import safe_int, md5, str2bool from rhodecode.model.auth_token import AuthTokenModel from rhodecode.model.comment import CommentsModel from rhodecode.model.db import ( - Repository, UserEmailMap, UserApiKeys, UserFollowing, joinedload, - PullRequest) -from rhodecode.model.forms import UserForm, UserExtraEmailForm + IntegrityError, joinedload, + Repository, UserEmailMap, UserApiKeys, UserFollowing, + PullRequest, UserBookmark, RepoGroup) from rhodecode.model.meta import Session from rhodecode.model.pull_request import PullRequestModel from rhodecode.model.scm import RepoList @@ -392,6 +393,140 @@ class MyAccountView(BaseAppView, DataGri @LoginRequired() @NotAnonymous() @view_config( + route_name='my_account_bookmarks', request_method='GET', + renderer='rhodecode:templates/admin/my_account/my_account.mako') + def my_account_bookmarks(self): + c = self.load_default_context() + c.active = 'bookmarks' + return self._get_template_context(c) + + def _process_entry(self, entry, user_id): + position = safe_int(entry.get('position')) + if position is None: + return + + # check if this is an existing entry + is_new = False + db_entry = UserBookmark().get_by_position_for_user(position, user_id) + + if db_entry and str2bool(entry.get('remove')): + log.debug('Marked bookmark %s for deletion', db_entry) + Session().delete(db_entry) + return + + if not db_entry: + # new + db_entry = UserBookmark() + is_new = True + + should_save = False + default_redirect_url = '' + + # save repo + if entry.get('bookmark_repo'): + repo = Repository.get(entry['bookmark_repo']) + perm_check = HasRepoPermissionAny( + 'repository.read', 'repository.write', 'repository.admin') + if repo and perm_check(repo_name=repo.repo_name): + db_entry.repository = repo + should_save = True + default_redirect_url = '${repo_url}' + # save repo group + elif entry.get('bookmark_repo_group'): + repo_group = RepoGroup.get(entry['bookmark_repo_group']) + perm_check = HasRepoGroupPermissionAny( + 'group.read', 'group.write', 'group.admin') + + if repo_group and perm_check(group_name=repo_group.group_name): + db_entry.repository_group = repo_group + should_save = True + default_redirect_url = '${repo_group_url}' + # save generic info + elif entry.get('title') and entry.get('redirect_url'): + should_save = True + + if should_save: + log.debug('Saving bookmark %s, new:%s', db_entry, is_new) + # mark user and position + db_entry.user_id = user_id + db_entry.position = position + db_entry.title = entry.get('title') + db_entry.redirect_url = entry.get('redirect_url') or default_redirect_url + + Session().add(db_entry) + + @LoginRequired() + @NotAnonymous() + @CSRFRequired() + @view_config( + route_name='my_account_bookmarks_update', request_method='POST') + def my_account_bookmarks_update(self): + _ = self.request.translate + c = self.load_default_context() + c.active = 'bookmarks' + + controls = peppercorn.parse(self.request.POST.items()) + user_id = c.user.user_id + + try: + for entry in controls.get('bookmarks', []): + self._process_entry(entry, user_id) + + Session().commit() + h.flash(_("Update Bookmarks"), category='success') + except IntegrityError: + h.flash(_("Failed to update bookmarks. " + "Make sure an unique position is used"), category='error') + + return HTTPFound(h.route_path('my_account_bookmarks')) + + @LoginRequired() + @NotAnonymous() + @view_config( + route_name='my_account_goto_bookmark', request_method='GET', + renderer='rhodecode:templates/admin/my_account/my_account.mako') + def my_account_goto_bookmark(self): + + bookmark_id = self.request.matchdict['bookmark_id'] + user_bookmark = UserBookmark().query()\ + .filter(UserBookmark.user_id == self.request.user.user_id) \ + .filter(UserBookmark.position == bookmark_id).scalar() + + redirect_url = h.route_path('my_account_bookmarks') + if not user_bookmark: + raise HTTPFound(redirect_url) + + if user_bookmark.repository: + repo_name = user_bookmark.repository.repo_name + base_redirect_url = h.route_path( + 'repo_summary', repo_name=repo_name) + if user_bookmark.redirect_url and \ + '${repo_url}' in user_bookmark.redirect_url: + redirect_url = string.Template(user_bookmark.redirect_url)\ + .safe_substitute({'repo_url': base_redirect_url}) + else: + redirect_url = base_redirect_url + + elif user_bookmark.repository_group: + repo_group_name = user_bookmark.repository_group.group_name + base_redirect_url = h.route_path( + 'repo_group_home', repo_group_name=repo_group_name) + if user_bookmark.redirect_url and \ + '${repo_group_url}' in user_bookmark.redirect_url: + redirect_url = string.Template(user_bookmark.redirect_url)\ + .safe_substitute({'repo_group_url': base_redirect_url}) + else: + redirect_url = base_redirect_url + + elif user_bookmark.redirect_url: + redirect_url = user_bookmark.redirect_url + + log.debug('Redirecting bookmark %s to %s', user_bookmark, redirect_url) + raise HTTPFound(redirect_url) + + @LoginRequired() + @NotAnonymous() + @view_config( route_name='my_account_perms', request_method='GET', renderer='rhodecode:templates/admin/my_account/my_account.mako') def my_account_perms(self): diff --git a/rhodecode/apps/my_account/views/my_account_notifications.py b/rhodecode/apps/my_account/views/my_account_notifications.py --- a/rhodecode/apps/my_account/views/my_account_notifications.py +++ b/rhodecode/apps/my_account/views/my_account_notifications.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/my_account/views/my_account_ssh_keys.py b/rhodecode/apps/my_account/views/my_account_ssh_keys.py --- a/rhodecode/apps/my_account/views/my_account_ssh_keys.py +++ b/rhodecode/apps/my_account/views/my_account_ssh_keys.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -71,10 +71,11 @@ class MyAccountSshKeysView(BaseAppView, c = self.load_default_context() c.active = 'ssh_keys_generate' - comment = 'RhodeCode-SSH {}'.format(c.user.email or '') - c.private, c.public = SshKeyModel().generate_keypair(comment=comment) - c.target_form_url = h.route_path( - 'my_account_ssh_keys', _query=dict(default_key=c.public)) + if c.ssh_key_generator_enabled: + comment = 'RhodeCode-SSH {}'.format(c.user.email or '') + c.private, c.public = SshKeyModel().generate_keypair(comment=comment) + c.target_form_url = h.route_path( + 'my_account_ssh_keys', _query=dict(default_key=c.public)) return self._get_template_context(c) @LoginRequired() diff --git a/rhodecode/apps/ops/__init__.py b/rhodecode/apps/ops/__init__.py --- a/rhodecode/apps/ops/__init__.py +++ b/rhodecode/apps/ops/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/ops/views.py b/rhodecode/apps/ops/views.py --- a/rhodecode/apps/ops/views.py +++ b/rhodecode/apps/ops/views.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -49,9 +49,13 @@ class OpsView(BaseAppView): 'instance': self.request.registry.settings.get('instance_id'), } if getattr(self.request, 'user'): + caller_name = 'anonymous' + if self.request.user.user_id: + caller_name = self.request.user.username + data.update({ 'caller_ip': self.request.user.ip_addr, - 'caller_name': self.request.user.username, + 'caller_name': caller_name, }) return {'ok': data} @@ -65,11 +69,13 @@ class OpsView(BaseAppView): """ Test exception handling and emails on errors """ + class TestException(Exception): pass - + # add timeout so we add some sort of rate limiter + time.sleep(2) msg = ('RhodeCode Enterprise test exception. ' - 'Generation time: {}'.format(time.time())) + 'Client:{}. Generation time: {}.'.format(self.request.user, time.time())) raise TestException(msg) @view_config( diff --git a/rhodecode/apps/repo_group/__init__.py b/rhodecode/apps/repo_group/__init__.py --- a/rhodecode/apps/repo_group/__init__.py +++ b/rhodecode/apps/repo_group/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repo_group/tests/__init__.py b/rhodecode/apps/repo_group/tests/__init__.py --- a/rhodecode/apps/repo_group/tests/__init__.py +++ b/rhodecode/apps/repo_group/tests/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repo_group/tests/test_repo_groups_advanced.py b/rhodecode/apps/repo_group/tests/test_repo_groups_advanced.py --- a/rhodecode/apps/repo_group/tests/test_repo_groups_advanced.py +++ b/rhodecode/apps/repo_group/tests/test_repo_groups_advanced.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repo_group/tests/test_repo_groups_permissions.py b/rhodecode/apps/repo_group/tests/test_repo_groups_permissions.py --- a/rhodecode/apps/repo_group/tests/test_repo_groups_permissions.py +++ b/rhodecode/apps/repo_group/tests/test_repo_groups_permissions.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repo_group/tests/test_repo_groups_settings.py b/rhodecode/apps/repo_group/tests/test_repo_groups_settings.py --- a/rhodecode/apps/repo_group/tests/test_repo_groups_settings.py +++ b/rhodecode/apps/repo_group/tests/test_repo_groups_settings.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repo_group/views/__init__.py b/rhodecode/apps/repo_group/views/__init__.py --- a/rhodecode/apps/repo_group/views/__init__.py +++ b/rhodecode/apps/repo_group/views/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repo_group/views/repo_group_advanced.py b/rhodecode/apps/repo_group/views/repo_group_advanced.py --- a/rhodecode/apps/repo_group/views/repo_group_advanced.py +++ b/rhodecode/apps/repo_group/views/repo_group_advanced.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repo_group/views/repo_group_permissions.py b/rhodecode/apps/repo_group/views/repo_group_permissions.py --- a/rhodecode/apps/repo_group/views/repo_group_permissions.py +++ b/rhodecode/apps/repo_group/views/repo_group_permissions.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repo_group/views/repo_group_settings.py b/rhodecode/apps/repo_group/views/repo_group_settings.py --- a/rhodecode/apps/repo_group/views/repo_group_settings.py +++ b/rhodecode/apps/repo_group/views/repo_group_settings.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/__init__.py b/rhodecode/apps/repository/__init__.py --- a/rhodecode/apps/repository/__init__.py +++ b/rhodecode/apps/repository/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -271,8 +271,8 @@ def includeme(config): repo_route=True) config.add_route( - name='pullrequest_repo_destinations', - pattern='/{repo_name:.*?[^/]}/pull-request/repo-destinations', + name='pullrequest_repo_targets', + pattern='/{repo_name:.*?[^/]}/pull-request/repo-targets', repo_route=True) config.add_route( diff --git a/rhodecode/apps/repository/tests/__init__.py b/rhodecode/apps/repository/tests/__init__.py --- a/rhodecode/apps/repository/tests/__init__.py +++ b/rhodecode/apps/repository/tests/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/tests/test_pull_requests_list.py b/rhodecode/apps/repository/tests/test_pull_requests_list.py --- a/rhodecode/apps/repository/tests/test_pull_requests_list.py +++ b/rhodecode/apps/repository/tests/test_pull_requests_list.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/tests/test_repo_bookmarks.py b/rhodecode/apps/repository/tests/test_repo_bookmarks.py --- a/rhodecode/apps/repository/tests/test_repo_bookmarks.py +++ b/rhodecode/apps/repository/tests/test_repo_bookmarks.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/tests/test_repo_branches.py b/rhodecode/apps/repository/tests/test_repo_branches.py --- a/rhodecode/apps/repository/tests/test_repo_branches.py +++ b/rhodecode/apps/repository/tests/test_repo_branches.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/tests/test_repo_changelog.py b/rhodecode/apps/repository/tests/test_repo_changelog.py --- a/rhodecode/apps/repository/tests/test_repo_changelog.py +++ b/rhodecode/apps/repository/tests/test_repo_changelog.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/tests/test_repo_commit_comments.py b/rhodecode/apps/repository/tests/test_repo_commit_comments.py --- a/rhodecode/apps/repository/tests/test_repo_commit_comments.py +++ b/rhodecode/apps/repository/tests/test_repo_commit_comments.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/tests/test_repo_commits.py b/rhodecode/apps/repository/tests/test_repo_commits.py --- a/rhodecode/apps/repository/tests/test_repo_commits.py +++ b/rhodecode/apps/repository/tests/test_repo_commits.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/tests/test_repo_compare.py b/rhodecode/apps/repository/tests/test_repo_compare.py --- a/rhodecode/apps/repository/tests/test_repo_compare.py +++ b/rhodecode/apps/repository/tests/test_repo_compare.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/tests/test_repo_compare_local.py b/rhodecode/apps/repository/tests/test_repo_compare_local.py --- a/rhodecode/apps/repository/tests/test_repo_compare_local.py +++ b/rhodecode/apps/repository/tests/test_repo_compare_local.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/tests/test_repo_compare_on_single_file.py b/rhodecode/apps/repository/tests/test_repo_compare_on_single_file.py --- a/rhodecode/apps/repository/tests/test_repo_compare_on_single_file.py +++ b/rhodecode/apps/repository/tests/test_repo_compare_on_single_file.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/tests/test_repo_feed.py b/rhodecode/apps/repository/tests/test_repo_feed.py --- a/rhodecode/apps/repository/tests/test_repo_feed.py +++ b/rhodecode/apps/repository/tests/test_repo_feed.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/tests/test_repo_files.py b/rhodecode/apps/repository/tests/test_repo_files.py --- a/rhodecode/apps/repository/tests/test_repo_files.py +++ b/rhodecode/apps/repository/tests/test_repo_files.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/tests/test_repo_forks.py b/rhodecode/apps/repository/tests/test_repo_forks.py --- a/rhodecode/apps/repository/tests/test_repo_forks.py +++ b/rhodecode/apps/repository/tests/test_repo_forks.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/tests/test_repo_issue_tracker.py b/rhodecode/apps/repository/tests/test_repo_issue_tracker.py --- a/rhodecode/apps/repository/tests/test_repo_issue_tracker.py +++ b/rhodecode/apps/repository/tests/test_repo_issue_tracker.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/tests/test_repo_permissions.py b/rhodecode/apps/repository/tests/test_repo_permissions.py --- a/rhodecode/apps/repository/tests/test_repo_permissions.py +++ b/rhodecode/apps/repository/tests/test_repo_permissions.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/tests/test_repo_pullrequests.py b/rhodecode/apps/repository/tests/test_repo_pullrequests.py --- a/rhodecode/apps/repository/tests/test_repo_pullrequests.py +++ b/rhodecode/apps/repository/tests/test_repo_pullrequests.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -32,7 +32,6 @@ from rhodecode.model.pull_request import from rhodecode.model.user import UserModel from rhodecode.tests import ( assert_session_flash, TEST_USER_ADMIN_LOGIN, TEST_USER_REGULAR_LOGIN) -from rhodecode.tests.utils import AssertResponse def route_path(name, params=None, **kwargs): @@ -45,7 +44,7 @@ def route_path(name, params=None, **kwar 'pullrequest_show_all': '/{repo_name}/pull-request', 'pullrequest_show_all_data': '/{repo_name}/pull-request-data', 'pullrequest_repo_refs': '/{repo_name}/pull-request/refs/{target_repo_name:.*?[^/]}', - 'pullrequest_repo_destinations': '/{repo_name}/pull-request/repo-destinations', + 'pullrequest_repo_targets': '/{repo_name}/pull-request/repo-destinations', 'pullrequest_new': '/{repo_name}/pull-request/new', 'pullrequest_create': '/{repo_name}/pull-request/create', 'pullrequest_update': '/{repo_name}/pull-request/{pull_request_id}/update', @@ -233,8 +232,7 @@ class TestPullrequestsView(object): route_path('pullrequest_update', repo_name=pull_request.target_repo.repo_name, pull_request_id=pull_request_id), - params={'update_commits': 'true', - 'csrf_token': csrf_token}) + params={'update_commits': 'true', 'csrf_token': csrf_token}) expected_msg = str(PullRequestModel.UPDATE_STATUS_MESSAGES[ UpdateFailureReason.MISSING_SOURCE_REF]) @@ -244,7 +242,8 @@ class TestPullrequestsView(object): from rhodecode.lib.vcs.backends.base import MergeFailureReason pull_request = pr_util.create_pull_request( approved=True, mergeable=True) - pull_request.target_ref = 'branch:invalid-branch:invalid-commit-id' + unicode_reference = u'branch:invalid-branch:invalid-commit-id' + pull_request.target_ref = unicode_reference Session().add(pull_request) Session().commit() @@ -255,12 +254,12 @@ class TestPullrequestsView(object): pull_request_id=pull_request_id) response = self.app.get(pull_request_url) - - assertr = AssertResponse(response) - expected_msg = PullRequestModel.MERGE_STATUS_MESSAGES[ - MergeFailureReason.MISSING_TARGET_REF] - assertr.element_contains( - 'span[data-role="merge-message"]', str(expected_msg)) + target_ref_id = 'invalid-branch' + merge_resp = MergeResponse( + True, True, '', MergeFailureReason.MISSING_TARGET_REF, + metadata={'target_ref': PullRequest.unicode_to_reference(unicode_reference)}) + response.assert_response().element_contains( + 'span[data-role="merge-message"]', merge_resp.merge_status_message) def test_comment_and_close_pull_request_custom_message_approved( self, pr_util, csrf_token, xhr_header): @@ -272,8 +271,8 @@ class TestPullrequestsView(object): self.app.post( route_path('pullrequest_comment_create', - repo_name=pull_request.target_repo.scm_instance().name, - pull_request_id=pull_request_id), + repo_name=pull_request.target_repo.scm_instance().name, + pull_request_id=pull_request_id), params={ 'close_pull_request': '1', 'text': 'Closing a PR', @@ -608,8 +607,7 @@ class TestPullrequestsView(object): response = self.app.post( route_path('pullrequest_merge', - repo_name=repo_name, - pull_request_id=pull_request_id), + repo_name=repo_name, pull_request_id=pull_request_id), params={'csrf_token': csrf_token}).follow() assert response.status_int == 200 @@ -624,10 +622,13 @@ class TestPullrequestsView(object): pull_request_id = pull_request.pull_request_id repo_name = pull_request.target_repo.scm_instance().name + merge_resp = MergeResponse(True, False, 'STUB_COMMIT_ID', + MergeFailureReason.PUSH_FAILED, + metadata={'target': 'shadow repo', + 'merge_commit': 'xxx'}) model_patcher = mock.patch.multiple( PullRequestModel, - merge_repo=mock.Mock(return_value=MergeResponse( - True, False, 'STUB_COMMIT_ID', MergeFailureReason.PUSH_FAILED)), + merge_repo=mock.Mock(return_value=merge_resp), merge_status=mock.Mock(return_value=(True, 'WRONG_MESSAGE'))) with model_patcher: @@ -637,8 +638,10 @@ class TestPullrequestsView(object): pull_request_id=pull_request_id), params={'csrf_token': csrf_token}, status=302) - assert_session_flash(response, PullRequestModel.MERGE_STATUS_MESSAGES[ - MergeFailureReason.PUSH_FAILED]) + merge_resp = MergeResponse(True, True, '', MergeFailureReason.PUSH_FAILED, + metadata={'target': 'shadow repo', + 'merge_commit': 'xxx'}) + assert_session_flash(response, merge_resp.merge_status_message) def test_update_source_revision(self, backend, csrf_token): commits = [ @@ -652,20 +655,20 @@ class TestPullrequestsView(object): # create pr from a in source to A in target pull_request = PullRequest() + pull_request.source_repo = source - # TODO: johbo: Make sure that we write the source ref this way! pull_request.source_ref = 'branch:{branch}:{commit_id}'.format( branch=backend.default_branch_name, commit_id=commit_ids['change']) + pull_request.target_repo = target - pull_request.target_ref = 'branch:{branch}:{commit_id}'.format( - branch=backend.default_branch_name, - commit_id=commit_ids['ancestor']) + branch=backend.default_branch_name, commit_id=commit_ids['ancestor']) + pull_request.revisions = [commit_ids['change']] pull_request.title = u"Test" pull_request.description = u"Description" - pull_request.author = UserModel().get_by_username( - TEST_USER_ADMIN_LOGIN) + pull_request.author = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN) + pull_request.pull_request_state = PullRequest.STATE_CREATED Session().add(pull_request) Session().commit() pull_request_id = pull_request.pull_request_id @@ -676,23 +679,21 @@ class TestPullrequestsView(object): # update PR self.app.post( route_path('pullrequest_update', - repo_name=target.repo_name, - pull_request_id=pull_request_id), - params={'update_commits': 'true', - 'csrf_token': csrf_token}) + repo_name=target.repo_name, pull_request_id=pull_request_id), + params={'update_commits': 'true', 'csrf_token': csrf_token}) + + response = self.app.get( + route_path('pullrequest_show', + repo_name=target.repo_name, + pull_request_id=pull_request.pull_request_id)) + + assert response.status_int == 200 + assert 'Pull request updated to' in response.body + assert 'with 1 added, 0 removed commits.' in response.body # check that we have now both revisions pull_request = PullRequest.get(pull_request_id) - assert pull_request.revisions == [ - commit_ids['change-2'], commit_ids['change']] - - # TODO: johbo: this should be a test on its own - response = self.app.get(route_path( - 'pullrequest_new', - repo_name=target.repo_name)) - assert response.status_int == 200 - assert 'Pull request updated to' in response.body - assert 'with 1 added, 0 removed commits.' in response.body + assert pull_request.revisions == [commit_ids['change-2'], commit_ids['change']] def test_update_target_revision(self, backend, csrf_token): commits = [ @@ -707,21 +708,21 @@ class TestPullrequestsView(object): # create pr from a in source to A in target pull_request = PullRequest() + pull_request.source_repo = source - # TODO: johbo: Make sure that we write the source ref this way! pull_request.source_ref = 'branch:{branch}:{commit_id}'.format( branch=backend.default_branch_name, commit_id=commit_ids['change']) + pull_request.target_repo = target - # TODO: johbo: Target ref should be branch based, since tip can jump - # from branch to branch pull_request.target_ref = 'branch:{branch}:{commit_id}'.format( - branch=backend.default_branch_name, - commit_id=commit_ids['ancestor']) + branch=backend.default_branch_name, commit_id=commit_ids['ancestor']) + pull_request.revisions = [commit_ids['change']] pull_request.title = u"Test" pull_request.description = u"Description" - pull_request.author = UserModel().get_by_username( - TEST_USER_ADMIN_LOGIN) + pull_request.author = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN) + pull_request.pull_request_state = PullRequest.STATE_CREATED + Session().add(pull_request) Session().commit() pull_request_id = pull_request.pull_request_id @@ -734,23 +735,21 @@ class TestPullrequestsView(object): # update PR self.app.post( route_path('pullrequest_update', - repo_name=target.repo_name, - pull_request_id=pull_request_id), - params={'update_commits': 'true', - 'csrf_token': csrf_token}, + repo_name=target.repo_name, + pull_request_id=pull_request_id), + params={'update_commits': 'true', 'csrf_token': csrf_token}, status=200) # check that we have now both revisions pull_request = PullRequest.get(pull_request_id) assert pull_request.revisions == [commit_ids['change-rebased']] assert pull_request.target_ref == 'branch:{branch}:{commit_id}'.format( - branch=backend.default_branch_name, - commit_id=commit_ids['ancestor-new']) + branch=backend.default_branch_name, commit_id=commit_ids['ancestor-new']) - # TODO: johbo: This should be a test on its own - response = self.app.get(route_path( - 'pullrequest_new', - repo_name=target.repo_name)) + response = self.app.get( + route_path('pullrequest_show', + repo_name=target.repo_name, + pull_request_id=pull_request.pull_request_id)) assert response.status_int == 200 assert 'Pull request updated to' in response.body assert 'with 1 added, 1 removed commits.' in response.body @@ -772,17 +771,14 @@ class TestPullrequestsView(object): # create pr from a in source to A in target pull_request = PullRequest() pull_request.source_repo = source - # TODO: johbo: Make sure that we write the source ref this way! + pull_request.source_ref = 'branch:{branch}:{commit_id}'.format( branch=backend.default_branch_name, commit_id=commit_ids['master-commit-3-change-2']) pull_request.target_repo = target - # TODO: johbo: Target ref should be branch based, since tip can jump - # from branch to branch pull_request.target_ref = 'branch:{branch}:{commit_id}'.format( - branch=backend.default_branch_name, - commit_id=commit_ids['feat-commit-2']) + branch=backend.default_branch_name, commit_id=commit_ids['feat-commit-2']) pull_request.revisions = [ commit_ids['feat-commit-1'], @@ -790,8 +786,8 @@ class TestPullrequestsView(object): ] pull_request.title = u"Test" pull_request.description = u"Description" - pull_request.author = UserModel().get_by_username( - TEST_USER_ADMIN_LOGIN) + pull_request.author = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN) + pull_request.pull_request_state = PullRequest.STATE_CREATED Session().add(pull_request) Session().commit() pull_request_id = pull_request.pull_request_id @@ -807,13 +803,10 @@ class TestPullrequestsView(object): route_path('pullrequest_update', repo_name=target.repo_name, pull_request_id=pull_request_id), - params={'update_commits': 'true', - 'csrf_token': csrf_token}, + params={'update_commits': 'true', 'csrf_token': csrf_token}, status=200) - response = self.app.get(route_path( - 'pullrequest_new', - repo_name=target.repo_name)) + response = self.app.get(route_path('pullrequest_new', repo_name=target.repo_name)) assert response.status_int == 200 response.mustcontain('Pull request updated to') response.mustcontain('with 0 added, 0 removed commits.') @@ -833,21 +826,17 @@ class TestPullrequestsView(object): # create pr from a in source to A in target pull_request = PullRequest() pull_request.source_repo = source - # TODO: johbo: Make sure that we write the source ref this way! + pull_request.source_ref = 'branch:{branch}:{commit_id}'.format( - branch=backend.default_branch_name, - commit_id=commit_ids['change']) + branch=backend.default_branch_name, commit_id=commit_ids['change']) pull_request.target_repo = target - # TODO: johbo: Target ref should be branch based, since tip can jump - # from branch to branch pull_request.target_ref = 'branch:{branch}:{commit_id}'.format( - branch=backend.default_branch_name, - commit_id=commit_ids['ancestor']) + branch=backend.default_branch_name, commit_id=commit_ids['ancestor']) pull_request.revisions = [commit_ids['change']] pull_request.title = u"Test" pull_request.description = u"Description" - pull_request.author = UserModel().get_by_username( - TEST_USER_ADMIN_LOGIN) + pull_request.author = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN) + pull_request.pull_request_state = PullRequest.STATE_CREATED Session().add(pull_request) Session().commit() pull_request_id = pull_request.pull_request_id @@ -860,10 +849,8 @@ class TestPullrequestsView(object): # update PR self.app.post( route_path('pullrequest_update', - repo_name=target.repo_name, - pull_request_id=pull_request_id), - params={'update_commits': 'true', - 'csrf_token': csrf_token}, + repo_name=target.repo_name, pull_request_id=pull_request_id), + params={'update_commits': 'true', 'csrf_token': csrf_token}, status=200) # Expect the target reference to be updated correctly @@ -890,13 +877,12 @@ class TestPullrequestsView(object): pull_request.source_ref = 'branch:{branch}:{commit_id}'.format( branch=branch_name, commit_id=commit_ids['new-feature']) pull_request.target_ref = 'branch:{branch}:{commit_id}'.format( - branch=backend_git.default_branch_name, - commit_id=commit_ids['old-feature']) + branch=backend_git.default_branch_name, commit_id=commit_ids['old-feature']) pull_request.revisions = [commit_ids['new-feature']] pull_request.title = u"Test" pull_request.description = u"Description" - pull_request.author = UserModel().get_by_username( - TEST_USER_ADMIN_LOGIN) + pull_request.author = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN) + pull_request.pull_request_state = PullRequest.STATE_CREATED Session().add(pull_request) Session().commit() @@ -909,11 +895,11 @@ class TestPullrequestsView(object): pull_request_id=pull_request.pull_request_id)) assert response.status_int == 200 - assert_response = AssertResponse(response) - assert_response.element_contains( + + response.assert_response().element_contains( '#changeset_compare_view_content .alert strong', 'Missing commits') - assert_response.element_contains( + response.assert_response().element_contains( '#changeset_compare_view_content .alert', 'This pull request cannot be displayed, because one or more' ' commits no longer exist in the source repository.') @@ -941,15 +927,15 @@ class TestPullrequestsView(object): pull_request_id=pull_request.pull_request_id)) assert response.status_int == 200 - assert_response = AssertResponse(response) - assert_response.element_contains( + + response.assert_response().element_contains( '#changeset_compare_view_content .alert strong', 'Missing commits') - assert_response.element_contains( + response.assert_response().element_contains( '#changeset_compare_view_content .alert', 'This pull request cannot be displayed, because one or more' ' commits no longer exist in the source repository.') - assert_response.element_contains( + response.assert_response().element_contains( '#update_commits', 'Update commits') @@ -987,8 +973,7 @@ class TestPullrequestsView(object): pull_request_id=pull_request.pull_request_id)) assert response.status_int == 200 - assert_response = AssertResponse(response) - assert_response.element_contains( + response.assert_response().element_contains( '#changeset_compare_view_content .alert strong', 'Missing commits') @@ -1004,12 +989,11 @@ class TestPullrequestsView(object): repo_name=pull_request.target_repo.scm_instance().name, pull_request_id=pull_request.pull_request_id)) assert response.status_int == 200 - assert_response = AssertResponse(response) - origin = assert_response.get_element('.pr-origininfo .tag') + origin = response.assert_response().get_element('.pr-origininfo .tag') origin_children = origin.getchildren() assert len(origin_children) == 1 - target = assert_response.get_element('.pr-targetinfo .tag') + target = response.assert_response().get_element('.pr-targetinfo .tag') target_children = target.getchildren() assert len(target_children) == 1 @@ -1038,13 +1022,12 @@ class TestPullrequestsView(object): repo_name=pull_request.target_repo.scm_instance().name, pull_request_id=pull_request.pull_request_id)) assert response.status_int == 200 - assert_response = AssertResponse(response) - origin = assert_response.get_element('.pr-origininfo .tag') + origin = response.assert_response().get_element('.pr-origininfo .tag') assert origin.text.strip() == 'bookmark: origin' assert origin.getchildren() == [] - target = assert_response.get_element('.pr-targetinfo .tag') + target = response.assert_response().get_element('.pr-targetinfo .tag') assert target.text.strip() == 'bookmark: target' assert target.getchildren() == [] @@ -1060,13 +1043,12 @@ class TestPullrequestsView(object): repo_name=pull_request.target_repo.scm_instance().name, pull_request_id=pull_request.pull_request_id)) assert response.status_int == 200 - assert_response = AssertResponse(response) - origin = assert_response.get_element('.pr-origininfo .tag') + origin = response.assert_response().get_element('.pr-origininfo .tag') assert origin.text.strip() == 'tag: origin' assert origin.getchildren() == [] - target = assert_response.get_element('.pr-targetinfo .tag') + target = response.assert_response().get_element('.pr-targetinfo .tag') assert target.text.strip() == 'tag: target' assert target.getchildren() == [] @@ -1090,12 +1072,13 @@ class TestPullrequestsView(object): repo_name=target_repo.name, pull_request_id=pr_id)) - assertr = AssertResponse(response) if mergeable: - assertr.element_value_contains('input.pr-mergeinfo', shadow_url) - assertr.element_value_contains('input.pr-mergeinfo ', 'pr-merge') + response.assert_response().element_value_contains( + 'input.pr-mergeinfo', shadow_url) + response.assert_response().element_value_contains( + 'input.pr-mergeinfo ', 'pr-merge') else: - assertr.no_element_exists('.pr-mergeinfo') + response.assert_response().no_element_exists('.pr-mergeinfo') @pytest.mark.usefixtures('app') diff --git a/rhodecode/apps/repository/tests/test_repo_settings.py b/rhodecode/apps/repository/tests/test_repo_settings.py --- a/rhodecode/apps/repository/tests/test_repo_settings.py +++ b/rhodecode/apps/repository/tests/test_repo_settings.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/tests/test_repo_settings_advanced.py b/rhodecode/apps/repository/tests/test_repo_settings_advanced.py --- a/rhodecode/apps/repository/tests/test_repo_settings_advanced.py +++ b/rhodecode/apps/repository/tests/test_repo_settings_advanced.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/tests/test_repo_summary.py b/rhodecode/apps/repository/tests/test_repo_summary.py --- a/rhodecode/apps/repository/tests/test_repo_summary.py +++ b/rhodecode/apps/repository/tests/test_repo_summary.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/tests/test_repo_tags.py b/rhodecode/apps/repository/tests/test_repo_tags.py --- a/rhodecode/apps/repository/tests/test_repo_tags.py +++ b/rhodecode/apps/repository/tests/test_repo_tags.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/tests/test_repo_vcs_settings.py b/rhodecode/apps/repository/tests/test_repo_vcs_settings.py --- a/rhodecode/apps/repository/tests/test_repo_vcs_settings.py +++ b/rhodecode/apps/repository/tests/test_repo_vcs_settings.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/tests/test_vcs_settings.py b/rhodecode/apps/repository/tests/test_vcs_settings.py --- a/rhodecode/apps/repository/tests/test_vcs_settings.py +++ b/rhodecode/apps/repository/tests/test_vcs_settings.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/utils.py b/rhodecode/apps/repository/utils.py --- a/rhodecode/apps/repository/utils.py +++ b/rhodecode/apps/repository/utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/views/__init__.py b/rhodecode/apps/repository/views/__init__.py --- a/rhodecode/apps/repository/views/__init__.py +++ b/rhodecode/apps/repository/views/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/views/repo_audit_logs.py b/rhodecode/apps/repository/views/repo_audit_logs.py --- a/rhodecode/apps/repository/views/repo_audit_logs.py +++ b/rhodecode/apps/repository/views/repo_audit_logs.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2017-2018 RhodeCode GmbH +# Copyright (C) 2017-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/views/repo_automation.py b/rhodecode/apps/repository/views/repo_automation.py --- a/rhodecode/apps/repository/views/repo_automation.py +++ b/rhodecode/apps/repository/views/repo_automation.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/views/repo_bookmarks.py b/rhodecode/apps/repository/views/repo_bookmarks.py --- a/rhodecode/apps/repository/views/repo_bookmarks.py +++ b/rhodecode/apps/repository/views/repo_bookmarks.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/views/repo_branch_permissions.py b/rhodecode/apps/repository/views/repo_branch_permissions.py --- a/rhodecode/apps/repository/views/repo_branch_permissions.py +++ b/rhodecode/apps/repository/views/repo_branch_permissions.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/views/repo_branches.py b/rhodecode/apps/repository/views/repo_branches.py --- a/rhodecode/apps/repository/views/repo_branches.py +++ b/rhodecode/apps/repository/views/repo_branches.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/views/repo_caches.py b/rhodecode/apps/repository/views/repo_caches.py --- a/rhodecode/apps/repository/views/repo_caches.py +++ b/rhodecode/apps/repository/views/repo_caches.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/views/repo_changelog.py b/rhodecode/apps/repository/views/repo_changelog.py --- a/rhodecode/apps/repository/views/repo_changelog.py +++ b/rhodecode/apps/repository/views/repo_changelog.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -203,7 +203,6 @@ class RepoChangelogView(RepoAppView): pre_load = self._get_preload_attrs() partial_xhr = self.request.environ.get('HTTP_X_PARTIAL_XHR') - try: if f_path: log.debug('generating changelog for path %s', f_path) @@ -231,7 +230,7 @@ class RepoChangelogView(RepoAppView): else: collection = self.rhodecode_vcs_repo.get_commits( branch_name=branch_name, show_hidden=show_hidden, - pre_load=pre_load) + pre_load=pre_load, translate_tags=False) self._load_changelog_data( c, collection, p, chunk_size, c.branch_name, @@ -320,7 +319,8 @@ class RepoChangelogView(RepoAppView): collection = list(reversed(collection)) else: collection = self.rhodecode_vcs_repo.get_commits( - branch_name=branch_name, show_hidden=show_hidden, pre_load=pre_load) + branch_name=branch_name, show_hidden=show_hidden, pre_load=pre_load, + translate_tags=False) p = safe_int(self.request.GET.get('page', 1), 1) try: diff --git a/rhodecode/apps/repository/views/repo_checks.py b/rhodecode/apps/repository/views/repo_checks.py --- a/rhodecode/apps/repository/views/repo_checks.py +++ b/rhodecode/apps/repository/views/repo_checks.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/views/repo_commits.py b/rhodecode/apps/repository/views/repo_commits.py --- a/rhodecode/apps/repository/views/repo_commits.py +++ b/rhodecode/apps/repository/views/repo_commits.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -97,7 +97,7 @@ class RepoCommitsView(RepoAppView): if len(commit_range) == 2: commits = self.rhodecode_vcs_repo.get_commits( start_id=commit_range[0], end_id=commit_range[1], - pre_load=pre_load) + pre_load=pre_load, translate_tags=False) commits = list(commits) else: commits = [self.rhodecode_vcs_repo.get_commit( diff --git a/rhodecode/apps/repository/views/repo_compare.py b/rhodecode/apps/repository/views/repo_compare.py --- a/rhodecode/apps/repository/views/repo_compare.py +++ b/rhodecode/apps/repository/views/repo_compare.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -27,12 +27,13 @@ from pyramid.renderers import render from pyramid.response import Response from rhodecode.apps._base import RepoAppView -from rhodecode.controllers.utils import parse_path_ref, get_commit_from_ref_name + from rhodecode.lib import helpers as h from rhodecode.lib import diffs, codeblocks from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator from rhodecode.lib.utils import safe_str from rhodecode.lib.utils2 import safe_unicode, str2bool +from rhodecode.lib.view_utils import parse_path_ref, get_commit_from_ref_name from rhodecode.lib.vcs.exceptions import ( EmptyRepositoryError, RepositoryError, RepositoryRequirementError, NodeDoesNotExistError) diff --git a/rhodecode/apps/repository/views/repo_feed.py b/rhodecode/apps/repository/views/repo_feed.py --- a/rhodecode/apps/repository/views/repo_feed.py +++ b/rhodecode/apps/repository/views/repo_feed.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2017-2018 RhodeCode GmbH +# Copyright (C) 2017-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/views/repo_files.py b/rhodecode/apps/repository/views/repo_files.py --- a/rhodecode/apps/repository/views/repo_files.py +++ b/rhodecode/apps/repository/views/repo_files.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -34,9 +34,10 @@ from pyramid.response import Response import rhodecode from rhodecode.apps._base import RepoAppView -from rhodecode.controllers.utils import parse_path_ref + from rhodecode.lib import diffs, helpers as h, rc_cache from rhodecode.lib import audit_logger +from rhodecode.lib.view_utils import parse_path_ref from rhodecode.lib.exceptions import NonRelativePathError from rhodecode.lib.codeblocks import ( filenode_as_lines_tokens, filenode_as_annotated_lines_tokens) @@ -83,6 +84,7 @@ class RepoFilesView(RepoAppView): def load_default_context(self): c = self._get_local_tmpl_context(include_app_defaults=True) c.rhodecode_repo = self.rhodecode_vcs_repo + c.enable_downloads = self.db_repo.enable_downloads return c def _ensure_not_locked(self): @@ -227,10 +229,11 @@ class RepoFilesView(RepoAppView): self, c, commit_id, f_path, full_load=False): repo_id = self.db_repo.repo_id + force_recache = self.get_recache_flag() cache_seconds = safe_int( rhodecode.CONFIG.get('rc_cache.cache_repo.expiration_time')) - cache_on = cache_seconds > 0 + cache_on = not force_recache and cache_seconds > 0 log.debug( 'Computing FILE TREE for repo_id %s commit_id `%s` and path `%s`' 'with caching: %s[TTL: %ss]' % ( diff --git a/rhodecode/apps/repository/views/repo_forks.py b/rhodecode/apps/repository/views/repo_forks.py --- a/rhodecode/apps/repository/views/repo_forks.py +++ b/rhodecode/apps/repository/views/repo_forks.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/views/repo_maintainance.py b/rhodecode/apps/repository/views/repo_maintainance.py --- a/rhodecode/apps/repository/views/repo_maintainance.py +++ b/rhodecode/apps/repository/views/repo_maintainance.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/views/repo_permissions.py b/rhodecode/apps/repository/views/repo_permissions.py --- a/rhodecode/apps/repository/views/repo_permissions.py +++ b/rhodecode/apps/repository/views/repo_permissions.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -50,8 +50,13 @@ class RepoSettingsPermissionsView(RepoAp route_name='edit_repo_perms', request_method='GET', renderer='rhodecode:templates/admin/repos/repo_edit.mako') def edit_permissions(self): + _ = self.request.translate c = self.load_default_context() c.active = 'permissions' + if self.request.GET.get('branch_permissions'): + h.flash(_('Explicitly add user or user group with write+ ' + 'permission to modify their branch permissions.'), + category='notice') return self._get_template_context(c) @LoginRequired() diff --git a/rhodecode/apps/repository/views/repo_pull_requests.py b/rhodecode/apps/repository/views/repo_pull_requests.py --- a/rhodecode/apps/repository/views/repo_pull_requests.py +++ b/rhodecode/apps/repository/views/repo_pull_requests.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -137,13 +137,6 @@ class RepoPullRequestsView(RepoAppView, }) return data - def get_recache_flag(self): - for flag_name in ['force_recache', 'force-recache', 'no-cache']: - flag_val = self.request.GET.get(flag_name) - if str2bool(flag_val): - return True - return False - @LoginRequired() @HasRepoPermissionAnyDecorator( 'repository.read', 'repository.write', 'repository.admin') @@ -272,9 +265,22 @@ class RepoPullRequestsView(RepoAppView, route_name='pullrequest_show', request_method='GET', renderer='rhodecode:templates/pullrequests/pullrequest_show.mako') def pull_request_show(self): - pull_request_id = self.request.matchdict['pull_request_id'] + _ = self.request.translate + c = self.load_default_context() + + pull_request = PullRequest.get_or_404( + self.request.matchdict['pull_request_id']) + pull_request_id = pull_request.pull_request_id - c = self.load_default_context() + if pull_request.pull_request_state != PullRequest.STATE_CREATED: + log.debug('show: forbidden because pull request is in state %s', + pull_request.pull_request_state) + msg = _(u'Cannot show pull requests in state other than `{}`. ' + u'Current state is: `{}`').format(PullRequest.STATE_CREATED, + pull_request.pull_request_state) + h.flash(msg, category='error') + raise HTTPFound(h.route_path('pullrequest_show_all', + repo_name=self.db_repo_name)) version = self.request.GET.get('version') from_version = self.request.GET.get('from_version') or version @@ -754,7 +760,7 @@ class RepoPullRequestsView(RepoAppView, default_target_repo = source_repo - if source_repo.parent: + if source_repo.parent and c.has_origin_repo_read_perm: parent_vcs_obj = source_repo.parent.scm_instance() if parent_vcs_obj and not parent_vcs_obj.is_empty(): # change default if we have a parent repo @@ -811,37 +817,51 @@ class RepoPullRequestsView(RepoAppView, @HasRepoPermissionAnyDecorator( 'repository.read', 'repository.write', 'repository.admin') @view_config( - route_name='pullrequest_repo_destinations', request_method='GET', + route_name='pullrequest_repo_targets', request_method='GET', renderer='json_ext', xhr=True) - def pull_request_repo_destinations(self): + def pullrequest_repo_targets(self): _ = self.request.translate filter_query = self.request.GET.get('query') + # get the parents + parent_target_repos = [] + if self.db_repo.parent: + parents_query = Repository.query() \ + .order_by(func.length(Repository.repo_name)) \ + .filter(Repository.fork_id == self.db_repo.parent.repo_id) + + if filter_query: + ilike_expression = u'%{}%'.format(safe_unicode(filter_query)) + parents_query = parents_query.filter( + Repository.repo_name.ilike(ilike_expression)) + parents = parents_query.limit(20).all() + + for parent in parents: + parent_vcs_obj = parent.scm_instance() + if parent_vcs_obj and not parent_vcs_obj.is_empty(): + parent_target_repos.append(parent) + + # get other forks, and repo itself query = Repository.query() \ .order_by(func.length(Repository.repo_name)) \ .filter( - or_(Repository.repo_name == self.db_repo.repo_name, - Repository.fork_id == self.db_repo.repo_id)) + or_(Repository.repo_id == self.db_repo.repo_id, # repo itself + Repository.fork_id == self.db_repo.repo_id) # forks of this repo + ) \ + .filter(~Repository.repo_id.in_([x.repo_id for x in parent_target_repos])) if filter_query: ilike_expression = u'%{}%'.format(safe_unicode(filter_query)) - query = query.filter( - Repository.repo_name.ilike(ilike_expression)) + query = query.filter(Repository.repo_name.ilike(ilike_expression)) - add_parent = False - if self.db_repo.parent: - if filter_query in self.db_repo.parent.repo_name: - parent_vcs_obj = self.db_repo.parent.scm_instance() - if parent_vcs_obj and not parent_vcs_obj.is_empty(): - add_parent = True + limit = max(20 - len(parent_target_repos), 5) # not less then 5 + target_repos = query.limit(limit).all() - limit = 20 - 1 if add_parent else 20 - all_repos = query.limit(limit).all() - if add_parent: - all_repos += [self.db_repo.parent] + all_target_repos = target_repos + parent_target_repos repos = [] - for obj in ScmModel().get_repos(all_repos): + # This checks permissions to the repositories + for obj in ScmModel().get_repos(all_target_repos): repos.append({ 'id': obj['name'], 'text': obj['name'], @@ -904,12 +924,17 @@ class RepoPullRequestsView(RepoAppView, source_db_repo = Repository.get_by_repo_name(_form['source_repo']) target_db_repo = Repository.get_by_repo_name(_form['target_repo']) + if not (source_db_repo or target_db_repo): + h.flash(_('source_repo or target repo not found'), category='error') + raise HTTPFound( + h.route_path('pullrequest_new', repo_name=self.db_repo_name)) + # re-check permissions again here # source_repo we must have read permissions source_perm = HasRepoPermissionAny( - 'repository.read', - 'repository.write', 'repository.admin')(source_db_repo.repo_name) + 'repository.read', 'repository.write', 'repository.admin')( + source_db_repo.repo_name) if not source_perm: msg = _('Not Enough permissions to source repo `{}`.'.format( source_db_repo.repo_name)) @@ -923,8 +948,8 @@ class RepoPullRequestsView(RepoAppView, # target repo we must have read permissions, and also later on # we want to check branch permissions here target_perm = HasRepoPermissionAny( - 'repository.read', - 'repository.write', 'repository.admin')(target_db_repo.repo_name) + 'repository.read', 'repository.write', 'repository.admin')( + target_db_repo.repo_name) if not target_perm: msg = _('Not Enough permissions to target repo `{}`.'.format( target_db_repo.repo_name)) @@ -1027,6 +1052,15 @@ class RepoPullRequestsView(RepoAppView, h.flash(msg, category='error') return True + if pull_request.pull_request_state != PullRequest.STATE_CREATED: + log.debug('update: forbidden because pull request is in state %s', + pull_request.pull_request_state) + msg = _(u'Cannot update pull requests in state other than `{}`. ' + u'Current state is: `{}`').format(PullRequest.STATE_CREATED, + pull_request.pull_request_state) + h.flash(msg, category='error') + return True + # only owner or admin can update it allowed_to_update = PullRequestModel().check_user_update( pull_request, self._rhodecode_user) @@ -1069,7 +1103,9 @@ class RepoPullRequestsView(RepoAppView, def _update_commits(self, pull_request): _ = self.request.translate - resp = PullRequestModel().update_commits(pull_request) + + with pull_request.set_state(PullRequest.STATE_UPDATING): + resp = PullRequestModel().update_commits(pull_request) if resp.executed: @@ -1082,10 +1118,9 @@ class RepoPullRequestsView(RepoAppView, else: changed = 'nothing' - msg = _( - u'Pull request updated to "{source_commit_id}" with ' - u'{count_added} added, {count_removed} removed commits. ' - u'Source of changes: {change_source}') + msg = _(u'Pull request updated to "{source_commit_id}" with ' + u'{count_added} added, {count_removed} removed commits. ' + u'Source of changes: {change_source}') msg = msg.format( source_commit_id=pull_request.source_ref_parts.commit_id, count_added=len(resp.changes.added), @@ -1094,8 +1129,7 @@ class RepoPullRequestsView(RepoAppView, h.flash(msg, category='success') channel = '/repo${}$/pr/{}'.format( - pull_request.target_repo.repo_name, - pull_request.pull_request_id) + pull_request.target_repo.repo_name, pull_request.pull_request_id) message = msg + ( ' - ' '{}'.format(_('Reload page'))) @@ -1128,11 +1162,26 @@ class RepoPullRequestsView(RepoAppView, """ pull_request = PullRequest.get_or_404( self.request.matchdict['pull_request_id']) + _ = self.request.translate + + if pull_request.pull_request_state != PullRequest.STATE_CREATED: + log.debug('show: forbidden because pull request is in state %s', + pull_request.pull_request_state) + msg = _(u'Cannot merge pull requests in state other than `{}`. ' + u'Current state is: `{}`').format(PullRequest.STATE_CREATED, + pull_request.pull_request_state) + h.flash(msg, category='error') + raise HTTPFound( + h.route_path('pullrequest_show', + repo_name=pull_request.target_repo.repo_name, + pull_request_id=pull_request.pull_request_id)) self.load_default_context() - check = MergeCheck.validate( - pull_request, auth_user=self._rhodecode_user, - translator=self.request.translate) + + with pull_request.set_state(PullRequest.STATE_UPDATING): + check = MergeCheck.validate( + pull_request, auth_user=self._rhodecode_user, + translator=self.request.translate) merge_possible = not check.failed for err_type, error_msg in check.errors: @@ -1144,8 +1193,9 @@ class RepoPullRequestsView(RepoAppView, self.request.environ, repo_name=pull_request.target_repo.repo_name, username=self._rhodecode_db_user.username, action='push', scm=pull_request.target_repo.repo_type) - self._merge_pull_request( - pull_request, self._rhodecode_db_user, extras) + with pull_request.set_state(PullRequest.STATE_UPDATING): + self._merge_pull_request( + pull_request, self._rhodecode_db_user, extras) else: log.debug("Pre-conditions failed, NOT merging.") @@ -1167,10 +1217,8 @@ class RepoPullRequestsView(RepoAppView, h.flash(msg, category='success') else: log.debug( - "The merge was not successful. Merge response: %s", - merge_resp) - msg = PullRequestModel().merge_status_message( - merge_resp.failure_reason) + "The merge was not successful. Merge response: %s", merge_resp) + msg = merge_resp.merge_status_message h.flash(msg, category='error') def _update_reviewers(self, pull_request, review_members, reviewer_rules): diff --git a/rhodecode/apps/repository/views/repo_review_rules.py b/rhodecode/apps/repository/views/repo_review_rules.py --- a/rhodecode/apps/repository/views/repo_review_rules.py +++ b/rhodecode/apps/repository/views/repo_review_rules.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/views/repo_settings.py b/rhodecode/apps/repository/views/repo_settings.py --- a/rhodecode/apps/repository/views/repo_settings.py +++ b/rhodecode/apps/repository/views/repo_settings.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/views/repo_settings_advanced.py b/rhodecode/apps/repository/views/repo_settings_advanced.py --- a/rhodecode/apps/repository/views/repo_settings_advanced.py +++ b/rhodecode/apps/repository/views/repo_settings_advanced.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -73,6 +73,8 @@ class RepoSettingsView(RepoAppView): 'repository.write', 'repository.read', 'repository.admin')( self.db_repo.fork.repo_name, 'repo set as fork page') + c.ver_info_dict = self.rhodecode_vcs_repo.get_hooks_info() + return self._get_template_context(c) @LoginRequired() diff --git a/rhodecode/apps/repository/views/repo_settings_fields.py b/rhodecode/apps/repository/views/repo_settings_fields.py --- a/rhodecode/apps/repository/views/repo_settings_fields.py +++ b/rhodecode/apps/repository/views/repo_settings_fields.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2017-2018 RhodeCode GmbH +# Copyright (C) 2017-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/views/repo_settings_issue_trackers.py b/rhodecode/apps/repository/views/repo_settings_issue_trackers.py --- a/rhodecode/apps/repository/views/repo_settings_issue_trackers.py +++ b/rhodecode/apps/repository/views/repo_settings_issue_trackers.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2017-2018 RhodeCode GmbH +# Copyright (C) 2017-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/views/repo_settings_remote.py b/rhodecode/apps/repository/views/repo_settings_remote.py --- a/rhodecode/apps/repository/views/repo_settings_remote.py +++ b/rhodecode/apps/repository/views/repo_settings_remote.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2017-2018 RhodeCode GmbH +# Copyright (C) 2017-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/views/repo_settings_vcs.py b/rhodecode/apps/repository/views/repo_settings_vcs.py --- a/rhodecode/apps/repository/views/repo_settings_vcs.py +++ b/rhodecode/apps/repository/views/repo_settings_vcs.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2017-2018 RhodeCode GmbH +# Copyright (C) 2017-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/views/repo_strip.py b/rhodecode/apps/repository/views/repo_strip.py --- a/rhodecode/apps/repository/views/repo_strip.py +++ b/rhodecode/apps/repository/views/repo_strip.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2017-2018 RhodeCode GmbH +# Copyright (C) 2017-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/repository/views/repo_summary.py b/rhodecode/apps/repository/views/repo_summary.py --- a/rhodecode/apps/repository/views/repo_summary.py +++ b/rhodecode/apps/repository/views/repo_summary.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -24,7 +24,7 @@ import rhodecode from pyramid.view import view_config -from rhodecode.controllers import utils +from rhodecode.lib.view_utils import get_format_ref_id from rhodecode.apps._base import RepoAppView from rhodecode.config.conf import (LANGUAGES_EXTENSIONS_MAP) from rhodecode.lib import helpers as h, rc_cache @@ -141,7 +141,8 @@ class RepoSummaryView(RepoAppView): pre_load = ['author', 'branch', 'date', 'message'] try: - collection = self.rhodecode_vcs_repo.get_commits(pre_load=pre_load) + collection = self.rhodecode_vcs_repo.get_commits( + pre_load=pre_load, translate_tags=False) except EmptyRepositoryError: collection = self.rhodecode_vcs_repo @@ -351,7 +352,7 @@ class RepoSummaryView(RepoAppView): return data def _create_reference_data(self, repo, full_repo_name, refs_to_create): - format_ref_id = utils.get_format_ref_id(repo) + format_ref_id = get_format_ref_id(repo) result = [] for title, refs, ref_type in refs_to_create: diff --git a/rhodecode/apps/repository/views/repo_tags.py b/rhodecode/apps/repository/views/repo_tags.py --- a/rhodecode/apps/repository/views/repo_tags.py +++ b/rhodecode/apps/repository/views/repo_tags.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/search/__init__.py b/rhodecode/apps/search/__init__.py --- a/rhodecode/apps/search/__init__.py +++ b/rhodecode/apps/search/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -28,7 +28,16 @@ def includeme(config): config.add_route( name='search_repo', + pattern='/{repo_name:.*?[^/]}/_search', repo_route=True) + + config.add_route( + name='search_repo_alt', pattern='/{repo_name:.*?[^/]}/search', repo_route=True) + config.add_route( + name='search_repo_group', + pattern='/{repo_group_name:.*?[^/]}/_search', + repo_group_route=True) + # Scan module for configuration decorators. config.scan('.views', ignore='.tests') diff --git a/rhodecode/apps/search/tests/test_search.py b/rhodecode/apps/search/tests/test_search.py --- a/rhodecode/apps/search/tests/test_search.py +++ b/rhodecode/apps/search/tests/test_search.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/search/views.py b/rhodecode/apps/search/views.py --- a/rhodecode/apps/search/views.py +++ b/rhodecode/apps/search/views.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -23,8 +23,9 @@ import urllib from pyramid.view import view_config from webhelpers.util import update_params -from rhodecode.apps._base import BaseAppView, RepoAppView -from rhodecode.lib.auth import (LoginRequired, HasRepoPermissionAnyDecorator) +from rhodecode.apps._base import BaseAppView, RepoAppView, RepoGroupAppView +from rhodecode.lib.auth import ( + LoginRequired, HasRepoPermissionAnyDecorator, HasRepoGroupPermissionAnyDecorator) from rhodecode.lib.helpers import Page from rhodecode.lib.utils2 import safe_str from rhodecode.lib.index import searcher_from_config @@ -34,22 +35,25 @@ from rhodecode.model.validation_schema.s log = logging.getLogger(__name__) -def search(request, tmpl_context, repo_name): +def perform_search(request, tmpl_context, repo_name=None, repo_group_name=None): searcher = searcher_from_config(request.registry.settings) formatted_results = [] execution_time = '' schema = search_schema.SearchParamsSchema() - + search_tags = [] search_params = {} errors = [] try: search_params = schema.deserialize( - dict(search_query=request.GET.get('q'), - search_type=request.GET.get('type'), - search_sort=request.GET.get('sort'), - page_limit=request.GET.get('page_limit'), - requested_page=request.GET.get('page')) + dict( + search_query=request.GET.get('q'), + search_type=request.GET.get('type'), + search_sort=request.GET.get('sort'), + search_max_lines=request.GET.get('max_lines'), + page_limit=request.GET.get('page_limit'), + requested_page=request.GET.get('page'), + ) ) except validation_schema.Invalid as e: errors = e.children @@ -57,20 +61,22 @@ def search(request, tmpl_context, repo_n def url_generator(**kw): q = urllib.quote(safe_str(search_query)) return update_params( - "?q=%s&type=%s" % (q, safe_str(search_type)), **kw) + "?q=%s&type=%s&max_lines=%s" % ( + q, safe_str(search_type), search_max_lines), **kw) c = tmpl_context search_query = search_params.get('search_query') search_type = search_params.get('search_type') search_sort = search_params.get('search_sort') + search_max_lines = search_params.get('search_max_lines') if search_params.get('search_query'): page_limit = search_params['page_limit'] requested_page = search_params['requested_page'] try: search_result = searcher.search( - search_query, search_type, c.auth_user, repo_name, - requested_page, page_limit, search_sort) + search_query, search_type, c.auth_user, repo_name, repo_group_name, + requested_page=requested_page, page_limit=page_limit, sort=search_sort) formatted_results = Page( search_result['results'], page=requested_page, @@ -79,6 +85,8 @@ def search(request, tmpl_context, repo_n finally: searcher.cleanup() + search_tags = searcher.extract_search_tags(search_query) + if not search_result['error']: execution_time = '%s results (%.3f seconds)' % ( search_result['count'], @@ -90,6 +98,7 @@ def search(request, tmpl_context, repo_n c.perm_user = c.auth_user c.repo_name = repo_name + c.repo_group_name = repo_group_name c.sort = search_sort c.url_generator = url_generator c.errors = errors @@ -98,12 +107,12 @@ def search(request, tmpl_context, repo_n c.cur_query = search_query c.search_type = search_type c.searcher = searcher + c.search_tags = search_tags class SearchView(BaseAppView): def load_default_context(self): c = self._get_local_tmpl_context() - return c @LoginRequired() @@ -112,14 +121,14 @@ class SearchView(BaseAppView): renderer='rhodecode:templates/search/search.mako') def search(self): c = self.load_default_context() - search(self.request, c, repo_name=None) + perform_search(self.request, c) return self._get_template_context(c) class SearchRepoView(RepoAppView): def load_default_context(self): c = self._get_local_tmpl_context() - + c.active = 'search' return c @LoginRequired() @@ -128,7 +137,28 @@ class SearchRepoView(RepoAppView): @view_config( route_name='search_repo', request_method='GET', renderer='rhodecode:templates/search/search.mako') + @view_config( + route_name='search_repo_alt', request_method='GET', + renderer='rhodecode:templates/search/search.mako') def search_repo(self): c = self.load_default_context() - search(self.request, c, repo_name=self.db_repo_name) + perform_search(self.request, c, repo_name=self.db_repo_name) return self._get_template_context(c) + + +class SearchRepoGroupView(RepoGroupAppView): + def load_default_context(self): + c = self._get_local_tmpl_context() + c.active = 'search' + return c + + @LoginRequired() + @HasRepoGroupPermissionAnyDecorator( + 'group.read', 'group.write', 'group.admin') + @view_config( + route_name='search_repo_group', request_method='GET', + renderer='rhodecode:templates/search/search.mako') + def search_repo_group(self): + c = self.load_default_context() + perform_search(self.request, c, repo_group_name=self.db_repo_group_name) + return self._get_template_context(c) diff --git a/rhodecode/apps/ssh_support/__init__.py b/rhodecode/apps/ssh_support/__init__.py --- a/rhodecode/apps/ssh_support/__init__.py +++ b/rhodecode/apps/ssh_support/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -36,6 +36,7 @@ def _sanitize_settings_and_apply_default _bool_setting(settings, config_keys.generate_authorized_keyfile, 'false') _bool_setting(settings, config_keys.wrapper_allow_shell, 'false') _bool_setting(settings, config_keys.enable_debug_logging, 'false') + _bool_setting(settings, config_keys.ssh_key_generator_enabled, 'true') _string_setting(settings, config_keys.authorized_keys_file_path, '~/.ssh/authorized_keys_rhodecode', diff --git a/rhodecode/apps/ssh_support/config_keys.py b/rhodecode/apps/ssh_support/config_keys.py --- a/rhodecode/apps/ssh_support/config_keys.py +++ b/rhodecode/apps/ssh_support/config_keys.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -24,6 +24,7 @@ generate_authorized_keyfile = 'ssh.generate_authorized_keyfile' authorized_keys_file_path = 'ssh.authorized_keys_file_path' authorized_keys_line_ssh_opts = 'ssh.authorized_keys_ssh_opts' +ssh_key_generator_enabled = 'ssh.enable_ui_key_generator' wrapper_cmd = 'ssh.wrapper_cmd' wrapper_allow_shell = 'ssh.wrapper_cmd_allow_shell' enable_debug_logging = 'ssh.enable_debug_logging' diff --git a/rhodecode/apps/ssh_support/events.py b/rhodecode/apps/ssh_support/events.py --- a/rhodecode/apps/ssh_support/events.py +++ b/rhodecode/apps/ssh_support/events.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/ssh_support/lib/__init__.py b/rhodecode/apps/ssh_support/lib/__init__.py --- a/rhodecode/apps/ssh_support/lib/__init__.py +++ b/rhodecode/apps/ssh_support/lib/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/ssh_support/lib/backends/__init__.py b/rhodecode/apps/ssh_support/lib/backends/__init__.py --- a/rhodecode/apps/ssh_support/lib/backends/__init__.py +++ b/rhodecode/apps/ssh_support/lib/backends/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/ssh_support/lib/backends/base.py b/rhodecode/apps/ssh_support/lib/backends/base.py --- a/rhodecode/apps/ssh_support/lib/backends/base.py +++ b/rhodecode/apps/ssh_support/lib/backends/base.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/ssh_support/lib/backends/git.py b/rhodecode/apps/ssh_support/lib/backends/git.py --- a/rhodecode/apps/ssh_support/lib/backends/git.py +++ b/rhodecode/apps/ssh_support/lib/backends/git.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/ssh_support/lib/backends/hg.py b/rhodecode/apps/ssh_support/lib/backends/hg.py --- a/rhodecode/apps/ssh_support/lib/backends/hg.py +++ b/rhodecode/apps/ssh_support/lib/backends/hg.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/ssh_support/lib/backends/svn.py b/rhodecode/apps/ssh_support/lib/backends/svn.py --- a/rhodecode/apps/ssh_support/lib/backends/svn.py +++ b/rhodecode/apps/ssh_support/lib/backends/svn.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/ssh_support/lib/ssh_wrapper.py b/rhodecode/apps/ssh_support/lib/ssh_wrapper.py --- a/rhodecode/apps/ssh_support/lib/ssh_wrapper.py +++ b/rhodecode/apps/ssh_support/lib/ssh_wrapper.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/ssh_support/subscribers.py b/rhodecode/apps/ssh_support/subscribers.py --- a/rhodecode/apps/ssh_support/subscribers.py +++ b/rhodecode/apps/ssh_support/subscribers.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/ssh_support/tests/__init__.py b/rhodecode/apps/ssh_support/tests/__init__.py --- a/rhodecode/apps/ssh_support/tests/__init__.py +++ b/rhodecode/apps/ssh_support/tests/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/ssh_support/tests/conftest.py b/rhodecode/apps/ssh_support/tests/conftest.py --- a/rhodecode/apps/ssh_support/tests/conftest.py +++ b/rhodecode/apps/ssh_support/tests/conftest.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/ssh_support/tests/test_server_git.py b/rhodecode/apps/ssh_support/tests/test_server_git.py --- a/rhodecode/apps/ssh_support/tests/test_server_git.py +++ b/rhodecode/apps/ssh_support/tests/test_server_git.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/ssh_support/tests/test_server_hg.py b/rhodecode/apps/ssh_support/tests/test_server_hg.py --- a/rhodecode/apps/ssh_support/tests/test_server_hg.py +++ b/rhodecode/apps/ssh_support/tests/test_server_hg.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/ssh_support/tests/test_server_svn.py b/rhodecode/apps/ssh_support/tests/test_server_svn.py --- a/rhodecode/apps/ssh_support/tests/test_server_svn.py +++ b/rhodecode/apps/ssh_support/tests/test_server_svn.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/ssh_support/tests/test_ssh_authorized_keys_gen.py b/rhodecode/apps/ssh_support/tests/test_ssh_authorized_keys_gen.py --- a/rhodecode/apps/ssh_support/tests/test_ssh_authorized_keys_gen.py +++ b/rhodecode/apps/ssh_support/tests/test_ssh_authorized_keys_gen.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/ssh_support/tests/test_ssh_wrapper.py b/rhodecode/apps/ssh_support/tests/test_ssh_wrapper.py --- a/rhodecode/apps/ssh_support/tests/test_ssh_wrapper.py +++ b/rhodecode/apps/ssh_support/tests/test_ssh_wrapper.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/ssh_support/utils.py b/rhodecode/apps/ssh_support/utils.py --- a/rhodecode/apps/ssh_support/utils.py +++ b/rhodecode/apps/ssh_support/utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/svn_support/__init__.py b/rhodecode/apps/svn_support/__init__.py --- a/rhodecode/apps/svn_support/__init__.py +++ b/rhodecode/apps/svn_support/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -17,10 +17,10 @@ # This program is dual-licensed. If you wish to learn more about the # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ - +import os import logging -import os import shlex +from pyramid import compat # Do not use `from rhodecode import events` here, it will be overridden by the # events module in this package due to pythons import mechanism. @@ -85,6 +85,6 @@ def _append_path_sep(path): """ Append the path separator if missing. """ - if isinstance(path, basestring) and not path.endswith(os.path.sep): + if isinstance(path, compat.string_types) and not path.endswith(os.path.sep): path += os.path.sep return path diff --git a/rhodecode/apps/svn_support/config_keys.py b/rhodecode/apps/svn_support/config_keys.py --- a/rhodecode/apps/svn_support/config_keys.py +++ b/rhodecode/apps/svn_support/config_keys.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/svn_support/events.py b/rhodecode/apps/svn_support/events.py --- a/rhodecode/apps/svn_support/events.py +++ b/rhodecode/apps/svn_support/events.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/svn_support/subscribers.py b/rhodecode/apps/svn_support/subscribers.py --- a/rhodecode/apps/svn_support/subscribers.py +++ b/rhodecode/apps/svn_support/subscribers.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/svn_support/tests/test_mod_dav_svn_config.py b/rhodecode/apps/svn_support/tests/test_mod_dav_svn_config.py --- a/rhodecode/apps/svn_support/tests/test_mod_dav_svn_config.py +++ b/rhodecode/apps/svn_support/tests/test_mod_dav_svn_config.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/svn_support/utils.py b/rhodecode/apps/svn_support/utils.py --- a/rhodecode/apps/svn_support/utils.py +++ b/rhodecode/apps/svn_support/utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/user_group/__init__.py b/rhodecode/apps/user_group/__init__.py --- a/rhodecode/apps/user_group/__init__.py +++ b/rhodecode/apps/user_group/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/user_group/tests/__init__.py b/rhodecode/apps/user_group/tests/__init__.py --- a/rhodecode/apps/user_group/tests/__init__.py +++ b/rhodecode/apps/user_group/tests/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/user_group/tests/test_user_groups.py b/rhodecode/apps/user_group/tests/test_user_groups.py --- a/rhodecode/apps/user_group/tests/test_user_groups.py +++ b/rhodecode/apps/user_group/tests/test_user_groups.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/user_group/tests/test_user_groups_permissions.py b/rhodecode/apps/user_group/tests/test_user_groups_permissions.py --- a/rhodecode/apps/user_group/tests/test_user_groups_permissions.py +++ b/rhodecode/apps/user_group/tests/test_user_groups_permissions.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/user_group/views/__init__.py b/rhodecode/apps/user_group/views/__init__.py --- a/rhodecode/apps/user_group/views/__init__.py +++ b/rhodecode/apps/user_group/views/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/user_group_profile/__init__.py b/rhodecode/apps/user_group_profile/__init__.py --- a/rhodecode/apps/user_group_profile/__init__.py +++ b/rhodecode/apps/user_group_profile/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/user_group_profile/tests/__init__.py b/rhodecode/apps/user_group_profile/tests/__init__.py --- a/rhodecode/apps/user_group_profile/tests/__init__.py +++ b/rhodecode/apps/user_group_profile/tests/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/user_group_profile/tests/test_user_group.py b/rhodecode/apps/user_group_profile/tests/test_user_group.py --- a/rhodecode/apps/user_group_profile/tests/test_user_group.py +++ b/rhodecode/apps/user_group_profile/tests/test_user_group.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/user_group_profile/views.py b/rhodecode/apps/user_group_profile/views.py --- a/rhodecode/apps/user_group_profile/views.py +++ b/rhodecode/apps/user_group_profile/views.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/user_profile/__init__.py b/rhodecode/apps/user_profile/__init__.py --- a/rhodecode/apps/user_profile/__init__.py +++ b/rhodecode/apps/user_profile/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/user_profile/tests/__init__.py b/rhodecode/apps/user_profile/tests/__init__.py --- a/rhodecode/apps/user_profile/tests/__init__.py +++ b/rhodecode/apps/user_profile/tests/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/user_profile/tests/test_users.py b/rhodecode/apps/user_profile/tests/test_users.py --- a/rhodecode/apps/user_profile/tests/test_users.py +++ b/rhodecode/apps/user_profile/tests/test_users.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/apps/user_profile/views.py b/rhodecode/apps/user_profile/views.py --- a/rhodecode/apps/user_profile/views.py +++ b/rhodecode/apps/user_profile/views.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/authentication/__init__.py b/rhodecode/authentication/__init__.py --- a/rhodecode/authentication/__init__.py +++ b/rhodecode/authentication/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/authentication/base.py b/rhodecode/authentication/base.py --- a/rhodecode/authentication/base.py +++ b/rhodecode/authentication/base.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/authentication/interface.py b/rhodecode/authentication/interface.py --- a/rhodecode/authentication/interface.py +++ b/rhodecode/authentication/interface.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/authentication/plugins/__init__.py b/rhodecode/authentication/plugins/__init__.py --- a/rhodecode/authentication/plugins/__init__.py +++ b/rhodecode/authentication/plugins/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/authentication/plugins/auth_crowd.py b/rhodecode/authentication/plugins/auth_crowd.py --- a/rhodecode/authentication/plugins/auth_crowd.py +++ b/rhodecode/authentication/plugins/auth_crowd.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/authentication/plugins/auth_headers.py b/rhodecode/authentication/plugins/auth_headers.py --- a/rhodecode/authentication/plugins/auth_headers.py +++ b/rhodecode/authentication/plugins/auth_headers.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/authentication/plugins/auth_jasig_cas.py b/rhodecode/authentication/plugins/auth_jasig_cas.py --- a/rhodecode/authentication/plugins/auth_jasig_cas.py +++ b/rhodecode/authentication/plugins/auth_jasig_cas.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/authentication/plugins/auth_ldap.py b/rhodecode/authentication/plugins/auth_ldap.py --- a/rhodecode/authentication/plugins/auth_ldap.py +++ b/rhodecode/authentication/plugins/auth_ldap.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/authentication/plugins/auth_pam.py b/rhodecode/authentication/plugins/auth_pam.py --- a/rhodecode/authentication/plugins/auth_pam.py +++ b/rhodecode/authentication/plugins/auth_pam.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/authentication/plugins/auth_rhodecode.py b/rhodecode/authentication/plugins/auth_rhodecode.py --- a/rhodecode/authentication/plugins/auth_rhodecode.py +++ b/rhodecode/authentication/plugins/auth_rhodecode.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -24,12 +24,15 @@ RhodeCode authentication plugin for buil import logging -from rhodecode.translation import _ +import colander -from rhodecode.authentication.base import RhodeCodeAuthPluginBase, hybrid_property -from rhodecode.authentication.routes import AuthnPluginResourceBase +from rhodecode.translation import _ from rhodecode.lib.utils2 import safe_str from rhodecode.model.db import User +from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase +from rhodecode.authentication.base import ( + RhodeCodeAuthPluginBase, hybrid_property, HTTP_TYPE, VCS_TYPE) +from rhodecode.authentication.routes import AuthnPluginResourceBase log = logging.getLogger(__name__) @@ -45,6 +48,11 @@ class RhodecodeAuthnResource(AuthnPlugin class RhodeCodeAuthPlugin(RhodeCodeAuthPluginBase): uid = 'rhodecode' + AUTH_RESTRICTION_NONE = 'user_all' + AUTH_RESTRICTION_SUPER_ADMIN = 'user_super_admin' + AUTH_RESTRICTION_SCOPE_ALL = 'scope_all' + AUTH_RESTRICTION_SCOPE_HTTP = 'scope_http' + AUTH_RESTRICTION_SCOPE_VCS = 'scope_vcs' def includeme(self, config): config.add_authn_plugin(self) @@ -64,6 +72,9 @@ class RhodeCodeAuthPlugin(RhodeCodeAuthP route_name='auth_home', context=RhodecodeAuthnResource) + def get_settings_schema(self): + return RhodeCodeSettingsSchema() + def get_display_name(self): return _('RhodeCode Internal') @@ -94,10 +105,34 @@ class RhodeCodeAuthPlugin(RhodeCodeAuthP if not userobj: log.debug('userobj was:%s skipping', userobj) return None + if userobj.extern_type != self.name: - log.warning( - "userobj:%s extern_type mismatch got:`%s` expected:`%s`", - userobj, userobj.extern_type, self.name) + log.warning("userobj:%s extern_type mismatch got:`%s` expected:`%s`", + userobj, userobj.extern_type, self.name) + return None + + # check scope of auth + scope_restriction = settings.get('scope_restriction', '') + + if scope_restriction == self.AUTH_RESTRICTION_SCOPE_HTTP \ + and self.auth_type != HTTP_TYPE: + log.warning("userobj:%s tried scope type %s and scope restriction is set to %s", + userobj, self.auth_type, scope_restriction) + return None + + if scope_restriction == self.AUTH_RESTRICTION_SCOPE_VCS \ + and self.auth_type != VCS_TYPE: + log.warning("userobj:%s tried scope type %s and scope restriction is set to %s", + userobj, self.auth_type, scope_restriction) + return None + + # check super-admin restriction + auth_restriction = settings.get('auth_restriction', '') + + if auth_restriction == self.AUTH_RESTRICTION_SUPER_ADMIN \ + and userobj.admin is False: + log.warning("userobj:%s is not super-admin and auth restriction is set to %s", + userobj, auth_restriction) return None user_attrs = { @@ -131,23 +166,55 @@ class RhodeCodeAuthPlugin(RhodeCodeAuthP user_attrs['_hash_migrate'] = new_hash if userobj.username == User.DEFAULT_USER and userobj.active: - log.info( - 'user `%s` authenticated correctly as anonymous user', userobj.username) + log.info('user `%s` authenticated correctly as anonymous user', + userobj.username) return user_attrs elif userobj.username == username and password_match: log.info('user `%s` authenticated correctly', userobj.username) return user_attrs - log.warn("user `%s` used a wrong password when " - "authenticating on this plugin", userobj.username) + log.warning("user `%s` used a wrong password when " + "authenticating on this plugin", userobj.username) return None else: - log.warning( - 'user `%s` failed to authenticate via %s, reason: account not ' - 'active.', username, self.name) + log.warning('user `%s` failed to authenticate via %s, reason: account not ' + 'active.', username, self.name) return None +class RhodeCodeSettingsSchema(AuthnPluginSettingsSchemaBase): + + auth_restriction_choices = [ + (RhodeCodeAuthPlugin.AUTH_RESTRICTION_NONE, 'All users'), + (RhodeCodeAuthPlugin.AUTH_RESTRICTION_SUPER_ADMIN, 'Super admins only'), + ] + + auth_scope_choices = [ + (RhodeCodeAuthPlugin.AUTH_RESTRICTION_SCOPE_ALL, 'HTTP and VCS'), + (RhodeCodeAuthPlugin.AUTH_RESTRICTION_SCOPE_HTTP, 'HTTP only'), + ] + + auth_restriction = colander.SchemaNode( + colander.String(), + default=auth_restriction_choices[0], + description=_('Allowed user types for authentication using this plugin.'), + title=_('User restriction'), + validator=colander.OneOf([x[0] for x in auth_restriction_choices]), + widget='select_with_labels', + choices=auth_restriction_choices + ) + scope_restriction = colander.SchemaNode( + colander.String(), + default=auth_scope_choices[0], + description=_('Allowed protocols for authentication using this plugin. ' + 'VCS means GIT/HG/SVN. HTTP is web based login.'), + title=_('Scope restriction'), + validator=colander.OneOf([x[0] for x in auth_scope_choices]), + widget='select_with_labels', + choices=auth_scope_choices + ) + + def includeme(config): plugin_id = 'egg:rhodecode-enterprise-ce#{}'.format(RhodeCodeAuthPlugin.uid) plugin_factory(plugin_id).includeme(config) diff --git a/rhodecode/authentication/plugins/auth_token.py b/rhodecode/authentication/plugins/auth_token.py --- a/rhodecode/authentication/plugins/auth_token.py +++ b/rhodecode/authentication/plugins/auth_token.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -23,7 +23,9 @@ RhodeCode authentication token plugin fo """ import logging +import colander +from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase from rhodecode.translation import _ from rhodecode.authentication.base import ( RhodeCodeAuthPluginBase, VCS_TYPE, hybrid_property) @@ -48,6 +50,7 @@ class RhodeCodeAuthPlugin(RhodeCodeAuthP Enables usage of authentication tokens for vcs operations. """ uid = 'token' + AUTH_RESTRICTION_SCOPE_VCS = 'scope_vcs' def includeme(self, config): config.add_authn_plugin(self) @@ -67,6 +70,9 @@ class RhodeCodeAuthPlugin(RhodeCodeAuthP route_name='auth_home', context=RhodecodeAuthnResource) + def get_settings_schema(self): + return RhodeCodeSettingsSchema() + def get_display_name(self): return _('Rhodecode Token') @@ -142,16 +148,30 @@ class RhodeCodeAuthPlugin(RhodeCodeAuthP 'user `%s` successfully authenticated via %s', user_attrs['username'], self.name) return user_attrs - log.warn( - 'user `%s` failed to authenticate via %s, reason: bad or ' - 'inactive token.', username, self.name) + log.warning('user `%s` failed to authenticate via %s, reason: bad or ' + 'inactive token.', username, self.name) else: - log.warning( - 'user `%s` failed to authenticate via %s, reason: account not ' - 'active.', username, self.name) + log.warning('user `%s` failed to authenticate via %s, reason: account not ' + 'active.', username, self.name) return None def includeme(config): plugin_id = 'egg:rhodecode-enterprise-ce#{}'.format(RhodeCodeAuthPlugin.uid) plugin_factory(plugin_id).includeme(config) + + +class RhodeCodeSettingsSchema(AuthnPluginSettingsSchemaBase): + auth_scope_choices = [ + (RhodeCodeAuthPlugin.AUTH_RESTRICTION_SCOPE_VCS, 'VCS only'), + ] + + scope_restriction = colander.SchemaNode( + colander.String(), + default=auth_scope_choices[0], + description=_('Choose operation scope restriction when authenticating.'), + title=_('Scope restriction'), + validator=colander.OneOf([x[0] for x in auth_scope_choices]), + widget='select_with_labels', + choices=auth_scope_choices + ) diff --git a/rhodecode/authentication/registry.py b/rhodecode/authentication/registry.py --- a/rhodecode/authentication/registry.py +++ b/rhodecode/authentication/registry.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/authentication/routes.py b/rhodecode/authentication/routes.py --- a/rhodecode/authentication/routes.py +++ b/rhodecode/authentication/routes.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/authentication/schema.py b/rhodecode/authentication/schema.py --- a/rhodecode/authentication/schema.py +++ b/rhodecode/authentication/schema.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/authentication/tests/conftest.py b/rhodecode/authentication/tests/conftest.py --- a/rhodecode/authentication/tests/conftest.py +++ b/rhodecode/authentication/tests/conftest.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/authentication/tests/functional/test_settings.py b/rhodecode/authentication/tests/functional/test_settings.py --- a/rhodecode/authentication/tests/functional/test_settings.py +++ b/rhodecode/authentication/tests/functional/test_settings.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/authentication/views.py b/rhodecode/authentication/views.py --- a/rhodecode/authentication/views.py +++ b/rhodecode/authentication/views.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/config/__init__.py b/rhodecode/config/__init__.py --- a/rhodecode/config/__init__.py +++ b/rhodecode/config/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/config/conf.py b/rhodecode/config/conf.py --- a/rhodecode/config/conf.py +++ b/rhodecode/config/conf.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2013-2018 RhodeCode GmbH +# Copyright (C) 2013-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/config/environment.py b/rhodecode/config/environment.py --- a/rhodecode/config/environment.py +++ b/rhodecode/config/environment.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/config/jsroutes.py b/rhodecode/config/jsroutes.py --- a/rhodecode/config/jsroutes.py +++ b/rhodecode/config/jsroutes.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py --- a/rhodecode/config/middleware.py +++ b/rhodecode/config/middleware.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -98,7 +98,7 @@ def make_pyramid_app(global_config, **se global_config = _substitute_values(global_config, environ) settings = _substitute_values(settings, environ) - sanitize_settings_and_apply_defaults(settings) + sanitize_settings_and_apply_defaults(global_config, settings) config = Configurator(settings=settings) @@ -165,7 +165,7 @@ def error_handler(exception, request): error_explanation = base_response.explanation or str(base_response) if base_response.status_code == 404: - error_explanation += " Or you don't have permission to access it." + error_explanation += " Optionally you don't have permission to access this page." c = AttributeDict() c.error_message = base_response.status c.error_explanation = error_explanation @@ -281,6 +281,7 @@ def includeme(config): config.include('rhodecode.apps.ops') config.include('rhodecode.apps.admin') config.include('rhodecode.apps.channelstream') + config.include('rhodecode.apps.file_store') config.include('rhodecode.apps.login') config.include('rhodecode.apps.home') config.include('rhodecode.apps.journal') @@ -381,7 +382,7 @@ def wrap_app_in_wsgi_middlewares(pyramid return pyramid_app_with_cleanup -def sanitize_settings_and_apply_defaults(settings): +def sanitize_settings_and_apply_defaults(global_config, settings): """ Applies settings defaults and does all type conversion. @@ -420,6 +421,7 @@ def sanitize_settings_and_apply_defaults # TODO: johbo: Re-think this, usually the call to config.include # should allow to pass in a prefix. settings.setdefault('rhodecode.api.url', '/_admin/api') + settings.setdefault('__file__', global_config.get('__file__')) # Sanitize generic settings. _list_setting(settings, 'default_encoding', 'UTF-8') @@ -708,18 +710,29 @@ def _string_setting(settings, name, defa def _substitute_values(mapping, substitutions): + result = {} try: - result = { + for key, value in mapping.items(): + # initialize without substitution first + result[key] = value + # Note: Cannot use regular replacements, since they would clash # with the implementation of ConfigParser. Using "format" instead. - key: value.format(**substitutions) - for key, value in mapping.items() - } - except KeyError as e: - raise ValueError( - 'Failed to substitute env variable: {}. ' - 'Make sure you have specified this env variable without ENV_ prefix'.format(e)) + try: + result[key] = value.format(**substitutions) + except KeyError as e: + env_var = '{}'.format(e.args[0]) + + msg = 'Failed to substitute: `{key}={{{var}}}` with environment entry. ' \ + 'Make sure your environment has {var} set, or remove this ' \ + 'variable from config file'.format(key=key, var=env_var) + + if env_var.startswith('ENV_'): + raise ValueError(msg) + else: + log.warning(msg) + except ValueError as e: log.warning('Failed to substitute ENV variable: %s', e) result = mapping diff --git a/rhodecode/config/patches.py b/rhodecode/config/patches.py --- a/rhodecode/config/patches.py +++ b/rhodecode/config/patches.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/config/rcextensions/__init__.py b/rhodecode/config/rcextensions/__init__.py --- a/rhodecode/config/rcextensions/__init__.py +++ b/rhodecode/config/rcextensions/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/config/rcextensions/examples/validate_commit_message_author.py b/rhodecode/config/rcextensions/examples/validate_commit_message_author.py --- a/rhodecode/config/rcextensions/examples/validate_commit_message_author.py +++ b/rhodecode/config/rcextensions/examples/validate_commit_message_author.py @@ -14,19 +14,6 @@ 'hook_type': '', 'user_agent': 'Client user agent, e.g git or mercurial CLI version', }) -@has_kwargs({ - 'server_url': 'url of instance that triggered this hook', - 'config': 'path to .ini config used', - 'scm': 'type of version control "git", "hg", "svn"', - 'username': 'username of actor who triggered this event', - 'ip': 'ip address of actor who triggered this hook', - 'action': '', - 'repository': 'repository name', - 'repo_store_path': 'full path to where repositories are stored', - 'commit_ids': 'pre transaction metadata for commit ids', - 'hook_type': '', - 'user_agent': 'Client user agent, e.g git or mercurial CLI version', -}) def _pre_push_hook(*args, **kwargs): """ Post push hook diff --git a/rhodecode/config/rcextensions/examples/validate_pushed_files_name_and_size.py b/rhodecode/config/rcextensions/examples/validate_pushed_files_name_and_size.py new file mode 100644 --- /dev/null +++ b/rhodecode/config/rcextensions/examples/validate_pushed_files_name_and_size.py @@ -0,0 +1,110 @@ +# Example to validate pushed files names and size using some sort of rules + + + +@has_kwargs({ + 'server_url': 'url of instance that triggered this hook', + 'config': 'path to .ini config used', + 'scm': 'type of version control "git", "hg", "svn"', + 'username': 'username of actor who triggered this event', + 'ip': 'ip address of actor who triggered this hook', + 'action': '', + 'repository': 'repository name', + 'repo_store_path': 'full path to where repositories are stored', + 'commit_ids': 'pre transaction metadata for commit ids', + 'hook_type': '', + 'user_agent': 'Client user agent, e.g git or mercurial CLI version', +}) +def _pre_push_hook(*args, **kwargs): + """ + Post push hook + To stop version control from storing the transaction and send a message to user + use non-zero HookResponse with a message, e.g return HookResponse(1, 'Not allowed') + + This message will be shown back to client during PUSH operation + + Commit ids might look like that:: + + [{u'hg_env|git_env': ..., + u'multiple_heads': [], + u'name': u'default', + u'new_rev': u'd0befe0692e722e01d5677f27a104631cf798b69', + u'old_rev': u'd0befe0692e722e01d5677f27a104631cf798b69', + u'ref': u'', + u'total_commits': 2, + u'type': u'branch'}] + """ + import fnmatch + from .helpers import extra_fields, extract_pre_files + from .utils import str2bool, aslist + from rhodecode.lib.helpers import format_byte_size_binary + + # returns list of dicts with key-val fetched from extra fields + repo_extra_fields = extra_fields.run(**kwargs) + + # optionally use 'extra fields' to control the logic per repo + # e.g store a list of patterns to be forbidden e.g `*.exe, *.dump` + forbid_files = repo_extra_fields.get('forbid_files_glob', {}).get('field_value') + forbid_files = aslist(forbid_files) + + # optionally get bytes limit for a single file, e.g 1024 for 1KB + forbid_size_over = repo_extra_fields.get('forbid_size_over', {}).get('field_value') + forbid_size_over = int(forbid_size_over or 0) + + def validate_file_name_and_size(file_data, forbidden_files=None, size_limit=None): + """ + This function validates commited files against some sort of rules. + It should return a valid boolean, and a reason for failure + + file_data =[ + 'raw_diff', 'old_revision', 'stats', 'original_filename', 'is_limited_diff', + 'chunks', 'new_revision', 'operation', 'exceeds_limit', 'filename' + ] + file_data['ops'] = { + # is file binary + 'binary': False, + + # lines + 'added': 32, + 'deleted': 0 + + 'ops': {3: 'modified file'}, + 'new_mode': '100644', + 'old_mode': None + } + """ + file_name = file_data['filename'] + operation = file_data['operation'] # can be A(dded), M(odified), D(eleted) + + # check files names + if forbidden_files: + reason = 'File {} is forbidden to be pushed'.format(file_name) + for forbidden_pattern in forbid_files: + # here we can also filter for operation, e.g if check for only ADDED files + # if operation == 'A': + if fnmatch.fnmatch(file_name, forbidden_pattern): + return False, reason + + # validate A(dded) files and size + if size_limit and operation == 'A': + size = len(file_data['raw_diff']) + + reason = 'File {} size of {} bytes exceeds limit {}'.format( + file_name, format_byte_size_binary(size), + format_byte_size_binary(size_limit)) + if size > size_limit: + return False, reason + + return True, '' + + if forbid_files or forbid_size_over: + # returns list of dicts with key-val fetched from extra fields + file_list = extract_pre_files.run(**kwargs) + + for file_data in file_list: + file_valid, reason = validate_file_name_and_size( + file_data, forbid_files, forbid_size_over) + if not file_valid: + return HookResponse(1, reason) + + return HookResponse(0, '') diff --git a/rhodecode/config/rcextensions/helpers/__init__.py b/rhodecode/config/rcextensions/helpers/__init__.py --- a/rhodecode/config/rcextensions/helpers/__init__.py +++ b/rhodecode/config/rcextensions/helpers/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/config/rcextensions/helpers/extra_fields.py b/rhodecode/config/rcextensions/helpers/extra_fields.py --- a/rhodecode/config/rcextensions/helpers/extra_fields.py +++ b/rhodecode/config/rcextensions/helpers/extra_fields.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/config/rcextensions/helpers/extract_post_commits.py b/rhodecode/config/rcextensions/helpers/extract_post_commits.py --- a/rhodecode/config/rcextensions/helpers/extract_post_commits.py +++ b/rhodecode/config/rcextensions/helpers/extract_post_commits.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/config/rcextensions/helpers/extract_pre_commits.py b/rhodecode/config/rcextensions/helpers/extract_pre_commits.py --- a/rhodecode/config/rcextensions/helpers/extract_pre_commits.py +++ b/rhodecode/config/rcextensions/helpers/extract_pre_commits.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/config/rcextensions/helpers/extract_pre_files.py b/rhodecode/config/rcextensions/helpers/extract_pre_files.py new file mode 100644 --- /dev/null +++ b/rhodecode/config/rcextensions/helpers/extract_pre_files.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2016-2019 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +""" +us in hooks:: + + from .helpers import extract_pre_files + # returns list of dicts with key-val fetched from extra fields + file_list = extract_pre_files.run(**kwargs) + +""" +import re +import collections +import json + +from rhodecode.lib import diffs +from rhodecode.lib.vcs.backends.hg.diff import MercurialDiff +from rhodecode.lib.vcs.backends.git.diff import GitDiff + + +def get_hg_files(repo, refs): + files = [] + return files + + +def get_git_files(repo, refs): + files = [] + + for data in refs: + # we should now extract commit data + old_rev = data['old_rev'] + new_rev = data['new_rev'] + + if '00000000' in old_rev: + # new branch, we don't need to extract nothing + return files + + git_env = dict(data['git_env']) + + cmd = [ + 'diff', old_rev, new_rev + ] + + stdout, stderr = repo.run_git_command(cmd, extra_env=git_env) + vcs_diff = GitDiff(stdout) + + diff_processor = diffs.DiffProcessor(vcs_diff, format='newdiff') + # this is list of dicts with diff information + # _parsed[0].keys() + # ['raw_diff', 'old_revision', 'stats', 'original_filename', + # 'is_limited_diff', 'chunks', 'new_revision', 'operation', + # 'exceeds_limit', 'filename'] + files = _parsed = diff_processor.prepare() + + return files + + +def run(*args, **kwargs): + from rhodecode.model.db import Repository + + vcs_type = kwargs['scm'] + # use temp name then the main one propagated + repo_name = kwargs.pop('REPOSITORY', None) or kwargs['repository'] + + repo = Repository.get_by_repo_name(repo_name) + vcs_repo = repo.scm_instance(cache=False) + + files = [] + + if vcs_type == 'git': + for rev_data in kwargs['commit_ids']: + new_environ = dict((k, v) for k, v in rev_data['git_env']) + files = get_git_files(vcs_repo, kwargs['commit_ids']) + + if vcs_type == 'hg': + for rev_data in kwargs['commit_ids']: + new_environ = dict((k, v) for k, v in rev_data['hg_env']) + files = get_hg_files(vcs_repo, kwargs['commit_ids']) + + return files diff --git a/rhodecode/config/rcextensions/helpers/http_call.py b/rhodecode/config/rcextensions/helpers/http_call.py --- a/rhodecode/config/rcextensions/helpers/http_call.py +++ b/rhodecode/config/rcextensions/helpers/http_call.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/config/rcextensions/hooks.py b/rhodecode/config/rcextensions/hooks.py --- a/rhodecode/config/rcextensions/hooks.py +++ b/rhodecode/config/rcextensions/hooks.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/config/rcextensions/utils.py b/rhodecode/config/rcextensions/utils.py --- a/rhodecode/config/rcextensions/utils.py +++ b/rhodecode/config/rcextensions/utils.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/config/routing_links.py b/rhodecode/config/routing_links.py --- a/rhodecode/config/routing_links.py +++ b/rhodecode/config/routing_links.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/config/utils.py b/rhodecode/config/utils.py --- a/rhodecode/config/utils.py +++ b/rhodecode/config/utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/controllers/__init__.py b/rhodecode/controllers/__init__.py deleted file mode 100644 --- a/rhodecode/controllers/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright (C) 2010-2018 RhodeCode GmbH -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License, version 3 -# (only), as published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -# This program is dual-licensed. If you wish to learn more about the -# RhodeCode Enterprise Edition, including its added features, Support services, -# and proprietary license terms, please see https://rhodecode.com/licenses/ diff --git a/rhodecode/events/__init__.py b/rhodecode/events/__init__.py --- a/rhodecode/events/__init__.py +++ b/rhodecode/events/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/events/base.py b/rhodecode/events/base.py --- a/rhodecode/events/base.py +++ b/rhodecode/events/base.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/events/interfaces.py b/rhodecode/events/interfaces.py --- a/rhodecode/events/interfaces.py +++ b/rhodecode/events/interfaces.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/events/pullrequest.py b/rhodecode/events/pullrequest.py --- a/rhodecode/events/pullrequest.py +++ b/rhodecode/events/pullrequest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/events/repo.py b/rhodecode/events/repo.py --- a/rhodecode/events/repo.py +++ b/rhodecode/events/repo.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/events/repo_group.py b/rhodecode/events/repo_group.py --- a/rhodecode/events/repo_group.py +++ b/rhodecode/events/repo_group.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/events/user.py b/rhodecode/events/user.py --- a/rhodecode/events/user.py +++ b/rhodecode/events/user.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/forms/__init__.py b/rhodecode/forms/__init__.py --- a/rhodecode/forms/__init__.py +++ b/rhodecode/forms/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/i18n/de/LC_MESSAGES/rhodecode.po b/rhodecode/i18n/de/LC_MESSAGES/rhodecode.po --- a/rhodecode/i18n/de/LC_MESSAGES/rhodecode.po +++ b/rhodecode/i18n/de/LC_MESSAGES/rhodecode.po @@ -7457,7 +7457,7 @@ msgid "" "Schema of clone url construction eg. '{scheme}://{user}@{netloc}/{repo}', available vars:\n" " {scheme} 'http' or 'https' sent from running RhodeCode server,\n" " {user} current user username,\n" -" {sys_user} current system user running this process, usefull for ssh,\n" +" {sys_user} current system user running this process, Useful for ssh,\n" " {hostname} hostname of this server running RhodeCode,\n" " {netloc} network location/server host of running RhodeCode server,\n" " {repo} full repository name,\n" diff --git a/rhodecode/i18n/es/LC_MESSAGES/rhodecode.po b/rhodecode/i18n/es/LC_MESSAGES/rhodecode.po --- a/rhodecode/i18n/es/LC_MESSAGES/rhodecode.po +++ b/rhodecode/i18n/es/LC_MESSAGES/rhodecode.po @@ -7455,7 +7455,7 @@ msgid "" "Schema of clone url construction eg. '{scheme}://{user}@{netloc}/{repo}', available vars:\n" " {scheme} 'http' or 'https' sent from running RhodeCode server,\n" " {user} current user username,\n" -" {sys_user} current system user running this process, usefull for ssh,\n" +" {sys_user} current system user running this process, Useful for ssh,\n" " {hostname} hostname of this server running RhodeCode,\n" " {netloc} network location/server host of running RhodeCode server,\n" " {repo} full repository name,\n" diff --git a/rhodecode/i18n/fr/LC_MESSAGES/rhodecode.po b/rhodecode/i18n/fr/LC_MESSAGES/rhodecode.po --- a/rhodecode/i18n/fr/LC_MESSAGES/rhodecode.po +++ b/rhodecode/i18n/fr/LC_MESSAGES/rhodecode.po @@ -7456,7 +7456,7 @@ msgid "" "Schema of clone url construction eg. '{scheme}://{user}@{netloc}/{repo}', available vars:\n" " {scheme} 'http' or 'https' sent from running RhodeCode server,\n" " {user} current user username,\n" -" {sys_user} current system user running this process, usefull for ssh,\n" +" {sys_user} current system user running this process, Useful for ssh,\n" " {hostname} hostname of this server running RhodeCode,\n" " {netloc} network location/server host of running RhodeCode server,\n" " {repo} full repository name,\n" diff --git a/rhodecode/i18n/it/LC_MESSAGES/rhodecode.po b/rhodecode/i18n/it/LC_MESSAGES/rhodecode.po --- a/rhodecode/i18n/it/LC_MESSAGES/rhodecode.po +++ b/rhodecode/i18n/it/LC_MESSAGES/rhodecode.po @@ -7456,7 +7456,7 @@ msgid "" "Schema of clone url construction eg. '{scheme}://{user}@{netloc}/{repo}', available vars:\n" " {scheme} 'http' or 'https' sent from running RhodeCode server,\n" " {user} current user username,\n" -" {sys_user} current system user running this process, usefull for ssh,\n" +" {sys_user} current system user running this process, Useful for ssh,\n" " {hostname} hostname of this server running RhodeCode,\n" " {netloc} network location/server host of running RhodeCode server,\n" " {repo} full repository name,\n" diff --git a/rhodecode/i18n/ja/LC_MESSAGES/rhodecode.po b/rhodecode/i18n/ja/LC_MESSAGES/rhodecode.po --- a/rhodecode/i18n/ja/LC_MESSAGES/rhodecode.po +++ b/rhodecode/i18n/ja/LC_MESSAGES/rhodecode.po @@ -7463,7 +7463,7 @@ msgid "" "Schema of clone url construction eg. '{scheme}://{user}@{netloc}/{repo}', available vars:\n" " {scheme} 'http' or 'https' sent from running RhodeCode server,\n" " {user} current user username,\n" -" {sys_user} current system user running this process, usefull for ssh,\n" +" {sys_user} current system user running this process, Useful for ssh,\n" " {hostname} hostname of this server running RhodeCode,\n" " {netloc} network location/server host of running RhodeCode server,\n" " {repo} full repository name,\n" diff --git a/rhodecode/i18n/pl/LC_MESSAGES/rhodecode.po b/rhodecode/i18n/pl/LC_MESSAGES/rhodecode.po --- a/rhodecode/i18n/pl/LC_MESSAGES/rhodecode.po +++ b/rhodecode/i18n/pl/LC_MESSAGES/rhodecode.po @@ -7461,7 +7461,7 @@ msgid "" "Schema of clone url construction eg. '{scheme}://{user}@{netloc}/{repo}', available vars:\n" " {scheme} 'http' or 'https' sent from running RhodeCode server,\n" " {user} current user username,\n" -" {sys_user} current system user running this process, usefull for ssh,\n" +" {sys_user} current system user running this process, Useful for ssh,\n" " {hostname} hostname of this server running RhodeCode,\n" " {netloc} network location/server host of running RhodeCode server,\n" " {repo} full repository name,\n" diff --git a/rhodecode/i18n/pt/LC_MESSAGES/rhodecode.po b/rhodecode/i18n/pt/LC_MESSAGES/rhodecode.po --- a/rhodecode/i18n/pt/LC_MESSAGES/rhodecode.po +++ b/rhodecode/i18n/pt/LC_MESSAGES/rhodecode.po @@ -7457,7 +7457,7 @@ msgid "" "Schema of clone url construction eg. '{scheme}://{user}@{netloc}/{repo}', available vars:\n" " {scheme} 'http' or 'https' sent from running RhodeCode server,\n" " {user} current user username,\n" -" {sys_user} current system user running this process, usefull for ssh,\n" +" {sys_user} current system user running this process, Useful for ssh,\n" " {hostname} hostname of this server running RhodeCode,\n" " {netloc} network location/server host of running RhodeCode server,\n" " {repo} full repository name,\n" diff --git a/rhodecode/i18n/rhodecode.pot b/rhodecode/i18n/rhodecode.pot --- a/rhodecode/i18n/rhodecode.pot +++ b/rhodecode/i18n/rhodecode.pot @@ -7497,7 +7497,7 @@ msgid "" "Schema of clone url construction eg. '{scheme}://{user}@{netloc}/{repo}', available vars:\n" " {scheme} 'http' or 'https' sent from running RhodeCode server,\n" " {user} current user username,\n" -" {sys_user} current system user running this process, usefull for ssh,\n" +" {sys_user} current system user running this process, Useful for ssh,\n" " {hostname} hostname of this server running RhodeCode,\n" " {netloc} network location/server host of running RhodeCode server,\n" " {repo} full repository name,\n" diff --git a/rhodecode/i18n/ru/LC_MESSAGES/rhodecode.po b/rhodecode/i18n/ru/LC_MESSAGES/rhodecode.po --- a/rhodecode/i18n/ru/LC_MESSAGES/rhodecode.po +++ b/rhodecode/i18n/ru/LC_MESSAGES/rhodecode.po @@ -7474,7 +7474,7 @@ msgid "" "Schema of clone url construction eg. '{scheme}://{user}@{netloc}/{repo}', available vars:\n" " {scheme} 'http' or 'https' sent from running RhodeCode server,\n" " {user} current user username,\n" -" {sys_user} current system user running this process, usefull for ssh,\n" +" {sys_user} current system user running this process, Useful for ssh,\n" " {hostname} hostname of this server running RhodeCode,\n" " {netloc} network location/server host of running RhodeCode server,\n" " {repo} full repository name,\n" diff --git a/rhodecode/i18n/zh/LC_MESSAGES/rhodecode.po b/rhodecode/i18n/zh/LC_MESSAGES/rhodecode.po --- a/rhodecode/i18n/zh/LC_MESSAGES/rhodecode.po +++ b/rhodecode/i18n/zh/LC_MESSAGES/rhodecode.po @@ -7458,7 +7458,7 @@ msgid "" "Schema of clone url construction eg. '{scheme}://{user}@{netloc}/{repo}', available vars:\n" " {scheme} 'http' or 'https' sent from running RhodeCode server,\n" " {user} current user username,\n" -" {sys_user} current system user running this process, usefull for ssh,\n" +" {sys_user} current system user running this process, Useful for ssh,\n" " {hostname} hostname of this server running RhodeCode,\n" " {netloc} network location/server host of running RhodeCode server,\n" " {repo} full repository name,\n" diff --git a/rhodecode/integrations/__init__.py b/rhodecode/integrations/__init__.py --- a/rhodecode/integrations/__init__.py +++ b/rhodecode/integrations/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/integrations/registry.py b/rhodecode/integrations/registry.py --- a/rhodecode/integrations/registry.py +++ b/rhodecode/integrations/registry.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/integrations/routes.py b/rhodecode/integrations/routes.py --- a/rhodecode/integrations/routes.py +++ b/rhodecode/integrations/routes.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/integrations/schema.py b/rhodecode/integrations/schema.py --- a/rhodecode/integrations/schema.py +++ b/rhodecode/integrations/schema.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/integrations/tests/__init__.py b/rhodecode/integrations/tests/__init__.py --- a/rhodecode/integrations/tests/__init__.py +++ b/rhodecode/integrations/tests/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/integrations/tests/test_integrations.py b/rhodecode/integrations/tests/test_integrations.py --- a/rhodecode/integrations/tests/test_integrations.py +++ b/rhodecode/integrations/tests/test_integrations.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/integrations/types/__init__.py b/rhodecode/integrations/types/__init__.py --- a/rhodecode/integrations/types/__init__.py +++ b/rhodecode/integrations/types/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/integrations/types/base.py b/rhodecode/integrations/types/base.py --- a/rhodecode/integrations/types/base.py +++ b/rhodecode/integrations/types/base.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -23,17 +23,27 @@ import string import collections import logging import requests +import urllib from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.retry import Retry from mako import exceptions +from rhodecode.lib.utils2 import safe_str from rhodecode.translation import _ log = logging.getLogger(__name__) +class UrlTmpl(string.Template): + + def safe_substitute(self, **kws): + # url encode the kw for usage in url + kws = {k: urllib.quote(safe_str(v)) for k, v in kws.items()} + return super(UrlTmpl, self).safe_substitute(**kws) + + class IntegrationTypeBase(object): """ Base class for IntegrationType plugins """ is_dummy = False @@ -217,7 +227,9 @@ class WebhookDataHandler(CommitParsingDa common_vars.update(extra_vars) template_url = self.template_url.replace('${extra:', '${extra__') - return string.Template(template_url).safe_substitute(**common_vars) + for k, v in common_vars.items(): + template_url = UrlTmpl(template_url).safe_substitute(**{k: v}) + return template_url def repo_push_event_handler(self, event, data): url = self.get_base_parsed_template(data) @@ -228,20 +240,18 @@ class WebhookDataHandler(CommitParsingDa if '${branch}' in url or '${branch_head}' in url or '${commit_id}' in url: # call it multiple times, for each branch if used in variables for branch, commit_ids in branches_commits.items(): - branch_url = string.Template(url).safe_substitute(branch=branch) + branch_url = UrlTmpl(url).safe_substitute(branch=branch) if '${branch_head}' in branch_url: # last commit in the aggregate is the head of the branch branch_head = commit_ids['branch_head'] - branch_url = string.Template(branch_url).safe_substitute( - branch_head=branch_head) + branch_url = UrlTmpl(branch_url).safe_substitute(branch_head=branch_head) # call further down for each commit if used if '${commit_id}' in branch_url: for commit_data in commit_ids['commits']: commit_id = commit_data['raw_id'] - commit_url = string.Template(branch_url).safe_substitute( - commit_id=commit_id) + commit_url = UrlTmpl(branch_url).safe_substitute(commit_id=commit_id) # register per-commit call log.debug( 'register %s call(%s) to url %s', @@ -251,36 +261,34 @@ class WebhookDataHandler(CommitParsingDa else: # register per-branch call - log.debug( - 'register %s call(%s) to url %s', - self.name, event, branch_url) - url_calls.append( - (branch_url, self.headers, data)) + log.debug('register %s call(%s) to url %s', + self.name, event, branch_url) + url_calls.append((branch_url, self.headers, data)) else: - log.debug( - 'register %s call(%s) to url %s', self.name, event, url) + log.debug('register %s call(%s) to url %s', self.name, event, url) url_calls.append((url, self.headers, data)) return url_calls def repo_create_event_handler(self, event, data): url = self.get_base_parsed_template(data) - log.debug( - 'register %s call(%s) to url %s', self.name, event, url) + log.debug('register %s call(%s) to url %s', self.name, event, url) return [(url, self.headers, data)] def pull_request_event_handler(self, event, data): url = self.get_base_parsed_template(data) - log.debug( - 'register %s call(%s) to url %s', self.name, event, url) - url = string.Template(url).safe_substitute( - pull_request_id=data['pullrequest']['pull_request_id'], - pull_request_title=data['pullrequest']['title'], - pull_request_url=data['pullrequest']['url'], - pull_request_shadow_url=data['pullrequest']['shadow_url'], - pull_request_commits_uid=data['pullrequest']['commits_uid'], - ) + log.debug('register %s call(%s) to url %s', self.name, event, url) + pr_vars = [ + ('pull_request_id', data['pullrequest']['pull_request_id']), + ('pull_request_title', data['pullrequest']['title']), + ('pull_request_url', data['pullrequest']['url']), + ('pull_request_shadow_url', data['pullrequest']['shadow_url']), + ('pull_request_commits_uid', data['pullrequest']['commits_uid']), + ] + for k, v in pr_vars: + url = UrlTmpl(url).safe_substitute(**{k: v}) + return [(url, self.headers, data)] def __call__(self, event, data): diff --git a/rhodecode/integrations/types/email.py b/rhodecode/integrations/types/email.py --- a/rhodecode/integrations/types/email.py +++ b/rhodecode/integrations/types/email.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/integrations/types/hipchat.py b/rhodecode/integrations/types/hipchat.py --- a/rhodecode/integrations/types/hipchat.py +++ b/rhodecode/integrations/types/hipchat.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -122,8 +122,10 @@ class HipchatIntegrationType(Integration log.debug('event not valid: %r', event) return - if event.name not in self.settings['events']: - log.debug('event ignored: %r', event) + allowed_events = self.settings['events'] + if event.name not in allowed_events: + log.debug('event ignored: %r event %s not in allowed events %s', + event, event.name, allowed_events) return data = event.as_dict() diff --git a/rhodecode/integrations/types/slack.py b/rhodecode/integrations/types/slack.py --- a/rhodecode/integrations/types/slack.py +++ b/rhodecode/integrations/types/slack.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -138,8 +138,10 @@ class SlackIntegrationType(IntegrationTy log.debug('event not valid: %r', event) return - if event.name not in self.settings['events']: - log.debug('event ignored: %r', event) + allowed_events = self.settings['events'] + if event.name not in allowed_events: + log.debug('event ignored: %r event %s not in allowed events %s', + event, event.name, allowed_events) return data = event.as_dict() diff --git a/rhodecode/integrations/types/webhook.py b/rhodecode/integrations/types/webhook.py --- a/rhodecode/integrations/types/webhook.py +++ b/rhodecode/integrations/types/webhook.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -118,12 +118,11 @@ class WebhookSettingsSchema(colander.Sch method_type = colander.SchemaNode( colander.String(), title=_('Call Method'), - description=_('Select if the Webhook call should be made ' - 'with POST or GET.'), + description=_('Select a HTTP method to use when calling the Webhook.'), default='post', missing='', widget=deform.widget.RadioChoiceWidget( - values=[('get', 'GET'), ('post', 'POST')], + values=[('get', 'GET'), ('post', 'POST'), ('put', 'PUT')], inline=True ), ) @@ -171,8 +170,10 @@ class WebhookIntegrationType(Integration log.debug('event not valid: %r', event) return - if event.name not in self.settings['events']: - log.debug('event ignored: %r', event) + allowed_events = self.settings['events'] + if event.name not in allowed_events: + log.debug('event ignored: %r event %s not in allowed events %s', + event, event.name, allowed_events) return data = event.as_dict() @@ -187,8 +188,7 @@ class WebhookIntegrationType(Integration handler = WebhookDataHandler(template_url, headers) url_calls = handler(event, data) - log.debug('webhook: calling following urls: %s', - [x[0] for x in url_calls]) + log.debug('Webhook: calling following urls: %s', [x[0] for x in url_calls]) run_task(post_to_webhook, url_calls, self.settings) @@ -202,37 +202,39 @@ def post_to_webhook(url_calls, settings) 'actor_ip': u'192.168.157.1', 'name': 'repo-push', 'push': {'branches': [{'name': u'default', - 'url': 'http://rc.local:8080/hg-repo/changelog?branch=default'}], - 'commits': [{'author': u'Marcin Kuzminski ', - 'branch': u'default', - 'date': datetime.datetime(2017, 11, 30, 12, 59, 48), - 'issues': [], - 'mentions': [], - 'message': u'commit Thu 30 Nov 2017 13:59:48 CET', - 'message_html': u'commit Thu 30 Nov 2017 13:59:48 CET', - 'message_html_title': u'commit Thu 30 Nov 2017 13:59:48 CET', - 'parents': [{'raw_id': '431b772a5353dad9974b810dd3707d79e3a7f6e0'}], - 'permalink_url': u'http://rc.local:8080/_7/changeset/a815cc738b9651eb5ffbcfb1ce6ccd7c701a5ddf', - 'raw_id': 'a815cc738b9651eb5ffbcfb1ce6ccd7c701a5ddf', - 'refs': {'bookmarks': [], 'branches': [u'default'], 'tags': [u'tip']}, - 'reviewers': [], - 'revision': 9L, - 'short_id': 'a815cc738b96', - 'url': u'http://rc.local:8080/hg-repo/changeset/a815cc738b9651eb5ffbcfb1ce6ccd7c701a5ddf'}], - 'issues': {}}, + 'url': 'http://rc.local:8080/hg-repo/changelog?branch=default'}], + 'commits': [{'author': u'Marcin Kuzminski ', + 'branch': u'default', + 'date': datetime.datetime(2017, 11, 30, 12, 59, 48), + 'issues': [], + 'mentions': [], + 'message': u'commit Thu 30 Nov 2017 13:59:48 CET', + 'message_html': u'commit Thu 30 Nov 2017 13:59:48 CET', + 'message_html_title': u'commit Thu 30 Nov 2017 13:59:48 CET', + 'parents': [{'raw_id': '431b772a5353dad9974b810dd3707d79e3a7f6e0'}], + 'permalink_url': u'http://rc.local:8080/_7/changeset/a815cc738b9651eb5ffbcfb1ce6ccd7c701a5ddf', + 'raw_id': 'a815cc738b9651eb5ffbcfb1ce6ccd7c701a5ddf', + 'refs': {'bookmarks': [], + 'branches': [u'default'], + 'tags': [u'tip']}, + 'reviewers': [], + 'revision': 9L, + 'short_id': 'a815cc738b96', + 'url': u'http://rc.local:8080/hg-repo/changeset/a815cc738b9651eb5ffbcfb1ce6ccd7c701a5ddf'}], + 'issues': {}}, 'repo': {'extra_fields': '', - 'permalink_url': u'http://rc.local:8080/_7', - 'repo_id': 7, - 'repo_name': u'hg-repo', - 'repo_type': u'hg', - 'url': u'http://rc.local:8080/hg-repo'}, + 'permalink_url': u'http://rc.local:8080/_7', + 'repo_id': 7, + 'repo_name': u'hg-repo', + 'repo_type': u'hg', + 'url': u'http://rc.local:8080/hg-repo'}, 'server_url': u'http://rc.local:8080', 'utc_timestamp': datetime.datetime(2017, 11, 30, 13, 0, 1, 569276) + } + """ - """ call_headers = { - 'User-Agent': 'RhodeCode-webhook-caller/{}'.format( - rhodecode.__version__) + 'User-Agent': 'RhodeCode-webhook-caller/{}'.format(rhodecode.__version__) } # updated below with custom ones, allows override auth = get_auth(settings) @@ -247,8 +249,7 @@ def post_to_webhook(url_calls, settings) headers = headers or {} call_headers.update(headers) - log.debug('calling Webhook with method: %s, and auth:%s', - call_method, auth) + log.debug('calling Webhook with method: %s, and auth:%s', call_method, auth) if settings.get('log_data'): log.debug('calling webhook with data: %s', data) resp = call_method(url, json={ diff --git a/rhodecode/integrations/views.py b/rhodecode/integrations/views.py --- a/rhodecode/integrations/views.py +++ b/rhodecode/integrations/views.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/__init__.py b/rhodecode/lib/__init__.py --- a/rhodecode/lib/__init__.py +++ b/rhodecode/lib/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/action_parser.py b/rhodecode/lib/action_parser.py --- a/rhodecode/lib/action_parser.py +++ b/rhodecode/lib/action_parser.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/audit_logger.py b/rhodecode/lib/audit_logger.py --- a/rhodecode/lib/audit_logger.py +++ b/rhodecode/lib/audit_logger.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2017-2018 RhodeCode GmbH +# Copyright (C) 2017-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/auth.py b/rhodecode/lib/auth.py --- a/rhodecode/lib/auth.py +++ b/rhodecode/lib/auth.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/base.py b/rhodecode/lib/base.py --- a/rhodecode/lib/base.py +++ b/rhodecode/lib/base.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -35,6 +35,7 @@ from paste.httpexceptions import HTTPUna from paste.httpheaders import WWW_AUTHENTICATE, AUTHORIZATION import rhodecode +from rhodecode.apps._base import TemplateArgs from rhodecode.authentication.base import VCS_TYPE from rhodecode.lib import auth, utils2 from rhodecode.lib import helpers as h @@ -43,7 +44,7 @@ from rhodecode.lib.exceptions import Use from rhodecode.lib.utils import (password_changed, get_enabled_hook_classes) from rhodecode.lib.utils2 import ( str2bool, safe_unicode, AttributeDict, safe_int, sha1, aslist, safe_str) -from rhodecode.model.db import Repository, User, ChangesetComment +from rhodecode.model.db import Repository, User, ChangesetComment, UserBookmark from rhodecode.model.notification import NotificationModel from rhodecode.model.settings import VcsSettingsModel, SettingsModel @@ -281,7 +282,7 @@ def get_current_lang(request): return getattr(request, '_LOCALE_', request.locale_name) -def attach_context_attributes(context, request, user_id): +def attach_context_attributes(context, request, user_id=None): """ Attach variables into template context called `c`. """ @@ -312,6 +313,10 @@ def attach_context_attributes(context, r rc_config.get('rhodecode_dashboard_items', 100)) context.visual.admin_grid_items = safe_int( rc_config.get('rhodecode_admin_grid_items', 100)) + context.visual.show_revision_number = str2bool( + rc_config.get('rhodecode_show_revision_number', True)) + context.visual.show_sha_length = safe_int( + rc_config.get('rhodecode_show_sha_length', 100)) context.visual.repository_fields = str2bool( rc_config.get('rhodecode_repository_fields')) context.visual.show_version = str2bool( @@ -343,6 +348,8 @@ def attach_context_attributes(context, r config.get('labs_settings_active', 'false')) context.ssh_enabled = str2bool( config.get('ssh.generate_authorized_keyfile', 'false')) + context.ssh_key_generator_enabled = str2bool( + config.get('ssh.enable_ui_key_generator', 'true')) context.visual.allow_repo_location_change = str2bool( config.get('allow_repo_location_change', True)) @@ -417,7 +424,13 @@ def attach_context_attributes(context, r context.csrf_token = auth.get_csrf_token(session=request.session) context.backends = rhodecode.BACKENDS.keys() context.backends.sort() - context.unread_notifications = NotificationModel().get_unread_cnt_for_user(user_id) + unread_count = 0 + user_bookmark_list = [] + if user_id: + unread_count = NotificationModel().get_unread_cnt_for_user(user_id) + user_bookmark_list = UserBookmark.get_bookmarks_for_user(user_id) + context.unread_notifications = unread_count + context.bookmark_items = user_bookmark_list # web case if hasattr(request, 'user'): @@ -551,7 +564,11 @@ def bootstrap_request(**kwargs): from rhodecode.lib.partial_renderer import get_partial_renderer return get_partial_renderer(request=self, tmpl_name=tmpl_name) - _call_context = {} + _call_context = TemplateArgs() + _call_context.visual = TemplateArgs() + _call_context.visual.show_sha_length = 12 + _call_context.visual.show_revision_number = True + @property def call_context(self): return self._call_context diff --git a/rhodecode/lib/caching_query.py b/rhodecode/lib/caching_query.py --- a/rhodecode/lib/caching_query.py +++ b/rhodecode/lib/caching_query.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/celerylib/__init__.py b/rhodecode/lib/celerylib/__init__.py --- a/rhodecode/lib/celerylib/__init__.py +++ b/rhodecode/lib/celerylib/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/celerylib/loader.py b/rhodecode/lib/celerylib/loader.py --- a/rhodecode/lib/celerylib/loader.py +++ b/rhodecode/lib/celerylib/loader.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -40,7 +40,7 @@ from pyramid.threadlocal import get_curr import rhodecode from rhodecode.lib.auth import AuthUser -from rhodecode.lib.celerylib.utils import get_ini_config, parse_ini_vars +from rhodecode.lib.celerylib.utils import get_ini_config, parse_ini_vars, ping_db from rhodecode.lib.ext_json import json from rhodecode.lib.pyramid_utils import bootstrap, setup_logging, prepare_request from rhodecode.lib.utils2 import str2bool @@ -144,6 +144,11 @@ def on_preload_parsed(options, **kwargs) rhodecode.CELERY_ENABLED = True +@signals.task_prerun.connect +def task_prerun_signal(task_id, task, args, **kwargs): + ping_db() + + @signals.task_success.connect def task_success_signal(result, **kwargs): meta.Session.commit() @@ -170,7 +175,7 @@ def task_failure_signal( # simulate sys.exc_info() exc_info = (einfo.type, einfo.exception, einfo.tb) - store_exception(id(exc_info), exc_info, prefix='celery_rhodecode') + store_exception(id(exc_info), exc_info, prefix='rhodecode-celery') closer = celery_app.conf['PYRAMID_CLOSER'] if closer: @@ -227,8 +232,7 @@ def maybe_prepare_env(req): environ.update({ 'PATH_INFO': req.environ['PATH_INFO'], 'SCRIPT_NAME': req.environ['SCRIPT_NAME'], - 'HTTP_HOST': - req.environ.get('HTTP_HOST', req.environ['SERVER_NAME']), + 'HTTP_HOST':req.environ.get('HTTP_HOST', req.environ['SERVER_NAME']), 'SERVER_NAME': req.environ['SERVER_NAME'], 'SERVER_PORT': req.environ['SERVER_PORT'], 'wsgi.url_scheme': req.environ['wsgi.url_scheme'], diff --git a/rhodecode/lib/celerylib/scheduler.py b/rhodecode/lib/celerylib/scheduler.py --- a/rhodecode/lib/celerylib/scheduler.py +++ b/rhodecode/lib/celerylib/scheduler.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/celerylib/tasks.py b/rhodecode/lib/celerylib/tasks.py --- a/rhodecode/lib/celerylib/tasks.py +++ b/rhodecode/lib/celerylib/tasks.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -26,6 +26,7 @@ by celery daemon import os import time +from pyramid import compat from pyramid_mailer.mailer import Mailer from pyramid_mailer.message import Message @@ -62,7 +63,7 @@ def send_email(recipients, subject, body subject = "%s %s" % (email_config.get('email_prefix', ''), subject) if recipients: - if isinstance(recipients, basestring): + if isinstance(recipients, compat.string_types): recipients = recipients.split(',') else: # if recipients are not defined we send to email_config + all admins diff --git a/rhodecode/lib/celerylib/utils.py b/rhodecode/lib/celerylib/utils.py --- a/rhodecode/lib/celerylib/utils.py +++ b/rhodecode/lib/celerylib/utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -22,6 +22,7 @@ import os import json import logging import datetime +import time from functools import partial @@ -30,7 +31,6 @@ from celery.result import AsyncResult import celery.loaders.base import celery.schedules - log = logging.getLogger(__name__) @@ -167,3 +167,21 @@ def parse_ini_vars(ini_vars): key, value = pairs.split('=') options[key] = value return options + + +def ping_db(): + from rhodecode.model import meta + from rhodecode.model.db import DbMigrateVersion + log.info('Testing DB connection...') + + for test in range(10): + try: + scalar = DbMigrateVersion.query().scalar() + log.debug('DB PING %s@%s', scalar, scalar.version) + break + except Exception: + retry = 1 + log.debug('DB not ready, next try in %ss', retry) + time.sleep(retry) + finally: + meta.Session.remove() diff --git a/rhodecode/lib/channelstream.py b/rhodecode/lib/channelstream.py --- a/rhodecode/lib/channelstream.py +++ b/rhodecode/lib/channelstream.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/codeblocks.py b/rhodecode/lib/codeblocks.py --- a/rhodecode/lib/codeblocks.py +++ b/rhodecode/lib/codeblocks.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -26,6 +26,7 @@ from pygments import lex from pygments.formatters.html import _get_ttype_class as pygment_token_class from pygments.lexers.special import TextLexer, Token from pygments.lexers import get_lexer_by_name +from pyramid import compat from rhodecode.lib.helpers import ( get_lexer_for_filenode, html_escape, get_custom_lexer) @@ -48,8 +49,9 @@ def filenode_as_lines_tokens(filenode, l lexer = lexer or get_lexer_for_filenode(filenode) log.debug('Generating file node pygment tokens for %s, %s, org_lexer:%s', lexer, filenode, org_lexer) - tokens = tokenize_string(filenode.content, lexer) - lines = split_token_stream(tokens) + content = filenode.content + tokens = tokenize_string(content, lexer) + lines = split_token_stream(tokens, content) rv = list(lines) return rv @@ -73,7 +75,7 @@ def tokenize_string(content, lexer): yield pygment_token_class(token_type), token_text -def split_token_stream(tokens): +def split_token_stream(tokens, content): """ Take a list of (TokenType, text) tuples and split them by a string @@ -82,18 +84,23 @@ def split_token_stream(tokens): (TEXT, 'more'), (TEXT, 'text')] """ - buffer = [] + token_buffer = [] for token_class, token_text in tokens: parts = token_text.split('\n') for part in parts[:-1]: - buffer.append((token_class, part)) - yield buffer - buffer = [] + token_buffer.append((token_class, part)) + yield token_buffer + token_buffer = [] + + token_buffer.append((token_class, parts[-1])) - buffer.append((token_class, parts[-1])) - - if buffer: - yield buffer + if token_buffer: + yield token_buffer + elif content: + # this is a special case, we have the content, but tokenization didn't produce + # any results. THis can happen if know file extensions like .css have some bogus + # unicode content without any newline characters + yield [(pygment_token_class(Token.Text), content)] def filenode_as_annotated_lines_tokens(filenode): @@ -695,7 +702,7 @@ class DiffSet(object): filenode = None filename = None - if isinstance(input_file, basestring): + if isinstance(input_file, compat.string_types): filename = input_file elif isinstance(input_file, FileNode): filenode = input_file @@ -720,7 +727,11 @@ class DiffSet(object): if filenode not in self.highlighted_filenodes: tokenized_lines = filenode_as_lines_tokens(filenode, lexer) self.highlighted_filenodes[filenode] = tokenized_lines - return self.highlighted_filenodes[filenode][line_number - 1] + + try: + return self.highlighted_filenodes[filenode][line_number - 1] + except Exception: + return [('', u'rhodecode diff rendering error')] def action_to_op(self, action): return { diff --git a/rhodecode/lib/colander_utils.py b/rhodecode/lib/colander_utils.py --- a/rhodecode/lib/colander_utils.py +++ b/rhodecode/lib/colander_utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -18,13 +18,15 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ +from pyramid import compat + def strip_whitespace(value): """ Removes leading/trailing whitespace, newlines, and tabs from the value. Implements the `colander.interface.Preparer` interface. """ - if isinstance(value, basestring): + if isinstance(value, compat.string_types): return value.strip(' \t\n\r') else: return value diff --git a/rhodecode/lib/colored_formatter.py b/rhodecode/lib/colored_formatter.py --- a/rhodecode/lib/colored_formatter.py +++ b/rhodecode/lib/colored_formatter.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/compat.py b/rhodecode/lib/compat.py --- a/rhodecode/lib/compat.py +++ b/rhodecode/lib/compat.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/datelib.py b/rhodecode/lib/datelib.py --- a/rhodecode/lib/datelib.py +++ b/rhodecode/lib/datelib.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/db_manage.py b/rhodecode/lib/db_manage.py --- a/rhodecode/lib/db_manage.py +++ b/rhodecode/lib/db_manage.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -156,8 +156,11 @@ class DbManage(object): 'rhodecode/lib/dbmigrate') db_uri = self.dburi + if version: + DbMigrateVersion.set_version(version) + try: - curr_version = version or api.db_version(db_uri, repository_path) + curr_version = api.db_version(db_uri, repository_path) msg = ('Found current database db_uri under version ' 'control with version {}'.format(curr_version)) diff --git a/rhodecode/lib/dbmigrate/__init__.py b/rhodecode/lib/dbmigrate/__init__.py --- a/rhodecode/lib/dbmigrate/__init__.py +++ b/rhodecode/lib/dbmigrate/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/dbmigrate/migrate/changeset/schema.py b/rhodecode/lib/dbmigrate/migrate/changeset/schema.py --- a/rhodecode/lib/dbmigrate/migrate/changeset/schema.py +++ b/rhodecode/lib/dbmigrate/migrate/changeset/schema.py @@ -9,6 +9,7 @@ import sqlalchemy from sqlalchemy.schema import ForeignKeyConstraint from sqlalchemy.schema import UniqueConstraint +from pyramid import compat from rhodecode.lib.dbmigrate.migrate.exceptions import * from rhodecode.lib.dbmigrate.migrate.changeset import SQLA_07, SQLA_08 @@ -229,7 +230,7 @@ class ColumnDelta(DictMixin, sqlalchemy. diffs = self.compare_1_column(*p, **kw) else: # Zero columns specified - if not len(p) or not isinstance(p[0], basestring): + if not len(p) or not isinstance(p[0], compat.string_types): raise ValueError("First argument must be column name") diffs = self.compare_parameters(*p, **kw) @@ -338,7 +339,7 @@ class ColumnDelta(DictMixin, sqlalchemy. """Extracts data from p and modifies diffs""" p = list(p) while len(p): - if isinstance(p[0], basestring): + if isinstance(p[0], compat.string_types): k.setdefault('name', p.pop(0)) elif isinstance(p[0], sqlalchemy.types.TypeEngine): k.setdefault('type', p.pop(0)) @@ -376,7 +377,7 @@ class ColumnDelta(DictMixin, sqlalchemy. return getattr(self, '_table', None) def _set_table(self, table): - if isinstance(table, basestring): + if isinstance(table, compat.string_types): if self.alter_metadata: if not self.meta: raise ValueError("metadata must be specified for table" @@ -593,7 +594,7 @@ populated with defaults if isinstance(cons,(ForeignKeyConstraint, UniqueConstraint)): for col_name in cons.columns: - if not isinstance(col_name,basestring): + if not isinstance(col_name, compat.string_types): col_name = col_name.name if self.name==col_name: to_drop.add(cons) @@ -628,7 +629,7 @@ populated with defaults if (getattr(self, name[:-5]) and not obj): raise InvalidConstraintError("Column.create() accepts index_name," " primary_key_name and unique_name to generate constraints") - if not isinstance(obj, basestring) and obj is not None: + if not isinstance(obj, compat.string_types) and obj is not None: raise InvalidConstraintError( "%s argument for column must be constraint name" % name) diff --git a/rhodecode/lib/dbmigrate/migrate/versioning/schema.py b/rhodecode/lib/dbmigrate/migrate/versioning/schema.py --- a/rhodecode/lib/dbmigrate/migrate/versioning/schema.py +++ b/rhodecode/lib/dbmigrate/migrate/versioning/schema.py @@ -9,6 +9,7 @@ from sqlalchemy import (Table, Column, M from sqlalchemy.sql import and_ from sqlalchemy import exc as sa_exceptions from sqlalchemy.sql import bindparam +from pyramid import compat from rhodecode.lib.dbmigrate.migrate import exceptions from rhodecode.lib.dbmigrate.migrate.changeset import SQLA_07 @@ -25,7 +26,7 @@ class ControlledSchema(object): """A database under version control""" def __init__(self, engine, repository): - if isinstance(repository, basestring): + if isinstance(repository, compat.string_types): repository = Repository(repository) self.engine = engine self.repository = repository @@ -134,7 +135,7 @@ class ControlledSchema(object): """ # Confirm that the version # is valid: positive, integer, # exists in repos - if isinstance(repository, basestring): + if isinstance(repository, compat.string_types): repository = Repository(repository) version = cls._validate_version(repository, version) table = cls._create_table_version(engine, repository, version) @@ -199,7 +200,7 @@ class ControlledSchema(object): """ Compare the current model against the current database. """ - if isinstance(repository, basestring): + if isinstance(repository, compat.string_types): repository = Repository(repository) model = load_model(model) @@ -212,7 +213,7 @@ class ControlledSchema(object): """ Dump the current database as a Python model. """ - if isinstance(repository, basestring): + if isinstance(repository, compat.string_types): repository = Repository(repository) diff = schemadiff.getDiffOfModelAgainstDatabase( diff --git a/rhodecode/lib/dbmigrate/migrate/versioning/script/py.py b/rhodecode/lib/dbmigrate/migrate/versioning/script/py.py --- a/rhodecode/lib/dbmigrate/migrate/versioning/script/py.py +++ b/rhodecode/lib/dbmigrate/migrate/versioning/script/py.py @@ -7,6 +7,7 @@ import logging import inspect from StringIO import StringIO +from pyramid import compat from rhodecode.lib.dbmigrate import migrate from rhodecode.lib.dbmigrate.migrate.versioning import genmodel, schemadiff from rhodecode.lib.dbmigrate.migrate.versioning.config import operations @@ -51,7 +52,7 @@ class PythonScript(base.BaseScript): :rtype: string """ - if isinstance(repository, basestring): + if isinstance(repository, compat.string_types): # oh dear, an import cycle! from rhodecode.lib.dbmigrate.migrate.versioning.repository import Repository repository = Repository(repository) diff --git a/rhodecode/lib/dbmigrate/migrate/versioning/util/__init__.py b/rhodecode/lib/dbmigrate/migrate/versioning/util/__init__.py --- a/rhodecode/lib/dbmigrate/migrate/versioning/util/__init__.py +++ b/rhodecode/lib/dbmigrate/migrate/versioning/util/__init__.py @@ -11,6 +11,7 @@ from sqlalchemy import create_engine from sqlalchemy.engine import Engine from sqlalchemy.pool import StaticPool +from pyramid import compat from rhodecode.lib.dbmigrate.migrate import exceptions from rhodecode.lib.dbmigrate.migrate.versioning.util.keyedinstance import KeyedInstance from rhodecode.lib.dbmigrate.migrate.versioning.util.importpath import import_path @@ -18,6 +19,7 @@ from rhodecode.lib.dbmigrate.migrate.ver log = logging.getLogger(__name__) + def load_model(dotted_name): """Import module and use module-level variable". @@ -26,7 +28,7 @@ def load_model(dotted_name): .. versionchanged:: 0.5.4 """ - if isinstance(dotted_name, basestring): + if isinstance(dotted_name, compat.string_types): if ':' not in dotted_name: # backwards compatibility warnings.warn('model should be in form of module.model:User ' @@ -39,7 +41,7 @@ def load_model(dotted_name): def asbool(obj): """Do everything to use object as bool""" - if isinstance(obj, basestring): + if isinstance(obj, compat.string_types): obj = obj.strip().lower() if obj in ['true', 'yes', 'on', 'y', 't', '1']: return True @@ -112,7 +114,7 @@ def construct_engine(engine, **opts): """ if isinstance(engine, Engine): return engine - elif not isinstance(engine, basestring): + elif not isinstance(engine, compat.string_types): raise ValueError("you need to pass either an existing engine or a database uri") # get options for create_engine diff --git a/rhodecode/lib/dbmigrate/schema/__init__.py b/rhodecode/lib/dbmigrate/schema/__init__.py --- a/rhodecode/lib/dbmigrate/schema/__init__.py +++ b/rhodecode/lib/dbmigrate/schema/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/dbmigrate/schema/db_1_1_0.py b/rhodecode/lib/dbmigrate/schema/db_1_1_0.py --- a/rhodecode/lib/dbmigrate/schema/db_1_1_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_1_1_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/dbmigrate/schema/db_1_2_0.py b/rhodecode/lib/dbmigrate/schema/db_1_2_0.py --- a/rhodecode/lib/dbmigrate/schema/db_1_2_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_1_2_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -28,6 +28,7 @@ from sqlalchemy import * from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import relationship, joinedload, class_mapper, validates from beaker.cache import cache_region, region_invalidate +from pyramid import compat from rhodecode.lib.vcs import get_backend from rhodecode.lib.vcs.utils.helpers import get_scm @@ -413,7 +414,7 @@ class UserGroup(Base, BaseModel): Session.flush() members_list = [] if v: - v = [v] if isinstance(v, basestring) else v + v = [v] if isinstance(v, compat.string_types) else v for u_id in set(v): member = UserGroupMember(users_group_id, u_id) members_list.append(member) @@ -679,7 +680,7 @@ class Repository(Base, BaseModel): class Group(Base, BaseModel): __tablename__ = 'groups' __table_args__ = (UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing':True},) + {'extend_existing':True},) __mapper_args__ = {'order_by':'group_name'} group_id = Column("group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) diff --git a/rhodecode/lib/dbmigrate/schema/db_1_3_0.py b/rhodecode/lib/dbmigrate/schema/db_1_3_0.py --- a/rhodecode/lib/dbmigrate/schema/db_1_3_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_1_3_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -699,7 +699,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine':'InnoDB', 'mysql_charset': 'utf8'}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_1_4_0.py b/rhodecode/lib/dbmigrate/schema/db_1_4_0.py --- a/rhodecode/lib/dbmigrate/schema/db_1_4_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_1_4_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -513,7 +513,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_1_5_0.py b/rhodecode/lib/dbmigrate/schema/db_1_5_0.py --- a/rhodecode/lib/dbmigrate/schema/db_1_5_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_1_5_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -526,7 +526,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_1_5_2.py b/rhodecode/lib/dbmigrate/schema/db_1_5_2.py --- a/rhodecode/lib/dbmigrate/schema/db_1_5_2.py +++ b/rhodecode/lib/dbmigrate/schema/db_1_5_2.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -543,7 +543,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_1_6_0.py b/rhodecode/lib/dbmigrate/schema/db_1_6_0.py --- a/rhodecode/lib/dbmigrate/schema/db_1_6_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_1_6_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -626,7 +626,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_1_7_0.py b/rhodecode/lib/dbmigrate/schema/db_1_7_0.py --- a/rhodecode/lib/dbmigrate/schema/db_1_7_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_1_7_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -597,7 +597,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_1_8_0.py b/rhodecode/lib/dbmigrate/schema/db_1_8_0.py --- a/rhodecode/lib/dbmigrate/schema/db_1_8_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_1_8_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -599,7 +599,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_2_0_0.py b/rhodecode/lib/dbmigrate/schema/db_2_0_0.py --- a/rhodecode/lib/dbmigrate/schema/db_2_0_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_2_0_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -622,7 +622,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_2_0_1.py b/rhodecode/lib/dbmigrate/schema/db_2_0_1.py --- a/rhodecode/lib/dbmigrate/schema/db_2_0_1.py +++ b/rhodecode/lib/dbmigrate/schema/db_2_0_1.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -621,7 +621,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_2_0_2.py b/rhodecode/lib/dbmigrate/schema/db_2_0_2.py --- a/rhodecode/lib/dbmigrate/schema/db_2_0_2.py +++ b/rhodecode/lib/dbmigrate/schema/db_2_0_2.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -639,7 +639,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_2_1_0.py b/rhodecode/lib/dbmigrate/schema/db_2_1_0.py --- a/rhodecode/lib/dbmigrate/schema/db_2_1_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_2_1_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -656,7 +656,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_2_2_0.py b/rhodecode/lib/dbmigrate/schema/db_2_2_0.py --- a/rhodecode/lib/dbmigrate/schema/db_2_2_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_2_2_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -672,7 +672,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_2_2_3.py b/rhodecode/lib/dbmigrate/schema/db_2_2_3.py --- a/rhodecode/lib/dbmigrate/schema/db_2_2_3.py +++ b/rhodecode/lib/dbmigrate/schema/db_2_2_3.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -675,7 +675,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_2_3_0_0.py b/rhodecode/lib/dbmigrate/schema/db_2_3_0_0.py --- a/rhodecode/lib/dbmigrate/schema/db_2_3_0_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_2_3_0_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -679,7 +679,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_2_3_0_1.py b/rhodecode/lib/dbmigrate/schema/db_2_3_0_1.py --- a/rhodecode/lib/dbmigrate/schema/db_2_3_0_1.py +++ b/rhodecode/lib/dbmigrate/schema/db_2_3_0_1.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -679,7 +679,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_2_3_0_2.py b/rhodecode/lib/dbmigrate/schema/db_2_3_0_2.py --- a/rhodecode/lib/dbmigrate/schema/db_2_3_0_2.py +++ b/rhodecode/lib/dbmigrate/schema/db_2_3_0_2.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -707,7 +707,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_3_0_0_0.py b/rhodecode/lib/dbmigrate/schema/db_3_0_0_0.py --- a/rhodecode/lib/dbmigrate/schema/db_3_0_0_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_3_0_0_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -717,7 +717,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_3_0_0_1.py b/rhodecode/lib/dbmigrate/schema/db_3_0_0_1.py --- a/rhodecode/lib/dbmigrate/schema/db_3_0_0_1.py +++ b/rhodecode/lib/dbmigrate/schema/db_3_0_0_1.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -760,7 +760,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_3_1_0_0.py b/rhodecode/lib/dbmigrate/schema/db_3_1_0_0.py --- a/rhodecode/lib/dbmigrate/schema/db_3_1_0_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_3_1_0_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -758,7 +758,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_3_1_0_1.py b/rhodecode/lib/dbmigrate/schema/db_3_1_0_1.py --- a/rhodecode/lib/dbmigrate/schema/db_3_1_0_1.py +++ b/rhodecode/lib/dbmigrate/schema/db_3_1_0_1.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -759,7 +759,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_3_2_0_0.py b/rhodecode/lib/dbmigrate/schema/db_3_2_0_0.py --- a/rhodecode/lib/dbmigrate/schema/db_3_2_0_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_3_2_0_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -759,7 +759,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_3_3_0_0.py b/rhodecode/lib/dbmigrate/schema/db_3_3_0_0.py --- a/rhodecode/lib/dbmigrate/schema/db_3_3_0_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_3_3_0_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -765,7 +765,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_3_5_0_0.py b/rhodecode/lib/dbmigrate/schema/db_3_5_0_0.py --- a/rhodecode/lib/dbmigrate/schema/db_3_5_0_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_3_5_0_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -887,7 +887,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_3_7_0_0.py b/rhodecode/lib/dbmigrate/schema/db_3_7_0_0.py --- a/rhodecode/lib/dbmigrate/schema/db_3_7_0_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_3_7_0_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -912,7 +912,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_4_11_0_0.py b/rhodecode/lib/dbmigrate/schema/db_4_11_0_0.py --- a/rhodecode/lib/dbmigrate/schema/db_4_11_0_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_4_11_0_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -49,7 +49,7 @@ from sqlalchemy.exc import IntegrityErro from sqlalchemy.dialects.mysql import LONGTEXT from beaker.cache import cache_region from zope.cachedescriptors.property import Lazy as LazyProperty - +from pyramid import compat from pyramid.threadlocal import get_current_request from rhodecode.translation import _ @@ -2110,7 +2110,7 @@ class Repository(Base, BaseModel): warnings.warn("Use get_commit", DeprecationWarning) commit_id = None commit_idx = None - if isinstance(rev, basestring): + if isinstance(rev, compat.string_types): commit_id = rev else: commit_idx = rev @@ -2295,7 +2295,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) @@ -3726,7 +3725,7 @@ class PullRequestReviewers(Base, BaseMod @reasons.setter def reasons(self, val): val = val or [] - if any(not isinstance(x, basestring) for x in val): + if any(not isinstance(x, compat.string_types) for x in val): raise Exception('invalid reasons type, must be list of strings') self._reasons = val diff --git a/rhodecode/lib/dbmigrate/schema/db_4_13_0_0.py b/rhodecode/lib/dbmigrate/schema/db_4_13_0_0.py --- a/rhodecode/lib/dbmigrate/schema/db_4_13_0_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_4_13_0_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -49,7 +49,7 @@ from sqlalchemy.exc import IntegrityErro from sqlalchemy.dialects.mysql import LONGTEXT from beaker.cache import cache_region from zope.cachedescriptors.property import Lazy as LazyProperty - +from pyramid import compat from pyramid.threadlocal import get_current_request from rhodecode.translation import _ @@ -2176,7 +2176,7 @@ class Repository(Base, BaseModel): warnings.warn("Use get_commit", DeprecationWarning) commit_id = None commit_idx = None - if isinstance(rev, basestring): + if isinstance(rev, compat.string_types): commit_id = rev else: commit_idx = rev @@ -2361,7 +2361,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) @@ -3809,7 +3808,7 @@ class PullRequestReviewers(Base, BaseMod @reasons.setter def reasons(self, val): val = val or [] - if any(not isinstance(x, basestring) for x in val): + if any(not isinstance(x, compat.string_types) for x in val): raise Exception('invalid reasons type, must be list of strings') self._reasons = val diff --git a/rhodecode/lib/dbmigrate/schema/db_4_16_0_0.py b/rhodecode/lib/dbmigrate/schema/db_4_16_0_0.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/dbmigrate/schema/db_4_16_0_0.py @@ -0,0 +1,4758 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2019 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +""" +Database Models for RhodeCode Enterprise +""" + +import re +import os +import time +import hashlib +import logging +import datetime +import warnings +import ipaddress +import functools +import traceback +import collections + +from sqlalchemy import ( + or_, and_, not_, func, TypeDecorator, event, + Index, Sequence, UniqueConstraint, ForeignKey, CheckConstraint, Column, + Boolean, String, Unicode, UnicodeText, DateTime, Integer, LargeBinary, + Text, Float, PickleType) +from sqlalchemy.sql.expression import true, false +from sqlalchemy.sql.functions import coalesce, count # pragma: no cover +from sqlalchemy.orm import ( + relationship, joinedload, class_mapper, validates, aliased) +from sqlalchemy.ext.declarative import declared_attr +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.exc import IntegrityError # pragma: no cover +from sqlalchemy.dialects.mysql import LONGTEXT +from zope.cachedescriptors.property import Lazy as LazyProperty +from pyramid import compat +from pyramid.threadlocal import get_current_request + +from rhodecode.translation import _ +from rhodecode.lib.vcs import get_vcs_instance +from rhodecode.lib.vcs.backends.base import EmptyCommit, Reference +from rhodecode.lib.utils2 import ( + str2bool, safe_str, get_commit_safe, safe_unicode, sha1_safe, + time_to_datetime, aslist, Optional, safe_int, get_clone_url, AttributeDict, + glob2re, StrictAttributeDict, cleaned_uri) +from rhodecode.lib.jsonalchemy import MutationObj, MutationList, JsonType, \ + JsonRaw +from rhodecode.lib.ext_json import json +from rhodecode.lib.caching_query import FromCache +from rhodecode.lib.encrypt import AESCipher + +from rhodecode.model.meta import Base, Session + +URL_SEP = '/' +log = logging.getLogger(__name__) + +# ============================================================================= +# BASE CLASSES +# ============================================================================= + +# this is propagated from .ini file rhodecode.encrypted_values.secret or +# beaker.session.secret if first is not set. +# and initialized at environment.py +ENCRYPTION_KEY = None + +# used to sort permissions by types, '#' used here is not allowed to be in +# usernames, and it's very early in sorted string.printable table. +PERMISSION_TYPE_SORT = { + 'admin': '####', + 'write': '###', + 'read': '##', + 'none': '#', +} + + +def display_user_sort(obj): + """ + Sort function used to sort permissions in .permissions() function of + Repository, RepoGroup, UserGroup. Also it put the default user in front + of all other resources + """ + + if obj.username == User.DEFAULT_USER: + return '#####' + prefix = PERMISSION_TYPE_SORT.get(obj.permission.split('.')[-1], '') + return prefix + obj.username + + +def display_user_group_sort(obj): + """ + Sort function used to sort permissions in .permissions() function of + Repository, RepoGroup, UserGroup. Also it put the default user in front + of all other resources + """ + + prefix = PERMISSION_TYPE_SORT.get(obj.permission.split('.')[-1], '') + return prefix + obj.users_group_name + + +def _hash_key(k): + return sha1_safe(k) + + +def in_filter_generator(qry, items, limit=500): + """ + Splits IN() into multiple with OR + e.g.:: + cnt = Repository.query().filter( + or_( + *in_filter_generator(Repository.repo_id, range(100000)) + )).count() + """ + if not items: + # empty list will cause empty query which might cause security issues + # this can lead to hidden unpleasant results + items = [-1] + + parts = [] + for chunk in xrange(0, len(items), limit): + parts.append( + qry.in_(items[chunk: chunk + limit]) + ) + + return parts + + +base_table_args = { + 'extend_existing': True, + 'mysql_engine': 'InnoDB', + 'mysql_charset': 'utf8', + 'sqlite_autoincrement': True +} + + +class EncryptedTextValue(TypeDecorator): + """ + Special column for encrypted long text data, use like:: + + value = Column("encrypted_value", EncryptedValue(), nullable=False) + + This column is intelligent so if value is in unencrypted form it return + unencrypted form, but on save it always encrypts + """ + impl = Text + + def process_bind_param(self, value, dialect): + if not value: + return value + if value.startswith('enc$aes$') or value.startswith('enc$aes_hmac$'): + # protect against double encrypting if someone manually starts + # doing + raise ValueError('value needs to be in unencrypted format, ie. ' + 'not starting with enc$aes') + return 'enc$aes_hmac$%s' % AESCipher( + ENCRYPTION_KEY, hmac=True).encrypt(value) + + def process_result_value(self, value, dialect): + import rhodecode + + if not value: + return value + + parts = value.split('$', 3) + if not len(parts) == 3: + # probably not encrypted values + return value + else: + if parts[0] != 'enc': + # parts ok but without our header ? + return value + enc_strict_mode = str2bool(rhodecode.CONFIG.get( + 'rhodecode.encrypted_values.strict') or True) + # at that stage we know it's our encryption + if parts[1] == 'aes': + decrypted_data = AESCipher(ENCRYPTION_KEY).decrypt(parts[2]) + elif parts[1] == 'aes_hmac': + decrypted_data = AESCipher( + ENCRYPTION_KEY, hmac=True, + strict_verification=enc_strict_mode).decrypt(parts[2]) + else: + raise ValueError( + 'Encryption type part is wrong, must be `aes` ' + 'or `aes_hmac`, got `%s` instead' % (parts[1])) + return decrypted_data + + +class BaseModel(object): + """ + Base Model for all classes + """ + + @classmethod + def _get_keys(cls): + """return column names for this model """ + return class_mapper(cls).c.keys() + + def get_dict(self): + """ + return dict with keys and values corresponding + to this model data """ + + d = {} + for k in self._get_keys(): + d[k] = getattr(self, k) + + # also use __json__() if present to get additional fields + _json_attr = getattr(self, '__json__', None) + if _json_attr: + # update with attributes from __json__ + if callable(_json_attr): + _json_attr = _json_attr() + for k, val in _json_attr.iteritems(): + d[k] = val + return d + + def get_appstruct(self): + """return list with keys and values tuples corresponding + to this model data """ + + lst = [] + for k in self._get_keys(): + lst.append((k, getattr(self, k),)) + return lst + + def populate_obj(self, populate_dict): + """populate model with data from given populate_dict""" + + for k in self._get_keys(): + if k in populate_dict: + setattr(self, k, populate_dict[k]) + + @classmethod + def query(cls): + return Session().query(cls) + + @classmethod + def get(cls, id_): + if id_: + return cls.query().get(id_) + + @classmethod + def get_or_404(cls, id_): + from pyramid.httpexceptions import HTTPNotFound + + try: + id_ = int(id_) + except (TypeError, ValueError): + raise HTTPNotFound() + + res = cls.query().get(id_) + if not res: + raise HTTPNotFound() + return res + + @classmethod + def getAll(cls): + # deprecated and left for backward compatibility + return cls.get_all() + + @classmethod + def get_all(cls): + return cls.query().all() + + @classmethod + def delete(cls, id_): + obj = cls.query().get(id_) + Session().delete(obj) + + @classmethod + def identity_cache(cls, session, attr_name, value): + exist_in_session = [] + for (item_cls, pkey), instance in session.identity_map.items(): + if cls == item_cls and getattr(instance, attr_name) == value: + exist_in_session.append(instance) + if exist_in_session: + if len(exist_in_session) == 1: + return exist_in_session[0] + log.exception( + 'multiple objects with attr %s and ' + 'value %s found with same name: %r', + attr_name, value, exist_in_session) + + def __repr__(self): + if hasattr(self, '__unicode__'): + # python repr needs to return str + try: + return safe_str(self.__unicode__()) + except UnicodeDecodeError: + pass + return '' % (self.__class__.__name__) + + +class RhodeCodeSetting(Base, BaseModel): + __tablename__ = 'rhodecode_settings' + __table_args__ = ( + UniqueConstraint('app_settings_name'), + base_table_args + ) + + SETTINGS_TYPES = { + 'str': safe_str, + 'int': safe_int, + 'unicode': safe_unicode, + 'bool': str2bool, + 'list': functools.partial(aslist, sep=',') + } + DEFAULT_UPDATE_URL = 'https://rhodecode.com/api/v1/info/versions' + GLOBAL_CONF_KEY = 'app_settings' + + app_settings_id = Column("app_settings_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + app_settings_name = Column("app_settings_name", String(255), nullable=True, unique=None, default=None) + _app_settings_value = Column("app_settings_value", String(4096), nullable=True, unique=None, default=None) + _app_settings_type = Column("app_settings_type", String(255), nullable=True, unique=None, default=None) + + def __init__(self, key='', val='', type='unicode'): + self.app_settings_name = key + self.app_settings_type = type + self.app_settings_value = val + + @validates('_app_settings_value') + def validate_settings_value(self, key, val): + assert type(val) == unicode + return val + + @hybrid_property + def app_settings_value(self): + v = self._app_settings_value + _type = self.app_settings_type + if _type: + _type = self.app_settings_type.split('.')[0] + # decode the encrypted value + if 'encrypted' in self.app_settings_type: + cipher = EncryptedTextValue() + v = safe_unicode(cipher.process_result_value(v, None)) + + converter = self.SETTINGS_TYPES.get(_type) or \ + self.SETTINGS_TYPES['unicode'] + return converter(v) + + @app_settings_value.setter + def app_settings_value(self, val): + """ + Setter that will always make sure we use unicode in app_settings_value + + :param val: + """ + val = safe_unicode(val) + # encode the encrypted value + if 'encrypted' in self.app_settings_type: + cipher = EncryptedTextValue() + val = safe_unicode(cipher.process_bind_param(val, None)) + self._app_settings_value = val + + @hybrid_property + def app_settings_type(self): + return self._app_settings_type + + @app_settings_type.setter + def app_settings_type(self, val): + if val.split('.')[0] not in self.SETTINGS_TYPES: + raise Exception('type must be one of %s got %s' + % (self.SETTINGS_TYPES.keys(), val)) + self._app_settings_type = val + + @classmethod + def get_by_prefix(cls, prefix): + return RhodeCodeSetting.query()\ + .filter(RhodeCodeSetting.app_settings_name.startswith(prefix))\ + .all() + + def __unicode__(self): + return u"<%s('%s:%s[%s]')>" % ( + self.__class__.__name__, + self.app_settings_name, self.app_settings_value, + self.app_settings_type + ) + + +class RhodeCodeUi(Base, BaseModel): + __tablename__ = 'rhodecode_ui' + __table_args__ = ( + UniqueConstraint('ui_key'), + base_table_args + ) + + HOOK_REPO_SIZE = 'changegroup.repo_size' + # HG + HOOK_PRE_PULL = 'preoutgoing.pre_pull' + HOOK_PULL = 'outgoing.pull_logger' + HOOK_PRE_PUSH = 'prechangegroup.pre_push' + HOOK_PRETX_PUSH = 'pretxnchangegroup.pre_push' + HOOK_PUSH = 'changegroup.push_logger' + HOOK_PUSH_KEY = 'pushkey.key_push' + + # TODO: johbo: Unify way how hooks are configured for git and hg, + # git part is currently hardcoded. + + # SVN PATTERNS + SVN_BRANCH_ID = 'vcs_svn_branch' + SVN_TAG_ID = 'vcs_svn_tag' + + ui_id = Column( + "ui_id", Integer(), nullable=False, unique=True, default=None, + primary_key=True) + ui_section = Column( + "ui_section", String(255), nullable=True, unique=None, default=None) + ui_key = Column( + "ui_key", String(255), nullable=True, unique=None, default=None) + ui_value = Column( + "ui_value", String(255), nullable=True, unique=None, default=None) + ui_active = Column( + "ui_active", Boolean(), nullable=True, unique=None, default=True) + + def __repr__(self): + return '<%s[%s]%s=>%s]>' % (self.__class__.__name__, self.ui_section, + self.ui_key, self.ui_value) + + +class RepoRhodeCodeSetting(Base, BaseModel): + __tablename__ = 'repo_rhodecode_settings' + __table_args__ = ( + UniqueConstraint( + 'app_settings_name', 'repository_id', + name='uq_repo_rhodecode_setting_name_repo_id'), + base_table_args + ) + + repository_id = Column( + "repository_id", Integer(), ForeignKey('repositories.repo_id'), + nullable=False) + app_settings_id = Column( + "app_settings_id", Integer(), nullable=False, unique=True, + default=None, primary_key=True) + app_settings_name = Column( + "app_settings_name", String(255), nullable=True, unique=None, + default=None) + _app_settings_value = Column( + "app_settings_value", String(4096), nullable=True, unique=None, + default=None) + _app_settings_type = Column( + "app_settings_type", String(255), nullable=True, unique=None, + default=None) + + repository = relationship('Repository') + + def __init__(self, repository_id, key='', val='', type='unicode'): + self.repository_id = repository_id + self.app_settings_name = key + self.app_settings_type = type + self.app_settings_value = val + + @validates('_app_settings_value') + def validate_settings_value(self, key, val): + assert type(val) == unicode + return val + + @hybrid_property + def app_settings_value(self): + v = self._app_settings_value + type_ = self.app_settings_type + SETTINGS_TYPES = RhodeCodeSetting.SETTINGS_TYPES + converter = SETTINGS_TYPES.get(type_) or SETTINGS_TYPES['unicode'] + return converter(v) + + @app_settings_value.setter + def app_settings_value(self, val): + """ + Setter that will always make sure we use unicode in app_settings_value + + :param val: + """ + self._app_settings_value = safe_unicode(val) + + @hybrid_property + def app_settings_type(self): + return self._app_settings_type + + @app_settings_type.setter + def app_settings_type(self, val): + SETTINGS_TYPES = RhodeCodeSetting.SETTINGS_TYPES + if val not in SETTINGS_TYPES: + raise Exception('type must be one of %s got %s' + % (SETTINGS_TYPES.keys(), val)) + self._app_settings_type = val + + def __unicode__(self): + return u"<%s('%s:%s:%s[%s]')>" % ( + self.__class__.__name__, self.repository.repo_name, + self.app_settings_name, self.app_settings_value, + self.app_settings_type + ) + + +class RepoRhodeCodeUi(Base, BaseModel): + __tablename__ = 'repo_rhodecode_ui' + __table_args__ = ( + UniqueConstraint( + 'repository_id', 'ui_section', 'ui_key', + name='uq_repo_rhodecode_ui_repository_id_section_key'), + base_table_args + ) + + repository_id = Column( + "repository_id", Integer(), ForeignKey('repositories.repo_id'), + nullable=False) + ui_id = Column( + "ui_id", Integer(), nullable=False, unique=True, default=None, + primary_key=True) + ui_section = Column( + "ui_section", String(255), nullable=True, unique=None, default=None) + ui_key = Column( + "ui_key", String(255), nullable=True, unique=None, default=None) + ui_value = Column( + "ui_value", String(255), nullable=True, unique=None, default=None) + ui_active = Column( + "ui_active", Boolean(), nullable=True, unique=None, default=True) + + repository = relationship('Repository') + + def __repr__(self): + return '<%s[%s:%s]%s=>%s]>' % ( + self.__class__.__name__, self.repository.repo_name, + self.ui_section, self.ui_key, self.ui_value) + + +class User(Base, BaseModel): + __tablename__ = 'users' + __table_args__ = ( + UniqueConstraint('username'), UniqueConstraint('email'), + Index('u_username_idx', 'username'), + Index('u_email_idx', 'email'), + base_table_args + ) + + DEFAULT_USER = 'default' + DEFAULT_USER_EMAIL = 'anonymous@rhodecode.org' + DEFAULT_GRAVATAR_URL = 'https://secure.gravatar.com/avatar/{md5email}?d=identicon&s={size}' + + user_id = Column("user_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + username = Column("username", String(255), nullable=True, unique=None, default=None) + password = Column("password", String(255), nullable=True, unique=None, default=None) + active = Column("active", Boolean(), nullable=True, unique=None, default=True) + admin = Column("admin", Boolean(), nullable=True, unique=None, default=False) + name = Column("firstname", String(255), nullable=True, unique=None, default=None) + lastname = Column("lastname", String(255), nullable=True, unique=None, default=None) + _email = Column("email", String(255), nullable=True, unique=None, default=None) + last_login = Column("last_login", DateTime(timezone=False), nullable=True, unique=None, default=None) + last_activity = Column('last_activity', DateTime(timezone=False), nullable=True, unique=None, default=None) + + extern_type = Column("extern_type", String(255), nullable=True, unique=None, default=None) + extern_name = Column("extern_name", String(255), nullable=True, unique=None, default=None) + _api_key = Column("api_key", String(255), nullable=True, unique=None, default=None) + inherit_default_permissions = Column("inherit_default_permissions", Boolean(), nullable=False, unique=None, default=True) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + _user_data = Column("user_data", LargeBinary(), nullable=True) # JSON data + + user_log = relationship('UserLog') + user_perms = relationship('UserToPerm', primaryjoin="User.user_id==UserToPerm.user_id", cascade='all') + + repositories = relationship('Repository') + repository_groups = relationship('RepoGroup') + user_groups = relationship('UserGroup') + + user_followers = relationship('UserFollowing', primaryjoin='UserFollowing.follows_user_id==User.user_id', cascade='all') + followings = relationship('UserFollowing', primaryjoin='UserFollowing.user_id==User.user_id', cascade='all') + + repo_to_perm = relationship('UserRepoToPerm', primaryjoin='UserRepoToPerm.user_id==User.user_id', cascade='all') + repo_group_to_perm = relationship('UserRepoGroupToPerm', primaryjoin='UserRepoGroupToPerm.user_id==User.user_id', cascade='all') + user_group_to_perm = relationship('UserUserGroupToPerm', primaryjoin='UserUserGroupToPerm.user_id==User.user_id', cascade='all') + + group_member = relationship('UserGroupMember', cascade='all') + + notifications = relationship('UserNotification', cascade='all') + # notifications assigned to this user + user_created_notifications = relationship('Notification', cascade='all') + # comments created by this user + user_comments = relationship('ChangesetComment', cascade='all') + # user profile extra info + user_emails = relationship('UserEmailMap', cascade='all') + user_ip_map = relationship('UserIpMap', cascade='all') + user_auth_tokens = relationship('UserApiKeys', cascade='all') + user_ssh_keys = relationship('UserSshKeys', cascade='all') + + # gists + user_gists = relationship('Gist', cascade='all') + # user pull requests + user_pull_requests = relationship('PullRequest', cascade='all') + # external identities + extenal_identities = relationship( + 'ExternalIdentity', + primaryjoin="User.user_id==ExternalIdentity.local_user_id", + cascade='all') + # review rules + user_review_rules = relationship('RepoReviewRuleUser', cascade='all') + + def __unicode__(self): + return u"<%s('id:%s:%s')>" % (self.__class__.__name__, + self.user_id, self.username) + + @hybrid_property + def email(self): + return self._email + + @email.setter + def email(self, val): + self._email = val.lower() if val else None + + @hybrid_property + def first_name(self): + from rhodecode.lib import helpers as h + if self.name: + return h.escape(self.name) + return self.name + + @hybrid_property + def last_name(self): + from rhodecode.lib import helpers as h + if self.lastname: + return h.escape(self.lastname) + return self.lastname + + @hybrid_property + def api_key(self): + """ + Fetch if exist an auth-token with role ALL connected to this user + """ + user_auth_token = UserApiKeys.query()\ + .filter(UserApiKeys.user_id == self.user_id)\ + .filter(or_(UserApiKeys.expires == -1, + UserApiKeys.expires >= time.time()))\ + .filter(UserApiKeys.role == UserApiKeys.ROLE_ALL).first() + if user_auth_token: + user_auth_token = user_auth_token.api_key + + return user_auth_token + + @api_key.setter + def api_key(self, val): + # don't allow to set API key this is deprecated for now + self._api_key = None + + @property + def reviewer_pull_requests(self): + return PullRequestReviewers.query() \ + .options(joinedload(PullRequestReviewers.pull_request)) \ + .filter(PullRequestReviewers.user_id == self.user_id) \ + .all() + + @property + def firstname(self): + # alias for future + return self.name + + @property + def emails(self): + other = UserEmailMap.query()\ + .filter(UserEmailMap.user == self) \ + .order_by(UserEmailMap.email_id.asc()) \ + .all() + return [self.email] + [x.email for x in other] + + @property + def auth_tokens(self): + auth_tokens = self.get_auth_tokens() + return [x.api_key for x in auth_tokens] + + def get_auth_tokens(self): + return UserApiKeys.query()\ + .filter(UserApiKeys.user == self)\ + .order_by(UserApiKeys.user_api_key_id.asc())\ + .all() + + @LazyProperty + def feed_token(self): + return self.get_feed_token() + + def get_feed_token(self, cache=True): + feed_tokens = UserApiKeys.query()\ + .filter(UserApiKeys.user == self)\ + .filter(UserApiKeys.role == UserApiKeys.ROLE_FEED) + if cache: + feed_tokens = feed_tokens.options( + FromCache("sql_cache_short", "get_user_feed_token_%s" % self.user_id)) + + feed_tokens = feed_tokens.all() + if feed_tokens: + return feed_tokens[0].api_key + return 'NO_FEED_TOKEN_AVAILABLE' + + @classmethod + def get(cls, user_id, cache=False): + if not user_id: + return + + user = cls.query() + if cache: + user = user.options( + FromCache("sql_cache_short", "get_users_%s" % user_id)) + return user.get(user_id) + + @classmethod + def extra_valid_auth_tokens(cls, user, role=None): + tokens = UserApiKeys.query().filter(UserApiKeys.user == user)\ + .filter(or_(UserApiKeys.expires == -1, + UserApiKeys.expires >= time.time())) + if role: + tokens = tokens.filter(or_(UserApiKeys.role == role, + UserApiKeys.role == UserApiKeys.ROLE_ALL)) + return tokens.all() + + def authenticate_by_token(self, auth_token, roles=None, scope_repo_id=None): + from rhodecode.lib import auth + + log.debug('Trying to authenticate user: %s via auth-token, ' + 'and roles: %s', self, roles) + + if not auth_token: + return False + + crypto_backend = auth.crypto_backend() + + roles = (roles or []) + [UserApiKeys.ROLE_ALL] + tokens_q = UserApiKeys.query()\ + .filter(UserApiKeys.user_id == self.user_id)\ + .filter(or_(UserApiKeys.expires == -1, + UserApiKeys.expires >= time.time())) + + tokens_q = tokens_q.filter(UserApiKeys.role.in_(roles)) + + plain_tokens = [] + hash_tokens = [] + + user_tokens = tokens_q.all() + log.debug('Found %s user tokens to check for authentication', len(user_tokens)) + for token in user_tokens: + log.debug('AUTH_TOKEN: checking if user token with id `%s` matches', + token.user_api_key_id) + # verify scope first, since it's way faster than hash calculation of + # encrypted tokens + if token.repo_id: + # token has a scope, we need to verify it + if scope_repo_id != token.repo_id: + log.debug( + 'AUTH_TOKEN: scope mismatch, token has a set repo scope: %s, ' + 'and calling scope is:%s, skipping further checks', + token.repo, scope_repo_id) + # token has a scope, and it doesn't match, skip token + continue + + if token.api_key.startswith(crypto_backend.ENC_PREF): + hash_tokens.append(token.api_key) + else: + plain_tokens.append(token.api_key) + + is_plain_match = auth_token in plain_tokens + if is_plain_match: + return True + + for hashed in hash_tokens: + # NOTE(marcink): this is expensive to calculate, but most secure + match = crypto_backend.hash_check(auth_token, hashed) + if match: + return True + + return False + + @property + def ip_addresses(self): + ret = UserIpMap.query().filter(UserIpMap.user == self).all() + return [x.ip_addr for x in ret] + + @property + def username_and_name(self): + return '%s (%s %s)' % (self.username, self.first_name, self.last_name) + + @property + def username_or_name_or_email(self): + full_name = self.full_name if self.full_name is not ' ' else None + return self.username or full_name or self.email + + @property + def full_name(self): + return '%s %s' % (self.first_name, self.last_name) + + @property + def full_name_or_username(self): + return ('%s %s' % (self.first_name, self.last_name) + if (self.first_name and self.last_name) else self.username) + + @property + def full_contact(self): + return '%s %s <%s>' % (self.first_name, self.last_name, self.email) + + @property + def short_contact(self): + return '%s %s' % (self.first_name, self.last_name) + + @property + def is_admin(self): + return self.admin + + def AuthUser(self, **kwargs): + """ + Returns instance of AuthUser for this user + """ + from rhodecode.lib.auth import AuthUser + return AuthUser(user_id=self.user_id, username=self.username, **kwargs) + + @hybrid_property + def user_data(self): + if not self._user_data: + return {} + + try: + return json.loads(self._user_data) + except TypeError: + return {} + + @user_data.setter + def user_data(self, val): + if not isinstance(val, dict): + raise Exception('user_data must be dict, got %s' % type(val)) + try: + self._user_data = json.dumps(val) + except Exception: + log.error(traceback.format_exc()) + + @classmethod + def get_by_username(cls, username, case_insensitive=False, + cache=False, identity_cache=False): + session = Session() + + if case_insensitive: + q = cls.query().filter( + func.lower(cls.username) == func.lower(username)) + else: + q = cls.query().filter(cls.username == username) + + if cache: + if identity_cache: + val = cls.identity_cache(session, 'username', username) + if val: + return val + else: + cache_key = "get_user_by_name_%s" % _hash_key(username) + q = q.options( + FromCache("sql_cache_short", cache_key)) + + return q.scalar() + + @classmethod + def get_by_auth_token(cls, auth_token, cache=False): + q = UserApiKeys.query()\ + .filter(UserApiKeys.api_key == auth_token)\ + .filter(or_(UserApiKeys.expires == -1, + UserApiKeys.expires >= time.time())) + if cache: + q = q.options( + FromCache("sql_cache_short", "get_auth_token_%s" % auth_token)) + + match = q.first() + if match: + return match.user + + @classmethod + def get_by_email(cls, email, case_insensitive=False, cache=False): + + if case_insensitive: + q = cls.query().filter(func.lower(cls.email) == func.lower(email)) + + else: + q = cls.query().filter(cls.email == email) + + email_key = _hash_key(email) + if cache: + q = q.options( + FromCache("sql_cache_short", "get_email_key_%s" % email_key)) + + ret = q.scalar() + if ret is None: + q = UserEmailMap.query() + # try fetching in alternate email map + if case_insensitive: + q = q.filter(func.lower(UserEmailMap.email) == func.lower(email)) + else: + q = q.filter(UserEmailMap.email == email) + q = q.options(joinedload(UserEmailMap.user)) + if cache: + q = q.options( + FromCache("sql_cache_short", "get_email_map_key_%s" % email_key)) + ret = getattr(q.scalar(), 'user', None) + + return ret + + @classmethod + def get_from_cs_author(cls, author): + """ + Tries to get User objects out of commit author string + + :param author: + """ + from rhodecode.lib.helpers import email, author_name + # Valid email in the attribute passed, see if they're in the system + _email = email(author) + if _email: + user = cls.get_by_email(_email, case_insensitive=True) + if user: + return user + # Maybe we can match by username? + _author = author_name(author) + user = cls.get_by_username(_author, case_insensitive=True) + if user: + return user + + def update_userdata(self, **kwargs): + usr = self + old = usr.user_data + old.update(**kwargs) + usr.user_data = old + Session().add(usr) + log.debug('updated userdata with ', kwargs) + + def update_lastlogin(self): + """Update user lastlogin""" + self.last_login = datetime.datetime.now() + Session().add(self) + log.debug('updated user %s lastlogin', self.username) + + def update_password(self, new_password): + from rhodecode.lib.auth import get_crypt_password + + self.password = get_crypt_password(new_password) + Session().add(self) + + @classmethod + def get_first_super_admin(cls): + user = User.query()\ + .filter(User.admin == true()) \ + .order_by(User.user_id.asc()) \ + .first() + + if user is None: + raise Exception('FATAL: Missing administrative account!') + return user + + @classmethod + def get_all_super_admins(cls): + """ + Returns all admin accounts sorted by username + """ + return User.query().filter(User.admin == true())\ + .order_by(User.username.asc()).all() + + @classmethod + def get_default_user(cls, cache=False, refresh=False): + user = User.get_by_username(User.DEFAULT_USER, cache=cache) + if user is None: + raise Exception('FATAL: Missing default account!') + if refresh: + # The default user might be based on outdated state which + # has been loaded from the cache. + # A call to refresh() ensures that the + # latest state from the database is used. + Session().refresh(user) + return user + + def _get_default_perms(self, user, suffix=''): + from rhodecode.model.permission import PermissionModel + return PermissionModel().get_default_perms(user.user_perms, suffix) + + def get_default_perms(self, suffix=''): + return self._get_default_perms(self, suffix) + + def get_api_data(self, include_secrets=False, details='full'): + """ + Common function for generating user related data for API + + :param include_secrets: By default secrets in the API data will be replaced + by a placeholder value to prevent exposing this data by accident. In case + this data shall be exposed, set this flag to ``True``. + + :param details: details can be 'basic|full' basic gives only a subset of + the available user information that includes user_id, name and emails. + """ + user = self + user_data = self.user_data + data = { + 'user_id': user.user_id, + 'username': user.username, + 'firstname': user.name, + 'lastname': user.lastname, + 'email': user.email, + 'emails': user.emails, + } + if details == 'basic': + return data + + auth_token_length = 40 + auth_token_replacement = '*' * auth_token_length + + extras = { + 'auth_tokens': [auth_token_replacement], + 'active': user.active, + 'admin': user.admin, + 'extern_type': user.extern_type, + 'extern_name': user.extern_name, + 'last_login': user.last_login, + 'last_activity': user.last_activity, + 'ip_addresses': user.ip_addresses, + 'language': user_data.get('language') + } + data.update(extras) + + if include_secrets: + data['auth_tokens'] = user.auth_tokens + return data + + def __json__(self): + data = { + 'full_name': self.full_name, + 'full_name_or_username': self.full_name_or_username, + 'short_contact': self.short_contact, + 'full_contact': self.full_contact, + } + data.update(self.get_api_data()) + return data + + +class UserApiKeys(Base, BaseModel): + __tablename__ = 'user_api_keys' + __table_args__ = ( + Index('uak_api_key_idx', 'api_key', unique=True), + Index('uak_api_key_expires_idx', 'api_key', 'expires'), + base_table_args + ) + __mapper_args__ = {} + + # ApiKey role + ROLE_ALL = 'token_role_all' + ROLE_HTTP = 'token_role_http' + ROLE_VCS = 'token_role_vcs' + ROLE_API = 'token_role_api' + ROLE_FEED = 'token_role_feed' + ROLE_PASSWORD_RESET = 'token_password_reset' + + ROLES = [ROLE_ALL, ROLE_HTTP, ROLE_VCS, ROLE_API, ROLE_FEED] + + user_api_key_id = Column("user_api_key_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None) + api_key = Column("api_key", String(255), nullable=False, unique=True) + description = Column('description', UnicodeText().with_variant(UnicodeText(1024), 'mysql')) + expires = Column('expires', Float(53), nullable=False) + role = Column('role', String(255), nullable=True) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + + # scope columns + repo_id = Column( + 'repo_id', Integer(), ForeignKey('repositories.repo_id'), + nullable=True, unique=None, default=None) + repo = relationship('Repository', lazy='joined') + + repo_group_id = Column( + 'repo_group_id', Integer(), ForeignKey('groups.group_id'), + nullable=True, unique=None, default=None) + repo_group = relationship('RepoGroup', lazy='joined') + + user = relationship('User', lazy='joined') + + def __unicode__(self): + return u"<%s('%s')>" % (self.__class__.__name__, self.role) + + def __json__(self): + data = { + 'auth_token': self.api_key, + 'role': self.role, + 'scope': self.scope_humanized, + 'expired': self.expired + } + return data + + def get_api_data(self, include_secrets=False): + data = self.__json__() + if include_secrets: + return data + else: + data['auth_token'] = self.token_obfuscated + return data + + @hybrid_property + def description_safe(self): + from rhodecode.lib import helpers as h + return h.escape(self.description) + + @property + def expired(self): + if self.expires == -1: + return False + return time.time() > self.expires + + @classmethod + def _get_role_name(cls, role): + return { + cls.ROLE_ALL: _('all'), + cls.ROLE_HTTP: _('http/web interface'), + cls.ROLE_VCS: _('vcs (git/hg/svn protocol)'), + cls.ROLE_API: _('api calls'), + cls.ROLE_FEED: _('feed access'), + }.get(role, role) + + @property + def role_humanized(self): + return self._get_role_name(self.role) + + def _get_scope(self): + if self.repo: + return repr(self.repo) + if self.repo_group: + return repr(self.repo_group) + ' (recursive)' + return 'global' + + @property + def scope_humanized(self): + return self._get_scope() + + @property + def token_obfuscated(self): + if self.api_key: + return self.api_key[:4] + "****" + + +class UserEmailMap(Base, BaseModel): + __tablename__ = 'user_email_map' + __table_args__ = ( + Index('uem_email_idx', 'email'), + UniqueConstraint('email'), + base_table_args + ) + __mapper_args__ = {} + + email_id = Column("email_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None) + _email = Column("email", String(255), nullable=True, unique=False, default=None) + user = relationship('User', lazy='joined') + + @validates('_email') + def validate_email(self, key, email): + # check if this email is not main one + main_email = Session().query(User).filter(User.email == email).scalar() + if main_email is not None: + raise AttributeError('email %s is present is user table' % email) + return email + + @hybrid_property + def email(self): + return self._email + + @email.setter + def email(self, val): + self._email = val.lower() if val else None + + +class UserIpMap(Base, BaseModel): + __tablename__ = 'user_ip_map' + __table_args__ = ( + UniqueConstraint('user_id', 'ip_addr'), + base_table_args + ) + __mapper_args__ = {} + + ip_id = Column("ip_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None) + ip_addr = Column("ip_addr", String(255), nullable=True, unique=False, default=None) + active = Column("active", Boolean(), nullable=True, unique=None, default=True) + description = Column("description", String(10000), nullable=True, unique=None, default=None) + user = relationship('User', lazy='joined') + + @hybrid_property + def description_safe(self): + from rhodecode.lib import helpers as h + return h.escape(self.description) + + @classmethod + def _get_ip_range(cls, ip_addr): + net = ipaddress.ip_network(safe_unicode(ip_addr), strict=False) + return [str(net.network_address), str(net.broadcast_address)] + + def __json__(self): + return { + 'ip_addr': self.ip_addr, + 'ip_range': self._get_ip_range(self.ip_addr), + } + + def __unicode__(self): + return u"<%s('user_id:%s=>%s')>" % (self.__class__.__name__, + self.user_id, self.ip_addr) + + +class UserSshKeys(Base, BaseModel): + __tablename__ = 'user_ssh_keys' + __table_args__ = ( + Index('usk_ssh_key_fingerprint_idx', 'ssh_key_fingerprint'), + + UniqueConstraint('ssh_key_fingerprint'), + + base_table_args + ) + __mapper_args__ = {} + + ssh_key_id = Column('ssh_key_id', Integer(), nullable=False, unique=True, default=None, primary_key=True) + ssh_key_data = Column('ssh_key_data', String(10240), nullable=False, unique=None, default=None) + ssh_key_fingerprint = Column('ssh_key_fingerprint', String(255), nullable=False, unique=None, default=None) + + description = Column('description', UnicodeText().with_variant(UnicodeText(1024), 'mysql')) + + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + accessed_on = Column('accessed_on', DateTime(timezone=False), nullable=True, default=None) + user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None) + + user = relationship('User', lazy='joined') + + def __json__(self): + data = { + 'ssh_fingerprint': self.ssh_key_fingerprint, + 'description': self.description, + 'created_on': self.created_on + } + return data + + def get_api_data(self): + data = self.__json__() + return data + + +class UserLog(Base, BaseModel): + __tablename__ = 'user_logs' + __table_args__ = ( + base_table_args, + ) + + VERSION_1 = 'v1' + VERSION_2 = 'v2' + VERSIONS = [VERSION_1, VERSION_2] + + user_log_id = Column("user_log_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id',ondelete='SET NULL'), nullable=True, unique=None, default=None) + username = Column("username", String(255), nullable=True, unique=None, default=None) + repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id', ondelete='SET NULL'), nullable=True, unique=None, default=None) + repository_name = Column("repository_name", String(255), nullable=True, unique=None, default=None) + user_ip = Column("user_ip", String(255), nullable=True, unique=None, default=None) + action = Column("action", Text().with_variant(Text(1200000), 'mysql'), nullable=True, unique=None, default=None) + action_date = Column("action_date", DateTime(timezone=False), nullable=True, unique=None, default=None) + + version = Column("version", String(255), nullable=True, default=VERSION_1) + user_data = Column('user_data_json', MutationObj.as_mutable(JsonType(dialect_map=dict(mysql=LONGTEXT())))) + action_data = Column('action_data_json', MutationObj.as_mutable(JsonType(dialect_map=dict(mysql=LONGTEXT())))) + + def __unicode__(self): + return u"<%s('id:%s:%s')>" % ( + self.__class__.__name__, self.repository_name, self.action) + + def __json__(self): + return { + 'user_id': self.user_id, + 'username': self.username, + 'repository_id': self.repository_id, + 'repository_name': self.repository_name, + 'user_ip': self.user_ip, + 'action_date': self.action_date, + 'action': self.action, + } + + @hybrid_property + def entry_id(self): + return self.user_log_id + + @property + def action_as_day(self): + return datetime.date(*self.action_date.timetuple()[:3]) + + user = relationship('User') + repository = relationship('Repository', cascade='') + + +class UserGroup(Base, BaseModel): + __tablename__ = 'users_groups' + __table_args__ = ( + base_table_args, + ) + + users_group_id = Column("users_group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + users_group_name = Column("users_group_name", String(255), nullable=False, unique=True, default=None) + user_group_description = Column("user_group_description", String(10000), nullable=True, unique=None, default=None) + users_group_active = Column("users_group_active", Boolean(), nullable=True, unique=None, default=None) + inherit_default_permissions = Column("users_group_inherit_default_permissions", Boolean(), nullable=False, unique=None, default=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=False, default=None) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + _group_data = Column("group_data", LargeBinary(), nullable=True) # JSON data + + members = relationship('UserGroupMember', cascade="all, delete, delete-orphan", lazy="joined") + users_group_to_perm = relationship('UserGroupToPerm', cascade='all') + users_group_repo_to_perm = relationship('UserGroupRepoToPerm', cascade='all') + users_group_repo_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all') + user_user_group_to_perm = relationship('UserUserGroupToPerm', cascade='all') + user_group_user_group_to_perm = relationship('UserGroupUserGroupToPerm ', primaryjoin="UserGroupUserGroupToPerm.target_user_group_id==UserGroup.users_group_id", cascade='all') + + user_group_review_rules = relationship('RepoReviewRuleUserGroup', cascade='all') + user = relationship('User', primaryjoin="User.user_id==UserGroup.user_id") + + @classmethod + def _load_group_data(cls, column): + if not column: + return {} + + try: + return json.loads(column) or {} + except TypeError: + return {} + + @hybrid_property + def description_safe(self): + from rhodecode.lib import helpers as h + return h.escape(self.user_group_description) + + @hybrid_property + def group_data(self): + return self._load_group_data(self._group_data) + + @group_data.expression + def group_data(self, **kwargs): + return self._group_data + + @group_data.setter + def group_data(self, val): + try: + self._group_data = json.dumps(val) + except Exception: + log.error(traceback.format_exc()) + + @classmethod + def _load_sync(cls, group_data): + if group_data: + return group_data.get('extern_type') + + @property + def sync(self): + return self._load_sync(self.group_data) + + def __unicode__(self): + return u"<%s('id:%s:%s')>" % (self.__class__.__name__, + self.users_group_id, + self.users_group_name) + + @classmethod + def get_by_group_name(cls, group_name, cache=False, + case_insensitive=False): + if case_insensitive: + q = cls.query().filter(func.lower(cls.users_group_name) == + func.lower(group_name)) + + else: + q = cls.query().filter(cls.users_group_name == group_name) + if cache: + q = q.options( + FromCache("sql_cache_short", "get_group_%s" % _hash_key(group_name))) + return q.scalar() + + @classmethod + def get(cls, user_group_id, cache=False): + if not user_group_id: + return + + user_group = cls.query() + if cache: + user_group = user_group.options( + FromCache("sql_cache_short", "get_users_group_%s" % user_group_id)) + return user_group.get(user_group_id) + + def permissions(self, with_admins=True, with_owner=True): + """ + Permissions for user groups + """ + _admin_perm = 'usergroup.admin' + + owner_row = [] + if with_owner: + usr = AttributeDict(self.user.get_dict()) + usr.owner_row = True + usr.permission = _admin_perm + owner_row.append(usr) + + super_admin_ids = [] + super_admin_rows = [] + if with_admins: + for usr in User.get_all_super_admins(): + super_admin_ids.append(usr.user_id) + # if this admin is also owner, don't double the record + if usr.user_id == owner_row[0].user_id: + owner_row[0].admin_row = True + else: + usr = AttributeDict(usr.get_dict()) + usr.admin_row = True + usr.permission = _admin_perm + super_admin_rows.append(usr) + + q = UserUserGroupToPerm.query().filter(UserUserGroupToPerm.user_group == self) + q = q.options(joinedload(UserUserGroupToPerm.user_group), + joinedload(UserUserGroupToPerm.user), + joinedload(UserUserGroupToPerm.permission),) + + # get owners and admins and permissions. We do a trick of re-writing + # objects from sqlalchemy to named-tuples due to sqlalchemy session + # has a global reference and changing one object propagates to all + # others. This means if admin is also an owner admin_row that change + # would propagate to both objects + perm_rows = [] + for _usr in q.all(): + usr = AttributeDict(_usr.user.get_dict()) + # if this user is also owner/admin, mark as duplicate record + if usr.user_id == owner_row[0].user_id or usr.user_id in super_admin_ids: + usr.duplicate_perm = True + usr.permission = _usr.permission.permission_name + perm_rows.append(usr) + + # filter the perm rows by 'default' first and then sort them by + # admin,write,read,none permissions sorted again alphabetically in + # each group + perm_rows = sorted(perm_rows, key=display_user_sort) + + return super_admin_rows + owner_row + perm_rows + + def permission_user_groups(self): + q = UserGroupUserGroupToPerm.query().filter(UserGroupUserGroupToPerm.target_user_group == self) + q = q.options(joinedload(UserGroupUserGroupToPerm.user_group), + joinedload(UserGroupUserGroupToPerm.target_user_group), + joinedload(UserGroupUserGroupToPerm.permission),) + + perm_rows = [] + for _user_group in q.all(): + usr = AttributeDict(_user_group.user_group.get_dict()) + usr.permission = _user_group.permission.permission_name + perm_rows.append(usr) + + perm_rows = sorted(perm_rows, key=display_user_group_sort) + return perm_rows + + def _get_default_perms(self, user_group, suffix=''): + from rhodecode.model.permission import PermissionModel + return PermissionModel().get_default_perms(user_group.users_group_to_perm, suffix) + + def get_default_perms(self, suffix=''): + return self._get_default_perms(self, suffix) + + def get_api_data(self, with_group_members=True, include_secrets=False): + """ + :param include_secrets: See :meth:`User.get_api_data`, this parameter is + basically forwarded. + + """ + user_group = self + data = { + 'users_group_id': user_group.users_group_id, + 'group_name': user_group.users_group_name, + 'group_description': user_group.user_group_description, + 'active': user_group.users_group_active, + 'owner': user_group.user.username, + 'sync': user_group.sync, + 'owner_email': user_group.user.email, + } + + if with_group_members: + users = [] + for user in user_group.members: + user = user.user + users.append(user.get_api_data(include_secrets=include_secrets)) + data['users'] = users + + return data + + +class UserGroupMember(Base, BaseModel): + __tablename__ = 'users_groups_members' + __table_args__ = ( + base_table_args, + ) + + users_group_member_id = Column("users_group_member_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) + + user = relationship('User', lazy='joined') + users_group = relationship('UserGroup') + + def __init__(self, gr_id='', u_id=''): + self.users_group_id = gr_id + self.user_id = u_id + + +class RepositoryField(Base, BaseModel): + __tablename__ = 'repositories_fields' + __table_args__ = ( + UniqueConstraint('repository_id', 'field_key'), # no-multi field + base_table_args, + ) + + PREFIX = 'ex_' # prefix used in form to not conflict with already existing fields + + repo_field_id = Column("repo_field_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None) + field_key = Column("field_key", String(250)) + field_label = Column("field_label", String(1024), nullable=False) + field_value = Column("field_value", String(10000), nullable=False) + field_desc = Column("field_desc", String(1024), nullable=False) + field_type = Column("field_type", String(255), nullable=False, unique=None) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + + repository = relationship('Repository') + + @property + def field_key_prefixed(self): + return 'ex_%s' % self.field_key + + @classmethod + def un_prefix_key(cls, key): + if key.startswith(cls.PREFIX): + return key[len(cls.PREFIX):] + return key + + @classmethod + def get_by_key_name(cls, key, repo): + row = cls.query()\ + .filter(cls.repository == repo)\ + .filter(cls.field_key == key).scalar() + return row + + +class Repository(Base, BaseModel): + __tablename__ = 'repositories' + __table_args__ = ( + Index('r_repo_name_idx', 'repo_name', mysql_length=255), + base_table_args, + ) + DEFAULT_CLONE_URI = '{scheme}://{user}@{netloc}/{repo}' + DEFAULT_CLONE_URI_ID = '{scheme}://{user}@{netloc}/_{repoid}' + DEFAULT_CLONE_URI_SSH = 'ssh://{sys_user}@{hostname}/{repo}' + + STATE_CREATED = 'repo_state_created' + STATE_PENDING = 'repo_state_pending' + STATE_ERROR = 'repo_state_error' + + LOCK_AUTOMATIC = 'lock_auto' + LOCK_API = 'lock_api' + LOCK_WEB = 'lock_web' + LOCK_PULL = 'lock_pull' + + NAME_SEP = URL_SEP + + repo_id = Column( + "repo_id", Integer(), nullable=False, unique=True, default=None, + primary_key=True) + _repo_name = Column( + "repo_name", Text(), nullable=False, default=None) + _repo_name_hash = Column( + "repo_name_hash", String(255), nullable=False, unique=True) + repo_state = Column("repo_state", String(255), nullable=True) + + clone_uri = Column( + "clone_uri", EncryptedTextValue(), nullable=True, unique=False, + default=None) + push_uri = Column( + "push_uri", EncryptedTextValue(), nullable=True, unique=False, + default=None) + repo_type = Column( + "repo_type", String(255), nullable=False, unique=False, default=None) + user_id = Column( + "user_id", Integer(), ForeignKey('users.user_id'), nullable=False, + unique=False, default=None) + private = Column( + "private", Boolean(), nullable=True, unique=None, default=None) + archived = Column( + "archived", Boolean(), nullable=True, unique=None, default=None) + enable_statistics = Column( + "statistics", Boolean(), nullable=True, unique=None, default=True) + enable_downloads = Column( + "downloads", Boolean(), nullable=True, unique=None, default=True) + description = Column( + "description", String(10000), nullable=True, unique=None, default=None) + created_on = Column( + 'created_on', DateTime(timezone=False), nullable=True, unique=None, + default=datetime.datetime.now) + updated_on = Column( + 'updated_on', DateTime(timezone=False), nullable=True, unique=None, + default=datetime.datetime.now) + _landing_revision = Column( + "landing_revision", String(255), nullable=False, unique=False, + default=None) + enable_locking = Column( + "enable_locking", Boolean(), nullable=False, unique=None, + default=False) + _locked = Column( + "locked", String(255), nullable=True, unique=False, default=None) + _changeset_cache = Column( + "changeset_cache", LargeBinary(), nullable=True) # JSON data + + fork_id = Column( + "fork_id", Integer(), ForeignKey('repositories.repo_id'), + nullable=True, unique=False, default=None) + group_id = Column( + "group_id", Integer(), ForeignKey('groups.group_id'), nullable=True, + unique=False, default=None) + + user = relationship('User', lazy='joined') + fork = relationship('Repository', remote_side=repo_id, lazy='joined') + group = relationship('RepoGroup', lazy='joined') + repo_to_perm = relationship( + 'UserRepoToPerm', cascade='all', + order_by='UserRepoToPerm.repo_to_perm_id') + users_group_to_perm = relationship('UserGroupRepoToPerm', cascade='all') + stats = relationship('Statistics', cascade='all', uselist=False) + + followers = relationship( + 'UserFollowing', + primaryjoin='UserFollowing.follows_repo_id==Repository.repo_id', + cascade='all') + extra_fields = relationship( + 'RepositoryField', cascade="all, delete, delete-orphan") + logs = relationship('UserLog') + comments = relationship( + 'ChangesetComment', cascade="all, delete, delete-orphan") + pull_requests_source = relationship( + 'PullRequest', + primaryjoin='PullRequest.source_repo_id==Repository.repo_id', + cascade="all, delete, delete-orphan") + pull_requests_target = relationship( + 'PullRequest', + primaryjoin='PullRequest.target_repo_id==Repository.repo_id', + cascade="all, delete, delete-orphan") + ui = relationship('RepoRhodeCodeUi', cascade="all") + settings = relationship('RepoRhodeCodeSetting', cascade="all") + integrations = relationship('Integration', + cascade="all, delete, delete-orphan") + + scoped_tokens = relationship('UserApiKeys', cascade="all") + + def __unicode__(self): + return u"<%s('%s:%s')>" % (self.__class__.__name__, self.repo_id, + safe_unicode(self.repo_name)) + + @hybrid_property + def description_safe(self): + from rhodecode.lib import helpers as h + return h.escape(self.description) + + @hybrid_property + def landing_rev(self): + # always should return [rev_type, rev] + if self._landing_revision: + _rev_info = self._landing_revision.split(':') + if len(_rev_info) < 2: + _rev_info.insert(0, 'rev') + return [_rev_info[0], _rev_info[1]] + return [None, None] + + @landing_rev.setter + def landing_rev(self, val): + if ':' not in val: + raise ValueError('value must be delimited with `:` and consist ' + 'of :, got %s instead' % val) + self._landing_revision = val + + @hybrid_property + def locked(self): + if self._locked: + user_id, timelocked, reason = self._locked.split(':') + lock_values = int(user_id), timelocked, reason + else: + lock_values = [None, None, None] + return lock_values + + @locked.setter + def locked(self, val): + if val and isinstance(val, (list, tuple)): + self._locked = ':'.join(map(str, val)) + else: + self._locked = None + + @hybrid_property + def changeset_cache(self): + from rhodecode.lib.vcs.backends.base import EmptyCommit + dummy = EmptyCommit().__json__() + if not self._changeset_cache: + return dummy + try: + return json.loads(self._changeset_cache) + except TypeError: + return dummy + except Exception: + log.error(traceback.format_exc()) + return dummy + + @changeset_cache.setter + def changeset_cache(self, val): + try: + self._changeset_cache = json.dumps(val) + except Exception: + log.error(traceback.format_exc()) + + @hybrid_property + def repo_name(self): + return self._repo_name + + @repo_name.setter + def repo_name(self, value): + self._repo_name = value + self._repo_name_hash = hashlib.sha1(safe_str(value)).hexdigest() + + @classmethod + def normalize_repo_name(cls, repo_name): + """ + Normalizes os specific repo_name to the format internally stored inside + database using URL_SEP + + :param cls: + :param repo_name: + """ + return cls.NAME_SEP.join(repo_name.split(os.sep)) + + @classmethod + def get_by_repo_name(cls, repo_name, cache=False, identity_cache=False): + session = Session() + q = session.query(cls).filter(cls.repo_name == repo_name) + + if cache: + if identity_cache: + val = cls.identity_cache(session, 'repo_name', repo_name) + if val: + return val + else: + cache_key = "get_repo_by_name_%s" % _hash_key(repo_name) + q = q.options( + FromCache("sql_cache_short", cache_key)) + + return q.scalar() + + @classmethod + def get_by_id_or_repo_name(cls, repoid): + if isinstance(repoid, (int, long)): + try: + repo = cls.get(repoid) + except ValueError: + repo = None + else: + repo = cls.get_by_repo_name(repoid) + return repo + + @classmethod + def get_by_full_path(cls, repo_full_path): + repo_name = repo_full_path.split(cls.base_path(), 1)[-1] + repo_name = cls.normalize_repo_name(repo_name) + return cls.get_by_repo_name(repo_name.strip(URL_SEP)) + + @classmethod + def get_repo_forks(cls, repo_id): + return cls.query().filter(Repository.fork_id == repo_id) + + @classmethod + def base_path(cls): + """ + Returns base path when all repos are stored + + :param cls: + """ + q = Session().query(RhodeCodeUi)\ + .filter(RhodeCodeUi.ui_key == cls.NAME_SEP) + q = q.options(FromCache("sql_cache_short", "repository_repo_path")) + return q.one().ui_value + + @classmethod + def get_all_repos(cls, user_id=Optional(None), group_id=Optional(None), + case_insensitive=True, archived=False): + q = Repository.query() + + if not archived: + q = q.filter(Repository.archived.isnot(true())) + + if not isinstance(user_id, Optional): + q = q.filter(Repository.user_id == user_id) + + if not isinstance(group_id, Optional): + q = q.filter(Repository.group_id == group_id) + + if case_insensitive: + q = q.order_by(func.lower(Repository.repo_name)) + else: + q = q.order_by(Repository.repo_name) + + return q.all() + + @property + def forks(self): + """ + Return forks of this repo + """ + return Repository.get_repo_forks(self.repo_id) + + @property + def parent(self): + """ + Returns fork parent + """ + return self.fork + + @property + def just_name(self): + return self.repo_name.split(self.NAME_SEP)[-1] + + @property + def groups_with_parents(self): + groups = [] + if self.group is None: + return groups + + cur_gr = self.group + groups.insert(0, cur_gr) + while 1: + gr = getattr(cur_gr, 'parent_group', None) + cur_gr = cur_gr.parent_group + if gr is None: + break + groups.insert(0, gr) + + return groups + + @property + def groups_and_repo(self): + return self.groups_with_parents, self + + @LazyProperty + def repo_path(self): + """ + Returns base full path for that repository means where it actually + exists on a filesystem + """ + q = Session().query(RhodeCodeUi).filter( + RhodeCodeUi.ui_key == self.NAME_SEP) + q = q.options(FromCache("sql_cache_short", "repository_repo_path")) + return q.one().ui_value + + @property + def repo_full_path(self): + p = [self.repo_path] + # we need to split the name by / since this is how we store the + # names in the database, but that eventually needs to be converted + # into a valid system path + p += self.repo_name.split(self.NAME_SEP) + return os.path.join(*map(safe_unicode, p)) + + @property + def cache_keys(self): + """ + Returns associated cache keys for that repo + """ + invalidation_namespace = CacheKey.REPO_INVALIDATION_NAMESPACE.format( + repo_id=self.repo_id) + return CacheKey.query()\ + .filter(CacheKey.cache_args == invalidation_namespace)\ + .order_by(CacheKey.cache_key)\ + .all() + + @property + def cached_diffs_relative_dir(self): + """ + Return a relative to the repository store path of cached diffs + used for safe display for users, who shouldn't know the absolute store + path + """ + return os.path.join( + os.path.dirname(self.repo_name), + self.cached_diffs_dir.split(os.path.sep)[-1]) + + @property + def cached_diffs_dir(self): + path = self.repo_full_path + return os.path.join( + os.path.dirname(path), + '.__shadow_diff_cache_repo_{}'.format(self.repo_id)) + + def cached_diffs(self): + diff_cache_dir = self.cached_diffs_dir + if os.path.isdir(diff_cache_dir): + return os.listdir(diff_cache_dir) + return [] + + def shadow_repos(self): + shadow_repos_pattern = '.__shadow_repo_{}'.format(self.repo_id) + return [ + x for x in os.listdir(os.path.dirname(self.repo_full_path)) + if x.startswith(shadow_repos_pattern)] + + def get_new_name(self, repo_name): + """ + returns new full repository name based on assigned group and new new + + :param group_name: + """ + path_prefix = self.group.full_path_splitted if self.group else [] + return self.NAME_SEP.join(path_prefix + [repo_name]) + + @property + def _config(self): + """ + Returns db based config object. + """ + from rhodecode.lib.utils import make_db_config + return make_db_config(clear_session=False, repo=self) + + def permissions(self, with_admins=True, with_owner=True): + """ + Permissions for repositories + """ + _admin_perm = 'repository.admin' + + owner_row = [] + if with_owner: + usr = AttributeDict(self.user.get_dict()) + usr.owner_row = True + usr.permission = _admin_perm + usr.permission_id = None + owner_row.append(usr) + + super_admin_ids = [] + super_admin_rows = [] + if with_admins: + for usr in User.get_all_super_admins(): + super_admin_ids.append(usr.user_id) + # if this admin is also owner, don't double the record + if usr.user_id == owner_row[0].user_id: + owner_row[0].admin_row = True + else: + usr = AttributeDict(usr.get_dict()) + usr.admin_row = True + usr.permission = _admin_perm + usr.permission_id = None + super_admin_rows.append(usr) + + q = UserRepoToPerm.query().filter(UserRepoToPerm.repository == self) + q = q.options(joinedload(UserRepoToPerm.repository), + joinedload(UserRepoToPerm.user), + joinedload(UserRepoToPerm.permission),) + + # get owners and admins and permissions. We do a trick of re-writing + # objects from sqlalchemy to named-tuples due to sqlalchemy session + # has a global reference and changing one object propagates to all + # others. This means if admin is also an owner admin_row that change + # would propagate to both objects + perm_rows = [] + for _usr in q.all(): + usr = AttributeDict(_usr.user.get_dict()) + # if this user is also owner/admin, mark as duplicate record + if usr.user_id == owner_row[0].user_id or usr.user_id in super_admin_ids: + usr.duplicate_perm = True + # also check if this permission is maybe used by branch_permissions + if _usr.branch_perm_entry: + usr.branch_rules = [x.branch_rule_id for x in _usr.branch_perm_entry] + + usr.permission = _usr.permission.permission_name + usr.permission_id = _usr.repo_to_perm_id + perm_rows.append(usr) + + # filter the perm rows by 'default' first and then sort them by + # admin,write,read,none permissions sorted again alphabetically in + # each group + perm_rows = sorted(perm_rows, key=display_user_sort) + + return super_admin_rows + owner_row + perm_rows + + def permission_user_groups(self): + q = UserGroupRepoToPerm.query().filter( + UserGroupRepoToPerm.repository == self) + q = q.options(joinedload(UserGroupRepoToPerm.repository), + joinedload(UserGroupRepoToPerm.users_group), + joinedload(UserGroupRepoToPerm.permission),) + + perm_rows = [] + for _user_group in q.all(): + usr = AttributeDict(_user_group.users_group.get_dict()) + usr.permission = _user_group.permission.permission_name + perm_rows.append(usr) + + perm_rows = sorted(perm_rows, key=display_user_group_sort) + return perm_rows + + def get_api_data(self, include_secrets=False): + """ + Common function for generating repo api data + + :param include_secrets: See :meth:`User.get_api_data`. + + """ + # TODO: mikhail: Here there is an anti-pattern, we probably need to + # move this methods on models level. + from rhodecode.model.settings import SettingsModel + from rhodecode.model.repo import RepoModel + + repo = self + _user_id, _time, _reason = self.locked + + data = { + 'repo_id': repo.repo_id, + 'repo_name': repo.repo_name, + 'repo_type': repo.repo_type, + 'clone_uri': repo.clone_uri or '', + 'push_uri': repo.push_uri or '', + 'url': RepoModel().get_url(self), + 'private': repo.private, + 'created_on': repo.created_on, + 'description': repo.description_safe, + 'landing_rev': repo.landing_rev, + 'owner': repo.user.username, + 'fork_of': repo.fork.repo_name if repo.fork else None, + 'fork_of_id': repo.fork.repo_id if repo.fork else None, + 'enable_statistics': repo.enable_statistics, + 'enable_locking': repo.enable_locking, + 'enable_downloads': repo.enable_downloads, + 'last_changeset': repo.changeset_cache, + 'locked_by': User.get(_user_id).get_api_data( + include_secrets=include_secrets) if _user_id else None, + 'locked_date': time_to_datetime(_time) if _time else None, + 'lock_reason': _reason if _reason else None, + } + + # TODO: mikhail: should be per-repo settings here + rc_config = SettingsModel().get_all_settings() + repository_fields = str2bool( + rc_config.get('rhodecode_repository_fields')) + if repository_fields: + for f in self.extra_fields: + data[f.field_key_prefixed] = f.field_value + + return data + + @classmethod + def lock(cls, repo, user_id, lock_time=None, lock_reason=None): + if not lock_time: + lock_time = time.time() + if not lock_reason: + lock_reason = cls.LOCK_AUTOMATIC + repo.locked = [user_id, lock_time, lock_reason] + Session().add(repo) + Session().commit() + + @classmethod + def unlock(cls, repo): + repo.locked = None + Session().add(repo) + Session().commit() + + @classmethod + def getlock(cls, repo): + return repo.locked + + def is_user_lock(self, user_id): + if self.lock[0]: + lock_user_id = safe_int(self.lock[0]) + user_id = safe_int(user_id) + # both are ints, and they are equal + return all([lock_user_id, user_id]) and lock_user_id == user_id + + return False + + def get_locking_state(self, action, user_id, only_when_enabled=True): + """ + Checks locking on this repository, if locking is enabled and lock is + present returns a tuple of make_lock, locked, locked_by. + make_lock can have 3 states None (do nothing) True, make lock + False release lock, This value is later propagated to hooks, which + do the locking. Think about this as signals passed to hooks what to do. + + """ + # TODO: johbo: This is part of the business logic and should be moved + # into the RepositoryModel. + + if action not in ('push', 'pull'): + raise ValueError("Invalid action value: %s" % repr(action)) + + # defines if locked error should be thrown to user + currently_locked = False + # defines if new lock should be made, tri-state + make_lock = None + repo = self + user = User.get(user_id) + + lock_info = repo.locked + + if repo and (repo.enable_locking or not only_when_enabled): + if action == 'push': + # check if it's already locked !, if it is compare users + locked_by_user_id = lock_info[0] + if user.user_id == locked_by_user_id: + log.debug( + 'Got `push` action from user %s, now unlocking', user) + # unlock if we have push from user who locked + make_lock = False + else: + # we're not the same user who locked, ban with + # code defined in settings (default is 423 HTTP Locked) ! + log.debug('Repo %s is currently locked by %s', repo, user) + currently_locked = True + elif action == 'pull': + # [0] user [1] date + if lock_info[0] and lock_info[1]: + log.debug('Repo %s is currently locked by %s', repo, user) + currently_locked = True + else: + log.debug('Setting lock on repo %s by %s', repo, user) + make_lock = True + + else: + log.debug('Repository %s do not have locking enabled', repo) + + log.debug('FINAL locking values make_lock:%s,locked:%s,locked_by:%s', + make_lock, currently_locked, lock_info) + + from rhodecode.lib.auth import HasRepoPermissionAny + perm_check = HasRepoPermissionAny('repository.write', 'repository.admin') + if make_lock and not perm_check(repo_name=repo.repo_name, user=user): + # if we don't have at least write permission we cannot make a lock + log.debug('lock state reset back to FALSE due to lack ' + 'of at least read permission') + make_lock = False + + return make_lock, currently_locked, lock_info + + @property + def last_db_change(self): + return self.updated_on + + @property + def clone_uri_hidden(self): + clone_uri = self.clone_uri + if clone_uri: + import urlobject + url_obj = urlobject.URLObject(cleaned_uri(clone_uri)) + if url_obj.password: + clone_uri = url_obj.with_password('*****') + return clone_uri + + @property + def push_uri_hidden(self): + push_uri = self.push_uri + if push_uri: + import urlobject + url_obj = urlobject.URLObject(cleaned_uri(push_uri)) + if url_obj.password: + push_uri = url_obj.with_password('*****') + return push_uri + + def clone_url(self, **override): + from rhodecode.model.settings import SettingsModel + + uri_tmpl = None + if 'with_id' in override: + uri_tmpl = self.DEFAULT_CLONE_URI_ID + del override['with_id'] + + if 'uri_tmpl' in override: + uri_tmpl = override['uri_tmpl'] + del override['uri_tmpl'] + + ssh = False + if 'ssh' in override: + ssh = True + del override['ssh'] + + # we didn't override our tmpl from **overrides + if not uri_tmpl: + rc_config = SettingsModel().get_all_settings(cache=True) + if ssh: + uri_tmpl = rc_config.get( + 'rhodecode_clone_uri_ssh_tmpl') or self.DEFAULT_CLONE_URI_SSH + else: + uri_tmpl = rc_config.get( + 'rhodecode_clone_uri_tmpl') or self.DEFAULT_CLONE_URI + + request = get_current_request() + return get_clone_url(request=request, + uri_tmpl=uri_tmpl, + repo_name=self.repo_name, + repo_id=self.repo_id, **override) + + def set_state(self, state): + self.repo_state = state + Session().add(self) + #========================================================================== + # SCM PROPERTIES + #========================================================================== + + def get_commit(self, commit_id=None, commit_idx=None, pre_load=None): + return get_commit_safe( + self.scm_instance(), commit_id, commit_idx, pre_load=pre_load) + + def get_changeset(self, rev=None, pre_load=None): + warnings.warn("Use get_commit", DeprecationWarning) + commit_id = None + commit_idx = None + if isinstance(rev, compat.string_types): + commit_id = rev + else: + commit_idx = rev + return self.get_commit(commit_id=commit_id, commit_idx=commit_idx, + pre_load=pre_load) + + def get_landing_commit(self): + """ + Returns landing commit, or if that doesn't exist returns the tip + """ + _rev_type, _rev = self.landing_rev + commit = self.get_commit(_rev) + if isinstance(commit, EmptyCommit): + return self.get_commit() + return commit + + def update_commit_cache(self, cs_cache=None, config=None): + """ + Update cache of last changeset for repository, keys should be:: + + short_id + raw_id + revision + parents + message + date + author + + :param cs_cache: + """ + from rhodecode.lib.vcs.backends.base import BaseChangeset + if cs_cache is None: + # use no-cache version here + scm_repo = self.scm_instance(cache=False, config=config) + + empty = scm_repo.is_empty() + if not empty: + cs_cache = scm_repo.get_commit( + pre_load=["author", "date", "message", "parents"]) + else: + cs_cache = EmptyCommit() + + if isinstance(cs_cache, BaseChangeset): + cs_cache = cs_cache.__json__() + + def is_outdated(new_cs_cache): + if (new_cs_cache['raw_id'] != self.changeset_cache['raw_id'] or + new_cs_cache['revision'] != self.changeset_cache['revision']): + return True + return False + + # check if we have maybe already latest cached revision + if is_outdated(cs_cache) or not self.changeset_cache: + _default = datetime.datetime.utcnow() + last_change = cs_cache.get('date') or _default + if self.updated_on and self.updated_on > last_change: + # we check if last update is newer than the new value + # if yes, we use the current timestamp instead. Imagine you get + # old commit pushed 1y ago, we'd set last update 1y to ago. + last_change = _default + log.debug('updated repo %s with new cs cache %s', + self.repo_name, cs_cache) + self.updated_on = last_change + self.changeset_cache = cs_cache + Session().add(self) + Session().commit() + else: + log.debug('Skipping update_commit_cache for repo:`%s` ' + 'commit already with latest changes', self.repo_name) + + @property + def tip(self): + return self.get_commit('tip') + + @property + def author(self): + return self.tip.author + + @property + def last_change(self): + return self.scm_instance().last_change + + def get_comments(self, revisions=None): + """ + Returns comments for this repository grouped by revisions + + :param revisions: filter query by revisions only + """ + cmts = ChangesetComment.query()\ + .filter(ChangesetComment.repo == self) + if revisions: + cmts = cmts.filter(ChangesetComment.revision.in_(revisions)) + grouped = collections.defaultdict(list) + for cmt in cmts.all(): + grouped[cmt.revision].append(cmt) + return grouped + + def statuses(self, revisions=None): + """ + Returns statuses for this repository + + :param revisions: list of revisions to get statuses for + """ + statuses = ChangesetStatus.query()\ + .filter(ChangesetStatus.repo == self)\ + .filter(ChangesetStatus.version == 0) + + if revisions: + # Try doing the filtering in chunks to avoid hitting limits + size = 500 + status_results = [] + for chunk in xrange(0, len(revisions), size): + status_results += statuses.filter( + ChangesetStatus.revision.in_( + revisions[chunk: chunk+size]) + ).all() + else: + status_results = statuses.all() + + grouped = {} + + # maybe we have open new pullrequest without a status? + stat = ChangesetStatus.STATUS_UNDER_REVIEW + status_lbl = ChangesetStatus.get_status_lbl(stat) + for pr in PullRequest.query().filter(PullRequest.source_repo == self).all(): + for rev in pr.revisions: + pr_id = pr.pull_request_id + pr_repo = pr.target_repo.repo_name + grouped[rev] = [stat, status_lbl, pr_id, pr_repo] + + for stat in status_results: + pr_id = pr_repo = None + if stat.pull_request: + pr_id = stat.pull_request.pull_request_id + pr_repo = stat.pull_request.target_repo.repo_name + grouped[stat.revision] = [str(stat.status), stat.status_lbl, + pr_id, pr_repo] + return grouped + + # ========================================================================== + # SCM CACHE INSTANCE + # ========================================================================== + + def scm_instance(self, **kwargs): + import rhodecode + + # Passing a config will not hit the cache currently only used + # for repo2dbmapper + config = kwargs.pop('config', None) + cache = kwargs.pop('cache', None) + full_cache = str2bool(rhodecode.CONFIG.get('vcs_full_cache')) + # if cache is NOT defined use default global, else we have a full + # control over cache behaviour + if cache is None and full_cache and not config: + return self._get_instance_cached() + return self._get_instance(cache=bool(cache), config=config) + + def _get_instance_cached(self): + from rhodecode.lib import rc_cache + + cache_namespace_uid = 'cache_repo_instance.{}'.format(self.repo_id) + invalidation_namespace = CacheKey.REPO_INVALIDATION_NAMESPACE.format( + repo_id=self.repo_id) + region = rc_cache.get_or_create_region('cache_repo_longterm', cache_namespace_uid) + + @region.conditional_cache_on_arguments(namespace=cache_namespace_uid) + def get_instance_cached(repo_id, context_id): + return self._get_instance() + + # we must use thread scoped cache here, + # because each thread of gevent needs it's own not shared connection and cache + # we also alter `args` so the cache key is individual for every green thread. + inv_context_manager = rc_cache.InvalidationContext( + uid=cache_namespace_uid, invalidation_namespace=invalidation_namespace, + thread_scoped=True) + with inv_context_manager as invalidation_context: + args = (self.repo_id, inv_context_manager.cache_key) + # re-compute and store cache if we get invalidate signal + if invalidation_context.should_invalidate(): + instance = get_instance_cached.refresh(*args) + else: + instance = get_instance_cached(*args) + + log.debug( + 'Repo instance fetched in %.3fs', inv_context_manager.compute_time) + return instance + + def _get_instance(self, cache=True, config=None): + config = config or self._config + custom_wire = { + 'cache': cache # controls the vcs.remote cache + } + repo = get_vcs_instance( + repo_path=safe_str(self.repo_full_path), + config=config, + with_wire=custom_wire, + create=False, + _vcs_alias=self.repo_type) + + return repo + + def __json__(self): + return {'landing_rev': self.landing_rev} + + def get_dict(self): + + # Since we transformed `repo_name` to a hybrid property, we need to + # keep compatibility with the code which uses `repo_name` field. + + result = super(Repository, self).get_dict() + result['repo_name'] = result.pop('_repo_name', None) + return result + + +class RepoGroup(Base, BaseModel): + __tablename__ = 'groups' + __table_args__ = ( + UniqueConstraint('group_name', 'group_parent_id'), + base_table_args, + ) + __mapper_args__ = {'order_by': 'group_name'} + + CHOICES_SEPARATOR = '/' # used to generate select2 choices for nested groups + + group_id = Column("group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + group_name = Column("group_name", String(255), nullable=False, unique=True, default=None) + group_parent_id = Column("group_parent_id", Integer(), ForeignKey('groups.group_id'), nullable=True, unique=None, default=None) + group_description = Column("group_description", String(10000), nullable=True, unique=None, default=None) + enable_locking = Column("enable_locking", Boolean(), nullable=False, unique=None, default=False) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=False, default=None) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + updated_on = Column('updated_on', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now) + personal = Column('personal', Boolean(), nullable=True, unique=None, default=None) + + repo_group_to_perm = relationship('UserRepoGroupToPerm', cascade='all', order_by='UserRepoGroupToPerm.group_to_perm_id') + users_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all') + parent_group = relationship('RepoGroup', remote_side=group_id) + user = relationship('User') + integrations = relationship('Integration', + cascade="all, delete, delete-orphan") + + def __init__(self, group_name='', parent_group=None): + self.group_name = group_name + self.parent_group = parent_group + + def __unicode__(self): + return u"<%s('id:%s:%s')>" % ( + self.__class__.__name__, self.group_id, self.group_name) + + @hybrid_property + def description_safe(self): + from rhodecode.lib import helpers as h + return h.escape(self.group_description) + + @classmethod + def _generate_choice(cls, repo_group): + from webhelpers.html import literal as _literal + _name = lambda k: _literal(cls.CHOICES_SEPARATOR.join(k)) + return repo_group.group_id, _name(repo_group.full_path_splitted) + + @classmethod + def groups_choices(cls, groups=None, show_empty_group=True): + if not groups: + groups = cls.query().all() + + repo_groups = [] + if show_empty_group: + repo_groups = [(-1, u'-- %s --' % _('No parent'))] + + repo_groups.extend([cls._generate_choice(x) for x in groups]) + + repo_groups = sorted( + repo_groups, key=lambda t: t[1].split(cls.CHOICES_SEPARATOR)[0]) + return repo_groups + + @classmethod + def url_sep(cls): + return URL_SEP + + @classmethod + def get_by_group_name(cls, group_name, cache=False, case_insensitive=False): + if case_insensitive: + gr = cls.query().filter(func.lower(cls.group_name) + == func.lower(group_name)) + else: + gr = cls.query().filter(cls.group_name == group_name) + if cache: + name_key = _hash_key(group_name) + gr = gr.options( + FromCache("sql_cache_short", "get_group_%s" % name_key)) + return gr.scalar() + + @classmethod + def get_user_personal_repo_group(cls, user_id): + user = User.get(user_id) + if user.username == User.DEFAULT_USER: + return None + + return cls.query()\ + .filter(cls.personal == true()) \ + .filter(cls.user == user) \ + .order_by(cls.group_id.asc()) \ + .first() + + @classmethod + def get_all_repo_groups(cls, user_id=Optional(None), group_id=Optional(None), + case_insensitive=True): + q = RepoGroup.query() + + if not isinstance(user_id, Optional): + q = q.filter(RepoGroup.user_id == user_id) + + if not isinstance(group_id, Optional): + q = q.filter(RepoGroup.group_parent_id == group_id) + + if case_insensitive: + q = q.order_by(func.lower(RepoGroup.group_name)) + else: + q = q.order_by(RepoGroup.group_name) + return q.all() + + @property + def parents(self): + parents_recursion_limit = 10 + groups = [] + if self.parent_group is None: + return groups + cur_gr = self.parent_group + groups.insert(0, cur_gr) + cnt = 0 + while 1: + cnt += 1 + gr = getattr(cur_gr, 'parent_group', None) + cur_gr = cur_gr.parent_group + if gr is None: + break + if cnt == parents_recursion_limit: + # this will prevent accidental infinit loops + log.error('more than %s parents found for group %s, stopping ' + 'recursive parent fetching', parents_recursion_limit, self) + break + + groups.insert(0, gr) + return groups + + @property + def last_db_change(self): + return self.updated_on + + @property + def children(self): + return RepoGroup.query().filter(RepoGroup.parent_group == self) + + @property + def name(self): + return self.group_name.split(RepoGroup.url_sep())[-1] + + @property + def full_path(self): + return self.group_name + + @property + def full_path_splitted(self): + return self.group_name.split(RepoGroup.url_sep()) + + @property + def repositories(self): + return Repository.query()\ + .filter(Repository.group == self)\ + .order_by(Repository.repo_name) + + @property + def repositories_recursive_count(self): + cnt = self.repositories.count() + + def children_count(group): + cnt = 0 + for child in group.children: + cnt += child.repositories.count() + cnt += children_count(child) + return cnt + + return cnt + children_count(self) + + def _recursive_objects(self, include_repos=True): + all_ = [] + + def _get_members(root_gr): + if include_repos: + for r in root_gr.repositories: + all_.append(r) + childs = root_gr.children.all() + if childs: + for gr in childs: + all_.append(gr) + _get_members(gr) + + _get_members(self) + return [self] + all_ + + def recursive_groups_and_repos(self): + """ + Recursive return all groups, with repositories in those groups + """ + return self._recursive_objects() + + def recursive_groups(self): + """ + Returns all children groups for this group including children of children + """ + return self._recursive_objects(include_repos=False) + + def get_new_name(self, group_name): + """ + returns new full group name based on parent and new name + + :param group_name: + """ + path_prefix = (self.parent_group.full_path_splitted if + self.parent_group else []) + return RepoGroup.url_sep().join(path_prefix + [group_name]) + + def permissions(self, with_admins=True, with_owner=True): + """ + Permissions for repository groups + """ + _admin_perm = 'group.admin' + + owner_row = [] + if with_owner: + usr = AttributeDict(self.user.get_dict()) + usr.owner_row = True + usr.permission = _admin_perm + owner_row.append(usr) + + super_admin_ids = [] + super_admin_rows = [] + if with_admins: + for usr in User.get_all_super_admins(): + super_admin_ids.append(usr.user_id) + # if this admin is also owner, don't double the record + if usr.user_id == owner_row[0].user_id: + owner_row[0].admin_row = True + else: + usr = AttributeDict(usr.get_dict()) + usr.admin_row = True + usr.permission = _admin_perm + super_admin_rows.append(usr) + + q = UserRepoGroupToPerm.query().filter(UserRepoGroupToPerm.group == self) + q = q.options(joinedload(UserRepoGroupToPerm.group), + joinedload(UserRepoGroupToPerm.user), + joinedload(UserRepoGroupToPerm.permission),) + + # get owners and admins and permissions. We do a trick of re-writing + # objects from sqlalchemy to named-tuples due to sqlalchemy session + # has a global reference and changing one object propagates to all + # others. This means if admin is also an owner admin_row that change + # would propagate to both objects + perm_rows = [] + for _usr in q.all(): + usr = AttributeDict(_usr.user.get_dict()) + # if this user is also owner/admin, mark as duplicate record + if usr.user_id == owner_row[0].user_id or usr.user_id in super_admin_ids: + usr.duplicate_perm = True + usr.permission = _usr.permission.permission_name + perm_rows.append(usr) + + # filter the perm rows by 'default' first and then sort them by + # admin,write,read,none permissions sorted again alphabetically in + # each group + perm_rows = sorted(perm_rows, key=display_user_sort) + + return super_admin_rows + owner_row + perm_rows + + def permission_user_groups(self): + q = UserGroupRepoGroupToPerm.query().filter( + UserGroupRepoGroupToPerm.group == self) + q = q.options(joinedload(UserGroupRepoGroupToPerm.group), + joinedload(UserGroupRepoGroupToPerm.users_group), + joinedload(UserGroupRepoGroupToPerm.permission),) + + perm_rows = [] + for _user_group in q.all(): + usr = AttributeDict(_user_group.users_group.get_dict()) + usr.permission = _user_group.permission.permission_name + perm_rows.append(usr) + + perm_rows = sorted(perm_rows, key=display_user_group_sort) + return perm_rows + + def get_api_data(self): + """ + Common function for generating api data + + """ + group = self + data = { + 'group_id': group.group_id, + 'group_name': group.group_name, + 'group_description': group.description_safe, + 'parent_group': group.parent_group.group_name if group.parent_group else None, + 'repositories': [x.repo_name for x in group.repositories], + 'owner': group.user.username, + } + return data + + +class Permission(Base, BaseModel): + __tablename__ = 'permissions' + __table_args__ = ( + Index('p_perm_name_idx', 'permission_name'), + base_table_args, + ) + + PERMS = [ + ('hg.admin', _('RhodeCode Super Administrator')), + + ('repository.none', _('Repository no access')), + ('repository.read', _('Repository read access')), + ('repository.write', _('Repository write access')), + ('repository.admin', _('Repository admin access')), + + ('group.none', _('Repository group no access')), + ('group.read', _('Repository group read access')), + ('group.write', _('Repository group write access')), + ('group.admin', _('Repository group admin access')), + + ('usergroup.none', _('User group no access')), + ('usergroup.read', _('User group read access')), + ('usergroup.write', _('User group write access')), + ('usergroup.admin', _('User group admin access')), + + ('branch.none', _('Branch no permissions')), + ('branch.merge', _('Branch access by web merge')), + ('branch.push', _('Branch access by push')), + ('branch.push_force', _('Branch access by push with force')), + + ('hg.repogroup.create.false', _('Repository Group creation disabled')), + ('hg.repogroup.create.true', _('Repository Group creation enabled')), + + ('hg.usergroup.create.false', _('User Group creation disabled')), + ('hg.usergroup.create.true', _('User Group creation enabled')), + + ('hg.create.none', _('Repository creation disabled')), + ('hg.create.repository', _('Repository creation enabled')), + ('hg.create.write_on_repogroup.true', _('Repository creation enabled with write permission to a repository group')), + ('hg.create.write_on_repogroup.false', _('Repository creation disabled with write permission to a repository group')), + + ('hg.fork.none', _('Repository forking disabled')), + ('hg.fork.repository', _('Repository forking enabled')), + + ('hg.register.none', _('Registration disabled')), + ('hg.register.manual_activate', _('User Registration with manual account activation')), + ('hg.register.auto_activate', _('User Registration with automatic account activation')), + + ('hg.password_reset.enabled', _('Password reset enabled')), + ('hg.password_reset.hidden', _('Password reset hidden')), + ('hg.password_reset.disabled', _('Password reset disabled')), + + ('hg.extern_activate.manual', _('Manual activation of external account')), + ('hg.extern_activate.auto', _('Automatic activation of external account')), + + ('hg.inherit_default_perms.false', _('Inherit object permissions from default user disabled')), + ('hg.inherit_default_perms.true', _('Inherit object permissions from default user enabled')), + ] + + # definition of system default permissions for DEFAULT user, created on + # system setup + DEFAULT_USER_PERMISSIONS = [ + # object perms + 'repository.read', + 'group.read', + 'usergroup.read', + # branch, for backward compat we need same value as before so forced pushed + 'branch.push_force', + # global + 'hg.create.repository', + 'hg.repogroup.create.false', + 'hg.usergroup.create.false', + 'hg.create.write_on_repogroup.true', + 'hg.fork.repository', + 'hg.register.manual_activate', + 'hg.password_reset.enabled', + 'hg.extern_activate.auto', + 'hg.inherit_default_perms.true', + ] + + # defines which permissions are more important higher the more important + # Weight defines which permissions are more important. + # The higher number the more important. + PERM_WEIGHTS = { + 'repository.none': 0, + 'repository.read': 1, + 'repository.write': 3, + 'repository.admin': 4, + + 'group.none': 0, + 'group.read': 1, + 'group.write': 3, + 'group.admin': 4, + + 'usergroup.none': 0, + 'usergroup.read': 1, + 'usergroup.write': 3, + 'usergroup.admin': 4, + + 'branch.none': 0, + 'branch.merge': 1, + 'branch.push': 3, + 'branch.push_force': 4, + + 'hg.repogroup.create.false': 0, + 'hg.repogroup.create.true': 1, + + 'hg.usergroup.create.false': 0, + 'hg.usergroup.create.true': 1, + + 'hg.fork.none': 0, + 'hg.fork.repository': 1, + 'hg.create.none': 0, + 'hg.create.repository': 1 + } + + permission_id = Column("permission_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + permission_name = Column("permission_name", String(255), nullable=True, unique=None, default=None) + permission_longname = Column("permission_longname", String(255), nullable=True, unique=None, default=None) + + def __unicode__(self): + return u"<%s('%s:%s')>" % ( + self.__class__.__name__, self.permission_id, self.permission_name + ) + + @classmethod + def get_by_key(cls, key): + return cls.query().filter(cls.permission_name == key).scalar() + + @classmethod + def get_default_repo_perms(cls, user_id, repo_id=None): + q = Session().query(UserRepoToPerm, Repository, Permission)\ + .join((Permission, UserRepoToPerm.permission_id == Permission.permission_id))\ + .join((Repository, UserRepoToPerm.repository_id == Repository.repo_id))\ + .filter(UserRepoToPerm.user_id == user_id) + if repo_id: + q = q.filter(UserRepoToPerm.repository_id == repo_id) + return q.all() + + @classmethod + def get_default_repo_branch_perms(cls, user_id, repo_id=None): + q = Session().query(UserToRepoBranchPermission, UserRepoToPerm, Permission) \ + .join( + Permission, + UserToRepoBranchPermission.permission_id == Permission.permission_id) \ + .join( + UserRepoToPerm, + UserToRepoBranchPermission.rule_to_perm_id == UserRepoToPerm.repo_to_perm_id) \ + .filter(UserRepoToPerm.user_id == user_id) + + if repo_id: + q = q.filter(UserToRepoBranchPermission.repository_id == repo_id) + return q.order_by(UserToRepoBranchPermission.rule_order).all() + + @classmethod + def get_default_repo_perms_from_user_group(cls, user_id, repo_id=None): + q = Session().query(UserGroupRepoToPerm, Repository, Permission)\ + .join( + Permission, + UserGroupRepoToPerm.permission_id == Permission.permission_id)\ + .join( + Repository, + UserGroupRepoToPerm.repository_id == Repository.repo_id)\ + .join( + UserGroup, + UserGroupRepoToPerm.users_group_id == + UserGroup.users_group_id)\ + .join( + UserGroupMember, + UserGroupRepoToPerm.users_group_id == + UserGroupMember.users_group_id)\ + .filter( + UserGroupMember.user_id == user_id, + UserGroup.users_group_active == true()) + if repo_id: + q = q.filter(UserGroupRepoToPerm.repository_id == repo_id) + return q.all() + + @classmethod + def get_default_repo_branch_perms_from_user_group(cls, user_id, repo_id=None): + q = Session().query(UserGroupToRepoBranchPermission, UserGroupRepoToPerm, Permission) \ + .join( + Permission, + UserGroupToRepoBranchPermission.permission_id == Permission.permission_id) \ + .join( + UserGroupRepoToPerm, + UserGroupToRepoBranchPermission.rule_to_perm_id == UserGroupRepoToPerm.users_group_to_perm_id) \ + .join( + UserGroup, + UserGroupRepoToPerm.users_group_id == UserGroup.users_group_id) \ + .join( + UserGroupMember, + UserGroupRepoToPerm.users_group_id == UserGroupMember.users_group_id) \ + .filter( + UserGroupMember.user_id == user_id, + UserGroup.users_group_active == true()) + + if repo_id: + q = q.filter(UserGroupToRepoBranchPermission.repository_id == repo_id) + return q.order_by(UserGroupToRepoBranchPermission.rule_order).all() + + @classmethod + def get_default_group_perms(cls, user_id, repo_group_id=None): + q = Session().query(UserRepoGroupToPerm, RepoGroup, Permission)\ + .join( + Permission, + UserRepoGroupToPerm.permission_id == Permission.permission_id)\ + .join( + RepoGroup, + UserRepoGroupToPerm.group_id == RepoGroup.group_id)\ + .filter(UserRepoGroupToPerm.user_id == user_id) + if repo_group_id: + q = q.filter(UserRepoGroupToPerm.group_id == repo_group_id) + return q.all() + + @classmethod + def get_default_group_perms_from_user_group( + cls, user_id, repo_group_id=None): + q = Session().query(UserGroupRepoGroupToPerm, RepoGroup, Permission)\ + .join( + Permission, + UserGroupRepoGroupToPerm.permission_id == + Permission.permission_id)\ + .join( + RepoGroup, + UserGroupRepoGroupToPerm.group_id == RepoGroup.group_id)\ + .join( + UserGroup, + UserGroupRepoGroupToPerm.users_group_id == + UserGroup.users_group_id)\ + .join( + UserGroupMember, + UserGroupRepoGroupToPerm.users_group_id == + UserGroupMember.users_group_id)\ + .filter( + UserGroupMember.user_id == user_id, + UserGroup.users_group_active == true()) + if repo_group_id: + q = q.filter(UserGroupRepoGroupToPerm.group_id == repo_group_id) + return q.all() + + @classmethod + def get_default_user_group_perms(cls, user_id, user_group_id=None): + q = Session().query(UserUserGroupToPerm, UserGroup, Permission)\ + .join((Permission, UserUserGroupToPerm.permission_id == Permission.permission_id))\ + .join((UserGroup, UserUserGroupToPerm.user_group_id == UserGroup.users_group_id))\ + .filter(UserUserGroupToPerm.user_id == user_id) + if user_group_id: + q = q.filter(UserUserGroupToPerm.user_group_id == user_group_id) + return q.all() + + @classmethod + def get_default_user_group_perms_from_user_group( + cls, user_id, user_group_id=None): + TargetUserGroup = aliased(UserGroup, name='target_user_group') + q = Session().query(UserGroupUserGroupToPerm, UserGroup, Permission)\ + .join( + Permission, + UserGroupUserGroupToPerm.permission_id == + Permission.permission_id)\ + .join( + TargetUserGroup, + UserGroupUserGroupToPerm.target_user_group_id == + TargetUserGroup.users_group_id)\ + .join( + UserGroup, + UserGroupUserGroupToPerm.user_group_id == + UserGroup.users_group_id)\ + .join( + UserGroupMember, + UserGroupUserGroupToPerm.user_group_id == + UserGroupMember.users_group_id)\ + .filter( + UserGroupMember.user_id == user_id, + UserGroup.users_group_active == true()) + if user_group_id: + q = q.filter( + UserGroupUserGroupToPerm.user_group_id == user_group_id) + + return q.all() + + +class UserRepoToPerm(Base, BaseModel): + __tablename__ = 'repo_to_perm' + __table_args__ = ( + UniqueConstraint('user_id', 'repository_id', 'permission_id'), + base_table_args + ) + + repo_to_perm_id = Column("repo_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None) + + user = relationship('User') + repository = relationship('Repository') + permission = relationship('Permission') + + branch_perm_entry = relationship('UserToRepoBranchPermission', cascade="all, delete, delete-orphan", lazy='joined') + + @classmethod + def create(cls, user, repository, permission): + n = cls() + n.user = user + n.repository = repository + n.permission = permission + Session().add(n) + return n + + def __unicode__(self): + return u'<%s => %s >' % (self.user, self.repository) + + +class UserUserGroupToPerm(Base, BaseModel): + __tablename__ = 'user_user_group_to_perm' + __table_args__ = ( + UniqueConstraint('user_id', 'user_group_id', 'permission_id'), + base_table_args + ) + + user_user_group_to_perm_id = Column("user_user_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + user_group_id = Column("user_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) + + user = relationship('User') + user_group = relationship('UserGroup') + permission = relationship('Permission') + + @classmethod + def create(cls, user, user_group, permission): + n = cls() + n.user = user + n.user_group = user_group + n.permission = permission + Session().add(n) + return n + + def __unicode__(self): + return u'<%s => %s >' % (self.user, self.user_group) + + +class UserToPerm(Base, BaseModel): + __tablename__ = 'user_to_perm' + __table_args__ = ( + UniqueConstraint('user_id', 'permission_id'), + base_table_args + ) + + user_to_perm_id = Column("user_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + + user = relationship('User') + permission = relationship('Permission', lazy='joined') + + def __unicode__(self): + return u'<%s => %s >' % (self.user, self.permission) + + +class UserGroupRepoToPerm(Base, BaseModel): + __tablename__ = 'users_group_repo_to_perm' + __table_args__ = ( + UniqueConstraint('repository_id', 'users_group_id', 'permission_id'), + base_table_args + ) + + users_group_to_perm_id = Column("users_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None) + + users_group = relationship('UserGroup') + permission = relationship('Permission') + repository = relationship('Repository') + user_group_branch_perms = relationship('UserGroupToRepoBranchPermission', cascade='all') + + @classmethod + def create(cls, users_group, repository, permission): + n = cls() + n.users_group = users_group + n.repository = repository + n.permission = permission + Session().add(n) + return n + + def __unicode__(self): + return u' %s >' % (self.users_group, self.repository) + + +class UserGroupUserGroupToPerm(Base, BaseModel): + __tablename__ = 'user_group_user_group_to_perm' + __table_args__ = ( + UniqueConstraint('target_user_group_id', 'user_group_id', 'permission_id'), + CheckConstraint('target_user_group_id != user_group_id'), + base_table_args + ) + + user_group_user_group_to_perm_id = Column("user_group_user_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + target_user_group_id = Column("target_user_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + user_group_id = Column("user_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) + + target_user_group = relationship('UserGroup', primaryjoin='UserGroupUserGroupToPerm.target_user_group_id==UserGroup.users_group_id') + user_group = relationship('UserGroup', primaryjoin='UserGroupUserGroupToPerm.user_group_id==UserGroup.users_group_id') + permission = relationship('Permission') + + @classmethod + def create(cls, target_user_group, user_group, permission): + n = cls() + n.target_user_group = target_user_group + n.user_group = user_group + n.permission = permission + Session().add(n) + return n + + def __unicode__(self): + return u' %s >' % (self.target_user_group, self.user_group) + + +class UserGroupToPerm(Base, BaseModel): + __tablename__ = 'users_group_to_perm' + __table_args__ = ( + UniqueConstraint('users_group_id', 'permission_id',), + base_table_args + ) + + users_group_to_perm_id = Column("users_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + + users_group = relationship('UserGroup') + permission = relationship('Permission') + + +class UserRepoGroupToPerm(Base, BaseModel): + __tablename__ = 'user_repo_group_to_perm' + __table_args__ = ( + UniqueConstraint('user_id', 'group_id', 'permission_id'), + base_table_args + ) + + group_to_perm_id = Column("group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) + group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + + user = relationship('User') + group = relationship('RepoGroup') + permission = relationship('Permission') + + @classmethod + def create(cls, user, repository_group, permission): + n = cls() + n.user = user + n.group = repository_group + n.permission = permission + Session().add(n) + return n + + +class UserGroupRepoGroupToPerm(Base, BaseModel): + __tablename__ = 'users_group_repo_group_to_perm' + __table_args__ = ( + UniqueConstraint('users_group_id', 'group_id'), + base_table_args + ) + + users_group_repo_group_to_perm_id = Column("users_group_repo_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) + group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + + users_group = relationship('UserGroup') + permission = relationship('Permission') + group = relationship('RepoGroup') + + @classmethod + def create(cls, user_group, repository_group, permission): + n = cls() + n.users_group = user_group + n.group = repository_group + n.permission = permission + Session().add(n) + return n + + def __unicode__(self): + return u' %s >' % (self.users_group, self.group) + + +class Statistics(Base, BaseModel): + __tablename__ = 'statistics' + __table_args__ = ( + base_table_args + ) + + stat_id = Column("stat_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=True, default=None) + stat_on_revision = Column("stat_on_revision", Integer(), nullable=False) + commit_activity = Column("commit_activity", LargeBinary(1000000), nullable=False)#JSON data + commit_activity_combined = Column("commit_activity_combined", LargeBinary(), nullable=False)#JSON data + languages = Column("languages", LargeBinary(1000000), nullable=False)#JSON data + + repository = relationship('Repository', single_parent=True) + + +class UserFollowing(Base, BaseModel): + __tablename__ = 'user_followings' + __table_args__ = ( + UniqueConstraint('user_id', 'follows_repository_id'), + UniqueConstraint('user_id', 'follows_user_id'), + base_table_args + ) + + user_following_id = Column("user_following_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) + follows_repo_id = Column("follows_repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=True, unique=None, default=None) + follows_user_id = Column("follows_user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None) + follows_from = Column('follows_from', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now) + + user = relationship('User', primaryjoin='User.user_id==UserFollowing.user_id') + + follows_user = relationship('User', primaryjoin='User.user_id==UserFollowing.follows_user_id') + follows_repository = relationship('Repository', order_by='Repository.repo_name') + + @classmethod + def get_repo_followers(cls, repo_id): + return cls.query().filter(cls.follows_repo_id == repo_id) + + +class CacheKey(Base, BaseModel): + __tablename__ = 'cache_invalidation' + __table_args__ = ( + UniqueConstraint('cache_key'), + Index('key_idx', 'cache_key'), + base_table_args, + ) + + CACHE_TYPE_FEED = 'FEED' + CACHE_TYPE_README = 'README' + # namespaces used to register process/thread aware caches + REPO_INVALIDATION_NAMESPACE = 'repo_cache:{repo_id}' + SETTINGS_INVALIDATION_NAMESPACE = 'system_settings' + + cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + cache_key = Column("cache_key", String(255), nullable=True, unique=None, default=None) + cache_args = Column("cache_args", String(255), nullable=True, unique=None, default=None) + cache_active = Column("cache_active", Boolean(), nullable=True, unique=None, default=False) + + def __init__(self, cache_key, cache_args=''): + self.cache_key = cache_key + self.cache_args = cache_args + self.cache_active = False + + def __unicode__(self): + return u"<%s('%s:%s[%s]')>" % ( + self.__class__.__name__, + self.cache_id, self.cache_key, self.cache_active) + + def _cache_key_partition(self): + prefix, repo_name, suffix = self.cache_key.partition(self.cache_args) + return prefix, repo_name, suffix + + def get_prefix(self): + """ + Try to extract prefix from existing cache key. The key could consist + of prefix, repo_name, suffix + """ + # this returns prefix, repo_name, suffix + return self._cache_key_partition()[0] + + def get_suffix(self): + """ + get suffix that might have been used in _get_cache_key to + generate self.cache_key. Only used for informational purposes + in repo_edit.mako. + """ + # prefix, repo_name, suffix + return self._cache_key_partition()[2] + + @classmethod + def delete_all_cache(cls): + """ + Delete all cache keys from database. + Should only be run when all instances are down and all entries + thus stale. + """ + cls.query().delete() + Session().commit() + + @classmethod + def set_invalidate(cls, cache_uid, delete=False): + """ + Mark all caches of a repo as invalid in the database. + """ + + try: + qry = Session().query(cls).filter(cls.cache_args == cache_uid) + if delete: + qry.delete() + log.debug('cache objects deleted for cache args %s', + safe_str(cache_uid)) + else: + qry.update({"cache_active": False}) + log.debug('cache objects marked as invalid for cache args %s', + safe_str(cache_uid)) + + Session().commit() + except Exception: + log.exception( + 'Cache key invalidation failed for cache args %s', + safe_str(cache_uid)) + Session().rollback() + + @classmethod + def get_active_cache(cls, cache_key): + inv_obj = cls.query().filter(cls.cache_key == cache_key).scalar() + if inv_obj: + return inv_obj + return None + + +class ChangesetComment(Base, BaseModel): + __tablename__ = 'changeset_comments' + __table_args__ = ( + Index('cc_revision_idx', 'revision'), + base_table_args, + ) + + COMMENT_OUTDATED = u'comment_outdated' + COMMENT_TYPE_NOTE = u'note' + COMMENT_TYPE_TODO = u'todo' + COMMENT_TYPES = [COMMENT_TYPE_NOTE, COMMENT_TYPE_TODO] + + comment_id = Column('comment_id', Integer(), nullable=False, primary_key=True) + repo_id = Column('repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False) + revision = Column('revision', String(40), nullable=True) + pull_request_id = Column("pull_request_id", Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=True) + pull_request_version_id = Column("pull_request_version_id", Integer(), ForeignKey('pull_request_versions.pull_request_version_id'), nullable=True) + line_no = Column('line_no', Unicode(10), nullable=True) + hl_lines = Column('hl_lines', Unicode(512), nullable=True) + f_path = Column('f_path', Unicode(1000), nullable=True) + user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False) + text = Column('text', UnicodeText().with_variant(UnicodeText(25000), 'mysql'), nullable=False) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + modified_at = Column('modified_at', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + renderer = Column('renderer', Unicode(64), nullable=True) + display_state = Column('display_state', Unicode(128), nullable=True) + + comment_type = Column('comment_type', Unicode(128), nullable=True, default=COMMENT_TYPE_NOTE) + resolved_comment_id = Column('resolved_comment_id', Integer(), ForeignKey('changeset_comments.comment_id'), nullable=True) + + resolved_comment = relationship('ChangesetComment', remote_side=comment_id, back_populates='resolved_by') + resolved_by = relationship('ChangesetComment', back_populates='resolved_comment') + + author = relationship('User', lazy='joined') + repo = relationship('Repository') + status_change = relationship('ChangesetStatus', cascade="all, delete, delete-orphan", lazy='joined') + pull_request = relationship('PullRequest', lazy='joined') + pull_request_version = relationship('PullRequestVersion') + + @classmethod + def get_users(cls, revision=None, pull_request_id=None): + """ + Returns user associated with this ChangesetComment. ie those + who actually commented + + :param cls: + :param revision: + """ + q = Session().query(User)\ + .join(ChangesetComment.author) + if revision: + q = q.filter(cls.revision == revision) + elif pull_request_id: + q = q.filter(cls.pull_request_id == pull_request_id) + return q.all() + + @classmethod + def get_index_from_version(cls, pr_version, versions): + num_versions = [x.pull_request_version_id for x in versions] + try: + return num_versions.index(pr_version) +1 + except (IndexError, ValueError): + return + + @property + def outdated(self): + return self.display_state == self.COMMENT_OUTDATED + + def outdated_at_version(self, version): + """ + Checks if comment is outdated for given pull request version + """ + return self.outdated and self.pull_request_version_id != version + + def older_than_version(self, version): + """ + Checks if comment is made from previous version than given + """ + if version is None: + return self.pull_request_version_id is not None + + return self.pull_request_version_id < version + + @property + def resolved(self): + return self.resolved_by[0] if self.resolved_by else None + + @property + def is_todo(self): + return self.comment_type == self.COMMENT_TYPE_TODO + + @property + def is_inline(self): + return self.line_no and self.f_path + + def get_index_version(self, versions): + return self.get_index_from_version( + self.pull_request_version_id, versions) + + def __repr__(self): + if self.comment_id: + return '' % self.comment_id + else: + return '' % id(self) + + def get_api_data(self): + comment = self + data = { + 'comment_id': comment.comment_id, + 'comment_type': comment.comment_type, + 'comment_text': comment.text, + 'comment_status': comment.status_change, + 'comment_f_path': comment.f_path, + 'comment_lineno': comment.line_no, + 'comment_author': comment.author, + 'comment_created_on': comment.created_on + } + return data + + def __json__(self): + data = dict() + data.update(self.get_api_data()) + return data + + +class ChangesetStatus(Base, BaseModel): + __tablename__ = 'changeset_statuses' + __table_args__ = ( + Index('cs_revision_idx', 'revision'), + Index('cs_version_idx', 'version'), + UniqueConstraint('repo_id', 'revision', 'version'), + base_table_args + ) + + STATUS_NOT_REVIEWED = DEFAULT = 'not_reviewed' + STATUS_APPROVED = 'approved' + STATUS_REJECTED = 'rejected' + STATUS_UNDER_REVIEW = 'under_review' + + STATUSES = [ + (STATUS_NOT_REVIEWED, _("Not Reviewed")), # (no icon) and default + (STATUS_APPROVED, _("Approved")), + (STATUS_REJECTED, _("Rejected")), + (STATUS_UNDER_REVIEW, _("Under Review")), + ] + + changeset_status_id = Column('changeset_status_id', Integer(), nullable=False, primary_key=True) + repo_id = Column('repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None) + revision = Column('revision', String(40), nullable=False) + status = Column('status', String(128), nullable=False, default=DEFAULT) + changeset_comment_id = Column('changeset_comment_id', Integer(), ForeignKey('changeset_comments.comment_id')) + modified_at = Column('modified_at', DateTime(), nullable=False, default=datetime.datetime.now) + version = Column('version', Integer(), nullable=False, default=0) + pull_request_id = Column("pull_request_id", Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=True) + + author = relationship('User', lazy='joined') + repo = relationship('Repository') + comment = relationship('ChangesetComment', lazy='joined') + pull_request = relationship('PullRequest', lazy='joined') + + def __unicode__(self): + return u"<%s('%s[v%s]:%s')>" % ( + self.__class__.__name__, + self.status, self.version, self.author + ) + + @classmethod + def get_status_lbl(cls, value): + return dict(cls.STATUSES).get(value) + + @property + def status_lbl(self): + return ChangesetStatus.get_status_lbl(self.status) + + def get_api_data(self): + status = self + data = { + 'status_id': status.changeset_status_id, + 'status': status.status, + } + return data + + def __json__(self): + data = dict() + data.update(self.get_api_data()) + return data + + +class _PullRequestBase(BaseModel): + """ + Common attributes of pull request and version entries. + """ + + # .status values + STATUS_NEW = u'new' + STATUS_OPEN = u'open' + STATUS_CLOSED = u'closed' + + # available states + STATE_CREATING = u'creating' + STATE_UPDATING = u'updating' + STATE_MERGING = u'merging' + STATE_CREATED = u'created' + + title = Column('title', Unicode(255), nullable=True) + description = Column( + 'description', UnicodeText().with_variant(UnicodeText(10240), 'mysql'), + nullable=True) + description_renderer = Column('description_renderer', Unicode(64), nullable=True) + + # new/open/closed status of pull request (not approve/reject/etc) + status = Column('status', Unicode(255), nullable=False, default=STATUS_NEW) + created_on = Column( + 'created_on', DateTime(timezone=False), nullable=False, + default=datetime.datetime.now) + updated_on = Column( + 'updated_on', DateTime(timezone=False), nullable=False, + default=datetime.datetime.now) + + pull_request_state = Column("pull_request_state", String(255), nullable=True) + + @declared_attr + def user_id(cls): + return Column( + "user_id", Integer(), ForeignKey('users.user_id'), nullable=False, + unique=None) + + # 500 revisions max + _revisions = Column( + 'revisions', UnicodeText().with_variant(UnicodeText(20500), 'mysql')) + + @declared_attr + def source_repo_id(cls): + # TODO: dan: rename column to source_repo_id + return Column( + 'org_repo_id', Integer(), ForeignKey('repositories.repo_id'), + nullable=False) + + _source_ref = Column('org_ref', Unicode(255), nullable=False) + + @hybrid_property + def source_ref(self): + return self._source_ref + + @source_ref.setter + def source_ref(self, val): + parts = (val or '').split(':') + if len(parts) != 3: + raise ValueError( + 'Invalid reference format given: {}, expected X:Y:Z'.format(val)) + self._source_ref = safe_unicode(val) + + _target_ref = Column('other_ref', Unicode(255), nullable=False) + + @hybrid_property + def target_ref(self): + return self._target_ref + + @target_ref.setter + def target_ref(self, val): + parts = (val or '').split(':') + if len(parts) != 3: + raise ValueError( + 'Invalid reference format given: {}, expected X:Y:Z'.format(val)) + self._target_ref = safe_unicode(val) + + @declared_attr + def target_repo_id(cls): + # TODO: dan: rename column to target_repo_id + return Column( + 'other_repo_id', Integer(), ForeignKey('repositories.repo_id'), + nullable=False) + + _shadow_merge_ref = Column('shadow_merge_ref', Unicode(255), nullable=True) + + # TODO: dan: rename column to last_merge_source_rev + _last_merge_source_rev = Column( + 'last_merge_org_rev', String(40), nullable=True) + # TODO: dan: rename column to last_merge_target_rev + _last_merge_target_rev = Column( + 'last_merge_other_rev', String(40), nullable=True) + _last_merge_status = Column('merge_status', Integer(), nullable=True) + merge_rev = Column('merge_rev', String(40), nullable=True) + + reviewer_data = Column( + 'reviewer_data_json', MutationObj.as_mutable( + JsonType(dialect_map=dict(mysql=UnicodeText(16384))))) + + @property + def reviewer_data_json(self): + return json.dumps(self.reviewer_data) + + @hybrid_property + def description_safe(self): + from rhodecode.lib import helpers as h + return h.escape(self.description) + + @hybrid_property + def revisions(self): + return self._revisions.split(':') if self._revisions else [] + + @revisions.setter + def revisions(self, val): + self._revisions = ':'.join(val) + + @hybrid_property + def last_merge_status(self): + return safe_int(self._last_merge_status) + + @last_merge_status.setter + def last_merge_status(self, val): + self._last_merge_status = val + + @declared_attr + def author(cls): + return relationship('User', lazy='joined') + + @declared_attr + def source_repo(cls): + return relationship( + 'Repository', + primaryjoin='%s.source_repo_id==Repository.repo_id' % cls.__name__) + + @property + def source_ref_parts(self): + return self.unicode_to_reference(self.source_ref) + + @declared_attr + def target_repo(cls): + return relationship( + 'Repository', + primaryjoin='%s.target_repo_id==Repository.repo_id' % cls.__name__) + + @property + def target_ref_parts(self): + return self.unicode_to_reference(self.target_ref) + + @property + def shadow_merge_ref(self): + return self.unicode_to_reference(self._shadow_merge_ref) + + @shadow_merge_ref.setter + def shadow_merge_ref(self, ref): + self._shadow_merge_ref = self.reference_to_unicode(ref) + + @staticmethod + def unicode_to_reference(raw): + """ + Convert a unicode (or string) to a reference object. + If unicode evaluates to False it returns None. + """ + if raw: + refs = raw.split(':') + return Reference(*refs) + else: + return None + + @staticmethod + def reference_to_unicode(ref): + """ + Convert a reference object to unicode. + If reference is None it returns None. + """ + if ref: + return u':'.join(ref) + else: + return None + + def get_api_data(self, with_merge_state=True): + from rhodecode.model.pull_request import PullRequestModel + + pull_request = self + if with_merge_state: + merge_status = PullRequestModel().merge_status(pull_request) + merge_state = { + 'status': merge_status[0], + 'message': safe_unicode(merge_status[1]), + } + else: + merge_state = {'status': 'not_available', + 'message': 'not_available'} + + merge_data = { + 'clone_url': PullRequestModel().get_shadow_clone_url(pull_request), + 'reference': ( + pull_request.shadow_merge_ref._asdict() + if pull_request.shadow_merge_ref else None), + } + + data = { + 'pull_request_id': pull_request.pull_request_id, + 'url': PullRequestModel().get_url(pull_request), + 'title': pull_request.title, + 'description': pull_request.description, + 'status': pull_request.status, + 'created_on': pull_request.created_on, + 'updated_on': pull_request.updated_on, + 'commit_ids': pull_request.revisions, + 'review_status': pull_request.calculated_review_status(), + 'mergeable': merge_state, + 'source': { + 'clone_url': pull_request.source_repo.clone_url(), + 'repository': pull_request.source_repo.repo_name, + 'reference': { + 'name': pull_request.source_ref_parts.name, + 'type': pull_request.source_ref_parts.type, + 'commit_id': pull_request.source_ref_parts.commit_id, + }, + }, + 'target': { + 'clone_url': pull_request.target_repo.clone_url(), + 'repository': pull_request.target_repo.repo_name, + 'reference': { + 'name': pull_request.target_ref_parts.name, + 'type': pull_request.target_ref_parts.type, + 'commit_id': pull_request.target_ref_parts.commit_id, + }, + }, + 'merge': merge_data, + 'author': pull_request.author.get_api_data(include_secrets=False, + details='basic'), + 'reviewers': [ + { + 'user': reviewer.get_api_data(include_secrets=False, + details='basic'), + 'reasons': reasons, + 'review_status': st[0][1].status if st else 'not_reviewed', + } + for obj, reviewer, reasons, mandatory, st in + pull_request.reviewers_statuses() + ] + } + + return data + + +class PullRequest(Base, _PullRequestBase): + __tablename__ = 'pull_requests' + __table_args__ = ( + base_table_args, + ) + + pull_request_id = Column( + 'pull_request_id', Integer(), nullable=False, primary_key=True) + + def __repr__(self): + if self.pull_request_id: + return '' % self.pull_request_id + else: + return '' % id(self) + + reviewers = relationship('PullRequestReviewers', + cascade="all, delete, delete-orphan") + statuses = relationship('ChangesetStatus', + cascade="all, delete, delete-orphan") + comments = relationship('ChangesetComment', + cascade="all, delete, delete-orphan") + versions = relationship('PullRequestVersion', + cascade="all, delete, delete-orphan", + lazy='dynamic') + + @classmethod + def get_pr_display_object(cls, pull_request_obj, org_pull_request_obj, + internal_methods=None): + + class PullRequestDisplay(object): + """ + Special object wrapper for showing PullRequest data via Versions + It mimics PR object as close as possible. This is read only object + just for display + """ + + def __init__(self, attrs, internal=None): + self.attrs = attrs + # internal have priority over the given ones via attrs + self.internal = internal or ['versions'] + + def __getattr__(self, item): + if item in self.internal: + return getattr(self, item) + try: + return self.attrs[item] + except KeyError: + raise AttributeError( + '%s object has no attribute %s' % (self, item)) + + def __repr__(self): + return '' % self.attrs.get('pull_request_id') + + def versions(self): + return pull_request_obj.versions.order_by( + PullRequestVersion.pull_request_version_id).all() + + def is_closed(self): + return pull_request_obj.is_closed() + + @property + def pull_request_version_id(self): + return getattr(pull_request_obj, 'pull_request_version_id', None) + + attrs = StrictAttributeDict(pull_request_obj.get_api_data()) + + attrs.author = StrictAttributeDict( + pull_request_obj.author.get_api_data()) + if pull_request_obj.target_repo: + attrs.target_repo = StrictAttributeDict( + pull_request_obj.target_repo.get_api_data()) + attrs.target_repo.clone_url = pull_request_obj.target_repo.clone_url + + if pull_request_obj.source_repo: + attrs.source_repo = StrictAttributeDict( + pull_request_obj.source_repo.get_api_data()) + attrs.source_repo.clone_url = pull_request_obj.source_repo.clone_url + + attrs.source_ref_parts = pull_request_obj.source_ref_parts + attrs.target_ref_parts = pull_request_obj.target_ref_parts + attrs.revisions = pull_request_obj.revisions + + attrs.shadow_merge_ref = org_pull_request_obj.shadow_merge_ref + attrs.reviewer_data = org_pull_request_obj.reviewer_data + attrs.reviewer_data_json = org_pull_request_obj.reviewer_data_json + + return PullRequestDisplay(attrs, internal=internal_methods) + + def is_closed(self): + return self.status == self.STATUS_CLOSED + + def __json__(self): + return { + 'revisions': self.revisions, + } + + def calculated_review_status(self): + from rhodecode.model.changeset_status import ChangesetStatusModel + return ChangesetStatusModel().calculated_review_status(self) + + def reviewers_statuses(self): + from rhodecode.model.changeset_status import ChangesetStatusModel + return ChangesetStatusModel().reviewers_statuses(self) + + @property + def workspace_id(self): + from rhodecode.model.pull_request import PullRequestModel + return PullRequestModel()._workspace_id(self) + + def get_shadow_repo(self): + workspace_id = self.workspace_id + vcs_obj = self.target_repo.scm_instance() + shadow_repository_path = vcs_obj._get_shadow_repository_path( + self.target_repo.repo_id, workspace_id) + if os.path.isdir(shadow_repository_path): + return vcs_obj._get_shadow_instance(shadow_repository_path) + + +class PullRequestVersion(Base, _PullRequestBase): + __tablename__ = 'pull_request_versions' + __table_args__ = ( + base_table_args, + ) + + pull_request_version_id = Column( + 'pull_request_version_id', Integer(), nullable=False, primary_key=True) + pull_request_id = Column( + 'pull_request_id', Integer(), + ForeignKey('pull_requests.pull_request_id'), nullable=False) + pull_request = relationship('PullRequest') + + def __repr__(self): + if self.pull_request_version_id: + return '' % self.pull_request_version_id + else: + return '' % id(self) + + @property + def reviewers(self): + return self.pull_request.reviewers + + @property + def versions(self): + return self.pull_request.versions + + def is_closed(self): + # calculate from original + return self.pull_request.status == self.STATUS_CLOSED + + def calculated_review_status(self): + return self.pull_request.calculated_review_status() + + def reviewers_statuses(self): + return self.pull_request.reviewers_statuses() + + +class PullRequestReviewers(Base, BaseModel): + __tablename__ = 'pull_request_reviewers' + __table_args__ = ( + base_table_args, + ) + + @hybrid_property + def reasons(self): + if not self._reasons: + return [] + return self._reasons + + @reasons.setter + def reasons(self, val): + val = val or [] + if any(not isinstance(x, compat.string_types) for x in val): + raise Exception('invalid reasons type, must be list of strings') + self._reasons = val + + pull_requests_reviewers_id = Column( + 'pull_requests_reviewers_id', Integer(), nullable=False, + primary_key=True) + pull_request_id = Column( + "pull_request_id", Integer(), + ForeignKey('pull_requests.pull_request_id'), nullable=False) + user_id = Column( + "user_id", Integer(), ForeignKey('users.user_id'), nullable=True) + _reasons = Column( + 'reason', MutationList.as_mutable( + JsonType('list', dialect_map=dict(mysql=UnicodeText(16384))))) + + mandatory = Column("mandatory", Boolean(), nullable=False, default=False) + user = relationship('User') + pull_request = relationship('PullRequest') + + rule_data = Column( + 'rule_data_json', + JsonType(dialect_map=dict(mysql=UnicodeText(16384)))) + + def rule_user_group_data(self): + """ + Returns the voting user group rule data for this reviewer + """ + + if self.rule_data and 'vote_rule' in self.rule_data: + user_group_data = {} + if 'rule_user_group_entry_id' in self.rule_data: + # means a group with voting rules ! + user_group_data['id'] = self.rule_data['rule_user_group_entry_id'] + user_group_data['name'] = self.rule_data['rule_name'] + user_group_data['vote_rule'] = self.rule_data['vote_rule'] + + return user_group_data + + def __unicode__(self): + return u"<%s('id:%s')>" % (self.__class__.__name__, + self.pull_requests_reviewers_id) + + +class Notification(Base, BaseModel): + __tablename__ = 'notifications' + __table_args__ = ( + Index('notification_type_idx', 'type'), + base_table_args, + ) + + TYPE_CHANGESET_COMMENT = u'cs_comment' + TYPE_MESSAGE = u'message' + TYPE_MENTION = u'mention' + TYPE_REGISTRATION = u'registration' + TYPE_PULL_REQUEST = u'pull_request' + TYPE_PULL_REQUEST_COMMENT = u'pull_request_comment' + + notification_id = Column('notification_id', Integer(), nullable=False, primary_key=True) + subject = Column('subject', Unicode(512), nullable=True) + body = Column('body', UnicodeText().with_variant(UnicodeText(50000), 'mysql'), nullable=True) + created_by = Column("created_by", Integer(), ForeignKey('users.user_id'), nullable=True) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + type_ = Column('type', Unicode(255)) + + created_by_user = relationship('User') + notifications_to_users = relationship('UserNotification', lazy='joined', + cascade="all, delete, delete-orphan") + + @property + def recipients(self): + return [x.user for x in UserNotification.query()\ + .filter(UserNotification.notification == self)\ + .order_by(UserNotification.user_id.asc()).all()] + + @classmethod + def create(cls, created_by, subject, body, recipients, type_=None): + if type_ is None: + type_ = Notification.TYPE_MESSAGE + + notification = cls() + notification.created_by_user = created_by + notification.subject = subject + notification.body = body + notification.type_ = type_ + notification.created_on = datetime.datetime.now() + + # For each recipient link the created notification to his account + for u in recipients: + assoc = UserNotification() + assoc.user_id = u.user_id + assoc.notification = notification + + # if created_by is inside recipients mark his notification + # as read + if u.user_id == created_by.user_id: + assoc.read = True + Session().add(assoc) + + Session().add(notification) + + return notification + + +class UserNotification(Base, BaseModel): + __tablename__ = 'user_to_notification' + __table_args__ = ( + UniqueConstraint('user_id', 'notification_id'), + base_table_args + ) + + user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), primary_key=True) + notification_id = Column("notification_id", Integer(), ForeignKey('notifications.notification_id'), primary_key=True) + read = Column('read', Boolean, default=False) + sent_on = Column('sent_on', DateTime(timezone=False), nullable=True, unique=None) + + user = relationship('User', lazy="joined") + notification = relationship('Notification', lazy="joined", + order_by=lambda: Notification.created_on.desc(),) + + def mark_as_read(self): + self.read = True + Session().add(self) + + +class Gist(Base, BaseModel): + __tablename__ = 'gists' + __table_args__ = ( + Index('g_gist_access_id_idx', 'gist_access_id'), + Index('g_created_on_idx', 'created_on'), + base_table_args + ) + + GIST_PUBLIC = u'public' + GIST_PRIVATE = u'private' + DEFAULT_FILENAME = u'gistfile1.txt' + + ACL_LEVEL_PUBLIC = u'acl_public' + ACL_LEVEL_PRIVATE = u'acl_private' + + gist_id = Column('gist_id', Integer(), primary_key=True) + gist_access_id = Column('gist_access_id', Unicode(250)) + gist_description = Column('gist_description', UnicodeText().with_variant(UnicodeText(1024), 'mysql')) + gist_owner = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=True) + gist_expires = Column('gist_expires', Float(53), nullable=False) + gist_type = Column('gist_type', Unicode(128), nullable=False) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + modified_at = Column('modified_at', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + acl_level = Column('acl_level', Unicode(128), nullable=True) + + owner = relationship('User') + + def __repr__(self): + return '' % (self.gist_type, self.gist_access_id) + + @hybrid_property + def description_safe(self): + from rhodecode.lib import helpers as h + return h.escape(self.gist_description) + + @classmethod + def get_or_404(cls, id_): + from pyramid.httpexceptions import HTTPNotFound + + res = cls.query().filter(cls.gist_access_id == id_).scalar() + if not res: + raise HTTPNotFound() + return res + + @classmethod + def get_by_access_id(cls, gist_access_id): + return cls.query().filter(cls.gist_access_id == gist_access_id).scalar() + + def gist_url(self): + from rhodecode.model.gist import GistModel + return GistModel().get_url(self) + + @classmethod + def base_path(cls): + """ + Returns base path when all gists are stored + + :param cls: + """ + from rhodecode.model.gist import GIST_STORE_LOC + q = Session().query(RhodeCodeUi)\ + .filter(RhodeCodeUi.ui_key == URL_SEP) + q = q.options(FromCache("sql_cache_short", "repository_repo_path")) + return os.path.join(q.one().ui_value, GIST_STORE_LOC) + + def get_api_data(self): + """ + Common function for generating gist related data for API + """ + gist = self + data = { + 'gist_id': gist.gist_id, + 'type': gist.gist_type, + 'access_id': gist.gist_access_id, + 'description': gist.gist_description, + 'url': gist.gist_url(), + 'expires': gist.gist_expires, + 'created_on': gist.created_on, + 'modified_at': gist.modified_at, + 'content': None, + 'acl_level': gist.acl_level, + } + return data + + def __json__(self): + data = dict( + ) + data.update(self.get_api_data()) + return data + # SCM functions + + def scm_instance(self, **kwargs): + full_repo_path = os.path.join(self.base_path(), self.gist_access_id) + return get_vcs_instance( + repo_path=safe_str(full_repo_path), create=False) + + +class ExternalIdentity(Base, BaseModel): + __tablename__ = 'external_identities' + __table_args__ = ( + Index('local_user_id_idx', 'local_user_id'), + Index('external_id_idx', 'external_id'), + base_table_args + ) + + external_id = Column('external_id', Unicode(255), default=u'', primary_key=True) + external_username = Column('external_username', Unicode(1024), default=u'') + local_user_id = Column('local_user_id', Integer(), ForeignKey('users.user_id'), primary_key=True) + provider_name = Column('provider_name', Unicode(255), default=u'', primary_key=True) + access_token = Column('access_token', String(1024), default=u'') + alt_token = Column('alt_token', String(1024), default=u'') + token_secret = Column('token_secret', String(1024), default=u'') + + @classmethod + def by_external_id_and_provider(cls, external_id, provider_name, local_user_id=None): + """ + Returns ExternalIdentity instance based on search params + + :param external_id: + :param provider_name: + :return: ExternalIdentity + """ + query = cls.query() + query = query.filter(cls.external_id == external_id) + query = query.filter(cls.provider_name == provider_name) + if local_user_id: + query = query.filter(cls.local_user_id == local_user_id) + return query.first() + + @classmethod + def user_by_external_id_and_provider(cls, external_id, provider_name): + """ + Returns User instance based on search params + + :param external_id: + :param provider_name: + :return: User + """ + query = User.query() + query = query.filter(cls.external_id == external_id) + query = query.filter(cls.provider_name == provider_name) + query = query.filter(User.user_id == cls.local_user_id) + return query.first() + + @classmethod + def by_local_user_id(cls, local_user_id): + """ + Returns all tokens for user + + :param local_user_id: + :return: ExternalIdentity + """ + query = cls.query() + query = query.filter(cls.local_user_id == local_user_id) + return query + + @classmethod + def load_provider_plugin(cls, plugin_id): + from rhodecode.authentication.base import loadplugin + _plugin_id = 'egg:rhodecode-enterprise-ee#{}'.format(plugin_id) + auth_plugin = loadplugin(_plugin_id) + return auth_plugin + + +class Integration(Base, BaseModel): + __tablename__ = 'integrations' + __table_args__ = ( + base_table_args + ) + + integration_id = Column('integration_id', Integer(), primary_key=True) + integration_type = Column('integration_type', String(255)) + enabled = Column('enabled', Boolean(), nullable=False) + name = Column('name', String(255), nullable=False) + child_repos_only = Column('child_repos_only', Boolean(), nullable=False, + default=False) + + settings = Column( + 'settings_json', MutationObj.as_mutable( + JsonType(dialect_map=dict(mysql=UnicodeText(16384))))) + repo_id = Column( + 'repo_id', Integer(), ForeignKey('repositories.repo_id'), + nullable=True, unique=None, default=None) + repo = relationship('Repository', lazy='joined') + + repo_group_id = Column( + 'repo_group_id', Integer(), ForeignKey('groups.group_id'), + nullable=True, unique=None, default=None) + repo_group = relationship('RepoGroup', lazy='joined') + + @property + def scope(self): + if self.repo: + return repr(self.repo) + if self.repo_group: + if self.child_repos_only: + return repr(self.repo_group) + ' (child repos only)' + else: + return repr(self.repo_group) + ' (recursive)' + if self.child_repos_only: + return 'root_repos' + return 'global' + + def __repr__(self): + return '' % (self.integration_type, self.scope) + + +class RepoReviewRuleUser(Base, BaseModel): + __tablename__ = 'repo_review_rules_users' + __table_args__ = ( + base_table_args + ) + + repo_review_rule_user_id = Column('repo_review_rule_user_id', Integer(), primary_key=True) + repo_review_rule_id = Column("repo_review_rule_id", Integer(), ForeignKey('repo_review_rules.repo_review_rule_id')) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False) + mandatory = Column("mandatory", Boolean(), nullable=False, default=False) + user = relationship('User') + + def rule_data(self): + return { + 'mandatory': self.mandatory + } + + +class RepoReviewRuleUserGroup(Base, BaseModel): + __tablename__ = 'repo_review_rules_users_groups' + __table_args__ = ( + base_table_args + ) + + VOTE_RULE_ALL = -1 + + repo_review_rule_users_group_id = Column('repo_review_rule_users_group_id', Integer(), primary_key=True) + repo_review_rule_id = Column("repo_review_rule_id", Integer(), ForeignKey('repo_review_rules.repo_review_rule_id')) + users_group_id = Column("users_group_id", Integer(),ForeignKey('users_groups.users_group_id'), nullable=False) + mandatory = Column("mandatory", Boolean(), nullable=False, default=False) + vote_rule = Column("vote_rule", Integer(), nullable=True, default=VOTE_RULE_ALL) + users_group = relationship('UserGroup') + + def rule_data(self): + return { + 'mandatory': self.mandatory, + 'vote_rule': self.vote_rule + } + + @property + def vote_rule_label(self): + if not self.vote_rule or self.vote_rule == self.VOTE_RULE_ALL: + return 'all must vote' + else: + return 'min. vote {}'.format(self.vote_rule) + + +class RepoReviewRule(Base, BaseModel): + __tablename__ = 'repo_review_rules' + __table_args__ = ( + base_table_args + ) + + repo_review_rule_id = Column( + 'repo_review_rule_id', Integer(), primary_key=True) + repo_id = Column( + "repo_id", Integer(), ForeignKey('repositories.repo_id')) + repo = relationship('Repository', backref='review_rules') + + review_rule_name = Column('review_rule_name', String(255)) + _branch_pattern = Column("branch_pattern", UnicodeText().with_variant(UnicodeText(255), 'mysql'), default=u'*') # glob + _target_branch_pattern = Column("target_branch_pattern", UnicodeText().with_variant(UnicodeText(255), 'mysql'), default=u'*') # glob + _file_pattern = Column("file_pattern", UnicodeText().with_variant(UnicodeText(255), 'mysql'), default=u'*') # glob + + use_authors_for_review = Column("use_authors_for_review", Boolean(), nullable=False, default=False) + forbid_author_to_review = Column("forbid_author_to_review", Boolean(), nullable=False, default=False) + forbid_commit_author_to_review = Column("forbid_commit_author_to_review", Boolean(), nullable=False, default=False) + forbid_adding_reviewers = Column("forbid_adding_reviewers", Boolean(), nullable=False, default=False) + + rule_users = relationship('RepoReviewRuleUser') + rule_user_groups = relationship('RepoReviewRuleUserGroup') + + def _validate_pattern(self, value): + re.compile('^' + glob2re(value) + '$') + + @hybrid_property + def source_branch_pattern(self): + return self._branch_pattern or '*' + + @source_branch_pattern.setter + def source_branch_pattern(self, value): + self._validate_pattern(value) + self._branch_pattern = value or '*' + + @hybrid_property + def target_branch_pattern(self): + return self._target_branch_pattern or '*' + + @target_branch_pattern.setter + def target_branch_pattern(self, value): + self._validate_pattern(value) + self._target_branch_pattern = value or '*' + + @hybrid_property + def file_pattern(self): + return self._file_pattern or '*' + + @file_pattern.setter + def file_pattern(self, value): + self._validate_pattern(value) + self._file_pattern = value or '*' + + def matches(self, source_branch, target_branch, files_changed): + """ + Check if this review rule matches a branch/files in a pull request + + :param source_branch: source branch name for the commit + :param target_branch: target branch name for the commit + :param files_changed: list of file paths changed in the pull request + """ + + source_branch = source_branch or '' + target_branch = target_branch or '' + files_changed = files_changed or [] + + branch_matches = True + if source_branch or target_branch: + if self.source_branch_pattern == '*': + source_branch_match = True + else: + if self.source_branch_pattern.startswith('re:'): + source_pattern = self.source_branch_pattern[3:] + else: + source_pattern = '^' + glob2re(self.source_branch_pattern) + '$' + source_branch_regex = re.compile(source_pattern) + source_branch_match = bool(source_branch_regex.search(source_branch)) + if self.target_branch_pattern == '*': + target_branch_match = True + else: + if self.target_branch_pattern.startswith('re:'): + target_pattern = self.target_branch_pattern[3:] + else: + target_pattern = '^' + glob2re(self.target_branch_pattern) + '$' + target_branch_regex = re.compile(target_pattern) + target_branch_match = bool(target_branch_regex.search(target_branch)) + + branch_matches = source_branch_match and target_branch_match + + files_matches = True + if self.file_pattern != '*': + files_matches = False + if self.file_pattern.startswith('re:'): + file_pattern = self.file_pattern[3:] + else: + file_pattern = glob2re(self.file_pattern) + file_regex = re.compile(file_pattern) + for filename in files_changed: + if file_regex.search(filename): + files_matches = True + break + + return branch_matches and files_matches + + @property + def review_users(self): + """ Returns the users which this rule applies to """ + + users = collections.OrderedDict() + + for rule_user in self.rule_users: + if rule_user.user.active: + if rule_user.user not in users: + users[rule_user.user.username] = { + 'user': rule_user.user, + 'source': 'user', + 'source_data': {}, + 'data': rule_user.rule_data() + } + + for rule_user_group in self.rule_user_groups: + source_data = { + 'user_group_id': rule_user_group.users_group.users_group_id, + 'name': rule_user_group.users_group.users_group_name, + 'members': len(rule_user_group.users_group.members) + } + for member in rule_user_group.users_group.members: + if member.user.active: + key = member.user.username + if key in users: + # skip this member as we have him already + # this prevents from override the "first" matched + # users with duplicates in multiple groups + continue + + users[key] = { + 'user': member.user, + 'source': 'user_group', + 'source_data': source_data, + 'data': rule_user_group.rule_data() + } + + return users + + def user_group_vote_rule(self, user_id): + + rules = [] + if not self.rule_user_groups: + return rules + + for user_group in self.rule_user_groups: + user_group_members = [x.user_id for x in user_group.users_group.members] + if user_id in user_group_members: + rules.append(user_group) + return rules + + def __repr__(self): + return '' % ( + self.repo_review_rule_id, self.repo) + + +class ScheduleEntry(Base, BaseModel): + __tablename__ = 'schedule_entries' + __table_args__ = ( + UniqueConstraint('schedule_name', name='s_schedule_name_idx'), + UniqueConstraint('task_uid', name='s_task_uid_idx'), + base_table_args, + ) + + schedule_types = ['crontab', 'timedelta', 'integer'] + schedule_entry_id = Column('schedule_entry_id', Integer(), primary_key=True) + + schedule_name = Column("schedule_name", String(255), nullable=False, unique=None, default=None) + schedule_description = Column("schedule_description", String(10000), nullable=True, unique=None, default=None) + schedule_enabled = Column("schedule_enabled", Boolean(), nullable=False, unique=None, default=True) + + _schedule_type = Column("schedule_type", String(255), nullable=False, unique=None, default=None) + schedule_definition = Column('schedule_definition_json', MutationObj.as_mutable(JsonType(default=lambda: "", dialect_map=dict(mysql=LONGTEXT())))) + + schedule_last_run = Column('schedule_last_run', DateTime(timezone=False), nullable=True, unique=None, default=None) + schedule_total_run_count = Column('schedule_total_run_count', Integer(), nullable=True, unique=None, default=0) + + # task + task_uid = Column("task_uid", String(255), nullable=False, unique=None, default=None) + task_dot_notation = Column("task_dot_notation", String(4096), nullable=False, unique=None, default=None) + task_args = Column('task_args_json', MutationObj.as_mutable(JsonType(default=list, dialect_map=dict(mysql=LONGTEXT())))) + task_kwargs = Column('task_kwargs_json', MutationObj.as_mutable(JsonType(default=dict, dialect_map=dict(mysql=LONGTEXT())))) + + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + updated_on = Column('updated_on', DateTime(timezone=False), nullable=True, unique=None, default=None) + + @hybrid_property + def schedule_type(self): + return self._schedule_type + + @schedule_type.setter + def schedule_type(self, val): + if val not in self.schedule_types: + raise ValueError('Value must be on of `{}` and got `{}`'.format( + val, self.schedule_type)) + + self._schedule_type = val + + @classmethod + def get_uid(cls, obj): + args = obj.task_args + kwargs = obj.task_kwargs + if isinstance(args, JsonRaw): + try: + args = json.loads(args) + except ValueError: + args = tuple() + + if isinstance(kwargs, JsonRaw): + try: + kwargs = json.loads(kwargs) + except ValueError: + kwargs = dict() + + dot_notation = obj.task_dot_notation + val = '.'.join(map(safe_str, [ + sorted(dot_notation), args, sorted(kwargs.items())])) + return hashlib.sha1(val).hexdigest() + + @classmethod + def get_by_schedule_name(cls, schedule_name): + return cls.query().filter(cls.schedule_name == schedule_name).scalar() + + @classmethod + def get_by_schedule_id(cls, schedule_id): + return cls.query().filter(cls.schedule_entry_id == schedule_id).scalar() + + @property + def task(self): + return self.task_dot_notation + + @property + def schedule(self): + from rhodecode.lib.celerylib.utils import raw_2_schedule + schedule = raw_2_schedule(self.schedule_definition, self.schedule_type) + return schedule + + @property + def args(self): + try: + return list(self.task_args or []) + except ValueError: + return list() + + @property + def kwargs(self): + try: + return dict(self.task_kwargs or {}) + except ValueError: + return dict() + + def _as_raw(self, val): + if hasattr(val, 'de_coerce'): + val = val.de_coerce() + if val: + val = json.dumps(val) + + return val + + @property + def schedule_definition_raw(self): + return self._as_raw(self.schedule_definition) + + @property + def args_raw(self): + return self._as_raw(self.task_args) + + @property + def kwargs_raw(self): + return self._as_raw(self.task_kwargs) + + def __repr__(self): + return ''.format( + self.schedule_entry_id, self.schedule_name) + + +@event.listens_for(ScheduleEntry, 'before_update') +def update_task_uid(mapper, connection, target): + target.task_uid = ScheduleEntry.get_uid(target) + + +@event.listens_for(ScheduleEntry, 'before_insert') +def set_task_uid(mapper, connection, target): + target.task_uid = ScheduleEntry.get_uid(target) + + +class _BaseBranchPerms(BaseModel): + @classmethod + def compute_hash(cls, value): + return sha1_safe(value) + + @hybrid_property + def branch_pattern(self): + return self._branch_pattern or '*' + + @hybrid_property + def branch_hash(self): + return self._branch_hash + + def _validate_glob(self, value): + re.compile('^' + glob2re(value) + '$') + + @branch_pattern.setter + def branch_pattern(self, value): + self._validate_glob(value) + self._branch_pattern = value or '*' + # set the Hash when setting the branch pattern + self._branch_hash = self.compute_hash(self._branch_pattern) + + def matches(self, branch): + """ + Check if this the branch matches entry + + :param branch: branch name for the commit + """ + + branch = branch or '' + + branch_matches = True + if branch: + branch_regex = re.compile('^' + glob2re(self.branch_pattern) + '$') + branch_matches = bool(branch_regex.search(branch)) + + return branch_matches + + +class UserToRepoBranchPermission(Base, _BaseBranchPerms): + __tablename__ = 'user_to_repo_branch_permissions' + __table_args__ = ( + {'extend_existing': True, 'mysql_engine': 'InnoDB', + 'mysql_charset': 'utf8', 'sqlite_autoincrement': True,} + ) + + branch_rule_id = Column('branch_rule_id', Integer(), primary_key=True) + + repository_id = Column('repository_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None) + repo = relationship('Repository', backref='user_branch_perms') + + permission_id = Column('permission_id', Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + permission = relationship('Permission') + + rule_to_perm_id = Column('rule_to_perm_id', Integer(), ForeignKey('repo_to_perm.repo_to_perm_id'), nullable=False, unique=None, default=None) + user_repo_to_perm = relationship('UserRepoToPerm') + + rule_order = Column('rule_order', Integer(), nullable=False) + _branch_pattern = Column('branch_pattern', UnicodeText().with_variant(UnicodeText(2048), 'mysql'), default=u'*') # glob + _branch_hash = Column('branch_hash', UnicodeText().with_variant(UnicodeText(2048), 'mysql')) + + def __unicode__(self): + return u' %r)>' % ( + self.user_repo_to_perm, self.branch_pattern) + + +class UserGroupToRepoBranchPermission(Base, _BaseBranchPerms): + __tablename__ = 'user_group_to_repo_branch_permissions' + __table_args__ = ( + {'extend_existing': True, 'mysql_engine': 'InnoDB', + 'mysql_charset': 'utf8', 'sqlite_autoincrement': True,} + ) + + branch_rule_id = Column('branch_rule_id', Integer(), primary_key=True) + + repository_id = Column('repository_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None) + repo = relationship('Repository', backref='user_group_branch_perms') + + permission_id = Column('permission_id', Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + permission = relationship('Permission') + + rule_to_perm_id = Column('rule_to_perm_id', Integer(), ForeignKey('users_group_repo_to_perm.users_group_to_perm_id'), nullable=False, unique=None, default=None) + user_group_repo_to_perm = relationship('UserGroupRepoToPerm') + + rule_order = Column('rule_order', Integer(), nullable=False) + _branch_pattern = Column('branch_pattern', UnicodeText().with_variant(UnicodeText(2048), 'mysql'), default=u'*') # glob + _branch_hash = Column('branch_hash', UnicodeText().with_variant(UnicodeText(2048), 'mysql')) + + def __unicode__(self): + return u' %r)>' % ( + self.user_group_repo_to_perm, self.branch_pattern) + + +class DbMigrateVersion(Base, BaseModel): + __tablename__ = 'db_migrate_version' + __table_args__ = ( + base_table_args, + ) + + repository_id = Column('repository_id', String(250), primary_key=True) + repository_path = Column('repository_path', Text) + version = Column('version', Integer) + + @classmethod + def set_version(cls, version): + """ + Helper for forcing a different version, usually for debugging purposes via ishell. + """ + ver = DbMigrateVersion.query().first() + ver.version = version + Session().commit() + + +class DbSession(Base, BaseModel): + __tablename__ = 'db_session' + __table_args__ = ( + base_table_args, + ) + + def __repr__(self): + return ''.format(self.id) + + id = Column('id', Integer()) + namespace = Column('namespace', String(255), primary_key=True) + accessed = Column('accessed', DateTime, nullable=False) + created = Column('created', DateTime, nullable=False) + data = Column('data', PickleType, nullable=False) diff --git a/rhodecode/lib/dbmigrate/schema/db_4_16_0_1.py b/rhodecode/lib/dbmigrate/schema/db_4_16_0_1.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/dbmigrate/schema/db_4_16_0_1.py @@ -0,0 +1,4857 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2019 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +""" +Database Models for RhodeCode Enterprise +""" + +import re +import os +import time +import hashlib +import logging +import datetime +import warnings +import ipaddress +import functools +import traceback +import collections + +from sqlalchemy import ( + or_, and_, not_, func, TypeDecorator, event, + Index, Sequence, UniqueConstraint, ForeignKey, CheckConstraint, Column, + Boolean, String, Unicode, UnicodeText, DateTime, Integer, LargeBinary, + Text, Float, PickleType) +from sqlalchemy.sql.expression import true, false +from sqlalchemy.sql.functions import coalesce, count # pragma: no cover +from sqlalchemy.orm import ( + relationship, joinedload, class_mapper, validates, aliased) +from sqlalchemy.ext.declarative import declared_attr +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.exc import IntegrityError # pragma: no cover +from sqlalchemy.dialects.mysql import LONGTEXT +from zope.cachedescriptors.property import Lazy as LazyProperty +from pyramid import compat +from pyramid.threadlocal import get_current_request + +from rhodecode.translation import _ +from rhodecode.lib.vcs import get_vcs_instance +from rhodecode.lib.vcs.backends.base import EmptyCommit, Reference +from rhodecode.lib.utils2 import ( + str2bool, safe_str, get_commit_safe, safe_unicode, sha1_safe, + time_to_datetime, aslist, Optional, safe_int, get_clone_url, AttributeDict, + glob2re, StrictAttributeDict, cleaned_uri) +from rhodecode.lib.jsonalchemy import MutationObj, MutationList, JsonType, \ + JsonRaw +from rhodecode.lib.ext_json import json +from rhodecode.lib.caching_query import FromCache +from rhodecode.lib.encrypt import AESCipher + +from rhodecode.model.meta import Base, Session + +URL_SEP = '/' +log = logging.getLogger(__name__) + +# ============================================================================= +# BASE CLASSES +# ============================================================================= + +# this is propagated from .ini file rhodecode.encrypted_values.secret or +# beaker.session.secret if first is not set. +# and initialized at environment.py +ENCRYPTION_KEY = None + +# used to sort permissions by types, '#' used here is not allowed to be in +# usernames, and it's very early in sorted string.printable table. +PERMISSION_TYPE_SORT = { + 'admin': '####', + 'write': '###', + 'read': '##', + 'none': '#', +} + + +def display_user_sort(obj): + """ + Sort function used to sort permissions in .permissions() function of + Repository, RepoGroup, UserGroup. Also it put the default user in front + of all other resources + """ + + if obj.username == User.DEFAULT_USER: + return '#####' + prefix = PERMISSION_TYPE_SORT.get(obj.permission.split('.')[-1], '') + return prefix + obj.username + + +def display_user_group_sort(obj): + """ + Sort function used to sort permissions in .permissions() function of + Repository, RepoGroup, UserGroup. Also it put the default user in front + of all other resources + """ + + prefix = PERMISSION_TYPE_SORT.get(obj.permission.split('.')[-1], '') + return prefix + obj.users_group_name + + +def _hash_key(k): + return sha1_safe(k) + + +def in_filter_generator(qry, items, limit=500): + """ + Splits IN() into multiple with OR + e.g.:: + cnt = Repository.query().filter( + or_( + *in_filter_generator(Repository.repo_id, range(100000)) + )).count() + """ + if not items: + # empty list will cause empty query which might cause security issues + # this can lead to hidden unpleasant results + items = [-1] + + parts = [] + for chunk in xrange(0, len(items), limit): + parts.append( + qry.in_(items[chunk: chunk + limit]) + ) + + return parts + + +base_table_args = { + 'extend_existing': True, + 'mysql_engine': 'InnoDB', + 'mysql_charset': 'utf8', + 'sqlite_autoincrement': True +} + + +class EncryptedTextValue(TypeDecorator): + """ + Special column for encrypted long text data, use like:: + + value = Column("encrypted_value", EncryptedValue(), nullable=False) + + This column is intelligent so if value is in unencrypted form it return + unencrypted form, but on save it always encrypts + """ + impl = Text + + def process_bind_param(self, value, dialect): + if not value: + return value + if value.startswith('enc$aes$') or value.startswith('enc$aes_hmac$'): + # protect against double encrypting if someone manually starts + # doing + raise ValueError('value needs to be in unencrypted format, ie. ' + 'not starting with enc$aes') + return 'enc$aes_hmac$%s' % AESCipher( + ENCRYPTION_KEY, hmac=True).encrypt(value) + + def process_result_value(self, value, dialect): + import rhodecode + + if not value: + return value + + parts = value.split('$', 3) + if not len(parts) == 3: + # probably not encrypted values + return value + else: + if parts[0] != 'enc': + # parts ok but without our header ? + return value + enc_strict_mode = str2bool(rhodecode.CONFIG.get( + 'rhodecode.encrypted_values.strict') or True) + # at that stage we know it's our encryption + if parts[1] == 'aes': + decrypted_data = AESCipher(ENCRYPTION_KEY).decrypt(parts[2]) + elif parts[1] == 'aes_hmac': + decrypted_data = AESCipher( + ENCRYPTION_KEY, hmac=True, + strict_verification=enc_strict_mode).decrypt(parts[2]) + else: + raise ValueError( + 'Encryption type part is wrong, must be `aes` ' + 'or `aes_hmac`, got `%s` instead' % (parts[1])) + return decrypted_data + + +class BaseModel(object): + """ + Base Model for all classes + """ + + @classmethod + def _get_keys(cls): + """return column names for this model """ + return class_mapper(cls).c.keys() + + def get_dict(self): + """ + return dict with keys and values corresponding + to this model data """ + + d = {} + for k in self._get_keys(): + d[k] = getattr(self, k) + + # also use __json__() if present to get additional fields + _json_attr = getattr(self, '__json__', None) + if _json_attr: + # update with attributes from __json__ + if callable(_json_attr): + _json_attr = _json_attr() + for k, val in _json_attr.iteritems(): + d[k] = val + return d + + def get_appstruct(self): + """return list with keys and values tuples corresponding + to this model data """ + + lst = [] + for k in self._get_keys(): + lst.append((k, getattr(self, k),)) + return lst + + def populate_obj(self, populate_dict): + """populate model with data from given populate_dict""" + + for k in self._get_keys(): + if k in populate_dict: + setattr(self, k, populate_dict[k]) + + @classmethod + def query(cls): + return Session().query(cls) + + @classmethod + def get(cls, id_): + if id_: + return cls.query().get(id_) + + @classmethod + def get_or_404(cls, id_): + from pyramid.httpexceptions import HTTPNotFound + + try: + id_ = int(id_) + except (TypeError, ValueError): + raise HTTPNotFound() + + res = cls.query().get(id_) + if not res: + raise HTTPNotFound() + return res + + @classmethod + def getAll(cls): + # deprecated and left for backward compatibility + return cls.get_all() + + @classmethod + def get_all(cls): + return cls.query().all() + + @classmethod + def delete(cls, id_): + obj = cls.query().get(id_) + Session().delete(obj) + + @classmethod + def identity_cache(cls, session, attr_name, value): + exist_in_session = [] + for (item_cls, pkey), instance in session.identity_map.items(): + if cls == item_cls and getattr(instance, attr_name) == value: + exist_in_session.append(instance) + if exist_in_session: + if len(exist_in_session) == 1: + return exist_in_session[0] + log.exception( + 'multiple objects with attr %s and ' + 'value %s found with same name: %r', + attr_name, value, exist_in_session) + + def __repr__(self): + if hasattr(self, '__unicode__'): + # python repr needs to return str + try: + return safe_str(self.__unicode__()) + except UnicodeDecodeError: + pass + return '' % (self.__class__.__name__) + + +class RhodeCodeSetting(Base, BaseModel): + __tablename__ = 'rhodecode_settings' + __table_args__ = ( + UniqueConstraint('app_settings_name'), + base_table_args + ) + + SETTINGS_TYPES = { + 'str': safe_str, + 'int': safe_int, + 'unicode': safe_unicode, + 'bool': str2bool, + 'list': functools.partial(aslist, sep=',') + } + DEFAULT_UPDATE_URL = 'https://rhodecode.com/api/v1/info/versions' + GLOBAL_CONF_KEY = 'app_settings' + + app_settings_id = Column("app_settings_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + app_settings_name = Column("app_settings_name", String(255), nullable=True, unique=None, default=None) + _app_settings_value = Column("app_settings_value", String(4096), nullable=True, unique=None, default=None) + _app_settings_type = Column("app_settings_type", String(255), nullable=True, unique=None, default=None) + + def __init__(self, key='', val='', type='unicode'): + self.app_settings_name = key + self.app_settings_type = type + self.app_settings_value = val + + @validates('_app_settings_value') + def validate_settings_value(self, key, val): + assert type(val) == unicode + return val + + @hybrid_property + def app_settings_value(self): + v = self._app_settings_value + _type = self.app_settings_type + if _type: + _type = self.app_settings_type.split('.')[0] + # decode the encrypted value + if 'encrypted' in self.app_settings_type: + cipher = EncryptedTextValue() + v = safe_unicode(cipher.process_result_value(v, None)) + + converter = self.SETTINGS_TYPES.get(_type) or \ + self.SETTINGS_TYPES['unicode'] + return converter(v) + + @app_settings_value.setter + def app_settings_value(self, val): + """ + Setter that will always make sure we use unicode in app_settings_value + + :param val: + """ + val = safe_unicode(val) + # encode the encrypted value + if 'encrypted' in self.app_settings_type: + cipher = EncryptedTextValue() + val = safe_unicode(cipher.process_bind_param(val, None)) + self._app_settings_value = val + + @hybrid_property + def app_settings_type(self): + return self._app_settings_type + + @app_settings_type.setter + def app_settings_type(self, val): + if val.split('.')[0] not in self.SETTINGS_TYPES: + raise Exception('type must be one of %s got %s' + % (self.SETTINGS_TYPES.keys(), val)) + self._app_settings_type = val + + @classmethod + def get_by_prefix(cls, prefix): + return RhodeCodeSetting.query()\ + .filter(RhodeCodeSetting.app_settings_name.startswith(prefix))\ + .all() + + def __unicode__(self): + return u"<%s('%s:%s[%s]')>" % ( + self.__class__.__name__, + self.app_settings_name, self.app_settings_value, + self.app_settings_type + ) + + +class RhodeCodeUi(Base, BaseModel): + __tablename__ = 'rhodecode_ui' + __table_args__ = ( + UniqueConstraint('ui_key'), + base_table_args + ) + + HOOK_REPO_SIZE = 'changegroup.repo_size' + # HG + HOOK_PRE_PULL = 'preoutgoing.pre_pull' + HOOK_PULL = 'outgoing.pull_logger' + HOOK_PRE_PUSH = 'prechangegroup.pre_push' + HOOK_PRETX_PUSH = 'pretxnchangegroup.pre_push' + HOOK_PUSH = 'changegroup.push_logger' + HOOK_PUSH_KEY = 'pushkey.key_push' + + # TODO: johbo: Unify way how hooks are configured for git and hg, + # git part is currently hardcoded. + + # SVN PATTERNS + SVN_BRANCH_ID = 'vcs_svn_branch' + SVN_TAG_ID = 'vcs_svn_tag' + + ui_id = Column( + "ui_id", Integer(), nullable=False, unique=True, default=None, + primary_key=True) + ui_section = Column( + "ui_section", String(255), nullable=True, unique=None, default=None) + ui_key = Column( + "ui_key", String(255), nullable=True, unique=None, default=None) + ui_value = Column( + "ui_value", String(255), nullable=True, unique=None, default=None) + ui_active = Column( + "ui_active", Boolean(), nullable=True, unique=None, default=True) + + def __repr__(self): + return '<%s[%s]%s=>%s]>' % (self.__class__.__name__, self.ui_section, + self.ui_key, self.ui_value) + + +class RepoRhodeCodeSetting(Base, BaseModel): + __tablename__ = 'repo_rhodecode_settings' + __table_args__ = ( + UniqueConstraint( + 'app_settings_name', 'repository_id', + name='uq_repo_rhodecode_setting_name_repo_id'), + base_table_args + ) + + repository_id = Column( + "repository_id", Integer(), ForeignKey('repositories.repo_id'), + nullable=False) + app_settings_id = Column( + "app_settings_id", Integer(), nullable=False, unique=True, + default=None, primary_key=True) + app_settings_name = Column( + "app_settings_name", String(255), nullable=True, unique=None, + default=None) + _app_settings_value = Column( + "app_settings_value", String(4096), nullable=True, unique=None, + default=None) + _app_settings_type = Column( + "app_settings_type", String(255), nullable=True, unique=None, + default=None) + + repository = relationship('Repository') + + def __init__(self, repository_id, key='', val='', type='unicode'): + self.repository_id = repository_id + self.app_settings_name = key + self.app_settings_type = type + self.app_settings_value = val + + @validates('_app_settings_value') + def validate_settings_value(self, key, val): + assert type(val) == unicode + return val + + @hybrid_property + def app_settings_value(self): + v = self._app_settings_value + type_ = self.app_settings_type + SETTINGS_TYPES = RhodeCodeSetting.SETTINGS_TYPES + converter = SETTINGS_TYPES.get(type_) or SETTINGS_TYPES['unicode'] + return converter(v) + + @app_settings_value.setter + def app_settings_value(self, val): + """ + Setter that will always make sure we use unicode in app_settings_value + + :param val: + """ + self._app_settings_value = safe_unicode(val) + + @hybrid_property + def app_settings_type(self): + return self._app_settings_type + + @app_settings_type.setter + def app_settings_type(self, val): + SETTINGS_TYPES = RhodeCodeSetting.SETTINGS_TYPES + if val not in SETTINGS_TYPES: + raise Exception('type must be one of %s got %s' + % (SETTINGS_TYPES.keys(), val)) + self._app_settings_type = val + + def __unicode__(self): + return u"<%s('%s:%s:%s[%s]')>" % ( + self.__class__.__name__, self.repository.repo_name, + self.app_settings_name, self.app_settings_value, + self.app_settings_type + ) + + +class RepoRhodeCodeUi(Base, BaseModel): + __tablename__ = 'repo_rhodecode_ui' + __table_args__ = ( + UniqueConstraint( + 'repository_id', 'ui_section', 'ui_key', + name='uq_repo_rhodecode_ui_repository_id_section_key'), + base_table_args + ) + + repository_id = Column( + "repository_id", Integer(), ForeignKey('repositories.repo_id'), + nullable=False) + ui_id = Column( + "ui_id", Integer(), nullable=False, unique=True, default=None, + primary_key=True) + ui_section = Column( + "ui_section", String(255), nullable=True, unique=None, default=None) + ui_key = Column( + "ui_key", String(255), nullable=True, unique=None, default=None) + ui_value = Column( + "ui_value", String(255), nullable=True, unique=None, default=None) + ui_active = Column( + "ui_active", Boolean(), nullable=True, unique=None, default=True) + + repository = relationship('Repository') + + def __repr__(self): + return '<%s[%s:%s]%s=>%s]>' % ( + self.__class__.__name__, self.repository.repo_name, + self.ui_section, self.ui_key, self.ui_value) + + +class User(Base, BaseModel): + __tablename__ = 'users' + __table_args__ = ( + UniqueConstraint('username'), UniqueConstraint('email'), + Index('u_username_idx', 'username'), + Index('u_email_idx', 'email'), + base_table_args + ) + + DEFAULT_USER = 'default' + DEFAULT_USER_EMAIL = 'anonymous@rhodecode.org' + DEFAULT_GRAVATAR_URL = 'https://secure.gravatar.com/avatar/{md5email}?d=identicon&s={size}' + + user_id = Column("user_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + username = Column("username", String(255), nullable=True, unique=None, default=None) + password = Column("password", String(255), nullable=True, unique=None, default=None) + active = Column("active", Boolean(), nullable=True, unique=None, default=True) + admin = Column("admin", Boolean(), nullable=True, unique=None, default=False) + name = Column("firstname", String(255), nullable=True, unique=None, default=None) + lastname = Column("lastname", String(255), nullable=True, unique=None, default=None) + _email = Column("email", String(255), nullable=True, unique=None, default=None) + last_login = Column("last_login", DateTime(timezone=False), nullable=True, unique=None, default=None) + last_activity = Column('last_activity', DateTime(timezone=False), nullable=True, unique=None, default=None) + + extern_type = Column("extern_type", String(255), nullable=True, unique=None, default=None) + extern_name = Column("extern_name", String(255), nullable=True, unique=None, default=None) + _api_key = Column("api_key", String(255), nullable=True, unique=None, default=None) + inherit_default_permissions = Column("inherit_default_permissions", Boolean(), nullable=False, unique=None, default=True) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + _user_data = Column("user_data", LargeBinary(), nullable=True) # JSON data + + user_log = relationship('UserLog') + user_perms = relationship('UserToPerm', primaryjoin="User.user_id==UserToPerm.user_id", cascade='all') + + repositories = relationship('Repository') + repository_groups = relationship('RepoGroup') + user_groups = relationship('UserGroup') + + user_followers = relationship('UserFollowing', primaryjoin='UserFollowing.follows_user_id==User.user_id', cascade='all') + followings = relationship('UserFollowing', primaryjoin='UserFollowing.user_id==User.user_id', cascade='all') + + repo_to_perm = relationship('UserRepoToPerm', primaryjoin='UserRepoToPerm.user_id==User.user_id', cascade='all') + repo_group_to_perm = relationship('UserRepoGroupToPerm', primaryjoin='UserRepoGroupToPerm.user_id==User.user_id', cascade='all') + user_group_to_perm = relationship('UserUserGroupToPerm', primaryjoin='UserUserGroupToPerm.user_id==User.user_id', cascade='all') + + group_member = relationship('UserGroupMember', cascade='all') + + notifications = relationship('UserNotification', cascade='all') + # notifications assigned to this user + user_created_notifications = relationship('Notification', cascade='all') + # comments created by this user + user_comments = relationship('ChangesetComment', cascade='all') + # user profile extra info + user_emails = relationship('UserEmailMap', cascade='all') + user_ip_map = relationship('UserIpMap', cascade='all') + user_auth_tokens = relationship('UserApiKeys', cascade='all') + user_ssh_keys = relationship('UserSshKeys', cascade='all') + + # gists + user_gists = relationship('Gist', cascade='all') + # user pull requests + user_pull_requests = relationship('PullRequest', cascade='all') + # external identities + extenal_identities = relationship( + 'ExternalIdentity', + primaryjoin="User.user_id==ExternalIdentity.local_user_id", + cascade='all') + # review rules + user_review_rules = relationship('RepoReviewRuleUser', cascade='all') + + def __unicode__(self): + return u"<%s('id:%s:%s')>" % (self.__class__.__name__, + self.user_id, self.username) + + @hybrid_property + def email(self): + return self._email + + @email.setter + def email(self, val): + self._email = val.lower() if val else None + + @hybrid_property + def first_name(self): + from rhodecode.lib import helpers as h + if self.name: + return h.escape(self.name) + return self.name + + @hybrid_property + def last_name(self): + from rhodecode.lib import helpers as h + if self.lastname: + return h.escape(self.lastname) + return self.lastname + + @hybrid_property + def api_key(self): + """ + Fetch if exist an auth-token with role ALL connected to this user + """ + user_auth_token = UserApiKeys.query()\ + .filter(UserApiKeys.user_id == self.user_id)\ + .filter(or_(UserApiKeys.expires == -1, + UserApiKeys.expires >= time.time()))\ + .filter(UserApiKeys.role == UserApiKeys.ROLE_ALL).first() + if user_auth_token: + user_auth_token = user_auth_token.api_key + + return user_auth_token + + @api_key.setter + def api_key(self, val): + # don't allow to set API key this is deprecated for now + self._api_key = None + + @property + def reviewer_pull_requests(self): + return PullRequestReviewers.query() \ + .options(joinedload(PullRequestReviewers.pull_request)) \ + .filter(PullRequestReviewers.user_id == self.user_id) \ + .all() + + @property + def firstname(self): + # alias for future + return self.name + + @property + def emails(self): + other = UserEmailMap.query()\ + .filter(UserEmailMap.user == self) \ + .order_by(UserEmailMap.email_id.asc()) \ + .all() + return [self.email] + [x.email for x in other] + + @property + def auth_tokens(self): + auth_tokens = self.get_auth_tokens() + return [x.api_key for x in auth_tokens] + + def get_auth_tokens(self): + return UserApiKeys.query()\ + .filter(UserApiKeys.user == self)\ + .order_by(UserApiKeys.user_api_key_id.asc())\ + .all() + + @LazyProperty + def feed_token(self): + return self.get_feed_token() + + def get_feed_token(self, cache=True): + feed_tokens = UserApiKeys.query()\ + .filter(UserApiKeys.user == self)\ + .filter(UserApiKeys.role == UserApiKeys.ROLE_FEED) + if cache: + feed_tokens = feed_tokens.options( + FromCache("sql_cache_short", "get_user_feed_token_%s" % self.user_id)) + + feed_tokens = feed_tokens.all() + if feed_tokens: + return feed_tokens[0].api_key + return 'NO_FEED_TOKEN_AVAILABLE' + + @classmethod + def get(cls, user_id, cache=False): + if not user_id: + return + + user = cls.query() + if cache: + user = user.options( + FromCache("sql_cache_short", "get_users_%s" % user_id)) + return user.get(user_id) + + @classmethod + def extra_valid_auth_tokens(cls, user, role=None): + tokens = UserApiKeys.query().filter(UserApiKeys.user == user)\ + .filter(or_(UserApiKeys.expires == -1, + UserApiKeys.expires >= time.time())) + if role: + tokens = tokens.filter(or_(UserApiKeys.role == role, + UserApiKeys.role == UserApiKeys.ROLE_ALL)) + return tokens.all() + + def authenticate_by_token(self, auth_token, roles=None, scope_repo_id=None): + from rhodecode.lib import auth + + log.debug('Trying to authenticate user: %s via auth-token, ' + 'and roles: %s', self, roles) + + if not auth_token: + return False + + crypto_backend = auth.crypto_backend() + + roles = (roles or []) + [UserApiKeys.ROLE_ALL] + tokens_q = UserApiKeys.query()\ + .filter(UserApiKeys.user_id == self.user_id)\ + .filter(or_(UserApiKeys.expires == -1, + UserApiKeys.expires >= time.time())) + + tokens_q = tokens_q.filter(UserApiKeys.role.in_(roles)) + + plain_tokens = [] + hash_tokens = [] + + user_tokens = tokens_q.all() + log.debug('Found %s user tokens to check for authentication', len(user_tokens)) + for token in user_tokens: + log.debug('AUTH_TOKEN: checking if user token with id `%s` matches', + token.user_api_key_id) + # verify scope first, since it's way faster than hash calculation of + # encrypted tokens + if token.repo_id: + # token has a scope, we need to verify it + if scope_repo_id != token.repo_id: + log.debug( + 'AUTH_TOKEN: scope mismatch, token has a set repo scope: %s, ' + 'and calling scope is:%s, skipping further checks', + token.repo, scope_repo_id) + # token has a scope, and it doesn't match, skip token + continue + + if token.api_key.startswith(crypto_backend.ENC_PREF): + hash_tokens.append(token.api_key) + else: + plain_tokens.append(token.api_key) + + is_plain_match = auth_token in plain_tokens + if is_plain_match: + return True + + for hashed in hash_tokens: + # NOTE(marcink): this is expensive to calculate, but most secure + match = crypto_backend.hash_check(auth_token, hashed) + if match: + return True + + return False + + @property + def ip_addresses(self): + ret = UserIpMap.query().filter(UserIpMap.user == self).all() + return [x.ip_addr for x in ret] + + @property + def username_and_name(self): + return '%s (%s %s)' % (self.username, self.first_name, self.last_name) + + @property + def username_or_name_or_email(self): + full_name = self.full_name if self.full_name is not ' ' else None + return self.username or full_name or self.email + + @property + def full_name(self): + return '%s %s' % (self.first_name, self.last_name) + + @property + def full_name_or_username(self): + return ('%s %s' % (self.first_name, self.last_name) + if (self.first_name and self.last_name) else self.username) + + @property + def full_contact(self): + return '%s %s <%s>' % (self.first_name, self.last_name, self.email) + + @property + def short_contact(self): + return '%s %s' % (self.first_name, self.last_name) + + @property + def is_admin(self): + return self.admin + + def AuthUser(self, **kwargs): + """ + Returns instance of AuthUser for this user + """ + from rhodecode.lib.auth import AuthUser + return AuthUser(user_id=self.user_id, username=self.username, **kwargs) + + @hybrid_property + def user_data(self): + if not self._user_data: + return {} + + try: + return json.loads(self._user_data) + except TypeError: + return {} + + @user_data.setter + def user_data(self, val): + if not isinstance(val, dict): + raise Exception('user_data must be dict, got %s' % type(val)) + try: + self._user_data = json.dumps(val) + except Exception: + log.error(traceback.format_exc()) + + @classmethod + def get_by_username(cls, username, case_insensitive=False, + cache=False, identity_cache=False): + session = Session() + + if case_insensitive: + q = cls.query().filter( + func.lower(cls.username) == func.lower(username)) + else: + q = cls.query().filter(cls.username == username) + + if cache: + if identity_cache: + val = cls.identity_cache(session, 'username', username) + if val: + return val + else: + cache_key = "get_user_by_name_%s" % _hash_key(username) + q = q.options( + FromCache("sql_cache_short", cache_key)) + + return q.scalar() + + @classmethod + def get_by_auth_token(cls, auth_token, cache=False): + q = UserApiKeys.query()\ + .filter(UserApiKeys.api_key == auth_token)\ + .filter(or_(UserApiKeys.expires == -1, + UserApiKeys.expires >= time.time())) + if cache: + q = q.options( + FromCache("sql_cache_short", "get_auth_token_%s" % auth_token)) + + match = q.first() + if match: + return match.user + + @classmethod + def get_by_email(cls, email, case_insensitive=False, cache=False): + + if case_insensitive: + q = cls.query().filter(func.lower(cls.email) == func.lower(email)) + + else: + q = cls.query().filter(cls.email == email) + + email_key = _hash_key(email) + if cache: + q = q.options( + FromCache("sql_cache_short", "get_email_key_%s" % email_key)) + + ret = q.scalar() + if ret is None: + q = UserEmailMap.query() + # try fetching in alternate email map + if case_insensitive: + q = q.filter(func.lower(UserEmailMap.email) == func.lower(email)) + else: + q = q.filter(UserEmailMap.email == email) + q = q.options(joinedload(UserEmailMap.user)) + if cache: + q = q.options( + FromCache("sql_cache_short", "get_email_map_key_%s" % email_key)) + ret = getattr(q.scalar(), 'user', None) + + return ret + + @classmethod + def get_from_cs_author(cls, author): + """ + Tries to get User objects out of commit author string + + :param author: + """ + from rhodecode.lib.helpers import email, author_name + # Valid email in the attribute passed, see if they're in the system + _email = email(author) + if _email: + user = cls.get_by_email(_email, case_insensitive=True) + if user: + return user + # Maybe we can match by username? + _author = author_name(author) + user = cls.get_by_username(_author, case_insensitive=True) + if user: + return user + + def update_userdata(self, **kwargs): + usr = self + old = usr.user_data + old.update(**kwargs) + usr.user_data = old + Session().add(usr) + log.debug('updated userdata with ', kwargs) + + def update_lastlogin(self): + """Update user lastlogin""" + self.last_login = datetime.datetime.now() + Session().add(self) + log.debug('updated user %s lastlogin', self.username) + + def update_password(self, new_password): + from rhodecode.lib.auth import get_crypt_password + + self.password = get_crypt_password(new_password) + Session().add(self) + + @classmethod + def get_first_super_admin(cls): + user = User.query()\ + .filter(User.admin == true()) \ + .order_by(User.user_id.asc()) \ + .first() + + if user is None: + raise Exception('FATAL: Missing administrative account!') + return user + + @classmethod + def get_all_super_admins(cls, only_active=False): + """ + Returns all admin accounts sorted by username + """ + qry = User.query().filter(User.admin == true()).order_by(User.username.asc()) + if only_active: + qry = qry.filter(User.active == true()) + return qry.all() + + @classmethod + def get_default_user(cls, cache=False, refresh=False): + user = User.get_by_username(User.DEFAULT_USER, cache=cache) + if user is None: + raise Exception('FATAL: Missing default account!') + if refresh: + # The default user might be based on outdated state which + # has been loaded from the cache. + # A call to refresh() ensures that the + # latest state from the database is used. + Session().refresh(user) + return user + + def _get_default_perms(self, user, suffix=''): + from rhodecode.model.permission import PermissionModel + return PermissionModel().get_default_perms(user.user_perms, suffix) + + def get_default_perms(self, suffix=''): + return self._get_default_perms(self, suffix) + + def get_api_data(self, include_secrets=False, details='full'): + """ + Common function for generating user related data for API + + :param include_secrets: By default secrets in the API data will be replaced + by a placeholder value to prevent exposing this data by accident. In case + this data shall be exposed, set this flag to ``True``. + + :param details: details can be 'basic|full' basic gives only a subset of + the available user information that includes user_id, name and emails. + """ + user = self + user_data = self.user_data + data = { + 'user_id': user.user_id, + 'username': user.username, + 'firstname': user.name, + 'lastname': user.lastname, + 'email': user.email, + 'emails': user.emails, + } + if details == 'basic': + return data + + auth_token_length = 40 + auth_token_replacement = '*' * auth_token_length + + extras = { + 'auth_tokens': [auth_token_replacement], + 'active': user.active, + 'admin': user.admin, + 'extern_type': user.extern_type, + 'extern_name': user.extern_name, + 'last_login': user.last_login, + 'last_activity': user.last_activity, + 'ip_addresses': user.ip_addresses, + 'language': user_data.get('language') + } + data.update(extras) + + if include_secrets: + data['auth_tokens'] = user.auth_tokens + return data + + def __json__(self): + data = { + 'full_name': self.full_name, + 'full_name_or_username': self.full_name_or_username, + 'short_contact': self.short_contact, + 'full_contact': self.full_contact, + } + data.update(self.get_api_data()) + return data + + +class UserApiKeys(Base, BaseModel): + __tablename__ = 'user_api_keys' + __table_args__ = ( + Index('uak_api_key_idx', 'api_key', unique=True), + Index('uak_api_key_expires_idx', 'api_key', 'expires'), + base_table_args + ) + __mapper_args__ = {} + + # ApiKey role + ROLE_ALL = 'token_role_all' + ROLE_HTTP = 'token_role_http' + ROLE_VCS = 'token_role_vcs' + ROLE_API = 'token_role_api' + ROLE_FEED = 'token_role_feed' + ROLE_PASSWORD_RESET = 'token_password_reset' + + ROLES = [ROLE_ALL, ROLE_HTTP, ROLE_VCS, ROLE_API, ROLE_FEED] + + user_api_key_id = Column("user_api_key_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None) + api_key = Column("api_key", String(255), nullable=False, unique=True) + description = Column('description', UnicodeText().with_variant(UnicodeText(1024), 'mysql')) + expires = Column('expires', Float(53), nullable=False) + role = Column('role', String(255), nullable=True) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + + # scope columns + repo_id = Column( + 'repo_id', Integer(), ForeignKey('repositories.repo_id'), + nullable=True, unique=None, default=None) + repo = relationship('Repository', lazy='joined') + + repo_group_id = Column( + 'repo_group_id', Integer(), ForeignKey('groups.group_id'), + nullable=True, unique=None, default=None) + repo_group = relationship('RepoGroup', lazy='joined') + + user = relationship('User', lazy='joined') + + def __unicode__(self): + return u"<%s('%s')>" % (self.__class__.__name__, self.role) + + def __json__(self): + data = { + 'auth_token': self.api_key, + 'role': self.role, + 'scope': self.scope_humanized, + 'expired': self.expired + } + return data + + def get_api_data(self, include_secrets=False): + data = self.__json__() + if include_secrets: + return data + else: + data['auth_token'] = self.token_obfuscated + return data + + @hybrid_property + def description_safe(self): + from rhodecode.lib import helpers as h + return h.escape(self.description) + + @property + def expired(self): + if self.expires == -1: + return False + return time.time() > self.expires + + @classmethod + def _get_role_name(cls, role): + return { + cls.ROLE_ALL: _('all'), + cls.ROLE_HTTP: _('http/web interface'), + cls.ROLE_VCS: _('vcs (git/hg/svn protocol)'), + cls.ROLE_API: _('api calls'), + cls.ROLE_FEED: _('feed access'), + }.get(role, role) + + @property + def role_humanized(self): + return self._get_role_name(self.role) + + def _get_scope(self): + if self.repo: + return 'Repository: {}'.format(self.repo.repo_name) + if self.repo_group: + return 'RepositoryGroup: {} (recursive)'.format(self.repo_group.group_name) + return 'Global' + + @property + def scope_humanized(self): + return self._get_scope() + + @property + def token_obfuscated(self): + if self.api_key: + return self.api_key[:4] + "****" + + +class UserEmailMap(Base, BaseModel): + __tablename__ = 'user_email_map' + __table_args__ = ( + Index('uem_email_idx', 'email'), + UniqueConstraint('email'), + base_table_args + ) + __mapper_args__ = {} + + email_id = Column("email_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None) + _email = Column("email", String(255), nullable=True, unique=False, default=None) + user = relationship('User', lazy='joined') + + @validates('_email') + def validate_email(self, key, email): + # check if this email is not main one + main_email = Session().query(User).filter(User.email == email).scalar() + if main_email is not None: + raise AttributeError('email %s is present is user table' % email) + return email + + @hybrid_property + def email(self): + return self._email + + @email.setter + def email(self, val): + self._email = val.lower() if val else None + + +class UserIpMap(Base, BaseModel): + __tablename__ = 'user_ip_map' + __table_args__ = ( + UniqueConstraint('user_id', 'ip_addr'), + base_table_args + ) + __mapper_args__ = {} + + ip_id = Column("ip_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None) + ip_addr = Column("ip_addr", String(255), nullable=True, unique=False, default=None) + active = Column("active", Boolean(), nullable=True, unique=None, default=True) + description = Column("description", String(10000), nullable=True, unique=None, default=None) + user = relationship('User', lazy='joined') + + @hybrid_property + def description_safe(self): + from rhodecode.lib import helpers as h + return h.escape(self.description) + + @classmethod + def _get_ip_range(cls, ip_addr): + net = ipaddress.ip_network(safe_unicode(ip_addr), strict=False) + return [str(net.network_address), str(net.broadcast_address)] + + def __json__(self): + return { + 'ip_addr': self.ip_addr, + 'ip_range': self._get_ip_range(self.ip_addr), + } + + def __unicode__(self): + return u"<%s('user_id:%s=>%s')>" % (self.__class__.__name__, + self.user_id, self.ip_addr) + + +class UserSshKeys(Base, BaseModel): + __tablename__ = 'user_ssh_keys' + __table_args__ = ( + Index('usk_ssh_key_fingerprint_idx', 'ssh_key_fingerprint'), + + UniqueConstraint('ssh_key_fingerprint'), + + base_table_args + ) + __mapper_args__ = {} + + ssh_key_id = Column('ssh_key_id', Integer(), nullable=False, unique=True, default=None, primary_key=True) + ssh_key_data = Column('ssh_key_data', String(10240), nullable=False, unique=None, default=None) + ssh_key_fingerprint = Column('ssh_key_fingerprint', String(255), nullable=False, unique=None, default=None) + + description = Column('description', UnicodeText().with_variant(UnicodeText(1024), 'mysql')) + + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + accessed_on = Column('accessed_on', DateTime(timezone=False), nullable=True, default=None) + user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None) + + user = relationship('User', lazy='joined') + + def __json__(self): + data = { + 'ssh_fingerprint': self.ssh_key_fingerprint, + 'description': self.description, + 'created_on': self.created_on + } + return data + + def get_api_data(self): + data = self.__json__() + return data + + +class UserLog(Base, BaseModel): + __tablename__ = 'user_logs' + __table_args__ = ( + base_table_args, + ) + + VERSION_1 = 'v1' + VERSION_2 = 'v2' + VERSIONS = [VERSION_1, VERSION_2] + + user_log_id = Column("user_log_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id',ondelete='SET NULL'), nullable=True, unique=None, default=None) + username = Column("username", String(255), nullable=True, unique=None, default=None) + repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id', ondelete='SET NULL'), nullable=True, unique=None, default=None) + repository_name = Column("repository_name", String(255), nullable=True, unique=None, default=None) + user_ip = Column("user_ip", String(255), nullable=True, unique=None, default=None) + action = Column("action", Text().with_variant(Text(1200000), 'mysql'), nullable=True, unique=None, default=None) + action_date = Column("action_date", DateTime(timezone=False), nullable=True, unique=None, default=None) + + version = Column("version", String(255), nullable=True, default=VERSION_1) + user_data = Column('user_data_json', MutationObj.as_mutable(JsonType(dialect_map=dict(mysql=LONGTEXT())))) + action_data = Column('action_data_json', MutationObj.as_mutable(JsonType(dialect_map=dict(mysql=LONGTEXT())))) + + def __unicode__(self): + return u"<%s('id:%s:%s')>" % ( + self.__class__.__name__, self.repository_name, self.action) + + def __json__(self): + return { + 'user_id': self.user_id, + 'username': self.username, + 'repository_id': self.repository_id, + 'repository_name': self.repository_name, + 'user_ip': self.user_ip, + 'action_date': self.action_date, + 'action': self.action, + } + + @hybrid_property + def entry_id(self): + return self.user_log_id + + @property + def action_as_day(self): + return datetime.date(*self.action_date.timetuple()[:3]) + + user = relationship('User') + repository = relationship('Repository', cascade='') + + +class UserGroup(Base, BaseModel): + __tablename__ = 'users_groups' + __table_args__ = ( + base_table_args, + ) + + users_group_id = Column("users_group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + users_group_name = Column("users_group_name", String(255), nullable=False, unique=True, default=None) + user_group_description = Column("user_group_description", String(10000), nullable=True, unique=None, default=None) + users_group_active = Column("users_group_active", Boolean(), nullable=True, unique=None, default=None) + inherit_default_permissions = Column("users_group_inherit_default_permissions", Boolean(), nullable=False, unique=None, default=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=False, default=None) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + _group_data = Column("group_data", LargeBinary(), nullable=True) # JSON data + + members = relationship('UserGroupMember', cascade="all, delete, delete-orphan", lazy="joined") + users_group_to_perm = relationship('UserGroupToPerm', cascade='all') + users_group_repo_to_perm = relationship('UserGroupRepoToPerm', cascade='all') + users_group_repo_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all') + user_user_group_to_perm = relationship('UserUserGroupToPerm', cascade='all') + user_group_user_group_to_perm = relationship('UserGroupUserGroupToPerm ', primaryjoin="UserGroupUserGroupToPerm.target_user_group_id==UserGroup.users_group_id", cascade='all') + + user_group_review_rules = relationship('RepoReviewRuleUserGroup', cascade='all') + user = relationship('User', primaryjoin="User.user_id==UserGroup.user_id") + + @classmethod + def _load_group_data(cls, column): + if not column: + return {} + + try: + return json.loads(column) or {} + except TypeError: + return {} + + @hybrid_property + def description_safe(self): + from rhodecode.lib import helpers as h + return h.escape(self.user_group_description) + + @hybrid_property + def group_data(self): + return self._load_group_data(self._group_data) + + @group_data.expression + def group_data(self, **kwargs): + return self._group_data + + @group_data.setter + def group_data(self, val): + try: + self._group_data = json.dumps(val) + except Exception: + log.error(traceback.format_exc()) + + @classmethod + def _load_sync(cls, group_data): + if group_data: + return group_data.get('extern_type') + + @property + def sync(self): + return self._load_sync(self.group_data) + + def __unicode__(self): + return u"<%s('id:%s:%s')>" % (self.__class__.__name__, + self.users_group_id, + self.users_group_name) + + @classmethod + def get_by_group_name(cls, group_name, cache=False, + case_insensitive=False): + if case_insensitive: + q = cls.query().filter(func.lower(cls.users_group_name) == + func.lower(group_name)) + + else: + q = cls.query().filter(cls.users_group_name == group_name) + if cache: + q = q.options( + FromCache("sql_cache_short", "get_group_%s" % _hash_key(group_name))) + return q.scalar() + + @classmethod + def get(cls, user_group_id, cache=False): + if not user_group_id: + return + + user_group = cls.query() + if cache: + user_group = user_group.options( + FromCache("sql_cache_short", "get_users_group_%s" % user_group_id)) + return user_group.get(user_group_id) + + def permissions(self, with_admins=True, with_owner=True, + expand_from_user_groups=False): + """ + Permissions for user groups + """ + _admin_perm = 'usergroup.admin' + + owner_row = [] + if with_owner: + usr = AttributeDict(self.user.get_dict()) + usr.owner_row = True + usr.permission = _admin_perm + owner_row.append(usr) + + super_admin_ids = [] + super_admin_rows = [] + if with_admins: + for usr in User.get_all_super_admins(): + super_admin_ids.append(usr.user_id) + # if this admin is also owner, don't double the record + if usr.user_id == owner_row[0].user_id: + owner_row[0].admin_row = True + else: + usr = AttributeDict(usr.get_dict()) + usr.admin_row = True + usr.permission = _admin_perm + super_admin_rows.append(usr) + + q = UserUserGroupToPerm.query().filter(UserUserGroupToPerm.user_group == self) + q = q.options(joinedload(UserUserGroupToPerm.user_group), + joinedload(UserUserGroupToPerm.user), + joinedload(UserUserGroupToPerm.permission),) + + # get owners and admins and permissions. We do a trick of re-writing + # objects from sqlalchemy to named-tuples due to sqlalchemy session + # has a global reference and changing one object propagates to all + # others. This means if admin is also an owner admin_row that change + # would propagate to both objects + perm_rows = [] + for _usr in q.all(): + usr = AttributeDict(_usr.user.get_dict()) + # if this user is also owner/admin, mark as duplicate record + if usr.user_id == owner_row[0].user_id or usr.user_id in super_admin_ids: + usr.duplicate_perm = True + usr.permission = _usr.permission.permission_name + perm_rows.append(usr) + + # filter the perm rows by 'default' first and then sort them by + # admin,write,read,none permissions sorted again alphabetically in + # each group + perm_rows = sorted(perm_rows, key=display_user_sort) + + user_groups_rows = [] + if expand_from_user_groups: + for ug in self.permission_user_groups(with_members=True): + for user_data in ug.members: + user_groups_rows.append(user_data) + + return super_admin_rows + owner_row + perm_rows + user_groups_rows + + def permission_user_groups(self, with_members=False): + q = UserGroupUserGroupToPerm.query()\ + .filter(UserGroupUserGroupToPerm.target_user_group == self) + q = q.options(joinedload(UserGroupUserGroupToPerm.user_group), + joinedload(UserGroupUserGroupToPerm.target_user_group), + joinedload(UserGroupUserGroupToPerm.permission),) + + perm_rows = [] + for _user_group in q.all(): + entry = AttributeDict(_user_group.user_group.get_dict()) + entry.permission = _user_group.permission.permission_name + if with_members: + entry.members = [x.user.get_dict() + for x in _user_group.users_group.members] + perm_rows.append(entry) + + perm_rows = sorted(perm_rows, key=display_user_group_sort) + return perm_rows + + def _get_default_perms(self, user_group, suffix=''): + from rhodecode.model.permission import PermissionModel + return PermissionModel().get_default_perms(user_group.users_group_to_perm, suffix) + + def get_default_perms(self, suffix=''): + return self._get_default_perms(self, suffix) + + def get_api_data(self, with_group_members=True, include_secrets=False): + """ + :param include_secrets: See :meth:`User.get_api_data`, this parameter is + basically forwarded. + + """ + user_group = self + data = { + 'users_group_id': user_group.users_group_id, + 'group_name': user_group.users_group_name, + 'group_description': user_group.user_group_description, + 'active': user_group.users_group_active, + 'owner': user_group.user.username, + 'sync': user_group.sync, + 'owner_email': user_group.user.email, + } + + if with_group_members: + users = [] + for user in user_group.members: + user = user.user + users.append(user.get_api_data(include_secrets=include_secrets)) + data['users'] = users + + return data + + +class UserGroupMember(Base, BaseModel): + __tablename__ = 'users_groups_members' + __table_args__ = ( + base_table_args, + ) + + users_group_member_id = Column("users_group_member_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) + + user = relationship('User', lazy='joined') + users_group = relationship('UserGroup') + + def __init__(self, gr_id='', u_id=''): + self.users_group_id = gr_id + self.user_id = u_id + + +class RepositoryField(Base, BaseModel): + __tablename__ = 'repositories_fields' + __table_args__ = ( + UniqueConstraint('repository_id', 'field_key'), # no-multi field + base_table_args, + ) + + PREFIX = 'ex_' # prefix used in form to not conflict with already existing fields + + repo_field_id = Column("repo_field_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None) + field_key = Column("field_key", String(250)) + field_label = Column("field_label", String(1024), nullable=False) + field_value = Column("field_value", String(10000), nullable=False) + field_desc = Column("field_desc", String(1024), nullable=False) + field_type = Column("field_type", String(255), nullable=False, unique=None) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + + repository = relationship('Repository') + + @property + def field_key_prefixed(self): + return 'ex_%s' % self.field_key + + @classmethod + def un_prefix_key(cls, key): + if key.startswith(cls.PREFIX): + return key[len(cls.PREFIX):] + return key + + @classmethod + def get_by_key_name(cls, key, repo): + row = cls.query()\ + .filter(cls.repository == repo)\ + .filter(cls.field_key == key).scalar() + return row + + +class Repository(Base, BaseModel): + __tablename__ = 'repositories' + __table_args__ = ( + Index('r_repo_name_idx', 'repo_name', mysql_length=255), + base_table_args, + ) + DEFAULT_CLONE_URI = '{scheme}://{user}@{netloc}/{repo}' + DEFAULT_CLONE_URI_ID = '{scheme}://{user}@{netloc}/_{repoid}' + DEFAULT_CLONE_URI_SSH = 'ssh://{sys_user}@{hostname}/{repo}' + + STATE_CREATED = 'repo_state_created' + STATE_PENDING = 'repo_state_pending' + STATE_ERROR = 'repo_state_error' + + LOCK_AUTOMATIC = 'lock_auto' + LOCK_API = 'lock_api' + LOCK_WEB = 'lock_web' + LOCK_PULL = 'lock_pull' + + NAME_SEP = URL_SEP + + repo_id = Column( + "repo_id", Integer(), nullable=False, unique=True, default=None, + primary_key=True) + _repo_name = Column( + "repo_name", Text(), nullable=False, default=None) + _repo_name_hash = Column( + "repo_name_hash", String(255), nullable=False, unique=True) + repo_state = Column("repo_state", String(255), nullable=True) + + clone_uri = Column( + "clone_uri", EncryptedTextValue(), nullable=True, unique=False, + default=None) + push_uri = Column( + "push_uri", EncryptedTextValue(), nullable=True, unique=False, + default=None) + repo_type = Column( + "repo_type", String(255), nullable=False, unique=False, default=None) + user_id = Column( + "user_id", Integer(), ForeignKey('users.user_id'), nullable=False, + unique=False, default=None) + private = Column( + "private", Boolean(), nullable=True, unique=None, default=None) + archived = Column( + "archived", Boolean(), nullable=True, unique=None, default=None) + enable_statistics = Column( + "statistics", Boolean(), nullable=True, unique=None, default=True) + enable_downloads = Column( + "downloads", Boolean(), nullable=True, unique=None, default=True) + description = Column( + "description", String(10000), nullable=True, unique=None, default=None) + created_on = Column( + 'created_on', DateTime(timezone=False), nullable=True, unique=None, + default=datetime.datetime.now) + updated_on = Column( + 'updated_on', DateTime(timezone=False), nullable=True, unique=None, + default=datetime.datetime.now) + _landing_revision = Column( + "landing_revision", String(255), nullable=False, unique=False, + default=None) + enable_locking = Column( + "enable_locking", Boolean(), nullable=False, unique=None, + default=False) + _locked = Column( + "locked", String(255), nullable=True, unique=False, default=None) + _changeset_cache = Column( + "changeset_cache", LargeBinary(), nullable=True) # JSON data + + fork_id = Column( + "fork_id", Integer(), ForeignKey('repositories.repo_id'), + nullable=True, unique=False, default=None) + group_id = Column( + "group_id", Integer(), ForeignKey('groups.group_id'), nullable=True, + unique=False, default=None) + + user = relationship('User', lazy='joined') + fork = relationship('Repository', remote_side=repo_id, lazy='joined') + group = relationship('RepoGroup', lazy='joined') + repo_to_perm = relationship( + 'UserRepoToPerm', cascade='all', + order_by='UserRepoToPerm.repo_to_perm_id') + users_group_to_perm = relationship('UserGroupRepoToPerm', cascade='all') + stats = relationship('Statistics', cascade='all', uselist=False) + + followers = relationship( + 'UserFollowing', + primaryjoin='UserFollowing.follows_repo_id==Repository.repo_id', + cascade='all') + extra_fields = relationship( + 'RepositoryField', cascade="all, delete, delete-orphan") + logs = relationship('UserLog') + comments = relationship( + 'ChangesetComment', cascade="all, delete, delete-orphan") + pull_requests_source = relationship( + 'PullRequest', + primaryjoin='PullRequest.source_repo_id==Repository.repo_id', + cascade="all, delete, delete-orphan") + pull_requests_target = relationship( + 'PullRequest', + primaryjoin='PullRequest.target_repo_id==Repository.repo_id', + cascade="all, delete, delete-orphan") + ui = relationship('RepoRhodeCodeUi', cascade="all") + settings = relationship('RepoRhodeCodeSetting', cascade="all") + integrations = relationship('Integration', + cascade="all, delete, delete-orphan") + + scoped_tokens = relationship('UserApiKeys', cascade="all") + + def __unicode__(self): + return u"<%s('%s:%s')>" % (self.__class__.__name__, self.repo_id, + safe_unicode(self.repo_name)) + + @hybrid_property + def description_safe(self): + from rhodecode.lib import helpers as h + return h.escape(self.description) + + @hybrid_property + def landing_rev(self): + # always should return [rev_type, rev] + if self._landing_revision: + _rev_info = self._landing_revision.split(':') + if len(_rev_info) < 2: + _rev_info.insert(0, 'rev') + return [_rev_info[0], _rev_info[1]] + return [None, None] + + @landing_rev.setter + def landing_rev(self, val): + if ':' not in val: + raise ValueError('value must be delimited with `:` and consist ' + 'of :, got %s instead' % val) + self._landing_revision = val + + @hybrid_property + def locked(self): + if self._locked: + user_id, timelocked, reason = self._locked.split(':') + lock_values = int(user_id), timelocked, reason + else: + lock_values = [None, None, None] + return lock_values + + @locked.setter + def locked(self, val): + if val and isinstance(val, (list, tuple)): + self._locked = ':'.join(map(str, val)) + else: + self._locked = None + + @hybrid_property + def changeset_cache(self): + from rhodecode.lib.vcs.backends.base import EmptyCommit + dummy = EmptyCommit().__json__() + if not self._changeset_cache: + return dummy + try: + return json.loads(self._changeset_cache) + except TypeError: + return dummy + except Exception: + log.error(traceback.format_exc()) + return dummy + + @changeset_cache.setter + def changeset_cache(self, val): + try: + self._changeset_cache = json.dumps(val) + except Exception: + log.error(traceback.format_exc()) + + @hybrid_property + def repo_name(self): + return self._repo_name + + @repo_name.setter + def repo_name(self, value): + self._repo_name = value + self._repo_name_hash = hashlib.sha1(safe_str(value)).hexdigest() + + @classmethod + def normalize_repo_name(cls, repo_name): + """ + Normalizes os specific repo_name to the format internally stored inside + database using URL_SEP + + :param cls: + :param repo_name: + """ + return cls.NAME_SEP.join(repo_name.split(os.sep)) + + @classmethod + def get_by_repo_name(cls, repo_name, cache=False, identity_cache=False): + session = Session() + q = session.query(cls).filter(cls.repo_name == repo_name) + + if cache: + if identity_cache: + val = cls.identity_cache(session, 'repo_name', repo_name) + if val: + return val + else: + cache_key = "get_repo_by_name_%s" % _hash_key(repo_name) + q = q.options( + FromCache("sql_cache_short", cache_key)) + + return q.scalar() + + @classmethod + def get_by_id_or_repo_name(cls, repoid): + if isinstance(repoid, (int, long)): + try: + repo = cls.get(repoid) + except ValueError: + repo = None + else: + repo = cls.get_by_repo_name(repoid) + return repo + + @classmethod + def get_by_full_path(cls, repo_full_path): + repo_name = repo_full_path.split(cls.base_path(), 1)[-1] + repo_name = cls.normalize_repo_name(repo_name) + return cls.get_by_repo_name(repo_name.strip(URL_SEP)) + + @classmethod + def get_repo_forks(cls, repo_id): + return cls.query().filter(Repository.fork_id == repo_id) + + @classmethod + def base_path(cls): + """ + Returns base path when all repos are stored + + :param cls: + """ + q = Session().query(RhodeCodeUi)\ + .filter(RhodeCodeUi.ui_key == cls.NAME_SEP) + q = q.options(FromCache("sql_cache_short", "repository_repo_path")) + return q.one().ui_value + + @classmethod + def get_all_repos(cls, user_id=Optional(None), group_id=Optional(None), + case_insensitive=True, archived=False): + q = Repository.query() + + if not archived: + q = q.filter(Repository.archived.isnot(true())) + + if not isinstance(user_id, Optional): + q = q.filter(Repository.user_id == user_id) + + if not isinstance(group_id, Optional): + q = q.filter(Repository.group_id == group_id) + + if case_insensitive: + q = q.order_by(func.lower(Repository.repo_name)) + else: + q = q.order_by(Repository.repo_name) + + return q.all() + + @property + def forks(self): + """ + Return forks of this repo + """ + return Repository.get_repo_forks(self.repo_id) + + @property + def parent(self): + """ + Returns fork parent + """ + return self.fork + + @property + def just_name(self): + return self.repo_name.split(self.NAME_SEP)[-1] + + @property + def groups_with_parents(self): + groups = [] + if self.group is None: + return groups + + cur_gr = self.group + groups.insert(0, cur_gr) + while 1: + gr = getattr(cur_gr, 'parent_group', None) + cur_gr = cur_gr.parent_group + if gr is None: + break + groups.insert(0, gr) + + return groups + + @property + def groups_and_repo(self): + return self.groups_with_parents, self + + @LazyProperty + def repo_path(self): + """ + Returns base full path for that repository means where it actually + exists on a filesystem + """ + q = Session().query(RhodeCodeUi).filter( + RhodeCodeUi.ui_key == self.NAME_SEP) + q = q.options(FromCache("sql_cache_short", "repository_repo_path")) + return q.one().ui_value + + @property + def repo_full_path(self): + p = [self.repo_path] + # we need to split the name by / since this is how we store the + # names in the database, but that eventually needs to be converted + # into a valid system path + p += self.repo_name.split(self.NAME_SEP) + return os.path.join(*map(safe_unicode, p)) + + @property + def cache_keys(self): + """ + Returns associated cache keys for that repo + """ + invalidation_namespace = CacheKey.REPO_INVALIDATION_NAMESPACE.format( + repo_id=self.repo_id) + return CacheKey.query()\ + .filter(CacheKey.cache_args == invalidation_namespace)\ + .order_by(CacheKey.cache_key)\ + .all() + + @property + def cached_diffs_relative_dir(self): + """ + Return a relative to the repository store path of cached diffs + used for safe display for users, who shouldn't know the absolute store + path + """ + return os.path.join( + os.path.dirname(self.repo_name), + self.cached_diffs_dir.split(os.path.sep)[-1]) + + @property + def cached_diffs_dir(self): + path = self.repo_full_path + return os.path.join( + os.path.dirname(path), + '.__shadow_diff_cache_repo_{}'.format(self.repo_id)) + + def cached_diffs(self): + diff_cache_dir = self.cached_diffs_dir + if os.path.isdir(diff_cache_dir): + return os.listdir(diff_cache_dir) + return [] + + def shadow_repos(self): + shadow_repos_pattern = '.__shadow_repo_{}'.format(self.repo_id) + return [ + x for x in os.listdir(os.path.dirname(self.repo_full_path)) + if x.startswith(shadow_repos_pattern)] + + def get_new_name(self, repo_name): + """ + returns new full repository name based on assigned group and new new + + :param group_name: + """ + path_prefix = self.group.full_path_splitted if self.group else [] + return self.NAME_SEP.join(path_prefix + [repo_name]) + + @property + def _config(self): + """ + Returns db based config object. + """ + from rhodecode.lib.utils import make_db_config + return make_db_config(clear_session=False, repo=self) + + def permissions(self, with_admins=True, with_owner=True, + expand_from_user_groups=False): + """ + Permissions for repositories + """ + _admin_perm = 'repository.admin' + + owner_row = [] + if with_owner: + usr = AttributeDict(self.user.get_dict()) + usr.owner_row = True + usr.permission = _admin_perm + usr.permission_id = None + owner_row.append(usr) + + super_admin_ids = [] + super_admin_rows = [] + if with_admins: + for usr in User.get_all_super_admins(): + super_admin_ids.append(usr.user_id) + # if this admin is also owner, don't double the record + if usr.user_id == owner_row[0].user_id: + owner_row[0].admin_row = True + else: + usr = AttributeDict(usr.get_dict()) + usr.admin_row = True + usr.permission = _admin_perm + usr.permission_id = None + super_admin_rows.append(usr) + + q = UserRepoToPerm.query().filter(UserRepoToPerm.repository == self) + q = q.options(joinedload(UserRepoToPerm.repository), + joinedload(UserRepoToPerm.user), + joinedload(UserRepoToPerm.permission),) + + # get owners and admins and permissions. We do a trick of re-writing + # objects from sqlalchemy to named-tuples due to sqlalchemy session + # has a global reference and changing one object propagates to all + # others. This means if admin is also an owner admin_row that change + # would propagate to both objects + perm_rows = [] + for _usr in q.all(): + usr = AttributeDict(_usr.user.get_dict()) + # if this user is also owner/admin, mark as duplicate record + if usr.user_id == owner_row[0].user_id or usr.user_id in super_admin_ids: + usr.duplicate_perm = True + # also check if this permission is maybe used by branch_permissions + if _usr.branch_perm_entry: + usr.branch_rules = [x.branch_rule_id for x in _usr.branch_perm_entry] + + usr.permission = _usr.permission.permission_name + usr.permission_id = _usr.repo_to_perm_id + perm_rows.append(usr) + + # filter the perm rows by 'default' first and then sort them by + # admin,write,read,none permissions sorted again alphabetically in + # each group + perm_rows = sorted(perm_rows, key=display_user_sort) + + user_groups_rows = [] + if expand_from_user_groups: + for ug in self.permission_user_groups(with_members=True): + for user_data in ug.members: + user_groups_rows.append(user_data) + + return super_admin_rows + owner_row + perm_rows + user_groups_rows + + def permission_user_groups(self, with_members=True): + q = UserGroupRepoToPerm.query()\ + .filter(UserGroupRepoToPerm.repository == self) + q = q.options(joinedload(UserGroupRepoToPerm.repository), + joinedload(UserGroupRepoToPerm.users_group), + joinedload(UserGroupRepoToPerm.permission),) + + perm_rows = [] + for _user_group in q.all(): + entry = AttributeDict(_user_group.users_group.get_dict()) + entry.permission = _user_group.permission.permission_name + if with_members: + entry.members = [x.user.get_dict() + for x in _user_group.users_group.members] + perm_rows.append(entry) + + perm_rows = sorted(perm_rows, key=display_user_group_sort) + return perm_rows + + def get_api_data(self, include_secrets=False): + """ + Common function for generating repo api data + + :param include_secrets: See :meth:`User.get_api_data`. + + """ + # TODO: mikhail: Here there is an anti-pattern, we probably need to + # move this methods on models level. + from rhodecode.model.settings import SettingsModel + from rhodecode.model.repo import RepoModel + + repo = self + _user_id, _time, _reason = self.locked + + data = { + 'repo_id': repo.repo_id, + 'repo_name': repo.repo_name, + 'repo_type': repo.repo_type, + 'clone_uri': repo.clone_uri or '', + 'push_uri': repo.push_uri or '', + 'url': RepoModel().get_url(self), + 'private': repo.private, + 'created_on': repo.created_on, + 'description': repo.description_safe, + 'landing_rev': repo.landing_rev, + 'owner': repo.user.username, + 'fork_of': repo.fork.repo_name if repo.fork else None, + 'fork_of_id': repo.fork.repo_id if repo.fork else None, + 'enable_statistics': repo.enable_statistics, + 'enable_locking': repo.enable_locking, + 'enable_downloads': repo.enable_downloads, + 'last_changeset': repo.changeset_cache, + 'locked_by': User.get(_user_id).get_api_data( + include_secrets=include_secrets) if _user_id else None, + 'locked_date': time_to_datetime(_time) if _time else None, + 'lock_reason': _reason if _reason else None, + } + + # TODO: mikhail: should be per-repo settings here + rc_config = SettingsModel().get_all_settings() + repository_fields = str2bool( + rc_config.get('rhodecode_repository_fields')) + if repository_fields: + for f in self.extra_fields: + data[f.field_key_prefixed] = f.field_value + + return data + + @classmethod + def lock(cls, repo, user_id, lock_time=None, lock_reason=None): + if not lock_time: + lock_time = time.time() + if not lock_reason: + lock_reason = cls.LOCK_AUTOMATIC + repo.locked = [user_id, lock_time, lock_reason] + Session().add(repo) + Session().commit() + + @classmethod + def unlock(cls, repo): + repo.locked = None + Session().add(repo) + Session().commit() + + @classmethod + def getlock(cls, repo): + return repo.locked + + def is_user_lock(self, user_id): + if self.lock[0]: + lock_user_id = safe_int(self.lock[0]) + user_id = safe_int(user_id) + # both are ints, and they are equal + return all([lock_user_id, user_id]) and lock_user_id == user_id + + return False + + def get_locking_state(self, action, user_id, only_when_enabled=True): + """ + Checks locking on this repository, if locking is enabled and lock is + present returns a tuple of make_lock, locked, locked_by. + make_lock can have 3 states None (do nothing) True, make lock + False release lock, This value is later propagated to hooks, which + do the locking. Think about this as signals passed to hooks what to do. + + """ + # TODO: johbo: This is part of the business logic and should be moved + # into the RepositoryModel. + + if action not in ('push', 'pull'): + raise ValueError("Invalid action value: %s" % repr(action)) + + # defines if locked error should be thrown to user + currently_locked = False + # defines if new lock should be made, tri-state + make_lock = None + repo = self + user = User.get(user_id) + + lock_info = repo.locked + + if repo and (repo.enable_locking or not only_when_enabled): + if action == 'push': + # check if it's already locked !, if it is compare users + locked_by_user_id = lock_info[0] + if user.user_id == locked_by_user_id: + log.debug( + 'Got `push` action from user %s, now unlocking', user) + # unlock if we have push from user who locked + make_lock = False + else: + # we're not the same user who locked, ban with + # code defined in settings (default is 423 HTTP Locked) ! + log.debug('Repo %s is currently locked by %s', repo, user) + currently_locked = True + elif action == 'pull': + # [0] user [1] date + if lock_info[0] and lock_info[1]: + log.debug('Repo %s is currently locked by %s', repo, user) + currently_locked = True + else: + log.debug('Setting lock on repo %s by %s', repo, user) + make_lock = True + + else: + log.debug('Repository %s do not have locking enabled', repo) + + log.debug('FINAL locking values make_lock:%s,locked:%s,locked_by:%s', + make_lock, currently_locked, lock_info) + + from rhodecode.lib.auth import HasRepoPermissionAny + perm_check = HasRepoPermissionAny('repository.write', 'repository.admin') + if make_lock and not perm_check(repo_name=repo.repo_name, user=user): + # if we don't have at least write permission we cannot make a lock + log.debug('lock state reset back to FALSE due to lack ' + 'of at least read permission') + make_lock = False + + return make_lock, currently_locked, lock_info + + @property + def last_db_change(self): + return self.updated_on + + @property + def clone_uri_hidden(self): + clone_uri = self.clone_uri + if clone_uri: + import urlobject + url_obj = urlobject.URLObject(cleaned_uri(clone_uri)) + if url_obj.password: + clone_uri = url_obj.with_password('*****') + return clone_uri + + @property + def push_uri_hidden(self): + push_uri = self.push_uri + if push_uri: + import urlobject + url_obj = urlobject.URLObject(cleaned_uri(push_uri)) + if url_obj.password: + push_uri = url_obj.with_password('*****') + return push_uri + + def clone_url(self, **override): + from rhodecode.model.settings import SettingsModel + + uri_tmpl = None + if 'with_id' in override: + uri_tmpl = self.DEFAULT_CLONE_URI_ID + del override['with_id'] + + if 'uri_tmpl' in override: + uri_tmpl = override['uri_tmpl'] + del override['uri_tmpl'] + + ssh = False + if 'ssh' in override: + ssh = True + del override['ssh'] + + # we didn't override our tmpl from **overrides + if not uri_tmpl: + rc_config = SettingsModel().get_all_settings(cache=True) + if ssh: + uri_tmpl = rc_config.get( + 'rhodecode_clone_uri_ssh_tmpl') or self.DEFAULT_CLONE_URI_SSH + else: + uri_tmpl = rc_config.get( + 'rhodecode_clone_uri_tmpl') or self.DEFAULT_CLONE_URI + + request = get_current_request() + return get_clone_url(request=request, + uri_tmpl=uri_tmpl, + repo_name=self.repo_name, + repo_id=self.repo_id, **override) + + def set_state(self, state): + self.repo_state = state + Session().add(self) + #========================================================================== + # SCM PROPERTIES + #========================================================================== + + def get_commit(self, commit_id=None, commit_idx=None, pre_load=None): + return get_commit_safe( + self.scm_instance(), commit_id, commit_idx, pre_load=pre_load) + + def get_changeset(self, rev=None, pre_load=None): + warnings.warn("Use get_commit", DeprecationWarning) + commit_id = None + commit_idx = None + if isinstance(rev, compat.string_types): + commit_id = rev + else: + commit_idx = rev + return self.get_commit(commit_id=commit_id, commit_idx=commit_idx, + pre_load=pre_load) + + def get_landing_commit(self): + """ + Returns landing commit, or if that doesn't exist returns the tip + """ + _rev_type, _rev = self.landing_rev + commit = self.get_commit(_rev) + if isinstance(commit, EmptyCommit): + return self.get_commit() + return commit + + def update_commit_cache(self, cs_cache=None, config=None): + """ + Update cache of last changeset for repository, keys should be:: + + short_id + raw_id + revision + parents + message + date + author + + :param cs_cache: + """ + from rhodecode.lib.vcs.backends.base import BaseChangeset + if cs_cache is None: + # use no-cache version here + scm_repo = self.scm_instance(cache=False, config=config) + + empty = not scm_repo or scm_repo.is_empty() + if not empty: + cs_cache = scm_repo.get_commit( + pre_load=["author", "date", "message", "parents"]) + else: + cs_cache = EmptyCommit() + + if isinstance(cs_cache, BaseChangeset): + cs_cache = cs_cache.__json__() + + def is_outdated(new_cs_cache): + if (new_cs_cache['raw_id'] != self.changeset_cache['raw_id'] or + new_cs_cache['revision'] != self.changeset_cache['revision']): + return True + return False + + # check if we have maybe already latest cached revision + if is_outdated(cs_cache) or not self.changeset_cache: + _default = datetime.datetime.utcnow() + last_change = cs_cache.get('date') or _default + if self.updated_on and self.updated_on > last_change: + # we check if last update is newer than the new value + # if yes, we use the current timestamp instead. Imagine you get + # old commit pushed 1y ago, we'd set last update 1y to ago. + last_change = _default + log.debug('updated repo %s with new cs cache %s', + self.repo_name, cs_cache) + self.updated_on = last_change + self.changeset_cache = cs_cache + Session().add(self) + Session().commit() + else: + log.debug('Skipping update_commit_cache for repo:`%s` ' + 'commit already with latest changes', self.repo_name) + + @property + def tip(self): + return self.get_commit('tip') + + @property + def author(self): + return self.tip.author + + @property + def last_change(self): + return self.scm_instance().last_change + + def get_comments(self, revisions=None): + """ + Returns comments for this repository grouped by revisions + + :param revisions: filter query by revisions only + """ + cmts = ChangesetComment.query()\ + .filter(ChangesetComment.repo == self) + if revisions: + cmts = cmts.filter(ChangesetComment.revision.in_(revisions)) + grouped = collections.defaultdict(list) + for cmt in cmts.all(): + grouped[cmt.revision].append(cmt) + return grouped + + def statuses(self, revisions=None): + """ + Returns statuses for this repository + + :param revisions: list of revisions to get statuses for + """ + statuses = ChangesetStatus.query()\ + .filter(ChangesetStatus.repo == self)\ + .filter(ChangesetStatus.version == 0) + + if revisions: + # Try doing the filtering in chunks to avoid hitting limits + size = 500 + status_results = [] + for chunk in xrange(0, len(revisions), size): + status_results += statuses.filter( + ChangesetStatus.revision.in_( + revisions[chunk: chunk+size]) + ).all() + else: + status_results = statuses.all() + + grouped = {} + + # maybe we have open new pullrequest without a status? + stat = ChangesetStatus.STATUS_UNDER_REVIEW + status_lbl = ChangesetStatus.get_status_lbl(stat) + for pr in PullRequest.query().filter(PullRequest.source_repo == self).all(): + for rev in pr.revisions: + pr_id = pr.pull_request_id + pr_repo = pr.target_repo.repo_name + grouped[rev] = [stat, status_lbl, pr_id, pr_repo] + + for stat in status_results: + pr_id = pr_repo = None + if stat.pull_request: + pr_id = stat.pull_request.pull_request_id + pr_repo = stat.pull_request.target_repo.repo_name + grouped[stat.revision] = [str(stat.status), stat.status_lbl, + pr_id, pr_repo] + return grouped + + # ========================================================================== + # SCM CACHE INSTANCE + # ========================================================================== + + def scm_instance(self, **kwargs): + import rhodecode + + # Passing a config will not hit the cache currently only used + # for repo2dbmapper + config = kwargs.pop('config', None) + cache = kwargs.pop('cache', None) + full_cache = str2bool(rhodecode.CONFIG.get('vcs_full_cache')) + # if cache is NOT defined use default global, else we have a full + # control over cache behaviour + if cache is None and full_cache and not config: + return self._get_instance_cached() + return self._get_instance(cache=bool(cache), config=config) + + def _get_instance_cached(self): + from rhodecode.lib import rc_cache + + cache_namespace_uid = 'cache_repo_instance.{}'.format(self.repo_id) + invalidation_namespace = CacheKey.REPO_INVALIDATION_NAMESPACE.format( + repo_id=self.repo_id) + region = rc_cache.get_or_create_region('cache_repo_longterm', cache_namespace_uid) + + @region.conditional_cache_on_arguments(namespace=cache_namespace_uid) + def get_instance_cached(repo_id, context_id): + return self._get_instance() + + # we must use thread scoped cache here, + # because each thread of gevent needs it's own not shared connection and cache + # we also alter `args` so the cache key is individual for every green thread. + inv_context_manager = rc_cache.InvalidationContext( + uid=cache_namespace_uid, invalidation_namespace=invalidation_namespace, + thread_scoped=True) + with inv_context_manager as invalidation_context: + args = (self.repo_id, inv_context_manager.cache_key) + # re-compute and store cache if we get invalidate signal + if invalidation_context.should_invalidate(): + instance = get_instance_cached.refresh(*args) + else: + instance = get_instance_cached(*args) + + log.debug( + 'Repo instance fetched in %.3fs', inv_context_manager.compute_time) + return instance + + def _get_instance(self, cache=True, config=None): + config = config or self._config + custom_wire = { + 'cache': cache # controls the vcs.remote cache + } + repo = get_vcs_instance( + repo_path=safe_str(self.repo_full_path), + config=config, + with_wire=custom_wire, + create=False, + _vcs_alias=self.repo_type) + + return repo + + def __json__(self): + return {'landing_rev': self.landing_rev} + + def get_dict(self): + + # Since we transformed `repo_name` to a hybrid property, we need to + # keep compatibility with the code which uses `repo_name` field. + + result = super(Repository, self).get_dict() + result['repo_name'] = result.pop('_repo_name', None) + return result + + +class RepoGroup(Base, BaseModel): + __tablename__ = 'groups' + __table_args__ = ( + UniqueConstraint('group_name', 'group_parent_id'), + base_table_args, + ) + __mapper_args__ = {'order_by': 'group_name'} + + CHOICES_SEPARATOR = '/' # used to generate select2 choices for nested groups + + group_id = Column("group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + group_name = Column("group_name", String(255), nullable=False, unique=True, default=None) + group_parent_id = Column("group_parent_id", Integer(), ForeignKey('groups.group_id'), nullable=True, unique=None, default=None) + group_description = Column("group_description", String(10000), nullable=True, unique=None, default=None) + enable_locking = Column("enable_locking", Boolean(), nullable=False, unique=None, default=False) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=False, default=None) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + updated_on = Column('updated_on', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now) + personal = Column('personal', Boolean(), nullable=True, unique=None, default=None) + + repo_group_to_perm = relationship('UserRepoGroupToPerm', cascade='all', order_by='UserRepoGroupToPerm.group_to_perm_id') + users_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all') + parent_group = relationship('RepoGroup', remote_side=group_id) + user = relationship('User') + integrations = relationship('Integration', + cascade="all, delete, delete-orphan") + + def __init__(self, group_name='', parent_group=None): + self.group_name = group_name + self.parent_group = parent_group + + def __unicode__(self): + return u"<%s('id:%s:%s')>" % ( + self.__class__.__name__, self.group_id, self.group_name) + + @hybrid_property + def description_safe(self): + from rhodecode.lib import helpers as h + return h.escape(self.group_description) + + @classmethod + def _generate_choice(cls, repo_group): + from webhelpers.html import literal as _literal + _name = lambda k: _literal(cls.CHOICES_SEPARATOR.join(k)) + return repo_group.group_id, _name(repo_group.full_path_splitted) + + @classmethod + def groups_choices(cls, groups=None, show_empty_group=True): + if not groups: + groups = cls.query().all() + + repo_groups = [] + if show_empty_group: + repo_groups = [(-1, u'-- %s --' % _('No parent'))] + + repo_groups.extend([cls._generate_choice(x) for x in groups]) + + repo_groups = sorted( + repo_groups, key=lambda t: t[1].split(cls.CHOICES_SEPARATOR)[0]) + return repo_groups + + @classmethod + def url_sep(cls): + return URL_SEP + + @classmethod + def get_by_group_name(cls, group_name, cache=False, case_insensitive=False): + if case_insensitive: + gr = cls.query().filter(func.lower(cls.group_name) + == func.lower(group_name)) + else: + gr = cls.query().filter(cls.group_name == group_name) + if cache: + name_key = _hash_key(group_name) + gr = gr.options( + FromCache("sql_cache_short", "get_group_%s" % name_key)) + return gr.scalar() + + @classmethod + def get_user_personal_repo_group(cls, user_id): + user = User.get(user_id) + if user.username == User.DEFAULT_USER: + return None + + return cls.query()\ + .filter(cls.personal == true()) \ + .filter(cls.user == user) \ + .order_by(cls.group_id.asc()) \ + .first() + + @classmethod + def get_all_repo_groups(cls, user_id=Optional(None), group_id=Optional(None), + case_insensitive=True): + q = RepoGroup.query() + + if not isinstance(user_id, Optional): + q = q.filter(RepoGroup.user_id == user_id) + + if not isinstance(group_id, Optional): + q = q.filter(RepoGroup.group_parent_id == group_id) + + if case_insensitive: + q = q.order_by(func.lower(RepoGroup.group_name)) + else: + q = q.order_by(RepoGroup.group_name) + return q.all() + + @property + def parents(self): + parents_recursion_limit = 10 + groups = [] + if self.parent_group is None: + return groups + cur_gr = self.parent_group + groups.insert(0, cur_gr) + cnt = 0 + while 1: + cnt += 1 + gr = getattr(cur_gr, 'parent_group', None) + cur_gr = cur_gr.parent_group + if gr is None: + break + if cnt == parents_recursion_limit: + # this will prevent accidental infinit loops + log.error('more than %s parents found for group %s, stopping ' + 'recursive parent fetching', parents_recursion_limit, self) + break + + groups.insert(0, gr) + return groups + + @property + def last_db_change(self): + return self.updated_on + + @property + def children(self): + return RepoGroup.query().filter(RepoGroup.parent_group == self) + + @property + def name(self): + return self.group_name.split(RepoGroup.url_sep())[-1] + + @property + def full_path(self): + return self.group_name + + @property + def full_path_splitted(self): + return self.group_name.split(RepoGroup.url_sep()) + + @property + def repositories(self): + return Repository.query()\ + .filter(Repository.group == self)\ + .order_by(Repository.repo_name) + + @property + def repositories_recursive_count(self): + cnt = self.repositories.count() + + def children_count(group): + cnt = 0 + for child in group.children: + cnt += child.repositories.count() + cnt += children_count(child) + return cnt + + return cnt + children_count(self) + + def _recursive_objects(self, include_repos=True): + all_ = [] + + def _get_members(root_gr): + if include_repos: + for r in root_gr.repositories: + all_.append(r) + childs = root_gr.children.all() + if childs: + for gr in childs: + all_.append(gr) + _get_members(gr) + + _get_members(self) + return [self] + all_ + + def recursive_groups_and_repos(self): + """ + Recursive return all groups, with repositories in those groups + """ + return self._recursive_objects() + + def recursive_groups(self): + """ + Returns all children groups for this group including children of children + """ + return self._recursive_objects(include_repos=False) + + def get_new_name(self, group_name): + """ + returns new full group name based on parent and new name + + :param group_name: + """ + path_prefix = (self.parent_group.full_path_splitted if + self.parent_group else []) + return RepoGroup.url_sep().join(path_prefix + [group_name]) + + def permissions(self, with_admins=True, with_owner=True, + expand_from_user_groups=False): + """ + Permissions for repository groups + """ + _admin_perm = 'group.admin' + + owner_row = [] + if with_owner: + usr = AttributeDict(self.user.get_dict()) + usr.owner_row = True + usr.permission = _admin_perm + owner_row.append(usr) + + super_admin_ids = [] + super_admin_rows = [] + if with_admins: + for usr in User.get_all_super_admins(): + super_admin_ids.append(usr.user_id) + # if this admin is also owner, don't double the record + if usr.user_id == owner_row[0].user_id: + owner_row[0].admin_row = True + else: + usr = AttributeDict(usr.get_dict()) + usr.admin_row = True + usr.permission = _admin_perm + super_admin_rows.append(usr) + + q = UserRepoGroupToPerm.query().filter(UserRepoGroupToPerm.group == self) + q = q.options(joinedload(UserRepoGroupToPerm.group), + joinedload(UserRepoGroupToPerm.user), + joinedload(UserRepoGroupToPerm.permission),) + + # get owners and admins and permissions. We do a trick of re-writing + # objects from sqlalchemy to named-tuples due to sqlalchemy session + # has a global reference and changing one object propagates to all + # others. This means if admin is also an owner admin_row that change + # would propagate to both objects + perm_rows = [] + for _usr in q.all(): + usr = AttributeDict(_usr.user.get_dict()) + # if this user is also owner/admin, mark as duplicate record + if usr.user_id == owner_row[0].user_id or usr.user_id in super_admin_ids: + usr.duplicate_perm = True + usr.permission = _usr.permission.permission_name + perm_rows.append(usr) + + # filter the perm rows by 'default' first and then sort them by + # admin,write,read,none permissions sorted again alphabetically in + # each group + perm_rows = sorted(perm_rows, key=display_user_sort) + + user_groups_rows = [] + if expand_from_user_groups: + for ug in self.permission_user_groups(with_members=True): + for user_data in ug.members: + user_groups_rows.append(user_data) + + return super_admin_rows + owner_row + perm_rows + user_groups_rows + + def permission_user_groups(self, with_members=False): + q = UserGroupRepoGroupToPerm.query()\ + .filter(UserGroupRepoGroupToPerm.group == self) + q = q.options(joinedload(UserGroupRepoGroupToPerm.group), + joinedload(UserGroupRepoGroupToPerm.users_group), + joinedload(UserGroupRepoGroupToPerm.permission),) + + perm_rows = [] + for _user_group in q.all(): + entry = AttributeDict(_user_group.users_group.get_dict()) + entry.permission = _user_group.permission.permission_name + if with_members: + entry.members = [x.user.get_dict() + for x in _user_group.users_group.members] + perm_rows.append(entry) + + perm_rows = sorted(perm_rows, key=display_user_group_sort) + return perm_rows + + def get_api_data(self): + """ + Common function for generating api data + + """ + group = self + data = { + 'group_id': group.group_id, + 'group_name': group.group_name, + 'group_description': group.description_safe, + 'parent_group': group.parent_group.group_name if group.parent_group else None, + 'repositories': [x.repo_name for x in group.repositories], + 'owner': group.user.username, + } + return data + + +class Permission(Base, BaseModel): + __tablename__ = 'permissions' + __table_args__ = ( + Index('p_perm_name_idx', 'permission_name'), + base_table_args, + ) + + PERMS = [ + ('hg.admin', _('RhodeCode Super Administrator')), + + ('repository.none', _('Repository no access')), + ('repository.read', _('Repository read access')), + ('repository.write', _('Repository write access')), + ('repository.admin', _('Repository admin access')), + + ('group.none', _('Repository group no access')), + ('group.read', _('Repository group read access')), + ('group.write', _('Repository group write access')), + ('group.admin', _('Repository group admin access')), + + ('usergroup.none', _('User group no access')), + ('usergroup.read', _('User group read access')), + ('usergroup.write', _('User group write access')), + ('usergroup.admin', _('User group admin access')), + + ('branch.none', _('Branch no permissions')), + ('branch.merge', _('Branch access by web merge')), + ('branch.push', _('Branch access by push')), + ('branch.push_force', _('Branch access by push with force')), + + ('hg.repogroup.create.false', _('Repository Group creation disabled')), + ('hg.repogroup.create.true', _('Repository Group creation enabled')), + + ('hg.usergroup.create.false', _('User Group creation disabled')), + ('hg.usergroup.create.true', _('User Group creation enabled')), + + ('hg.create.none', _('Repository creation disabled')), + ('hg.create.repository', _('Repository creation enabled')), + ('hg.create.write_on_repogroup.true', _('Repository creation enabled with write permission to a repository group')), + ('hg.create.write_on_repogroup.false', _('Repository creation disabled with write permission to a repository group')), + + ('hg.fork.none', _('Repository forking disabled')), + ('hg.fork.repository', _('Repository forking enabled')), + + ('hg.register.none', _('Registration disabled')), + ('hg.register.manual_activate', _('User Registration with manual account activation')), + ('hg.register.auto_activate', _('User Registration with automatic account activation')), + + ('hg.password_reset.enabled', _('Password reset enabled')), + ('hg.password_reset.hidden', _('Password reset hidden')), + ('hg.password_reset.disabled', _('Password reset disabled')), + + ('hg.extern_activate.manual', _('Manual activation of external account')), + ('hg.extern_activate.auto', _('Automatic activation of external account')), + + ('hg.inherit_default_perms.false', _('Inherit object permissions from default user disabled')), + ('hg.inherit_default_perms.true', _('Inherit object permissions from default user enabled')), + ] + + # definition of system default permissions for DEFAULT user, created on + # system setup + DEFAULT_USER_PERMISSIONS = [ + # object perms + 'repository.read', + 'group.read', + 'usergroup.read', + # branch, for backward compat we need same value as before so forced pushed + 'branch.push_force', + # global + 'hg.create.repository', + 'hg.repogroup.create.false', + 'hg.usergroup.create.false', + 'hg.create.write_on_repogroup.true', + 'hg.fork.repository', + 'hg.register.manual_activate', + 'hg.password_reset.enabled', + 'hg.extern_activate.auto', + 'hg.inherit_default_perms.true', + ] + + # defines which permissions are more important higher the more important + # Weight defines which permissions are more important. + # The higher number the more important. + PERM_WEIGHTS = { + 'repository.none': 0, + 'repository.read': 1, + 'repository.write': 3, + 'repository.admin': 4, + + 'group.none': 0, + 'group.read': 1, + 'group.write': 3, + 'group.admin': 4, + + 'usergroup.none': 0, + 'usergroup.read': 1, + 'usergroup.write': 3, + 'usergroup.admin': 4, + + 'branch.none': 0, + 'branch.merge': 1, + 'branch.push': 3, + 'branch.push_force': 4, + + 'hg.repogroup.create.false': 0, + 'hg.repogroup.create.true': 1, + + 'hg.usergroup.create.false': 0, + 'hg.usergroup.create.true': 1, + + 'hg.fork.none': 0, + 'hg.fork.repository': 1, + 'hg.create.none': 0, + 'hg.create.repository': 1 + } + + permission_id = Column("permission_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + permission_name = Column("permission_name", String(255), nullable=True, unique=None, default=None) + permission_longname = Column("permission_longname", String(255), nullable=True, unique=None, default=None) + + def __unicode__(self): + return u"<%s('%s:%s')>" % ( + self.__class__.__name__, self.permission_id, self.permission_name + ) + + @classmethod + def get_by_key(cls, key): + return cls.query().filter(cls.permission_name == key).scalar() + + @classmethod + def get_default_repo_perms(cls, user_id, repo_id=None): + q = Session().query(UserRepoToPerm, Repository, Permission)\ + .join((Permission, UserRepoToPerm.permission_id == Permission.permission_id))\ + .join((Repository, UserRepoToPerm.repository_id == Repository.repo_id))\ + .filter(UserRepoToPerm.user_id == user_id) + if repo_id: + q = q.filter(UserRepoToPerm.repository_id == repo_id) + return q.all() + + @classmethod + def get_default_repo_branch_perms(cls, user_id, repo_id=None): + q = Session().query(UserToRepoBranchPermission, UserRepoToPerm, Permission) \ + .join( + Permission, + UserToRepoBranchPermission.permission_id == Permission.permission_id) \ + .join( + UserRepoToPerm, + UserToRepoBranchPermission.rule_to_perm_id == UserRepoToPerm.repo_to_perm_id) \ + .filter(UserRepoToPerm.user_id == user_id) + + if repo_id: + q = q.filter(UserToRepoBranchPermission.repository_id == repo_id) + return q.order_by(UserToRepoBranchPermission.rule_order).all() + + @classmethod + def get_default_repo_perms_from_user_group(cls, user_id, repo_id=None): + q = Session().query(UserGroupRepoToPerm, Repository, Permission)\ + .join( + Permission, + UserGroupRepoToPerm.permission_id == Permission.permission_id)\ + .join( + Repository, + UserGroupRepoToPerm.repository_id == Repository.repo_id)\ + .join( + UserGroup, + UserGroupRepoToPerm.users_group_id == + UserGroup.users_group_id)\ + .join( + UserGroupMember, + UserGroupRepoToPerm.users_group_id == + UserGroupMember.users_group_id)\ + .filter( + UserGroupMember.user_id == user_id, + UserGroup.users_group_active == true()) + if repo_id: + q = q.filter(UserGroupRepoToPerm.repository_id == repo_id) + return q.all() + + @classmethod + def get_default_repo_branch_perms_from_user_group(cls, user_id, repo_id=None): + q = Session().query(UserGroupToRepoBranchPermission, UserGroupRepoToPerm, Permission) \ + .join( + Permission, + UserGroupToRepoBranchPermission.permission_id == Permission.permission_id) \ + .join( + UserGroupRepoToPerm, + UserGroupToRepoBranchPermission.rule_to_perm_id == UserGroupRepoToPerm.users_group_to_perm_id) \ + .join( + UserGroup, + UserGroupRepoToPerm.users_group_id == UserGroup.users_group_id) \ + .join( + UserGroupMember, + UserGroupRepoToPerm.users_group_id == UserGroupMember.users_group_id) \ + .filter( + UserGroupMember.user_id == user_id, + UserGroup.users_group_active == true()) + + if repo_id: + q = q.filter(UserGroupToRepoBranchPermission.repository_id == repo_id) + return q.order_by(UserGroupToRepoBranchPermission.rule_order).all() + + @classmethod + def get_default_group_perms(cls, user_id, repo_group_id=None): + q = Session().query(UserRepoGroupToPerm, RepoGroup, Permission)\ + .join( + Permission, + UserRepoGroupToPerm.permission_id == Permission.permission_id)\ + .join( + RepoGroup, + UserRepoGroupToPerm.group_id == RepoGroup.group_id)\ + .filter(UserRepoGroupToPerm.user_id == user_id) + if repo_group_id: + q = q.filter(UserRepoGroupToPerm.group_id == repo_group_id) + return q.all() + + @classmethod + def get_default_group_perms_from_user_group( + cls, user_id, repo_group_id=None): + q = Session().query(UserGroupRepoGroupToPerm, RepoGroup, Permission)\ + .join( + Permission, + UserGroupRepoGroupToPerm.permission_id == + Permission.permission_id)\ + .join( + RepoGroup, + UserGroupRepoGroupToPerm.group_id == RepoGroup.group_id)\ + .join( + UserGroup, + UserGroupRepoGroupToPerm.users_group_id == + UserGroup.users_group_id)\ + .join( + UserGroupMember, + UserGroupRepoGroupToPerm.users_group_id == + UserGroupMember.users_group_id)\ + .filter( + UserGroupMember.user_id == user_id, + UserGroup.users_group_active == true()) + if repo_group_id: + q = q.filter(UserGroupRepoGroupToPerm.group_id == repo_group_id) + return q.all() + + @classmethod + def get_default_user_group_perms(cls, user_id, user_group_id=None): + q = Session().query(UserUserGroupToPerm, UserGroup, Permission)\ + .join((Permission, UserUserGroupToPerm.permission_id == Permission.permission_id))\ + .join((UserGroup, UserUserGroupToPerm.user_group_id == UserGroup.users_group_id))\ + .filter(UserUserGroupToPerm.user_id == user_id) + if user_group_id: + q = q.filter(UserUserGroupToPerm.user_group_id == user_group_id) + return q.all() + + @classmethod + def get_default_user_group_perms_from_user_group( + cls, user_id, user_group_id=None): + TargetUserGroup = aliased(UserGroup, name='target_user_group') + q = Session().query(UserGroupUserGroupToPerm, UserGroup, Permission)\ + .join( + Permission, + UserGroupUserGroupToPerm.permission_id == + Permission.permission_id)\ + .join( + TargetUserGroup, + UserGroupUserGroupToPerm.target_user_group_id == + TargetUserGroup.users_group_id)\ + .join( + UserGroup, + UserGroupUserGroupToPerm.user_group_id == + UserGroup.users_group_id)\ + .join( + UserGroupMember, + UserGroupUserGroupToPerm.user_group_id == + UserGroupMember.users_group_id)\ + .filter( + UserGroupMember.user_id == user_id, + UserGroup.users_group_active == true()) + if user_group_id: + q = q.filter( + UserGroupUserGroupToPerm.user_group_id == user_group_id) + + return q.all() + + +class UserRepoToPerm(Base, BaseModel): + __tablename__ = 'repo_to_perm' + __table_args__ = ( + UniqueConstraint('user_id', 'repository_id', 'permission_id'), + base_table_args + ) + + repo_to_perm_id = Column("repo_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None) + + user = relationship('User') + repository = relationship('Repository') + permission = relationship('Permission') + + branch_perm_entry = relationship('UserToRepoBranchPermission', cascade="all, delete, delete-orphan", lazy='joined') + + @classmethod + def create(cls, user, repository, permission): + n = cls() + n.user = user + n.repository = repository + n.permission = permission + Session().add(n) + return n + + def __unicode__(self): + return u'<%s => %s >' % (self.user, self.repository) + + +class UserUserGroupToPerm(Base, BaseModel): + __tablename__ = 'user_user_group_to_perm' + __table_args__ = ( + UniqueConstraint('user_id', 'user_group_id', 'permission_id'), + base_table_args + ) + + user_user_group_to_perm_id = Column("user_user_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + user_group_id = Column("user_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) + + user = relationship('User') + user_group = relationship('UserGroup') + permission = relationship('Permission') + + @classmethod + def create(cls, user, user_group, permission): + n = cls() + n.user = user + n.user_group = user_group + n.permission = permission + Session().add(n) + return n + + def __unicode__(self): + return u'<%s => %s >' % (self.user, self.user_group) + + +class UserToPerm(Base, BaseModel): + __tablename__ = 'user_to_perm' + __table_args__ = ( + UniqueConstraint('user_id', 'permission_id'), + base_table_args + ) + + user_to_perm_id = Column("user_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + + user = relationship('User') + permission = relationship('Permission', lazy='joined') + + def __unicode__(self): + return u'<%s => %s >' % (self.user, self.permission) + + +class UserGroupRepoToPerm(Base, BaseModel): + __tablename__ = 'users_group_repo_to_perm' + __table_args__ = ( + UniqueConstraint('repository_id', 'users_group_id', 'permission_id'), + base_table_args + ) + + users_group_to_perm_id = Column("users_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None) + + users_group = relationship('UserGroup') + permission = relationship('Permission') + repository = relationship('Repository') + user_group_branch_perms = relationship('UserGroupToRepoBranchPermission', cascade='all') + + @classmethod + def create(cls, users_group, repository, permission): + n = cls() + n.users_group = users_group + n.repository = repository + n.permission = permission + Session().add(n) + return n + + def __unicode__(self): + return u' %s >' % (self.users_group, self.repository) + + +class UserGroupUserGroupToPerm(Base, BaseModel): + __tablename__ = 'user_group_user_group_to_perm' + __table_args__ = ( + UniqueConstraint('target_user_group_id', 'user_group_id', 'permission_id'), + CheckConstraint('target_user_group_id != user_group_id'), + base_table_args + ) + + user_group_user_group_to_perm_id = Column("user_group_user_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + target_user_group_id = Column("target_user_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + user_group_id = Column("user_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) + + target_user_group = relationship('UserGroup', primaryjoin='UserGroupUserGroupToPerm.target_user_group_id==UserGroup.users_group_id') + user_group = relationship('UserGroup', primaryjoin='UserGroupUserGroupToPerm.user_group_id==UserGroup.users_group_id') + permission = relationship('Permission') + + @classmethod + def create(cls, target_user_group, user_group, permission): + n = cls() + n.target_user_group = target_user_group + n.user_group = user_group + n.permission = permission + Session().add(n) + return n + + def __unicode__(self): + return u' %s >' % (self.target_user_group, self.user_group) + + +class UserGroupToPerm(Base, BaseModel): + __tablename__ = 'users_group_to_perm' + __table_args__ = ( + UniqueConstraint('users_group_id', 'permission_id',), + base_table_args + ) + + users_group_to_perm_id = Column("users_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + + users_group = relationship('UserGroup') + permission = relationship('Permission') + + +class UserRepoGroupToPerm(Base, BaseModel): + __tablename__ = 'user_repo_group_to_perm' + __table_args__ = ( + UniqueConstraint('user_id', 'group_id', 'permission_id'), + base_table_args + ) + + group_to_perm_id = Column("group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) + group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + + user = relationship('User') + group = relationship('RepoGroup') + permission = relationship('Permission') + + @classmethod + def create(cls, user, repository_group, permission): + n = cls() + n.user = user + n.group = repository_group + n.permission = permission + Session().add(n) + return n + + +class UserGroupRepoGroupToPerm(Base, BaseModel): + __tablename__ = 'users_group_repo_group_to_perm' + __table_args__ = ( + UniqueConstraint('users_group_id', 'group_id'), + base_table_args + ) + + users_group_repo_group_to_perm_id = Column("users_group_repo_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) + group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + + users_group = relationship('UserGroup') + permission = relationship('Permission') + group = relationship('RepoGroup') + + @classmethod + def create(cls, user_group, repository_group, permission): + n = cls() + n.users_group = user_group + n.group = repository_group + n.permission = permission + Session().add(n) + return n + + def __unicode__(self): + return u' %s >' % (self.users_group, self.group) + + +class Statistics(Base, BaseModel): + __tablename__ = 'statistics' + __table_args__ = ( + base_table_args + ) + + stat_id = Column("stat_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=True, default=None) + stat_on_revision = Column("stat_on_revision", Integer(), nullable=False) + commit_activity = Column("commit_activity", LargeBinary(1000000), nullable=False)#JSON data + commit_activity_combined = Column("commit_activity_combined", LargeBinary(), nullable=False)#JSON data + languages = Column("languages", LargeBinary(1000000), nullable=False)#JSON data + + repository = relationship('Repository', single_parent=True) + + +class UserFollowing(Base, BaseModel): + __tablename__ = 'user_followings' + __table_args__ = ( + UniqueConstraint('user_id', 'follows_repository_id'), + UniqueConstraint('user_id', 'follows_user_id'), + base_table_args + ) + + user_following_id = Column("user_following_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) + follows_repo_id = Column("follows_repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=True, unique=None, default=None) + follows_user_id = Column("follows_user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None) + follows_from = Column('follows_from', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now) + + user = relationship('User', primaryjoin='User.user_id==UserFollowing.user_id') + + follows_user = relationship('User', primaryjoin='User.user_id==UserFollowing.follows_user_id') + follows_repository = relationship('Repository', order_by='Repository.repo_name') + + @classmethod + def get_repo_followers(cls, repo_id): + return cls.query().filter(cls.follows_repo_id == repo_id) + + +class CacheKey(Base, BaseModel): + __tablename__ = 'cache_invalidation' + __table_args__ = ( + UniqueConstraint('cache_key'), + Index('key_idx', 'cache_key'), + base_table_args, + ) + + CACHE_TYPE_FEED = 'FEED' + CACHE_TYPE_README = 'README' + # namespaces used to register process/thread aware caches + REPO_INVALIDATION_NAMESPACE = 'repo_cache:{repo_id}' + SETTINGS_INVALIDATION_NAMESPACE = 'system_settings' + + cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + cache_key = Column("cache_key", String(255), nullable=True, unique=None, default=None) + cache_args = Column("cache_args", String(255), nullable=True, unique=None, default=None) + cache_active = Column("cache_active", Boolean(), nullable=True, unique=None, default=False) + + def __init__(self, cache_key, cache_args=''): + self.cache_key = cache_key + self.cache_args = cache_args + self.cache_active = False + + def __unicode__(self): + return u"<%s('%s:%s[%s]')>" % ( + self.__class__.__name__, + self.cache_id, self.cache_key, self.cache_active) + + def _cache_key_partition(self): + prefix, repo_name, suffix = self.cache_key.partition(self.cache_args) + return prefix, repo_name, suffix + + def get_prefix(self): + """ + Try to extract prefix from existing cache key. The key could consist + of prefix, repo_name, suffix + """ + # this returns prefix, repo_name, suffix + return self._cache_key_partition()[0] + + def get_suffix(self): + """ + get suffix that might have been used in _get_cache_key to + generate self.cache_key. Only used for informational purposes + in repo_edit.mako. + """ + # prefix, repo_name, suffix + return self._cache_key_partition()[2] + + @classmethod + def delete_all_cache(cls): + """ + Delete all cache keys from database. + Should only be run when all instances are down and all entries + thus stale. + """ + cls.query().delete() + Session().commit() + + @classmethod + def set_invalidate(cls, cache_uid, delete=False): + """ + Mark all caches of a repo as invalid in the database. + """ + + try: + qry = Session().query(cls).filter(cls.cache_args == cache_uid) + if delete: + qry.delete() + log.debug('cache objects deleted for cache args %s', + safe_str(cache_uid)) + else: + qry.update({"cache_active": False}) + log.debug('cache objects marked as invalid for cache args %s', + safe_str(cache_uid)) + + Session().commit() + except Exception: + log.exception( + 'Cache key invalidation failed for cache args %s', + safe_str(cache_uid)) + Session().rollback() + + @classmethod + def get_active_cache(cls, cache_key): + inv_obj = cls.query().filter(cls.cache_key == cache_key).scalar() + if inv_obj: + return inv_obj + return None + + +class ChangesetComment(Base, BaseModel): + __tablename__ = 'changeset_comments' + __table_args__ = ( + Index('cc_revision_idx', 'revision'), + base_table_args, + ) + + COMMENT_OUTDATED = u'comment_outdated' + COMMENT_TYPE_NOTE = u'note' + COMMENT_TYPE_TODO = u'todo' + COMMENT_TYPES = [COMMENT_TYPE_NOTE, COMMENT_TYPE_TODO] + + comment_id = Column('comment_id', Integer(), nullable=False, primary_key=True) + repo_id = Column('repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False) + revision = Column('revision', String(40), nullable=True) + pull_request_id = Column("pull_request_id", Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=True) + pull_request_version_id = Column("pull_request_version_id", Integer(), ForeignKey('pull_request_versions.pull_request_version_id'), nullable=True) + line_no = Column('line_no', Unicode(10), nullable=True) + hl_lines = Column('hl_lines', Unicode(512), nullable=True) + f_path = Column('f_path', Unicode(1000), nullable=True) + user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False) + text = Column('text', UnicodeText().with_variant(UnicodeText(25000), 'mysql'), nullable=False) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + modified_at = Column('modified_at', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + renderer = Column('renderer', Unicode(64), nullable=True) + display_state = Column('display_state', Unicode(128), nullable=True) + + comment_type = Column('comment_type', Unicode(128), nullable=True, default=COMMENT_TYPE_NOTE) + resolved_comment_id = Column('resolved_comment_id', Integer(), ForeignKey('changeset_comments.comment_id'), nullable=True) + + resolved_comment = relationship('ChangesetComment', remote_side=comment_id, back_populates='resolved_by') + resolved_by = relationship('ChangesetComment', back_populates='resolved_comment') + + author = relationship('User', lazy='joined') + repo = relationship('Repository') + status_change = relationship('ChangesetStatus', cascade="all, delete, delete-orphan", lazy='joined') + pull_request = relationship('PullRequest', lazy='joined') + pull_request_version = relationship('PullRequestVersion') + + @classmethod + def get_users(cls, revision=None, pull_request_id=None): + """ + Returns user associated with this ChangesetComment. ie those + who actually commented + + :param cls: + :param revision: + """ + q = Session().query(User)\ + .join(ChangesetComment.author) + if revision: + q = q.filter(cls.revision == revision) + elif pull_request_id: + q = q.filter(cls.pull_request_id == pull_request_id) + return q.all() + + @classmethod + def get_index_from_version(cls, pr_version, versions): + num_versions = [x.pull_request_version_id for x in versions] + try: + return num_versions.index(pr_version) +1 + except (IndexError, ValueError): + return + + @property + def outdated(self): + return self.display_state == self.COMMENT_OUTDATED + + def outdated_at_version(self, version): + """ + Checks if comment is outdated for given pull request version + """ + return self.outdated and self.pull_request_version_id != version + + def older_than_version(self, version): + """ + Checks if comment is made from previous version than given + """ + if version is None: + return self.pull_request_version_id is not None + + return self.pull_request_version_id < version + + @property + def resolved(self): + return self.resolved_by[0] if self.resolved_by else None + + @property + def is_todo(self): + return self.comment_type == self.COMMENT_TYPE_TODO + + @property + def is_inline(self): + return self.line_no and self.f_path + + def get_index_version(self, versions): + return self.get_index_from_version( + self.pull_request_version_id, versions) + + def __repr__(self): + if self.comment_id: + return '' % self.comment_id + else: + return '' % id(self) + + def get_api_data(self): + comment = self + data = { + 'comment_id': comment.comment_id, + 'comment_type': comment.comment_type, + 'comment_text': comment.text, + 'comment_status': comment.status_change, + 'comment_f_path': comment.f_path, + 'comment_lineno': comment.line_no, + 'comment_author': comment.author, + 'comment_created_on': comment.created_on + } + return data + + def __json__(self): + data = dict() + data.update(self.get_api_data()) + return data + + +class ChangesetStatus(Base, BaseModel): + __tablename__ = 'changeset_statuses' + __table_args__ = ( + Index('cs_revision_idx', 'revision'), + Index('cs_version_idx', 'version'), + UniqueConstraint('repo_id', 'revision', 'version'), + base_table_args + ) + + STATUS_NOT_REVIEWED = DEFAULT = 'not_reviewed' + STATUS_APPROVED = 'approved' + STATUS_REJECTED = 'rejected' + STATUS_UNDER_REVIEW = 'under_review' + + STATUSES = [ + (STATUS_NOT_REVIEWED, _("Not Reviewed")), # (no icon) and default + (STATUS_APPROVED, _("Approved")), + (STATUS_REJECTED, _("Rejected")), + (STATUS_UNDER_REVIEW, _("Under Review")), + ] + + changeset_status_id = Column('changeset_status_id', Integer(), nullable=False, primary_key=True) + repo_id = Column('repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None) + revision = Column('revision', String(40), nullable=False) + status = Column('status', String(128), nullable=False, default=DEFAULT) + changeset_comment_id = Column('changeset_comment_id', Integer(), ForeignKey('changeset_comments.comment_id')) + modified_at = Column('modified_at', DateTime(), nullable=False, default=datetime.datetime.now) + version = Column('version', Integer(), nullable=False, default=0) + pull_request_id = Column("pull_request_id", Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=True) + + author = relationship('User', lazy='joined') + repo = relationship('Repository') + comment = relationship('ChangesetComment', lazy='joined') + pull_request = relationship('PullRequest', lazy='joined') + + def __unicode__(self): + return u"<%s('%s[v%s]:%s')>" % ( + self.__class__.__name__, + self.status, self.version, self.author + ) + + @classmethod + def get_status_lbl(cls, value): + return dict(cls.STATUSES).get(value) + + @property + def status_lbl(self): + return ChangesetStatus.get_status_lbl(self.status) + + def get_api_data(self): + status = self + data = { + 'status_id': status.changeset_status_id, + 'status': status.status, + } + return data + + def __json__(self): + data = dict() + data.update(self.get_api_data()) + return data + + +class _SetState(object): + """ + Context processor allowing changing state for sensitive operation such as + pull request update or merge + """ + + def __init__(self, pull_request, pr_state, back_state=None): + self._pr = pull_request + self._org_state = back_state or pull_request.pull_request_state + self._pr_state = pr_state + + def __enter__(self): + log.debug('StateLock: entering set state context, setting state to: `%s`', + self._pr_state) + self._pr.pull_request_state = self._pr_state + Session().add(self._pr) + Session().commit() + + def __exit__(self, exc_type, exc_val, exc_tb): + log.debug('StateLock: exiting set state context, setting state to: `%s`', + self._org_state) + self._pr.pull_request_state = self._org_state + Session().add(self._pr) + Session().commit() + + +class _PullRequestBase(BaseModel): + """ + Common attributes of pull request and version entries. + """ + + # .status values + STATUS_NEW = u'new' + STATUS_OPEN = u'open' + STATUS_CLOSED = u'closed' + + # available states + STATE_CREATING = u'creating' + STATE_UPDATING = u'updating' + STATE_MERGING = u'merging' + STATE_CREATED = u'created' + + title = Column('title', Unicode(255), nullable=True) + description = Column( + 'description', UnicodeText().with_variant(UnicodeText(10240), 'mysql'), + nullable=True) + description_renderer = Column('description_renderer', Unicode(64), nullable=True) + + # new/open/closed status of pull request (not approve/reject/etc) + status = Column('status', Unicode(255), nullable=False, default=STATUS_NEW) + created_on = Column( + 'created_on', DateTime(timezone=False), nullable=False, + default=datetime.datetime.now) + updated_on = Column( + 'updated_on', DateTime(timezone=False), nullable=False, + default=datetime.datetime.now) + + pull_request_state = Column("pull_request_state", String(255), nullable=True) + + @declared_attr + def user_id(cls): + return Column( + "user_id", Integer(), ForeignKey('users.user_id'), nullable=False, + unique=None) + + # 500 revisions max + _revisions = Column( + 'revisions', UnicodeText().with_variant(UnicodeText(20500), 'mysql')) + + @declared_attr + def source_repo_id(cls): + # TODO: dan: rename column to source_repo_id + return Column( + 'org_repo_id', Integer(), ForeignKey('repositories.repo_id'), + nullable=False) + + _source_ref = Column('org_ref', Unicode(255), nullable=False) + + @hybrid_property + def source_ref(self): + return self._source_ref + + @source_ref.setter + def source_ref(self, val): + parts = (val or '').split(':') + if len(parts) != 3: + raise ValueError( + 'Invalid reference format given: {}, expected X:Y:Z'.format(val)) + self._source_ref = safe_unicode(val) + + _target_ref = Column('other_ref', Unicode(255), nullable=False) + + @hybrid_property + def target_ref(self): + return self._target_ref + + @target_ref.setter + def target_ref(self, val): + parts = (val or '').split(':') + if len(parts) != 3: + raise ValueError( + 'Invalid reference format given: {}, expected X:Y:Z'.format(val)) + self._target_ref = safe_unicode(val) + + @declared_attr + def target_repo_id(cls): + # TODO: dan: rename column to target_repo_id + return Column( + 'other_repo_id', Integer(), ForeignKey('repositories.repo_id'), + nullable=False) + + _shadow_merge_ref = Column('shadow_merge_ref', Unicode(255), nullable=True) + + # TODO: dan: rename column to last_merge_source_rev + _last_merge_source_rev = Column( + 'last_merge_org_rev', String(40), nullable=True) + # TODO: dan: rename column to last_merge_target_rev + _last_merge_target_rev = Column( + 'last_merge_other_rev', String(40), nullable=True) + _last_merge_status = Column('merge_status', Integer(), nullable=True) + merge_rev = Column('merge_rev', String(40), nullable=True) + + reviewer_data = Column( + 'reviewer_data_json', MutationObj.as_mutable( + JsonType(dialect_map=dict(mysql=UnicodeText(16384))))) + + @property + def reviewer_data_json(self): + return json.dumps(self.reviewer_data) + + @hybrid_property + def description_safe(self): + from rhodecode.lib import helpers as h + return h.escape(self.description) + + @hybrid_property + def revisions(self): + return self._revisions.split(':') if self._revisions else [] + + @revisions.setter + def revisions(self, val): + self._revisions = ':'.join(val) + + @hybrid_property + def last_merge_status(self): + return safe_int(self._last_merge_status) + + @last_merge_status.setter + def last_merge_status(self, val): + self._last_merge_status = val + + @declared_attr + def author(cls): + return relationship('User', lazy='joined') + + @declared_attr + def source_repo(cls): + return relationship( + 'Repository', + primaryjoin='%s.source_repo_id==Repository.repo_id' % cls.__name__) + + @property + def source_ref_parts(self): + return self.unicode_to_reference(self.source_ref) + + @declared_attr + def target_repo(cls): + return relationship( + 'Repository', + primaryjoin='%s.target_repo_id==Repository.repo_id' % cls.__name__) + + @property + def target_ref_parts(self): + return self.unicode_to_reference(self.target_ref) + + @property + def shadow_merge_ref(self): + return self.unicode_to_reference(self._shadow_merge_ref) + + @shadow_merge_ref.setter + def shadow_merge_ref(self, ref): + self._shadow_merge_ref = self.reference_to_unicode(ref) + + @staticmethod + def unicode_to_reference(raw): + """ + Convert a unicode (or string) to a reference object. + If unicode evaluates to False it returns None. + """ + if raw: + refs = raw.split(':') + return Reference(*refs) + else: + return None + + @staticmethod + def reference_to_unicode(ref): + """ + Convert a reference object to unicode. + If reference is None it returns None. + """ + if ref: + return u':'.join(ref) + else: + return None + + def get_api_data(self, with_merge_state=True): + from rhodecode.model.pull_request import PullRequestModel + + pull_request = self + if with_merge_state: + merge_status = PullRequestModel().merge_status(pull_request) + merge_state = { + 'status': merge_status[0], + 'message': safe_unicode(merge_status[1]), + } + else: + merge_state = {'status': 'not_available', + 'message': 'not_available'} + + merge_data = { + 'clone_url': PullRequestModel().get_shadow_clone_url(pull_request), + 'reference': ( + pull_request.shadow_merge_ref._asdict() + if pull_request.shadow_merge_ref else None), + } + + data = { + 'pull_request_id': pull_request.pull_request_id, + 'url': PullRequestModel().get_url(pull_request), + 'title': pull_request.title, + 'description': pull_request.description, + 'status': pull_request.status, + 'state': pull_request.pull_request_state, + 'created_on': pull_request.created_on, + 'updated_on': pull_request.updated_on, + 'commit_ids': pull_request.revisions, + 'review_status': pull_request.calculated_review_status(), + 'mergeable': merge_state, + 'source': { + 'clone_url': pull_request.source_repo.clone_url(), + 'repository': pull_request.source_repo.repo_name, + 'reference': { + 'name': pull_request.source_ref_parts.name, + 'type': pull_request.source_ref_parts.type, + 'commit_id': pull_request.source_ref_parts.commit_id, + }, + }, + 'target': { + 'clone_url': pull_request.target_repo.clone_url(), + 'repository': pull_request.target_repo.repo_name, + 'reference': { + 'name': pull_request.target_ref_parts.name, + 'type': pull_request.target_ref_parts.type, + 'commit_id': pull_request.target_ref_parts.commit_id, + }, + }, + 'merge': merge_data, + 'author': pull_request.author.get_api_data(include_secrets=False, + details='basic'), + 'reviewers': [ + { + 'user': reviewer.get_api_data(include_secrets=False, + details='basic'), + 'reasons': reasons, + 'review_status': st[0][1].status if st else 'not_reviewed', + } + for obj, reviewer, reasons, mandatory, st in + pull_request.reviewers_statuses() + ] + } + + return data + + def set_state(self, pull_request_state, final_state=None): + """ + # goes from initial state to updating to initial state. + # initial state can be changed by specifying back_state= + with pull_request_obj.set_state(PullRequest.STATE_UPDATING): + pull_request.merge() + + :param pull_request_state: + :param final_state: + + """ + + return _SetState(self, pull_request_state, back_state=final_state) + + +class PullRequest(Base, _PullRequestBase): + __tablename__ = 'pull_requests' + __table_args__ = ( + base_table_args, + ) + + pull_request_id = Column( + 'pull_request_id', Integer(), nullable=False, primary_key=True) + + def __repr__(self): + if self.pull_request_id: + return '' % self.pull_request_id + else: + return '' % id(self) + + reviewers = relationship('PullRequestReviewers', + cascade="all, delete, delete-orphan") + statuses = relationship('ChangesetStatus', + cascade="all, delete, delete-orphan") + comments = relationship('ChangesetComment', + cascade="all, delete, delete-orphan") + versions = relationship('PullRequestVersion', + cascade="all, delete, delete-orphan", + lazy='dynamic') + + @classmethod + def get_pr_display_object(cls, pull_request_obj, org_pull_request_obj, + internal_methods=None): + + class PullRequestDisplay(object): + """ + Special object wrapper for showing PullRequest data via Versions + It mimics PR object as close as possible. This is read only object + just for display + """ + + def __init__(self, attrs, internal=None): + self.attrs = attrs + # internal have priority over the given ones via attrs + self.internal = internal or ['versions'] + + def __getattr__(self, item): + if item in self.internal: + return getattr(self, item) + try: + return self.attrs[item] + except KeyError: + raise AttributeError( + '%s object has no attribute %s' % (self, item)) + + def __repr__(self): + return '' % self.attrs.get('pull_request_id') + + def versions(self): + return pull_request_obj.versions.order_by( + PullRequestVersion.pull_request_version_id).all() + + def is_closed(self): + return pull_request_obj.is_closed() + + @property + def pull_request_version_id(self): + return getattr(pull_request_obj, 'pull_request_version_id', None) + + attrs = StrictAttributeDict(pull_request_obj.get_api_data()) + + attrs.author = StrictAttributeDict( + pull_request_obj.author.get_api_data()) + if pull_request_obj.target_repo: + attrs.target_repo = StrictAttributeDict( + pull_request_obj.target_repo.get_api_data()) + attrs.target_repo.clone_url = pull_request_obj.target_repo.clone_url + + if pull_request_obj.source_repo: + attrs.source_repo = StrictAttributeDict( + pull_request_obj.source_repo.get_api_data()) + attrs.source_repo.clone_url = pull_request_obj.source_repo.clone_url + + attrs.source_ref_parts = pull_request_obj.source_ref_parts + attrs.target_ref_parts = pull_request_obj.target_ref_parts + attrs.revisions = pull_request_obj.revisions + + attrs.shadow_merge_ref = org_pull_request_obj.shadow_merge_ref + attrs.reviewer_data = org_pull_request_obj.reviewer_data + attrs.reviewer_data_json = org_pull_request_obj.reviewer_data_json + + return PullRequestDisplay(attrs, internal=internal_methods) + + def is_closed(self): + return self.status == self.STATUS_CLOSED + + def __json__(self): + return { + 'revisions': self.revisions, + } + + def calculated_review_status(self): + from rhodecode.model.changeset_status import ChangesetStatusModel + return ChangesetStatusModel().calculated_review_status(self) + + def reviewers_statuses(self): + from rhodecode.model.changeset_status import ChangesetStatusModel + return ChangesetStatusModel().reviewers_statuses(self) + + @property + def workspace_id(self): + from rhodecode.model.pull_request import PullRequestModel + return PullRequestModel()._workspace_id(self) + + def get_shadow_repo(self): + workspace_id = self.workspace_id + vcs_obj = self.target_repo.scm_instance() + shadow_repository_path = vcs_obj._get_shadow_repository_path( + self.target_repo.repo_id, workspace_id) + if os.path.isdir(shadow_repository_path): + return vcs_obj._get_shadow_instance(shadow_repository_path) + + +class PullRequestVersion(Base, _PullRequestBase): + __tablename__ = 'pull_request_versions' + __table_args__ = ( + base_table_args, + ) + + pull_request_version_id = Column( + 'pull_request_version_id', Integer(), nullable=False, primary_key=True) + pull_request_id = Column( + 'pull_request_id', Integer(), + ForeignKey('pull_requests.pull_request_id'), nullable=False) + pull_request = relationship('PullRequest') + + def __repr__(self): + if self.pull_request_version_id: + return '' % self.pull_request_version_id + else: + return '' % id(self) + + @property + def reviewers(self): + return self.pull_request.reviewers + + @property + def versions(self): + return self.pull_request.versions + + def is_closed(self): + # calculate from original + return self.pull_request.status == self.STATUS_CLOSED + + def calculated_review_status(self): + return self.pull_request.calculated_review_status() + + def reviewers_statuses(self): + return self.pull_request.reviewers_statuses() + + +class PullRequestReviewers(Base, BaseModel): + __tablename__ = 'pull_request_reviewers' + __table_args__ = ( + base_table_args, + ) + + @hybrid_property + def reasons(self): + if not self._reasons: + return [] + return self._reasons + + @reasons.setter + def reasons(self, val): + val = val or [] + if any(not isinstance(x, compat.string_types) for x in val): + raise Exception('invalid reasons type, must be list of strings') + self._reasons = val + + pull_requests_reviewers_id = Column( + 'pull_requests_reviewers_id', Integer(), nullable=False, + primary_key=True) + pull_request_id = Column( + "pull_request_id", Integer(), + ForeignKey('pull_requests.pull_request_id'), nullable=False) + user_id = Column( + "user_id", Integer(), ForeignKey('users.user_id'), nullable=True) + _reasons = Column( + 'reason', MutationList.as_mutable( + JsonType('list', dialect_map=dict(mysql=UnicodeText(16384))))) + + mandatory = Column("mandatory", Boolean(), nullable=False, default=False) + user = relationship('User') + pull_request = relationship('PullRequest') + + rule_data = Column( + 'rule_data_json', + JsonType(dialect_map=dict(mysql=UnicodeText(16384)))) + + def rule_user_group_data(self): + """ + Returns the voting user group rule data for this reviewer + """ + + if self.rule_data and 'vote_rule' in self.rule_data: + user_group_data = {} + if 'rule_user_group_entry_id' in self.rule_data: + # means a group with voting rules ! + user_group_data['id'] = self.rule_data['rule_user_group_entry_id'] + user_group_data['name'] = self.rule_data['rule_name'] + user_group_data['vote_rule'] = self.rule_data['vote_rule'] + + return user_group_data + + def __unicode__(self): + return u"<%s('id:%s')>" % (self.__class__.__name__, + self.pull_requests_reviewers_id) + + +class Notification(Base, BaseModel): + __tablename__ = 'notifications' + __table_args__ = ( + Index('notification_type_idx', 'type'), + base_table_args, + ) + + TYPE_CHANGESET_COMMENT = u'cs_comment' + TYPE_MESSAGE = u'message' + TYPE_MENTION = u'mention' + TYPE_REGISTRATION = u'registration' + TYPE_PULL_REQUEST = u'pull_request' + TYPE_PULL_REQUEST_COMMENT = u'pull_request_comment' + + notification_id = Column('notification_id', Integer(), nullable=False, primary_key=True) + subject = Column('subject', Unicode(512), nullable=True) + body = Column('body', UnicodeText().with_variant(UnicodeText(50000), 'mysql'), nullable=True) + created_by = Column("created_by", Integer(), ForeignKey('users.user_id'), nullable=True) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + type_ = Column('type', Unicode(255)) + + created_by_user = relationship('User') + notifications_to_users = relationship('UserNotification', lazy='joined', + cascade="all, delete, delete-orphan") + + @property + def recipients(self): + return [x.user for x in UserNotification.query()\ + .filter(UserNotification.notification == self)\ + .order_by(UserNotification.user_id.asc()).all()] + + @classmethod + def create(cls, created_by, subject, body, recipients, type_=None): + if type_ is None: + type_ = Notification.TYPE_MESSAGE + + notification = cls() + notification.created_by_user = created_by + notification.subject = subject + notification.body = body + notification.type_ = type_ + notification.created_on = datetime.datetime.now() + + # For each recipient link the created notification to his account + for u in recipients: + assoc = UserNotification() + assoc.user_id = u.user_id + assoc.notification = notification + + # if created_by is inside recipients mark his notification + # as read + if u.user_id == created_by.user_id: + assoc.read = True + Session().add(assoc) + + Session().add(notification) + + return notification + + +class UserNotification(Base, BaseModel): + __tablename__ = 'user_to_notification' + __table_args__ = ( + UniqueConstraint('user_id', 'notification_id'), + base_table_args + ) + + user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), primary_key=True) + notification_id = Column("notification_id", Integer(), ForeignKey('notifications.notification_id'), primary_key=True) + read = Column('read', Boolean, default=False) + sent_on = Column('sent_on', DateTime(timezone=False), nullable=True, unique=None) + + user = relationship('User', lazy="joined") + notification = relationship('Notification', lazy="joined", + order_by=lambda: Notification.created_on.desc(),) + + def mark_as_read(self): + self.read = True + Session().add(self) + + +class Gist(Base, BaseModel): + __tablename__ = 'gists' + __table_args__ = ( + Index('g_gist_access_id_idx', 'gist_access_id'), + Index('g_created_on_idx', 'created_on'), + base_table_args + ) + + GIST_PUBLIC = u'public' + GIST_PRIVATE = u'private' + DEFAULT_FILENAME = u'gistfile1.txt' + + ACL_LEVEL_PUBLIC = u'acl_public' + ACL_LEVEL_PRIVATE = u'acl_private' + + gist_id = Column('gist_id', Integer(), primary_key=True) + gist_access_id = Column('gist_access_id', Unicode(250)) + gist_description = Column('gist_description', UnicodeText().with_variant(UnicodeText(1024), 'mysql')) + gist_owner = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=True) + gist_expires = Column('gist_expires', Float(53), nullable=False) + gist_type = Column('gist_type', Unicode(128), nullable=False) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + modified_at = Column('modified_at', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + acl_level = Column('acl_level', Unicode(128), nullable=True) + + owner = relationship('User') + + def __repr__(self): + return '' % (self.gist_type, self.gist_access_id) + + @hybrid_property + def description_safe(self): + from rhodecode.lib import helpers as h + return h.escape(self.gist_description) + + @classmethod + def get_or_404(cls, id_): + from pyramid.httpexceptions import HTTPNotFound + + res = cls.query().filter(cls.gist_access_id == id_).scalar() + if not res: + raise HTTPNotFound() + return res + + @classmethod + def get_by_access_id(cls, gist_access_id): + return cls.query().filter(cls.gist_access_id == gist_access_id).scalar() + + def gist_url(self): + from rhodecode.model.gist import GistModel + return GistModel().get_url(self) + + @classmethod + def base_path(cls): + """ + Returns base path when all gists are stored + + :param cls: + """ + from rhodecode.model.gist import GIST_STORE_LOC + q = Session().query(RhodeCodeUi)\ + .filter(RhodeCodeUi.ui_key == URL_SEP) + q = q.options(FromCache("sql_cache_short", "repository_repo_path")) + return os.path.join(q.one().ui_value, GIST_STORE_LOC) + + def get_api_data(self): + """ + Common function for generating gist related data for API + """ + gist = self + data = { + 'gist_id': gist.gist_id, + 'type': gist.gist_type, + 'access_id': gist.gist_access_id, + 'description': gist.gist_description, + 'url': gist.gist_url(), + 'expires': gist.gist_expires, + 'created_on': gist.created_on, + 'modified_at': gist.modified_at, + 'content': None, + 'acl_level': gist.acl_level, + } + return data + + def __json__(self): + data = dict( + ) + data.update(self.get_api_data()) + return data + # SCM functions + + def scm_instance(self, **kwargs): + full_repo_path = os.path.join(self.base_path(), self.gist_access_id) + return get_vcs_instance( + repo_path=safe_str(full_repo_path), create=False) + + +class ExternalIdentity(Base, BaseModel): + __tablename__ = 'external_identities' + __table_args__ = ( + Index('local_user_id_idx', 'local_user_id'), + Index('external_id_idx', 'external_id'), + base_table_args + ) + + external_id = Column('external_id', Unicode(255), default=u'', primary_key=True) + external_username = Column('external_username', Unicode(1024), default=u'') + local_user_id = Column('local_user_id', Integer(), ForeignKey('users.user_id'), primary_key=True) + provider_name = Column('provider_name', Unicode(255), default=u'', primary_key=True) + access_token = Column('access_token', String(1024), default=u'') + alt_token = Column('alt_token', String(1024), default=u'') + token_secret = Column('token_secret', String(1024), default=u'') + + @classmethod + def by_external_id_and_provider(cls, external_id, provider_name, local_user_id=None): + """ + Returns ExternalIdentity instance based on search params + + :param external_id: + :param provider_name: + :return: ExternalIdentity + """ + query = cls.query() + query = query.filter(cls.external_id == external_id) + query = query.filter(cls.provider_name == provider_name) + if local_user_id: + query = query.filter(cls.local_user_id == local_user_id) + return query.first() + + @classmethod + def user_by_external_id_and_provider(cls, external_id, provider_name): + """ + Returns User instance based on search params + + :param external_id: + :param provider_name: + :return: User + """ + query = User.query() + query = query.filter(cls.external_id == external_id) + query = query.filter(cls.provider_name == provider_name) + query = query.filter(User.user_id == cls.local_user_id) + return query.first() + + @classmethod + def by_local_user_id(cls, local_user_id): + """ + Returns all tokens for user + + :param local_user_id: + :return: ExternalIdentity + """ + query = cls.query() + query = query.filter(cls.local_user_id == local_user_id) + return query + + @classmethod + def load_provider_plugin(cls, plugin_id): + from rhodecode.authentication.base import loadplugin + _plugin_id = 'egg:rhodecode-enterprise-ee#{}'.format(plugin_id) + auth_plugin = loadplugin(_plugin_id) + return auth_plugin + + +class Integration(Base, BaseModel): + __tablename__ = 'integrations' + __table_args__ = ( + base_table_args + ) + + integration_id = Column('integration_id', Integer(), primary_key=True) + integration_type = Column('integration_type', String(255)) + enabled = Column('enabled', Boolean(), nullable=False) + name = Column('name', String(255), nullable=False) + child_repos_only = Column('child_repos_only', Boolean(), nullable=False, + default=False) + + settings = Column( + 'settings_json', MutationObj.as_mutable( + JsonType(dialect_map=dict(mysql=UnicodeText(16384))))) + repo_id = Column( + 'repo_id', Integer(), ForeignKey('repositories.repo_id'), + nullable=True, unique=None, default=None) + repo = relationship('Repository', lazy='joined') + + repo_group_id = Column( + 'repo_group_id', Integer(), ForeignKey('groups.group_id'), + nullable=True, unique=None, default=None) + repo_group = relationship('RepoGroup', lazy='joined') + + @property + def scope(self): + if self.repo: + return repr(self.repo) + if self.repo_group: + if self.child_repos_only: + return repr(self.repo_group) + ' (child repos only)' + else: + return repr(self.repo_group) + ' (recursive)' + if self.child_repos_only: + return 'root_repos' + return 'global' + + def __repr__(self): + return '' % (self.integration_type, self.scope) + + +class RepoReviewRuleUser(Base, BaseModel): + __tablename__ = 'repo_review_rules_users' + __table_args__ = ( + base_table_args + ) + + repo_review_rule_user_id = Column('repo_review_rule_user_id', Integer(), primary_key=True) + repo_review_rule_id = Column("repo_review_rule_id", Integer(), ForeignKey('repo_review_rules.repo_review_rule_id')) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False) + mandatory = Column("mandatory", Boolean(), nullable=False, default=False) + user = relationship('User') + + def rule_data(self): + return { + 'mandatory': self.mandatory + } + + +class RepoReviewRuleUserGroup(Base, BaseModel): + __tablename__ = 'repo_review_rules_users_groups' + __table_args__ = ( + base_table_args + ) + + VOTE_RULE_ALL = -1 + + repo_review_rule_users_group_id = Column('repo_review_rule_users_group_id', Integer(), primary_key=True) + repo_review_rule_id = Column("repo_review_rule_id", Integer(), ForeignKey('repo_review_rules.repo_review_rule_id')) + users_group_id = Column("users_group_id", Integer(),ForeignKey('users_groups.users_group_id'), nullable=False) + mandatory = Column("mandatory", Boolean(), nullable=False, default=False) + vote_rule = Column("vote_rule", Integer(), nullable=True, default=VOTE_RULE_ALL) + users_group = relationship('UserGroup') + + def rule_data(self): + return { + 'mandatory': self.mandatory, + 'vote_rule': self.vote_rule + } + + @property + def vote_rule_label(self): + if not self.vote_rule or self.vote_rule == self.VOTE_RULE_ALL: + return 'all must vote' + else: + return 'min. vote {}'.format(self.vote_rule) + + +class RepoReviewRule(Base, BaseModel): + __tablename__ = 'repo_review_rules' + __table_args__ = ( + base_table_args + ) + + repo_review_rule_id = Column( + 'repo_review_rule_id', Integer(), primary_key=True) + repo_id = Column( + "repo_id", Integer(), ForeignKey('repositories.repo_id')) + repo = relationship('Repository', backref='review_rules') + + review_rule_name = Column('review_rule_name', String(255)) + _branch_pattern = Column("branch_pattern", UnicodeText().with_variant(UnicodeText(255), 'mysql'), default=u'*') # glob + _target_branch_pattern = Column("target_branch_pattern", UnicodeText().with_variant(UnicodeText(255), 'mysql'), default=u'*') # glob + _file_pattern = Column("file_pattern", UnicodeText().with_variant(UnicodeText(255), 'mysql'), default=u'*') # glob + + use_authors_for_review = Column("use_authors_for_review", Boolean(), nullable=False, default=False) + forbid_author_to_review = Column("forbid_author_to_review", Boolean(), nullable=False, default=False) + forbid_commit_author_to_review = Column("forbid_commit_author_to_review", Boolean(), nullable=False, default=False) + forbid_adding_reviewers = Column("forbid_adding_reviewers", Boolean(), nullable=False, default=False) + + rule_users = relationship('RepoReviewRuleUser') + rule_user_groups = relationship('RepoReviewRuleUserGroup') + + def _validate_pattern(self, value): + re.compile('^' + glob2re(value) + '$') + + @hybrid_property + def source_branch_pattern(self): + return self._branch_pattern or '*' + + @source_branch_pattern.setter + def source_branch_pattern(self, value): + self._validate_pattern(value) + self._branch_pattern = value or '*' + + @hybrid_property + def target_branch_pattern(self): + return self._target_branch_pattern or '*' + + @target_branch_pattern.setter + def target_branch_pattern(self, value): + self._validate_pattern(value) + self._target_branch_pattern = value or '*' + + @hybrid_property + def file_pattern(self): + return self._file_pattern or '*' + + @file_pattern.setter + def file_pattern(self, value): + self._validate_pattern(value) + self._file_pattern = value or '*' + + def matches(self, source_branch, target_branch, files_changed): + """ + Check if this review rule matches a branch/files in a pull request + + :param source_branch: source branch name for the commit + :param target_branch: target branch name for the commit + :param files_changed: list of file paths changed in the pull request + """ + + source_branch = source_branch or '' + target_branch = target_branch or '' + files_changed = files_changed or [] + + branch_matches = True + if source_branch or target_branch: + if self.source_branch_pattern == '*': + source_branch_match = True + else: + if self.source_branch_pattern.startswith('re:'): + source_pattern = self.source_branch_pattern[3:] + else: + source_pattern = '^' + glob2re(self.source_branch_pattern) + '$' + source_branch_regex = re.compile(source_pattern) + source_branch_match = bool(source_branch_regex.search(source_branch)) + if self.target_branch_pattern == '*': + target_branch_match = True + else: + if self.target_branch_pattern.startswith('re:'): + target_pattern = self.target_branch_pattern[3:] + else: + target_pattern = '^' + glob2re(self.target_branch_pattern) + '$' + target_branch_regex = re.compile(target_pattern) + target_branch_match = bool(target_branch_regex.search(target_branch)) + + branch_matches = source_branch_match and target_branch_match + + files_matches = True + if self.file_pattern != '*': + files_matches = False + if self.file_pattern.startswith('re:'): + file_pattern = self.file_pattern[3:] + else: + file_pattern = glob2re(self.file_pattern) + file_regex = re.compile(file_pattern) + for filename in files_changed: + if file_regex.search(filename): + files_matches = True + break + + return branch_matches and files_matches + + @property + def review_users(self): + """ Returns the users which this rule applies to """ + + users = collections.OrderedDict() + + for rule_user in self.rule_users: + if rule_user.user.active: + if rule_user.user not in users: + users[rule_user.user.username] = { + 'user': rule_user.user, + 'source': 'user', + 'source_data': {}, + 'data': rule_user.rule_data() + } + + for rule_user_group in self.rule_user_groups: + source_data = { + 'user_group_id': rule_user_group.users_group.users_group_id, + 'name': rule_user_group.users_group.users_group_name, + 'members': len(rule_user_group.users_group.members) + } + for member in rule_user_group.users_group.members: + if member.user.active: + key = member.user.username + if key in users: + # skip this member as we have him already + # this prevents from override the "first" matched + # users with duplicates in multiple groups + continue + + users[key] = { + 'user': member.user, + 'source': 'user_group', + 'source_data': source_data, + 'data': rule_user_group.rule_data() + } + + return users + + def user_group_vote_rule(self, user_id): + + rules = [] + if not self.rule_user_groups: + return rules + + for user_group in self.rule_user_groups: + user_group_members = [x.user_id for x in user_group.users_group.members] + if user_id in user_group_members: + rules.append(user_group) + return rules + + def __repr__(self): + return '' % ( + self.repo_review_rule_id, self.repo) + + +class ScheduleEntry(Base, BaseModel): + __tablename__ = 'schedule_entries' + __table_args__ = ( + UniqueConstraint('schedule_name', name='s_schedule_name_idx'), + UniqueConstraint('task_uid', name='s_task_uid_idx'), + base_table_args, + ) + + schedule_types = ['crontab', 'timedelta', 'integer'] + schedule_entry_id = Column('schedule_entry_id', Integer(), primary_key=True) + + schedule_name = Column("schedule_name", String(255), nullable=False, unique=None, default=None) + schedule_description = Column("schedule_description", String(10000), nullable=True, unique=None, default=None) + schedule_enabled = Column("schedule_enabled", Boolean(), nullable=False, unique=None, default=True) + + _schedule_type = Column("schedule_type", String(255), nullable=False, unique=None, default=None) + schedule_definition = Column('schedule_definition_json', MutationObj.as_mutable(JsonType(default=lambda: "", dialect_map=dict(mysql=LONGTEXT())))) + + schedule_last_run = Column('schedule_last_run', DateTime(timezone=False), nullable=True, unique=None, default=None) + schedule_total_run_count = Column('schedule_total_run_count', Integer(), nullable=True, unique=None, default=0) + + # task + task_uid = Column("task_uid", String(255), nullable=False, unique=None, default=None) + task_dot_notation = Column("task_dot_notation", String(4096), nullable=False, unique=None, default=None) + task_args = Column('task_args_json', MutationObj.as_mutable(JsonType(default=list, dialect_map=dict(mysql=LONGTEXT())))) + task_kwargs = Column('task_kwargs_json', MutationObj.as_mutable(JsonType(default=dict, dialect_map=dict(mysql=LONGTEXT())))) + + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + updated_on = Column('updated_on', DateTime(timezone=False), nullable=True, unique=None, default=None) + + @hybrid_property + def schedule_type(self): + return self._schedule_type + + @schedule_type.setter + def schedule_type(self, val): + if val not in self.schedule_types: + raise ValueError('Value must be on of `{}` and got `{}`'.format( + val, self.schedule_type)) + + self._schedule_type = val + + @classmethod + def get_uid(cls, obj): + args = obj.task_args + kwargs = obj.task_kwargs + if isinstance(args, JsonRaw): + try: + args = json.loads(args) + except ValueError: + args = tuple() + + if isinstance(kwargs, JsonRaw): + try: + kwargs = json.loads(kwargs) + except ValueError: + kwargs = dict() + + dot_notation = obj.task_dot_notation + val = '.'.join(map(safe_str, [ + sorted(dot_notation), args, sorted(kwargs.items())])) + return hashlib.sha1(val).hexdigest() + + @classmethod + def get_by_schedule_name(cls, schedule_name): + return cls.query().filter(cls.schedule_name == schedule_name).scalar() + + @classmethod + def get_by_schedule_id(cls, schedule_id): + return cls.query().filter(cls.schedule_entry_id == schedule_id).scalar() + + @property + def task(self): + return self.task_dot_notation + + @property + def schedule(self): + from rhodecode.lib.celerylib.utils import raw_2_schedule + schedule = raw_2_schedule(self.schedule_definition, self.schedule_type) + return schedule + + @property + def args(self): + try: + return list(self.task_args or []) + except ValueError: + return list() + + @property + def kwargs(self): + try: + return dict(self.task_kwargs or {}) + except ValueError: + return dict() + + def _as_raw(self, val): + if hasattr(val, 'de_coerce'): + val = val.de_coerce() + if val: + val = json.dumps(val) + + return val + + @property + def schedule_definition_raw(self): + return self._as_raw(self.schedule_definition) + + @property + def args_raw(self): + return self._as_raw(self.task_args) + + @property + def kwargs_raw(self): + return self._as_raw(self.task_kwargs) + + def __repr__(self): + return ''.format( + self.schedule_entry_id, self.schedule_name) + + +@event.listens_for(ScheduleEntry, 'before_update') +def update_task_uid(mapper, connection, target): + target.task_uid = ScheduleEntry.get_uid(target) + + +@event.listens_for(ScheduleEntry, 'before_insert') +def set_task_uid(mapper, connection, target): + target.task_uid = ScheduleEntry.get_uid(target) + + +class _BaseBranchPerms(BaseModel): + @classmethod + def compute_hash(cls, value): + return sha1_safe(value) + + @hybrid_property + def branch_pattern(self): + return self._branch_pattern or '*' + + @hybrid_property + def branch_hash(self): + return self._branch_hash + + def _validate_glob(self, value): + re.compile('^' + glob2re(value) + '$') + + @branch_pattern.setter + def branch_pattern(self, value): + self._validate_glob(value) + self._branch_pattern = value or '*' + # set the Hash when setting the branch pattern + self._branch_hash = self.compute_hash(self._branch_pattern) + + def matches(self, branch): + """ + Check if this the branch matches entry + + :param branch: branch name for the commit + """ + + branch = branch or '' + + branch_matches = True + if branch: + branch_regex = re.compile('^' + glob2re(self.branch_pattern) + '$') + branch_matches = bool(branch_regex.search(branch)) + + return branch_matches + + +class UserToRepoBranchPermission(Base, _BaseBranchPerms): + __tablename__ = 'user_to_repo_branch_permissions' + __table_args__ = ( + {'extend_existing': True, 'mysql_engine': 'InnoDB', + 'mysql_charset': 'utf8', 'sqlite_autoincrement': True,} + ) + + branch_rule_id = Column('branch_rule_id', Integer(), primary_key=True) + + repository_id = Column('repository_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None) + repo = relationship('Repository', backref='user_branch_perms') + + permission_id = Column('permission_id', Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + permission = relationship('Permission') + + rule_to_perm_id = Column('rule_to_perm_id', Integer(), ForeignKey('repo_to_perm.repo_to_perm_id'), nullable=False, unique=None, default=None) + user_repo_to_perm = relationship('UserRepoToPerm') + + rule_order = Column('rule_order', Integer(), nullable=False) + _branch_pattern = Column('branch_pattern', UnicodeText().with_variant(UnicodeText(2048), 'mysql'), default=u'*') # glob + _branch_hash = Column('branch_hash', UnicodeText().with_variant(UnicodeText(2048), 'mysql')) + + def __unicode__(self): + return u' %r)>' % ( + self.user_repo_to_perm, self.branch_pattern) + + +class UserGroupToRepoBranchPermission(Base, _BaseBranchPerms): + __tablename__ = 'user_group_to_repo_branch_permissions' + __table_args__ = ( + {'extend_existing': True, 'mysql_engine': 'InnoDB', + 'mysql_charset': 'utf8', 'sqlite_autoincrement': True,} + ) + + branch_rule_id = Column('branch_rule_id', Integer(), primary_key=True) + + repository_id = Column('repository_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None) + repo = relationship('Repository', backref='user_group_branch_perms') + + permission_id = Column('permission_id', Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + permission = relationship('Permission') + + rule_to_perm_id = Column('rule_to_perm_id', Integer(), ForeignKey('users_group_repo_to_perm.users_group_to_perm_id'), nullable=False, unique=None, default=None) + user_group_repo_to_perm = relationship('UserGroupRepoToPerm') + + rule_order = Column('rule_order', Integer(), nullable=False) + _branch_pattern = Column('branch_pattern', UnicodeText().with_variant(UnicodeText(2048), 'mysql'), default=u'*') # glob + _branch_hash = Column('branch_hash', UnicodeText().with_variant(UnicodeText(2048), 'mysql')) + + def __unicode__(self): + return u' %r)>' % ( + self.user_group_repo_to_perm, self.branch_pattern) + + +class UserBookmark(Base, BaseModel): + __tablename__ = 'user_bookmarks' + __table_args__ = ( + UniqueConstraint('user_id', 'bookmark_repo_id'), + UniqueConstraint('user_id', 'bookmark_repo_group_id'), + UniqueConstraint('user_id', 'bookmark_position'), + base_table_args + ) + + user_bookmark_id = Column("user_bookmark_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) + position = Column("bookmark_position", Integer(), nullable=False) + title = Column("bookmark_title", String(255), nullable=True, unique=None, default=None) + redirect_url = Column("bookmark_redirect_url", String(10240), nullable=True, unique=None, default=None) + created_on = Column("created_on", DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + + bookmark_repo_id = Column("bookmark_repo_id", Integer(), ForeignKey("repositories.repo_id"), nullable=True, unique=None, default=None) + bookmark_repo_group_id = Column("bookmark_repo_group_id", Integer(), ForeignKey("groups.group_id"), nullable=True, unique=None, default=None) + + user = relationship("User") + + repository = relationship("Repository") + repository_group = relationship("RepoGroup") + + +class DbMigrateVersion(Base, BaseModel): + __tablename__ = 'db_migrate_version' + __table_args__ = ( + base_table_args, + ) + + repository_id = Column('repository_id', String(250), primary_key=True) + repository_path = Column('repository_path', Text) + version = Column('version', Integer) + + @classmethod + def set_version(cls, version): + """ + Helper for forcing a different version, usually for debugging purposes via ishell. + """ + ver = DbMigrateVersion.query().first() + ver.version = version + Session().commit() + + +class DbSession(Base, BaseModel): + __tablename__ = 'db_session' + __table_args__ = ( + base_table_args, + ) + + def __repr__(self): + return ''.format(self.id) + + id = Column('id', Integer()) + namespace = Column('namespace', String(255), primary_key=True) + accessed = Column('accessed', DateTime, nullable=False) + created = Column('created', DateTime, nullable=False) + data = Column('data', PickleType, nullable=False) diff --git a/rhodecode/lib/dbmigrate/schema/db_4_16_0_2.py b/rhodecode/lib/dbmigrate/schema/db_4_16_0_2.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/dbmigrate/schema/db_4_16_0_2.py @@ -0,0 +1,4932 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2019 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +""" +Database Models for RhodeCode Enterprise +""" + +import re +import os +import time +import hashlib +import logging +import datetime +import warnings +import ipaddress +import functools +import traceback +import collections + +from sqlalchemy import ( + or_, and_, not_, func, TypeDecorator, event, + Index, Sequence, UniqueConstraint, ForeignKey, CheckConstraint, Column, + Boolean, String, Unicode, UnicodeText, DateTime, Integer, LargeBinary, + Text, Float, PickleType) +from sqlalchemy.sql.expression import true, false +from sqlalchemy.sql.functions import coalesce, count # pragma: no cover +from sqlalchemy.orm import ( + relationship, joinedload, class_mapper, validates, aliased) +from sqlalchemy.ext.declarative import declared_attr +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.exc import IntegrityError # pragma: no cover +from sqlalchemy.dialects.mysql import LONGTEXT +from zope.cachedescriptors.property import Lazy as LazyProperty +from pyramid import compat +from pyramid.threadlocal import get_current_request + +from rhodecode.translation import _ +from rhodecode.lib.vcs import get_vcs_instance +from rhodecode.lib.vcs.backends.base import EmptyCommit, Reference +from rhodecode.lib.utils2 import ( + str2bool, safe_str, get_commit_safe, safe_unicode, sha1_safe, + time_to_datetime, aslist, Optional, safe_int, get_clone_url, AttributeDict, + glob2re, StrictAttributeDict, cleaned_uri) +from rhodecode.lib.jsonalchemy import MutationObj, MutationList, JsonType, \ + JsonRaw +from rhodecode.lib.ext_json import json +from rhodecode.lib.caching_query import FromCache +from rhodecode.lib.encrypt import AESCipher + +from rhodecode.model.meta import Base, Session + +URL_SEP = '/' +log = logging.getLogger(__name__) + +# ============================================================================= +# BASE CLASSES +# ============================================================================= + +# this is propagated from .ini file rhodecode.encrypted_values.secret or +# beaker.session.secret if first is not set. +# and initialized at environment.py +ENCRYPTION_KEY = None + +# used to sort permissions by types, '#' used here is not allowed to be in +# usernames, and it's very early in sorted string.printable table. +PERMISSION_TYPE_SORT = { + 'admin': '####', + 'write': '###', + 'read': '##', + 'none': '#', +} + + +def display_user_sort(obj): + """ + Sort function used to sort permissions in .permissions() function of + Repository, RepoGroup, UserGroup. Also it put the default user in front + of all other resources + """ + + if obj.username == User.DEFAULT_USER: + return '#####' + prefix = PERMISSION_TYPE_SORT.get(obj.permission.split('.')[-1], '') + return prefix + obj.username + + +def display_user_group_sort(obj): + """ + Sort function used to sort permissions in .permissions() function of + Repository, RepoGroup, UserGroup. Also it put the default user in front + of all other resources + """ + + prefix = PERMISSION_TYPE_SORT.get(obj.permission.split('.')[-1], '') + return prefix + obj.users_group_name + + +def _hash_key(k): + return sha1_safe(k) + + +def in_filter_generator(qry, items, limit=500): + """ + Splits IN() into multiple with OR + e.g.:: + cnt = Repository.query().filter( + or_( + *in_filter_generator(Repository.repo_id, range(100000)) + )).count() + """ + if not items: + # empty list will cause empty query which might cause security issues + # this can lead to hidden unpleasant results + items = [-1] + + parts = [] + for chunk in xrange(0, len(items), limit): + parts.append( + qry.in_(items[chunk: chunk + limit]) + ) + + return parts + + +base_table_args = { + 'extend_existing': True, + 'mysql_engine': 'InnoDB', + 'mysql_charset': 'utf8', + 'sqlite_autoincrement': True +} + + +class EncryptedTextValue(TypeDecorator): + """ + Special column for encrypted long text data, use like:: + + value = Column("encrypted_value", EncryptedValue(), nullable=False) + + This column is intelligent so if value is in unencrypted form it return + unencrypted form, but on save it always encrypts + """ + impl = Text + + def process_bind_param(self, value, dialect): + if not value: + return value + if value.startswith('enc$aes$') or value.startswith('enc$aes_hmac$'): + # protect against double encrypting if someone manually starts + # doing + raise ValueError('value needs to be in unencrypted format, ie. ' + 'not starting with enc$aes') + return 'enc$aes_hmac$%s' % AESCipher( + ENCRYPTION_KEY, hmac=True).encrypt(value) + + def process_result_value(self, value, dialect): + import rhodecode + + if not value: + return value + + parts = value.split('$', 3) + if not len(parts) == 3: + # probably not encrypted values + return value + else: + if parts[0] != 'enc': + # parts ok but without our header ? + return value + enc_strict_mode = str2bool(rhodecode.CONFIG.get( + 'rhodecode.encrypted_values.strict') or True) + # at that stage we know it's our encryption + if parts[1] == 'aes': + decrypted_data = AESCipher(ENCRYPTION_KEY).decrypt(parts[2]) + elif parts[1] == 'aes_hmac': + decrypted_data = AESCipher( + ENCRYPTION_KEY, hmac=True, + strict_verification=enc_strict_mode).decrypt(parts[2]) + else: + raise ValueError( + 'Encryption type part is wrong, must be `aes` ' + 'or `aes_hmac`, got `%s` instead' % (parts[1])) + return decrypted_data + + +class BaseModel(object): + """ + Base Model for all classes + """ + + @classmethod + def _get_keys(cls): + """return column names for this model """ + return class_mapper(cls).c.keys() + + def get_dict(self): + """ + return dict with keys and values corresponding + to this model data """ + + d = {} + for k in self._get_keys(): + d[k] = getattr(self, k) + + # also use __json__() if present to get additional fields + _json_attr = getattr(self, '__json__', None) + if _json_attr: + # update with attributes from __json__ + if callable(_json_attr): + _json_attr = _json_attr() + for k, val in _json_attr.iteritems(): + d[k] = val + return d + + def get_appstruct(self): + """return list with keys and values tuples corresponding + to this model data """ + + lst = [] + for k in self._get_keys(): + lst.append((k, getattr(self, k),)) + return lst + + def populate_obj(self, populate_dict): + """populate model with data from given populate_dict""" + + for k in self._get_keys(): + if k in populate_dict: + setattr(self, k, populate_dict[k]) + + @classmethod + def query(cls): + return Session().query(cls) + + @classmethod + def get(cls, id_): + if id_: + return cls.query().get(id_) + + @classmethod + def get_or_404(cls, id_): + from pyramid.httpexceptions import HTTPNotFound + + try: + id_ = int(id_) + except (TypeError, ValueError): + raise HTTPNotFound() + + res = cls.query().get(id_) + if not res: + raise HTTPNotFound() + return res + + @classmethod + def getAll(cls): + # deprecated and left for backward compatibility + return cls.get_all() + + @classmethod + def get_all(cls): + return cls.query().all() + + @classmethod + def delete(cls, id_): + obj = cls.query().get(id_) + Session().delete(obj) + + @classmethod + def identity_cache(cls, session, attr_name, value): + exist_in_session = [] + for (item_cls, pkey), instance in session.identity_map.items(): + if cls == item_cls and getattr(instance, attr_name) == value: + exist_in_session.append(instance) + if exist_in_session: + if len(exist_in_session) == 1: + return exist_in_session[0] + log.exception( + 'multiple objects with attr %s and ' + 'value %s found with same name: %r', + attr_name, value, exist_in_session) + + def __repr__(self): + if hasattr(self, '__unicode__'): + # python repr needs to return str + try: + return safe_str(self.__unicode__()) + except UnicodeDecodeError: + pass + return '' % (self.__class__.__name__) + + +class RhodeCodeSetting(Base, BaseModel): + __tablename__ = 'rhodecode_settings' + __table_args__ = ( + UniqueConstraint('app_settings_name'), + base_table_args + ) + + SETTINGS_TYPES = { + 'str': safe_str, + 'int': safe_int, + 'unicode': safe_unicode, + 'bool': str2bool, + 'list': functools.partial(aslist, sep=',') + } + DEFAULT_UPDATE_URL = 'https://rhodecode.com/api/v1/info/versions' + GLOBAL_CONF_KEY = 'app_settings' + + app_settings_id = Column("app_settings_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + app_settings_name = Column("app_settings_name", String(255), nullable=True, unique=None, default=None) + _app_settings_value = Column("app_settings_value", String(4096), nullable=True, unique=None, default=None) + _app_settings_type = Column("app_settings_type", String(255), nullable=True, unique=None, default=None) + + def __init__(self, key='', val='', type='unicode'): + self.app_settings_name = key + self.app_settings_type = type + self.app_settings_value = val + + @validates('_app_settings_value') + def validate_settings_value(self, key, val): + assert type(val) == unicode + return val + + @hybrid_property + def app_settings_value(self): + v = self._app_settings_value + _type = self.app_settings_type + if _type: + _type = self.app_settings_type.split('.')[0] + # decode the encrypted value + if 'encrypted' in self.app_settings_type: + cipher = EncryptedTextValue() + v = safe_unicode(cipher.process_result_value(v, None)) + + converter = self.SETTINGS_TYPES.get(_type) or \ + self.SETTINGS_TYPES['unicode'] + return converter(v) + + @app_settings_value.setter + def app_settings_value(self, val): + """ + Setter that will always make sure we use unicode in app_settings_value + + :param val: + """ + val = safe_unicode(val) + # encode the encrypted value + if 'encrypted' in self.app_settings_type: + cipher = EncryptedTextValue() + val = safe_unicode(cipher.process_bind_param(val, None)) + self._app_settings_value = val + + @hybrid_property + def app_settings_type(self): + return self._app_settings_type + + @app_settings_type.setter + def app_settings_type(self, val): + if val.split('.')[0] not in self.SETTINGS_TYPES: + raise Exception('type must be one of %s got %s' + % (self.SETTINGS_TYPES.keys(), val)) + self._app_settings_type = val + + @classmethod + def get_by_prefix(cls, prefix): + return RhodeCodeSetting.query()\ + .filter(RhodeCodeSetting.app_settings_name.startswith(prefix))\ + .all() + + def __unicode__(self): + return u"<%s('%s:%s[%s]')>" % ( + self.__class__.__name__, + self.app_settings_name, self.app_settings_value, + self.app_settings_type + ) + + +class RhodeCodeUi(Base, BaseModel): + __tablename__ = 'rhodecode_ui' + __table_args__ = ( + UniqueConstraint('ui_key'), + base_table_args + ) + + HOOK_REPO_SIZE = 'changegroup.repo_size' + # HG + HOOK_PRE_PULL = 'preoutgoing.pre_pull' + HOOK_PULL = 'outgoing.pull_logger' + HOOK_PRE_PUSH = 'prechangegroup.pre_push' + HOOK_PRETX_PUSH = 'pretxnchangegroup.pre_push' + HOOK_PUSH = 'changegroup.push_logger' + HOOK_PUSH_KEY = 'pushkey.key_push' + + # TODO: johbo: Unify way how hooks are configured for git and hg, + # git part is currently hardcoded. + + # SVN PATTERNS + SVN_BRANCH_ID = 'vcs_svn_branch' + SVN_TAG_ID = 'vcs_svn_tag' + + ui_id = Column( + "ui_id", Integer(), nullable=False, unique=True, default=None, + primary_key=True) + ui_section = Column( + "ui_section", String(255), nullable=True, unique=None, default=None) + ui_key = Column( + "ui_key", String(255), nullable=True, unique=None, default=None) + ui_value = Column( + "ui_value", String(255), nullable=True, unique=None, default=None) + ui_active = Column( + "ui_active", Boolean(), nullable=True, unique=None, default=True) + + def __repr__(self): + return '<%s[%s]%s=>%s]>' % (self.__class__.__name__, self.ui_section, + self.ui_key, self.ui_value) + + +class RepoRhodeCodeSetting(Base, BaseModel): + __tablename__ = 'repo_rhodecode_settings' + __table_args__ = ( + UniqueConstraint( + 'app_settings_name', 'repository_id', + name='uq_repo_rhodecode_setting_name_repo_id'), + base_table_args + ) + + repository_id = Column( + "repository_id", Integer(), ForeignKey('repositories.repo_id'), + nullable=False) + app_settings_id = Column( + "app_settings_id", Integer(), nullable=False, unique=True, + default=None, primary_key=True) + app_settings_name = Column( + "app_settings_name", String(255), nullable=True, unique=None, + default=None) + _app_settings_value = Column( + "app_settings_value", String(4096), nullable=True, unique=None, + default=None) + _app_settings_type = Column( + "app_settings_type", String(255), nullable=True, unique=None, + default=None) + + repository = relationship('Repository') + + def __init__(self, repository_id, key='', val='', type='unicode'): + self.repository_id = repository_id + self.app_settings_name = key + self.app_settings_type = type + self.app_settings_value = val + + @validates('_app_settings_value') + def validate_settings_value(self, key, val): + assert type(val) == unicode + return val + + @hybrid_property + def app_settings_value(self): + v = self._app_settings_value + type_ = self.app_settings_type + SETTINGS_TYPES = RhodeCodeSetting.SETTINGS_TYPES + converter = SETTINGS_TYPES.get(type_) or SETTINGS_TYPES['unicode'] + return converter(v) + + @app_settings_value.setter + def app_settings_value(self, val): + """ + Setter that will always make sure we use unicode in app_settings_value + + :param val: + """ + self._app_settings_value = safe_unicode(val) + + @hybrid_property + def app_settings_type(self): + return self._app_settings_type + + @app_settings_type.setter + def app_settings_type(self, val): + SETTINGS_TYPES = RhodeCodeSetting.SETTINGS_TYPES + if val not in SETTINGS_TYPES: + raise Exception('type must be one of %s got %s' + % (SETTINGS_TYPES.keys(), val)) + self._app_settings_type = val + + def __unicode__(self): + return u"<%s('%s:%s:%s[%s]')>" % ( + self.__class__.__name__, self.repository.repo_name, + self.app_settings_name, self.app_settings_value, + self.app_settings_type + ) + + +class RepoRhodeCodeUi(Base, BaseModel): + __tablename__ = 'repo_rhodecode_ui' + __table_args__ = ( + UniqueConstraint( + 'repository_id', 'ui_section', 'ui_key', + name='uq_repo_rhodecode_ui_repository_id_section_key'), + base_table_args + ) + + repository_id = Column( + "repository_id", Integer(), ForeignKey('repositories.repo_id'), + nullable=False) + ui_id = Column( + "ui_id", Integer(), nullable=False, unique=True, default=None, + primary_key=True) + ui_section = Column( + "ui_section", String(255), nullable=True, unique=None, default=None) + ui_key = Column( + "ui_key", String(255), nullable=True, unique=None, default=None) + ui_value = Column( + "ui_value", String(255), nullable=True, unique=None, default=None) + ui_active = Column( + "ui_active", Boolean(), nullable=True, unique=None, default=True) + + repository = relationship('Repository') + + def __repr__(self): + return '<%s[%s:%s]%s=>%s]>' % ( + self.__class__.__name__, self.repository.repo_name, + self.ui_section, self.ui_key, self.ui_value) + + +class User(Base, BaseModel): + __tablename__ = 'users' + __table_args__ = ( + UniqueConstraint('username'), UniqueConstraint('email'), + Index('u_username_idx', 'username'), + Index('u_email_idx', 'email'), + base_table_args + ) + + DEFAULT_USER = 'default' + DEFAULT_USER_EMAIL = 'anonymous@rhodecode.org' + DEFAULT_GRAVATAR_URL = 'https://secure.gravatar.com/avatar/{md5email}?d=identicon&s={size}' + + user_id = Column("user_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + username = Column("username", String(255), nullable=True, unique=None, default=None) + password = Column("password", String(255), nullable=True, unique=None, default=None) + active = Column("active", Boolean(), nullable=True, unique=None, default=True) + admin = Column("admin", Boolean(), nullable=True, unique=None, default=False) + name = Column("firstname", String(255), nullable=True, unique=None, default=None) + lastname = Column("lastname", String(255), nullable=True, unique=None, default=None) + _email = Column("email", String(255), nullable=True, unique=None, default=None) + last_login = Column("last_login", DateTime(timezone=False), nullable=True, unique=None, default=None) + last_activity = Column('last_activity', DateTime(timezone=False), nullable=True, unique=None, default=None) + + extern_type = Column("extern_type", String(255), nullable=True, unique=None, default=None) + extern_name = Column("extern_name", String(255), nullable=True, unique=None, default=None) + _api_key = Column("api_key", String(255), nullable=True, unique=None, default=None) + inherit_default_permissions = Column("inherit_default_permissions", Boolean(), nullable=False, unique=None, default=True) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + _user_data = Column("user_data", LargeBinary(), nullable=True) # JSON data + + user_log = relationship('UserLog') + user_perms = relationship('UserToPerm', primaryjoin="User.user_id==UserToPerm.user_id", cascade='all') + + repositories = relationship('Repository') + repository_groups = relationship('RepoGroup') + user_groups = relationship('UserGroup') + + user_followers = relationship('UserFollowing', primaryjoin='UserFollowing.follows_user_id==User.user_id', cascade='all') + followings = relationship('UserFollowing', primaryjoin='UserFollowing.user_id==User.user_id', cascade='all') + + repo_to_perm = relationship('UserRepoToPerm', primaryjoin='UserRepoToPerm.user_id==User.user_id', cascade='all') + repo_group_to_perm = relationship('UserRepoGroupToPerm', primaryjoin='UserRepoGroupToPerm.user_id==User.user_id', cascade='all') + user_group_to_perm = relationship('UserUserGroupToPerm', primaryjoin='UserUserGroupToPerm.user_id==User.user_id', cascade='all') + + group_member = relationship('UserGroupMember', cascade='all') + + notifications = relationship('UserNotification', cascade='all') + # notifications assigned to this user + user_created_notifications = relationship('Notification', cascade='all') + # comments created by this user + user_comments = relationship('ChangesetComment', cascade='all') + # user profile extra info + user_emails = relationship('UserEmailMap', cascade='all') + user_ip_map = relationship('UserIpMap', cascade='all') + user_auth_tokens = relationship('UserApiKeys', cascade='all') + user_ssh_keys = relationship('UserSshKeys', cascade='all') + + # gists + user_gists = relationship('Gist', cascade='all') + # user pull requests + user_pull_requests = relationship('PullRequest', cascade='all') + # external identities + extenal_identities = relationship( + 'ExternalIdentity', + primaryjoin="User.user_id==ExternalIdentity.local_user_id", + cascade='all') + # review rules + user_review_rules = relationship('RepoReviewRuleUser', cascade='all') + + def __unicode__(self): + return u"<%s('id:%s:%s')>" % (self.__class__.__name__, + self.user_id, self.username) + + @hybrid_property + def email(self): + return self._email + + @email.setter + def email(self, val): + self._email = val.lower() if val else None + + @hybrid_property + def first_name(self): + from rhodecode.lib import helpers as h + if self.name: + return h.escape(self.name) + return self.name + + @hybrid_property + def last_name(self): + from rhodecode.lib import helpers as h + if self.lastname: + return h.escape(self.lastname) + return self.lastname + + @hybrid_property + def api_key(self): + """ + Fetch if exist an auth-token with role ALL connected to this user + """ + user_auth_token = UserApiKeys.query()\ + .filter(UserApiKeys.user_id == self.user_id)\ + .filter(or_(UserApiKeys.expires == -1, + UserApiKeys.expires >= time.time()))\ + .filter(UserApiKeys.role == UserApiKeys.ROLE_ALL).first() + if user_auth_token: + user_auth_token = user_auth_token.api_key + + return user_auth_token + + @api_key.setter + def api_key(self, val): + # don't allow to set API key this is deprecated for now + self._api_key = None + + @property + def reviewer_pull_requests(self): + return PullRequestReviewers.query() \ + .options(joinedload(PullRequestReviewers.pull_request)) \ + .filter(PullRequestReviewers.user_id == self.user_id) \ + .all() + + @property + def firstname(self): + # alias for future + return self.name + + @property + def emails(self): + other = UserEmailMap.query()\ + .filter(UserEmailMap.user == self) \ + .order_by(UserEmailMap.email_id.asc()) \ + .all() + return [self.email] + [x.email for x in other] + + @property + def auth_tokens(self): + auth_tokens = self.get_auth_tokens() + return [x.api_key for x in auth_tokens] + + def get_auth_tokens(self): + return UserApiKeys.query()\ + .filter(UserApiKeys.user == self)\ + .order_by(UserApiKeys.user_api_key_id.asc())\ + .all() + + @LazyProperty + def feed_token(self): + return self.get_feed_token() + + def get_feed_token(self, cache=True): + feed_tokens = UserApiKeys.query()\ + .filter(UserApiKeys.user == self)\ + .filter(UserApiKeys.role == UserApiKeys.ROLE_FEED) + if cache: + feed_tokens = feed_tokens.options( + FromCache("sql_cache_short", "get_user_feed_token_%s" % self.user_id)) + + feed_tokens = feed_tokens.all() + if feed_tokens: + return feed_tokens[0].api_key + return 'NO_FEED_TOKEN_AVAILABLE' + + @classmethod + def get(cls, user_id, cache=False): + if not user_id: + return + + user = cls.query() + if cache: + user = user.options( + FromCache("sql_cache_short", "get_users_%s" % user_id)) + return user.get(user_id) + + @classmethod + def extra_valid_auth_tokens(cls, user, role=None): + tokens = UserApiKeys.query().filter(UserApiKeys.user == user)\ + .filter(or_(UserApiKeys.expires == -1, + UserApiKeys.expires >= time.time())) + if role: + tokens = tokens.filter(or_(UserApiKeys.role == role, + UserApiKeys.role == UserApiKeys.ROLE_ALL)) + return tokens.all() + + def authenticate_by_token(self, auth_token, roles=None, scope_repo_id=None): + from rhodecode.lib import auth + + log.debug('Trying to authenticate user: %s via auth-token, ' + 'and roles: %s', self, roles) + + if not auth_token: + return False + + crypto_backend = auth.crypto_backend() + + roles = (roles or []) + [UserApiKeys.ROLE_ALL] + tokens_q = UserApiKeys.query()\ + .filter(UserApiKeys.user_id == self.user_id)\ + .filter(or_(UserApiKeys.expires == -1, + UserApiKeys.expires >= time.time())) + + tokens_q = tokens_q.filter(UserApiKeys.role.in_(roles)) + + plain_tokens = [] + hash_tokens = [] + + user_tokens = tokens_q.all() + log.debug('Found %s user tokens to check for authentication', len(user_tokens)) + for token in user_tokens: + log.debug('AUTH_TOKEN: checking if user token with id `%s` matches', + token.user_api_key_id) + # verify scope first, since it's way faster than hash calculation of + # encrypted tokens + if token.repo_id: + # token has a scope, we need to verify it + if scope_repo_id != token.repo_id: + log.debug( + 'AUTH_TOKEN: scope mismatch, token has a set repo scope: %s, ' + 'and calling scope is:%s, skipping further checks', + token.repo, scope_repo_id) + # token has a scope, and it doesn't match, skip token + continue + + if token.api_key.startswith(crypto_backend.ENC_PREF): + hash_tokens.append(token.api_key) + else: + plain_tokens.append(token.api_key) + + is_plain_match = auth_token in plain_tokens + if is_plain_match: + return True + + for hashed in hash_tokens: + # NOTE(marcink): this is expensive to calculate, but most secure + match = crypto_backend.hash_check(auth_token, hashed) + if match: + return True + + return False + + @property + def ip_addresses(self): + ret = UserIpMap.query().filter(UserIpMap.user == self).all() + return [x.ip_addr for x in ret] + + @property + def username_and_name(self): + return '%s (%s %s)' % (self.username, self.first_name, self.last_name) + + @property + def username_or_name_or_email(self): + full_name = self.full_name if self.full_name is not ' ' else None + return self.username or full_name or self.email + + @property + def full_name(self): + return '%s %s' % (self.first_name, self.last_name) + + @property + def full_name_or_username(self): + return ('%s %s' % (self.first_name, self.last_name) + if (self.first_name and self.last_name) else self.username) + + @property + def full_contact(self): + return '%s %s <%s>' % (self.first_name, self.last_name, self.email) + + @property + def short_contact(self): + return '%s %s' % (self.first_name, self.last_name) + + @property + def is_admin(self): + return self.admin + + def AuthUser(self, **kwargs): + """ + Returns instance of AuthUser for this user + """ + from rhodecode.lib.auth import AuthUser + return AuthUser(user_id=self.user_id, username=self.username, **kwargs) + + @hybrid_property + def user_data(self): + if not self._user_data: + return {} + + try: + return json.loads(self._user_data) + except TypeError: + return {} + + @user_data.setter + def user_data(self, val): + if not isinstance(val, dict): + raise Exception('user_data must be dict, got %s' % type(val)) + try: + self._user_data = json.dumps(val) + except Exception: + log.error(traceback.format_exc()) + + @classmethod + def get_by_username(cls, username, case_insensitive=False, + cache=False, identity_cache=False): + session = Session() + + if case_insensitive: + q = cls.query().filter( + func.lower(cls.username) == func.lower(username)) + else: + q = cls.query().filter(cls.username == username) + + if cache: + if identity_cache: + val = cls.identity_cache(session, 'username', username) + if val: + return val + else: + cache_key = "get_user_by_name_%s" % _hash_key(username) + q = q.options( + FromCache("sql_cache_short", cache_key)) + + return q.scalar() + + @classmethod + def get_by_auth_token(cls, auth_token, cache=False): + q = UserApiKeys.query()\ + .filter(UserApiKeys.api_key == auth_token)\ + .filter(or_(UserApiKeys.expires == -1, + UserApiKeys.expires >= time.time())) + if cache: + q = q.options( + FromCache("sql_cache_short", "get_auth_token_%s" % auth_token)) + + match = q.first() + if match: + return match.user + + @classmethod + def get_by_email(cls, email, case_insensitive=False, cache=False): + + if case_insensitive: + q = cls.query().filter(func.lower(cls.email) == func.lower(email)) + + else: + q = cls.query().filter(cls.email == email) + + email_key = _hash_key(email) + if cache: + q = q.options( + FromCache("sql_cache_short", "get_email_key_%s" % email_key)) + + ret = q.scalar() + if ret is None: + q = UserEmailMap.query() + # try fetching in alternate email map + if case_insensitive: + q = q.filter(func.lower(UserEmailMap.email) == func.lower(email)) + else: + q = q.filter(UserEmailMap.email == email) + q = q.options(joinedload(UserEmailMap.user)) + if cache: + q = q.options( + FromCache("sql_cache_short", "get_email_map_key_%s" % email_key)) + ret = getattr(q.scalar(), 'user', None) + + return ret + + @classmethod + def get_from_cs_author(cls, author): + """ + Tries to get User objects out of commit author string + + :param author: + """ + from rhodecode.lib.helpers import email, author_name + # Valid email in the attribute passed, see if they're in the system + _email = email(author) + if _email: + user = cls.get_by_email(_email, case_insensitive=True) + if user: + return user + # Maybe we can match by username? + _author = author_name(author) + user = cls.get_by_username(_author, case_insensitive=True) + if user: + return user + + def update_userdata(self, **kwargs): + usr = self + old = usr.user_data + old.update(**kwargs) + usr.user_data = old + Session().add(usr) + log.debug('updated userdata with ', kwargs) + + def update_lastlogin(self): + """Update user lastlogin""" + self.last_login = datetime.datetime.now() + Session().add(self) + log.debug('updated user %s lastlogin', self.username) + + def update_password(self, new_password): + from rhodecode.lib.auth import get_crypt_password + + self.password = get_crypt_password(new_password) + Session().add(self) + + @classmethod + def get_first_super_admin(cls): + user = User.query()\ + .filter(User.admin == true()) \ + .order_by(User.user_id.asc()) \ + .first() + + if user is None: + raise Exception('FATAL: Missing administrative account!') + return user + + @classmethod + def get_all_super_admins(cls, only_active=False): + """ + Returns all admin accounts sorted by username + """ + qry = User.query().filter(User.admin == true()).order_by(User.username.asc()) + if only_active: + qry = qry.filter(User.active == true()) + return qry.all() + + @classmethod + def get_default_user(cls, cache=False, refresh=False): + user = User.get_by_username(User.DEFAULT_USER, cache=cache) + if user is None: + raise Exception('FATAL: Missing default account!') + if refresh: + # The default user might be based on outdated state which + # has been loaded from the cache. + # A call to refresh() ensures that the + # latest state from the database is used. + Session().refresh(user) + return user + + def _get_default_perms(self, user, suffix=''): + from rhodecode.model.permission import PermissionModel + return PermissionModel().get_default_perms(user.user_perms, suffix) + + def get_default_perms(self, suffix=''): + return self._get_default_perms(self, suffix) + + def get_api_data(self, include_secrets=False, details='full'): + """ + Common function for generating user related data for API + + :param include_secrets: By default secrets in the API data will be replaced + by a placeholder value to prevent exposing this data by accident. In case + this data shall be exposed, set this flag to ``True``. + + :param details: details can be 'basic|full' basic gives only a subset of + the available user information that includes user_id, name and emails. + """ + user = self + user_data = self.user_data + data = { + 'user_id': user.user_id, + 'username': user.username, + 'firstname': user.name, + 'lastname': user.lastname, + 'email': user.email, + 'emails': user.emails, + } + if details == 'basic': + return data + + auth_token_length = 40 + auth_token_replacement = '*' * auth_token_length + + extras = { + 'auth_tokens': [auth_token_replacement], + 'active': user.active, + 'admin': user.admin, + 'extern_type': user.extern_type, + 'extern_name': user.extern_name, + 'last_login': user.last_login, + 'last_activity': user.last_activity, + 'ip_addresses': user.ip_addresses, + 'language': user_data.get('language') + } + data.update(extras) + + if include_secrets: + data['auth_tokens'] = user.auth_tokens + return data + + def __json__(self): + data = { + 'full_name': self.full_name, + 'full_name_or_username': self.full_name_or_username, + 'short_contact': self.short_contact, + 'full_contact': self.full_contact, + } + data.update(self.get_api_data()) + return data + + +class UserApiKeys(Base, BaseModel): + __tablename__ = 'user_api_keys' + __table_args__ = ( + Index('uak_api_key_idx', 'api_key', unique=True), + Index('uak_api_key_expires_idx', 'api_key', 'expires'), + base_table_args + ) + __mapper_args__ = {} + + # ApiKey role + ROLE_ALL = 'token_role_all' + ROLE_HTTP = 'token_role_http' + ROLE_VCS = 'token_role_vcs' + ROLE_API = 'token_role_api' + ROLE_FEED = 'token_role_feed' + ROLE_PASSWORD_RESET = 'token_password_reset' + + ROLES = [ROLE_ALL, ROLE_HTTP, ROLE_VCS, ROLE_API, ROLE_FEED] + + user_api_key_id = Column("user_api_key_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None) + api_key = Column("api_key", String(255), nullable=False, unique=True) + description = Column('description', UnicodeText().with_variant(UnicodeText(1024), 'mysql')) + expires = Column('expires', Float(53), nullable=False) + role = Column('role', String(255), nullable=True) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + + # scope columns + repo_id = Column( + 'repo_id', Integer(), ForeignKey('repositories.repo_id'), + nullable=True, unique=None, default=None) + repo = relationship('Repository', lazy='joined') + + repo_group_id = Column( + 'repo_group_id', Integer(), ForeignKey('groups.group_id'), + nullable=True, unique=None, default=None) + repo_group = relationship('RepoGroup', lazy='joined') + + user = relationship('User', lazy='joined') + + def __unicode__(self): + return u"<%s('%s')>" % (self.__class__.__name__, self.role) + + def __json__(self): + data = { + 'auth_token': self.api_key, + 'role': self.role, + 'scope': self.scope_humanized, + 'expired': self.expired + } + return data + + def get_api_data(self, include_secrets=False): + data = self.__json__() + if include_secrets: + return data + else: + data['auth_token'] = self.token_obfuscated + return data + + @hybrid_property + def description_safe(self): + from rhodecode.lib import helpers as h + return h.escape(self.description) + + @property + def expired(self): + if self.expires == -1: + return False + return time.time() > self.expires + + @classmethod + def _get_role_name(cls, role): + return { + cls.ROLE_ALL: _('all'), + cls.ROLE_HTTP: _('http/web interface'), + cls.ROLE_VCS: _('vcs (git/hg/svn protocol)'), + cls.ROLE_API: _('api calls'), + cls.ROLE_FEED: _('feed access'), + }.get(role, role) + + @property + def role_humanized(self): + return self._get_role_name(self.role) + + def _get_scope(self): + if self.repo: + return 'Repository: {}'.format(self.repo.repo_name) + if self.repo_group: + return 'RepositoryGroup: {} (recursive)'.format(self.repo_group.group_name) + return 'Global' + + @property + def scope_humanized(self): + return self._get_scope() + + @property + def token_obfuscated(self): + if self.api_key: + return self.api_key[:4] + "****" + + +class UserEmailMap(Base, BaseModel): + __tablename__ = 'user_email_map' + __table_args__ = ( + Index('uem_email_idx', 'email'), + UniqueConstraint('email'), + base_table_args + ) + __mapper_args__ = {} + + email_id = Column("email_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None) + _email = Column("email", String(255), nullable=True, unique=False, default=None) + user = relationship('User', lazy='joined') + + @validates('_email') + def validate_email(self, key, email): + # check if this email is not main one + main_email = Session().query(User).filter(User.email == email).scalar() + if main_email is not None: + raise AttributeError('email %s is present is user table' % email) + return email + + @hybrid_property + def email(self): + return self._email + + @email.setter + def email(self, val): + self._email = val.lower() if val else None + + +class UserIpMap(Base, BaseModel): + __tablename__ = 'user_ip_map' + __table_args__ = ( + UniqueConstraint('user_id', 'ip_addr'), + base_table_args + ) + __mapper_args__ = {} + + ip_id = Column("ip_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None) + ip_addr = Column("ip_addr", String(255), nullable=True, unique=False, default=None) + active = Column("active", Boolean(), nullable=True, unique=None, default=True) + description = Column("description", String(10000), nullable=True, unique=None, default=None) + user = relationship('User', lazy='joined') + + @hybrid_property + def description_safe(self): + from rhodecode.lib import helpers as h + return h.escape(self.description) + + @classmethod + def _get_ip_range(cls, ip_addr): + net = ipaddress.ip_network(safe_unicode(ip_addr), strict=False) + return [str(net.network_address), str(net.broadcast_address)] + + def __json__(self): + return { + 'ip_addr': self.ip_addr, + 'ip_range': self._get_ip_range(self.ip_addr), + } + + def __unicode__(self): + return u"<%s('user_id:%s=>%s')>" % (self.__class__.__name__, + self.user_id, self.ip_addr) + + +class UserSshKeys(Base, BaseModel): + __tablename__ = 'user_ssh_keys' + __table_args__ = ( + Index('usk_ssh_key_fingerprint_idx', 'ssh_key_fingerprint'), + + UniqueConstraint('ssh_key_fingerprint'), + + base_table_args + ) + __mapper_args__ = {} + + ssh_key_id = Column('ssh_key_id', Integer(), nullable=False, unique=True, default=None, primary_key=True) + ssh_key_data = Column('ssh_key_data', String(10240), nullable=False, unique=None, default=None) + ssh_key_fingerprint = Column('ssh_key_fingerprint', String(255), nullable=False, unique=None, default=None) + + description = Column('description', UnicodeText().with_variant(UnicodeText(1024), 'mysql')) + + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + accessed_on = Column('accessed_on', DateTime(timezone=False), nullable=True, default=None) + user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None) + + user = relationship('User', lazy='joined') + + def __json__(self): + data = { + 'ssh_fingerprint': self.ssh_key_fingerprint, + 'description': self.description, + 'created_on': self.created_on + } + return data + + def get_api_data(self): + data = self.__json__() + return data + + +class UserLog(Base, BaseModel): + __tablename__ = 'user_logs' + __table_args__ = ( + base_table_args, + ) + + VERSION_1 = 'v1' + VERSION_2 = 'v2' + VERSIONS = [VERSION_1, VERSION_2] + + user_log_id = Column("user_log_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id',ondelete='SET NULL'), nullable=True, unique=None, default=None) + username = Column("username", String(255), nullable=True, unique=None, default=None) + repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id', ondelete='SET NULL'), nullable=True, unique=None, default=None) + repository_name = Column("repository_name", String(255), nullable=True, unique=None, default=None) + user_ip = Column("user_ip", String(255), nullable=True, unique=None, default=None) + action = Column("action", Text().with_variant(Text(1200000), 'mysql'), nullable=True, unique=None, default=None) + action_date = Column("action_date", DateTime(timezone=False), nullable=True, unique=None, default=None) + + version = Column("version", String(255), nullable=True, default=VERSION_1) + user_data = Column('user_data_json', MutationObj.as_mutable(JsonType(dialect_map=dict(mysql=LONGTEXT())))) + action_data = Column('action_data_json', MutationObj.as_mutable(JsonType(dialect_map=dict(mysql=LONGTEXT())))) + + def __unicode__(self): + return u"<%s('id:%s:%s')>" % ( + self.__class__.__name__, self.repository_name, self.action) + + def __json__(self): + return { + 'user_id': self.user_id, + 'username': self.username, + 'repository_id': self.repository_id, + 'repository_name': self.repository_name, + 'user_ip': self.user_ip, + 'action_date': self.action_date, + 'action': self.action, + } + + @hybrid_property + def entry_id(self): + return self.user_log_id + + @property + def action_as_day(self): + return datetime.date(*self.action_date.timetuple()[:3]) + + user = relationship('User') + repository = relationship('Repository', cascade='') + + +class UserGroup(Base, BaseModel): + __tablename__ = 'users_groups' + __table_args__ = ( + base_table_args, + ) + + users_group_id = Column("users_group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + users_group_name = Column("users_group_name", String(255), nullable=False, unique=True, default=None) + user_group_description = Column("user_group_description", String(10000), nullable=True, unique=None, default=None) + users_group_active = Column("users_group_active", Boolean(), nullable=True, unique=None, default=None) + inherit_default_permissions = Column("users_group_inherit_default_permissions", Boolean(), nullable=False, unique=None, default=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=False, default=None) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + _group_data = Column("group_data", LargeBinary(), nullable=True) # JSON data + + members = relationship('UserGroupMember', cascade="all, delete, delete-orphan", lazy="joined") + users_group_to_perm = relationship('UserGroupToPerm', cascade='all') + users_group_repo_to_perm = relationship('UserGroupRepoToPerm', cascade='all') + users_group_repo_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all') + user_user_group_to_perm = relationship('UserUserGroupToPerm', cascade='all') + user_group_user_group_to_perm = relationship('UserGroupUserGroupToPerm ', primaryjoin="UserGroupUserGroupToPerm.target_user_group_id==UserGroup.users_group_id", cascade='all') + + user_group_review_rules = relationship('RepoReviewRuleUserGroup', cascade='all') + user = relationship('User', primaryjoin="User.user_id==UserGroup.user_id") + + @classmethod + def _load_group_data(cls, column): + if not column: + return {} + + try: + return json.loads(column) or {} + except TypeError: + return {} + + @hybrid_property + def description_safe(self): + from rhodecode.lib import helpers as h + return h.escape(self.user_group_description) + + @hybrid_property + def group_data(self): + return self._load_group_data(self._group_data) + + @group_data.expression + def group_data(self, **kwargs): + return self._group_data + + @group_data.setter + def group_data(self, val): + try: + self._group_data = json.dumps(val) + except Exception: + log.error(traceback.format_exc()) + + @classmethod + def _load_sync(cls, group_data): + if group_data: + return group_data.get('extern_type') + + @property + def sync(self): + return self._load_sync(self.group_data) + + def __unicode__(self): + return u"<%s('id:%s:%s')>" % (self.__class__.__name__, + self.users_group_id, + self.users_group_name) + + @classmethod + def get_by_group_name(cls, group_name, cache=False, + case_insensitive=False): + if case_insensitive: + q = cls.query().filter(func.lower(cls.users_group_name) == + func.lower(group_name)) + + else: + q = cls.query().filter(cls.users_group_name == group_name) + if cache: + q = q.options( + FromCache("sql_cache_short", "get_group_%s" % _hash_key(group_name))) + return q.scalar() + + @classmethod + def get(cls, user_group_id, cache=False): + if not user_group_id: + return + + user_group = cls.query() + if cache: + user_group = user_group.options( + FromCache("sql_cache_short", "get_users_group_%s" % user_group_id)) + return user_group.get(user_group_id) + + def permissions(self, with_admins=True, with_owner=True, + expand_from_user_groups=False): + """ + Permissions for user groups + """ + _admin_perm = 'usergroup.admin' + + owner_row = [] + if with_owner: + usr = AttributeDict(self.user.get_dict()) + usr.owner_row = True + usr.permission = _admin_perm + owner_row.append(usr) + + super_admin_ids = [] + super_admin_rows = [] + if with_admins: + for usr in User.get_all_super_admins(): + super_admin_ids.append(usr.user_id) + # if this admin is also owner, don't double the record + if usr.user_id == owner_row[0].user_id: + owner_row[0].admin_row = True + else: + usr = AttributeDict(usr.get_dict()) + usr.admin_row = True + usr.permission = _admin_perm + super_admin_rows.append(usr) + + q = UserUserGroupToPerm.query().filter(UserUserGroupToPerm.user_group == self) + q = q.options(joinedload(UserUserGroupToPerm.user_group), + joinedload(UserUserGroupToPerm.user), + joinedload(UserUserGroupToPerm.permission),) + + # get owners and admins and permissions. We do a trick of re-writing + # objects from sqlalchemy to named-tuples due to sqlalchemy session + # has a global reference and changing one object propagates to all + # others. This means if admin is also an owner admin_row that change + # would propagate to both objects + perm_rows = [] + for _usr in q.all(): + usr = AttributeDict(_usr.user.get_dict()) + # if this user is also owner/admin, mark as duplicate record + if usr.user_id == owner_row[0].user_id or usr.user_id in super_admin_ids: + usr.duplicate_perm = True + usr.permission = _usr.permission.permission_name + perm_rows.append(usr) + + # filter the perm rows by 'default' first and then sort them by + # admin,write,read,none permissions sorted again alphabetically in + # each group + perm_rows = sorted(perm_rows, key=display_user_sort) + + user_groups_rows = [] + if expand_from_user_groups: + for ug in self.permission_user_groups(with_members=True): + for user_data in ug.members: + user_groups_rows.append(user_data) + + return super_admin_rows + owner_row + perm_rows + user_groups_rows + + def permission_user_groups(self, with_members=False): + q = UserGroupUserGroupToPerm.query()\ + .filter(UserGroupUserGroupToPerm.target_user_group == self) + q = q.options(joinedload(UserGroupUserGroupToPerm.user_group), + joinedload(UserGroupUserGroupToPerm.target_user_group), + joinedload(UserGroupUserGroupToPerm.permission),) + + perm_rows = [] + for _user_group in q.all(): + entry = AttributeDict(_user_group.user_group.get_dict()) + entry.permission = _user_group.permission.permission_name + if with_members: + entry.members = [x.user.get_dict() + for x in _user_group.users_group.members] + perm_rows.append(entry) + + perm_rows = sorted(perm_rows, key=display_user_group_sort) + return perm_rows + + def _get_default_perms(self, user_group, suffix=''): + from rhodecode.model.permission import PermissionModel + return PermissionModel().get_default_perms(user_group.users_group_to_perm, suffix) + + def get_default_perms(self, suffix=''): + return self._get_default_perms(self, suffix) + + def get_api_data(self, with_group_members=True, include_secrets=False): + """ + :param include_secrets: See :meth:`User.get_api_data`, this parameter is + basically forwarded. + + """ + user_group = self + data = { + 'users_group_id': user_group.users_group_id, + 'group_name': user_group.users_group_name, + 'group_description': user_group.user_group_description, + 'active': user_group.users_group_active, + 'owner': user_group.user.username, + 'sync': user_group.sync, + 'owner_email': user_group.user.email, + } + + if with_group_members: + users = [] + for user in user_group.members: + user = user.user + users.append(user.get_api_data(include_secrets=include_secrets)) + data['users'] = users + + return data + + +class UserGroupMember(Base, BaseModel): + __tablename__ = 'users_groups_members' + __table_args__ = ( + base_table_args, + ) + + users_group_member_id = Column("users_group_member_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) + + user = relationship('User', lazy='joined') + users_group = relationship('UserGroup') + + def __init__(self, gr_id='', u_id=''): + self.users_group_id = gr_id + self.user_id = u_id + + +class RepositoryField(Base, BaseModel): + __tablename__ = 'repositories_fields' + __table_args__ = ( + UniqueConstraint('repository_id', 'field_key'), # no-multi field + base_table_args, + ) + + PREFIX = 'ex_' # prefix used in form to not conflict with already existing fields + + repo_field_id = Column("repo_field_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None) + field_key = Column("field_key", String(250)) + field_label = Column("field_label", String(1024), nullable=False) + field_value = Column("field_value", String(10000), nullable=False) + field_desc = Column("field_desc", String(1024), nullable=False) + field_type = Column("field_type", String(255), nullable=False, unique=None) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + + repository = relationship('Repository') + + @property + def field_key_prefixed(self): + return 'ex_%s' % self.field_key + + @classmethod + def un_prefix_key(cls, key): + if key.startswith(cls.PREFIX): + return key[len(cls.PREFIX):] + return key + + @classmethod + def get_by_key_name(cls, key, repo): + row = cls.query()\ + .filter(cls.repository == repo)\ + .filter(cls.field_key == key).scalar() + return row + + +class Repository(Base, BaseModel): + __tablename__ = 'repositories' + __table_args__ = ( + Index('r_repo_name_idx', 'repo_name', mysql_length=255), + base_table_args, + ) + DEFAULT_CLONE_URI = '{scheme}://{user}@{netloc}/{repo}' + DEFAULT_CLONE_URI_ID = '{scheme}://{user}@{netloc}/_{repoid}' + DEFAULT_CLONE_URI_SSH = 'ssh://{sys_user}@{hostname}/{repo}' + + STATE_CREATED = 'repo_state_created' + STATE_PENDING = 'repo_state_pending' + STATE_ERROR = 'repo_state_error' + + LOCK_AUTOMATIC = 'lock_auto' + LOCK_API = 'lock_api' + LOCK_WEB = 'lock_web' + LOCK_PULL = 'lock_pull' + + NAME_SEP = URL_SEP + + repo_id = Column( + "repo_id", Integer(), nullable=False, unique=True, default=None, + primary_key=True) + _repo_name = Column( + "repo_name", Text(), nullable=False, default=None) + _repo_name_hash = Column( + "repo_name_hash", String(255), nullable=False, unique=True) + repo_state = Column("repo_state", String(255), nullable=True) + + clone_uri = Column( + "clone_uri", EncryptedTextValue(), nullable=True, unique=False, + default=None) + push_uri = Column( + "push_uri", EncryptedTextValue(), nullable=True, unique=False, + default=None) + repo_type = Column( + "repo_type", String(255), nullable=False, unique=False, default=None) + user_id = Column( + "user_id", Integer(), ForeignKey('users.user_id'), nullable=False, + unique=False, default=None) + private = Column( + "private", Boolean(), nullable=True, unique=None, default=None) + archived = Column( + "archived", Boolean(), nullable=True, unique=None, default=None) + enable_statistics = Column( + "statistics", Boolean(), nullable=True, unique=None, default=True) + enable_downloads = Column( + "downloads", Boolean(), nullable=True, unique=None, default=True) + description = Column( + "description", String(10000), nullable=True, unique=None, default=None) + created_on = Column( + 'created_on', DateTime(timezone=False), nullable=True, unique=None, + default=datetime.datetime.now) + updated_on = Column( + 'updated_on', DateTime(timezone=False), nullable=True, unique=None, + default=datetime.datetime.now) + _landing_revision = Column( + "landing_revision", String(255), nullable=False, unique=False, + default=None) + enable_locking = Column( + "enable_locking", Boolean(), nullable=False, unique=None, + default=False) + _locked = Column( + "locked", String(255), nullable=True, unique=False, default=None) + _changeset_cache = Column( + "changeset_cache", LargeBinary(), nullable=True) # JSON data + + fork_id = Column( + "fork_id", Integer(), ForeignKey('repositories.repo_id'), + nullable=True, unique=False, default=None) + group_id = Column( + "group_id", Integer(), ForeignKey('groups.group_id'), nullable=True, + unique=False, default=None) + + user = relationship('User', lazy='joined') + fork = relationship('Repository', remote_side=repo_id, lazy='joined') + group = relationship('RepoGroup', lazy='joined') + repo_to_perm = relationship( + 'UserRepoToPerm', cascade='all', + order_by='UserRepoToPerm.repo_to_perm_id') + users_group_to_perm = relationship('UserGroupRepoToPerm', cascade='all') + stats = relationship('Statistics', cascade='all', uselist=False) + + followers = relationship( + 'UserFollowing', + primaryjoin='UserFollowing.follows_repo_id==Repository.repo_id', + cascade='all') + extra_fields = relationship( + 'RepositoryField', cascade="all, delete, delete-orphan") + logs = relationship('UserLog') + comments = relationship( + 'ChangesetComment', cascade="all, delete, delete-orphan") + pull_requests_source = relationship( + 'PullRequest', + primaryjoin='PullRequest.source_repo_id==Repository.repo_id', + cascade="all, delete, delete-orphan") + pull_requests_target = relationship( + 'PullRequest', + primaryjoin='PullRequest.target_repo_id==Repository.repo_id', + cascade="all, delete, delete-orphan") + ui = relationship('RepoRhodeCodeUi', cascade="all") + settings = relationship('RepoRhodeCodeSetting', cascade="all") + integrations = relationship('Integration', + cascade="all, delete, delete-orphan") + + scoped_tokens = relationship('UserApiKeys', cascade="all") + + def __unicode__(self): + return u"<%s('%s:%s')>" % (self.__class__.__name__, self.repo_id, + safe_unicode(self.repo_name)) + + @hybrid_property + def description_safe(self): + from rhodecode.lib import helpers as h + return h.escape(self.description) + + @hybrid_property + def landing_rev(self): + # always should return [rev_type, rev] + if self._landing_revision: + _rev_info = self._landing_revision.split(':') + if len(_rev_info) < 2: + _rev_info.insert(0, 'rev') + return [_rev_info[0], _rev_info[1]] + return [None, None] + + @landing_rev.setter + def landing_rev(self, val): + if ':' not in val: + raise ValueError('value must be delimited with `:` and consist ' + 'of :, got %s instead' % val) + self._landing_revision = val + + @hybrid_property + def locked(self): + if self._locked: + user_id, timelocked, reason = self._locked.split(':') + lock_values = int(user_id), timelocked, reason + else: + lock_values = [None, None, None] + return lock_values + + @locked.setter + def locked(self, val): + if val and isinstance(val, (list, tuple)): + self._locked = ':'.join(map(str, val)) + else: + self._locked = None + + @hybrid_property + def changeset_cache(self): + from rhodecode.lib.vcs.backends.base import EmptyCommit + dummy = EmptyCommit().__json__() + if not self._changeset_cache: + return dummy + try: + return json.loads(self._changeset_cache) + except TypeError: + return dummy + except Exception: + log.error(traceback.format_exc()) + return dummy + + @changeset_cache.setter + def changeset_cache(self, val): + try: + self._changeset_cache = json.dumps(val) + except Exception: + log.error(traceback.format_exc()) + + @hybrid_property + def repo_name(self): + return self._repo_name + + @repo_name.setter + def repo_name(self, value): + self._repo_name = value + self._repo_name_hash = hashlib.sha1(safe_str(value)).hexdigest() + + @classmethod + def normalize_repo_name(cls, repo_name): + """ + Normalizes os specific repo_name to the format internally stored inside + database using URL_SEP + + :param cls: + :param repo_name: + """ + return cls.NAME_SEP.join(repo_name.split(os.sep)) + + @classmethod + def get_by_repo_name(cls, repo_name, cache=False, identity_cache=False): + session = Session() + q = session.query(cls).filter(cls.repo_name == repo_name) + + if cache: + if identity_cache: + val = cls.identity_cache(session, 'repo_name', repo_name) + if val: + return val + else: + cache_key = "get_repo_by_name_%s" % _hash_key(repo_name) + q = q.options( + FromCache("sql_cache_short", cache_key)) + + return q.scalar() + + @classmethod + def get_by_id_or_repo_name(cls, repoid): + if isinstance(repoid, (int, long)): + try: + repo = cls.get(repoid) + except ValueError: + repo = None + else: + repo = cls.get_by_repo_name(repoid) + return repo + + @classmethod + def get_by_full_path(cls, repo_full_path): + repo_name = repo_full_path.split(cls.base_path(), 1)[-1] + repo_name = cls.normalize_repo_name(repo_name) + return cls.get_by_repo_name(repo_name.strip(URL_SEP)) + + @classmethod + def get_repo_forks(cls, repo_id): + return cls.query().filter(Repository.fork_id == repo_id) + + @classmethod + def base_path(cls): + """ + Returns base path when all repos are stored + + :param cls: + """ + q = Session().query(RhodeCodeUi)\ + .filter(RhodeCodeUi.ui_key == cls.NAME_SEP) + q = q.options(FromCache("sql_cache_short", "repository_repo_path")) + return q.one().ui_value + + @classmethod + def get_all_repos(cls, user_id=Optional(None), group_id=Optional(None), + case_insensitive=True, archived=False): + q = Repository.query() + + if not archived: + q = q.filter(Repository.archived.isnot(true())) + + if not isinstance(user_id, Optional): + q = q.filter(Repository.user_id == user_id) + + if not isinstance(group_id, Optional): + q = q.filter(Repository.group_id == group_id) + + if case_insensitive: + q = q.order_by(func.lower(Repository.repo_name)) + else: + q = q.order_by(Repository.repo_name) + + return q.all() + + @property + def forks(self): + """ + Return forks of this repo + """ + return Repository.get_repo_forks(self.repo_id) + + @property + def parent(self): + """ + Returns fork parent + """ + return self.fork + + @property + def just_name(self): + return self.repo_name.split(self.NAME_SEP)[-1] + + @property + def groups_with_parents(self): + groups = [] + if self.group is None: + return groups + + cur_gr = self.group + groups.insert(0, cur_gr) + while 1: + gr = getattr(cur_gr, 'parent_group', None) + cur_gr = cur_gr.parent_group + if gr is None: + break + groups.insert(0, gr) + + return groups + + @property + def groups_and_repo(self): + return self.groups_with_parents, self + + @LazyProperty + def repo_path(self): + """ + Returns base full path for that repository means where it actually + exists on a filesystem + """ + q = Session().query(RhodeCodeUi).filter( + RhodeCodeUi.ui_key == self.NAME_SEP) + q = q.options(FromCache("sql_cache_short", "repository_repo_path")) + return q.one().ui_value + + @property + def repo_full_path(self): + p = [self.repo_path] + # we need to split the name by / since this is how we store the + # names in the database, but that eventually needs to be converted + # into a valid system path + p += self.repo_name.split(self.NAME_SEP) + return os.path.join(*map(safe_unicode, p)) + + @property + def cache_keys(self): + """ + Returns associated cache keys for that repo + """ + invalidation_namespace = CacheKey.REPO_INVALIDATION_NAMESPACE.format( + repo_id=self.repo_id) + return CacheKey.query()\ + .filter(CacheKey.cache_args == invalidation_namespace)\ + .order_by(CacheKey.cache_key)\ + .all() + + @property + def cached_diffs_relative_dir(self): + """ + Return a relative to the repository store path of cached diffs + used for safe display for users, who shouldn't know the absolute store + path + """ + return os.path.join( + os.path.dirname(self.repo_name), + self.cached_diffs_dir.split(os.path.sep)[-1]) + + @property + def cached_diffs_dir(self): + path = self.repo_full_path + return os.path.join( + os.path.dirname(path), + '.__shadow_diff_cache_repo_{}'.format(self.repo_id)) + + def cached_diffs(self): + diff_cache_dir = self.cached_diffs_dir + if os.path.isdir(diff_cache_dir): + return os.listdir(diff_cache_dir) + return [] + + def shadow_repos(self): + shadow_repos_pattern = '.__shadow_repo_{}'.format(self.repo_id) + return [ + x for x in os.listdir(os.path.dirname(self.repo_full_path)) + if x.startswith(shadow_repos_pattern)] + + def get_new_name(self, repo_name): + """ + returns new full repository name based on assigned group and new new + + :param group_name: + """ + path_prefix = self.group.full_path_splitted if self.group else [] + return self.NAME_SEP.join(path_prefix + [repo_name]) + + @property + def _config(self): + """ + Returns db based config object. + """ + from rhodecode.lib.utils import make_db_config + return make_db_config(clear_session=False, repo=self) + + def permissions(self, with_admins=True, with_owner=True, + expand_from_user_groups=False): + """ + Permissions for repositories + """ + _admin_perm = 'repository.admin' + + owner_row = [] + if with_owner: + usr = AttributeDict(self.user.get_dict()) + usr.owner_row = True + usr.permission = _admin_perm + usr.permission_id = None + owner_row.append(usr) + + super_admin_ids = [] + super_admin_rows = [] + if with_admins: + for usr in User.get_all_super_admins(): + super_admin_ids.append(usr.user_id) + # if this admin is also owner, don't double the record + if usr.user_id == owner_row[0].user_id: + owner_row[0].admin_row = True + else: + usr = AttributeDict(usr.get_dict()) + usr.admin_row = True + usr.permission = _admin_perm + usr.permission_id = None + super_admin_rows.append(usr) + + q = UserRepoToPerm.query().filter(UserRepoToPerm.repository == self) + q = q.options(joinedload(UserRepoToPerm.repository), + joinedload(UserRepoToPerm.user), + joinedload(UserRepoToPerm.permission),) + + # get owners and admins and permissions. We do a trick of re-writing + # objects from sqlalchemy to named-tuples due to sqlalchemy session + # has a global reference and changing one object propagates to all + # others. This means if admin is also an owner admin_row that change + # would propagate to both objects + perm_rows = [] + for _usr in q.all(): + usr = AttributeDict(_usr.user.get_dict()) + # if this user is also owner/admin, mark as duplicate record + if usr.user_id == owner_row[0].user_id or usr.user_id in super_admin_ids: + usr.duplicate_perm = True + # also check if this permission is maybe used by branch_permissions + if _usr.branch_perm_entry: + usr.branch_rules = [x.branch_rule_id for x in _usr.branch_perm_entry] + + usr.permission = _usr.permission.permission_name + usr.permission_id = _usr.repo_to_perm_id + perm_rows.append(usr) + + # filter the perm rows by 'default' first and then sort them by + # admin,write,read,none permissions sorted again alphabetically in + # each group + perm_rows = sorted(perm_rows, key=display_user_sort) + + user_groups_rows = [] + if expand_from_user_groups: + for ug in self.permission_user_groups(with_members=True): + for user_data in ug.members: + user_groups_rows.append(user_data) + + return super_admin_rows + owner_row + perm_rows + user_groups_rows + + def permission_user_groups(self, with_members=True): + q = UserGroupRepoToPerm.query()\ + .filter(UserGroupRepoToPerm.repository == self) + q = q.options(joinedload(UserGroupRepoToPerm.repository), + joinedload(UserGroupRepoToPerm.users_group), + joinedload(UserGroupRepoToPerm.permission),) + + perm_rows = [] + for _user_group in q.all(): + entry = AttributeDict(_user_group.users_group.get_dict()) + entry.permission = _user_group.permission.permission_name + if with_members: + entry.members = [x.user.get_dict() + for x in _user_group.users_group.members] + perm_rows.append(entry) + + perm_rows = sorted(perm_rows, key=display_user_group_sort) + return perm_rows + + def get_api_data(self, include_secrets=False): + """ + Common function for generating repo api data + + :param include_secrets: See :meth:`User.get_api_data`. + + """ + # TODO: mikhail: Here there is an anti-pattern, we probably need to + # move this methods on models level. + from rhodecode.model.settings import SettingsModel + from rhodecode.model.repo import RepoModel + + repo = self + _user_id, _time, _reason = self.locked + + data = { + 'repo_id': repo.repo_id, + 'repo_name': repo.repo_name, + 'repo_type': repo.repo_type, + 'clone_uri': repo.clone_uri or '', + 'push_uri': repo.push_uri or '', + 'url': RepoModel().get_url(self), + 'private': repo.private, + 'created_on': repo.created_on, + 'description': repo.description_safe, + 'landing_rev': repo.landing_rev, + 'owner': repo.user.username, + 'fork_of': repo.fork.repo_name if repo.fork else None, + 'fork_of_id': repo.fork.repo_id if repo.fork else None, + 'enable_statistics': repo.enable_statistics, + 'enable_locking': repo.enable_locking, + 'enable_downloads': repo.enable_downloads, + 'last_changeset': repo.changeset_cache, + 'locked_by': User.get(_user_id).get_api_data( + include_secrets=include_secrets) if _user_id else None, + 'locked_date': time_to_datetime(_time) if _time else None, + 'lock_reason': _reason if _reason else None, + } + + # TODO: mikhail: should be per-repo settings here + rc_config = SettingsModel().get_all_settings() + repository_fields = str2bool( + rc_config.get('rhodecode_repository_fields')) + if repository_fields: + for f in self.extra_fields: + data[f.field_key_prefixed] = f.field_value + + return data + + @classmethod + def lock(cls, repo, user_id, lock_time=None, lock_reason=None): + if not lock_time: + lock_time = time.time() + if not lock_reason: + lock_reason = cls.LOCK_AUTOMATIC + repo.locked = [user_id, lock_time, lock_reason] + Session().add(repo) + Session().commit() + + @classmethod + def unlock(cls, repo): + repo.locked = None + Session().add(repo) + Session().commit() + + @classmethod + def getlock(cls, repo): + return repo.locked + + def is_user_lock(self, user_id): + if self.lock[0]: + lock_user_id = safe_int(self.lock[0]) + user_id = safe_int(user_id) + # both are ints, and they are equal + return all([lock_user_id, user_id]) and lock_user_id == user_id + + return False + + def get_locking_state(self, action, user_id, only_when_enabled=True): + """ + Checks locking on this repository, if locking is enabled and lock is + present returns a tuple of make_lock, locked, locked_by. + make_lock can have 3 states None (do nothing) True, make lock + False release lock, This value is later propagated to hooks, which + do the locking. Think about this as signals passed to hooks what to do. + + """ + # TODO: johbo: This is part of the business logic and should be moved + # into the RepositoryModel. + + if action not in ('push', 'pull'): + raise ValueError("Invalid action value: %s" % repr(action)) + + # defines if locked error should be thrown to user + currently_locked = False + # defines if new lock should be made, tri-state + make_lock = None + repo = self + user = User.get(user_id) + + lock_info = repo.locked + + if repo and (repo.enable_locking or not only_when_enabled): + if action == 'push': + # check if it's already locked !, if it is compare users + locked_by_user_id = lock_info[0] + if user.user_id == locked_by_user_id: + log.debug( + 'Got `push` action from user %s, now unlocking', user) + # unlock if we have push from user who locked + make_lock = False + else: + # we're not the same user who locked, ban with + # code defined in settings (default is 423 HTTP Locked) ! + log.debug('Repo %s is currently locked by %s', repo, user) + currently_locked = True + elif action == 'pull': + # [0] user [1] date + if lock_info[0] and lock_info[1]: + log.debug('Repo %s is currently locked by %s', repo, user) + currently_locked = True + else: + log.debug('Setting lock on repo %s by %s', repo, user) + make_lock = True + + else: + log.debug('Repository %s do not have locking enabled', repo) + + log.debug('FINAL locking values make_lock:%s,locked:%s,locked_by:%s', + make_lock, currently_locked, lock_info) + + from rhodecode.lib.auth import HasRepoPermissionAny + perm_check = HasRepoPermissionAny('repository.write', 'repository.admin') + if make_lock and not perm_check(repo_name=repo.repo_name, user=user): + # if we don't have at least write permission we cannot make a lock + log.debug('lock state reset back to FALSE due to lack ' + 'of at least read permission') + make_lock = False + + return make_lock, currently_locked, lock_info + + @property + def last_db_change(self): + return self.updated_on + + @property + def clone_uri_hidden(self): + clone_uri = self.clone_uri + if clone_uri: + import urlobject + url_obj = urlobject.URLObject(cleaned_uri(clone_uri)) + if url_obj.password: + clone_uri = url_obj.with_password('*****') + return clone_uri + + @property + def push_uri_hidden(self): + push_uri = self.push_uri + if push_uri: + import urlobject + url_obj = urlobject.URLObject(cleaned_uri(push_uri)) + if url_obj.password: + push_uri = url_obj.with_password('*****') + return push_uri + + def clone_url(self, **override): + from rhodecode.model.settings import SettingsModel + + uri_tmpl = None + if 'with_id' in override: + uri_tmpl = self.DEFAULT_CLONE_URI_ID + del override['with_id'] + + if 'uri_tmpl' in override: + uri_tmpl = override['uri_tmpl'] + del override['uri_tmpl'] + + ssh = False + if 'ssh' in override: + ssh = True + del override['ssh'] + + # we didn't override our tmpl from **overrides + if not uri_tmpl: + rc_config = SettingsModel().get_all_settings(cache=True) + if ssh: + uri_tmpl = rc_config.get( + 'rhodecode_clone_uri_ssh_tmpl') or self.DEFAULT_CLONE_URI_SSH + else: + uri_tmpl = rc_config.get( + 'rhodecode_clone_uri_tmpl') or self.DEFAULT_CLONE_URI + + request = get_current_request() + return get_clone_url(request=request, + uri_tmpl=uri_tmpl, + repo_name=self.repo_name, + repo_id=self.repo_id, **override) + + def set_state(self, state): + self.repo_state = state + Session().add(self) + #========================================================================== + # SCM PROPERTIES + #========================================================================== + + def get_commit(self, commit_id=None, commit_idx=None, pre_load=None): + return get_commit_safe( + self.scm_instance(), commit_id, commit_idx, pre_load=pre_load) + + def get_changeset(self, rev=None, pre_load=None): + warnings.warn("Use get_commit", DeprecationWarning) + commit_id = None + commit_idx = None + if isinstance(rev, compat.string_types): + commit_id = rev + else: + commit_idx = rev + return self.get_commit(commit_id=commit_id, commit_idx=commit_idx, + pre_load=pre_load) + + def get_landing_commit(self): + """ + Returns landing commit, or if that doesn't exist returns the tip + """ + _rev_type, _rev = self.landing_rev + commit = self.get_commit(_rev) + if isinstance(commit, EmptyCommit): + return self.get_commit() + return commit + + def update_commit_cache(self, cs_cache=None, config=None): + """ + Update cache of last changeset for repository, keys should be:: + + short_id + raw_id + revision + parents + message + date + author + + :param cs_cache: + """ + from rhodecode.lib.vcs.backends.base import BaseChangeset + if cs_cache is None: + # use no-cache version here + scm_repo = self.scm_instance(cache=False, config=config) + + empty = not scm_repo or scm_repo.is_empty() + if not empty: + cs_cache = scm_repo.get_commit( + pre_load=["author", "date", "message", "parents"]) + else: + cs_cache = EmptyCommit() + + if isinstance(cs_cache, BaseChangeset): + cs_cache = cs_cache.__json__() + + def is_outdated(new_cs_cache): + if (new_cs_cache['raw_id'] != self.changeset_cache['raw_id'] or + new_cs_cache['revision'] != self.changeset_cache['revision']): + return True + return False + + # check if we have maybe already latest cached revision + if is_outdated(cs_cache) or not self.changeset_cache: + _default = datetime.datetime.utcnow() + last_change = cs_cache.get('date') or _default + if self.updated_on and self.updated_on > last_change: + # we check if last update is newer than the new value + # if yes, we use the current timestamp instead. Imagine you get + # old commit pushed 1y ago, we'd set last update 1y to ago. + last_change = _default + log.debug('updated repo %s with new cs cache %s', + self.repo_name, cs_cache) + self.updated_on = last_change + self.changeset_cache = cs_cache + Session().add(self) + Session().commit() + else: + log.debug('Skipping update_commit_cache for repo:`%s` ' + 'commit already with latest changes', self.repo_name) + + @property + def tip(self): + return self.get_commit('tip') + + @property + def author(self): + return self.tip.author + + @property + def last_change(self): + return self.scm_instance().last_change + + def get_comments(self, revisions=None): + """ + Returns comments for this repository grouped by revisions + + :param revisions: filter query by revisions only + """ + cmts = ChangesetComment.query()\ + .filter(ChangesetComment.repo == self) + if revisions: + cmts = cmts.filter(ChangesetComment.revision.in_(revisions)) + grouped = collections.defaultdict(list) + for cmt in cmts.all(): + grouped[cmt.revision].append(cmt) + return grouped + + def statuses(self, revisions=None): + """ + Returns statuses for this repository + + :param revisions: list of revisions to get statuses for + """ + statuses = ChangesetStatus.query()\ + .filter(ChangesetStatus.repo == self)\ + .filter(ChangesetStatus.version == 0) + + if revisions: + # Try doing the filtering in chunks to avoid hitting limits + size = 500 + status_results = [] + for chunk in xrange(0, len(revisions), size): + status_results += statuses.filter( + ChangesetStatus.revision.in_( + revisions[chunk: chunk+size]) + ).all() + else: + status_results = statuses.all() + + grouped = {} + + # maybe we have open new pullrequest without a status? + stat = ChangesetStatus.STATUS_UNDER_REVIEW + status_lbl = ChangesetStatus.get_status_lbl(stat) + for pr in PullRequest.query().filter(PullRequest.source_repo == self).all(): + for rev in pr.revisions: + pr_id = pr.pull_request_id + pr_repo = pr.target_repo.repo_name + grouped[rev] = [stat, status_lbl, pr_id, pr_repo] + + for stat in status_results: + pr_id = pr_repo = None + if stat.pull_request: + pr_id = stat.pull_request.pull_request_id + pr_repo = stat.pull_request.target_repo.repo_name + grouped[stat.revision] = [str(stat.status), stat.status_lbl, + pr_id, pr_repo] + return grouped + + # ========================================================================== + # SCM CACHE INSTANCE + # ========================================================================== + + def scm_instance(self, **kwargs): + import rhodecode + + # Passing a config will not hit the cache currently only used + # for repo2dbmapper + config = kwargs.pop('config', None) + cache = kwargs.pop('cache', None) + full_cache = str2bool(rhodecode.CONFIG.get('vcs_full_cache')) + # if cache is NOT defined use default global, else we have a full + # control over cache behaviour + if cache is None and full_cache and not config: + return self._get_instance_cached() + return self._get_instance(cache=bool(cache), config=config) + + def _get_instance_cached(self): + from rhodecode.lib import rc_cache + + cache_namespace_uid = 'cache_repo_instance.{}'.format(self.repo_id) + invalidation_namespace = CacheKey.REPO_INVALIDATION_NAMESPACE.format( + repo_id=self.repo_id) + region = rc_cache.get_or_create_region('cache_repo_longterm', cache_namespace_uid) + + @region.conditional_cache_on_arguments(namespace=cache_namespace_uid) + def get_instance_cached(repo_id, context_id): + return self._get_instance() + + # we must use thread scoped cache here, + # because each thread of gevent needs it's own not shared connection and cache + # we also alter `args` so the cache key is individual for every green thread. + inv_context_manager = rc_cache.InvalidationContext( + uid=cache_namespace_uid, invalidation_namespace=invalidation_namespace, + thread_scoped=True) + with inv_context_manager as invalidation_context: + args = (self.repo_id, inv_context_manager.cache_key) + # re-compute and store cache if we get invalidate signal + if invalidation_context.should_invalidate(): + instance = get_instance_cached.refresh(*args) + else: + instance = get_instance_cached(*args) + + log.debug( + 'Repo instance fetched in %.3fs', inv_context_manager.compute_time) + return instance + + def _get_instance(self, cache=True, config=None): + config = config or self._config + custom_wire = { + 'cache': cache # controls the vcs.remote cache + } + repo = get_vcs_instance( + repo_path=safe_str(self.repo_full_path), + config=config, + with_wire=custom_wire, + create=False, + _vcs_alias=self.repo_type) + + return repo + + def __json__(self): + return {'landing_rev': self.landing_rev} + + def get_dict(self): + + # Since we transformed `repo_name` to a hybrid property, we need to + # keep compatibility with the code which uses `repo_name` field. + + result = super(Repository, self).get_dict() + result['repo_name'] = result.pop('_repo_name', None) + return result + + +class RepoGroup(Base, BaseModel): + __tablename__ = 'groups' + __table_args__ = ( + UniqueConstraint('group_name', 'group_parent_id'), + base_table_args, + ) + __mapper_args__ = {'order_by': 'group_name'} + + CHOICES_SEPARATOR = '/' # used to generate select2 choices for nested groups + + group_id = Column("group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + group_name = Column("group_name", String(255), nullable=False, unique=True, default=None) + group_parent_id = Column("group_parent_id", Integer(), ForeignKey('groups.group_id'), nullable=True, unique=None, default=None) + group_description = Column("group_description", String(10000), nullable=True, unique=None, default=None) + enable_locking = Column("enable_locking", Boolean(), nullable=False, unique=None, default=False) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=False, default=None) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + updated_on = Column('updated_on', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now) + personal = Column('personal', Boolean(), nullable=True, unique=None, default=None) + + repo_group_to_perm = relationship('UserRepoGroupToPerm', cascade='all', order_by='UserRepoGroupToPerm.group_to_perm_id') + users_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all') + parent_group = relationship('RepoGroup', remote_side=group_id) + user = relationship('User') + integrations = relationship('Integration', + cascade="all, delete, delete-orphan") + + def __init__(self, group_name='', parent_group=None): + self.group_name = group_name + self.parent_group = parent_group + + def __unicode__(self): + return u"<%s('id:%s:%s')>" % ( + self.__class__.__name__, self.group_id, self.group_name) + + @hybrid_property + def description_safe(self): + from rhodecode.lib import helpers as h + return h.escape(self.group_description) + + @classmethod + def _generate_choice(cls, repo_group): + from webhelpers.html import literal as _literal + _name = lambda k: _literal(cls.CHOICES_SEPARATOR.join(k)) + return repo_group.group_id, _name(repo_group.full_path_splitted) + + @classmethod + def groups_choices(cls, groups=None, show_empty_group=True): + if not groups: + groups = cls.query().all() + + repo_groups = [] + if show_empty_group: + repo_groups = [(-1, u'-- %s --' % _('No parent'))] + + repo_groups.extend([cls._generate_choice(x) for x in groups]) + + repo_groups = sorted( + repo_groups, key=lambda t: t[1].split(cls.CHOICES_SEPARATOR)[0]) + return repo_groups + + @classmethod + def url_sep(cls): + return URL_SEP + + @classmethod + def get_by_group_name(cls, group_name, cache=False, case_insensitive=False): + if case_insensitive: + gr = cls.query().filter(func.lower(cls.group_name) + == func.lower(group_name)) + else: + gr = cls.query().filter(cls.group_name == group_name) + if cache: + name_key = _hash_key(group_name) + gr = gr.options( + FromCache("sql_cache_short", "get_group_%s" % name_key)) + return gr.scalar() + + @classmethod + def get_user_personal_repo_group(cls, user_id): + user = User.get(user_id) + if user.username == User.DEFAULT_USER: + return None + + return cls.query()\ + .filter(cls.personal == true()) \ + .filter(cls.user == user) \ + .order_by(cls.group_id.asc()) \ + .first() + + @classmethod + def get_all_repo_groups(cls, user_id=Optional(None), group_id=Optional(None), + case_insensitive=True): + q = RepoGroup.query() + + if not isinstance(user_id, Optional): + q = q.filter(RepoGroup.user_id == user_id) + + if not isinstance(group_id, Optional): + q = q.filter(RepoGroup.group_parent_id == group_id) + + if case_insensitive: + q = q.order_by(func.lower(RepoGroup.group_name)) + else: + q = q.order_by(RepoGroup.group_name) + return q.all() + + @property + def parents(self): + parents_recursion_limit = 10 + groups = [] + if self.parent_group is None: + return groups + cur_gr = self.parent_group + groups.insert(0, cur_gr) + cnt = 0 + while 1: + cnt += 1 + gr = getattr(cur_gr, 'parent_group', None) + cur_gr = cur_gr.parent_group + if gr is None: + break + if cnt == parents_recursion_limit: + # this will prevent accidental infinit loops + log.error('more than %s parents found for group %s, stopping ' + 'recursive parent fetching', parents_recursion_limit, self) + break + + groups.insert(0, gr) + return groups + + @property + def last_db_change(self): + return self.updated_on + + @property + def children(self): + return RepoGroup.query().filter(RepoGroup.parent_group == self) + + @property + def name(self): + return self.group_name.split(RepoGroup.url_sep())[-1] + + @property + def full_path(self): + return self.group_name + + @property + def full_path_splitted(self): + return self.group_name.split(RepoGroup.url_sep()) + + @property + def repositories(self): + return Repository.query()\ + .filter(Repository.group == self)\ + .order_by(Repository.repo_name) + + @property + def repositories_recursive_count(self): + cnt = self.repositories.count() + + def children_count(group): + cnt = 0 + for child in group.children: + cnt += child.repositories.count() + cnt += children_count(child) + return cnt + + return cnt + children_count(self) + + def _recursive_objects(self, include_repos=True): + all_ = [] + + def _get_members(root_gr): + if include_repos: + for r in root_gr.repositories: + all_.append(r) + childs = root_gr.children.all() + if childs: + for gr in childs: + all_.append(gr) + _get_members(gr) + + _get_members(self) + return [self] + all_ + + def recursive_groups_and_repos(self): + """ + Recursive return all groups, with repositories in those groups + """ + return self._recursive_objects() + + def recursive_groups(self): + """ + Returns all children groups for this group including children of children + """ + return self._recursive_objects(include_repos=False) + + def get_new_name(self, group_name): + """ + returns new full group name based on parent and new name + + :param group_name: + """ + path_prefix = (self.parent_group.full_path_splitted if + self.parent_group else []) + return RepoGroup.url_sep().join(path_prefix + [group_name]) + + def permissions(self, with_admins=True, with_owner=True, + expand_from_user_groups=False): + """ + Permissions for repository groups + """ + _admin_perm = 'group.admin' + + owner_row = [] + if with_owner: + usr = AttributeDict(self.user.get_dict()) + usr.owner_row = True + usr.permission = _admin_perm + owner_row.append(usr) + + super_admin_ids = [] + super_admin_rows = [] + if with_admins: + for usr in User.get_all_super_admins(): + super_admin_ids.append(usr.user_id) + # if this admin is also owner, don't double the record + if usr.user_id == owner_row[0].user_id: + owner_row[0].admin_row = True + else: + usr = AttributeDict(usr.get_dict()) + usr.admin_row = True + usr.permission = _admin_perm + super_admin_rows.append(usr) + + q = UserRepoGroupToPerm.query().filter(UserRepoGroupToPerm.group == self) + q = q.options(joinedload(UserRepoGroupToPerm.group), + joinedload(UserRepoGroupToPerm.user), + joinedload(UserRepoGroupToPerm.permission),) + + # get owners and admins and permissions. We do a trick of re-writing + # objects from sqlalchemy to named-tuples due to sqlalchemy session + # has a global reference and changing one object propagates to all + # others. This means if admin is also an owner admin_row that change + # would propagate to both objects + perm_rows = [] + for _usr in q.all(): + usr = AttributeDict(_usr.user.get_dict()) + # if this user is also owner/admin, mark as duplicate record + if usr.user_id == owner_row[0].user_id or usr.user_id in super_admin_ids: + usr.duplicate_perm = True + usr.permission = _usr.permission.permission_name + perm_rows.append(usr) + + # filter the perm rows by 'default' first and then sort them by + # admin,write,read,none permissions sorted again alphabetically in + # each group + perm_rows = sorted(perm_rows, key=display_user_sort) + + user_groups_rows = [] + if expand_from_user_groups: + for ug in self.permission_user_groups(with_members=True): + for user_data in ug.members: + user_groups_rows.append(user_data) + + return super_admin_rows + owner_row + perm_rows + user_groups_rows + + def permission_user_groups(self, with_members=False): + q = UserGroupRepoGroupToPerm.query()\ + .filter(UserGroupRepoGroupToPerm.group == self) + q = q.options(joinedload(UserGroupRepoGroupToPerm.group), + joinedload(UserGroupRepoGroupToPerm.users_group), + joinedload(UserGroupRepoGroupToPerm.permission),) + + perm_rows = [] + for _user_group in q.all(): + entry = AttributeDict(_user_group.users_group.get_dict()) + entry.permission = _user_group.permission.permission_name + if with_members: + entry.members = [x.user.get_dict() + for x in _user_group.users_group.members] + perm_rows.append(entry) + + perm_rows = sorted(perm_rows, key=display_user_group_sort) + return perm_rows + + def get_api_data(self): + """ + Common function for generating api data + + """ + group = self + data = { + 'group_id': group.group_id, + 'group_name': group.group_name, + 'group_description': group.description_safe, + 'parent_group': group.parent_group.group_name if group.parent_group else None, + 'repositories': [x.repo_name for x in group.repositories], + 'owner': group.user.username, + } + return data + + +class Permission(Base, BaseModel): + __tablename__ = 'permissions' + __table_args__ = ( + Index('p_perm_name_idx', 'permission_name'), + base_table_args, + ) + + PERMS = [ + ('hg.admin', _('RhodeCode Super Administrator')), + + ('repository.none', _('Repository no access')), + ('repository.read', _('Repository read access')), + ('repository.write', _('Repository write access')), + ('repository.admin', _('Repository admin access')), + + ('group.none', _('Repository group no access')), + ('group.read', _('Repository group read access')), + ('group.write', _('Repository group write access')), + ('group.admin', _('Repository group admin access')), + + ('usergroup.none', _('User group no access')), + ('usergroup.read', _('User group read access')), + ('usergroup.write', _('User group write access')), + ('usergroup.admin', _('User group admin access')), + + ('branch.none', _('Branch no permissions')), + ('branch.merge', _('Branch access by web merge')), + ('branch.push', _('Branch access by push')), + ('branch.push_force', _('Branch access by push with force')), + + ('hg.repogroup.create.false', _('Repository Group creation disabled')), + ('hg.repogroup.create.true', _('Repository Group creation enabled')), + + ('hg.usergroup.create.false', _('User Group creation disabled')), + ('hg.usergroup.create.true', _('User Group creation enabled')), + + ('hg.create.none', _('Repository creation disabled')), + ('hg.create.repository', _('Repository creation enabled')), + ('hg.create.write_on_repogroup.true', _('Repository creation enabled with write permission to a repository group')), + ('hg.create.write_on_repogroup.false', _('Repository creation disabled with write permission to a repository group')), + + ('hg.fork.none', _('Repository forking disabled')), + ('hg.fork.repository', _('Repository forking enabled')), + + ('hg.register.none', _('Registration disabled')), + ('hg.register.manual_activate', _('User Registration with manual account activation')), + ('hg.register.auto_activate', _('User Registration with automatic account activation')), + + ('hg.password_reset.enabled', _('Password reset enabled')), + ('hg.password_reset.hidden', _('Password reset hidden')), + ('hg.password_reset.disabled', _('Password reset disabled')), + + ('hg.extern_activate.manual', _('Manual activation of external account')), + ('hg.extern_activate.auto', _('Automatic activation of external account')), + + ('hg.inherit_default_perms.false', _('Inherit object permissions from default user disabled')), + ('hg.inherit_default_perms.true', _('Inherit object permissions from default user enabled')), + ] + + # definition of system default permissions for DEFAULT user, created on + # system setup + DEFAULT_USER_PERMISSIONS = [ + # object perms + 'repository.read', + 'group.read', + 'usergroup.read', + # branch, for backward compat we need same value as before so forced pushed + 'branch.push_force', + # global + 'hg.create.repository', + 'hg.repogroup.create.false', + 'hg.usergroup.create.false', + 'hg.create.write_on_repogroup.true', + 'hg.fork.repository', + 'hg.register.manual_activate', + 'hg.password_reset.enabled', + 'hg.extern_activate.auto', + 'hg.inherit_default_perms.true', + ] + + # defines which permissions are more important higher the more important + # Weight defines which permissions are more important. + # The higher number the more important. + PERM_WEIGHTS = { + 'repository.none': 0, + 'repository.read': 1, + 'repository.write': 3, + 'repository.admin': 4, + + 'group.none': 0, + 'group.read': 1, + 'group.write': 3, + 'group.admin': 4, + + 'usergroup.none': 0, + 'usergroup.read': 1, + 'usergroup.write': 3, + 'usergroup.admin': 4, + + 'branch.none': 0, + 'branch.merge': 1, + 'branch.push': 3, + 'branch.push_force': 4, + + 'hg.repogroup.create.false': 0, + 'hg.repogroup.create.true': 1, + + 'hg.usergroup.create.false': 0, + 'hg.usergroup.create.true': 1, + + 'hg.fork.none': 0, + 'hg.fork.repository': 1, + 'hg.create.none': 0, + 'hg.create.repository': 1 + } + + permission_id = Column("permission_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + permission_name = Column("permission_name", String(255), nullable=True, unique=None, default=None) + permission_longname = Column("permission_longname", String(255), nullable=True, unique=None, default=None) + + def __unicode__(self): + return u"<%s('%s:%s')>" % ( + self.__class__.__name__, self.permission_id, self.permission_name + ) + + @classmethod + def get_by_key(cls, key): + return cls.query().filter(cls.permission_name == key).scalar() + + @classmethod + def get_default_repo_perms(cls, user_id, repo_id=None): + q = Session().query(UserRepoToPerm, Repository, Permission)\ + .join((Permission, UserRepoToPerm.permission_id == Permission.permission_id))\ + .join((Repository, UserRepoToPerm.repository_id == Repository.repo_id))\ + .filter(UserRepoToPerm.user_id == user_id) + if repo_id: + q = q.filter(UserRepoToPerm.repository_id == repo_id) + return q.all() + + @classmethod + def get_default_repo_branch_perms(cls, user_id, repo_id=None): + q = Session().query(UserToRepoBranchPermission, UserRepoToPerm, Permission) \ + .join( + Permission, + UserToRepoBranchPermission.permission_id == Permission.permission_id) \ + .join( + UserRepoToPerm, + UserToRepoBranchPermission.rule_to_perm_id == UserRepoToPerm.repo_to_perm_id) \ + .filter(UserRepoToPerm.user_id == user_id) + + if repo_id: + q = q.filter(UserToRepoBranchPermission.repository_id == repo_id) + return q.order_by(UserToRepoBranchPermission.rule_order).all() + + @classmethod + def get_default_repo_perms_from_user_group(cls, user_id, repo_id=None): + q = Session().query(UserGroupRepoToPerm, Repository, Permission)\ + .join( + Permission, + UserGroupRepoToPerm.permission_id == Permission.permission_id)\ + .join( + Repository, + UserGroupRepoToPerm.repository_id == Repository.repo_id)\ + .join( + UserGroup, + UserGroupRepoToPerm.users_group_id == + UserGroup.users_group_id)\ + .join( + UserGroupMember, + UserGroupRepoToPerm.users_group_id == + UserGroupMember.users_group_id)\ + .filter( + UserGroupMember.user_id == user_id, + UserGroup.users_group_active == true()) + if repo_id: + q = q.filter(UserGroupRepoToPerm.repository_id == repo_id) + return q.all() + + @classmethod + def get_default_repo_branch_perms_from_user_group(cls, user_id, repo_id=None): + q = Session().query(UserGroupToRepoBranchPermission, UserGroupRepoToPerm, Permission) \ + .join( + Permission, + UserGroupToRepoBranchPermission.permission_id == Permission.permission_id) \ + .join( + UserGroupRepoToPerm, + UserGroupToRepoBranchPermission.rule_to_perm_id == UserGroupRepoToPerm.users_group_to_perm_id) \ + .join( + UserGroup, + UserGroupRepoToPerm.users_group_id == UserGroup.users_group_id) \ + .join( + UserGroupMember, + UserGroupRepoToPerm.users_group_id == UserGroupMember.users_group_id) \ + .filter( + UserGroupMember.user_id == user_id, + UserGroup.users_group_active == true()) + + if repo_id: + q = q.filter(UserGroupToRepoBranchPermission.repository_id == repo_id) + return q.order_by(UserGroupToRepoBranchPermission.rule_order).all() + + @classmethod + def get_default_group_perms(cls, user_id, repo_group_id=None): + q = Session().query(UserRepoGroupToPerm, RepoGroup, Permission)\ + .join( + Permission, + UserRepoGroupToPerm.permission_id == Permission.permission_id)\ + .join( + RepoGroup, + UserRepoGroupToPerm.group_id == RepoGroup.group_id)\ + .filter(UserRepoGroupToPerm.user_id == user_id) + if repo_group_id: + q = q.filter(UserRepoGroupToPerm.group_id == repo_group_id) + return q.all() + + @classmethod + def get_default_group_perms_from_user_group( + cls, user_id, repo_group_id=None): + q = Session().query(UserGroupRepoGroupToPerm, RepoGroup, Permission)\ + .join( + Permission, + UserGroupRepoGroupToPerm.permission_id == + Permission.permission_id)\ + .join( + RepoGroup, + UserGroupRepoGroupToPerm.group_id == RepoGroup.group_id)\ + .join( + UserGroup, + UserGroupRepoGroupToPerm.users_group_id == + UserGroup.users_group_id)\ + .join( + UserGroupMember, + UserGroupRepoGroupToPerm.users_group_id == + UserGroupMember.users_group_id)\ + .filter( + UserGroupMember.user_id == user_id, + UserGroup.users_group_active == true()) + if repo_group_id: + q = q.filter(UserGroupRepoGroupToPerm.group_id == repo_group_id) + return q.all() + + @classmethod + def get_default_user_group_perms(cls, user_id, user_group_id=None): + q = Session().query(UserUserGroupToPerm, UserGroup, Permission)\ + .join((Permission, UserUserGroupToPerm.permission_id == Permission.permission_id))\ + .join((UserGroup, UserUserGroupToPerm.user_group_id == UserGroup.users_group_id))\ + .filter(UserUserGroupToPerm.user_id == user_id) + if user_group_id: + q = q.filter(UserUserGroupToPerm.user_group_id == user_group_id) + return q.all() + + @classmethod + def get_default_user_group_perms_from_user_group( + cls, user_id, user_group_id=None): + TargetUserGroup = aliased(UserGroup, name='target_user_group') + q = Session().query(UserGroupUserGroupToPerm, UserGroup, Permission)\ + .join( + Permission, + UserGroupUserGroupToPerm.permission_id == + Permission.permission_id)\ + .join( + TargetUserGroup, + UserGroupUserGroupToPerm.target_user_group_id == + TargetUserGroup.users_group_id)\ + .join( + UserGroup, + UserGroupUserGroupToPerm.user_group_id == + UserGroup.users_group_id)\ + .join( + UserGroupMember, + UserGroupUserGroupToPerm.user_group_id == + UserGroupMember.users_group_id)\ + .filter( + UserGroupMember.user_id == user_id, + UserGroup.users_group_active == true()) + if user_group_id: + q = q.filter( + UserGroupUserGroupToPerm.user_group_id == user_group_id) + + return q.all() + + +class UserRepoToPerm(Base, BaseModel): + __tablename__ = 'repo_to_perm' + __table_args__ = ( + UniqueConstraint('user_id', 'repository_id', 'permission_id'), + base_table_args + ) + + repo_to_perm_id = Column("repo_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None) + + user = relationship('User') + repository = relationship('Repository') + permission = relationship('Permission') + + branch_perm_entry = relationship('UserToRepoBranchPermission', cascade="all, delete, delete-orphan", lazy='joined') + + @classmethod + def create(cls, user, repository, permission): + n = cls() + n.user = user + n.repository = repository + n.permission = permission + Session().add(n) + return n + + def __unicode__(self): + return u'<%s => %s >' % (self.user, self.repository) + + +class UserUserGroupToPerm(Base, BaseModel): + __tablename__ = 'user_user_group_to_perm' + __table_args__ = ( + UniqueConstraint('user_id', 'user_group_id', 'permission_id'), + base_table_args + ) + + user_user_group_to_perm_id = Column("user_user_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + user_group_id = Column("user_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) + + user = relationship('User') + user_group = relationship('UserGroup') + permission = relationship('Permission') + + @classmethod + def create(cls, user, user_group, permission): + n = cls() + n.user = user + n.user_group = user_group + n.permission = permission + Session().add(n) + return n + + def __unicode__(self): + return u'<%s => %s >' % (self.user, self.user_group) + + +class UserToPerm(Base, BaseModel): + __tablename__ = 'user_to_perm' + __table_args__ = ( + UniqueConstraint('user_id', 'permission_id'), + base_table_args + ) + + user_to_perm_id = Column("user_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + + user = relationship('User') + permission = relationship('Permission', lazy='joined') + + def __unicode__(self): + return u'<%s => %s >' % (self.user, self.permission) + + +class UserGroupRepoToPerm(Base, BaseModel): + __tablename__ = 'users_group_repo_to_perm' + __table_args__ = ( + UniqueConstraint('repository_id', 'users_group_id', 'permission_id'), + base_table_args + ) + + users_group_to_perm_id = Column("users_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None) + + users_group = relationship('UserGroup') + permission = relationship('Permission') + repository = relationship('Repository') + user_group_branch_perms = relationship('UserGroupToRepoBranchPermission', cascade='all') + + @classmethod + def create(cls, users_group, repository, permission): + n = cls() + n.users_group = users_group + n.repository = repository + n.permission = permission + Session().add(n) + return n + + def __unicode__(self): + return u' %s >' % (self.users_group, self.repository) + + +class UserGroupUserGroupToPerm(Base, BaseModel): + __tablename__ = 'user_group_user_group_to_perm' + __table_args__ = ( + UniqueConstraint('target_user_group_id', 'user_group_id', 'permission_id'), + CheckConstraint('target_user_group_id != user_group_id'), + base_table_args + ) + + user_group_user_group_to_perm_id = Column("user_group_user_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + target_user_group_id = Column("target_user_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + user_group_id = Column("user_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) + + target_user_group = relationship('UserGroup', primaryjoin='UserGroupUserGroupToPerm.target_user_group_id==UserGroup.users_group_id') + user_group = relationship('UserGroup', primaryjoin='UserGroupUserGroupToPerm.user_group_id==UserGroup.users_group_id') + permission = relationship('Permission') + + @classmethod + def create(cls, target_user_group, user_group, permission): + n = cls() + n.target_user_group = target_user_group + n.user_group = user_group + n.permission = permission + Session().add(n) + return n + + def __unicode__(self): + return u' %s >' % (self.target_user_group, self.user_group) + + +class UserGroupToPerm(Base, BaseModel): + __tablename__ = 'users_group_to_perm' + __table_args__ = ( + UniqueConstraint('users_group_id', 'permission_id',), + base_table_args + ) + + users_group_to_perm_id = Column("users_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + + users_group = relationship('UserGroup') + permission = relationship('Permission') + + +class UserRepoGroupToPerm(Base, BaseModel): + __tablename__ = 'user_repo_group_to_perm' + __table_args__ = ( + UniqueConstraint('user_id', 'group_id', 'permission_id'), + base_table_args + ) + + group_to_perm_id = Column("group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) + group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + + user = relationship('User') + group = relationship('RepoGroup') + permission = relationship('Permission') + + @classmethod + def create(cls, user, repository_group, permission): + n = cls() + n.user = user + n.group = repository_group + n.permission = permission + Session().add(n) + return n + + +class UserGroupRepoGroupToPerm(Base, BaseModel): + __tablename__ = 'users_group_repo_group_to_perm' + __table_args__ = ( + UniqueConstraint('users_group_id', 'group_id'), + base_table_args + ) + + users_group_repo_group_to_perm_id = Column("users_group_repo_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) + group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=False, unique=None, default=None) + permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + + users_group = relationship('UserGroup') + permission = relationship('Permission') + group = relationship('RepoGroup') + + @classmethod + def create(cls, user_group, repository_group, permission): + n = cls() + n.users_group = user_group + n.group = repository_group + n.permission = permission + Session().add(n) + return n + + def __unicode__(self): + return u' %s >' % (self.users_group, self.group) + + +class Statistics(Base, BaseModel): + __tablename__ = 'statistics' + __table_args__ = ( + base_table_args + ) + + stat_id = Column("stat_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=True, default=None) + stat_on_revision = Column("stat_on_revision", Integer(), nullable=False) + commit_activity = Column("commit_activity", LargeBinary(1000000), nullable=False)#JSON data + commit_activity_combined = Column("commit_activity_combined", LargeBinary(), nullable=False)#JSON data + languages = Column("languages", LargeBinary(1000000), nullable=False)#JSON data + + repository = relationship('Repository', single_parent=True) + + +class UserFollowing(Base, BaseModel): + __tablename__ = 'user_followings' + __table_args__ = ( + UniqueConstraint('user_id', 'follows_repository_id'), + UniqueConstraint('user_id', 'follows_user_id'), + base_table_args + ) + + user_following_id = Column("user_following_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) + follows_repo_id = Column("follows_repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=True, unique=None, default=None) + follows_user_id = Column("follows_user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None) + follows_from = Column('follows_from', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now) + + user = relationship('User', primaryjoin='User.user_id==UserFollowing.user_id') + + follows_user = relationship('User', primaryjoin='User.user_id==UserFollowing.follows_user_id') + follows_repository = relationship('Repository', order_by='Repository.repo_name') + + @classmethod + def get_repo_followers(cls, repo_id): + return cls.query().filter(cls.follows_repo_id == repo_id) + + +class CacheKey(Base, BaseModel): + __tablename__ = 'cache_invalidation' + __table_args__ = ( + UniqueConstraint('cache_key'), + Index('key_idx', 'cache_key'), + base_table_args, + ) + + CACHE_TYPE_FEED = 'FEED' + CACHE_TYPE_README = 'README' + # namespaces used to register process/thread aware caches + REPO_INVALIDATION_NAMESPACE = 'repo_cache:{repo_id}' + SETTINGS_INVALIDATION_NAMESPACE = 'system_settings' + + cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + cache_key = Column("cache_key", String(255), nullable=True, unique=None, default=None) + cache_args = Column("cache_args", String(255), nullable=True, unique=None, default=None) + cache_active = Column("cache_active", Boolean(), nullable=True, unique=None, default=False) + + def __init__(self, cache_key, cache_args=''): + self.cache_key = cache_key + self.cache_args = cache_args + self.cache_active = False + + def __unicode__(self): + return u"<%s('%s:%s[%s]')>" % ( + self.__class__.__name__, + self.cache_id, self.cache_key, self.cache_active) + + def _cache_key_partition(self): + prefix, repo_name, suffix = self.cache_key.partition(self.cache_args) + return prefix, repo_name, suffix + + def get_prefix(self): + """ + Try to extract prefix from existing cache key. The key could consist + of prefix, repo_name, suffix + """ + # this returns prefix, repo_name, suffix + return self._cache_key_partition()[0] + + def get_suffix(self): + """ + get suffix that might have been used in _get_cache_key to + generate self.cache_key. Only used for informational purposes + in repo_edit.mako. + """ + # prefix, repo_name, suffix + return self._cache_key_partition()[2] + + @classmethod + def delete_all_cache(cls): + """ + Delete all cache keys from database. + Should only be run when all instances are down and all entries + thus stale. + """ + cls.query().delete() + Session().commit() + + @classmethod + def set_invalidate(cls, cache_uid, delete=False): + """ + Mark all caches of a repo as invalid in the database. + """ + + try: + qry = Session().query(cls).filter(cls.cache_args == cache_uid) + if delete: + qry.delete() + log.debug('cache objects deleted for cache args %s', + safe_str(cache_uid)) + else: + qry.update({"cache_active": False}) + log.debug('cache objects marked as invalid for cache args %s', + safe_str(cache_uid)) + + Session().commit() + except Exception: + log.exception( + 'Cache key invalidation failed for cache args %s', + safe_str(cache_uid)) + Session().rollback() + + @classmethod + def get_active_cache(cls, cache_key): + inv_obj = cls.query().filter(cls.cache_key == cache_key).scalar() + if inv_obj: + return inv_obj + return None + + +class ChangesetComment(Base, BaseModel): + __tablename__ = 'changeset_comments' + __table_args__ = ( + Index('cc_revision_idx', 'revision'), + base_table_args, + ) + + COMMENT_OUTDATED = u'comment_outdated' + COMMENT_TYPE_NOTE = u'note' + COMMENT_TYPE_TODO = u'todo' + COMMENT_TYPES = [COMMENT_TYPE_NOTE, COMMENT_TYPE_TODO] + + comment_id = Column('comment_id', Integer(), nullable=False, primary_key=True) + repo_id = Column('repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False) + revision = Column('revision', String(40), nullable=True) + pull_request_id = Column("pull_request_id", Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=True) + pull_request_version_id = Column("pull_request_version_id", Integer(), ForeignKey('pull_request_versions.pull_request_version_id'), nullable=True) + line_no = Column('line_no', Unicode(10), nullable=True) + hl_lines = Column('hl_lines', Unicode(512), nullable=True) + f_path = Column('f_path', Unicode(1000), nullable=True) + user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False) + text = Column('text', UnicodeText().with_variant(UnicodeText(25000), 'mysql'), nullable=False) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + modified_at = Column('modified_at', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + renderer = Column('renderer', Unicode(64), nullable=True) + display_state = Column('display_state', Unicode(128), nullable=True) + + comment_type = Column('comment_type', Unicode(128), nullable=True, default=COMMENT_TYPE_NOTE) + resolved_comment_id = Column('resolved_comment_id', Integer(), ForeignKey('changeset_comments.comment_id'), nullable=True) + + resolved_comment = relationship('ChangesetComment', remote_side=comment_id, back_populates='resolved_by') + resolved_by = relationship('ChangesetComment', back_populates='resolved_comment') + + author = relationship('User', lazy='joined') + repo = relationship('Repository') + status_change = relationship('ChangesetStatus', cascade="all, delete, delete-orphan", lazy='joined') + pull_request = relationship('PullRequest', lazy='joined') + pull_request_version = relationship('PullRequestVersion') + + @classmethod + def get_users(cls, revision=None, pull_request_id=None): + """ + Returns user associated with this ChangesetComment. ie those + who actually commented + + :param cls: + :param revision: + """ + q = Session().query(User)\ + .join(ChangesetComment.author) + if revision: + q = q.filter(cls.revision == revision) + elif pull_request_id: + q = q.filter(cls.pull_request_id == pull_request_id) + return q.all() + + @classmethod + def get_index_from_version(cls, pr_version, versions): + num_versions = [x.pull_request_version_id for x in versions] + try: + return num_versions.index(pr_version) +1 + except (IndexError, ValueError): + return + + @property + def outdated(self): + return self.display_state == self.COMMENT_OUTDATED + + def outdated_at_version(self, version): + """ + Checks if comment is outdated for given pull request version + """ + return self.outdated and self.pull_request_version_id != version + + def older_than_version(self, version): + """ + Checks if comment is made from previous version than given + """ + if version is None: + return self.pull_request_version_id is not None + + return self.pull_request_version_id < version + + @property + def resolved(self): + return self.resolved_by[0] if self.resolved_by else None + + @property + def is_todo(self): + return self.comment_type == self.COMMENT_TYPE_TODO + + @property + def is_inline(self): + return self.line_no and self.f_path + + def get_index_version(self, versions): + return self.get_index_from_version( + self.pull_request_version_id, versions) + + def __repr__(self): + if self.comment_id: + return '' % self.comment_id + else: + return '' % id(self) + + def get_api_data(self): + comment = self + data = { + 'comment_id': comment.comment_id, + 'comment_type': comment.comment_type, + 'comment_text': comment.text, + 'comment_status': comment.status_change, + 'comment_f_path': comment.f_path, + 'comment_lineno': comment.line_no, + 'comment_author': comment.author, + 'comment_created_on': comment.created_on, + 'comment_resolved_by': self.resolved + } + return data + + def __json__(self): + data = dict() + data.update(self.get_api_data()) + return data + + +class ChangesetStatus(Base, BaseModel): + __tablename__ = 'changeset_statuses' + __table_args__ = ( + Index('cs_revision_idx', 'revision'), + Index('cs_version_idx', 'version'), + UniqueConstraint('repo_id', 'revision', 'version'), + base_table_args + ) + + STATUS_NOT_REVIEWED = DEFAULT = 'not_reviewed' + STATUS_APPROVED = 'approved' + STATUS_REJECTED = 'rejected' + STATUS_UNDER_REVIEW = 'under_review' + + STATUSES = [ + (STATUS_NOT_REVIEWED, _("Not Reviewed")), # (no icon) and default + (STATUS_APPROVED, _("Approved")), + (STATUS_REJECTED, _("Rejected")), + (STATUS_UNDER_REVIEW, _("Under Review")), + ] + + changeset_status_id = Column('changeset_status_id', Integer(), nullable=False, primary_key=True) + repo_id = Column('repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None) + revision = Column('revision', String(40), nullable=False) + status = Column('status', String(128), nullable=False, default=DEFAULT) + changeset_comment_id = Column('changeset_comment_id', Integer(), ForeignKey('changeset_comments.comment_id')) + modified_at = Column('modified_at', DateTime(), nullable=False, default=datetime.datetime.now) + version = Column('version', Integer(), nullable=False, default=0) + pull_request_id = Column("pull_request_id", Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=True) + + author = relationship('User', lazy='joined') + repo = relationship('Repository') + comment = relationship('ChangesetComment', lazy='joined') + pull_request = relationship('PullRequest', lazy='joined') + + def __unicode__(self): + return u"<%s('%s[v%s]:%s')>" % ( + self.__class__.__name__, + self.status, self.version, self.author + ) + + @classmethod + def get_status_lbl(cls, value): + return dict(cls.STATUSES).get(value) + + @property + def status_lbl(self): + return ChangesetStatus.get_status_lbl(self.status) + + def get_api_data(self): + status = self + data = { + 'status_id': status.changeset_status_id, + 'status': status.status, + } + return data + + def __json__(self): + data = dict() + data.update(self.get_api_data()) + return data + + +class _SetState(object): + """ + Context processor allowing changing state for sensitive operation such as + pull request update or merge + """ + + def __init__(self, pull_request, pr_state, back_state=None): + self._pr = pull_request + self._org_state = back_state or pull_request.pull_request_state + self._pr_state = pr_state + + def __enter__(self): + log.debug('StateLock: entering set state context, setting state to: `%s`', + self._pr_state) + self._pr.pull_request_state = self._pr_state + Session().add(self._pr) + Session().commit() + + def __exit__(self, exc_type, exc_val, exc_tb): + log.debug('StateLock: exiting set state context, setting state to: `%s`', + self._org_state) + self._pr.pull_request_state = self._org_state + Session().add(self._pr) + Session().commit() + + +class _PullRequestBase(BaseModel): + """ + Common attributes of pull request and version entries. + """ + + # .status values + STATUS_NEW = u'new' + STATUS_OPEN = u'open' + STATUS_CLOSED = u'closed' + + # available states + STATE_CREATING = u'creating' + STATE_UPDATING = u'updating' + STATE_MERGING = u'merging' + STATE_CREATED = u'created' + + title = Column('title', Unicode(255), nullable=True) + description = Column( + 'description', UnicodeText().with_variant(UnicodeText(10240), 'mysql'), + nullable=True) + description_renderer = Column('description_renderer', Unicode(64), nullable=True) + + # new/open/closed status of pull request (not approve/reject/etc) + status = Column('status', Unicode(255), nullable=False, default=STATUS_NEW) + created_on = Column( + 'created_on', DateTime(timezone=False), nullable=False, + default=datetime.datetime.now) + updated_on = Column( + 'updated_on', DateTime(timezone=False), nullable=False, + default=datetime.datetime.now) + + pull_request_state = Column("pull_request_state", String(255), nullable=True) + + @declared_attr + def user_id(cls): + return Column( + "user_id", Integer(), ForeignKey('users.user_id'), nullable=False, + unique=None) + + # 500 revisions max + _revisions = Column( + 'revisions', UnicodeText().with_variant(UnicodeText(20500), 'mysql')) + + @declared_attr + def source_repo_id(cls): + # TODO: dan: rename column to source_repo_id + return Column( + 'org_repo_id', Integer(), ForeignKey('repositories.repo_id'), + nullable=False) + + _source_ref = Column('org_ref', Unicode(255), nullable=False) + + @hybrid_property + def source_ref(self): + return self._source_ref + + @source_ref.setter + def source_ref(self, val): + parts = (val or '').split(':') + if len(parts) != 3: + raise ValueError( + 'Invalid reference format given: {}, expected X:Y:Z'.format(val)) + self._source_ref = safe_unicode(val) + + _target_ref = Column('other_ref', Unicode(255), nullable=False) + + @hybrid_property + def target_ref(self): + return self._target_ref + + @target_ref.setter + def target_ref(self, val): + parts = (val or '').split(':') + if len(parts) != 3: + raise ValueError( + 'Invalid reference format given: {}, expected X:Y:Z'.format(val)) + self._target_ref = safe_unicode(val) + + @declared_attr + def target_repo_id(cls): + # TODO: dan: rename column to target_repo_id + return Column( + 'other_repo_id', Integer(), ForeignKey('repositories.repo_id'), + nullable=False) + + _shadow_merge_ref = Column('shadow_merge_ref', Unicode(255), nullable=True) + + # TODO: dan: rename column to last_merge_source_rev + _last_merge_source_rev = Column( + 'last_merge_org_rev', String(40), nullable=True) + # TODO: dan: rename column to last_merge_target_rev + _last_merge_target_rev = Column( + 'last_merge_other_rev', String(40), nullable=True) + _last_merge_status = Column('merge_status', Integer(), nullable=True) + merge_rev = Column('merge_rev', String(40), nullable=True) + + reviewer_data = Column( + 'reviewer_data_json', MutationObj.as_mutable( + JsonType(dialect_map=dict(mysql=UnicodeText(16384))))) + + @property + def reviewer_data_json(self): + return json.dumps(self.reviewer_data) + + @hybrid_property + def description_safe(self): + from rhodecode.lib import helpers as h + return h.escape(self.description) + + @hybrid_property + def revisions(self): + return self._revisions.split(':') if self._revisions else [] + + @revisions.setter + def revisions(self, val): + self._revisions = ':'.join(val) + + @hybrid_property + def last_merge_status(self): + return safe_int(self._last_merge_status) + + @last_merge_status.setter + def last_merge_status(self, val): + self._last_merge_status = val + + @declared_attr + def author(cls): + return relationship('User', lazy='joined') + + @declared_attr + def source_repo(cls): + return relationship( + 'Repository', + primaryjoin='%s.source_repo_id==Repository.repo_id' % cls.__name__) + + @property + def source_ref_parts(self): + return self.unicode_to_reference(self.source_ref) + + @declared_attr + def target_repo(cls): + return relationship( + 'Repository', + primaryjoin='%s.target_repo_id==Repository.repo_id' % cls.__name__) + + @property + def target_ref_parts(self): + return self.unicode_to_reference(self.target_ref) + + @property + def shadow_merge_ref(self): + return self.unicode_to_reference(self._shadow_merge_ref) + + @shadow_merge_ref.setter + def shadow_merge_ref(self, ref): + self._shadow_merge_ref = self.reference_to_unicode(ref) + + @staticmethod + def unicode_to_reference(raw): + """ + Convert a unicode (or string) to a reference object. + If unicode evaluates to False it returns None. + """ + if raw: + refs = raw.split(':') + return Reference(*refs) + else: + return None + + @staticmethod + def reference_to_unicode(ref): + """ + Convert a reference object to unicode. + If reference is None it returns None. + """ + if ref: + return u':'.join(ref) + else: + return None + + def get_api_data(self, with_merge_state=True): + from rhodecode.model.pull_request import PullRequestModel + + pull_request = self + if with_merge_state: + merge_status = PullRequestModel().merge_status(pull_request) + merge_state = { + 'status': merge_status[0], + 'message': safe_unicode(merge_status[1]), + } + else: + merge_state = {'status': 'not_available', + 'message': 'not_available'} + + merge_data = { + 'clone_url': PullRequestModel().get_shadow_clone_url(pull_request), + 'reference': ( + pull_request.shadow_merge_ref._asdict() + if pull_request.shadow_merge_ref else None), + } + + data = { + 'pull_request_id': pull_request.pull_request_id, + 'url': PullRequestModel().get_url(pull_request), + 'title': pull_request.title, + 'description': pull_request.description, + 'status': pull_request.status, + 'state': pull_request.pull_request_state, + 'created_on': pull_request.created_on, + 'updated_on': pull_request.updated_on, + 'commit_ids': pull_request.revisions, + 'review_status': pull_request.calculated_review_status(), + 'mergeable': merge_state, + 'source': { + 'clone_url': pull_request.source_repo.clone_url(), + 'repository': pull_request.source_repo.repo_name, + 'reference': { + 'name': pull_request.source_ref_parts.name, + 'type': pull_request.source_ref_parts.type, + 'commit_id': pull_request.source_ref_parts.commit_id, + }, + }, + 'target': { + 'clone_url': pull_request.target_repo.clone_url(), + 'repository': pull_request.target_repo.repo_name, + 'reference': { + 'name': pull_request.target_ref_parts.name, + 'type': pull_request.target_ref_parts.type, + 'commit_id': pull_request.target_ref_parts.commit_id, + }, + }, + 'merge': merge_data, + 'author': pull_request.author.get_api_data(include_secrets=False, + details='basic'), + 'reviewers': [ + { + 'user': reviewer.get_api_data(include_secrets=False, + details='basic'), + 'reasons': reasons, + 'review_status': st[0][1].status if st else 'not_reviewed', + } + for obj, reviewer, reasons, mandatory, st in + pull_request.reviewers_statuses() + ] + } + + return data + + def set_state(self, pull_request_state, final_state=None): + """ + # goes from initial state to updating to initial state. + # initial state can be changed by specifying back_state= + with pull_request_obj.set_state(PullRequest.STATE_UPDATING): + pull_request.merge() + + :param pull_request_state: + :param final_state: + + """ + + return _SetState(self, pull_request_state, back_state=final_state) + + +class PullRequest(Base, _PullRequestBase): + __tablename__ = 'pull_requests' + __table_args__ = ( + base_table_args, + ) + + pull_request_id = Column( + 'pull_request_id', Integer(), nullable=False, primary_key=True) + + def __repr__(self): + if self.pull_request_id: + return '' % self.pull_request_id + else: + return '' % id(self) + + reviewers = relationship('PullRequestReviewers', + cascade="all, delete, delete-orphan") + statuses = relationship('ChangesetStatus', + cascade="all, delete, delete-orphan") + comments = relationship('ChangesetComment', + cascade="all, delete, delete-orphan") + versions = relationship('PullRequestVersion', + cascade="all, delete, delete-orphan", + lazy='dynamic') + + @classmethod + def get_pr_display_object(cls, pull_request_obj, org_pull_request_obj, + internal_methods=None): + + class PullRequestDisplay(object): + """ + Special object wrapper for showing PullRequest data via Versions + It mimics PR object as close as possible. This is read only object + just for display + """ + + def __init__(self, attrs, internal=None): + self.attrs = attrs + # internal have priority over the given ones via attrs + self.internal = internal or ['versions'] + + def __getattr__(self, item): + if item in self.internal: + return getattr(self, item) + try: + return self.attrs[item] + except KeyError: + raise AttributeError( + '%s object has no attribute %s' % (self, item)) + + def __repr__(self): + return '' % self.attrs.get('pull_request_id') + + def versions(self): + return pull_request_obj.versions.order_by( + PullRequestVersion.pull_request_version_id).all() + + def is_closed(self): + return pull_request_obj.is_closed() + + @property + def pull_request_version_id(self): + return getattr(pull_request_obj, 'pull_request_version_id', None) + + attrs = StrictAttributeDict(pull_request_obj.get_api_data()) + + attrs.author = StrictAttributeDict( + pull_request_obj.author.get_api_data()) + if pull_request_obj.target_repo: + attrs.target_repo = StrictAttributeDict( + pull_request_obj.target_repo.get_api_data()) + attrs.target_repo.clone_url = pull_request_obj.target_repo.clone_url + + if pull_request_obj.source_repo: + attrs.source_repo = StrictAttributeDict( + pull_request_obj.source_repo.get_api_data()) + attrs.source_repo.clone_url = pull_request_obj.source_repo.clone_url + + attrs.source_ref_parts = pull_request_obj.source_ref_parts + attrs.target_ref_parts = pull_request_obj.target_ref_parts + attrs.revisions = pull_request_obj.revisions + + attrs.shadow_merge_ref = org_pull_request_obj.shadow_merge_ref + attrs.reviewer_data = org_pull_request_obj.reviewer_data + attrs.reviewer_data_json = org_pull_request_obj.reviewer_data_json + + return PullRequestDisplay(attrs, internal=internal_methods) + + def is_closed(self): + return self.status == self.STATUS_CLOSED + + def __json__(self): + return { + 'revisions': self.revisions, + } + + def calculated_review_status(self): + from rhodecode.model.changeset_status import ChangesetStatusModel + return ChangesetStatusModel().calculated_review_status(self) + + def reviewers_statuses(self): + from rhodecode.model.changeset_status import ChangesetStatusModel + return ChangesetStatusModel().reviewers_statuses(self) + + @property + def workspace_id(self): + from rhodecode.model.pull_request import PullRequestModel + return PullRequestModel()._workspace_id(self) + + def get_shadow_repo(self): + workspace_id = self.workspace_id + vcs_obj = self.target_repo.scm_instance() + shadow_repository_path = vcs_obj._get_shadow_repository_path( + self.target_repo.repo_id, workspace_id) + if os.path.isdir(shadow_repository_path): + return vcs_obj._get_shadow_instance(shadow_repository_path) + + +class PullRequestVersion(Base, _PullRequestBase): + __tablename__ = 'pull_request_versions' + __table_args__ = ( + base_table_args, + ) + + pull_request_version_id = Column( + 'pull_request_version_id', Integer(), nullable=False, primary_key=True) + pull_request_id = Column( + 'pull_request_id', Integer(), + ForeignKey('pull_requests.pull_request_id'), nullable=False) + pull_request = relationship('PullRequest') + + def __repr__(self): + if self.pull_request_version_id: + return '' % self.pull_request_version_id + else: + return '' % id(self) + + @property + def reviewers(self): + return self.pull_request.reviewers + + @property + def versions(self): + return self.pull_request.versions + + def is_closed(self): + # calculate from original + return self.pull_request.status == self.STATUS_CLOSED + + def calculated_review_status(self): + return self.pull_request.calculated_review_status() + + def reviewers_statuses(self): + return self.pull_request.reviewers_statuses() + + +class PullRequestReviewers(Base, BaseModel): + __tablename__ = 'pull_request_reviewers' + __table_args__ = ( + base_table_args, + ) + + @hybrid_property + def reasons(self): + if not self._reasons: + return [] + return self._reasons + + @reasons.setter + def reasons(self, val): + val = val or [] + if any(not isinstance(x, compat.string_types) for x in val): + raise Exception('invalid reasons type, must be list of strings') + self._reasons = val + + pull_requests_reviewers_id = Column( + 'pull_requests_reviewers_id', Integer(), nullable=False, + primary_key=True) + pull_request_id = Column( + "pull_request_id", Integer(), + ForeignKey('pull_requests.pull_request_id'), nullable=False) + user_id = Column( + "user_id", Integer(), ForeignKey('users.user_id'), nullable=True) + _reasons = Column( + 'reason', MutationList.as_mutable( + JsonType('list', dialect_map=dict(mysql=UnicodeText(16384))))) + + mandatory = Column("mandatory", Boolean(), nullable=False, default=False) + user = relationship('User') + pull_request = relationship('PullRequest') + + rule_data = Column( + 'rule_data_json', + JsonType(dialect_map=dict(mysql=UnicodeText(16384)))) + + def rule_user_group_data(self): + """ + Returns the voting user group rule data for this reviewer + """ + + if self.rule_data and 'vote_rule' in self.rule_data: + user_group_data = {} + if 'rule_user_group_entry_id' in self.rule_data: + # means a group with voting rules ! + user_group_data['id'] = self.rule_data['rule_user_group_entry_id'] + user_group_data['name'] = self.rule_data['rule_name'] + user_group_data['vote_rule'] = self.rule_data['vote_rule'] + + return user_group_data + + def __unicode__(self): + return u"<%s('id:%s')>" % (self.__class__.__name__, + self.pull_requests_reviewers_id) + + +class Notification(Base, BaseModel): + __tablename__ = 'notifications' + __table_args__ = ( + Index('notification_type_idx', 'type'), + base_table_args, + ) + + TYPE_CHANGESET_COMMENT = u'cs_comment' + TYPE_MESSAGE = u'message' + TYPE_MENTION = u'mention' + TYPE_REGISTRATION = u'registration' + TYPE_PULL_REQUEST = u'pull_request' + TYPE_PULL_REQUEST_COMMENT = u'pull_request_comment' + + notification_id = Column('notification_id', Integer(), nullable=False, primary_key=True) + subject = Column('subject', Unicode(512), nullable=True) + body = Column('body', UnicodeText().with_variant(UnicodeText(50000), 'mysql'), nullable=True) + created_by = Column("created_by", Integer(), ForeignKey('users.user_id'), nullable=True) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + type_ = Column('type', Unicode(255)) + + created_by_user = relationship('User') + notifications_to_users = relationship('UserNotification', lazy='joined', + cascade="all, delete, delete-orphan") + + @property + def recipients(self): + return [x.user for x in UserNotification.query()\ + .filter(UserNotification.notification == self)\ + .order_by(UserNotification.user_id.asc()).all()] + + @classmethod + def create(cls, created_by, subject, body, recipients, type_=None): + if type_ is None: + type_ = Notification.TYPE_MESSAGE + + notification = cls() + notification.created_by_user = created_by + notification.subject = subject + notification.body = body + notification.type_ = type_ + notification.created_on = datetime.datetime.now() + + # For each recipient link the created notification to his account + for u in recipients: + assoc = UserNotification() + assoc.user_id = u.user_id + assoc.notification = notification + + # if created_by is inside recipients mark his notification + # as read + if u.user_id == created_by.user_id: + assoc.read = True + Session().add(assoc) + + Session().add(notification) + + return notification + + +class UserNotification(Base, BaseModel): + __tablename__ = 'user_to_notification' + __table_args__ = ( + UniqueConstraint('user_id', 'notification_id'), + base_table_args + ) + + user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), primary_key=True) + notification_id = Column("notification_id", Integer(), ForeignKey('notifications.notification_id'), primary_key=True) + read = Column('read', Boolean, default=False) + sent_on = Column('sent_on', DateTime(timezone=False), nullable=True, unique=None) + + user = relationship('User', lazy="joined") + notification = relationship('Notification', lazy="joined", + order_by=lambda: Notification.created_on.desc(),) + + def mark_as_read(self): + self.read = True + Session().add(self) + + +class Gist(Base, BaseModel): + __tablename__ = 'gists' + __table_args__ = ( + Index('g_gist_access_id_idx', 'gist_access_id'), + Index('g_created_on_idx', 'created_on'), + base_table_args + ) + + GIST_PUBLIC = u'public' + GIST_PRIVATE = u'private' + DEFAULT_FILENAME = u'gistfile1.txt' + + ACL_LEVEL_PUBLIC = u'acl_public' + ACL_LEVEL_PRIVATE = u'acl_private' + + gist_id = Column('gist_id', Integer(), primary_key=True) + gist_access_id = Column('gist_access_id', Unicode(250)) + gist_description = Column('gist_description', UnicodeText().with_variant(UnicodeText(1024), 'mysql')) + gist_owner = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=True) + gist_expires = Column('gist_expires', Float(53), nullable=False) + gist_type = Column('gist_type', Unicode(128), nullable=False) + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + modified_at = Column('modified_at', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + acl_level = Column('acl_level', Unicode(128), nullable=True) + + owner = relationship('User') + + def __repr__(self): + return '' % (self.gist_type, self.gist_access_id) + + @hybrid_property + def description_safe(self): + from rhodecode.lib import helpers as h + return h.escape(self.gist_description) + + @classmethod + def get_or_404(cls, id_): + from pyramid.httpexceptions import HTTPNotFound + + res = cls.query().filter(cls.gist_access_id == id_).scalar() + if not res: + raise HTTPNotFound() + return res + + @classmethod + def get_by_access_id(cls, gist_access_id): + return cls.query().filter(cls.gist_access_id == gist_access_id).scalar() + + def gist_url(self): + from rhodecode.model.gist import GistModel + return GistModel().get_url(self) + + @classmethod + def base_path(cls): + """ + Returns base path when all gists are stored + + :param cls: + """ + from rhodecode.model.gist import GIST_STORE_LOC + q = Session().query(RhodeCodeUi)\ + .filter(RhodeCodeUi.ui_key == URL_SEP) + q = q.options(FromCache("sql_cache_short", "repository_repo_path")) + return os.path.join(q.one().ui_value, GIST_STORE_LOC) + + def get_api_data(self): + """ + Common function for generating gist related data for API + """ + gist = self + data = { + 'gist_id': gist.gist_id, + 'type': gist.gist_type, + 'access_id': gist.gist_access_id, + 'description': gist.gist_description, + 'url': gist.gist_url(), + 'expires': gist.gist_expires, + 'created_on': gist.created_on, + 'modified_at': gist.modified_at, + 'content': None, + 'acl_level': gist.acl_level, + } + return data + + def __json__(self): + data = dict( + ) + data.update(self.get_api_data()) + return data + # SCM functions + + def scm_instance(self, **kwargs): + full_repo_path = os.path.join(self.base_path(), self.gist_access_id) + return get_vcs_instance( + repo_path=safe_str(full_repo_path), create=False) + + +class ExternalIdentity(Base, BaseModel): + __tablename__ = 'external_identities' + __table_args__ = ( + Index('local_user_id_idx', 'local_user_id'), + Index('external_id_idx', 'external_id'), + base_table_args + ) + + external_id = Column('external_id', Unicode(255), default=u'', primary_key=True) + external_username = Column('external_username', Unicode(1024), default=u'') + local_user_id = Column('local_user_id', Integer(), ForeignKey('users.user_id'), primary_key=True) + provider_name = Column('provider_name', Unicode(255), default=u'', primary_key=True) + access_token = Column('access_token', String(1024), default=u'') + alt_token = Column('alt_token', String(1024), default=u'') + token_secret = Column('token_secret', String(1024), default=u'') + + @classmethod + def by_external_id_and_provider(cls, external_id, provider_name, local_user_id=None): + """ + Returns ExternalIdentity instance based on search params + + :param external_id: + :param provider_name: + :return: ExternalIdentity + """ + query = cls.query() + query = query.filter(cls.external_id == external_id) + query = query.filter(cls.provider_name == provider_name) + if local_user_id: + query = query.filter(cls.local_user_id == local_user_id) + return query.first() + + @classmethod + def user_by_external_id_and_provider(cls, external_id, provider_name): + """ + Returns User instance based on search params + + :param external_id: + :param provider_name: + :return: User + """ + query = User.query() + query = query.filter(cls.external_id == external_id) + query = query.filter(cls.provider_name == provider_name) + query = query.filter(User.user_id == cls.local_user_id) + return query.first() + + @classmethod + def by_local_user_id(cls, local_user_id): + """ + Returns all tokens for user + + :param local_user_id: + :return: ExternalIdentity + """ + query = cls.query() + query = query.filter(cls.local_user_id == local_user_id) + return query + + @classmethod + def load_provider_plugin(cls, plugin_id): + from rhodecode.authentication.base import loadplugin + _plugin_id = 'egg:rhodecode-enterprise-ee#{}'.format(plugin_id) + auth_plugin = loadplugin(_plugin_id) + return auth_plugin + + +class Integration(Base, BaseModel): + __tablename__ = 'integrations' + __table_args__ = ( + base_table_args + ) + + integration_id = Column('integration_id', Integer(), primary_key=True) + integration_type = Column('integration_type', String(255)) + enabled = Column('enabled', Boolean(), nullable=False) + name = Column('name', String(255), nullable=False) + child_repos_only = Column('child_repos_only', Boolean(), nullable=False, + default=False) + + settings = Column( + 'settings_json', MutationObj.as_mutable( + JsonType(dialect_map=dict(mysql=UnicodeText(16384))))) + repo_id = Column( + 'repo_id', Integer(), ForeignKey('repositories.repo_id'), + nullable=True, unique=None, default=None) + repo = relationship('Repository', lazy='joined') + + repo_group_id = Column( + 'repo_group_id', Integer(), ForeignKey('groups.group_id'), + nullable=True, unique=None, default=None) + repo_group = relationship('RepoGroup', lazy='joined') + + @property + def scope(self): + if self.repo: + return repr(self.repo) + if self.repo_group: + if self.child_repos_only: + return repr(self.repo_group) + ' (child repos only)' + else: + return repr(self.repo_group) + ' (recursive)' + if self.child_repos_only: + return 'root_repos' + return 'global' + + def __repr__(self): + return '' % (self.integration_type, self.scope) + + +class RepoReviewRuleUser(Base, BaseModel): + __tablename__ = 'repo_review_rules_users' + __table_args__ = ( + base_table_args + ) + + repo_review_rule_user_id = Column('repo_review_rule_user_id', Integer(), primary_key=True) + repo_review_rule_id = Column("repo_review_rule_id", Integer(), ForeignKey('repo_review_rules.repo_review_rule_id')) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False) + mandatory = Column("mandatory", Boolean(), nullable=False, default=False) + user = relationship('User') + + def rule_data(self): + return { + 'mandatory': self.mandatory + } + + +class RepoReviewRuleUserGroup(Base, BaseModel): + __tablename__ = 'repo_review_rules_users_groups' + __table_args__ = ( + base_table_args + ) + + VOTE_RULE_ALL = -1 + + repo_review_rule_users_group_id = Column('repo_review_rule_users_group_id', Integer(), primary_key=True) + repo_review_rule_id = Column("repo_review_rule_id", Integer(), ForeignKey('repo_review_rules.repo_review_rule_id')) + users_group_id = Column("users_group_id", Integer(),ForeignKey('users_groups.users_group_id'), nullable=False) + mandatory = Column("mandatory", Boolean(), nullable=False, default=False) + vote_rule = Column("vote_rule", Integer(), nullable=True, default=VOTE_RULE_ALL) + users_group = relationship('UserGroup') + + def rule_data(self): + return { + 'mandatory': self.mandatory, + 'vote_rule': self.vote_rule + } + + @property + def vote_rule_label(self): + if not self.vote_rule or self.vote_rule == self.VOTE_RULE_ALL: + return 'all must vote' + else: + return 'min. vote {}'.format(self.vote_rule) + + +class RepoReviewRule(Base, BaseModel): + __tablename__ = 'repo_review_rules' + __table_args__ = ( + base_table_args + ) + + repo_review_rule_id = Column( + 'repo_review_rule_id', Integer(), primary_key=True) + repo_id = Column( + "repo_id", Integer(), ForeignKey('repositories.repo_id')) + repo = relationship('Repository', backref='review_rules') + + review_rule_name = Column('review_rule_name', String(255)) + _branch_pattern = Column("branch_pattern", UnicodeText().with_variant(UnicodeText(255), 'mysql'), default=u'*') # glob + _target_branch_pattern = Column("target_branch_pattern", UnicodeText().with_variant(UnicodeText(255), 'mysql'), default=u'*') # glob + _file_pattern = Column("file_pattern", UnicodeText().with_variant(UnicodeText(255), 'mysql'), default=u'*') # glob + + use_authors_for_review = Column("use_authors_for_review", Boolean(), nullable=False, default=False) + forbid_author_to_review = Column("forbid_author_to_review", Boolean(), nullable=False, default=False) + forbid_commit_author_to_review = Column("forbid_commit_author_to_review", Boolean(), nullable=False, default=False) + forbid_adding_reviewers = Column("forbid_adding_reviewers", Boolean(), nullable=False, default=False) + + rule_users = relationship('RepoReviewRuleUser') + rule_user_groups = relationship('RepoReviewRuleUserGroup') + + def _validate_pattern(self, value): + re.compile('^' + glob2re(value) + '$') + + @hybrid_property + def source_branch_pattern(self): + return self._branch_pattern or '*' + + @source_branch_pattern.setter + def source_branch_pattern(self, value): + self._validate_pattern(value) + self._branch_pattern = value or '*' + + @hybrid_property + def target_branch_pattern(self): + return self._target_branch_pattern or '*' + + @target_branch_pattern.setter + def target_branch_pattern(self, value): + self._validate_pattern(value) + self._target_branch_pattern = value or '*' + + @hybrid_property + def file_pattern(self): + return self._file_pattern or '*' + + @file_pattern.setter + def file_pattern(self, value): + self._validate_pattern(value) + self._file_pattern = value or '*' + + def matches(self, source_branch, target_branch, files_changed): + """ + Check if this review rule matches a branch/files in a pull request + + :param source_branch: source branch name for the commit + :param target_branch: target branch name for the commit + :param files_changed: list of file paths changed in the pull request + """ + + source_branch = source_branch or '' + target_branch = target_branch or '' + files_changed = files_changed or [] + + branch_matches = True + if source_branch or target_branch: + if self.source_branch_pattern == '*': + source_branch_match = True + else: + if self.source_branch_pattern.startswith('re:'): + source_pattern = self.source_branch_pattern[3:] + else: + source_pattern = '^' + glob2re(self.source_branch_pattern) + '$' + source_branch_regex = re.compile(source_pattern) + source_branch_match = bool(source_branch_regex.search(source_branch)) + if self.target_branch_pattern == '*': + target_branch_match = True + else: + if self.target_branch_pattern.startswith('re:'): + target_pattern = self.target_branch_pattern[3:] + else: + target_pattern = '^' + glob2re(self.target_branch_pattern) + '$' + target_branch_regex = re.compile(target_pattern) + target_branch_match = bool(target_branch_regex.search(target_branch)) + + branch_matches = source_branch_match and target_branch_match + + files_matches = True + if self.file_pattern != '*': + files_matches = False + if self.file_pattern.startswith('re:'): + file_pattern = self.file_pattern[3:] + else: + file_pattern = glob2re(self.file_pattern) + file_regex = re.compile(file_pattern) + for filename in files_changed: + if file_regex.search(filename): + files_matches = True + break + + return branch_matches and files_matches + + @property + def review_users(self): + """ Returns the users which this rule applies to """ + + users = collections.OrderedDict() + + for rule_user in self.rule_users: + if rule_user.user.active: + if rule_user.user not in users: + users[rule_user.user.username] = { + 'user': rule_user.user, + 'source': 'user', + 'source_data': {}, + 'data': rule_user.rule_data() + } + + for rule_user_group in self.rule_user_groups: + source_data = { + 'user_group_id': rule_user_group.users_group.users_group_id, + 'name': rule_user_group.users_group.users_group_name, + 'members': len(rule_user_group.users_group.members) + } + for member in rule_user_group.users_group.members: + if member.user.active: + key = member.user.username + if key in users: + # skip this member as we have him already + # this prevents from override the "first" matched + # users with duplicates in multiple groups + continue + + users[key] = { + 'user': member.user, + 'source': 'user_group', + 'source_data': source_data, + 'data': rule_user_group.rule_data() + } + + return users + + def user_group_vote_rule(self, user_id): + + rules = [] + if not self.rule_user_groups: + return rules + + for user_group in self.rule_user_groups: + user_group_members = [x.user_id for x in user_group.users_group.members] + if user_id in user_group_members: + rules.append(user_group) + return rules + + def __repr__(self): + return '' % ( + self.repo_review_rule_id, self.repo) + + +class ScheduleEntry(Base, BaseModel): + __tablename__ = 'schedule_entries' + __table_args__ = ( + UniqueConstraint('schedule_name', name='s_schedule_name_idx'), + UniqueConstraint('task_uid', name='s_task_uid_idx'), + base_table_args, + ) + + schedule_types = ['crontab', 'timedelta', 'integer'] + schedule_entry_id = Column('schedule_entry_id', Integer(), primary_key=True) + + schedule_name = Column("schedule_name", String(255), nullable=False, unique=None, default=None) + schedule_description = Column("schedule_description", String(10000), nullable=True, unique=None, default=None) + schedule_enabled = Column("schedule_enabled", Boolean(), nullable=False, unique=None, default=True) + + _schedule_type = Column("schedule_type", String(255), nullable=False, unique=None, default=None) + schedule_definition = Column('schedule_definition_json', MutationObj.as_mutable(JsonType(default=lambda: "", dialect_map=dict(mysql=LONGTEXT())))) + + schedule_last_run = Column('schedule_last_run', DateTime(timezone=False), nullable=True, unique=None, default=None) + schedule_total_run_count = Column('schedule_total_run_count', Integer(), nullable=True, unique=None, default=0) + + # task + task_uid = Column("task_uid", String(255), nullable=False, unique=None, default=None) + task_dot_notation = Column("task_dot_notation", String(4096), nullable=False, unique=None, default=None) + task_args = Column('task_args_json', MutationObj.as_mutable(JsonType(default=list, dialect_map=dict(mysql=LONGTEXT())))) + task_kwargs = Column('task_kwargs_json', MutationObj.as_mutable(JsonType(default=dict, dialect_map=dict(mysql=LONGTEXT())))) + + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + updated_on = Column('updated_on', DateTime(timezone=False), nullable=True, unique=None, default=None) + + @hybrid_property + def schedule_type(self): + return self._schedule_type + + @schedule_type.setter + def schedule_type(self, val): + if val not in self.schedule_types: + raise ValueError('Value must be on of `{}` and got `{}`'.format( + val, self.schedule_type)) + + self._schedule_type = val + + @classmethod + def get_uid(cls, obj): + args = obj.task_args + kwargs = obj.task_kwargs + if isinstance(args, JsonRaw): + try: + args = json.loads(args) + except ValueError: + args = tuple() + + if isinstance(kwargs, JsonRaw): + try: + kwargs = json.loads(kwargs) + except ValueError: + kwargs = dict() + + dot_notation = obj.task_dot_notation + val = '.'.join(map(safe_str, [ + sorted(dot_notation), args, sorted(kwargs.items())])) + return hashlib.sha1(val).hexdigest() + + @classmethod + def get_by_schedule_name(cls, schedule_name): + return cls.query().filter(cls.schedule_name == schedule_name).scalar() + + @classmethod + def get_by_schedule_id(cls, schedule_id): + return cls.query().filter(cls.schedule_entry_id == schedule_id).scalar() + + @property + def task(self): + return self.task_dot_notation + + @property + def schedule(self): + from rhodecode.lib.celerylib.utils import raw_2_schedule + schedule = raw_2_schedule(self.schedule_definition, self.schedule_type) + return schedule + + @property + def args(self): + try: + return list(self.task_args or []) + except ValueError: + return list() + + @property + def kwargs(self): + try: + return dict(self.task_kwargs or {}) + except ValueError: + return dict() + + def _as_raw(self, val): + if hasattr(val, 'de_coerce'): + val = val.de_coerce() + if val: + val = json.dumps(val) + + return val + + @property + def schedule_definition_raw(self): + return self._as_raw(self.schedule_definition) + + @property + def args_raw(self): + return self._as_raw(self.task_args) + + @property + def kwargs_raw(self): + return self._as_raw(self.task_kwargs) + + def __repr__(self): + return ''.format( + self.schedule_entry_id, self.schedule_name) + + +@event.listens_for(ScheduleEntry, 'before_update') +def update_task_uid(mapper, connection, target): + target.task_uid = ScheduleEntry.get_uid(target) + + +@event.listens_for(ScheduleEntry, 'before_insert') +def set_task_uid(mapper, connection, target): + target.task_uid = ScheduleEntry.get_uid(target) + + +class _BaseBranchPerms(BaseModel): + @classmethod + def compute_hash(cls, value): + return sha1_safe(value) + + @hybrid_property + def branch_pattern(self): + return self._branch_pattern or '*' + + @hybrid_property + def branch_hash(self): + return self._branch_hash + + def _validate_glob(self, value): + re.compile('^' + glob2re(value) + '$') + + @branch_pattern.setter + def branch_pattern(self, value): + self._validate_glob(value) + self._branch_pattern = value or '*' + # set the Hash when setting the branch pattern + self._branch_hash = self.compute_hash(self._branch_pattern) + + def matches(self, branch): + """ + Check if this the branch matches entry + + :param branch: branch name for the commit + """ + + branch = branch or '' + + branch_matches = True + if branch: + branch_regex = re.compile('^' + glob2re(self.branch_pattern) + '$') + branch_matches = bool(branch_regex.search(branch)) + + return branch_matches + + +class UserToRepoBranchPermission(Base, _BaseBranchPerms): + __tablename__ = 'user_to_repo_branch_permissions' + __table_args__ = ( + {'extend_existing': True, 'mysql_engine': 'InnoDB', + 'mysql_charset': 'utf8', 'sqlite_autoincrement': True,} + ) + + branch_rule_id = Column('branch_rule_id', Integer(), primary_key=True) + + repository_id = Column('repository_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None) + repo = relationship('Repository', backref='user_branch_perms') + + permission_id = Column('permission_id', Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + permission = relationship('Permission') + + rule_to_perm_id = Column('rule_to_perm_id', Integer(), ForeignKey('repo_to_perm.repo_to_perm_id'), nullable=False, unique=None, default=None) + user_repo_to_perm = relationship('UserRepoToPerm') + + rule_order = Column('rule_order', Integer(), nullable=False) + _branch_pattern = Column('branch_pattern', UnicodeText().with_variant(UnicodeText(2048), 'mysql'), default=u'*') # glob + _branch_hash = Column('branch_hash', UnicodeText().with_variant(UnicodeText(2048), 'mysql')) + + def __unicode__(self): + return u' %r)>' % ( + self.user_repo_to_perm, self.branch_pattern) + + +class UserGroupToRepoBranchPermission(Base, _BaseBranchPerms): + __tablename__ = 'user_group_to_repo_branch_permissions' + __table_args__ = ( + {'extend_existing': True, 'mysql_engine': 'InnoDB', + 'mysql_charset': 'utf8', 'sqlite_autoincrement': True,} + ) + + branch_rule_id = Column('branch_rule_id', Integer(), primary_key=True) + + repository_id = Column('repository_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None) + repo = relationship('Repository', backref='user_group_branch_perms') + + permission_id = Column('permission_id', Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) + permission = relationship('Permission') + + rule_to_perm_id = Column('rule_to_perm_id', Integer(), ForeignKey('users_group_repo_to_perm.users_group_to_perm_id'), nullable=False, unique=None, default=None) + user_group_repo_to_perm = relationship('UserGroupRepoToPerm') + + rule_order = Column('rule_order', Integer(), nullable=False) + _branch_pattern = Column('branch_pattern', UnicodeText().with_variant(UnicodeText(2048), 'mysql'), default=u'*') # glob + _branch_hash = Column('branch_hash', UnicodeText().with_variant(UnicodeText(2048), 'mysql')) + + def __unicode__(self): + return u' %r)>' % ( + self.user_group_repo_to_perm, self.branch_pattern) + + +class UserBookmark(Base, BaseModel): + __tablename__ = 'user_bookmarks' + __table_args__ = ( + UniqueConstraint('user_id', 'bookmark_repo_id'), + UniqueConstraint('user_id', 'bookmark_repo_group_id'), + UniqueConstraint('user_id', 'bookmark_position'), + base_table_args + ) + + user_bookmark_id = Column("user_bookmark_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) + position = Column("bookmark_position", Integer(), nullable=False) + title = Column("bookmark_title", String(255), nullable=True, unique=None, default=None) + redirect_url = Column("bookmark_redirect_url", String(10240), nullable=True, unique=None, default=None) + created_on = Column("created_on", DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + + bookmark_repo_id = Column("bookmark_repo_id", Integer(), ForeignKey("repositories.repo_id"), nullable=True, unique=None, default=None) + bookmark_repo_group_id = Column("bookmark_repo_group_id", Integer(), ForeignKey("groups.group_id"), nullable=True, unique=None, default=None) + + user = relationship("User") + + repository = relationship("Repository") + repository_group = relationship("RepoGroup") + + @classmethod + def get_by_position_for_user(cls, position, user_id): + return cls.query() \ + .filter(UserBookmark.user_id == user_id) \ + .filter(UserBookmark.position == position).scalar() + + @classmethod + def get_bookmarks_for_user(cls, user_id): + return cls.query() \ + .filter(UserBookmark.user_id == user_id) \ + .options(joinedload(UserBookmark.repository)) \ + .options(joinedload(UserBookmark.repository_group)) \ + .order_by(UserBookmark.position.asc()) \ + .all() + + def __unicode__(self): + return u'' % (self.position, self.redirect_url) + + +class FileStore(Base, BaseModel): + __tablename__ = 'file_store' + __table_args__ = ( + base_table_args + ) + + file_store_id = Column('file_store_id', Integer(), primary_key=True) + file_uid = Column('file_uid', String(1024), nullable=False) + file_display_name = Column('file_display_name', UnicodeText().with_variant(UnicodeText(2048), 'mysql'), nullable=True) + file_description = Column('file_description', UnicodeText().with_variant(UnicodeText(10240), 'mysql'), nullable=True) + file_org_name = Column('file_org_name', UnicodeText().with_variant(UnicodeText(10240), 'mysql'), nullable=False) + + # sha256 hash + file_hash = Column('file_hash', String(512), nullable=False) + file_size = Column('file_size', Integer(), nullable=False) + + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + accessed_on = Column('accessed_on', DateTime(timezone=False), nullable=True) + accessed_count = Column('accessed_count', Integer(), default=0) + + enabled = Column('enabled', Boolean(), nullable=False, default=True) + + # if repo/repo_group reference is set, check for permissions + check_acl = Column('check_acl', Boolean(), nullable=False, default=True) + + user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False) + upload_user = relationship('User', lazy='joined', primaryjoin='User.user_id==FileStore.user_id') + + # scope limited to user, which requester have access to + scope_user_id = Column( + 'scope_user_id', Integer(), ForeignKey('users.user_id'), + nullable=True, unique=None, default=None) + user = relationship('User', lazy='joined', primaryjoin='User.user_id==FileStore.scope_user_id') + + # scope limited to user group, which requester have access to + scope_user_group_id = Column( + 'scope_user_group_id', Integer(), ForeignKey('users_groups.users_group_id'), + nullable=True, unique=None, default=None) + user_group = relationship('UserGroup', lazy='joined') + + # scope limited to repo, which requester have access to + scope_repo_id = Column( + 'scope_repo_id', Integer(), ForeignKey('repositories.repo_id'), + nullable=True, unique=None, default=None) + repo = relationship('Repository', lazy='joined') + + # scope limited to repo group, which requester have access to + scope_repo_group_id = Column( + 'scope_repo_group_id', Integer(), ForeignKey('groups.group_id'), + nullable=True, unique=None, default=None) + repo_group = relationship('RepoGroup', lazy='joined') + + def __repr__(self): + return ''.format(self.file_store_id) + + +class DbMigrateVersion(Base, BaseModel): + __tablename__ = 'db_migrate_version' + __table_args__ = ( + base_table_args, + ) + + repository_id = Column('repository_id', String(250), primary_key=True) + repository_path = Column('repository_path', Text) + version = Column('version', Integer) + + @classmethod + def set_version(cls, version): + """ + Helper for forcing a different version, usually for debugging purposes via ishell. + """ + ver = DbMigrateVersion.query().first() + ver.version = version + Session().commit() + + +class DbSession(Base, BaseModel): + __tablename__ = 'db_session' + __table_args__ = ( + base_table_args, + ) + + def __repr__(self): + return ''.format(self.id) + + id = Column('id', Integer()) + namespace = Column('namespace', String(255), primary_key=True) + accessed = Column('accessed', DateTime, nullable=False) + created = Column('created', DateTime, nullable=False) + data = Column('data', PickleType, nullable=False) diff --git a/rhodecode/lib/dbmigrate/schema/db_4_3_0_0.py b/rhodecode/lib/dbmigrate/schema/db_4_3_0_0.py --- a/rhodecode/lib/dbmigrate/schema/db_4_3_0_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_4_3_0_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -34,7 +34,6 @@ import functools import traceback import collections - from sqlalchemy import * from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.declarative import declared_attr @@ -45,6 +44,7 @@ from sqlalchemy.sql.expression import tr from beaker.cache import cache_region, region_invalidate from webob.exc import HTTPNotFound from zope.cachedescriptors.property import Lazy as LazyProperty +from pyramid import compat # replace pylons with fake url for migration from rhodecode.lib.dbmigrate.schema import url @@ -1811,7 +1811,7 @@ class Repository(Base, BaseModel): warnings.warn("Use get_commit", DeprecationWarning) commit_id = None commit_idx = None - if isinstance(rev, basestring): + if isinstance(rev, compat.string_types): commit_id = rev else: commit_idx = rev @@ -2007,7 +2007,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_4_4_0_0.py b/rhodecode/lib/dbmigrate/schema/db_4_4_0_0.py --- a/rhodecode/lib/dbmigrate/schema/db_4_4_0_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_4_4_0_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -34,7 +34,6 @@ import functools import traceback import collections - from sqlalchemy import * from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.declarative import declared_attr @@ -45,6 +44,7 @@ from sqlalchemy.sql.expression import tr from beaker.cache import cache_region, region_invalidate from webob.exc import HTTPNotFound from zope.cachedescriptors.property import Lazy as LazyProperty +from pyramid import compat # replace pylons with fake url for migration from rhodecode.lib.dbmigrate.schema import url @@ -1814,7 +1814,7 @@ class Repository(Base, BaseModel): warnings.warn("Use get_commit", DeprecationWarning) commit_id = None commit_idx = None - if isinstance(rev, basestring): + if isinstance(rev, compat.string_types): commit_id = rev else: commit_idx = rev @@ -1999,7 +1999,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_4_4_0_1.py b/rhodecode/lib/dbmigrate/schema/db_4_4_0_1.py --- a/rhodecode/lib/dbmigrate/schema/db_4_4_0_1.py +++ b/rhodecode/lib/dbmigrate/schema/db_4_4_0_1.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -34,7 +34,6 @@ import functools import traceback import collections - from sqlalchemy import * from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.declarative import declared_attr @@ -45,7 +44,7 @@ from sqlalchemy.sql.expression import tr from beaker.cache import cache_region, region_invalidate from webob.exc import HTTPNotFound from zope.cachedescriptors.property import Lazy as LazyProperty - +from pyramid import compat # replace pylons with fake url for migration from rhodecode.lib.dbmigrate.schema import url from rhodecode.translation import _ @@ -1814,7 +1813,7 @@ class Repository(Base, BaseModel): warnings.warn("Use get_commit", DeprecationWarning) commit_id = None commit_idx = None - if isinstance(rev, basestring): + if isinstance(rev, compat.string_types): commit_id = rev else: commit_idx = rev @@ -1999,7 +1998,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_4_4_0_2.py b/rhodecode/lib/dbmigrate/schema/db_4_4_0_2.py --- a/rhodecode/lib/dbmigrate/schema/db_4_4_0_2.py +++ b/rhodecode/lib/dbmigrate/schema/db_4_4_0_2.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -35,7 +35,6 @@ import functools import traceback import collections - from sqlalchemy import * from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.declarative import declared_attr @@ -46,7 +45,7 @@ from sqlalchemy.sql.expression import tr from beaker.cache import cache_region, region_invalidate from webob.exc import HTTPNotFound from zope.cachedescriptors.property import Lazy as LazyProperty - +from pyramid import compat # replace pylons with fake url for migration from rhodecode.lib.dbmigrate.schema import url from rhodecode.translation import _ @@ -1816,7 +1815,7 @@ class Repository(Base, BaseModel): warnings.warn("Use get_commit", DeprecationWarning) commit_id = None commit_idx = None - if isinstance(rev, basestring): + if isinstance(rev, compat.string_types): commit_id = rev else: commit_idx = rev @@ -2001,7 +2000,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) diff --git a/rhodecode/lib/dbmigrate/schema/db_4_5_0_0.py b/rhodecode/lib/dbmigrate/schema/db_4_5_0_0.py --- a/rhodecode/lib/dbmigrate/schema/db_4_5_0_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_4_5_0_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -35,7 +35,6 @@ import functools import traceback import collections - from sqlalchemy import * from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.declarative import declared_attr @@ -46,7 +45,7 @@ from sqlalchemy.sql.expression import tr from beaker.cache import cache_region, region_invalidate from webob.exc import HTTPNotFound from zope.cachedescriptors.property import Lazy as LazyProperty - +from pyramid import compat # replace pylons with fake url for migration from rhodecode.lib.dbmigrate.schema import url from rhodecode.translation import _ @@ -1816,7 +1815,7 @@ class Repository(Base, BaseModel): warnings.warn("Use get_commit", DeprecationWarning) commit_id = None commit_idx = None - if isinstance(rev, basestring): + if isinstance(rev, compat.string_types): commit_id = rev else: commit_idx = rev @@ -2001,7 +2000,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) @@ -3180,7 +3178,7 @@ class PullRequestReviewers(Base, BaseMod @reasons.setter def reasons(self, val): val = val or [] - if any(not isinstance(x, basestring) for x in val): + if any(not isinstance(x, compat.string_types) for x in val): raise Exception('invalid reasons type, must be list of strings') self._reasons = val diff --git a/rhodecode/lib/dbmigrate/schema/db_4_7_0_0.py b/rhodecode/lib/dbmigrate/schema/db_4_7_0_0.py --- a/rhodecode/lib/dbmigrate/schema/db_4_7_0_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_4_7_0_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -34,7 +34,6 @@ import functools import traceback import collections - from sqlalchemy import * from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.ext.hybrid import hybrid_property @@ -44,6 +43,7 @@ from sqlalchemy.sql.expression import tr from beaker.cache import cache_region from webob.exc import HTTPNotFound from zope.cachedescriptors.property import Lazy as LazyProperty +from pyramid import compat # replace pylons with fake url for migration from rhodecode.lib.dbmigrate.schema import url @@ -1858,7 +1858,7 @@ class Repository(Base, BaseModel): warnings.warn("Use get_commit", DeprecationWarning) commit_id = None commit_idx = None - if isinstance(rev, basestring): + if isinstance(rev, compat.string_types): commit_id = rev else: commit_idx = rev @@ -2043,7 +2043,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) @@ -3407,7 +3406,7 @@ class PullRequestReviewers(Base, BaseMod @reasons.setter def reasons(self, val): val = val or [] - if any(not isinstance(x, basestring) for x in val): + if any(not isinstance(x, compat.string_types) for x in val): raise Exception('invalid reasons type, must be list of strings') self._reasons = val diff --git a/rhodecode/lib/dbmigrate/schema/db_4_7_0_1.py b/rhodecode/lib/dbmigrate/schema/db_4_7_0_1.py --- a/rhodecode/lib/dbmigrate/schema/db_4_7_0_1.py +++ b/rhodecode/lib/dbmigrate/schema/db_4_7_0_1.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -34,7 +34,6 @@ import functools import traceback import collections - from sqlalchemy import * from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.ext.hybrid import hybrid_property @@ -44,6 +43,7 @@ from sqlalchemy.sql.expression import tr from beaker.cache import cache_region from webob.exc import HTTPNotFound from zope.cachedescriptors.property import Lazy as LazyProperty +from pyramid import compat # replace pylons with fake url for migration from rhodecode.lib.dbmigrate.schema import url @@ -1859,7 +1859,7 @@ class Repository(Base, BaseModel): warnings.warn("Use get_commit", DeprecationWarning) commit_id = None commit_idx = None - if isinstance(rev, basestring): + if isinstance(rev, compat.string_types): commit_id = rev else: commit_idx = rev @@ -2044,7 +2044,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) @@ -3408,7 +3407,7 @@ class PullRequestReviewers(Base, BaseMod @reasons.setter def reasons(self, val): val = val or [] - if any(not isinstance(x, basestring) for x in val): + if any(not isinstance(x, compat.string_types) for x in val): raise Exception('invalid reasons type, must be list of strings') self._reasons = val diff --git a/rhodecode/lib/dbmigrate/schema/db_4_9_0_0.py b/rhodecode/lib/dbmigrate/schema/db_4_9_0_0.py --- a/rhodecode/lib/dbmigrate/schema/db_4_9_0_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_4_9_0_0.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -34,7 +34,6 @@ import functools import traceback import collections - from sqlalchemy import * from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.ext.hybrid import hybrid_property @@ -44,7 +43,7 @@ from sqlalchemy.sql.expression import tr from sqlalchemy.sql.functions import coalesce, count # pragma: no cover from beaker.cache import cache_region from zope.cachedescriptors.property import Lazy as LazyProperty - +from pyramid import compat from pyramid.threadlocal import get_current_request from rhodecode.translation import _ @@ -2047,7 +2046,7 @@ class Repository(Base, BaseModel): warnings.warn("Use get_commit", DeprecationWarning) commit_id = None commit_idx = None - if isinstance(rev, basestring): + if isinstance(rev, compat.string_types): commit_id = rev else: commit_idx = rev @@ -2232,7 +2231,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), {'extend_existing': True, 'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8', 'sqlite_autoincrement': True}, ) @@ -3662,7 +3660,7 @@ class PullRequestReviewers(Base, BaseMod @reasons.setter def reasons(self, val): val = val or [] - if any(not isinstance(x, basestring) for x in val): + if any(not isinstance(x, compat.string_types) for x in val): raise Exception('invalid reasons type, must be list of strings') self._reasons = val diff --git a/rhodecode/lib/dbmigrate/utils.py b/rhodecode/lib/dbmigrate/utils.py --- a/rhodecode/lib/dbmigrate/utils.py +++ b/rhodecode/lib/dbmigrate/utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/dbmigrate/versions/070_version_4_7_0.py b/rhodecode/lib/dbmigrate/versions/070_version_4_7_0.py --- a/rhodecode/lib/dbmigrate/versions/070_version_4_7_0.py +++ b/rhodecode/lib/dbmigrate/versions/070_version_4_7_0.py @@ -2,6 +2,7 @@ import logging import datetime from sqlalchemy import * +from pyramid import compat from rhodecode.lib.utils2 import safe_str from rhodecode.model import meta @@ -12,7 +13,7 @@ log = logging.getLogger(__name__) def time_to_datetime(tm): if tm: - if isinstance(tm, basestring): + if isinstance(tm, compat.string_types): try: tm = float(tm) except ValueError: diff --git a/rhodecode/lib/dbmigrate/versions/092_version_4_16_0.py b/rhodecode/lib/dbmigrate/versions/092_version_4_16_0.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/dbmigrate/versions/092_version_4_16_0.py @@ -0,0 +1,37 @@ +import logging + +from sqlalchemy import * + +from rhodecode.model import meta +from rhodecode.lib.dbmigrate.versions import _reset_base, notify + +log = logging.getLogger(__name__) + + +def upgrade(migrate_engine): + """ + Upgrade operations go here. + Don't create your own engine; bind migrate_engine to your metadata + """ + _reset_base(migrate_engine) + from rhodecode.lib.dbmigrate.schema import db_4_13_0_0 as db + + pull_request = db.PullRequest.__table__ + pull_request_version = db.PullRequestVersion.__table__ + + repo_state_1 = Column("pull_request_state", String(255), nullable=True) + repo_state_1.create(table=pull_request) + + repo_state_2 = Column("pull_request_state", String(255), nullable=True) + repo_state_2.create(table=pull_request_version) + + fixups(db, meta.Session) + + +def downgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + +def fixups(models, _SESSION): + pass diff --git a/rhodecode/lib/dbmigrate/versions/093_version_4_16_0.py b/rhodecode/lib/dbmigrate/versions/093_version_4_16_0.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/dbmigrate/versions/093_version_4_16_0.py @@ -0,0 +1,41 @@ +import logging + +from sqlalchemy import * + +from rhodecode.model import meta +from rhodecode.lib.dbmigrate.versions import _reset_base, notify + +log = logging.getLogger(__name__) + + +def upgrade(migrate_engine): + """ + Upgrade operations go here. + Don't create your own engine; bind migrate_engine to your metadata + """ + _reset_base(migrate_engine) + from rhodecode.lib.dbmigrate.schema import db_4_16_0_0 as db + + fixups(db, meta.Session) + + +def downgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + +def fixups(models, _SESSION): + # move the builtin token to external tokens + + log.info('Updating pull request pull_request_state to %s', + models.PullRequest.STATE_CREATED) + qry = _SESSION().query(models.PullRequest) + qry.update({"pull_request_state": models.PullRequest.STATE_CREATED}) + _SESSION().commit() + + log.info('Updating pull_request_version pull_request_state to %s', + models.PullRequest.STATE_CREATED) + qry = _SESSION().query(models.PullRequestVersion) + qry.update({"pull_request_state": models.PullRequest.STATE_CREATED}) + _SESSION().commit() + diff --git a/rhodecode/lib/dbmigrate/versions/094_version_4_16_0.py b/rhodecode/lib/dbmigrate/versions/094_version_4_16_0.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/dbmigrate/versions/094_version_4_16_0.py @@ -0,0 +1,30 @@ +import logging + +from sqlalchemy import * + +from rhodecode.model import meta +from rhodecode.lib.dbmigrate.versions import _reset_base, notify + +log = logging.getLogger(__name__) + + +def upgrade(migrate_engine): + """ + Upgrade operations go here. + Don't create your own engine; bind migrate_engine to your metadata + """ + _reset_base(migrate_engine) + from rhodecode.lib.dbmigrate.schema import db_4_16_0_1 as db + + db.UserBookmark.__table__.create() + + fixups(db, meta.Session) + + +def downgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + +def fixups(models, _SESSION): + pass diff --git a/rhodecode/lib/dbmigrate/versions/095_version_4_16_0.py b/rhodecode/lib/dbmigrate/versions/095_version_4_16_0.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/dbmigrate/versions/095_version_4_16_0.py @@ -0,0 +1,30 @@ +import logging + +from sqlalchemy import * + +from rhodecode.model import meta +from rhodecode.lib.dbmigrate.versions import _reset_base, notify + +log = logging.getLogger(__name__) + + +def upgrade(migrate_engine): + """ + Upgrade operations go here. + Don't create your own engine; bind migrate_engine to your metadata + """ + _reset_base(migrate_engine) + from rhodecode.lib.dbmigrate.schema import db_4_16_0_2 as db + + db.FileStore.__table__.create() + + fixups(db, meta.Session) + + +def downgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + +def fixups(models, _SESSION): + pass diff --git a/rhodecode/lib/dbmigrate/versions/__init__.py b/rhodecode/lib/dbmigrate/versions/__init__.py --- a/rhodecode/lib/dbmigrate/versions/__init__.py +++ b/rhodecode/lib/dbmigrate/versions/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/diff_match_patch.py b/rhodecode/lib/diff_match_patch.py --- a/rhodecode/lib/diff_match_patch.py +++ b/rhodecode/lib/diff_match_patch.py @@ -1,4 +1,3 @@ -#!/usr/bin/python2.4 from __future__ import division @@ -33,6 +32,8 @@ import re import sys import time import urllib +from pyramid import compat + class diff_match_patch: """Class containing the diff, match and patch methods. @@ -1438,7 +1439,7 @@ class diff_match_patch: text1 = None diffs = None # Note that texts may arrive as 'str' or 'unicode'. - if isinstance(a, basestring) and isinstance(b, basestring) and c is None: + if isinstance(a, compat.string_types) and isinstance(b, compat.string_types) and c is None: # Method 1: text1, text2 # Compute diffs from text1 and text2. text1 = a @@ -1451,11 +1452,11 @@ class diff_match_patch: # Compute text1 from diffs. diffs = a text1 = self.diff_text1(diffs) - elif isinstance(a, basestring) and isinstance(b, list) and c is None: + elif isinstance(a, compat.string_types) and isinstance(b, list) and c is None: # Method 3: text1, diffs text1 = a diffs = b - elif (isinstance(a, basestring) and isinstance(b, basestring) and + elif (isinstance(a, compat.string_types) and isinstance(b, compat.string_types) and isinstance(c, list)): # Method 4: text1, text2, diffs # text2 is not used. diff --git a/rhodecode/lib/diffs.py b/rhodecode/lib/diffs.py --- a/rhodecode/lib/diffs.py +++ b/rhodecode/lib/diffs.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/encrypt.py b/rhodecode/lib/encrypt.py --- a/rhodecode/lib/encrypt.py +++ b/rhodecode/lib/encrypt.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/exc_tracking.py b/rhodecode/lib/exc_tracking.py --- a/rhodecode/lib/exc_tracking.py +++ b/rhodecode/lib/exc_tracking.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -67,14 +67,13 @@ def get_exc_store(): return _exc_store_path -def _store_exception(exc_id, exc_info, prefix): - exc_type, exc_value, exc_traceback = exc_info - tb = ''.join(traceback.format_exception( - exc_type, exc_value, exc_traceback, None)) +def _store_exception(exc_id, exc_type_name, exc_traceback, prefix): + """ + Low level function to store exception in the exception tracker + """ - exc_type_name = exc_type.__name__ exc_store_path = get_exc_store() - exc_data, org_data = exc_serialize(exc_id, tb, exc_type_name) + exc_data, org_data = exc_serialize(exc_id, exc_traceback, exc_type_name) exc_pref_id = '{}_{}_{}'.format(exc_id, prefix, org_data['exc_timestamp']) if not os.path.isdir(exc_store_path): os.makedirs(exc_store_path) @@ -84,6 +83,16 @@ def _store_exception(exc_id, exc_info, p log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path) +def _prepare_exception(exc_info): + exc_type, exc_value, exc_traceback = exc_info + exc_type_name = exc_type.__name__ + + tb = ''.join(traceback.format_exception( + exc_type, exc_value, exc_traceback, None)) + + return exc_type_name, tb + + def store_exception(exc_id, exc_info, prefix=global_prefix): """ Example usage:: @@ -93,7 +102,9 @@ def store_exception(exc_id, exc_info, pr """ try: - _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix) + exc_type_name, exc_traceback = _prepare_exception(exc_info) + _store_exception(exc_id=exc_id, exc_type_name=exc_type_name, + exc_traceback=exc_traceback, prefix=prefix) except Exception: log.exception('Failed to store exception `%s` information', exc_id) # there's no way this can fail, it will crash server badly if it does. @@ -149,3 +160,7 @@ def delete_exception(exc_id, prefix=glob log.exception('Failed to remove exception `%s` information', exc_id) # there's no way this can fail, it will crash server badly if it does. pass + + +def generate_id(): + return id(object()) diff --git a/rhodecode/lib/exceptions.py b/rhodecode/lib/exceptions.py --- a/rhodecode/lib/exceptions.py +++ b/rhodecode/lib/exceptions.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/ext_json_renderer.py b/rhodecode/lib/ext_json_renderer.py --- a/rhodecode/lib/ext_json_renderer.py +++ b/rhodecode/lib/ext_json_renderer.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/graphmod.py b/rhodecode/lib/graphmod.py --- a/rhodecode/lib/graphmod.py +++ b/rhodecode/lib/graphmod.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/helpers.py b/rhodecode/lib/helpers.py --- a/rhodecode/lib/helpers.py +++ b/rhodecode/lib/helpers.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -34,7 +34,6 @@ import urllib import math import logging import re -import urlparse import time import string import hashlib @@ -45,10 +44,10 @@ import itertools import fnmatch import bleach +from pyramid import compat from datetime import datetime from functools import partial from pygments.formatters.html import HtmlFormatter -from pygments import highlight as code_highlight from pygments.lexers import ( get_lexer_by_name, get_lexer_for_filename, get_lexer_for_mimetype) @@ -81,12 +80,14 @@ from rhodecode.lib.utils2 import str2boo from rhodecode.lib.markup_renderer import MarkupRenderer, relative_links from rhodecode.lib.vcs.exceptions import CommitDoesNotExistError from rhodecode.lib.vcs.backends.base import BaseChangeset, EmptyCommit +from rhodecode.lib.index.search_utils import get_matching_line_offsets from rhodecode.config.conf import DATE_FORMAT, DATETIME_FORMAT from rhodecode.model.changeset_status import ChangesetStatusModel from rhodecode.model.db import Permission, User, Repository from rhodecode.model.repo_group import RepoGroupModel from rhodecode.model.settings import IssueTrackerSettingsModel + log = logging.getLogger(__name__) @@ -260,6 +261,21 @@ def files_breadcrumbs(repo_name, commit_ return literal('/'.join(url_segments)) +def code_highlight(code, lexer, formatter, use_hl_filter=False): + """ + Lex ``code`` with ``lexer`` and format it with the formatter ``formatter``. + + If ``outfile`` is given and a valid file object (an object + with a ``write`` method), the result will be written to it, otherwise + it is returned as a string. + """ + if use_hl_filter: + # add HL filter + from rhodecode.lib.index import search_utils + lexer.add_filter(search_utils.ElasticSearchHLFilter()) + return pygments.format(pygments.lex(code, lexer), formatter) + + class CodeHtmlFormatter(HtmlFormatter): """ My code Html Formatter for source codes @@ -386,110 +402,9 @@ class SearchContentCodeHtmlFormatter(Cod current_line_number += 1 - yield 0, '' -def extract_phrases(text_query): - """ - Extracts phrases from search term string making sure phrases - contained in double quotes are kept together - and discarding empty values - or fully whitespace values eg. - - 'some text "a phrase" more' => ['some', 'text', 'a phrase', 'more'] - - """ - - in_phrase = False - buf = '' - phrases = [] - for char in text_query: - if in_phrase: - if char == '"': # end phrase - phrases.append(buf) - buf = '' - in_phrase = False - continue - else: - buf += char - continue - else: - if char == '"': # start phrase - in_phrase = True - phrases.append(buf) - buf = '' - continue - elif char == ' ': - phrases.append(buf) - buf = '' - continue - else: - buf += char - - phrases.append(buf) - phrases = [phrase.strip() for phrase in phrases if phrase.strip()] - return phrases - - -def get_matching_offsets(text, phrases): - """ - Returns a list of string offsets in `text` that the list of `terms` match - - >>> get_matching_offsets('some text here', ['some', 'here']) - [(0, 4), (10, 14)] - - """ - offsets = [] - for phrase in phrases: - for match in re.finditer(phrase, text): - offsets.append((match.start(), match.end())) - - return offsets - - -def normalize_text_for_matching(x): - """ - Replaces all non alnum characters to spaces and lower cases the string, - useful for comparing two text strings without punctuation - """ - return re.sub(r'[^\w]', ' ', x.lower()) - - -def get_matching_line_offsets(lines, terms): - """ Return a set of `lines` indices (starting from 1) matching a - text search query, along with `context` lines above/below matching lines - - :param lines: list of strings representing lines - :param terms: search term string to match in lines eg. 'some text' - :param context: number of lines above/below a matching line to add to result - :param max_lines: cut off for lines of interest - eg. - - text = ''' - words words words - words words words - some text some - words words words - words words words - text here what - ''' - get_matching_line_offsets(text, 'text', context=1) - {3: [(5, 9)], 6: [(0, 4)]] - - """ - matching_lines = {} - phrases = [normalize_text_for_matching(phrase) - for phrase in extract_phrases(terms)] - - for line_index, line in enumerate(lines, start=1): - match_offsets = get_matching_offsets( - normalize_text_for_matching(line), phrases) - if match_offsets: - matching_lines[line_index] = match_offsets - - return matching_lines - - def hsv_to_rgb(h, s, v): """ Convert hsv color values to rgb """ @@ -735,10 +650,9 @@ flash = Flash() # SCM FILTERS available via h. #============================================================================== from rhodecode.lib.vcs.utils import author_name, author_email -from rhodecode.lib.utils2 import credentials_filter, age as _age +from rhodecode.lib.utils2 import credentials_filter, age, age_from_seconds from rhodecode.model.db import User, ChangesetStatus -age = _age capitalize = lambda x: x.capitalize() email = author_email short_id = lambda x: x[:12] @@ -769,23 +683,25 @@ def age_component(datetime_iso, value=No datetime_iso, title, tzinfo)) -def _shorten_commit_id(commit_id): - from rhodecode import CONFIG - def_len = safe_int(CONFIG.get('rhodecode_show_sha_length', 12)) - return commit_id[:def_len] +def _shorten_commit_id(commit_id, commit_len=None): + if commit_len is None: + request = get_current_request() + commit_len = request.call_context.visual.show_sha_length + return commit_id[:commit_len] -def show_id(commit): +def show_id(commit, show_idx=None, commit_len=None): """ Configurable function that shows ID by default it's r123:fffeeefffeee :param commit: commit instance """ - from rhodecode import CONFIG - show_idx = str2bool(CONFIG.get('rhodecode_show_revision_number', True)) + if show_idx is None: + request = get_current_request() + show_idx = request.call_context.visual.show_revision_number - raw_id = _shorten_commit_id(commit.raw_id) + raw_id = _shorten_commit_id(commit.raw_id, commit_len=commit_len) if show_idx: return 'r%s:%s' % (commit.idx, raw_id) else: @@ -821,6 +737,7 @@ class _RepoChecker(object): _type = repository return _type == self._backend_alias + is_git = _RepoChecker('git') is_hg = _RepoChecker('hg') is_svn = _RepoChecker('svn') @@ -828,7 +745,8 @@ is_svn = _RepoChecker('svn') def get_repo_type_by_name(repo_name): repo = Repository.get_by_repo_name(repo_name) - return repo.repo_type + if repo: + return repo.repo_type def is_svn_without_proxy(repository): @@ -1577,6 +1495,28 @@ def breadcrumb_repo_link(repo): return literal(' » '.join(path)) +def breadcrumb_repo_group_link(repo_group): + """ + Makes a breadcrumbs path link to repo + + ex:: + group >> subgroup + + :param repo_group: a Repository Group instance + """ + + path = [ + link_to(group.name, + route_path('repo_group_home', repo_group_name=group.group_name)) + for group in repo_group.parents + ] + [ + link_to(repo_group.name, + route_path('repo_group_home', repo_group_name=repo_group.group_name)) + ] + + return literal(' » '.join(path)) + + def format_byte_size_binary(file_size): """ Formats file/folder sizes to standard. @@ -1629,8 +1569,7 @@ def urlify_commits(text_, repository): return tmpl % { 'pref': pref, 'cls': 'revision-link', - 'url': route_url('repo_commit', repo_name=repository, - commit_id=commit_id), + 'url': route_url('repo_commit', repo_name=repository, commit_id=commit_id), 'commit_id': commit_id, 'suf': suf } @@ -1661,8 +1600,7 @@ def _process_url_func(match_obj, repo_na raise ValueError('Bad link_format:{}'.format(link_format)) (repo_name_cleaned, - parent_group_name) = RepoGroupModel().\ - _get_group_name_and_parent(repo_name) + parent_group_name) = RepoGroupModel()._get_group_name_and_parent(repo_name) # variables replacement named_vars = { @@ -1675,10 +1613,14 @@ def _process_url_func(match_obj, repo_na named_vars.update(match_obj.groupdict()) _url = string.Template(entry['url']).safe_substitute(**named_vars) + def quote_cleaner(input_str): + """Remove quotes as it's HTML""" + return input_str.replace('"', '') + data = { 'pref': pref, - 'cls': 'issue-tracker-link', - 'url': _url, + 'cls': quote_cleaner('issue-tracker-link'), + 'url': quote_cleaner(_url), 'id-repr': issue_id, 'issue-prefix': entry['pref'], 'serv': entry['url'], @@ -1703,8 +1645,7 @@ def get_active_pattern_entries(repo_name return active_entries -def process_patterns(text_string, repo_name, link_format='html', - active_entries=None): +def process_patterns(text_string, repo_name, link_format='html', active_entries=None): allowed_formats = ['html', 'rst', 'markdown'] if link_format not in allowed_formats: @@ -1750,8 +1691,7 @@ def process_patterns(text_string, repo_n return newtext, issues_data -def urlify_commit_message(commit_text, repository=None, - active_pattern_entries=None): +def urlify_commit_message(commit_text, repository=None, active_pattern_entries=None): """ Parses given text message and makes proper links. issues are linked to given issue-server, and rest is a commit link @@ -1904,25 +1844,6 @@ def journal_filter_help(request): ).format(actions=actions) -def search_filter_help(searcher, request): - _ = request.translate - - terms = '' - return _( - 'Example filter terms for `{searcher}` search:\n' + - '{terms}\n' + - 'Generate wildcards using \'*\' character:\n' + - ' "repo_name:vcs*" - search everything starting with \'vcs\'\n' + - ' "repo_name:*vcs*" - search for repository containing \'vcs\'\n' + - '\n' + - 'Optional AND / OR operators in queries\n' + - ' "repo_name:vcs OR repo_name:test"\n' + - ' "owner:test AND repo_name:test*"\n' + - 'More: {search_doc}' - ).format(searcher=searcher.name, - terms=terms, search_doc=searcher.query_lang_doc) - - def not_mapped_error(repo_name): from rhodecode.translation import _ flash(_('%s repository is not mapped to db perhaps' @@ -2107,3 +2028,15 @@ def go_import_header(request, db_repo=No def reviewer_as_json(*args, **kwargs): from rhodecode.apps.repository.utils import reviewer_as_json as _reviewer_as_json return _reviewer_as_json(*args, **kwargs) + + +def get_repo_view_type(request): + route_name = request.matched_route.name + route_to_view_type = { + 'repo_changelog': 'changelog', + 'repo_files': 'files', + 'repo_summary': 'summary', + 'repo_commit': 'commit' + + } + return route_to_view_type.get(route_name) diff --git a/rhodecode/lib/hooks_base.py b/rhodecode/lib/hooks_base.py --- a/rhodecode/lib/hooks_base.py +++ b/rhodecode/lib/hooks_base.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2013-2018 RhodeCode GmbH +# Copyright (C) 2013-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -284,17 +284,16 @@ def post_push(extras): output += _http_ret.title if extras.new_refs: - tmpl = \ - extras.server_url + '/' + \ - extras.repository + \ - "/pull-request/new?{ref_type}={ref_name}" + tmpl = extras.server_url + '/' + extras.repository + \ + "/pull-request/new?{ref_type}={ref_name}" + for branch_name in extras.new_refs['branches']: output += 'RhodeCode: open pull request link: {}\n'.format( - tmpl.format(ref_type='branch', ref_name=branch_name)) + tmpl.format(ref_type='branch', ref_name=safe_str(branch_name))) for book_name in extras.new_refs['bookmarks']: output += 'RhodeCode: open pull request link: {}\n'.format( - tmpl.format(ref_type='bookmark', ref_name=book_name)) + tmpl.format(ref_type='bookmark', ref_name=safe_str(book_name))) hook_response = '' if not is_shadow_repo(extras): diff --git a/rhodecode/lib/hooks_daemon.py b/rhodecode/lib/hooks_daemon.py --- a/rhodecode/lib/hooks_daemon.py +++ b/rhodecode/lib/hooks_daemon.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/hooks_utils.py b/rhodecode/lib/hooks_utils.py --- a/rhodecode/lib/hooks_utils.py +++ b/rhodecode/lib/hooks_utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/index/__init__.py b/rhodecode/lib/index/__init__.py --- a/rhodecode/lib/index/__init__.py +++ b/rhodecode/lib/index/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -25,15 +25,27 @@ Index schema for RhodeCode import importlib import logging +from rhodecode.lib.index.search_utils import normalize_text_for_matching + log = logging.getLogger(__name__) # leave defaults for backward compat default_searcher = 'rhodecode.lib.index.whoosh' default_location = '%(here)s/data/index' +ES_VERSION_2 = '2' +ES_VERSION_6 = '6' +# for legacy reasons we keep 2 compat as default +DEFAULT_ES_VERSION = ES_VERSION_2 -class BaseSearch(object): +from rhodecode_tools.lib.fts_index.elasticsearch_engine_6 import \ + ES_CONFIG # pragma: no cover + + +class BaseSearcher(object): query_lang_doc = '' + es_version = None + name = None def __init__(self): pass @@ -41,19 +53,51 @@ class BaseSearch(object): def cleanup(self): pass - def search(self, query, document_type, search_user, repo_name=None, + def search(self, query, document_type, search_user, + repo_name=None, repo_group_name=None, raise_on_exc=True): raise Exception('NotImplemented') + @staticmethod + def query_to_mark(query, default_field=None): + """ + Formats the query to mark token for jquery.mark.js highlighting. ES could + have a different format optionally. -def searcher_from_config(config, prefix='search.'): + :param default_field: + :param query: + """ + return ' '.join(normalize_text_for_matching(query).split()) + + @property + def is_es_6(self): + return self.es_version == ES_VERSION_6 + + def get_handlers(self): + return {} + + @staticmethod + def extract_search_tags(query): + return [] + + +def search_config(config, prefix='search.'): _config = {} for key in config.keys(): if key.startswith(prefix): _config[key[len(prefix):]] = config[key] + return _config + + +def searcher_from_config(config, prefix='search.'): + _config = search_config(config, prefix) if 'location' not in _config: _config['location'] = default_location + if 'es_version' not in _config: + # use old legacy ES version set to 2 + _config['es_version'] = '2' + imported = importlib.import_module(_config.get('module', default_searcher)) - searcher = imported.Search(config=_config) + searcher = imported.Searcher(config=_config) return searcher diff --git a/rhodecode/lib/index/search_utils.py b/rhodecode/lib/index/search_utils.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/index/search_utils.py @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2012-2019 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ +import re + +import pygments.filter +import pygments.filters +from pygments.token import Comment + +HL_BEG_MARKER = '__RCSearchHLMarkBEG__' +HL_END_MARKER = '__RCSearchHLMarkEND__' +HL_MARKER_RE = '{}(.*?){}'.format(HL_BEG_MARKER, HL_END_MARKER) + + +class ElasticSearchHLFilter(pygments.filters.Filter): + _names = [HL_BEG_MARKER, HL_END_MARKER] + + def __init__(self, **options): + pygments.filters.Filter.__init__(self, **options) + + def filter(self, lexer, stream): + def tokenize(_value): + for token in re.split('({}|{})'.format( + self._names[0], self._names[1]), _value): + if token: + yield token + + hl = False + for ttype, value in stream: + + if self._names[0] in value or self._names[1] in value: + for item in tokenize(value): + if item == self._names[0]: + # skip marker, but start HL + hl = True + continue + elif item == self._names[1]: + hl = False + continue + + if hl: + yield Comment.ElasticMatch, item + else: + yield ttype, item + else: + if hl: + yield Comment.ElasticMatch, value + else: + yield ttype, value + + +def extract_phrases(text_query): + """ + Extracts phrases from search term string making sure phrases + contained in double quotes are kept together - and discarding empty values + or fully whitespace values eg. + + 'some text "a phrase" more' => ['some', 'text', 'a phrase', 'more'] + + """ + + in_phrase = False + buf = '' + phrases = [] + for char in text_query: + if in_phrase: + if char == '"': # end phrase + phrases.append(buf) + buf = '' + in_phrase = False + continue + else: + buf += char + continue + else: + if char == '"': # start phrase + in_phrase = True + phrases.append(buf) + buf = '' + continue + elif char == ' ': + phrases.append(buf) + buf = '' + continue + else: + buf += char + + phrases.append(buf) + phrases = [phrase.strip() for phrase in phrases if phrase.strip()] + return phrases + + +def get_matching_phrase_offsets(text, phrases): + """ + Returns a list of string offsets in `text` that the list of `terms` match + + >>> get_matching_phrase_offsets('some text here', ['some', 'here']) + [(0, 4), (10, 14)] + + """ + phrases = phrases or [] + offsets = [] + + for phrase in phrases: + for match in re.finditer(phrase, text): + offsets.append((match.start(), match.end())) + + return offsets + + +def get_matching_markers_offsets(text, markers=None): + """ + Returns a list of string offsets in `text` that the are between matching markers + + >>> get_matching_markers_offsets('$1some$2 text $1here$2 marked', ['\$1(.*?)\$2']) + [(0, 5), (16, 22)] + + """ + markers = markers or [HL_MARKER_RE] + offsets = [] + + if markers: + for mark in markers: + for match in re.finditer(mark, text): + offsets.append((match.start(), match.end())) + + return offsets + + +def normalize_text_for_matching(x): + """ + Replaces all non alfanum characters to spaces and lower cases the string, + useful for comparing two text strings without punctuation + """ + return re.sub(r'[^\w]', ' ', x.lower()) + + +def get_matching_line_offsets(lines, terms=None, markers=None): + """ Return a set of `lines` indices (starting from 1) matching a + text search query, along with `context` lines above/below matching lines + + :param lines: list of strings representing lines + :param terms: search term string to match in lines eg. 'some text' + :param markers: instead of terms, use highlight markers instead that + mark beginning and end for matched item. eg. ['START(.*?)END'] + + eg. + + text = ''' + words words words + words words words + some text some + words words words + words words words + text here what + ''' + get_matching_line_offsets(text, 'text', context=1) + 6, {3: [(5, 9)], 6: [(0, 4)]] + + """ + matching_lines = {} + line_index = 0 + + if terms: + phrases = [normalize_text_for_matching(phrase) + for phrase in extract_phrases(terms)] + + for line_index, line in enumerate(lines.splitlines(), start=1): + normalized_line = normalize_text_for_matching(line) + match_offsets = get_matching_phrase_offsets(normalized_line, phrases) + if match_offsets: + matching_lines[line_index] = match_offsets + + else: + markers = markers or [HL_MARKER_RE] + for line_index, line in enumerate(lines.splitlines(), start=1): + match_offsets = get_matching_markers_offsets(line, markers=markers) + if match_offsets: + matching_lines[line_index] = match_offsets + + return line_index, matching_lines diff --git a/rhodecode/lib/index/whoosh.py b/rhodecode/lib/index/whoosh.py --- a/rhodecode/lib/index/whoosh.py +++ b/rhodecode/lib/index/whoosh.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -33,7 +33,7 @@ from whoosh.index import create_in, open from whoosh.qparser import QueryParser, QueryParserError import rhodecode.lib.helpers as h -from rhodecode.lib.index import BaseSearch +from rhodecode.lib.index import BaseSearcher from rhodecode.lib.utils2 import safe_unicode log = logging.getLogger(__name__) @@ -59,13 +59,13 @@ FRAGMENTER = ContextFragmenter(200) log = logging.getLogger(__name__) -class Search(BaseSearch): +class WhooshSearcher(BaseSearcher): # this also shows in UI query_lang_doc = 'http://whoosh.readthedocs.io/en/latest/querylang.html' name = 'whoosh' def __init__(self, config): - super(Search, self).__init__() + super(Searcher, self).__init__() self.config = config if not os.path.isdir(self.config['location']): os.makedirs(self.config['location']) @@ -100,8 +100,8 @@ class Search(BaseSearch): return query def search(self, query, document_type, search_user, - repo_name=None, requested_page=1, page_limit=10, sort=None, - raise_on_exc=True): + repo_name=None, repo_group_name=None, + requested_page=1, page_limit=10, sort=None, raise_on_exc=True): original_query = query query = self._extend_query(query) @@ -162,16 +162,17 @@ class Search(BaseSearch): _ = translator stats = [ {'key': _('Index Type'), 'value': 'Whoosh'}, + {'sep': True}, + {'key': _('File Index'), 'value': str(self.file_index)}, - {'key': _('Indexed documents'), - 'value': self.file_index.doc_count()}, - {'key': _('Last update'), - 'value': h.time_to_datetime(self.file_index.last_modified())}, + {'key': _('Indexed documents'), 'value': self.file_index.doc_count()}, + {'key': _('Last update'), 'value': h.time_to_datetime(self.file_index.last_modified())}, + + {'sep': True}, + {'key': _('Commit index'), 'value': str(self.commit_index)}, - {'key': _('Indexed documents'), - 'value': str(self.commit_index.doc_count())}, - {'key': _('Last update'), - 'value': h.time_to_datetime(self.commit_index.last_modified())} + {'key': _('Indexed documents'), 'value': str(self.commit_index.doc_count())}, + {'key': _('Last update'), 'value': h.time_to_datetime(self.commit_index.last_modified())} ] return stats @@ -227,6 +228,9 @@ class Search(BaseSearch): return self.searcher +Searcher = WhooshSearcher + + class WhooshResultWrapper(object): def __init__(self, search_type, total_hits, results): self.search_type = search_type @@ -263,6 +267,8 @@ class WhooshResultWrapper(object): # TODO: marcink: this feels like an overkill, there's a lot of data # inside hit object, and we don't need all res = dict(hit) + # elastic search uses that, we set it empty so it fallbacks to regular HL logic + res['content_highlight'] = '' f_path = '' # pragma: no cover if self.search_type in ['content', 'path']: diff --git a/rhodecode/lib/index/whoosh_fallback_schema.py b/rhodecode/lib/index/whoosh_fallback_schema.py --- a/rhodecode/lib/index/whoosh_fallback_schema.py +++ b/rhodecode/lib/index/whoosh_fallback_schema.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/jsonalchemy.py b/rhodecode/lib/jsonalchemy.py --- a/rhodecode/lib/jsonalchemy.py +++ b/rhodecode/lib/jsonalchemy.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/logging_formatter.py b/rhodecode/lib/logging_formatter.py --- a/rhodecode/lib/logging_formatter.py +++ b/rhodecode/lib/logging_formatter.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/markdown_ext.py b/rhodecode/lib/markdown_ext.py --- a/rhodecode/lib/markdown_ext.py +++ b/rhodecode/lib/markdown_ext.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/markup_renderer.py b/rhodecode/lib/markup_renderer.py --- a/rhodecode/lib/markup_renderer.py +++ b/rhodecode/lib/markup_renderer.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -357,7 +357,9 @@ class MarkupRenderer(object): if leading_newline: source += '
' source += rendered_source.replace("\n", '
') - return source + + rendered = cls.bleach_clean(source) + return rendered @classmethod def markdown(cls, source, safe=True, flavored=True, mentions=False, diff --git a/rhodecode/lib/memory_lru_dict.py b/rhodecode/lib/memory_lru_dict.py --- a/rhodecode/lib/memory_lru_dict.py +++ b/rhodecode/lib/memory_lru_dict.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/middleware/__init__.py b/rhodecode/lib/middleware/__init__.py --- a/rhodecode/lib/middleware/__init__.py +++ b/rhodecode/lib/middleware/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/middleware/appenlight.py b/rhodecode/lib/middleware/appenlight.py --- a/rhodecode/lib/middleware/appenlight.py +++ b/rhodecode/lib/middleware/appenlight.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/middleware/csrf.py b/rhodecode/lib/middleware/csrf.py --- a/rhodecode/lib/middleware/csrf.py +++ b/rhodecode/lib/middleware/csrf.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/middleware/https_fixup.py b/rhodecode/lib/middleware/https_fixup.py --- a/rhodecode/lib/middleware/https_fixup.py +++ b/rhodecode/lib/middleware/https_fixup.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/middleware/request_wrapper.py b/rhodecode/lib/middleware/request_wrapper.py --- a/rhodecode/lib/middleware/request_wrapper.py +++ b/rhodecode/lib/middleware/request_wrapper.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -45,8 +45,8 @@ class RequestWrapperTween(object): end = time.time() total = end - start log.info( - 'IP: %s Request to %s time: %.3fs [%s]', - get_ip_addr(request.environ), + 'IP: %s %s Request to %s time: %.3fs [%s]', + get_ip_addr(request.environ), request.environ.get('REQUEST_METHOD'), safe_str(get_access_path(request.environ)), total, get_user_agent(request. environ) ) @@ -57,4 +57,4 @@ class RequestWrapperTween(object): def includeme(config): config.add_tween( 'rhodecode.lib.middleware.request_wrapper.RequestWrapperTween', - ) \ No newline at end of file + ) diff --git a/rhodecode/lib/middleware/simplegit.py b/rhodecode/lib/middleware/simplegit.py --- a/rhodecode/lib/middleware/simplegit.py +++ b/rhodecode/lib/middleware/simplegit.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/middleware/simplehg.py b/rhodecode/lib/middleware/simplehg.py --- a/rhodecode/lib/middleware/simplehg.py +++ b/rhodecode/lib/middleware/simplehg.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/middleware/simplesvn.py b/rhodecode/lib/middleware/simplesvn.py --- a/rhodecode/lib/middleware/simplesvn.py +++ b/rhodecode/lib/middleware/simplesvn.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/middleware/simplevcs.py b/rhodecode/lib/middleware/simplevcs.py --- a/rhodecode/lib/middleware/simplevcs.py +++ b/rhodecode/lib/middleware/simplevcs.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/middleware/utils/__init__.py b/rhodecode/lib/middleware/utils/__init__.py --- a/rhodecode/lib/middleware/utils/__init__.py +++ b/rhodecode/lib/middleware/utils/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/middleware/utils/scm_app.py b/rhodecode/lib/middleware/utils/scm_app.py --- a/rhodecode/lib/middleware/utils/scm_app.py +++ b/rhodecode/lib/middleware/utils/scm_app.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/middleware/utils/scm_app_http.py b/rhodecode/lib/middleware/utils/scm_app_http.py --- a/rhodecode/lib/middleware/utils/scm_app_http.py +++ b/rhodecode/lib/middleware/utils/scm_app_http.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/middleware/utils/wsgi_app_caller_client.py b/rhodecode/lib/middleware/utils/wsgi_app_caller_client.py --- a/rhodecode/lib/middleware/utils/wsgi_app_caller_client.py +++ b/rhodecode/lib/middleware/utils/wsgi_app_caller_client.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/middleware/vcs.py b/rhodecode/lib/middleware/vcs.py --- a/rhodecode/lib/middleware/vcs.py +++ b/rhodecode/lib/middleware/vcs.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/partial_renderer.py b/rhodecode/lib/partial_renderer.py --- a/rhodecode/lib/partial_renderer.py +++ b/rhodecode/lib/partial_renderer.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/paster_commands/__init__.py b/rhodecode/lib/paster_commands/__init__.py --- a/rhodecode/lib/paster_commands/__init__.py +++ b/rhodecode/lib/paster_commands/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/paster_commands/deprecated/celeryd.py b/rhodecode/lib/paster_commands/deprecated/celeryd.py --- a/rhodecode/lib/paster_commands/deprecated/celeryd.py +++ b/rhodecode/lib/paster_commands/deprecated/celeryd.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2013-2018 RhodeCode GmbH +# Copyright (C) 2013-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/paster_commands/deprecated/setup_rhodecode.py b/rhodecode/lib/paster_commands/deprecated/setup_rhodecode.py --- a/rhodecode/lib/paster_commands/deprecated/setup_rhodecode.py +++ b/rhodecode/lib/paster_commands/deprecated/setup_rhodecode.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/paster_commands/ishell.py b/rhodecode/lib/paster_commands/ishell.py --- a/rhodecode/lib/paster_commands/ishell.py +++ b/rhodecode/lib/paster_commands/ishell.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2013-2018 RhodeCode GmbH +# Copyright (C) 2013-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/paster_commands/upgrade_db.py b/rhodecode/lib/paster_commands/upgrade_db.py --- a/rhodecode/lib/paster_commands/upgrade_db.py +++ b/rhodecode/lib/paster_commands/upgrade_db.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/pidlock.py b/rhodecode/lib/pidlock.py --- a/rhodecode/lib/pidlock.py +++ b/rhodecode/lib/pidlock.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/plugins/__init__.py b/rhodecode/lib/plugins/__init__.py --- a/rhodecode/lib/plugins/__init__.py +++ b/rhodecode/lib/plugins/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/plugins/utils.py b/rhodecode/lib/plugins/utils.py --- a/rhodecode/lib/plugins/utils.py +++ b/rhodecode/lib/plugins/utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/pyramid_shell/__init__.py b/rhodecode/lib/pyramid_shell/__init__.py --- a/rhodecode/lib/pyramid_shell/__init__.py +++ b/rhodecode/lib/pyramid_shell/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/pyramid_utils.py b/rhodecode/lib/pyramid_utils.py --- a/rhodecode/lib/pyramid_utils.py +++ b/rhodecode/lib/pyramid_utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/rc_cache/__init__.py b/rhodecode/lib/rc_cache/__init__.py --- a/rhodecode/lib/rc_cache/__init__.py +++ b/rhodecode/lib/rc_cache/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2015-2018 RhodeCode GmbH +# Copyright (C) 2015-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/rc_cache/backends.py b/rhodecode/lib/rc_cache/backends.py --- a/rhodecode/lib/rc_cache/backends.py +++ b/rhodecode/lib/rc_cache/backends.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2015-2018 RhodeCode GmbH +# Copyright (C) 2015-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -98,7 +98,7 @@ class CustomLockFactory(FileLock): # set non-blocking, this will cause an exception if we cannot acquire a lock operation |= fcntl.LOCK_NB start_lock_time = time.time() - timeout = 60 * 5 # 5min + timeout = 60 * 15 # 15min while True: try: flock_org(fd, operation) @@ -203,3 +203,12 @@ class RedisPickleBackend(Serializer, red for key, value in mapping.items(): pipe.setex(key, self.redis_expiration_time, value) pipe.execute() + + def get_mutex(self, key): + u = redis_backend.u + if self.distributed_lock: + lock_key = u('_lock_{0}').format(key) + log.debug('Trying to acquire Redis lock for key %s', lock_key) + return self.client.lock(lock_key, self.lock_timeout, self.lock_sleep) + else: + return None diff --git a/rhodecode/lib/rc_cache/region_meta.py b/rhodecode/lib/rc_cache/region_meta.py --- a/rhodecode/lib/rc_cache/region_meta.py +++ b/rhodecode/lib/rc_cache/region_meta.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2015-2018 RhodeCode GmbH +# Copyright (C) 2015-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/rc_cache/utils.py b/rhodecode/lib/rc_cache/utils.py --- a/rhodecode/lib/rc_cache/utils.py +++ b/rhodecode/lib/rc_cache/utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2015-2018 RhodeCode GmbH +# Copyright (C) 2015-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -102,6 +102,7 @@ class RhodeCodeCacheRegion(CacheRegion): decorate.get = get decorate.original = fn decorate.key_generator = key_generator + decorate.__wrapped__ = fn return decorate @@ -120,7 +121,7 @@ def get_default_cache_settings(settings, if key.startswith(prefix): name = key.split(prefix)[1].strip() val = settings[key] - if isinstance(val, basestring): + if isinstance(val, compat.string_types): val = val.strip() cache_settings[name] = val return cache_settings diff --git a/rhodecode/lib/rc_commands/ishell.py b/rhodecode/lib/rc_commands/ishell.py --- a/rhodecode/lib/rc_commands/ishell.py +++ b/rhodecode/lib/rc_commands/ishell.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/rc_commands/setup_rc.py b/rhodecode/lib/rc_commands/setup_rc.py --- a/rhodecode/lib/rc_commands/setup_rc.py +++ b/rhodecode/lib/rc_commands/setup_rc.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/rc_commands/upgrade_db.py b/rhodecode/lib/rc_commands/upgrade_db.py --- a/rhodecode/lib/rc_commands/upgrade_db.py +++ b/rhodecode/lib/rc_commands/upgrade_db.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -25,6 +25,7 @@ import pyramid.paster from rhodecode.lib.pyramid_utils import bootstrap from rhodecode.lib.db_manage import DbManage +from rhodecode.lib.utils2 import safe_int log = logging.getLogger(__name__) @@ -33,11 +34,13 @@ log = logging.getLogger(__name__) @click.argument('ini_path', type=click.Path(exists=True)) @click.option('--force-yes/--force-no', default=None, help="Force yes/no to every question") -def main(ini_path, force_yes): - return command(ini_path, force_yes) +@click.option('--force-version', default=None, + help="Force upgrade from version") +def main(ini_path, force_yes, force_version): + return command(ini_path, force_yes, force_version) -def command(ini_path, force_yes): +def command(ini_path, force_yes, force_version): pyramid.paster.setup_logging(ini_path) with bootstrap(ini_path, env={'RC_CMD_UPGRADE_DB': '1'}) as env: @@ -49,4 +52,4 @@ def command(ini_path, force_yes): dbmanage = DbManage( log_sql=True, dbconf=db_uri, root='.', tests=False, cli_args=options) - dbmanage.upgrade() + dbmanage.upgrade(version=safe_int(force_version)) diff --git a/rhodecode/lib/repo_maintenance.py b/rhodecode/lib/repo_maintenance.py --- a/rhodecode/lib/repo_maintenance.py +++ b/rhodecode/lib/repo_maintenance.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2017-2018 RhodeCode GmbH +# Copyright (C) 2017-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/request.py b/rhodecode/lib/request.py --- a/rhodecode/lib/request.py +++ b/rhodecode/lib/request.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2017-2018 RhodeCode GmbH +# Copyright (C) 2017-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/system_info.py b/rhodecode/lib/system_info.py --- a/rhodecode/lib/system_info.py +++ b/rhodecode/lib/system_info.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2017-2018 RhodeCode GmbH +# Copyright (C) 2017-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -67,8 +67,7 @@ def get_storage_size(storage_path): try: sizes.append(os.path.getsize(storage_file)) except OSError: - log.exception('Failed to get size of storage file %s', - storage_file) + log.exception('Failed to get size of storage file %s', storage_file) pass return sum(sizes) @@ -81,6 +80,16 @@ def get_resource(resource_type): return 'NOT_SUPPORTED' +def get_cert_path(ini_path): + default = '/etc/ssl/certs/ca-certificates.crt' + control_ca_bundle = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(ini_path)))), + '.rccontrol-profile/etc/ca-bundle.crt') + if os.path.isfile(control_ca_bundle): + default = control_ca_bundle + + return default + class SysInfoRes(object): def __init__(self, value, state=None, human_value=None): self.value = value @@ -604,6 +613,7 @@ def rhodecode_config(): import rhodecode path = rhodecode.CONFIG.get('__file__') rhodecode_ini_safe = rhodecode.CONFIG.copy() + cert_path = get_cert_path(path) try: config = configparser.ConfigParser() @@ -615,10 +625,6 @@ def rhodecode_config(): log.exception('Failed to read .ini file for display') parsed_ini = {} - cert_path = os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(path)))), - '.rccontrol-profile/etc/ca-bundle.crt') - rhodecode_ini_safe['server:main'] = parsed_ini blacklist = [ diff --git a/rhodecode/lib/user_log_filter.py b/rhodecode/lib/user_log_filter.py --- a/rhodecode/lib/user_log_filter.py +++ b/rhodecode/lib/user_log_filter.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/user_sessions.py b/rhodecode/lib/user_sessions.py --- a/rhodecode/lib/user_sessions.py +++ b/rhodecode/lib/user_sessions.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2017-2018 RhodeCode GmbH +# Copyright (C) 2017-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/utils.py b/rhodecode/lib/utils.py --- a/rhodecode/lib/utils.py +++ b/rhodecode/lib/utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/utils2.py b/rhodecode/lib/utils2.py --- a/rhodecode/lib/utils2.py +++ b/rhodecode/lib/utils2.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -43,6 +43,7 @@ import sqlalchemy.exc import sqlalchemy.sql import webob import pyramid.threadlocal +from pyramid import compat from pyramid.settings import asbool import rhodecode @@ -261,7 +262,7 @@ def safe_str(unicode_, to_encoding=None) """ # if it's not basestr cast to str - if not isinstance(unicode_, basestring): + if not isinstance(unicode_, compat.string_types): return str(unicode_) if isinstance(unicode_, str): @@ -564,6 +565,12 @@ def age(prevdate, now=None, show_short_v return _(u'just now') +def age_from_seconds(seconds): + seconds = safe_int(seconds) or 0 + prevdate = time_to_datetime(time.time() + seconds) + return age(prevdate, show_suffix=False, show_short_version=True) + + def cleaned_uri(uri): """ Quotes '[' and ']' from uri if there is only one of them. @@ -681,7 +688,7 @@ def datetime_to_time(dt): def time_to_datetime(tm): if tm: - if isinstance(tm, basestring): + if isinstance(tm, compat.string_types): try: tm = float(tm) except ValueError: @@ -691,7 +698,7 @@ def time_to_datetime(tm): def time_to_utcdatetime(tm): if tm: - if isinstance(tm, basestring): + if isinstance(tm, compat.string_types): try: tm = float(tm) except ValueError: @@ -1009,3 +1016,14 @@ def glob2re(pat): else: res = res + re.escape(c) return res + '\Z(?ms)' + + +def parse_byte_string(size_str): + match = re.match(r'(\d+)(MB|KB)', size_str, re.IGNORECASE) + if not match: + raise ValueError('Given size:%s is invalid, please make sure ' + 'to use format of (MB|KB)' % size_str) + + _parts = match.groups() + num, type_ = _parts + return long(num) * {'mb': 1024*1024, 'kb': 1024}[type_.lower()] diff --git a/rhodecode/lib/vcs/__init__.py b/rhodecode/lib/vcs/__init__.py --- a/rhodecode/lib/vcs/__init__.py +++ b/rhodecode/lib/vcs/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/vcs/backends/__init__.py b/rhodecode/lib/vcs/backends/__init__.py --- a/rhodecode/lib/vcs/backends/__init__.py +++ b/rhodecode/lib/vcs/backends/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/vcs/backends/base.py b/rhodecode/lib/vcs/backends/base.py --- a/rhodecode/lib/vcs/backends/base.py +++ b/rhodecode/lib/vcs/backends/base.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -21,20 +21,21 @@ """ Base module for all VCS systems """ - -import collections +import os +import re +import time +import shutil import datetime import fnmatch import itertools import logging -import os -import re -import time +import collections import warnings -import shutil from zope.cachedescriptors.property import Lazy as LazyProperty +from pyramid import compat +from rhodecode.translation import lazy_ugettext from rhodecode.lib.utils2 import safe_str, safe_unicode from rhodecode.lib.vcs import connection from rhodecode.lib.vcs.utils import author_name, author_email @@ -54,9 +55,6 @@ FILEMODE_DEFAULT = 0o100644 FILEMODE_EXECUTABLE = 0o100755 Reference = collections.namedtuple('Reference', ('type', 'name', 'commit_id')) -MergeResponse = collections.namedtuple( - 'MergeResponse', - ('possible', 'executed', 'merge_ref', 'failure_reason')) class MergeFailureReason(object): @@ -142,6 +140,93 @@ class UpdateFailureReason(object): MISSING_SOURCE_REF = 5 +class MergeResponse(object): + + # uses .format(**metadata) for variables + MERGE_STATUS_MESSAGES = { + MergeFailureReason.NONE: lazy_ugettext( + u'This pull request can be automatically merged.'), + MergeFailureReason.UNKNOWN: lazy_ugettext( + u'This pull request cannot be merged because of an unhandled exception. ' + u'{exception}'), + MergeFailureReason.MERGE_FAILED: lazy_ugettext( + u'This pull request cannot be merged because of merge conflicts.'), + MergeFailureReason.PUSH_FAILED: lazy_ugettext( + u'This pull request could not be merged because push to ' + u'target:`{target}@{merge_commit}` failed.'), + MergeFailureReason.TARGET_IS_NOT_HEAD: lazy_ugettext( + u'This pull request cannot be merged because the target ' + u'`{target_ref.name}` is not a head.'), + MergeFailureReason.HG_SOURCE_HAS_MORE_BRANCHES: lazy_ugettext( + u'This pull request cannot be merged because the source contains ' + u'more branches than the target.'), + MergeFailureReason.HG_TARGET_HAS_MULTIPLE_HEADS: lazy_ugettext( + u'This pull request cannot be merged because the target ' + u'has multiple heads: `{heads}`.'), + MergeFailureReason.TARGET_IS_LOCKED: lazy_ugettext( + u'This pull request cannot be merged because the target repository is ' + u'locked by {locked_by}.'), + + MergeFailureReason.MISSING_TARGET_REF: lazy_ugettext( + u'This pull request cannot be merged because the target ' + u'reference `{target_ref.name}` is missing.'), + MergeFailureReason.MISSING_SOURCE_REF: lazy_ugettext( + u'This pull request cannot be merged because the source ' + u'reference `{source_ref.name}` is missing.'), + MergeFailureReason.SUBREPO_MERGE_FAILED: lazy_ugettext( + u'This pull request cannot be merged because of conflicts related ' + u'to sub repositories.'), + + # Deprecations + MergeFailureReason._DEPRECATED_MISSING_COMMIT: lazy_ugettext( + u'This pull request cannot be merged because the target or the ' + u'source reference is missing.'), + + } + + def __init__(self, possible, executed, merge_ref, failure_reason, metadata=None): + self.possible = possible + self.executed = executed + self.merge_ref = merge_ref + self.failure_reason = failure_reason + self.metadata = metadata or {} + + def __repr__(self): + return ''.format(self.label, self.failure_reason) + + def __eq__(self, other): + same_instance = isinstance(other, self.__class__) + return same_instance \ + and self.possible == other.possible \ + and self.executed == other.executed \ + and self.failure_reason == other.failure_reason + + @property + def label(self): + label_dict = dict((v, k) for k, v in MergeFailureReason.__dict__.items() if + not k.startswith('_')) + return label_dict.get(self.failure_reason) + + @property + def merge_status_message(self): + """ + Return a human friendly error message for the given merge status code. + """ + msg = safe_unicode(self.MERGE_STATUS_MESSAGES[self.failure_reason]) + try: + return msg.format(**self.metadata) + except Exception: + log.exception('Failed to format %s message', self) + return msg + + def asdict(self): + data = {} + for k in ['possible', 'executed', 'merge_ref', 'failure_reason', + 'merge_status_message']: + data[k] = getattr(self, k) + return data + + class BaseRepository(object): """ Base Repository for final backends @@ -316,7 +401,7 @@ class BaseRepository(object): # COMMITS # ========================================================================== - def get_commit(self, commit_id=None, commit_idx=None, pre_load=None): + def get_commit(self, commit_id=None, commit_idx=None, pre_load=None, translate_tag=None): """ Returns instance of `BaseCommit` class. If `commit_id` and `commit_idx` are both None, most recent commit is returned. @@ -333,7 +418,7 @@ class BaseRepository(object): def get_commits( self, start_id=None, end_id=None, start_date=None, end_date=None, - branch_name=None, show_hidden=False, pre_load=None): + branch_name=None, show_hidden=False, pre_load=None, translate_tags=None): """ Returns iterator of `BaseCommit` objects from start to end not inclusive. This should behave just like a list, ie. end is not @@ -346,6 +431,7 @@ class BaseRepository(object): :param branch_name: :param show_hidden: :param pre_load: + :param translate_tags: """ raise NotImplementedError @@ -501,12 +587,11 @@ class BaseRepository(object): repo_id, workspace_id, target_ref, source_repo, source_ref, message, user_name, user_email, dry_run=dry_run, use_rebase=use_rebase, close_branch=close_branch) - except RepositoryError: - log.exception( - 'Unexpected failure when running merge, dry-run=%s', - dry_run) + except RepositoryError as exc: + log.exception('Unexpected failure when running merge, dry-run=%s', dry_run) return MergeResponse( - False, False, None, MergeFailureReason.UNKNOWN) + False, False, None, MergeFailureReason.UNKNOWN, + metadata={'exception': str(exc)}) def _merge_repo(self, repo_id, workspace_id, target_ref, source_repo, source_ref, merge_message, @@ -610,7 +695,7 @@ class BaseRepository(object): (commit, self, commit.repository)) def _validate_commit_id(self, commit_id): - if not isinstance(commit_id, basestring): + if not isinstance(commit_id, compat.string_types): raise TypeError("commit_id must be a string value") def _validate_commit_idx(self, commit_idx): @@ -647,7 +732,7 @@ class BaseRepository(object): warnings.warn("Use get_commit instead", DeprecationWarning) commit_id = None commit_idx = None - if isinstance(revision, basestring): + if isinstance(revision, compat.string_types): commit_id = revision else: commit_idx = revision @@ -674,7 +759,7 @@ class BaseRepository(object): if revision is None: return revision - if isinstance(revision, basestring): + if isinstance(revision, compat.string_types): commit_id = revision else: commit_id = self.commit_ids[revision] @@ -697,6 +782,9 @@ class BaseRepository(object): def install_hooks(self, force=False): return self._remote.install_hooks(force) + def get_hooks_info(self): + return self._remote.get_hooks_info() + class BaseCommit(object): """ @@ -1559,12 +1647,13 @@ class EmptyRepository(BaseRepository): class CollectionGenerator(object): - def __init__(self, repo, commit_ids, collection_size=None, pre_load=None): + def __init__(self, repo, commit_ids, collection_size=None, pre_load=None, translate_tag=None): self.repo = repo self.commit_ids = commit_ids # TODO: (oliver) this isn't currently hooked up self.collection_size = None self.pre_load = pre_load + self.translate_tag = translate_tag def __len__(self): if self.collection_size is not None: @@ -1580,8 +1669,9 @@ class CollectionGenerator(object): """ Allows backends to override the way commits are generated. """ - return self.repo.get_commit(commit_id=commit_id, - pre_load=self.pre_load) + return self.repo.get_commit( + commit_id=commit_id, pre_load=self.pre_load, + translate_tag=self.translate_tag) def __getslice__(self, i, j): """ @@ -1589,7 +1679,8 @@ class CollectionGenerator(object): """ commit_ids = self.commit_ids[i:j] return self.__class__( - self.repo, commit_ids, pre_load=self.pre_load) + self.repo, commit_ids, pre_load=self.pre_load, + translate_tag=self.translate_tag) def __repr__(self): return '' % (self.__len__()) diff --git a/rhodecode/lib/vcs/backends/git/__init__.py b/rhodecode/lib/vcs/backends/git/__init__.py --- a/rhodecode/lib/vcs/backends/git/__init__.py +++ b/rhodecode/lib/vcs/backends/git/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/vcs/backends/git/commit.py b/rhodecode/lib/vcs/backends/git/commit.py --- a/rhodecode/lib/vcs/backends/git/commit.py +++ b/rhodecode/lib/vcs/backends/git/commit.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -269,7 +269,8 @@ class GitCommit(base.BaseCommit): def _make_commits(self, commit_ids, pre_load=None): return [ - self.repository.get_commit(commit_id=commit_id, pre_load=pre_load) + self.repository.get_commit(commit_id=commit_id, pre_load=pre_load, + translate_tag=False) for commit_id in commit_ids] def get_file_mode(self, path): @@ -309,10 +310,14 @@ class GitCommit(base.BaseCommit): self._get_filectx(path) f_path = safe_str(path) - cmd = ['log'] - if limit: - cmd.extend(['-n', str(safe_int(limit, 0))]) - cmd.extend(['--pretty=format: %H', '-s', self.raw_id, '--', f_path]) + # optimize for n==1, rev-list is much faster for that use-case + if limit == 1: + cmd = ['rev-list', '-1', self.raw_id, '--', f_path] + else: + cmd = ['log'] + if limit: + cmd.extend(['-n', str(safe_int(limit, 0))]) + cmd.extend(['--pretty=format: %H', '-s', self.raw_id, '--', f_path]) output, __ = self.repository.run_git_command(cmd) commit_ids = re.findall(r'[0-9a-fA-F]{40}', output) diff --git a/rhodecode/lib/vcs/backends/git/diff.py b/rhodecode/lib/vcs/backends/git/diff.py --- a/rhodecode/lib/vcs/backends/git/diff.py +++ b/rhodecode/lib/vcs/backends/git/diff.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/vcs/backends/git/inmemory.py b/rhodecode/lib/vcs/backends/git/inmemory.py --- a/rhodecode/lib/vcs/backends/git/inmemory.py +++ b/rhodecode/lib/vcs/backends/git/inmemory.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/vcs/backends/git/repository.py b/rhodecode/lib/vcs/backends/git/repository.py --- a/rhodecode/lib/vcs/backends/git/repository.py +++ b/rhodecode/lib/vcs/backends/git/repository.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -426,7 +426,7 @@ class GitRepository(BaseRepository): except Exception: return - def get_commit(self, commit_id=None, commit_idx=None, pre_load=None): + def get_commit(self, commit_id=None, commit_idx=None, pre_load=None, translate_tag=True): """ Returns `GitCommit` object representing commit from git repository at the given `commit_id` or head (most recent commit) if None given. @@ -438,8 +438,9 @@ class GitRepository(BaseRepository): commit_id = commit_idx commit_id = self._get_commit_id(commit_id) try: - # Need to call remote to translate id for tagging scenario - commit_id = self._remote.get_object(commit_id)["commit_id"] + if translate_tag: + # Need to call remote to translate id for tagging scenario + commit_id = self._remote.get_object(commit_id)["commit_id"] idx = self._commit_ids[commit_id] except KeyError: raise RepositoryError("Cannot get object with id %s" % commit_id) @@ -448,7 +449,7 @@ class GitRepository(BaseRepository): def get_commits( self, start_id=None, end_id=None, start_date=None, end_date=None, - branch_name=None, show_hidden=False, pre_load=None): + branch_name=None, show_hidden=False, pre_load=None, translate_tags=True): """ Returns generator of `GitCommit` objects from start to end (both are inclusive), in ascending date order. @@ -528,7 +529,8 @@ class GitRepository(BaseRepository): if start_pos or end_pos: commit_ids = commit_ids[start_pos: end_pos] - return CollectionGenerator(self, commit_ids, pre_load=pre_load) + return CollectionGenerator(self, commit_ids, pre_load=pre_load, + translate_tag=translate_tags) def get_diff( self, commit1, commit2, path='', ignore_whitespace=False, @@ -911,11 +913,15 @@ class GitRepository(BaseRepository): source_repo, source_ref, merge_message, merger_name, merger_email, dry_run=False, use_rebase=False, close_branch=False): + + log.debug('Executing merge_repo with %s strategy, dry_run mode:%s', + 'rebase' if use_rebase else 'merge', dry_run) if target_ref.commit_id != self.branches[target_ref.name]: log.warning('Target ref %s commit mismatch %s vs %s', target_ref, target_ref.commit_id, self.branches[target_ref.name]) return MergeResponse( - False, False, None, MergeFailureReason.TARGET_IS_NOT_HEAD) + False, False, None, MergeFailureReason.TARGET_IS_NOT_HEAD, + metadata={'target_ref': target_ref}) shadow_repository_path = self._maybe_prepare_merge_workspace( repo_id, workspace_id, target_ref, source_ref) @@ -943,7 +949,8 @@ class GitRepository(BaseRepository): target_ref, target_ref.commit_id, shadow_repo.branches[target_ref.name]) return MergeResponse( - False, False, None, MergeFailureReason.TARGET_IS_NOT_HEAD) + False, False, None, MergeFailureReason.TARGET_IS_NOT_HEAD, + metadata={'target_ref': target_ref}) # calculate new branch pr_branch = shadow_repo._get_new_pr_branch( @@ -954,12 +961,15 @@ class GitRepository(BaseRepository): try: shadow_repo._local_fetch(source_repo.path, source_ref.name) except RepositoryError: - log.exception('Failure when doing local fetch on git shadow repo') + log.exception('Failure when doing local fetch on ' + 'shadow repo: %s', shadow_repo) return MergeResponse( - False, False, None, MergeFailureReason.MISSING_SOURCE_REF) + False, False, None, MergeFailureReason.MISSING_SOURCE_REF, + metadata={'source_ref': source_ref}) merge_ref = None merge_failure_reason = MergeFailureReason.NONE + metadata = {} try: shadow_repo._local_merge(merge_message, merger_name, merger_email, [source_ref.commit_id]) @@ -988,12 +998,15 @@ class GitRepository(BaseRepository): merge_succeeded = True except RepositoryError: log.exception( - 'Failure when doing local push on git shadow repo') + 'Failure when doing local push from the shadow ' + 'repository to the target repository at %s.', self.path) merge_succeeded = False merge_failure_reason = MergeFailureReason.PUSH_FAILED + metadata['target'] = 'git shadow repo' + metadata['merge_commit'] = pr_branch else: merge_succeeded = False return MergeResponse( - merge_possible, merge_succeeded, merge_ref, - merge_failure_reason) + merge_possible, merge_succeeded, merge_ref, merge_failure_reason, + metadata=metadata) diff --git a/rhodecode/lib/vcs/backends/hg/__init__.py b/rhodecode/lib/vcs/backends/hg/__init__.py --- a/rhodecode/lib/vcs/backends/hg/__init__.py +++ b/rhodecode/lib/vcs/backends/hg/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/vcs/backends/hg/commit.py b/rhodecode/lib/vcs/backends/hg/commit.py --- a/rhodecode/lib/vcs/backends/hg/commit.py +++ b/rhodecode/lib/vcs/backends/hg/commit.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -303,10 +303,10 @@ class MercurialCommit(base.BaseCommit): alias = self.repository.alias for k, vals in self._submodules.iteritems(): - loc = vals[0] - commit = vals[1] - dirnodes.append( - SubModuleNode(k, url=loc, commit=commit, alias=alias)) + if vcspath.dirname(k) == path: + loc = vals[0] + commit = vals[1] + dirnodes.append(SubModuleNode(k, url=loc, commit=commit, alias=alias)) nodes = dirnodes + filenodes # cache nodes for node in nodes: diff --git a/rhodecode/lib/vcs/backends/hg/diff.py b/rhodecode/lib/vcs/backends/hg/diff.py --- a/rhodecode/lib/vcs/backends/hg/diff.py +++ b/rhodecode/lib/vcs/backends/hg/diff.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/vcs/backends/hg/inmemory.py b/rhodecode/lib/vcs/backends/hg/inmemory.py --- a/rhodecode/lib/vcs/backends/hg/inmemory.py +++ b/rhodecode/lib/vcs/backends/hg/inmemory.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/vcs/backends/hg/repository.py b/rhodecode/lib/vcs/backends/hg/repository.py --- a/rhodecode/lib/vcs/backends/hg/repository.py +++ b/rhodecode/lib/vcs/backends/hg/repository.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -414,7 +414,7 @@ class MercurialRepository(BaseRepository """ return os.path.join(self.path, '.hg', '.hgrc') - def get_commit(self, commit_id=None, commit_idx=None, pre_load=None): + def get_commit(self, commit_id=None, commit_idx=None, pre_load=None, translate_tag=None): """ Returns ``MercurialCommit`` object representing repository's commit at the given `commit_id` or `commit_idx`. @@ -456,7 +456,7 @@ class MercurialRepository(BaseRepository def get_commits( self, start_id=None, end_id=None, start_date=None, end_date=None, - branch_name=None, show_hidden=False, pre_load=None): + branch_name=None, show_hidden=False, pre_load=None, translate_tags=None): """ Returns generator of ``MercurialCommit`` objects from start to end (both are inclusive) @@ -610,7 +610,7 @@ class MercurialRepository(BaseRepository Returns the commit id of the merge and a boolean indicating if the commit needs to be pushed. """ - self._update(target_ref.commit_id) + self._update(target_ref.commit_id, clean=True) ancestor = self._ancestor(target_ref.commit_id, source_ref.commit_id) is_the_same_branch = self._is_the_same_branch(target_ref, source_ref) @@ -631,7 +631,7 @@ class MercurialRepository(BaseRepository self._remote.rebase( source=source_ref.commit_id, dest=target_ref.commit_id) self._remote.invalidate_vcs_cache() - self._update(bookmark_name) + self._update(bookmark_name, clean=True) return self._identify(), True except RepositoryError: # The rebase-abort may raise another exception which 'hides' @@ -710,18 +710,21 @@ class MercurialRepository(BaseRepository 'rebase' if use_rebase else 'merge', dry_run) if target_ref.commit_id not in self._heads(): return MergeResponse( - False, False, None, MergeFailureReason.TARGET_IS_NOT_HEAD) + False, False, None, MergeFailureReason.TARGET_IS_NOT_HEAD, + metadata={'target_ref': target_ref}) try: - if (target_ref.type == 'branch' and - len(self._heads(target_ref.name)) != 1): + if target_ref.type == 'branch' and len(self._heads(target_ref.name)) != 1: + heads = ','.join(self._heads(target_ref.name)) return MergeResponse( False, False, None, - MergeFailureReason.HG_TARGET_HAS_MULTIPLE_HEADS) + MergeFailureReason.HG_TARGET_HAS_MULTIPLE_HEADS, + metadata={'heads': heads}) except CommitDoesNotExistError: log.exception('Failure when looking up branch heads on hg target') return MergeResponse( - False, False, None, MergeFailureReason.MISSING_TARGET_REF) + False, False, None, MergeFailureReason.MISSING_TARGET_REF, + metadata={'target_ref': target_ref}) shadow_repository_path = self._maybe_prepare_merge_workspace( repo_id, workspace_id, target_ref, source_ref) @@ -730,6 +733,7 @@ class MercurialRepository(BaseRepository log.debug('Pulling in target reference %s', target_ref) self._validate_pull_reference(target_ref) shadow_repo._local_pull(self.path, target_ref) + try: log.debug('Pulling in source reference %s', source_ref) source_repo._validate_pull_reference(source_ref) @@ -737,12 +741,14 @@ class MercurialRepository(BaseRepository except CommitDoesNotExistError: log.exception('Failure when doing local pull on hg shadow repo') return MergeResponse( - False, False, None, MergeFailureReason.MISSING_SOURCE_REF) + False, False, None, MergeFailureReason.MISSING_SOURCE_REF, + metadata={'source_ref': source_ref}) merge_ref = None merge_commit_id = None close_commit_id = None merge_failure_reason = MergeFailureReason.NONE + metadata = {} # enforce that close branch should be used only in case we source from # an actual Branch @@ -758,8 +764,8 @@ class MercurialRepository(BaseRepository target_ref, merger_name, merger_email, source_ref) merge_possible = True except RepositoryError: - log.exception( - 'Failure when doing close branch on hg shadow repo') + log.exception('Failure when doing close branch on ' + 'shadow repo: %s', shadow_repo) merge_possible = False merge_failure_reason = MergeFailureReason.MERGE_FAILED else: @@ -824,19 +830,21 @@ class MercurialRepository(BaseRepository except RepositoryError: log.exception( 'Failure when doing local push from the shadow ' - 'repository to the target repository.') + 'repository to the target repository at %s.', self.path) merge_succeeded = False merge_failure_reason = MergeFailureReason.PUSH_FAILED + metadata['target'] = 'hg shadow repo' + metadata['merge_commit'] = merge_commit_id else: merge_succeeded = True else: merge_succeeded = False return MergeResponse( - merge_possible, merge_succeeded, merge_ref, merge_failure_reason) + merge_possible, merge_succeeded, merge_ref, merge_failure_reason, + metadata=metadata) - def _get_shadow_instance( - self, shadow_repository_path, enable_hooks=False): + def _get_shadow_instance(self, shadow_repository_path, enable_hooks=False): config = self.config.copy() if not enable_hooks: config.clear_section('hooks') diff --git a/rhodecode/lib/vcs/backends/svn/__init__.py b/rhodecode/lib/vcs/backends/svn/__init__.py --- a/rhodecode/lib/vcs/backends/svn/__init__.py +++ b/rhodecode/lib/vcs/backends/svn/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/vcs/backends/svn/commit.py b/rhodecode/lib/vcs/backends/svn/commit.py --- a/rhodecode/lib/vcs/backends/svn/commit.py +++ b/rhodecode/lib/vcs/backends/svn/commit.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/vcs/backends/svn/diff.py b/rhodecode/lib/vcs/backends/svn/diff.py --- a/rhodecode/lib/vcs/backends/svn/diff.py +++ b/rhodecode/lib/vcs/backends/svn/diff.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/vcs/backends/svn/inmemory.py b/rhodecode/lib/vcs/backends/svn/inmemory.py --- a/rhodecode/lib/vcs/backends/svn/inmemory.py +++ b/rhodecode/lib/vcs/backends/svn/inmemory.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/vcs/backends/svn/repository.py b/rhodecode/lib/vcs/backends/svn/repository.py --- a/rhodecode/lib/vcs/backends/svn/repository.py +++ b/rhodecode/lib/vcs/backends/svn/repository.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -250,7 +250,7 @@ class SubversionRepository(base.BaseRepo """ return os.path.join(self.path, 'hooks') - def get_commit(self, commit_id=None, commit_idx=None, pre_load=None): + def get_commit(self, commit_id=None, commit_idx=None, pre_load=None, translate_tag=None): if self.is_empty(): raise EmptyRepositoryError("There are no commits yet") if commit_id is not None: @@ -268,7 +268,7 @@ class SubversionRepository(base.BaseRepo def get_commits( self, start_id=None, end_id=None, start_date=None, end_date=None, - branch_name=None, show_hidden=False, pre_load=None): + branch_name=None, show_hidden=False, pre_load=None, translate_tags=None): if self.is_empty(): raise EmptyRepositoryError("There are no commit_ids yet") self._validate_branch_name(branch_name) diff --git a/rhodecode/lib/vcs/client_http.py b/rhodecode/lib/vcs/client_http.py --- a/rhodecode/lib/vcs/client_http.py +++ b/rhodecode/lib/vcs/client_http.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -35,7 +35,9 @@ import msgpack import requests from requests.packages.urllib3.util.retry import Retry -from . import exceptions, CurlSession +import rhodecode +from rhodecode.lib.system_info import get_cert_path +from rhodecode.lib.vcs import exceptions, CurlSession log = logging.getLogger(__name__) @@ -121,6 +123,8 @@ class RemoteRepo(object): if log.isEnabledFor(logging.DEBUG): self._call = self._call_with_logging + self.cert_dir = get_cert_path(rhodecode.CONFIG.get('__file__')) + def __getattr__(self, name): def f(*args, **kwargs): return self._call(name, *args, **kwargs) @@ -132,6 +136,8 @@ class RemoteRepo(object): # config object is being changed for hooking scenarios wire = copy.deepcopy(self._wire) wire["config"] = wire["config"].serialize() + + wire["config"].append(('vcs', 'ssl_dir', self.cert_dir)) payload = { 'id': str(uuid.uuid4()), 'method': name, @@ -231,6 +237,8 @@ def _remote_call(url, payload, exception try: exc._vcs_server_traceback = error['traceback'] + exc._vcs_server_org_exc_name = error['org_exc'] + exc._vcs_server_org_exc_tb = error['org_exc_tb'] except KeyError: pass @@ -243,8 +251,6 @@ class VcsHttpProxy(object): CHUNK_SIZE = 16384 def __init__(self, server_and_port, backend_endpoint): - - retries = Retry(total=5, connect=None, read=None, redirect=None) adapter = requests.adapters.HTTPAdapter(max_retries=retries) diff --git a/rhodecode/lib/vcs/conf/__init__.py b/rhodecode/lib/vcs/conf/__init__.py --- a/rhodecode/lib/vcs/conf/__init__.py +++ b/rhodecode/lib/vcs/conf/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/vcs/conf/mtypes.py b/rhodecode/lib/vcs/conf/mtypes.py --- a/rhodecode/lib/vcs/conf/mtypes.py +++ b/rhodecode/lib/vcs/conf/mtypes.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/vcs/conf/settings.py b/rhodecode/lib/vcs/conf/settings.py --- a/rhodecode/lib/vcs/conf/settings.py +++ b/rhodecode/lib/vcs/conf/settings.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/vcs/connection.py b/rhodecode/lib/vcs/connection.py --- a/rhodecode/lib/vcs/connection.py +++ b/rhodecode/lib/vcs/connection.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/vcs/exceptions.py b/rhodecode/lib/vcs/exceptions.py --- a/rhodecode/lib/vcs/exceptions.py +++ b/rhodecode/lib/vcs/exceptions.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -24,6 +24,8 @@ Custom vcs exceptions module. import logging import functools import urllib2 +import rhodecode +from pyramid import compat log = logging.getLogger(__name__) @@ -180,17 +182,26 @@ def map_vcs_exceptions(func): try: return func(*args, **kwargs) except Exception as e: + from rhodecode.lib.utils2 import str2bool + debug = str2bool(rhodecode.CONFIG.get('debug')) + # The error middleware adds information if it finds # __traceback_info__ in a frame object. This way the remote # traceback information is made available in error reports. remote_tb = getattr(e, '_vcs_server_traceback', None) + org_remote_tb = getattr(e, '_vcs_server_org_exc_tb', '') __traceback_info__ = None if remote_tb: - if isinstance(remote_tb, basestring): + if isinstance(remote_tb, compat.string_types): remote_tb = [remote_tb] __traceback_info__ = ( - 'Found VCSServer remote traceback information:\n\n' + - '\n'.join(remote_tb)) + 'Found VCSServer remote traceback information:\n' + '{}\n' + '+++ BEG SOURCE EXCEPTION +++\n\n' + '{}\n' + '+++ END SOURCE EXCEPTION +++\n' + ''.format('\n'.join(remote_tb), org_remote_tb) + ) # Avoid that remote_tb also appears in the frame del remote_tb @@ -205,9 +216,9 @@ def map_vcs_exceptions(func): args = e.args else: args = [__traceback_info__ or 'unhandledException'] - if __traceback_info__ and kind not in ['unhandled', 'lookup']: + if debug or __traceback_info__ and kind not in ['unhandled', 'lookup']: # for other than unhandled errors also log the traceback - # can be usefull for debugging + # can be useful for debugging log.error(__traceback_info__) raise _EXCEPTION_MAP[kind](*args) else: diff --git a/rhodecode/lib/vcs/geventcurl.py b/rhodecode/lib/vcs/geventcurl.py --- a/rhodecode/lib/vcs/geventcurl.py +++ b/rhodecode/lib/vcs/geventcurl.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/vcs/nodes.py b/rhodecode/lib/vcs/nodes.py --- a/rhodecode/lib/vcs/nodes.py +++ b/rhodecode/lib/vcs/nodes.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -200,8 +200,12 @@ class Node(object): """ Comparator using name of the node, needed for quick list sorting. """ + kind_cmp = cmp(self.kind, other.kind) if kind_cmp: + if isinstance(self, SubModuleNode): + # we make submodules equal to dirnode for "sorting" purposes + return NodeKind.DIR return kind_cmp return cmp(self.name, other.name) @@ -372,6 +376,31 @@ class FileNode(Node): """ return md5(self.raw_bytes) + def metadata_uncached(self): + """ + Returns md5, binary flag of the file node, without any cache usage. + """ + + content = self.content_uncached() + + is_binary = content and '\0' in content + size = 0 + if content: + size = len(content) + + return is_binary, md5(content), size, content + + def content_uncached(self): + """ + Returns lazily content of the FileNode. If possible, would try to + decode content from UTF-8. + """ + if self.commit: + content = self.commit.get_file_content(self.path) + else: + content = self._content + return content + @LazyProperty def content(self): """ diff --git a/rhodecode/lib/vcs/path.py b/rhodecode/lib/vcs/path.py --- a/rhodecode/lib/vcs/path.py +++ b/rhodecode/lib/vcs/path.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/vcs/utils/__init__.py b/rhodecode/lib/vcs/utils/__init__.py --- a/rhodecode/lib/vcs/utils/__init__.py +++ b/rhodecode/lib/vcs/utils/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -58,7 +58,7 @@ def author_name(author): to get the username """ - if not author or not '@' in author: + if not author or '@' not in author: return author else: return author.replace(author_email(author), '').replace('<', '')\ diff --git a/rhodecode/lib/vcs/utils/helpers.py b/rhodecode/lib/vcs/utils/helpers.py --- a/rhodecode/lib/vcs/utils/helpers.py +++ b/rhodecode/lib/vcs/utils/helpers.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2014-2018 RhodeCode GmbH +# Copyright (C) 2014-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/vcs/utils/imports.py b/rhodecode/lib/vcs/utils/imports.py --- a/rhodecode/lib/vcs/utils/imports.py +++ b/rhodecode/lib/vcs/utils/imports.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/lib/vcs/utils/paths.py b/rhodecode/lib/vcs/utils/paths.py --- a/rhodecode/lib/vcs/utils/paths.py +++ b/rhodecode/lib/vcs/utils/paths.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/controllers/utils.py b/rhodecode/lib/view_utils.py rename from rhodecode/controllers/utils.py rename to rhodecode/lib/view_utils.py --- a/rhodecode/controllers/utils.py +++ b/rhodecode/lib/view_utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/__init__.py b/rhodecode/model/__init__.py --- a/rhodecode/model/__init__.py +++ b/rhodecode/model/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/auth_token.py b/rhodecode/model/auth_token.py --- a/rhodecode/model/auth_token.py +++ b/rhodecode/model/auth_token.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2013-2018 RhodeCode GmbH +# Copyright (C) 2013-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/changeset_status.py b/rhodecode/model/changeset_status.py --- a/rhodecode/model/changeset_status.py +++ b/rhodecode/model/changeset_status.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/comment.py b/rhodecode/model/comment.py --- a/rhodecode/model/comment.py +++ b/rhodecode/model/comment.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -125,7 +125,35 @@ class CommentsModel(BaseModel): return comment_versions - def get_unresolved_todos(self, pull_request, show_outdated=True): + def get_repository_comments(self, repo, comment_type=None, user=None, commit_id=None): + qry = Session().query(ChangesetComment) \ + .filter(ChangesetComment.repo == repo) + + if comment_type and comment_type in ChangesetComment.COMMENT_TYPES: + qry = qry.filter(ChangesetComment.comment_type == comment_type) + + if user: + user = self._get_user(user) + if user: + qry = qry.filter(ChangesetComment.user_id == user.user_id) + + if commit_id: + qry = qry.filter(ChangesetComment.revision == commit_id) + + qry = qry.order_by(ChangesetComment.created_on) + return qry.all() + + def get_repository_unresolved_todos(self, repo): + todos = Session().query(ChangesetComment) \ + .filter(ChangesetComment.repo == repo) \ + .filter(ChangesetComment.resolved_by == None) \ + .filter(ChangesetComment.comment_type + == ChangesetComment.COMMENT_TYPE_TODO) + todos = todos.all() + + return todos + + def get_pull_request_unresolved_todos(self, pull_request, show_outdated=True): todos = Session().query(ChangesetComment) \ .filter(ChangesetComment.pull_request == pull_request) \ diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py --- a/rhodecode/model/db.py +++ b/rhodecode/model/db.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -48,7 +48,7 @@ from sqlalchemy.ext.hybrid import hybrid from sqlalchemy.exc import IntegrityError # pragma: no cover from sqlalchemy.dialects.mysql import LONGTEXT from zope.cachedescriptors.property import Lazy as LazyProperty - +from pyramid import compat from pyramid.threadlocal import get_current_request from rhodecode.translation import _ @@ -732,8 +732,6 @@ class User(Base, BaseModel): if not auth_token: return False - crypto_backend = auth.crypto_backend() - roles = (roles or []) + [UserApiKeys.ROLE_ALL] tokens_q = UserApiKeys.query()\ .filter(UserApiKeys.user_id == self.user_id)\ @@ -742,39 +740,42 @@ class User(Base, BaseModel): tokens_q = tokens_q.filter(UserApiKeys.role.in_(roles)) - plain_tokens = [] - hash_tokens = [] - - user_tokens = tokens_q.all() - log.debug('Found %s user tokens to check for authentication', len(user_tokens)) - for token in user_tokens: - log.debug('AUTH_TOKEN: checking if user token with id `%s` matches', - token.user_api_key_id) - # verify scope first, since it's way faster than hash calculation of - # encrypted tokens - if token.repo_id: - # token has a scope, we need to verify it - if scope_repo_id != token.repo_id: + crypto_backend = auth.crypto_backend() + enc_token_map = {} + plain_token_map = {} + for token in tokens_q: + if token.api_key.startswith(crypto_backend.ENC_PREF): + enc_token_map[token.api_key] = token + else: + plain_token_map[token.api_key] = token + log.debug( + 'Found %s plain and %s encrypted user tokens to check for authentication', + len(plain_token_map), len(enc_token_map)) + + # plain token match comes first + match = plain_token_map.get(auth_token) + + # check encrypted tokens now + if not match: + for token_hash, token in enc_token_map.items(): + # NOTE(marcink): this is expensive to calculate, but most secure + if crypto_backend.hash_check(auth_token, token_hash): + match = token + break + + if match: + log.debug('Found matching token %s', match) + if match.repo_id: + log.debug('Found scope, checking for scope match of token %s', match) + if match.repo_id == scope_repo_id: + return True + else: log.debug( 'AUTH_TOKEN: scope mismatch, token has a set repo scope: %s, ' 'and calling scope is:%s, skipping further checks', - token.repo, scope_repo_id) - # token has a scope, and it doesn't match, skip token - continue - - if token.api_key.startswith(crypto_backend.ENC_PREF): - hash_tokens.append(token.api_key) + match.repo, scope_repo_id) + return False else: - plain_tokens.append(token.api_key) - - is_plain_match = auth_token in plain_tokens - if is_plain_match: - return True - - for hashed in hash_tokens: - # NOTE(marcink): this is expensive to calculate, but most secure - match = crypto_backend.hash_check(auth_token, hashed) - if match: return True return False @@ -1130,10 +1131,10 @@ class UserApiKeys(Base, BaseModel): def _get_scope(self): if self.repo: - return repr(self.repo) + return 'Repository: {}'.format(self.repo.repo_name) if self.repo_group: - return repr(self.repo_group) + ' (recursive)' - return 'global' + return 'RepositoryGroup: {} (recursive)'.format(self.repo_group.group_name) + return 'Global' @property def scope_humanized(self): @@ -2240,7 +2241,7 @@ class Repository(Base, BaseModel): warnings.warn("Use get_commit", DeprecationWarning) commit_id = None commit_idx = None - if isinstance(rev, basestring): + if isinstance(rev, compat.string_types): commit_id = rev else: commit_idx = rev @@ -2276,7 +2277,7 @@ class Repository(Base, BaseModel): # use no-cache version here scm_repo = self.scm_instance(cache=False, config=config) - empty = scm_repo.is_empty() + empty = not scm_repo or scm_repo.is_empty() if not empty: cs_cache = scm_repo.get_commit( pre_load=["author", "date", "message", "parents"]) @@ -2459,7 +2460,6 @@ class RepoGroup(Base, BaseModel): __tablename__ = 'groups' __table_args__ = ( UniqueConstraint('group_name', 'group_parent_id'), - CheckConstraint('group_id != group_parent_id'), base_table_args, ) __mapper_args__ = {'order_by': 'group_name'} @@ -2480,8 +2480,7 @@ class RepoGroup(Base, BaseModel): users_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all') parent_group = relationship('RepoGroup', remote_side=group_id) user = relationship('User') - integrations = relationship('Integration', - cascade="all, delete, delete-orphan") + integrations = relationship('Integration', cascade="all, delete, delete-orphan") def __init__(self, group_name='', parent_group=None): self.group_name = group_name @@ -2491,6 +2490,16 @@ class RepoGroup(Base, BaseModel): return u"<%s('id:%s:%s')>" % ( self.__class__.__name__, self.group_id, self.group_name) + @validates('group_parent_id') + def validate_group_parent_id(self, key, val): + """ + Check cycle references for a parent group to self + """ + if self.group_id and val: + assert val != self.group_id + + return val + @hybrid_property def description_safe(self): from rhodecode.lib import helpers as h @@ -3411,7 +3420,10 @@ class ChangesetComment(Base, BaseModel): comment_type = Column('comment_type', Unicode(128), nullable=True, default=COMMENT_TYPE_NOTE) resolved_comment_id = Column('resolved_comment_id', Integer(), ForeignKey('changeset_comments.comment_id'), nullable=True) - resolved_comment = relationship('ChangesetComment', remote_side=comment_id, backref='resolved_by') + + resolved_comment = relationship('ChangesetComment', remote_side=comment_id, back_populates='resolved_by') + resolved_by = relationship('ChangesetComment', back_populates='resolved_comment') + author = relationship('User', lazy='joined') repo = relationship('Repository') status_change = relationship('ChangesetStatus', cascade="all, delete, delete-orphan", lazy='joined') @@ -3494,7 +3506,8 @@ class ChangesetComment(Base, BaseModel): 'comment_f_path': comment.f_path, 'comment_lineno': comment.line_no, 'comment_author': comment.author, - 'comment_created_on': comment.created_on + 'comment_created_on': comment.created_on, + 'comment_resolved_by': self.resolved } return data @@ -3568,6 +3581,32 @@ class ChangesetStatus(Base, BaseModel): return data +class _SetState(object): + """ + Context processor allowing changing state for sensitive operation such as + pull request update or merge + """ + + def __init__(self, pull_request, pr_state, back_state=None): + self._pr = pull_request + self._org_state = back_state or pull_request.pull_request_state + self._pr_state = pr_state + + def __enter__(self): + log.debug('StateLock: entering set state context, setting state to: `%s`', + self._pr_state) + self._pr.pull_request_state = self._pr_state + Session().add(self._pr) + Session().commit() + + def __exit__(self, exc_type, exc_val, exc_tb): + log.debug('StateLock: exiting set state context, setting state to: `%s`', + self._org_state) + self._pr.pull_request_state = self._org_state + Session().add(self._pr) + Session().commit() + + class _PullRequestBase(BaseModel): """ Common attributes of pull request and version entries. @@ -3578,6 +3617,12 @@ class _PullRequestBase(BaseModel): STATUS_OPEN = u'open' STATUS_CLOSED = u'closed' + # available states + STATE_CREATING = u'creating' + STATE_UPDATING = u'updating' + STATE_MERGING = u'merging' + STATE_CREATED = u'created' + title = Column('title', Unicode(255), nullable=True) description = Column( 'description', UnicodeText().with_variant(UnicodeText(10240), 'mysql'), @@ -3593,6 +3638,8 @@ class _PullRequestBase(BaseModel): 'updated_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + pull_request_state = Column("pull_request_state", String(255), nullable=True) + @declared_attr def user_id(cls): return Column( @@ -3610,7 +3657,33 @@ class _PullRequestBase(BaseModel): 'org_repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False) - source_ref = Column('org_ref', Unicode(255), nullable=False) + _source_ref = Column('org_ref', Unicode(255), nullable=False) + + @hybrid_property + def source_ref(self): + return self._source_ref + + @source_ref.setter + def source_ref(self, val): + parts = (val or '').split(':') + if len(parts) != 3: + raise ValueError( + 'Invalid reference format given: {}, expected X:Y:Z'.format(val)) + self._source_ref = safe_unicode(val) + + _target_ref = Column('other_ref', Unicode(255), nullable=False) + + @hybrid_property + def target_ref(self): + return self._target_ref + + @target_ref.setter + def target_ref(self, val): + parts = (val or '').split(':') + if len(parts) != 3: + raise ValueError( + 'Invalid reference format given: {}, expected X:Y:Z'.format(val)) + self._target_ref = safe_unicode(val) @declared_attr def target_repo_id(cls): @@ -3619,7 +3692,6 @@ class _PullRequestBase(BaseModel): 'other_repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False) - target_ref = Column('other_ref', Unicode(255), nullable=False) _shadow_merge_ref = Column('shadow_merge_ref', Unicode(255), nullable=True) # TODO: dan: rename column to last_merge_source_rev @@ -3692,7 +3764,8 @@ class _PullRequestBase(BaseModel): def shadow_merge_ref(self, ref): self._shadow_merge_ref = self.reference_to_unicode(ref) - def unicode_to_reference(self, raw): + @staticmethod + def unicode_to_reference(raw): """ Convert a unicode (or string) to a reference object. If unicode evaluates to False it returns None. @@ -3703,7 +3776,8 @@ class _PullRequestBase(BaseModel): else: return None - def reference_to_unicode(self, ref): + @staticmethod + def reference_to_unicode(ref): """ Convert a reference object to unicode. If reference is None it returns None. @@ -3740,6 +3814,7 @@ class _PullRequestBase(BaseModel): 'title': pull_request.title, 'description': pull_request.description, 'status': pull_request.status, + 'state': pull_request.pull_request_state, 'created_on': pull_request.created_on, 'updated_on': pull_request.updated_on, 'commit_ids': pull_request.revisions, @@ -3780,6 +3855,20 @@ class _PullRequestBase(BaseModel): return data + def set_state(self, pull_request_state, final_state=None): + """ + # goes from initial state to updating to initial state. + # initial state can be changed by specifying back_state= + with pull_request_obj.set_state(PullRequest.STATE_UPDATING): + pull_request.merge() + + :param pull_request_state: + :param final_state: + + """ + + return _SetState(self, pull_request_state, back_state=final_state) + class PullRequest(Base, _PullRequestBase): __tablename__ = 'pull_requests' @@ -3952,7 +4041,7 @@ class PullRequestReviewers(Base, BaseMod @reasons.setter def reasons(self, val): val = val or [] - if any(not isinstance(x, basestring) for x in val): + if any(not isinstance(x, compat.string_types) for x in val): raise Exception('invalid reasons type, must be list of strings') self._reasons = val @@ -4718,6 +4807,135 @@ class UserGroupToRepoBranchPermission(Ba self.user_group_repo_to_perm, self.branch_pattern) +class UserBookmark(Base, BaseModel): + __tablename__ = 'user_bookmarks' + __table_args__ = ( + UniqueConstraint('user_id', 'bookmark_repo_id'), + UniqueConstraint('user_id', 'bookmark_repo_group_id'), + UniqueConstraint('user_id', 'bookmark_position'), + base_table_args + ) + + user_bookmark_id = Column("user_bookmark_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) + user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) + position = Column("bookmark_position", Integer(), nullable=False) + title = Column("bookmark_title", String(255), nullable=True, unique=None, default=None) + redirect_url = Column("bookmark_redirect_url", String(10240), nullable=True, unique=None, default=None) + created_on = Column("created_on", DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + + bookmark_repo_id = Column("bookmark_repo_id", Integer(), ForeignKey("repositories.repo_id"), nullable=True, unique=None, default=None) + bookmark_repo_group_id = Column("bookmark_repo_group_id", Integer(), ForeignKey("groups.group_id"), nullable=True, unique=None, default=None) + + user = relationship("User") + + repository = relationship("Repository") + repository_group = relationship("RepoGroup") + + @classmethod + def get_by_position_for_user(cls, position, user_id): + return cls.query() \ + .filter(UserBookmark.user_id == user_id) \ + .filter(UserBookmark.position == position).scalar() + + @classmethod + def get_bookmarks_for_user(cls, user_id): + return cls.query() \ + .filter(UserBookmark.user_id == user_id) \ + .options(joinedload(UserBookmark.repository)) \ + .options(joinedload(UserBookmark.repository_group)) \ + .order_by(UserBookmark.position.asc()) \ + .all() + + def __unicode__(self): + return u'' % (self.position, self.redirect_url) + + +class FileStore(Base, BaseModel): + __tablename__ = 'file_store' + __table_args__ = ( + base_table_args + ) + + file_store_id = Column('file_store_id', Integer(), primary_key=True) + file_uid = Column('file_uid', String(1024), nullable=False) + file_display_name = Column('file_display_name', UnicodeText().with_variant(UnicodeText(2048), 'mysql'), nullable=True) + file_description = Column('file_description', UnicodeText().with_variant(UnicodeText(10240), 'mysql'), nullable=True) + file_org_name = Column('file_org_name', UnicodeText().with_variant(UnicodeText(10240), 'mysql'), nullable=False) + + # sha256 hash + file_hash = Column('file_hash', String(512), nullable=False) + file_size = Column('file_size', Integer(), nullable=False) + + created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) + accessed_on = Column('accessed_on', DateTime(timezone=False), nullable=True) + accessed_count = Column('accessed_count', Integer(), default=0) + + enabled = Column('enabled', Boolean(), nullable=False, default=True) + + # if repo/repo_group reference is set, check for permissions + check_acl = Column('check_acl', Boolean(), nullable=False, default=True) + + user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False) + upload_user = relationship('User', lazy='joined', primaryjoin='User.user_id==FileStore.user_id') + + # scope limited to user, which requester have access to + scope_user_id = Column( + 'scope_user_id', Integer(), ForeignKey('users.user_id'), + nullable=True, unique=None, default=None) + user = relationship('User', lazy='joined', primaryjoin='User.user_id==FileStore.scope_user_id') + + # scope limited to user group, which requester have access to + scope_user_group_id = Column( + 'scope_user_group_id', Integer(), ForeignKey('users_groups.users_group_id'), + nullable=True, unique=None, default=None) + user_group = relationship('UserGroup', lazy='joined') + + # scope limited to repo, which requester have access to + scope_repo_id = Column( + 'scope_repo_id', Integer(), ForeignKey('repositories.repo_id'), + nullable=True, unique=None, default=None) + repo = relationship('Repository', lazy='joined') + + # scope limited to repo group, which requester have access to + scope_repo_group_id = Column( + 'scope_repo_group_id', Integer(), ForeignKey('groups.group_id'), + nullable=True, unique=None, default=None) + repo_group = relationship('RepoGroup', lazy='joined') + + @classmethod + def create(cls, file_uid, filename, file_hash, file_size, file_display_name='', + file_description='', enabled=True, check_acl=True, + user_id=None, scope_repo_id=None, scope_repo_group_id=None): + + store_entry = FileStore() + store_entry.file_uid = file_uid + store_entry.file_display_name = file_display_name + store_entry.file_org_name = filename + store_entry.file_size = file_size + store_entry.file_hash = file_hash + store_entry.file_description = file_description + + store_entry.check_acl = check_acl + store_entry.enabled = enabled + + store_entry.user_id = user_id + store_entry.scope_repo_id = scope_repo_id + store_entry.scope_repo_group_id = scope_repo_group_id + return store_entry + + @classmethod + def bump_access_counter(cls, file_uid, commit=True): + FileStore().query()\ + .filter(FileStore.file_uid == file_uid)\ + .update({FileStore.accessed_count: (FileStore.accessed_count + 1), + FileStore.accessed_on: datetime.datetime.now()}) + if commit: + Session().commit() + + def __repr__(self): + return ''.format(self.file_store_id) + + class DbMigrateVersion(Base, BaseModel): __tablename__ = 'db_migrate_version' __table_args__ = ( diff --git a/rhodecode/model/forms.py b/rhodecode/model/forms.py --- a/rhodecode/model/forms.py +++ b/rhodecode/model/forms.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/gist.py b/rhodecode/model/gist.py --- a/rhodecode/model/gist.py +++ b/rhodecode/model/gist.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2013-2018 RhodeCode GmbH +# Copyright (C) 2013-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/integration.py b/rhodecode/model/integration.py --- a/rhodecode/model/integration.py +++ b/rhodecode/model/integration.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/meta.py b/rhodecode/model/meta.py --- a/rhodecode/model/meta.py +++ b/rhodecode/model/meta.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/notification.py b/rhodecode/model/notification.py --- a/rhodecode/model/notification.py +++ b/rhodecode/model/notification.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/permission.py b/rhodecode/model/permission.py --- a/rhodecode/model/permission.py +++ b/rhodecode/model/permission.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/pull_request.py b/rhodecode/model/pull_request.py --- a/rhodecode/model/pull_request.py +++ b/rhodecode/model/pull_request.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2012-2018 RhodeCode GmbH +# Copyright (C) 2012-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -30,10 +30,11 @@ import datetime import urllib import collections +from pyramid import compat from pyramid.threadlocal import get_current_request from rhodecode import events -from rhodecode.translation import lazy_ugettext#, _ +from rhodecode.translation import lazy_ugettext from rhodecode.lib import helpers as h, hooks_utils, diffs from rhodecode.lib import audit_logger from rhodecode.lib.compat import OrderedDict @@ -75,43 +76,6 @@ class PullRequestModel(BaseModel): DIFF_CONTEXT = diffs.DEFAULT_CONTEXT - MERGE_STATUS_MESSAGES = { - MergeFailureReason.NONE: lazy_ugettext( - 'This pull request can be automatically merged.'), - MergeFailureReason.UNKNOWN: lazy_ugettext( - 'This pull request cannot be merged because of an unhandled' - ' exception.'), - MergeFailureReason.MERGE_FAILED: lazy_ugettext( - 'This pull request cannot be merged because of merge conflicts.'), - MergeFailureReason.PUSH_FAILED: lazy_ugettext( - 'This pull request could not be merged because push to target' - ' failed.'), - MergeFailureReason.TARGET_IS_NOT_HEAD: lazy_ugettext( - 'This pull request cannot be merged because the target is not a' - ' head.'), - MergeFailureReason.HG_SOURCE_HAS_MORE_BRANCHES: lazy_ugettext( - 'This pull request cannot be merged because the source contains' - ' more branches than the target.'), - MergeFailureReason.HG_TARGET_HAS_MULTIPLE_HEADS: lazy_ugettext( - 'This pull request cannot be merged because the target has' - ' multiple heads.'), - MergeFailureReason.TARGET_IS_LOCKED: lazy_ugettext( - 'This pull request cannot be merged because the target repository' - ' is locked.'), - MergeFailureReason._DEPRECATED_MISSING_COMMIT: lazy_ugettext( - 'This pull request cannot be merged because the target or the ' - 'source reference is missing.'), - MergeFailureReason.MISSING_TARGET_REF: lazy_ugettext( - 'This pull request cannot be merged because the target ' - 'reference is missing.'), - MergeFailureReason.MISSING_SOURCE_REF: lazy_ugettext( - 'This pull request cannot be merged because the source ' - 'reference is missing.'), - MergeFailureReason.SUBREPO_MERGE_FAILED: lazy_ugettext( - 'This pull request cannot be merged because of conflicts related ' - 'to sub repositories.'), - } - UPDATE_STATUS_MESSAGES = { UpdateFailureReason.NONE: lazy_ugettext( 'Pull request update successful.'), @@ -175,7 +139,7 @@ class PullRequestModel(BaseModel): def _prepare_get_all_query(self, repo_name, source=False, statuses=None, opened_by=None, order_by=None, - order_dir='desc'): + order_dir='desc', only_created=True): repo = None if repo_name: repo = self._get_repo(repo_name) @@ -196,9 +160,14 @@ class PullRequestModel(BaseModel): if opened_by: q = q.filter(PullRequest.user_id.in_(opened_by)) + # only get those that are in "created" state + if only_created: + q = q.filter(PullRequest.pull_request_state == PullRequest.STATE_CREATED) + if order_by: order_map = { 'name_raw': PullRequest.pull_request_id, + 'id': PullRequest.pull_request_id, 'title': PullRequest.title, 'updated_on_raw': PullRequest.updated_on, 'target_repo': PullRequest.target_repo_id @@ -466,7 +435,7 @@ class PullRequestModel(BaseModel): pull_request.description_renderer = description_renderer pull_request.author = created_by_user pull_request.reviewer_data = reviewer_data - + pull_request.pull_request_state = pull_request.STATE_CREATING Session().add(pull_request) Session().flush() @@ -534,9 +503,16 @@ class PullRequestModel(BaseModel): # that for large repos could be long resulting in long row locks Session().commit() - # prepare workspace, and run initial merge simulation - MergeCheck.validate( - pull_request, auth_user=auth_user, translator=translator) + # prepare workspace, and run initial merge simulation. Set state during that + # operation + pull_request = PullRequest.get(pull_request.pull_request_id) + + # set as merging, for simulation, and if finished to created so we mark + # simulation is working fine + with pull_request.set_state(PullRequest.STATE_MERGING, + final_state=PullRequest.STATE_CREATED): + MergeCheck.validate( + pull_request, auth_user=auth_user, translator=translator) self.notify_reviewers(pull_request, reviewer_ids) self.trigger_pull_request_hook( @@ -602,8 +578,7 @@ class PullRequestModel(BaseModel): extras['user_agent'] = 'internal-merge' merge_state = self._merge_pull_request(pull_request, user, extras) if merge_state.executed: - log.debug( - "Merge was successful, updating the pull request comments.") + log.debug("Merge was successful, updating the pull request comments.") self._comment_and_close_pr(pull_request, user, merge_state) self._log_audit_action( @@ -698,9 +673,8 @@ class PullRequestModel(BaseModel): target_ref_id = pull_request.target_ref_parts.commit_id if not self.has_valid_update_type(pull_request): - log.debug( - "Skipping update of pull request %s due to ref type: %s", - pull_request, source_ref_type) + log.debug("Skipping update of pull request %s due to ref type: %s", + pull_request, source_ref_type) return UpdateResponse( executed=False, reason=UpdateFailureReason.WRONG_REF_TYPE, @@ -858,6 +832,7 @@ class PullRequestModel(BaseModel): version.title = pull_request.title version.description = pull_request.description version.status = pull_request.status + version.pull_request_state = pull_request.pull_request_state version.created_on = datetime.datetime.now() version.updated_on = pull_request.updated_on version.user_id = pull_request.user_id @@ -1028,7 +1003,7 @@ class PullRequestModel(BaseModel): reviewers = {} for user_id, reasons, mandatory, rules in reviewer_data: - if isinstance(user_id, (int, basestring)): + if isinstance(user_id, (int, compat.string_types)): user_id = self._get_user(user_id).user_id reviewers[user_id] = { 'reasons': reasons, 'mandatory': mandatory} @@ -1263,8 +1238,7 @@ class PullRequestModel(BaseModel): pull_request, force_shadow_repo_refresh=force_shadow_repo_refresh) log.debug("Merge response: %s", resp) - status = resp.possible, self.merge_status_message( - resp.failure_reason) + status = resp.possible, resp.merge_status_message except NotImplementedError: status = False, _('Pull request merging is not supported.') @@ -1306,21 +1280,23 @@ class PullRequestModel(BaseModel): "Trying out if the pull request %s can be merged. Force_refresh=%s", pull_request.pull_request_id, force_shadow_repo_refresh) target_vcs = pull_request.target_repo.scm_instance() - # Refresh the target reference. try: target_ref = self._refresh_reference( pull_request.target_ref_parts, target_vcs) except CommitDoesNotExistError: merge_state = MergeResponse( - False, False, None, MergeFailureReason.MISSING_TARGET_REF) + False, False, None, MergeFailureReason.MISSING_TARGET_REF, + metadata={'target_ref': pull_request.target_ref_parts}) return merge_state target_locked = pull_request.target_repo.locked if target_locked and target_locked[0]: - log.debug("The target repository is locked.") + locked_by = 'user:{}'.format(target_locked[0]) + log.debug("The target repository is locked by %s.", locked_by) merge_state = MergeResponse( - False, False, None, MergeFailureReason.TARGET_IS_LOCKED) + False, False, None, MergeFailureReason.TARGET_IS_LOCKED, + metadata={'locked_by': locked_by}) elif force_shadow_repo_refresh or self._needs_merge_state_refresh( pull_request, target_ref): log.debug("Refreshing the merge status of the repository.") @@ -1378,12 +1354,6 @@ class PullRequestModel(BaseModel): workspace_id = 'pr-%s' % pull_request.pull_request_id return workspace_id - def merge_status_message(self, status_code): - """ - Return a human friendly error message for the given merge status code. - """ - return self.MERGE_STATUS_MESSAGES[status_code] - def generate_repo_data(self, repo, commit_id=None, branch=None, bookmark=None, translator=None): from rhodecode.model.repo import RepoModel @@ -1664,17 +1634,16 @@ class MergeCheck(object): msg = _('Pull request reviewer approval is pending.') - merge_check.push_error( - 'warning', msg, cls.REVIEW_CHECK, review_status) + merge_check.push_error('warning', msg, cls.REVIEW_CHECK, review_status) if fail_early: return merge_check # left over TODOs - todos = CommentsModel().get_unresolved_todos(pull_request) + todos = CommentsModel().get_pull_request_unresolved_todos(pull_request) if todos: log.debug("MergeCheck: cannot merge, {} " - "unresolved todos left.".format(len(todos))) + "unresolved TODOs left.".format(len(todos))) if len(todos) == 1: msg = _('Cannot merge, {} TODO still not resolved.').format( @@ -1695,8 +1664,7 @@ class MergeCheck(object): merge_check.merge_possible = merge_status merge_check.merge_msg = msg if not merge_status: - log.debug( - "MergeCheck: cannot merge, pull request merge not possible.") + log.debug("MergeCheck: cannot merge, pull request merge not possible.") merge_check.push_error('warning', msg, cls.MERGE_CHECK, None) if fail_early: @@ -1727,6 +1695,7 @@ class MergeCheck(object): close_branch = model._close_branch_before_merging(pull_request) if close_branch: repo_type = pull_request.target_repo.repo_type + close_msg = '' if repo_type == 'hg': close_msg = _('Source branch will be closed after merge.') elif repo_type == 'git': @@ -1739,6 +1708,7 @@ class MergeCheck(object): return merge_details + ChangeTuple = collections.namedtuple( 'ChangeTuple', ['added', 'common', 'removed', 'total']) diff --git a/rhodecode/model/repo.py b/rhodecode/model/repo.py --- a/rhodecode/model/repo.py +++ b/rhodecode/model/repo.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -462,8 +462,7 @@ class RepoModel(BaseModel): UserGroupRepoToPerm.create( perm.users_group, new_repo, perm.permission) # in case we copy permissions and also set this repo to private - # override the default user permission to make it a private - # repo + # override the default user permission to make it a private repo if private: RepoModel(self.sa).grant_user_permission( repo=new_repo, user=User.DEFAULT_USER, perm=EMPTY_PERM) @@ -485,8 +484,7 @@ class RepoModel(BaseModel): perm_name = perm.permission.permission_name.replace( 'group.', 'repository.') perm_obj = Permission.get_by_key(perm_name) - UserGroupRepoToPerm.create( - perm.users_group, new_repo, perm_obj) + UserGroupRepoToPerm.create(perm.users_group, new_repo, perm_obj) if private: RepoModel(self.sa).grant_user_permission( @@ -497,8 +495,7 @@ class RepoModel(BaseModel): self.sa.add(perm_obj) # now automatically start following this repository as owner - ScmModel(self.sa).toggle_following_repo(new_repo.repo_id, - owner.user_id) + ScmModel(self.sa).toggle_following_repo(new_repo.repo_id, owner.user_id) # we need to flush here, in order to check if database won't # throw any exceptions, create filesystem dirs at the very end diff --git a/rhodecode/model/repo_group.py b/rhodecode/model/repo_group.py --- a/rhodecode/model/repo_group.py +++ b/rhodecode/model/repo_group.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/repo_permission.py b/rhodecode/model/repo_permission.py --- a/rhodecode/model/repo_permission.py +++ b/rhodecode/model/repo_permission.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/scm.py b/rhodecode/model/scm.py --- a/rhodecode/model/scm.py +++ b/rhodecode/model/scm.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -513,6 +513,8 @@ class ScmModel(BaseModel): :param commit_id: commit id for which to list nodes :param root_path: root path to list :param flat: return as a list, if False returns a dict with description + :param extended_info: show additional info such as md5, binary, size etc + :param content: add nodes content to the return data :param max_file_bytes: will not return file contents over this limit """ @@ -523,15 +525,14 @@ class ScmModel(BaseModel): commit = _repo.scm_instance().get_commit(commit_id=commit_id) root_path = root_path.lstrip('/') for __, dirs, files in commit.walk(root_path): + for f in files: _content = None - _data = f.unicode_path - over_size_limit = (max_file_bytes is not None - and f.size > max_file_bytes) + _data = f_name = f.unicode_path if not flat: _data = { - "name": h.escape(f.unicode_path), + "name": h.escape(f_name), "type": "file", } if extended_info: @@ -545,6 +546,8 @@ class ScmModel(BaseModel): }) if content: + over_size_limit = (max_file_bytes is not None + and f.size > max_file_bytes) full_content = None if not f.is_binary and not over_size_limit: full_content = safe_str(f.content) @@ -553,11 +556,12 @@ class ScmModel(BaseModel): "content": full_content, }) _files.append(_data) + for d in dirs: - _data = d.unicode_path + _data = d_name = d.unicode_path if not flat: _data = { - "name": h.escape(d.unicode_path), + "name": h.escape(d_name), "type": "dir", } if extended_info: @@ -573,11 +577,114 @@ class ScmModel(BaseModel): }) _dirs.append(_data) except RepositoryError: - log.debug("Exception in get_nodes", exc_info=True) + log.exception("Exception in get_nodes") raise return _dirs, _files + def get_node(self, repo_name, commit_id, file_path, + extended_info=False, content=False, max_file_bytes=None, cache=True): + """ + retrieve single node from commit + """ + try: + + _repo = self._get_repo(repo_name) + commit = _repo.scm_instance().get_commit(commit_id=commit_id) + + file_node = commit.get_node(file_path) + if file_node.is_dir(): + raise RepositoryError('The given path is a directory') + + _content = None + f_name = file_node.unicode_path + + file_data = { + "name": h.escape(f_name), + "type": "file", + } + + if extended_info: + file_data.update({ + "extension": file_node.extension, + "mimetype": file_node.mimetype, + }) + + if cache: + md5 = file_node.md5 + is_binary = file_node.is_binary + size = file_node.size + else: + is_binary, md5, size, _content = file_node.metadata_uncached() + + file_data.update({ + "md5": md5, + "binary": is_binary, + "size": size, + }) + + if content and cache: + # get content + cache + size = file_node.size + over_size_limit = (max_file_bytes is not None and size > max_file_bytes) + full_content = None + if not file_node.is_binary and not over_size_limit: + full_content = safe_unicode(file_node.content) + + file_data.update({ + "content": full_content, + }) + elif content: + # get content *without* cache + if _content is None: + is_binary, md5, size, _content = file_node.metadata_uncached() + + over_size_limit = (max_file_bytes is not None and size > max_file_bytes) + full_content = None + if not is_binary and not over_size_limit: + full_content = safe_unicode(_content) + + file_data.update({ + "content": full_content, + }) + + except RepositoryError: + log.exception("Exception in get_node") + raise + + return file_data + + def get_fts_data(self, repo_name, commit_id, root_path='/'): + """ + Fetch node tree for usage in full text search + """ + + tree_info = list() + + try: + _repo = self._get_repo(repo_name) + commit = _repo.scm_instance().get_commit(commit_id=commit_id) + root_path = root_path.lstrip('/') + for __, dirs, files in commit.walk(root_path): + + for f in files: + is_binary, md5, size, _content = f.metadata_uncached() + _data = { + "name": f.unicode_path, + "md5": md5, + "extension": f.extension, + "binary": is_binary, + "size": size + } + + tree_info.append(_data) + + except RepositoryError: + log.exception("Exception in get_nodes") + raise + + return tree_info + def create_nodes(self, user, repo, message, nodes, parent_commit=None, author=None, trigger_push_hook=True): """ diff --git a/rhodecode/model/settings.py b/rhodecode/model/settings.py --- a/rhodecode/model/settings.py +++ b/rhodecode/model/settings.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -350,18 +350,26 @@ class IssueTrackerSettingsModel(object): uid = k[len(prefix_match):] issuetracker_entries[uid] = None + def url_cleaner(input_str): + input_str = input_str.replace('"', '').replace("'", '') + input_str = bleach.clean(input_str, strip=True) + return input_str + # populate for uid in issuetracker_entries: + url_data = qs.get(self._get_keyname('url', uid, 'rhodecode_')) + issuetracker_entries[uid] = AttributeDict({ 'pat': qs.get( self._get_keyname('pat', uid, 'rhodecode_')), - 'url': bleach.clean( + 'url': url_cleaner( qs.get(self._get_keyname('url', uid, 'rhodecode_')) or ''), 'pref': bleach.clean( qs.get(self._get_keyname('pref', uid, 'rhodecode_')) or ''), 'desc': qs.get( self._get_keyname('desc', uid, 'rhodecode_')), }) + return issuetracker_entries def get_global_settings(self, cache=False): diff --git a/rhodecode/model/ssh_key.py b/rhodecode/model/ssh_key.py --- a/rhodecode/model/ssh_key.py +++ b/rhodecode/model/ssh_key.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2013-2018 RhodeCode GmbH +# Copyright (C) 2013-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/update.py b/rhodecode/model/update.py --- a/rhodecode/model/update.py +++ b/rhodecode/model/update.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2013-2018 RhodeCode GmbH +# Copyright (C) 2013-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/user.py b/rhodecode/model/user.py --- a/rhodecode/model/user.py +++ b/rhodecode/model/user.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/user_group.py b/rhodecode/model/user_group.py --- a/rhodecode/model/user_group.py +++ b/rhodecode/model/user_group.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -20,6 +20,7 @@ import logging import traceback +from pyramid import compat from rhodecode.lib.utils2 import safe_str, safe_unicode from rhodecode.lib.exceptions import ( @@ -247,7 +248,7 @@ class UserGroupModel(BaseModel): # handle owner change if 'user' in form_data: owner = form_data['user'] - if isinstance(owner, basestring): + if isinstance(owner, compat.string_types): owner = User.get_by_username(form_data['user']) if not isinstance(owner, User): diff --git a/rhodecode/model/validation_schema/__init__.py b/rhodecode/model/validation_schema/__init__.py --- a/rhodecode/model/validation_schema/__init__.py +++ b/rhodecode/model/validation_schema/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/validation_schema/preparers.py b/rhodecode/model/validation_schema/preparers.py --- a/rhodecode/model/validation_schema/preparers.py +++ b/rhodecode/model/validation_schema/preparers.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -19,6 +19,7 @@ # and proprietary license terms, please see https://rhodecode.com/licenses/ import unicodedata +from pyramid import compat def strip_preparer(value): @@ -83,6 +84,6 @@ def unique_list_from_str_preparer(value) """ from rhodecode.lib.utils2 import aslist - if isinstance(value, basestring): + if isinstance(value, compat.string_types): value = aslist(value, ',') return unique_list_preparer(value) \ No newline at end of file diff --git a/rhodecode/model/validation_schema/schemas/__init__.py b/rhodecode/model/validation_schema/schemas/__init__.py --- a/rhodecode/model/validation_schema/schemas/__init__.py +++ b/rhodecode/model/validation_schema/schemas/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/validation_schema/schemas/comment_schema.py b/rhodecode/model/validation_schema/schemas/comment_schema.py --- a/rhodecode/model/validation_schema/schemas/comment_schema.py +++ b/rhodecode/model/validation_schema/schemas/comment_schema.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2017-2018 RhodeCode GmbH +# Copyright (C) 2017-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/validation_schema/schemas/gist_schema.py b/rhodecode/model/validation_schema/schemas/gist_schema.py --- a/rhodecode/model/validation_schema/schemas/gist_schema.py +++ b/rhodecode/model/validation_schema/schemas/gist_schema.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/validation_schema/schemas/integration_schema.py b/rhodecode/model/validation_schema/schemas/integration_schema.py --- a/rhodecode/model/validation_schema/schemas/integration_schema.py +++ b/rhodecode/model/validation_schema/schemas/integration_schema.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/validation_schema/schemas/repo_group_schema.py b/rhodecode/model/validation_schema/schemas/repo_group_schema.py --- a/rhodecode/model/validation_schema/schemas/repo_group_schema.py +++ b/rhodecode/model/validation_schema/schemas/repo_group_schema.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/validation_schema/schemas/repo_schema.py b/rhodecode/model/validation_schema/schemas/repo_schema.py --- a/rhodecode/model/validation_schema/schemas/repo_schema.py +++ b/rhodecode/model/validation_schema/schemas/repo_schema.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/validation_schema/schemas/reviewer_schema.py b/rhodecode/model/validation_schema/schemas/reviewer_schema.py --- a/rhodecode/model/validation_schema/schemas/reviewer_schema.py +++ b/rhodecode/model/validation_schema/schemas/reviewer_schema.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/validation_schema/schemas/search_schema.py b/rhodecode/model/validation_schema/schemas/search_schema.py --- a/rhodecode/model/validation_schema/schemas/search_schema.py +++ b/rhodecode/model/validation_schema/schemas/search_schema.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -34,6 +34,9 @@ class SearchParamsSchema(colander.Mappin colander.String(), missing='newfirst', validator=colander.OneOf(['oldfirst', 'newfirst'])) + search_max_lines = colander.SchemaNode( + colander.Integer(), + missing=10) page_limit = colander.SchemaNode( colander.Integer(), missing=10, diff --git a/rhodecode/model/validation_schema/schemas/user_group_schema.py b/rhodecode/model/validation_schema/schemas/user_group_schema.py --- a/rhodecode/model/validation_schema/schemas/user_group_schema.py +++ b/rhodecode/model/validation_schema/schemas/user_group_schema.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/validation_schema/schemas/user_schema.py b/rhodecode/model/validation_schema/schemas/user_schema.py --- a/rhodecode/model/validation_schema/schemas/user_schema.py +++ b/rhodecode/model/validation_schema/schemas/user_schema.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/validation_schema/types.py b/rhodecode/model/validation_schema/types.py --- a/rhodecode/model/validation_schema/types.py +++ b/rhodecode/model/validation_schema/types.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -21,6 +21,8 @@ import re import colander +from pyramid import compat + from rhodecode.model.validation_schema import preparers from rhodecode.model.db import User, UserGroup @@ -106,7 +108,7 @@ class StringBooleanType(colander.String) if isinstance(cstruct, bool): return cstruct - if not isinstance(cstruct, basestring): + if not isinstance(cstruct, compat.string_types): raise colander.Invalid(node, '%r is not a string' % cstruct) value = cstruct.lower() @@ -190,7 +192,7 @@ class UserGroupType(UserOrUserGroupType) class StrOrIntType(colander.String): def deserialize(self, node, cstruct): - if isinstance(cstruct, basestring): + if isinstance(cstruct, compat.string_types): return super(StrOrIntType, self).deserialize(node, cstruct) else: return colander.Integer().deserialize(node, cstruct) diff --git a/rhodecode/model/validation_schema/utils.py b/rhodecode/model/validation_schema/utils.py --- a/rhodecode/model/validation_schema/utils.py +++ b/rhodecode/model/validation_schema/utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2016-2018 RhodeCode GmbH +# Copyright (C) 2016-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/validation_schema/validators.py b/rhodecode/model/validation_schema/validators.py --- a/rhodecode/model/validation_schema/validators.py +++ b/rhodecode/model/validation_schema/validators.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/validation_schema/widgets.py b/rhodecode/model/validation_schema/widgets.py --- a/rhodecode/model/validation_schema/widgets.py +++ b/rhodecode/model/validation_schema/widgets.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2011-2018 RhodeCode GmbH +# Copyright (C) 2011-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/model/validators.py b/rhodecode/model/validators.py --- a/rhodecode/model/validators.py +++ b/rhodecode/model/validators.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -37,6 +37,7 @@ from formencode.validators import ( from sqlalchemy.sql.expression import true from sqlalchemy.util import OrderedSet +from pyramid import compat from rhodecode.authentication import ( legacy_plugin_prefix, _import_legacy_plugin) @@ -125,7 +126,7 @@ def UniqueListFromString(localizer): class _validator(UniqueList(localizer)): def _to_python(self, value, state): - if isinstance(value, basestring): + if isinstance(value, compat.string_types): value = aslist(value, ',') return super(_validator, self)._to_python(value, state) return _validator diff --git a/rhodecode/public/css/code-block.less b/rhodecode/public/css/code-block.less --- a/rhodecode/public/css/code-block.less +++ b/rhodecode/public/css/code-block.less @@ -572,6 +572,7 @@ div.annotatediv { margin-left: 2px; marg .code-highlight, /* TODO: dan: merge codehilite into code-highlight */ /* This can be generated with `pygmentize -S default -f html` */ .codehilite { + .c-ElasticMatch { background-color: #faffa6; padding: 0.2em;} .hll { background-color: #ffffcc } .c { color: #408080; font-style: italic } /* Comment */ .err, .codehilite .err { border: none } /* Error */ @@ -640,6 +641,7 @@ div.annotatediv { margin-left: 2px; marg .vi { color: #19177C } /* Name.Variable.Instance */ .vm { color: #19177C } /* Name.Variable.Magic */ .il { color: #666666 } /* Literal.Number.Integer.Long */ + } /* customized pre blocks for markdown/rst */ diff --git a/rhodecode/public/css/login.less b/rhodecode/public/css/login.less --- a/rhodecode/public/css/login.less +++ b/rhodecode/public/css/login.less @@ -73,9 +73,6 @@ } .sign-in-title { - h1 { - margin: 0; - } h4 { margin: @padding*2 0; @@ -109,6 +106,12 @@ width: 100%; margin: @padding 0; } + .pwd_reset { + font-weight: normal; + } + .new_account { + font-weight: bold; + } } .register_message, .activation_msg { @@ -197,7 +200,27 @@ .user-menu.submenu { right: 0; left: auto; + min-width: 290px; } + + +.user-menu { + .bookmark-items { + padding: 4px 2px; + color: @grey3; + border-bottom: @grey3; + + a { + padding: 0 !important; + color: @rcblue !important; + } + } + a.bookmark-item { + color: @rcblue !important; + } +} + + #quick_login { left: auto; right: 0; diff --git a/rhodecode/public/css/main.less b/rhodecode/public/css/main.less --- a/rhodecode/public/css/main.less +++ b/rhodecode/public/css/main.less @@ -1263,7 +1263,7 @@ table.integrations { margin-bottom: @padding; clear: both; - .stats{ + .stats { overflow: hidden; } @@ -2236,6 +2236,10 @@ h3.files_location{ clear: both; margin: 0 0 @padding; } + + .search-tags { + padding: 5px 0; + } } div.search-feedback-items { diff --git a/rhodecode/public/css/navigation.less b/rhodecode/public/css/navigation.less --- a/rhodecode/public/css/navigation.less +++ b/rhodecode/public/css/navigation.less @@ -335,6 +335,7 @@ } } } + } @@ -635,8 +636,10 @@ ul#context-pages { border-bottom: 1px solid @grey4; display: inline-block; vertical-align: top; - margin-left: -7px; - background: @grey3; + background: inherit; + position: absolute; + right: 8px; + top: 9px; } .main_filter_input_box { @@ -651,24 +654,36 @@ ul#context-pages { background: @grey3; border: 1px solid black; position: absolute; - white-space: pre-wrap; + white-space: pre; z-index: 9999; color: @nav-grey; margin: 1px 7px; - padding: 0 2px; + padding: 0 10px; } .main_filter_input { padding: 5px; - min-width: 220px; + min-width: 260px; color: @nav-grey; background: @grey3; min-height: 18px; + + + &:active { + color: @grey2 !important; + background: white !important; + } + &:focus { + color: @grey2 !important; + background: white !important; + } } + + .main_filter_input::placeholder { - color: @nav-grey; - opacity: 1; + color: @nav-grey; + opacity: 1; } .notice-box { diff --git a/rhodecode/public/css/summary.less b/rhodecode/public/css/summary.less --- a/rhodecode/public/css/summary.less +++ b/rhodecode/public/css/summary.less @@ -255,7 +255,6 @@ .stats { float: left; - width: 50%; } .stats-filename { font-size: 120%; @@ -266,10 +265,15 @@ .buttons { float: right; - width: 50%; text-align: right; color: @grey4; } + + .file-container { + display: inline-block; + width: 100%; + } + } #summary-menu-stats { diff --git a/rhodecode/public/css/tables.less b/rhodecode/public/css/tables.less --- a/rhodecode/public/css/tables.less +++ b/rhodecode/public/css/tables.less @@ -172,7 +172,9 @@ table.dataTable { &.td-buttons { padding: .3em 0; } - + &.td-align-top { + vertical-align: text-top + } &.td-action { // this is for the remove/delete/edit buttons padding-right: 0; diff --git a/rhodecode/public/css/type.less b/rhodecode/public/css/type.less --- a/rhodecode/public/css/type.less +++ b/rhodecode/public/css/type.less @@ -166,7 +166,6 @@ small, mark, .mark { - background-color: @rclightblue; padding: .2em; } @@ -301,6 +300,10 @@ mark, margin-top: @padding; } } + + .repo-group-desc { + padding: 8px 0px 0px 0px; + } } .title-main { @@ -527,6 +530,10 @@ address { font-family: @text-regular; } +.help-block-inline { + margin: 0; +} + // help block text .help-block { display: block; diff --git a/rhodecode/public/js/rhodecode/base/keyboard-bindings.js b/rhodecode/public/js/rhodecode/base/keyboard-bindings.js --- a/rhodecode/public/js/rhodecode/base/keyboard-bindings.js +++ b/rhodecode/public/js/rhodecode/base/keyboard-bindings.js @@ -51,6 +51,38 @@ function setRCMouseBindings(repoName, re Mousetrap.bind(['g G'], function(e) { window.location = pyroutes.url('gists_show', {'public': 1}); }); + + Mousetrap.bind(['g 0'], function(e) { + window.location = pyroutes.url('my_account_goto_bookmark', {'bookmark_id': 0}); + }); + Mousetrap.bind(['g 1'], function(e) { + window.location = pyroutes.url('my_account_goto_bookmark', {'bookmark_id': 1}); + }); + Mousetrap.bind(['g 2'], function(e) { + window.location = pyroutes.url('my_account_goto_bookmark', {'bookmark_id': 2}); + }); + Mousetrap.bind(['g 3'], function(e) { + window.location = pyroutes.url('my_account_goto_bookmark', {'bookmark_id': 3}); + }); + Mousetrap.bind(['g 4'], function(e) { + window.location = pyroutes.url('my_account_goto_bookmark', {'bookmark_id': 4}); + }); + Mousetrap.bind(['g 5'], function(e) { + window.location = pyroutes.url('my_account_goto_bookmark', {'bookmark_id': 5}); + }); + Mousetrap.bind(['g 6'], function(e) { + window.location = pyroutes.url('my_account_goto_bookmark', {'bookmark_id': 6}); + }); + Mousetrap.bind(['g 7'], function(e) { + window.location = pyroutes.url('my_account_goto_bookmark', {'bookmark_id': 7}); + }); + Mousetrap.bind(['g 8'], function(e) { + window.location = pyroutes.url('my_account_goto_bookmark', {'bookmark_id': 8}); + }); + Mousetrap.bind(['g 9'], function(e) { + window.location = pyroutes.url('my_account_goto_bookmark', {'bookmark_id': 9}); + }); + Mousetrap.bind(['n g'], function(e) { window.location = pyroutes.url('gists_new'); }); @@ -58,7 +90,7 @@ function setRCMouseBindings(repoName, re window.location = pyroutes.url('repo_new'); }); - if (repoName && repoName != '') { + if (repoName && repoName !== '') { // nav in repo context Mousetrap.bind(['g s'], function(e) { window.location = pyroutes.url( diff --git a/rhodecode/public/js/rhodecode/routes.js b/rhodecode/public/js/rhodecode/routes.js --- a/rhodecode/public/js/rhodecode/routes.js +++ b/rhodecode/public/js/rhodecode/routes.js @@ -102,7 +102,8 @@ function registerRCRoutes() { pyroutes.register('user_edit_global_perms_update', '/_admin/users/%(user_id)s/edit/global_permissions/update', ['user_id']); pyroutes.register('user_update', '/_admin/users/%(user_id)s/update', ['user_id']); pyroutes.register('user_delete', '/_admin/users/%(user_id)s/delete', ['user_id']); - pyroutes.register('user_force_password_reset', '/_admin/users/%(user_id)s/password_reset', ['user_id']); + pyroutes.register('user_enable_force_password_reset', '/_admin/users/%(user_id)s/password_reset_enable', ['user_id']); + pyroutes.register('user_disable_force_password_reset', '/_admin/users/%(user_id)s/password_reset_disable', ['user_id']); pyroutes.register('user_create_personal_repo_group', '/_admin/users/%(user_id)s/create_repo_group', ['user_id']); pyroutes.register('edit_user_auth_tokens_delete', '/_admin/users/%(user_id)s/edit/auth_tokens/delete', ['user_id']); pyroutes.register('edit_user_ssh_keys', '/_admin/users/%(user_id)s/edit/ssh_keys', ['user_id']); @@ -135,6 +136,8 @@ function registerRCRoutes() { pyroutes.register('channelstream_connect', '/_admin/channelstream/connect', []); pyroutes.register('channelstream_subscribe', '/_admin/channelstream/subscribe', []); pyroutes.register('channelstream_proxy', '/_channelstream', []); + pyroutes.register('upload_file', '/_file_store/upload', []); + pyroutes.register('download_file', '/_file_store/download/%(fid)s', ['fid']); pyroutes.register('logout', '/_admin/logout', []); pyroutes.register('reset_password', '/_admin/password_reset', []); pyroutes.register('reset_password_confirmation', '/_admin/password_reset_confirmation', []); @@ -142,6 +145,7 @@ function registerRCRoutes() { pyroutes.register('user_autocomplete_data', '/_users', []); pyroutes.register('user_group_autocomplete_data', '/_user_groups', []); pyroutes.register('repo_list_data', '/_repos', []); + pyroutes.register('repo_group_list_data', '/_repo_groups', []); pyroutes.register('goto_switcher_data', '/_goto_data', []); pyroutes.register('markup_preview', '/_markup_preview', []); pyroutes.register('store_user_session_value', '/_store_session_attr', []); @@ -212,7 +216,7 @@ function registerRCRoutes() { pyroutes.register('pullrequest_show_all', '/%(repo_name)s/pull-request', ['repo_name']); pyroutes.register('pullrequest_show_all_data', '/%(repo_name)s/pull-request-data', ['repo_name']); pyroutes.register('pullrequest_repo_refs', '/%(repo_name)s/pull-request/refs/%(target_repo_name)s', ['repo_name', 'target_repo_name']); - pyroutes.register('pullrequest_repo_destinations', '/%(repo_name)s/pull-request/repo-destinations', ['repo_name']); + pyroutes.register('pullrequest_repo_targets', '/%(repo_name)s/pull-request/repo-targets', ['repo_name']); pyroutes.register('pullrequest_new', '/%(repo_name)s/pull-request/new', ['repo_name']); pyroutes.register('pullrequest_create', '/%(repo_name)s/pull-request/create', ['repo_name']); pyroutes.register('pullrequest_update', '/%(repo_name)s/pull-request/%(pull_request_id)s/update', ['repo_name', 'pull_request_id']); @@ -277,7 +281,9 @@ function registerRCRoutes() { pyroutes.register('edit_user_group_advanced_sync', '/_admin/user_groups/%(user_group_id)s/edit/advanced/sync', ['user_group_id']); pyroutes.register('user_groups_delete', '/_admin/user_groups/%(user_group_id)s/delete', ['user_group_id']); pyroutes.register('search', '/_admin/search', []); - pyroutes.register('search_repo', '/%(repo_name)s/search', ['repo_name']); + pyroutes.register('search_repo', '/%(repo_name)s/_search', ['repo_name']); + pyroutes.register('search_repo_alt', '/%(repo_name)s/search', ['repo_name']); + pyroutes.register('search_repo_group', '/%(repo_group_name)s/_search', ['repo_group_name']); pyroutes.register('user_profile', '/_profiles/%(username)s', ['username']); pyroutes.register('user_group_profile', '/_profile_user_group/%(user_group_name)s', ['user_group_name']); pyroutes.register('my_account_profile', '/_admin/my_account/profile', []); @@ -296,6 +302,9 @@ function registerRCRoutes() { pyroutes.register('my_account_emails_delete', '/_admin/my_account/emails/delete', []); pyroutes.register('my_account_repos', '/_admin/my_account/repos', []); pyroutes.register('my_account_watched', '/_admin/my_account/watched', []); + pyroutes.register('my_account_bookmarks', '/_admin/my_account/bookmarks', []); + pyroutes.register('my_account_bookmarks_update', '/_admin/my_account/bookmarks/update', []); + pyroutes.register('my_account_goto_bookmark', '/_admin/my_account/bookmark/%(bookmark_id)s', ['bookmark_id']); pyroutes.register('my_account_perms', '/_admin/my_account/perms', []); pyroutes.register('my_account_notifications', '/_admin/my_account/notifications', []); pyroutes.register('my_account_notifications_toggle_visibility', '/_admin/my_account/toggle_visibility', []); diff --git a/rhodecode/public/js/src/ejs_templates/utils.js b/rhodecode/public/js/src/ejs_templates/utils.js --- a/rhodecode/public/js/src/ejs_templates/utils.js +++ b/rhodecode/public/js/src/ejs_templates/utils.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/i18n_utils.js b/rhodecode/public/js/src/i18n_utils.js --- a/rhodecode/public/js/src/i18n_utils.js +++ b/rhodecode/public/js/src/i18n_utils.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2016-2018 RhodeCode GmbH +// # Copyright (C) 2016-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/plugins/jquery.mark.js b/rhodecode/public/js/src/plugins/jquery.mark.js deleted file mode 100755 --- a/rhodecode/public/js/src/plugins/jquery.mark.js +++ /dev/null @@ -1,490 +0,0 @@ -/*!*************************************************** - * mark.js v6.1.0 - * https://github.com/julmot/mark.js - * Copyright (c) 2014–2016, Julian Motz - * Released under the MIT license https://git.io/vwTVl - *****************************************************/ - -"use strict"; - -var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; }; - -var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); - -var _typeof = typeof Symbol === "function" && typeof Symbol.iterator === "symbol" ? function (obj) { return typeof obj; } : function (obj) { return obj && typeof Symbol === "function" && obj.constructor === Symbol ? "symbol" : typeof obj; }; - -function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } - -(function (factory, window, document) { - if (typeof define === "function" && define.amd) { - define(["jquery"], function (jQuery) { - return factory(window, document, jQuery); - }); - } else if ((typeof exports === "undefined" ? "undefined" : _typeof(exports)) === "object") { - factory(window, document, require("jquery")); - } else { - factory(window, document, jQuery); - } -})(function (window, document, $) { - var Mark = function () { - function Mark(ctx) { - _classCallCheck(this, Mark); - - this.ctx = ctx; - } - - _createClass(Mark, [{ - key: "log", - value: function log(msg) { - var level = arguments.length <= 1 || arguments[1] === undefined ? "debug" : arguments[1]; - - var log = this.opt.log; - if (!this.opt.debug) { - return; - } - if ((typeof log === "undefined" ? "undefined" : _typeof(log)) === "object" && typeof log[level] === "function") { - log[level]("mark.js: " + msg); - } - } - }, { - key: "escapeStr", - value: function escapeStr(str) { - return str.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&"); - } - }, { - key: "createRegExp", - value: function createRegExp(str) { - str = this.escapeStr(str); - if (Object.keys(this.opt.synonyms).length) { - str = this.createSynonymsRegExp(str); - } - if (this.opt.diacritics) { - str = this.createDiacriticsRegExp(str); - } - str = this.createAccuracyRegExp(str); - return str; - } - }, { - key: "createSynonymsRegExp", - value: function createSynonymsRegExp(str) { - var syn = this.opt.synonyms; - for (var index in syn) { - if (syn.hasOwnProperty(index)) { - var value = syn[index], - k1 = this.escapeStr(index), - k2 = this.escapeStr(value); - str = str.replace(new RegExp("(" + k1 + "|" + k2 + ")", "gmi"), "(" + k1 + "|" + k2 + ")"); - } - } - return str; - } - }, { - key: "createDiacriticsRegExp", - value: function createDiacriticsRegExp(str) { - var dct = ["aÀÁÂÃÄÅàáâãäåĀāąĄ", "cÇçćĆčČ", "dđĐďĎ", "eÈÉÊËèéêëěĚĒēęĘ", "iÌÍÎÏìíîïĪī", "lłŁ", "nÑñňŇńŃ", "oÒÓÔÕÕÖØòóôõöøŌō", "rřŘ", "sŠšśŚ", "tťŤ", "uÙÚÛÜùúûüůŮŪū", "yŸÿýÝ", "zŽžżŻźŹ"]; - var handled = []; - str.split("").forEach(function (ch) { - dct.every(function (dct) { - if (dct.indexOf(ch) !== -1) { - if (handled.indexOf(dct) > -1) { - return false; - } - - str = str.replace(new RegExp("[" + dct + "]", "gmi"), "[" + dct + "]"); - handled.push(dct); - } - return true; - }); - }); - return str; - } - }, { - key: "createAccuracyRegExp", - value: function createAccuracyRegExp(str) { - switch (this.opt.accuracy) { - case "partially": - return "()(" + str + ")"; - case "complementary": - return "()(\\S*" + str + "\\S*)"; - case "exactly": - return "(^|\\s)(" + str + ")(?=\\s|$)"; - } - } - }, { - key: "getSeparatedKeywords", - value: function getSeparatedKeywords(sv) { - var _this = this; - - var stack = []; - sv.forEach(function (kw) { - if (!_this.opt.separateWordSearch) { - if (kw.trim()) { - stack.push(kw); - } - } else { - kw.split(" ").forEach(function (kwSplitted) { - if (kwSplitted.trim()) { - stack.push(kwSplitted); - } - }); - } - }); - return { - "keywords": stack, - "length": stack.length - }; - } - }, { - key: "getElements", - value: function getElements() { - var ctx = void 0, - stack = []; - if (typeof this.ctx === "undefined") { - ctx = []; - } else if (this.ctx instanceof HTMLElement) { - ctx = [this.ctx]; - } else if (Array.isArray(this.ctx)) { - ctx = this.ctx; - } else { - ctx = Array.prototype.slice.call(this.ctx); - } - ctx.forEach(function (ctx) { - stack.push(ctx); - var childs = ctx.querySelectorAll("*"); - if (childs.length) { - stack = stack.concat(Array.prototype.slice.call(childs)); - } - }); - if (!ctx.length) { - this.log("Empty context", "warn"); - } - return { - "elements": stack, - "length": stack.length - }; - } - }, { - key: "matches", - value: function matches(el, selector) { - return (el.matches || el.matchesSelector || el.msMatchesSelector || el.mozMatchesSelector || el.webkitMatchesSelector || el.oMatchesSelector).call(el, selector); - } - }, { - key: "matchesFilter", - value: function matchesFilter(el, exclM) { - var _this2 = this; - - var remain = true; - var fltr = this.opt.filter.concat(["script", "style", "title"]); - if (!this.opt.iframes) { - fltr = fltr.concat(["iframe"]); - } - if (exclM) { - fltr = fltr.concat(["*[data-markjs='true']"]); - } - fltr.every(function (filter) { - if (_this2.matches(el, filter)) { - return remain = false; - } - return true; - }); - return !remain; - } - }, { - key: "onIframeReady", - value: function onIframeReady(ifr, successFn, errorFn) { - try { - (function () { - var ifrWin = ifr.contentWindow, - bl = "about:blank", - compl = "complete"; - var callCallback = function callCallback() { - try { - if (ifrWin.document === null) { - throw new Error("iframe inaccessible"); - } - successFn(ifrWin.document); - } catch (e) { - errorFn(); - } - }; - var isBlank = function isBlank() { - var src = ifr.getAttribute("src").trim(), - href = ifrWin.location.href; - return href === bl && src !== bl && src; - }; - var observeOnload = function observeOnload() { - var listener = function listener() { - try { - if (!isBlank()) { - ifr.removeEventListener("load", listener); - callCallback(); - } - } catch (e) { - errorFn(); - } - }; - ifr.addEventListener("load", listener); - }; - if (ifrWin.document.readyState === compl) { - if (isBlank()) { - observeOnload(); - } else { - callCallback(); - } - } else { - observeOnload(); - } - })(); - } catch (e) { - errorFn(); - } - } - }, { - key: "forEachElementInIframe", - value: function forEachElementInIframe(ifr, cb) { - var _this3 = this; - - var end = arguments.length <= 2 || arguments[2] === undefined ? function () {} : arguments[2]; - - var open = 0; - var checkEnd = function checkEnd() { - if (--open < 1) { - end(); - } - }; - this.onIframeReady(ifr, function (con) { - var stack = Array.prototype.slice.call(con.querySelectorAll("*")); - if ((open = stack.length) === 0) { - checkEnd(); - } - stack.forEach(function (el) { - if (el.tagName.toLowerCase() === "iframe") { - (function () { - var j = 0; - _this3.forEachElementInIframe(el, function (iel, len) { - cb(iel, len); - if (len - 1 === j) { - checkEnd(); - } - j++; - }, checkEnd); - })(); - } else { - cb(el, stack.length); - checkEnd(); - } - }); - }, function () { - var src = ifr.getAttribute("src"); - _this3.log("iframe '" + src + "' could not be accessed", "warn"); - checkEnd(); - }); - } - }, { - key: "forEachElement", - value: function forEachElement(cb) { - var _this4 = this; - - var end = arguments.length <= 1 || arguments[1] === undefined ? function () {} : arguments[1]; - var exclM = arguments.length <= 2 || arguments[2] === undefined ? true : arguments[2]; - - var _getElements = this.getElements(); - - var stack = _getElements.elements; - var open = _getElements.length; - - var checkEnd = function checkEnd() { - if (--open === 0) { - end(); - } - }; - checkEnd(++open); - stack.forEach(function (el) { - if (!_this4.matchesFilter(el, exclM)) { - if (el.tagName.toLowerCase() === "iframe") { - _this4.forEachElementInIframe(el, function (iel) { - if (!_this4.matchesFilter(iel, exclM)) { - cb(iel); - } - }, checkEnd); - return; - } else { - cb(el); - } - } - checkEnd(); - }); - } - }, { - key: "forEachNode", - value: function forEachNode(cb) { - var end = arguments.length <= 1 || arguments[1] === undefined ? function () {} : arguments[1]; - - this.forEachElement(function (n) { - for (n = n.firstChild; n; n = n.nextSibling) { - if (n.nodeType === 3 && n.textContent.trim()) { - cb(n); - } - } - }, end); - } - }, { - key: "wrapMatches", - value: function wrapMatches(node, regex, custom, cb) { - var hEl = !this.opt.element ? "mark" : this.opt.element, - index = custom ? 0 : 2; - var match = void 0; - while ((match = regex.exec(node.textContent)) !== null) { - var pos = match.index; - if (!custom) { - pos += match[index - 1].length; - } - var startNode = node.splitText(pos); - - node = startNode.splitText(match[index].length); - if (startNode.parentNode !== null) { - var repl = document.createElement(hEl); - repl.setAttribute("data-markjs", "true"); - if (this.opt.className) { - repl.setAttribute("class", this.opt.className); - } - repl.textContent = match[index]; - startNode.parentNode.replaceChild(repl, startNode); - cb(repl); - } - regex.lastIndex = 0; - } - } - }, { - key: "unwrapMatches", - value: function unwrapMatches(node) { - var parent = node.parentNode; - var docFrag = document.createDocumentFragment(); - while (node.firstChild) { - docFrag.appendChild(node.removeChild(node.firstChild)); - } - parent.replaceChild(docFrag, node); - parent.normalize(); - } - }, { - key: "markRegExp", - value: function markRegExp(regexp, opt) { - var _this5 = this; - - this.opt = opt; - this.log("Searching with expression \"" + regexp + "\""); - var found = false; - var eachCb = function eachCb(element) { - found = true; - _this5.opt.each(element); - }; - this.forEachNode(function (node) { - _this5.wrapMatches(node, regexp, true, eachCb); - }, function () { - if (!found) { - _this5.opt.noMatch(regexp); - } - _this5.opt.complete(); - _this5.opt.done(); - }); - } - }, { - key: "mark", - value: function mark(sv, opt) { - var _this6 = this; - - this.opt = opt; - sv = typeof sv === "string" ? [sv] : sv; - - var _getSeparatedKeywords = this.getSeparatedKeywords(sv); - - var kwArr = _getSeparatedKeywords.keywords; - var kwArrLen = _getSeparatedKeywords.length; - - if (kwArrLen === 0) { - this.opt.complete(); - this.opt.done(); - } - kwArr.forEach(function (kw) { - var regex = new RegExp(_this6.createRegExp(kw), "gmi"), - found = false; - var eachCb = function eachCb(element) { - found = true; - _this6.opt.each(element); - }; - _this6.log("Searching with expression \"" + regex + "\""); - _this6.forEachNode(function (node) { - _this6.wrapMatches(node, regex, false, eachCb); - }, function () { - if (!found) { - _this6.opt.noMatch(kw); - } - if (kwArr[kwArrLen - 1] === kw) { - _this6.opt.complete(); - _this6.opt.done(); - } - }); - }); - } - }, { - key: "unmark", - value: function unmark(opt) { - var _this7 = this; - - this.opt = opt; - var sel = this.opt.element ? this.opt.element : "*"; - sel += "[data-markjs]"; - if (this.opt.className) { - sel += "." + this.opt.className; - } - this.log("Removal selector \"" + sel + "\""); - this.forEachElement(function (el) { - if (_this7.matches(el, sel)) { - _this7.unwrapMatches(el); - } - }, function () { - _this7.opt.complete(); - _this7.opt.done(); - }, false); - } - }, { - key: "opt", - set: function set(val) { - this._opt = _extends({}, { - "element": "", - "className": "", - "filter": [], - "iframes": false, - "separateWordSearch": true, - "diacritics": true, - "synonyms": {}, - "accuracy": "partially", - "each": function each() {}, - "noMatch": function noMatch() {}, - "done": function done() {}, - "complete": function complete() {}, - "debug": false, - "log": window.console - }, val); - }, - get: function get() { - return this._opt; - } - }]); - - return Mark; - }(); - - $.fn.mark = function (sv, opt) { - new Mark(this).mark(sv, opt); - return this; - }; - $.fn.markRegExp = function (regexp, opt) { - new Mark(this).markRegExp(regexp, opt); - return this; - }; - $.fn.unmark = function (opt) { - new Mark(this).unmark(opt); - return this; - }; -}, window, document); diff --git a/rhodecode/public/js/src/rhodecode.js b/rhodecode/public/js/src/rhodecode.js --- a/rhodecode/public/js/src/rhodecode.js +++ b/rhodecode/public/js/src/rhodecode.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 @@ -76,6 +76,7 @@ var showRepoSize = function(target, repo var url = pyroutes.url('repo_stats', {"repo_name": repo_name, "commit_id": commit_id}); + container.show(); if (!container.hasClass('loaded')) { $.ajax({url: url}) .complete(function (data) { diff --git a/rhodecode/public/js/src/rhodecode/appenlight.js b/rhodecode/public/js/src/rhodecode/appenlight.js --- a/rhodecode/public/js/src/rhodecode/appenlight.js +++ b/rhodecode/public/js/src/rhodecode/appenlight.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/changelog.js b/rhodecode/public/js/src/rhodecode/changelog.js --- a/rhodecode/public/js/src/rhodecode/changelog.js +++ b/rhodecode/public/js/src/rhodecode/changelog.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2016-2018 RhodeCode GmbH +// # Copyright (C) 2016-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/codemirror.js b/rhodecode/public/js/src/rhodecode/codemirror.js --- a/rhodecode/public/js/src/rhodecode/codemirror.js +++ b/rhodecode/public/js/src/rhodecode/codemirror.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/comments.js b/rhodecode/public/js/src/rhodecode/comments.js --- a/rhodecode/public/js/src/rhodecode/comments.js +++ b/rhodecode/public/js/src/rhodecode/comments.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/constants.js b/rhodecode/public/js/src/rhodecode/constants.js --- a/rhodecode/public/js/src/rhodecode/constants.js +++ b/rhodecode/public/js/src/rhodecode/constants.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/files.js b/rhodecode/public/js/src/rhodecode/files.js --- a/rhodecode/public/js/src/rhodecode/files.js +++ b/rhodecode/public/js/src/rhodecode/files.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 @@ -137,11 +137,12 @@ var fileBrowserListeners = function(node var new_url = url_base.replace('__FPATH__',n); var typeObj = { - dir: 'icon-folder browser-dir', - file: 'icon-file browser-file' + dir: 'icon-directory browser-dir', + file: 'icon-file-text browser-file' }; + var typeIcon = ''.format(typeObj[t]); - match.push('{2}{3}'.format(t,new_url,typeIcon, n_hl)); + match.push('{1}{2}'.format(new_url,typeIcon, n_hl)); } } if(results.length > limit){ diff --git a/rhodecode/public/js/src/rhodecode/followers.js b/rhodecode/public/js/src/rhodecode/followers.js --- a/rhodecode/public/js/src/rhodecode/followers.js +++ b/rhodecode/public/js/src/rhodecode/followers.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/init.js b/rhodecode/public/js/src/rhodecode/init.js --- a/rhodecode/public/js/src/rhodecode/init.js +++ b/rhodecode/public/js/src/rhodecode/init.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/menus.js b/rhodecode/public/js/src/rhodecode/menus.js --- a/rhodecode/public/js/src/rhodecode/menus.js +++ b/rhodecode/public/js/src/rhodecode/menus.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/notifications.js b/rhodecode/public/js/src/rhodecode/notifications.js --- a/rhodecode/public/js/src/rhodecode/notifications.js +++ b/rhodecode/public/js/src/rhodecode/notifications.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/permissions.js b/rhodecode/public/js/src/rhodecode/permissions.js --- a/rhodecode/public/js/src/rhodecode/permissions.js +++ b/rhodecode/public/js/src/rhodecode/permissions.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/pjax.js b/rhodecode/public/js/src/rhodecode/pjax.js --- a/rhodecode/public/js/src/rhodecode/pjax.js +++ b/rhodecode/public/js/src/rhodecode/pjax.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/pullrequests.js b/rhodecode/public/js/src/rhodecode/pullrequests.js --- a/rhodecode/public/js/src/rhodecode/pullrequests.js +++ b/rhodecode/public/js/src/rhodecode/pullrequests.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/settings.js b/rhodecode/public/js/src/rhodecode/settings.js --- a/rhodecode/public/js/src/rhodecode/settings.js +++ b/rhodecode/public/js/src/rhodecode/settings.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/tooltips.js b/rhodecode/public/js/src/rhodecode/tooltips.js --- a/rhodecode/public/js/src/rhodecode/tooltips.js +++ b/rhodecode/public/js/src/rhodecode/tooltips.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/users.js b/rhodecode/public/js/src/rhodecode/users.js --- a/rhodecode/public/js/src/rhodecode/users.js +++ b/rhodecode/public/js/src/rhodecode/users.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/utils/ajax.js b/rhodecode/public/js/src/rhodecode/utils/ajax.js --- a/rhodecode/public/js/src/rhodecode/utils/ajax.js +++ b/rhodecode/public/js/src/rhodecode/utils/ajax.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/utils/array.js b/rhodecode/public/js/src/rhodecode/utils/array.js --- a/rhodecode/public/js/src/rhodecode/utils/array.js +++ b/rhodecode/public/js/src/rhodecode/utils/array.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/utils/autocomplete.js b/rhodecode/public/js/src/rhodecode/utils/autocomplete.js --- a/rhodecode/public/js/src/rhodecode/utils/autocomplete.js +++ b/rhodecode/public/js/src/rhodecode/utils/autocomplete.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/utils/colorgenerator.js b/rhodecode/public/js/src/rhodecode/utils/colorgenerator.js --- a/rhodecode/public/js/src/rhodecode/utils/colorgenerator.js +++ b/rhodecode/public/js/src/rhodecode/utils/colorgenerator.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/utils/ie.js b/rhodecode/public/js/src/rhodecode/utils/ie.js --- a/rhodecode/public/js/src/rhodecode/utils/ie.js +++ b/rhodecode/public/js/src/rhodecode/utils/ie.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/utils/os.js b/rhodecode/public/js/src/rhodecode/utils/os.js --- a/rhodecode/public/js/src/rhodecode/utils/os.js +++ b/rhodecode/public/js/src/rhodecode/utils/os.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/utils/pyroutes.js b/rhodecode/public/js/src/rhodecode/utils/pyroutes.js --- a/rhodecode/public/js/src/rhodecode/utils/pyroutes.js +++ b/rhodecode/public/js/src/rhodecode/utils/pyroutes.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/utils/string.js b/rhodecode/public/js/src/rhodecode/utils/string.js --- a/rhodecode/public/js/src/rhodecode/utils/string.js +++ b/rhodecode/public/js/src/rhodecode/utils/string.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/public/js/src/rhodecode/utils/topics.js b/rhodecode/public/js/src/rhodecode/utils/topics.js --- a/rhodecode/public/js/src/rhodecode/utils/topics.js +++ b/rhodecode/public/js/src/rhodecode/utils/topics.js @@ -1,4 +1,4 @@ -// # Copyright (C) 2010-2018 RhodeCode GmbH +// # Copyright (C) 2010-2019 RhodeCode GmbH // # // # This program is free software: you can redistribute it and/or modify // # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/rcserver.py b/rhodecode/rcserver.py deleted file mode 100644 --- a/rhodecode/rcserver.py +++ /dev/null @@ -1,1024 +0,0 @@ -# (c) 2005 Ian Bicking and contributors; written for Paste -# (http://pythonpaste.org) Licensed under the MIT license: -# http://www.opensource.org/licenses/mit-license.php -# -# For discussion of daemonizing: -# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/278731 -# -# Code taken also from QP: http://www.mems-exchange.org/software/qp/ From -# lib/site.py - -import atexit -import errno -import fnmatch -import logging -import optparse -import os -import re -import subprocess32 -import sys -import textwrap -import threading -import time -import traceback - -from logging.config import fileConfig -import ConfigParser as configparser -from paste.deploy import loadserver -from paste.deploy import loadapp - -import rhodecode -from rhodecode.lib.compat import kill - - -def make_web_build_callback(filename): - p = subprocess32.Popen('make web-build', shell=True, - stdout=subprocess32.PIPE, - stderr=subprocess32.PIPE, - cwd=os.path.dirname(os.path.dirname(__file__))) - stdout, stderr = p.communicate() - stdout = ''.join(stdout) - stderr = ''.join(stderr) - if stdout: - print(stdout) - if stderr: - print('%s %s %s' % ('-' * 20, 'ERRORS', '-' * 20)) - print(stderr) - - -MAXFD = 1024 -HERE = os.path.dirname(os.path.abspath(__file__)) -SERVER_RUNNING_FILE = None - - -# watch those extra files for changes, server gets restarted if file changes -GLOBAL_EXTRA_FILES = { - 'rhodecode/public/css/*.less': make_web_build_callback, - 'rhodecode/public/js/src/**/*.js': make_web_build_callback, -} - - - -## HOOKS - inspired by gunicorn # - -def when_ready(server): - """ - Called just after the server is started. - """ - - def _remove_server_running_file(): - if os.path.isfile(SERVER_RUNNING_FILE): - os.remove(SERVER_RUNNING_FILE) - - if SERVER_RUNNING_FILE: - with open(SERVER_RUNNING_FILE, 'wb') as f: - f.write(str(os.getpid())) - # register cleanup of that file when server exits - atexit.register(_remove_server_running_file) - - -def setup_logging(config_uri, fileConfig=fileConfig, - configparser=configparser): - """ - Set up logging via the logging module's fileConfig function with the - filename specified via ``config_uri`` (a string in the form - ``filename#sectionname``). - - ConfigParser defaults are specified for the special ``__file__`` - and ``here`` variables, similar to PasteDeploy config loading. - """ - path, _ = _getpathsec(config_uri, None) - parser = configparser.ConfigParser() - parser.read([path]) - if parser.has_section('loggers'): - config_file = os.path.abspath(path) - return fileConfig( - config_file, - {'__file__': config_file, 'here': os.path.dirname(config_file)} - ) - - -def set_rhodecode_is_test(config_uri): - """If is_test is defined in the config file sets rhodecode.is_test.""" - path, _ = _getpathsec(config_uri, None) - parser = configparser.ConfigParser() - parser.read(path) - rhodecode.is_test = ( - parser.has_option('app:main', 'is_test') and - parser.getboolean('app:main', 'is_test')) - - -def _getpathsec(config_uri, name): - if '#' in config_uri: - path, section = config_uri.split('#', 1) - else: - path, section = config_uri, 'main' - if name: - section = name - return path, section - - -def parse_vars(args): - """ - Given variables like ``['a=b', 'c=d']`` turns it into ``{'a': - 'b', 'c': 'd'}`` - """ - result = {} - for arg in args: - if '=' not in arg: - raise ValueError( - 'Variable assignment %r invalid (no "=")' - % arg) - name, value = arg.split('=', 1) - result[name] = value - return result - - -def _match_pattern(filename): - for pattern in GLOBAL_EXTRA_FILES: - if fnmatch.fnmatch(filename, pattern): - return pattern - return False - - -def generate_extra_file_list(): - - extra_list = [] - for root, dirs, files in os.walk(HERE, topdown=True): - for fname in files: - stripped_src = os.path.join( - 'rhodecode', os.path.relpath(os.path.join(root, fname), HERE)) - - if _match_pattern(stripped_src): - extra_list.append(stripped_src) - - return extra_list - - -def run_callback_for_pattern(filename): - pattern = _match_pattern(filename) - if pattern: - _file_callback = GLOBAL_EXTRA_FILES.get(pattern) - if callable(_file_callback): - _file_callback(filename) - - -class DaemonizeException(Exception): - pass - - -class RcServerCommand(object): - - usage = '%prog config_uri [start|stop|restart|status] [var=value]' - description = """\ - This command serves a web application that uses a PasteDeploy - configuration file for the server and application. - - If start/stop/restart is given, then --daemon is implied, and it will - start (normal operation), stop (--stop-daemon), or do both. - - You can also include variable assignments like 'http_port=8080' - and then use %(http_port)s in your config files. - """ - default_verbosity = 1 - - parser = optparse.OptionParser( - usage, - description=textwrap.dedent(description) - ) - parser.add_option( - '-n', '--app-name', - dest='app_name', - metavar='NAME', - help="Load the named application (default main)") - parser.add_option( - '-s', '--server', - dest='server', - metavar='SERVER_TYPE', - help="Use the named server.") - parser.add_option( - '--server-name', - dest='server_name', - metavar='SECTION_NAME', - help=("Use the named server as defined in the configuration file " - "(default: main)")) - parser.add_option( - '--with-vcsserver', - dest='vcs_server', - action='store_true', - help=("Start the vcsserver instance together with the RhodeCode server")) - if hasattr(os, 'fork'): - parser.add_option( - '--daemon', - dest="daemon", - action="store_true", - help="Run in daemon (background) mode") - parser.add_option( - '--pid-file', - dest='pid_file', - metavar='FILENAME', - help=("Save PID to file (default to pyramid.pid if running in " - "daemon mode)")) - parser.add_option( - '--running-file', - dest='running_file', - metavar='RUNNING_FILE', - help="Create a running file after the server is initalized with " - "stored PID of process") - parser.add_option( - '--log-file', - dest='log_file', - metavar='LOG_FILE', - help="Save output to the given log file (redirects stdout)") - parser.add_option( - '--reload', - dest='reload', - action='store_true', - help="Use auto-restart file monitor") - parser.add_option( - '--reload-interval', - dest='reload_interval', - default=1, - help=("Seconds between checking files (low number can cause " - "significant CPU usage)")) - parser.add_option( - '--monitor-restart', - dest='monitor_restart', - action='store_true', - help="Auto-restart server if it dies") - parser.add_option( - '--status', - action='store_true', - dest='show_status', - help="Show the status of the (presumably daemonized) server") - parser.add_option( - '-v', '--verbose', - default=default_verbosity, - dest='verbose', - action='count', - help="Set verbose level (default "+str(default_verbosity)+")") - parser.add_option( - '-q', '--quiet', - action='store_const', - const=0, - dest='verbose', - help="Suppress verbose output") - - if hasattr(os, 'setuid'): - # I don't think these are available on Windows - parser.add_option( - '--user', - dest='set_user', - metavar="USERNAME", - help="Set the user (usually only possible when run as root)") - parser.add_option( - '--group', - dest='set_group', - metavar="GROUP", - help="Set the group (usually only possible when run as root)") - - parser.add_option( - '--stop-daemon', - dest='stop_daemon', - action='store_true', - help=('Stop a daemonized server (given a PID file, or default ' - 'pyramid.pid file)')) - - _scheme_re = re.compile(r'^[a-z][a-z]+:', re.I) - - _reloader_environ_key = 'PYTHON_RELOADER_SHOULD_RUN' - _monitor_environ_key = 'PASTE_MONITOR_SHOULD_RUN' - - possible_subcommands = ('start', 'stop', 'restart', 'status') - - def __init__(self, argv, quiet=False): - self.options, self.args = self.parser.parse_args(argv[1:]) - if quiet: - self.options.verbose = 0 - - def out(self, msg): # pragma: no cover - if self.options.verbose > 0: - print(msg) - - def get_options(self): - if (len(self.args) > 1 - and self.args[1] in self.possible_subcommands): - restvars = self.args[2:] - else: - restvars = self.args[1:] - - return parse_vars(restvars) - - def run(self): # pragma: no cover - if self.options.stop_daemon: - return self.stop_daemon() - - if not hasattr(self.options, 'set_user'): - # Windows case: - self.options.set_user = self.options.set_group = None - - # @@: Is this the right stage to set the user at? - self.change_user_group( - self.options.set_user, self.options.set_group) - - if not self.args: - self.out('Please provide configuration file as first argument, ' - 'most likely it should be production.ini') - return 2 - app_spec = self.args[0] - - if (len(self.args) > 1 - and self.args[1] in self.possible_subcommands): - cmd = self.args[1] - else: - cmd = None - - if self.options.reload: - if os.environ.get(self._reloader_environ_key): - if self.options.verbose > 1: - self.out('Running reloading file monitor') - - install_reloader(int(self.options.reload_interval), - [app_spec] + generate_extra_file_list()) - # if self.requires_config_file: - # watch_file(self.args[0]) - else: - return self.restart_with_reloader() - - if cmd not in (None, 'start', 'stop', 'restart', 'status'): - self.out( - 'Error: must give start|stop|restart (not %s)' % cmd) - return 2 - - if cmd == 'status' or self.options.show_status: - return self.show_status() - - if cmd == 'restart' or cmd == 'stop': - result = self.stop_daemon() - if result: - if cmd == 'restart': - self.out("Could not stop daemon; aborting") - else: - self.out("Could not stop daemon") - return result - if cmd == 'stop': - return result - self.options.daemon = True - - if cmd == 'start': - self.options.daemon = True - - app_name = self.options.app_name - - vars = self.get_options() - - if self.options.vcs_server: - vars['vcs.start_server'] = 'true' - - if self.options.running_file: - global SERVER_RUNNING_FILE - SERVER_RUNNING_FILE = self.options.running_file - - if not self._scheme_re.search(app_spec): - app_spec = 'config:' + app_spec - server_name = self.options.server_name - if self.options.server: - server_spec = 'egg:pyramid' - assert server_name is None - server_name = self.options.server - else: - server_spec = app_spec - base = os.getcwd() - - if getattr(self.options, 'daemon', False): - if not self.options.pid_file: - self.options.pid_file = 'pyramid.pid' - if not self.options.log_file: - self.options.log_file = 'pyramid.log' - - # Ensure the log file is writeable - if self.options.log_file: - try: - writeable_log_file = open(self.options.log_file, 'a') - except IOError as ioe: - msg = 'Error: Unable to write to log file: %s' % ioe - raise ValueError(msg) - writeable_log_file.close() - - # Ensure the pid file is writeable - if self.options.pid_file: - try: - writeable_pid_file = open(self.options.pid_file, 'a') - except IOError as ioe: - msg = 'Error: Unable to write to pid file: %s' % ioe - raise ValueError(msg) - writeable_pid_file.close() - - - if getattr(self.options, 'daemon', False): - try: - self.daemonize() - except DaemonizeException as ex: - if self.options.verbose > 0: - self.out(str(ex)) - return 2 - - if (self.options.monitor_restart - and not os.environ.get(self._monitor_environ_key)): - return self.restart_with_monitor() - - if self.options.pid_file: - self.record_pid(self.options.pid_file) - - if self.options.log_file: - stdout_log = LazyWriter(self.options.log_file, 'a') - sys.stdout = stdout_log - sys.stderr = stdout_log - logging.basicConfig(stream=stdout_log) - - log_fn = app_spec - if log_fn.startswith('config:'): - log_fn = app_spec[len('config:'):] - elif log_fn.startswith('egg:'): - log_fn = None - if log_fn: - log_fn = os.path.join(base, log_fn) - setup_logging(log_fn) - set_rhodecode_is_test(log_fn) - - server = self.loadserver(server_spec, name=server_name, - relative_to=base, global_conf=vars) - # starting hooks - app = self.loadapp(app_spec, name=app_name, relative_to=base, - global_conf=vars) - - if self.options.verbose > 0: - if hasattr(os, 'getpid'): - msg = 'Starting %s in PID %i.' % (__name__, os.getpid()) - else: - msg = 'Starting %s.' % (__name__,) - self.out(msg) - if SERVER_RUNNING_FILE: - self.out('PID file written as %s' % (SERVER_RUNNING_FILE, )) - elif not self.options.pid_file: - self.out('No PID file written by default.') - - try: - when_ready(server) - server(app) - except (SystemExit, KeyboardInterrupt) as e: - if self.options.verbose > 1: - raise - if str(e): - msg = ' ' + str(e) - else: - msg = '' - self.out('Exiting%s (-v to see traceback)' % msg) - - def loadapp(self, app_spec, name, relative_to, **kw): # pragma: no cover - return loadapp(app_spec, name=name, relative_to=relative_to, **kw) - - def loadserver(self, server_spec, name, relative_to, **kw): # pragma: no cover - return loadserver( - server_spec, name=name, relative_to=relative_to, **kw) - - def quote_first_command_arg(self, arg): # pragma: no cover - """ - There's a bug in Windows when running an executable that's - located inside a path with a space in it. This method handles - that case, or on non-Windows systems or an executable with no - spaces, it just leaves well enough alone. - """ - if sys.platform != 'win32' or ' ' not in arg: - # Problem does not apply: - return arg - try: - import win32api - except ImportError: - raise ValueError( - "The executable %r contains a space, and in order to " - "handle this issue you must have the win32api module " - "installed" % arg) - arg = win32api.GetShortPathName(arg) - return arg - - def daemonize(self): # pragma: no cover - pid = live_pidfile(self.options.pid_file) - if pid: - raise DaemonizeException( - "Daemon is already running (PID: %s from PID file %s)" - % (pid, self.options.pid_file)) - - if self.options.verbose > 0: - self.out('Entering daemon mode') - pid = os.fork() - if pid: - # The forked process also has a handle on resources, so we - # *don't* want proper termination of the process, we just - # want to exit quick (which os._exit() does) - os._exit(0) - # Make this the session leader - os.setsid() - # Fork again for good measure! - pid = os.fork() - if pid: - os._exit(0) - - # @@: Should we set the umask and cwd now? - - import resource # Resource usage information. - maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1] - if maxfd == resource.RLIM_INFINITY: - maxfd = MAXFD - # Iterate through and close all file descriptors. - for fd in range(0, maxfd): - try: - os.close(fd) - except OSError: # ERROR, fd wasn't open to begin with (ignored) - pass - - if hasattr(os, "devnull"): - REDIRECT_TO = os.devnull - else: - REDIRECT_TO = "/dev/null" - os.open(REDIRECT_TO, os.O_RDWR) # standard input (0) - # Duplicate standard input to standard output and standard error. - os.dup2(0, 1) # standard output (1) - os.dup2(0, 2) # standard error (2) - - def _remove_pid_file(self, written_pid, filename, verbosity): - current_pid = os.getpid() - if written_pid != current_pid: - # A forked process must be exiting, not the process that - # wrote the PID file - return - if not os.path.exists(filename): - return - with open(filename) as f: - content = f.read().strip() - try: - pid_in_file = int(content) - except ValueError: - pass - else: - if pid_in_file != current_pid: - msg = "PID file %s contains %s, not expected PID %s" - self.out(msg % (filename, pid_in_file, current_pid)) - return - if verbosity > 0: - self.out("Removing PID file %s" % filename) - try: - os.unlink(filename) - return - except OSError as e: - # Record, but don't give traceback - self.out("Cannot remove PID file: (%s)" % e) - # well, at least lets not leave the invalid PID around... - try: - with open(filename, 'w') as f: - f.write('') - except OSError as e: - self.out('Stale PID left in file: %s (%s)' % (filename, e)) - else: - self.out('Stale PID removed') - - def record_pid(self, pid_file): - pid = os.getpid() - if self.options.verbose > 1: - self.out('Writing PID %s to %s' % (pid, pid_file)) - with open(pid_file, 'w') as f: - f.write(str(pid)) - atexit.register(self._remove_pid_file, pid, pid_file, self.options.verbose) - - def stop_daemon(self): # pragma: no cover - pid_file = self.options.pid_file or 'pyramid.pid' - if not os.path.exists(pid_file): - self.out('No PID file exists in %s' % pid_file) - return 1 - pid = read_pidfile(pid_file) - if not pid: - self.out("Not a valid PID file in %s" % pid_file) - return 1 - pid = live_pidfile(pid_file) - if not pid: - self.out("PID in %s is not valid (deleting)" % pid_file) - try: - os.unlink(pid_file) - except (OSError, IOError) as e: - self.out("Could not delete: %s" % e) - return 2 - return 1 - for j in range(10): - if not live_pidfile(pid_file): - break - import signal - kill(pid, signal.SIGTERM) - time.sleep(1) - else: - self.out("failed to kill web process %s" % pid) - return 3 - if os.path.exists(pid_file): - os.unlink(pid_file) - return 0 - - def show_status(self): # pragma: no cover - pid_file = self.options.pid_file or 'pyramid.pid' - if not os.path.exists(pid_file): - self.out('No PID file %s' % pid_file) - return 1 - pid = read_pidfile(pid_file) - if not pid: - self.out('No PID in file %s' % pid_file) - return 1 - pid = live_pidfile(pid_file) - if not pid: - self.out('PID %s in %s is not running' % (pid, pid_file)) - return 1 - self.out('Server running in PID %s' % pid) - return 0 - - def restart_with_reloader(self): # pragma: no cover - self.restart_with_monitor(reloader=True) - - def restart_with_monitor(self, reloader=False): # pragma: no cover - if self.options.verbose > 0: - if reloader: - self.out('Starting subprocess with file monitor') - else: - self.out('Starting subprocess with monitor parent') - while 1: - args = [self.quote_first_command_arg(sys.executable)] + sys.argv - new_environ = os.environ.copy() - if reloader: - new_environ[self._reloader_environ_key] = 'true' - else: - new_environ[self._monitor_environ_key] = 'true' - proc = None - try: - try: - _turn_sigterm_into_systemexit() - proc = subprocess32.Popen(args, env=new_environ) - exit_code = proc.wait() - proc = None - except KeyboardInterrupt: - self.out('^C caught in monitor process') - if self.options.verbose > 1: - raise - return 1 - finally: - if proc is not None: - import signal - try: - kill(proc.pid, signal.SIGTERM) - except (OSError, IOError): - pass - - if reloader: - # Reloader always exits with code 3; but if we are - # a monitor, any exit code will restart - if exit_code != 3: - return exit_code - if self.options.verbose > 0: - self.out('%s %s %s' % ('-' * 20, 'Restarting', '-' * 20)) - - def change_user_group(self, user, group): # pragma: no cover - if not user and not group: - return - import pwd - import grp - uid = gid = None - if group: - try: - gid = int(group) - group = grp.getgrgid(gid).gr_name - except ValueError: - try: - entry = grp.getgrnam(group) - except KeyError: - raise ValueError( - "Bad group: %r; no such group exists" % group) - gid = entry.gr_gid - try: - uid = int(user) - user = pwd.getpwuid(uid).pw_name - except ValueError: - try: - entry = pwd.getpwnam(user) - except KeyError: - raise ValueError( - "Bad username: %r; no such user exists" % user) - if not gid: - gid = entry.pw_gid - uid = entry.pw_uid - if self.options.verbose > 0: - self.out('Changing user to %s:%s (%s:%s)' % ( - user, group or '(unknown)', uid, gid)) - if gid: - os.setgid(gid) - if uid: - os.setuid(uid) - - -class LazyWriter(object): - - """ - File-like object that opens a file lazily when it is first written - to. - """ - - def __init__(self, filename, mode='w'): - self.filename = filename - self.fileobj = None - self.lock = threading.Lock() - self.mode = mode - - def open(self): - if self.fileobj is None: - with self.lock: - self.fileobj = open(self.filename, self.mode) - return self.fileobj - - def close(self): - fileobj = self.fileobj - if fileobj is not None: - fileobj.close() - - def __del__(self): - self.close() - - def write(self, text): - fileobj = self.open() - fileobj.write(text) - fileobj.flush() - - def writelines(self, text): - fileobj = self.open() - fileobj.writelines(text) - fileobj.flush() - - def flush(self): - self.open().flush() - - -def live_pidfile(pidfile): # pragma: no cover - """ - (pidfile:str) -> int | None - Returns an int found in the named file, if there is one, - and if there is a running process with that process id. - Return None if no such process exists. - """ - pid = read_pidfile(pidfile) - if pid: - try: - kill(int(pid), 0) - return pid - except OSError as e: - if e.errno == errno.EPERM: - return pid - return None - - -def read_pidfile(filename): - if os.path.exists(filename): - try: - with open(filename) as f: - content = f.read() - return int(content.strip()) - except (ValueError, IOError): - return None - else: - return None - - -def ensure_port_cleanup( - bound_addresses, maxtries=30, sleeptime=2): # pragma: no cover - """ - This makes sure any open ports are closed. - - Does this by connecting to them until they give connection - refused. Servers should call like:: - - ensure_port_cleanup([80, 443]) - """ - atexit.register(_cleanup_ports, bound_addresses, maxtries=maxtries, - sleeptime=sleeptime) - - -def _cleanup_ports( - bound_addresses, maxtries=30, sleeptime=2): # pragma: no cover - # Wait for the server to bind to the port. - import socket - import errno - for bound_address in bound_addresses: - for attempt in range(maxtries): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - sock.connect(bound_address) - except socket.error as e: - if e.args[0] != errno.ECONNREFUSED: - raise - break - else: - time.sleep(sleeptime) - else: - raise SystemExit('Timeout waiting for port.') - sock.close() - - -def _turn_sigterm_into_systemexit(): # pragma: no cover - """ - Attempts to turn a SIGTERM exception into a SystemExit exception. - """ - try: - import signal - except ImportError: - return - def handle_term(signo, frame): - raise SystemExit - signal.signal(signal.SIGTERM, handle_term) - - -def install_reloader(poll_interval=1, extra_files=None): # pragma: no cover - """ - Install the reloading monitor. - - On some platforms server threads may not terminate when the main - thread does, causing ports to remain open/locked. The - ``raise_keyboard_interrupt`` option creates a unignorable signal - which causes the whole application to shut-down (rudely). - """ - mon = Monitor(poll_interval=poll_interval) - if extra_files is None: - extra_files = [] - mon.extra_files.extend(extra_files) - t = threading.Thread(target=mon.periodic_reload) - t.setDaemon(True) - t.start() - - -class classinstancemethod(object): - """ - Acts like a class method when called from a class, like an - instance method when called by an instance. The method should - take two arguments, 'self' and 'cls'; one of these will be None - depending on how the method was called. - """ - - def __init__(self, func): - self.func = func - self.__doc__ = func.__doc__ - - def __get__(self, obj, type=None): - return _methodwrapper(self.func, obj=obj, type=type) - - -class _methodwrapper(object): - - def __init__(self, func, obj, type): - self.func = func - self.obj = obj - self.type = type - - def __call__(self, *args, **kw): - assert not 'self' in kw and not 'cls' in kw, ( - "You cannot use 'self' or 'cls' arguments to a " - "classinstancemethod") - return self.func(*((self.obj, self.type) + args), **kw) - - -class Monitor(object): # pragma: no cover - """ - A file monitor and server restarter. - - Use this like: - - ..code-block:: Python - - install_reloader() - - Then make sure your server is installed with a shell script like:: - - err=3 - while test "$err" -eq 3 ; do - python server.py - err="$?" - done - - or is run from this .bat file (if you use Windows):: - - @echo off - :repeat - python server.py - if %errorlevel% == 3 goto repeat - - or run a monitoring process in Python (``pserve --reload`` does - this). - - Use the ``watch_file(filename)`` function to cause a reload/restart for - other non-Python files (e.g., configuration files). If you have - a dynamic set of files that grows over time you can use something like:: - - def watch_config_files(): - return CONFIG_FILE_CACHE.keys() - add_file_callback(watch_config_files) - - Then every time the reloader polls files it will call - ``watch_config_files`` and check all the filenames it returns. - """ - instances = [] - global_extra_files = [] - global_file_callbacks = [] - - def __init__(self, poll_interval): - self.module_mtimes = {} - self.keep_running = True - self.poll_interval = poll_interval - self.extra_files = list(self.global_extra_files) - self.instances.append(self) - self.file_callbacks = list(self.global_file_callbacks) - - def _exit(self): - # use os._exit() here and not sys.exit() since within a - # thread sys.exit() just closes the given thread and - # won't kill the process; note os._exit does not call - # any atexit callbacks, nor does it do finally blocks, - # flush open files, etc. In otherwords, it is rude. - os._exit(3) - - def periodic_reload(self): - while True: - if not self.check_reload(): - self._exit() - break - time.sleep(self.poll_interval) - - def check_reload(self): - filenames = list(self.extra_files) - for file_callback in self.file_callbacks: - try: - filenames.extend(file_callback()) - except: - print( - "Error calling reloader callback %r:" % file_callback) - traceback.print_exc() - for module in list(sys.modules.values()): - try: - filename = module.__file__ - except (AttributeError, ImportError): - continue - if filename is not None: - filenames.append(filename) - - for filename in filenames: - try: - stat = os.stat(filename) - if stat: - mtime = stat.st_mtime - else: - mtime = 0 - except (OSError, IOError): - continue - if filename.endswith('.pyc') and os.path.exists(filename[:-1]): - mtime = max(os.stat(filename[:-1]).st_mtime, mtime) - if not filename in self.module_mtimes: - self.module_mtimes[filename] = mtime - elif self.module_mtimes[filename] < mtime: - print("%s changed; reloading..." % filename) - run_callback_for_pattern(filename) - return False - return True - - def watch_file(self, cls, filename): - """Watch the named file for changes""" - filename = os.path.abspath(filename) - if self is None: - for instance in cls.instances: - instance.watch_file(filename) - cls.global_extra_files.append(filename) - else: - self.extra_files.append(filename) - - watch_file = classinstancemethod(watch_file) - - def add_file_callback(self, cls, callback): - """Add a callback -- a function that takes no parameters -- that will - return a list of filenames to watch for changes.""" - if self is None: - for instance in cls.instances: - instance.add_file_callback(callback) - cls.global_file_callbacks.append(callback) - else: - self.file_callbacks.append(callback) - - add_file_callback = classinstancemethod(add_file_callback) - -watch_file = Monitor.watch_file -add_file_callback = Monitor.add_file_callback - - -def main(argv=sys.argv, quiet=False): - command = RcServerCommand(argv, quiet=quiet) - return command.run() - -if __name__ == '__main__': # pragma: no cover - sys.exit(main() or 0) diff --git a/rhodecode/subscribers.py b/rhodecode/subscribers.py --- a/rhodecode/subscribers.py +++ b/rhodecode/subscribers.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2010-2018 RhodeCode GmbH +# Copyright (C) 2010-2019 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 diff --git a/rhodecode/templates/admin/auth/plugin_settings.mako b/rhodecode/templates/admin/auth/plugin_settings.mako --- a/rhodecode/templates/admin/auth/plugin_settings.mako +++ b/rhodecode/templates/admin/auth/plugin_settings.mako @@ -66,6 +66,8 @@
${h.checkbox(node.name, True, checked=defaults.get(node.name))}
%elif node.widget == "select": ${h.select(node.name, defaults.get(node.name), node.validator.choices, class_="select2AuthSetting")} + %elif node.widget == "select_with_labels": + ${h.select(node.name, defaults.get(node.name), node.choices, class_="select2AuthSetting")} %elif node.widget == "textarea":
${h.textarea(node.name, defaults.get(node.name), rows=10)}
%elif node.widget == "readonly": diff --git a/rhodecode/templates/admin/integrations/new.mako b/rhodecode/templates/admin/integrations/new.mako --- a/rhodecode/templates/admin/integrations/new.mako +++ b/rhodecode/templates/admin/integrations/new.mako @@ -4,17 +4,9 @@ <%def name="breadcrumbs_links()"> %if c.repo: - ${h.link_to('Settings',h.route_path('edit_repo', repo_name=c.repo.repo_name))} - » - ${h.link_to(_('Integrations'),request.route_url(route_name='repo_integrations_home', repo_name=c.repo.repo_name))} + ${_('Settings')} %elif c.repo_group: - ${h.link_to(_('Admin'),h.route_path('admin_home'))} - » - ${h.link_to(_('Repository Groups'),h.route_path('repo_groups'))} - » - ${h.link_to(c.repo_group.group_name,h.route_path('edit_repo_group', repo_group_name=c.repo_group.group_name))} - » - ${h.link_to(_('Integrations'),request.route_url(route_name='repo_group_integrations_home', repo_group_name=c.repo_group.group_name))} + ${_('Settings')} %else: ${h.link_to(_('Admin'),h.route_path('admin_home'))} » diff --git a/rhodecode/templates/admin/my_account/my_account.mako b/rhodecode/templates/admin/my_account/my_account.mako --- a/rhodecode/templates/admin/my_account/my_account.mako +++ b/rhodecode/templates/admin/my_account/my_account.mako @@ -28,6 +28,7 @@
  • ${_('Profile')}
  • ${_('Password')}
    • +
  • ${_('Bookmarks')}
  • ${_('Auth Tokens')}
  • ${_('SSH Keys')}
  • ${_('User Group Membership')}
    • diff --git a/rhodecode/templates/admin/my_account/my_account_auth_tokens.mako b/rhodecode/templates/admin/my_account/my_account_auth_tokens.mako --- a/rhodecode/templates/admin/my_account/my_account_auth_tokens.mako +++ b/rhodecode/templates/admin/my_account/my_account_auth_tokens.mako @@ -1,35 +1,36 @@
    -
    +

    ${_('Authentication Tokens')}

    -
    -
    +
    +

    - ${_('Each token can have a role. Token with a role can be used only in given context, ' + ${_('Authentication tokens can be used to interact with the API, or VCS-over-http. ' + 'Each token can have a role. Token with a role can be used only in given context, ' 'e.g. VCS tokens can be used together with the authtoken auth plugin for git/hg/svn operations only.')}

    - + %if c.user_auth_tokens: %for auth_token in c.user_auth_tokens: - + - +
    ${_('Token')}${_('Scope')} ${_('Description')} ${_('Role')}${_('Repository Scope')} ${_('Expiration')} ${_('Action')}
    ${auth_token.api_key}
    		${auth_token.scope_humanized} ${auth_token.description} ${auth_token.role_humanized} ${auth_token.scope_humanized} %if auth_token.expires == -1: ${_('never')} @@ -58,40 +59,41 @@
    -
    - ${h.secure_form(h.route_path('my_account_auth_tokens_add'), request=request)} -
    - -
    -
    -
    - -
    -
    - ${h.text('description', class_='medium', placeholder=_('Description'))} - ${h.hidden('lifetime')} - ${h.select('role', '', c.role_options)} +
    + ${h.secure_form(h.route_path('my_account_auth_tokens_add'), request=request)} +
    + +
    +
    +
    + +
    +
    + ${h.text('description', class_='medium', placeholder=_('Description'))} + ${h.hidden('lifetime')} + ${h.select('role', '', c.role_options)} - % if c.allow_scoped_tokens: - ${h.hidden('scope_repo_id')} - % else: - ${h.select('scope_repo_id_disabled', '', ['Scopes available in EE edition'], disabled='disabled')} - % endif -
    -

    - ${_('Repository scope works only with tokens with VCS type.')} -

    -
    -
    - ${h.submit('save',_('Add'),class_="btn")} - ${h.reset('reset',_('Reset'),class_="btn")} + % if c.allow_scoped_tokens: + ${h.hidden('scope_repo_id')} + % else: + ${h.select('scope_repo_id_disabled', '', ['Scopes available in EE edition'], disabled='disabled')} + % endif
    +

    + ${_('Repository scope works only with tokens with VCS type.')} +

    +
    +
    + ${h.submit('save',_('Add'),class_="btn")} + ${h.reset('reset',_('Reset'),class_="btn")}
    - ${h.end_form()}
    + ${h.end_form()} +
    + diff --git a/rhodecode/templates/admin/my_account/my_account_ssh_keys.mako b/rhodecode/templates/admin/my_account/my_account_ssh_keys.mako --- a/rhodecode/templates/admin/my_account/my_account_ssh_keys.mako +++ b/rhodecode/templates/admin/my_account/my_account_ssh_keys.mako @@ -55,7 +55,9 @@
    ${h.text('description', class_='medium', placeholder=_('Description'))} - ${_('Generate random RSA key')} + % if c.ssh_key_generator_enabled: + ${_('Generate random RSA key')} + % endif
    @@ -70,7 +72,7 @@ ${h.reset('reset',_('Reset'),class_="btn")}
    % if c.default_key: - ${_('Click add to use this generate SSH key')} + ${_('Click add to use this generated SSH key')} % endif
    diff --git a/rhodecode/templates/admin/repo_groups/repo_group_add.mako b/rhodecode/templates/admin/repo_groups/repo_group_add.mako --- a/rhodecode/templates/admin/repo_groups/repo_group_add.mako +++ b/rhodecode/templates/admin/repo_groups/repo_group_add.mako @@ -33,7 +33,7 @@
    - +
    ${h.text('group_name', class_="medium")} @@ -41,6 +41,15 @@
    +
    + +
    +
    + ${h.select('group_parent_id',request.GET.get('parent_group'),c.repo_groups,class_="medium")} +
    +
    + +
    @@ -55,22 +64,13 @@
    -
    -
    - -
    -
    - ${h.select('group_parent_id',request.GET.get('parent_group'),c.repo_groups,class_="medium")} -
    -
    -
    ${h.checkbox('group_copy_permissions', value="True", checked="checked")} - ${_('Copy permission settings from parent repository group.')} + ${_('Copy permissions from parent repository group.')}
    @@ -84,22 +84,22 @@ diff --git a/rhodecode/templates/admin/repo_groups/repo_group_edit.mako b/rhodecode/templates/admin/repo_groups/repo_group_edit.mako --- a/rhodecode/templates/admin/repo_groups/repo_group_edit.mako +++ b/rhodecode/templates/admin/repo_groups/repo_group_edit.mako @@ -8,26 +8,12 @@ %endif -<%def name="breadcrumbs_links()"> - ${h.link_to(_('Admin'),h.route_path('admin_home'))} - » - ${h.link_to(_('Repository Groups'),h.route_path('repo_groups'))} - %if c.repo_group.parent_group: - » ${h.link_to(c.repo_group.parent_group.name, h.route_path('repo_group_home', repo_group_name=c.repo_group.parent_group.group_name))} - %endif - » ${c.repo_group.name} +<%def name="menu_bar_nav()"> + ${self.menu_items(active='admin')} -<%def name="breadcrumbs_side_links()"> - - - -<%def name="menu_bar_nav()"> - ${self.menu_items(active='admin')} +<%def name="menu_bar_subnav()"> + ${self.repo_group_menu(active='options')} <%def name="main_content()"> @@ -35,10 +21,10 @@ <%def name="main()"> +
    - ${self.breadcrumbs()} - ${self.breadcrumbs_side_links()} + ${self.repo_group_page_title(c.repo_group)}
    @@ -58,4 +44,5 @@
    + diff --git a/rhodecode/templates/admin/repo_groups/repo_group_edit_advanced.mako b/rhodecode/templates/admin/repo_groups/repo_group_edit_advanced.mako --- a/rhodecode/templates/admin/repo_groups/repo_group_edit_advanced.mako +++ b/rhodecode/templates/admin/repo_groups/repo_group_edit_advanced.mako @@ -2,6 +2,7 @@ <% elems = [ + (_('Repository Group ID'), c.repo_group.group_id, '', ''), (_('Owner'), lambda:base.gravatar_with_user(c.repo_group.user.email), '', ''), (_('Created on'), h.format_date(c.repo_group.created_on), '', ''), (_('Is Personal Group'), c.repo_group.personal or False, '', ''), diff --git a/rhodecode/templates/admin/repo_groups/repo_group_edit_permissions.mako b/rhodecode/templates/admin/repo_groups/repo_group_edit_permissions.mako --- a/rhodecode/templates/admin/repo_groups/repo_group_edit_permissions.mako +++ b/rhodecode/templates/admin/repo_groups/repo_group_edit_permissions.mako @@ -133,7 +133,7 @@ %endfor ## USER GROUPS - %for _user_group in c.repo_group.permission_user_groups(): + %for _user_group in c.repo_group.permission_user_groups(with_members=True): ${h.radio('g_perm_%s' % _user_group.users_group_id,'group.none', checked=_user_group.permission=='group.none')} ${h.radio('g_perm_%s' % _user_group.users_group_id,'group.read', checked=_user_group.permission=='group.read')} @@ -148,6 +148,7 @@ %else: ${h.link_to_group(_user_group.users_group_name)} %endif + (${_('members')}: ${len(_user_group.members)})
    - +
    ${c.form['repo_group_name'].render(css_class='medium', oid='group_name')|n} @@ -22,7 +22,7 @@
    - +
    ${c.form['repo_group'].render(css_class='medium', oid='repo_group')|n} diff --git a/rhodecode/templates/admin/repos/repo_add_base.mako b/rhodecode/templates/admin/repos/repo_add_base.mako --- a/rhodecode/templates/admin/repos/repo_add_base.mako +++ b/rhodecode/templates/admin/repos/repo_add_base.mako @@ -6,7 +6,7 @@
    - +
    ${h.text('repo_name', class_="medium")} @@ -48,22 +48,8 @@
    -
    - -
    -
    - ${h.textarea('repo_description')} - <% metatags_url = h.literal('''meta-tags''') %> - ${_('Plain text format with support of {metatags}. Add a README file for longer descriptions').format(metatags=metatags_url)|n} - -
    -
    -
    - +
    ${h.select('repo_group',request.GET.get('parent_group'),c.repo_groups,class_="medium")} @@ -77,6 +63,20 @@
    + +
    +
    + ${h.textarea('repo_description',cols=23,rows=5,class_="medium")} + <% metatags_url = h.literal('''meta-tags''') %> + ${_('Plain text format with support of {metatags}. Add a README file for longer descriptions').format(metatags=metatags_url)|n} + +
    +
    +
    +
    @@ -90,7 +90,7 @@
    ${h.checkbox('repo_copy_permissions', value="True", checked="checked")} - ${_('Copy permission set from the parent repository group.')} + ${_('Copy permissions from parent repository group.')}
    diff --git a/rhodecode/templates/admin/repos/repo_edit.mako b/rhodecode/templates/admin/repos/repo_edit.mako --- a/rhodecode/templates/admin/repos/repo_edit.mako +++ b/rhodecode/templates/admin/repos/repo_edit.mako @@ -36,7 +36,7 @@
    ${self.repo_page_title(c.rhodecode_db_repo)} - ${self.breadcrumbs()} +
    diff --git a/rhodecode/templates/admin/repos/repo_edit_advanced.mako b/rhodecode/templates/admin/repos/repo_edit_advanced.mako --- a/rhodecode/templates/admin/repos/repo_edit_advanced.mako +++ b/rhodecode/templates/admin/repos/repo_edit_advanced.mako @@ -2,6 +2,7 @@ <% elems = [ + (_('Repository ID'), c.rhodecode_db_repo.repo_id, '', ''), (_('Owner'), lambda:base.gravatar_with_user(c.rhodecode_db_repo.user.email), '', ''), (_('Created on'), h.format_date(c.rhodecode_db_repo.created_on), '', ''), (_('Updated on'), h.format_date(c.rhodecode_db_repo.updated_on), '', ''), @@ -112,6 +113,40 @@
    +
    +
    +

    ${_('Hooks')}

    +
    +
    + + + + + % if c.ver_info_dict: + + + + + + + + + + + % else: + + + + % endif +
    ${_('Hook type')}${_('Hook version')}${_('Current version')}
    ${_('PRE HOOK')}${c.ver_info_dict['pre_version']}${c.rhodecode_version}
    ${_('POST HOOK')}${c.ver_info_dict['post_version']}${c.rhodecode_version}
    ${_('Unable to read hook information from VCS Server')}
    + + + ${_('Update Hooks')} + +
    +
    +

    ${_('Archive repository')}

    diff --git a/rhodecode/templates/admin/repos/repo_edit_permissions.mako b/rhodecode/templates/admin/repos/repo_edit_permissions.mako --- a/rhodecode/templates/admin/repos/repo_edit_permissions.mako +++ b/rhodecode/templates/admin/repos/repo_edit_permissions.mako @@ -128,7 +128,7 @@ %endfor ## USER GROUPS - %for _user_group in c.rhodecode_db_repo.permission_user_groups(): + %for _user_group in c.rhodecode_db_repo.permission_user_groups(with_members=True): ${h.radio('g_perm_%s' % _user_group.users_group_id,'repository.none', checked=_user_group.permission=='repository.none')} ${h.radio('g_perm_%s' % _user_group.users_group_id,'repository.read', checked=_user_group.permission=='repository.read')} @@ -143,6 +143,7 @@ %else: ${h.link_to_group(_user_group.users_group_name)} %endif + (${_('members')}: ${len(_user_group.members)}) ${c.form['repo_landing_commit_ref'].render(css_class='medium', oid='repo_landing_commit_ref')|n} ${c.form.render_error(request, c.form['repo_landing_commit_ref'])|n} -

    ${_('Default commit for files page, downloads, full text search index and readme')}

    +

    ${_('The default commit for file pages, downloads, full text search index, and README generation.')}

    diff --git a/rhodecode/templates/admin/settings/settings_search.mako b/rhodecode/templates/admin/settings/settings_search.mako --- a/rhodecode/templates/admin/settings/settings_search.mako +++ b/rhodecode/templates/admin/settings/settings_search.mako @@ -5,8 +5,13 @@
    % for stat in c.statistics: -
    ${stat['key']}
    -
    ${stat['value']}
    + % if stat.get('sep'): +
    +
    --
    + % else: +
    ${stat['key']}
    +
    ${stat['value']}
    + % endif % endfor
    diff --git a/rhodecode/templates/admin/settings/settings_visual.mako b/rhodecode/templates/admin/settings/settings_visual.mako --- a/rhodecode/templates/admin/settings/settings_visual.mako +++ b/rhodecode/templates/admin/settings/settings_visual.mako @@ -181,7 +181,7 @@ ${_('''Schema of clone url construction eg. '{scheme}://{user}@{netloc}/{repo}', available vars: {scheme} 'http' or 'https' sent from running RhodeCode server, {user} current user username, - {sys_user} current system user running this process, usefull for ssh, + {sys_user} current system user running this process, Useful for ssh, {hostname} hostname of this server running RhodeCode, {netloc} network location/server host of running RhodeCode server, {repo} full repository name, diff --git a/rhodecode/templates/admin/user_groups/user_group_edit_advanced.mako b/rhodecode/templates/admin/user_groups/user_group_edit_advanced.mako --- a/rhodecode/templates/admin/user_groups/user_group_edit_advanced.mako +++ b/rhodecode/templates/admin/user_groups/user_group_edit_advanced.mako @@ -2,6 +2,7 @@ <% elems = [ + (_('User Group ID'), c.user_group.users_group_id, '', ''), (_('Owner'), lambda:base.gravatar_with_user(c.user_group.user.email), '', ''), (_('Created on'), h.format_date(c.user_group.created_on), '', '',), diff --git a/rhodecode/templates/admin/user_groups/user_group_edit_perms.mako b/rhodecode/templates/admin/user_groups/user_group_edit_perms.mako --- a/rhodecode/templates/admin/user_groups/user_group_edit_perms.mako +++ b/rhodecode/templates/admin/user_groups/user_group_edit_perms.mako @@ -135,7 +135,7 @@ %endfor ## USER GROUPS - %for _user_group in c.user_group.permission_user_groups(): + %for _user_group in c.user_group.permission_user_groups(with_members=True): ${h.radio('g_perm_%s' % _user_group.users_group_id,'usergroup.none')} ${h.radio('g_perm_%s' % _user_group.users_group_id,'usergroup.read')} @@ -150,6 +150,7 @@ %else: ${h.link_to_group(_user_group.users_group_name)} %endif + (${_('members')}: ${len(_user_group.members)}) ${_('Force Password Reset')}
    - ${h.secure_form(h.route_path('user_force_password_reset', user_id=c.user.user_id), request=request)} + ${h.secure_form(h.route_path('user_disable_force_password_reset', user_id=c.user.user_id), request=request)}
    +
    +
    + + ${_("Clear the forced password change flag.")} + +
    + ${h.end_form()} + + ${h.secure_form(h.route_path('user_enable_force_password_reset', user_id=c.user.user_id), request=request)} +
    +
    @@ -51,6 +60,7 @@
    ${h.end_form()} +
    @@ -86,20 +96,16 @@
    ${h.secure_form(h.route_path('user_delete', user_id=c.user.user_id), request=request)} - +
    @@ -108,14 +114,10 @@ ${_ungettext('This user owns %s repository group.', 'This user owns %s repository groups.', len(c.user.repository_groups)) % len(c.user.repository_groups)} @@ -124,18 +126,37 @@ ${_ungettext('This user owns %s user group.', 'This user owns %s user groups.', len(c.user.user_groups)) % len(c.user.user_groups)}
    ${_ungettext('This user owns %s repository.', 'This user owns %s repositories.', len(c.user.repositories)) % len(c.user.repositories)} - %if len(c.user.repositories) > 0: - - %endif + - %if len(c.user.repositories) > 0: - - %endif +
    - %if len(c.user.repository_groups) > 0: - - %endif + - %if len(c.user.repository_groups) > 0: - - %endif +
    - %if len(c.user.user_groups) > 0: - - %endif + - %if len(c.user.user_groups) > 0: - - %endif +
    +
    + % if len(c.user.repositories) > 0 or len(c.user.repository_groups) > 0 or len(c.user.user_groups) > 0: + % endif + + ${_('New owner for detached objects')}: +
    ${base.gravatar_with_user(c.first_admin.email, 16)}
    +
    +
    + +
    +

    + ${_("When selecting the detach option, the depending objects owned by this user will be assigned to the above user.")} +
    + ${_("The delete option will delete the user and all his owned objects!")} +

    +
    + + % if c.can_delete_user_message: +

    ${c.can_delete_user_message}

    + % endif +
    + +
    - % if c.can_delete_user_message: -

    ${c.can_delete_user_message}

    - % endif - -
    - - %if len(c.user.repositories) > 0 or len(c.user.repository_groups) > 0 or len(c.user.user_groups) > 0: -

    ${_("When selecting the detach option, the depending objects owned by this user will be assigned to the `%s` super admin in the system. The delete option will delete the user's repositories!") % (c.first_admin.full_name)}

    - %endif -     -
    ${h.end_form()}
    diff --git a/rhodecode/templates/admin/users/user_edit_auth_tokens.mako b/rhodecode/templates/admin/users/user_edit_auth_tokens.mako --- a/rhodecode/templates/admin/users/user_edit_auth_tokens.mako +++ b/rhodecode/templates/admin/users/user_edit_auth_tokens.mako @@ -5,27 +5,32 @@

    - ${_('Each token can have a role. Token with a role can be used only in given context, ' + ${_('Authentication tokens can be used to interact with the API, or VCS-over-http. ' + 'Each token can have a role. Token with a role can be used only in given context, ' 'e.g. VCS tokens can be used together with the authtoken auth plugin for git/hg/svn operations only.')}

    - + %if c.user_auth_tokens: %for auth_token in c.user_auth_tokens: - - - + + +
    ${_('Token')}${_('Scope')} ${_('Description')} ${_('Role')}${_('Repository Scope')} ${_('Expiration')} ${_('Action')}
    ${auth_token.api_key}
    		${auth_token.scope_humanized}
    +
    + ${auth_token.api_key} +
    +     		${auth_token.description} ${auth_token.role_humanized} ${auth_token.scope_humanized} %if auth_token.expires == -1: ${_('never')} diff --git a/rhodecode/templates/admin/users/user_edit_ssh_keys.mako b/rhodecode/templates/admin/users/user_edit_ssh_keys.mako --- a/rhodecode/templates/admin/users/user_edit_ssh_keys.mako +++ b/rhodecode/templates/admin/users/user_edit_ssh_keys.mako @@ -50,7 +50,9 @@
    ${h.text('description', class_='medium', placeholder=_('Description'))} - ${_('Generate random RSA key')} + % if c.ssh_key_generator_enabled: + ${_('Generate random RSA key')} + % endif
    diff --git a/rhodecode/templates/admin/users/user_edit_ssh_keys_generate.mako b/rhodecode/templates/admin/users/user_edit_ssh_keys_generate.mako --- a/rhodecode/templates/admin/users/user_edit_ssh_keys_generate.mako +++ b/rhodecode/templates/admin/users/user_edit_ssh_keys_generate.mako @@ -1,14 +1,15 @@
    -

    ${_('New SSH Key generated')}

    +

    ${_('New SSH Key generation')}

    -

    - ${_('Below is a 2048 bit generated SSH RSA key.')}
    - ${_('If You wish to use it to access RhodeCode via the SSH please save the private key and click `Use this generated key` at the bottom.')} -

    -

    ${_('Private key')}

    - 
    +        %if c.ssh_enabled and c.ssh_key_generator_enabled:
+            
    
+                ${_('Below is a 2048 bit generated SSH RSA key.')}
    
+                ${_('If You wish to use it to access RhodeCode via the SSH please save the private key and click `Use this generated key` at the bottom.')}
+            
    
+            
    ${_('Private key')}
    
+            
     # Save the below content as
 # Windows: /Users/{username}/.ssh/id_rsa_rhodecode_access_priv.key
 # macOS: /Users/{yourname}/.ssh/id_rsa_rhodecode_access_priv.key
@@ -16,30 +17,35 @@
 
 # Change permissions to 0600 to make it secure, and usable.
 e.g chmod 0600 /home/{username}/.ssh/id_rsa_rhodecode_access_priv.key
-        
    
+
    -
    - -
    -
    +
    + +
    +
    -

    ${_('Public key')}

    - 
    +            
    ${_('Public key')}
    
+            
     # Save the below content as
 # Windows: /Users/{username}/.ssh/id_rsa_rhodecode_access_pub.key
 # macOS: /Users/{yourname}/.ssh/id_rsa_rhodecode_access_pub.key
 # Linux: /home/{username}/.ssh/id_rsa_rhodecode_access_pub.key
-        
    
+
    - -

    - % if hasattr(c, 'target_form_url'): - ${_('Use this generated key')}. - % else: - ${_('Use this generated key')}. - % endif - ${_('Confirmation required on the next screen')}. -

    + +

    + % if hasattr(c, 'target_form_url'): + ${_('Use this generated key')}. + % else: + ${_('Use this generated key')}. + % endif + ${_('Confirmation required on the next screen')}. +

    + % else: +

    + ${_('SSH key generator has been disabled.')} +

    + % endif
    diff --git a/rhodecode/templates/admin/users/users.mako b/rhodecode/templates/admin/users/users.mako --- a/rhodecode/templates/admin/users/users.mako +++ b/rhodecode/templates/admin/users/users.mako @@ -81,7 +81,7 @@ { data: {"_": "active", "sort": "active"}, title: "${_('Active')}", className: "td-active" }, { data: {"_": "admin", - "sort": "admin"}, title: "${_('Admin')}", className: "td-admin" }, + "sort": "admin"}, title: "${_('Super admin')}", className: "td-admin" }, { data: {"_": "extern_type", "sort": "extern_type"}, title: "${_('Auth type')}", className: "td-type" }, { data: {"_": "action", diff --git a/rhodecode/templates/base/base.mako b/rhodecode/templates/base/base.mako --- a/rhodecode/templates/base/base.mako +++ b/rhodecode/templates/base/base.mako @@ -52,7 +52,7 @@

    ## display hidden instance ID if specially defined % if c.rhodecode_instanceid: - ${_('RhodeCode instance id: %s') % c.rhodecode_instanceid} + ${_('RhodeCode instance id: {}').format(c.rhodecode_instanceid)} % endif

    @@ -180,7 +180,7 @@ %endif ## repo name with group name - ${h.breadcrumb_repo_link(c.rhodecode_db_repo)} + ${h.breadcrumb_repo_link(repo_instance)} @@ -188,7 +188,7 @@ %if repo_instance.fork:

    ${_('Fork of')} - ${repo_instance.fork.repo_name} + ${h.link_to_if(c.has_origin_repo_read_perm,repo_instance.fork.repo_name, h.route_path('repo_summary', repo_name=repo_instance.fork.repo_name))}

    %endif @@ -231,6 +231,8 @@
  • ${_('Changelog')}
  • ${_('Files')}
  • ${_('Compare')}
    • +
  • ${_('Search')}
    • + ## TODO: anderson: ideally it would have a function on the scm_instance "enable_pullrequest() and enable_fork()" %if c.rhodecode_db_repo.repo_type in ['git','hg']:
  • @@ -242,13 +244,14 @@
    • %endif +
  • ${_('Options')}
      %if h.HasRepoPermissionAll('repository.admin')(c.repo_name): -
    • ${_('Settings')}
      • +
    • ${_('Repository Settings')}
      • %endif %if c.rhodecode_db_repo.fork:
    • @@ -266,8 +269,6 @@
      • %endif -
    • ${_('Search')}
      • - %if h.HasRepoPermissionAny('repository.write','repository.admin')(c.repo_name) and c.rhodecode_db_repo.enable_locking: %if c.rhodecode_db_repo.locked[0]:
    • ${_('Unlock')}
      • @@ -296,90 +297,168 @@ +<%def name="repo_group_page_title(repo_group_instance)"> +
      +
      + ## Repository Group icon + + + ## repo name with group name + ${h.breadcrumb_repo_group_link(repo_group_instance)} +
      + + <%namespace name="dt" file="/data_table/_dt_elements.mako"/> +
      + ${dt.repo_group_desc(repo_group_instance.description_safe, repo_group_instance.personal, c.visual.stylify_metatags)} +
      + +
      + + +<%def name="repo_group_menu(active=None)"> + <% + def is_active(selected): + if selected == active: + return "active" + + is_admin = h.HasPermissionAny('hg.admin')('can create repos index page') + + gr_name = c.repo_group.group_name if c.repo_group else None + # create repositories with write permission on group is set to true + create_on_write = h.HasPermissionAny('hg.create.write_on_repogroup.true')() + group_admin = h.HasRepoGroupPermissionAny('group.admin')(gr_name, 'group admin index page') + group_write = h.HasRepoGroupPermissionAny('group.write')(gr_name, 'can write into group index page') + + %> + + +
      +
      + +
      +
      +
      + + + + + + <%def name="usermenu(active=False)"> ## USER MENU
    • - - ${gravatar(c.rhodecode_user.email, 20)} - - %if c.rhodecode_user.username != h.DEFAULT_USER: - ${c.rhodecode_user.username}
      - %else: - ${_('Sign in')} - %endif -       -       - -
      -
      - %if c.rhodecode_user.username == h.DEFAULT_USER: -

      ${_('Sign in to your account')}

      - ${h.form(h.route_path('login', _query={'came_from': h.current_route_path(request)}), needs_csrf_token=False)} -
      -
      -
      -
      - -
      -
      - ${h.text('username',class_='focus',tabindex=1)} -
      - -
      -
      -
      - - %if h.HasPermissionAny('hg.password_reset.enabled')(): - ${h.link_to(_('(Forgot password?)'),h.route_path('reset_password'), class_='pwd_reset')} - %endif -
      -
      - ${h.password('password',class_='focus',tabindex=2)} -
      + % if c.rhodecode_user.username == h.DEFAULT_USER: + + ${gravatar(c.rhodecode_user.email, 20)} + + ${_('Sign in')} + + + % else: + ## logged in user + + ${gravatar(c.rhodecode_user.email, 20)} + + ${c.rhodecode_user.username} +
      +       +       + ## subnav with menu for logged in user +
      +
      + %if c.rhodecode_user.username != h.DEFAULT_USER: +
      +
      ${gravatar(c.rhodecode_user.email, 48)}
      +
      ${c.rhodecode_user.full_name_or_username}
      +
      -
      -
      - %if h.HasPermissionAny('hg.admin', 'hg.register.auto_activate', 'hg.register.manual_activate')(): - ${h.link_to(_("Don't have an account?"),h.route_path('register'))}
      - %endif - ${h.link_to(_("Using external auth? Sign In here."),h.route_path('login'))} -
      -
      - ${h.submit('sign_in',_('Sign In'),class_="btn btn-small",tabindex=3)} -
      +
      +
        +
      1. ${h.link_to(_(u'My account'),h.route_path('my_account_profile'))}
        2. + % if c.rhodecode_user.personal_repo_group: +
      2. ${h.link_to(_(u'My personal group'), h.route_path('repo_group_home', repo_group_name=c.rhodecode_user.personal_repo_group.group_name))}
        3. + % endif +
      3. ${h.link_to(_(u'Pull Requests'), h.route_path('my_account_pullrequests'))}
        4. + ## bookmark-items +
      4. + ${_('Bookmarks')} +
        + ${_('Manage')} +
        +
        5. + % if not c.bookmark_items: +
      5. + ${_('No Bookmarks yet.')} +
        6. + % endif + % for item in c.bookmark_items: +
      6. + % if item.repository: + + % elif item.repository_group: + + % else: + + ${item.position} + ${item.title} + + % endif +
        7. + % endfor + +
      7. + ${h.secure_form(h.route_path('logout'), request=request)} + ${h.submit('log_out', _(u'Sign Out'),class_="btn btn-primary")} + ${h.end_form()} +
        8. +
      -
      -
      - ${h.end_form()} - %else: -
      -
      ${gravatar(c.rhodecode_user.email, 48)}
      -
      ${c.rhodecode_user.full_name_or_username}
      - -
      -
      -
        -
      1. ${h.link_to(_(u'My account'),h.route_path('my_account_profile'))}
        2. - % if c.rhodecode_user.personal_repo_group: -
      2. ${h.link_to(_(u'My personal group'), h.route_path('repo_group_home', repo_group_name=c.rhodecode_user.personal_repo_group.group_name))}
        3. - % endif -
      3. ${h.link_to(_(u'Pull Requests'), h.route_path('my_account_pullrequests'))}
        4. - -
      4. - ${h.secure_form(h.route_path('logout'), request=request)} - ${h.submit('log_out', _(u'Sign Out'),class_="btn btn-primary")} - ${h.end_form()} -
        5. -
      -
      - %endif -
      -
      - %if c.rhodecode_user.username != h.DEFAULT_USER: -
      - ${c.unread_notifications} -
      - % endif + %endif +
      +
      + ## unread counter +
      + ${c.unread_notifications} +
      + % endif
      • @@ -413,17 +492,17 @@ @@ -516,6 +595,26 @@ commit:efced4, to search for commits }(result, escapeMarkup); }; + var formatRepoGroupResult = function(result, container, query, escapeMarkup) { + return function(data, escapeMarkup) { + if (!data.repo_group_id){ + return data.text; // optgroup text Repositories + } + + var tmpl = ''; + var repoGroupName = data['text']; + + if(data){ + + tmpl += ' '; + + } + tmpl += escapeMarkup(repoGroupName); + return tmpl; + + }(result, escapeMarkup); + }; + var autocompleteMainFilterFormatResult = function (data, value, org_formatter) { @@ -527,10 +626,23 @@ commit:efced4, to search for commits var valueDisplay = data['value_display']; var escapeRegExChars = function (value) { - return value.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&"); + return value.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&"); }; var pattern = '(' + escapeRegExChars(value) + ')'; + var getRepoIcon = function(repo_type) { + if (repo_type === 'hg') { + return ' '; + } + else if (repo_type === 'git') { + return ' '; + } + else if (repo_type === 'svn') { + return ' '; + } + return '' + }; + // highlight match valueDisplay = Select2.util.escapeMarkup(valueDisplay); valueDisplay = valueDisplay.replace(new RegExp(pattern, 'gi'), '$1<\/strong>'); @@ -540,19 +652,16 @@ commit:efced4, to search for commits if (searchType === 'hint') { icon += ' '; } + // full text search else if (searchType === 'search') { icon += ' '; } + // repository else if (searchType === 'repo') { - if (data['repo_type'] === 'hg') { - icon += ' '; - } - else if (data['repo_type'] === 'git') { - icon += ' '; - } - else if (data['repo_type'] === 'svn') { - icon += ' '; - } + + var repoIcon = getRepoIcon(data['repo_type']); + icon += repoIcon; + if (data['private']) { icon += ' '; } @@ -560,17 +669,26 @@ commit:efced4, to search for commits icon += ' '; } } + // repository groups else if (searchType === 'repo_group') { icon += ' '; } + // user group else if (searchType === 'user_group') { icon += ' '; } else if (searchType === 'user') { icon += ''.format(data['icon_link']); } + // commit else if (searchType === 'commit') { - icon += ''; + var repo_data = data['repo_data']; + var repoIcon = getRepoIcon(repo_data['repository_type']); + if (repoIcon) { + icon += repoIcon; + } else { + icon += ''; + } } var tmpl = '
      {0}{1}
      '; @@ -594,7 +712,7 @@ commit:efced4, to search for commits $('#main_filter').autocomplete({ serviceUrl: pyroutes.url('goto_switcher_data'), - params: {"repo_group_id": templateContext.repo_group_id}, + params: {"search_context": templateContext.search_context}, minChars:2, maxHeight:400, deferRequestBy: 300, //miliseconds @@ -602,15 +720,21 @@ commit:efced4, to search for commits autoSelectFirst: true, formatResult: autocompleteMainFilterFormatResult, lookupFilter: autocompleteMainFilterResult, - onSelect: function(element, suggestion){ + onSelect: function (element, suggestion) { handleSelect(element, suggestion); return false; + }, + onSearchError: function (element, query, jqXHR, textStatus, errorThrown) { + if (jqXHR !== 'abort') { + alert("Error during search.\nError code: {0}".format(textStatus)); + window.location = ''; + } } }); showMainFilterBox = function () { $('#main_filter_help').toggle(); - } + }; @@ -637,6 +761,7 @@ commit:efced4, to search for commits ('g h', 'Goto home page'), ('g g', 'Goto my private gists page'), ('g G', 'Goto my public gists page'), + ('g 0-9', 'Goto bookmarked items from 0-9'), ('n r', 'New repository page'), ('n g', 'New gist page'), ] diff --git a/rhodecode/templates/base/root.mako b/rhodecode/templates/base/root.mako --- a/rhodecode/templates/base/root.mako +++ b/rhodecode/templates/base/root.mako @@ -7,9 +7,12 @@ go_import_header = '' if hasattr(c, 'rhodecode_db_repo'): c.template_context['repo_type'] = c.rhodecode_db_repo.repo_type c.template_context['repo_landing_commit'] = c.rhodecode_db_repo.landing_rev[1] + ## check repo context + c.template_context['repo_view_type'] = h.get_repo_view_type(request) if getattr(c, 'repo_group', None): c.template_context['repo_group_id'] = c.repo_group.group_id + c.template_context['repo_group_name'] = c.repo_group.group_name if getattr(c, 'rhodecode_user', None) and c.rhodecode_user.user_id: c.template_context['rhodecode_user']['username'] = c.rhodecode_user.username @@ -23,6 +26,12 @@ c.template_context['default_user'] = { 'username': h.DEFAULT_USER, 'user_id': 1 } +c.template_context['search_context'] = { + 'repo_group_id': c.template_context.get('repo_group_id'), + 'repo_group_name': c.template_context.get('repo_group_name'), + 'repo_name': c.template_context.get('repo_name'), + 'repo_view_type': c.template_context.get('repo_view_type'), +} %> diff --git a/rhodecode/templates/codeblocks/diffs.mako b/rhodecode/templates/codeblocks/diffs.mako --- a/rhodecode/templates/codeblocks/diffs.mako +++ b/rhodecode/templates/codeblocks/diffs.mako @@ -520,7 +520,7 @@ def get_comments_for(diff_type, comments if hasattr(filename, 'unicode_path'): filename = filename.unicode_path - if not isinstance(filename, basestring): + if not isinstance(filename, (unicode, str)): return None line_key = '{}{}'.format(line_version, line_number) ## e.g o37, n12 diff --git a/rhodecode/templates/data_table/_dt_elements.mako b/rhodecode/templates/data_table/_dt_elements.mako --- a/rhodecode/templates/data_table/_dt_elements.mako +++ b/rhodecode/templates/data_table/_dt_elements.mako @@ -88,9 +88,9 @@ %endif ##PRIVATE/PUBLIC - %if private and c.visual.show_private_icon: + %if private is True and c.visual.show_private_icon: - %elif not private and c.visual.show_public_icon: + %elif private is False and c.visual.show_public_icon: %else: diff --git a/rhodecode/templates/files/files.mako b/rhodecode/templates/files/files.mako --- a/rhodecode/templates/files/files.mako +++ b/rhodecode/templates/files/files.mako @@ -314,7 +314,7 @@ $(document).ready(function() { callbacks(); var search_GET = "${request.GET.get('search','')}"; - if (search_GET == "1") { + if (search_GET === "1") { _NODEFILTER.initFilter(); } }); diff --git a/rhodecode/templates/files/files_browser.mako b/rhodecode/templates/files/files_browser.mako --- a/rhodecode/templates/files/files_browser.mako +++ b/rhodecode/templates/files/files_browser.mako @@ -27,6 +27,14 @@ ${_('Add File')} % endif + % if c.enable_downloads: + <% at_path = '{}.zip'.format(request.GET.get('at') or c.commit.raw_id[:6]) %> +
      + + ${_('Download tree at {}').format(at_path)} + +
      + % endif
      diff --git a/rhodecode/templates/files/files_source.mako b/rhodecode/templates/files/files_source.mako --- a/rhodecode/templates/files/files_source.mako +++ b/rhodecode/templates/files/files_source.mako @@ -106,4 +106,25 @@ %endif %endif
      - \ No newline at end of file + + + diff --git a/rhodecode/templates/forks/fork.mako b/rhodecode/templates/forks/fork.mako --- a/rhodecode/templates/forks/fork.mako +++ b/rhodecode/templates/forks/fork.mako @@ -8,9 +8,7 @@ %endif -<%def name="breadcrumbs_links()"> - ${_('New Fork')} - +<%def name="breadcrumbs_links()"> <%def name="menu_bar_nav()"> ${self.menu_items(active='repositories')} @@ -24,7 +22,6 @@
      ${self.repo_page_title(c.rhodecode_db_repo)} - ${self.breadcrumbs()}
      ${h.secure_form(h.route_path('repo_fork_create',repo_name=c.rhodecode_db_repo.repo_name), request=request)} @@ -44,21 +41,6 @@
      -
      - -
      -
      - ${h.textarea('description')} - <% metatags_url = h.literal('''meta-tags''') %> - ${_('Plain text format with support of {metatags}. Add a README file for longer descriptions').format(metatags=metatags_url)|n} - -
      -
      - -
      @@ -73,13 +55,28 @@
      -
      +
      +
      + +
      +
      + ${h.textarea('description',cols=23,rows=5,class_="medium")} + <% metatags_url = h.literal('''meta-tags''') %> + ${_('Plain text format with support of {metatags}. Add a README file for longer descriptions').format(metatags=metatags_url)|n} + +
      +
      + +
      ${h.select('landing_rev','',c.landing_revs,class_="medium")} - ${_('Default commit for files page, downloads, whoosh and readme')} + ${_('The default commit for file pages, downloads, full text search index, and README generation.')}
      @@ -89,7 +86,7 @@
      ${h.checkbox('copy_permissions',value="True", checked="checked")} - ${_('Copy permissions from forked repository')} + ${_('Copy permissions from parent repository.')}
      diff --git a/rhodecode/templates/index_base.mako b/rhodecode/templates/index_base.mako --- a/rhodecode/templates/index_base.mako +++ b/rhodecode/templates/index_base.mako @@ -1,13 +1,27 @@ <%inherit file="/base/base.mako"/> + +<%def name="menu_bar_subnav()"> + % if c.repo_group: + ${self.repo_group_menu(active='home')} + % endif + + + <%def name="main()">
      -
      - ${self.breadcrumbs()} - + % if c.repo_group: + ${self.repo_group_page_title(c.repo_group)} + ## context actions +
      +
        +
      • +
      + % endif + %if c.rhodecode_user.username != h.DEFAULT_USER:
      <% @@ -15,12 +29,6 @@ create_repo = h.HasPermissionAny('hg.create.repository')('can create repository index page') create_repo_group = h.HasPermissionAny('hg.repogroup.create.true')('can create repository groups index page') create_user_group = h.HasPermissionAny('hg.usergroup.create.true')('can create user groups index page') - - gr_name = c.repo_group.group_name if c.repo_group else None - # create repositories with write permission on group is set to true - create_on_write = h.HasPermissionAny('hg.create.write_on_repogroup.true')() - group_admin = h.HasRepoGroupPermissionAny('group.admin')(gr_name, 'group admin index page') - group_write = h.HasRepoGroupPermissionAny('group.write')(gr_name, 'can write into group index page') %> %if not c.repo_group: @@ -32,17 +40,6 @@ %if is_admin or create_repo_group: ${_(u'Add Repository Group')} %endif - %else: - ##we're inside other repository group other terms apply - %if is_admin or group_admin or (group_write and create_on_write): - ${_('Add Repository')} - %endif - %if is_admin or group_admin: - ${_(u'Add Repository Group')} - %endif - %if is_admin or group_admin: - ${_('Edit Repository Group')} - %endif %endif
      %endif diff --git a/rhodecode/templates/login.mako b/rhodecode/templates/login.mako --- a/rhodecode/templates/login.mako +++ b/rhodecode/templates/login.mako @@ -25,17 +25,16 @@
      +
      + <%block name="above_login_button" />
      ${h.form(request.route_path('login', _query={'came_from': c.came_from}), needs_csrf_token=False)} @@ -47,7 +46,12 @@
      %endif - + ${h.password('password', class_='focus')} %if 'password' in errors: ${errors.get('password')} @@ -55,15 +59,25 @@ %endif ${h.checkbox('remember', value=True, checked=defaults.get('remember'))} - + <% timeout = request.registry.settings.get('beaker.session.timeout', '0') %> + % if timeout == '0': + <% remember_label = _('Remember my indefinitely') %> + % else: + <% remember_label = _('Remember me for {}').format(h.age_from_seconds(timeout)) %> + % endif + - %if h.HasPermissionAny('hg.password_reset.enabled')(): -

      - ${h.link_to(_('Forgot your password?'), h.route_path('reset_password'), class_='pwd_reset')} -

      - %elif h.HasPermissionAny('hg.password_reset.hidden')(): +

      + %if h.HasPermissionAny('hg.admin', 'hg.register.auto_activate', 'hg.register.manual_activate')(): + ${h.link_to(_("Create a new account."), request.route_path('register'), class_='new_account')} + %endif +

      + + %if not h.HasPermissionAny('hg.password_reset.enabled')(): + ## password reset hidden or disabled.

      - ${_('Password reset is disabled. Please contact ')} + ${_('Password reset is disabled.')}
      + ${_('Please contact ')} % if c.visual.rhodecode_support_url: ${_('Support')} ${_('or')} @@ -72,18 +86,18 @@

      %endif - ${h.submit('sign_in', _('Sign In'), class_="btn sign-in")} -

      - RhodeCode ${c.rhodecode_edition} -

      + ${h.submit('sign_in', _('Sign In'), class_="btn sign-in", title=_('Sign in to {}').format(c.rhodecode_edition))} + ${h.end_form()} +
      + <%block name="below_login_button" />
      diff --git a/rhodecode/templates/pullrequests/pullrequest.mako b/rhodecode/templates/pullrequests/pullrequest.mako --- a/rhodecode/templates/pullrequests/pullrequest.mako +++ b/rhodecode/templates/pullrequests/pullrequest.mako @@ -33,7 +33,7 @@
      - ${_('Pull request summary')} + ${_('Summary')}
      @@ -257,7 +257,7 @@ query.callback({results: cachedData.results}); } else { $.ajax({ - url: pyroutes.url('pullrequest_repo_destinations', {'repo_name': templateContext.repo_name}), + url: pyroutes.url('pullrequest_repo_targets', {'repo_name': templateContext.repo_name}), data: {query: query.term}, dataType: 'json', type: 'GET', diff --git a/rhodecode/templates/register.mako b/rhodecode/templates/register.mako --- a/rhodecode/templates/register.mako +++ b/rhodecode/templates/register.mako @@ -118,10 +118,7 @@ ${register_message|n}

      - ${h.submit('sign_up',_('Create Account'),class_="btn sign-in")} -

      - RhodeCode ${c.rhodecode_edition} -

      + ${h.submit('sign_up',_('Create Account'), class_="btn sign-in", title=_('Create Account in {}').format(c.rhodecode_edition))} ${h.end_form()}
      <%block name="below_register_button" /> diff --git a/rhodecode/templates/search/search.mako b/rhodecode/templates/search/search.mako --- a/rhodecode/templates/search/search.mako +++ b/rhodecode/templates/search/search.mako @@ -3,7 +3,9 @@ <%def name="title()"> %if c.repo_name: - ${_('Search inside repository %(repo_name)s') % {'repo_name': c.repo_name}} + ${_('Search inside repository {repo_name}').format(repo_name=c.repo_name)} + %elif c.repo_group_name: + ${_('Search inside repository group {repo_group_name}').format(repo_group_name=c.repo_group_name)} %else: ${_('Search inside all accessible repositories')} %endif @@ -14,19 +16,20 @@ <%def name="breadcrumbs_links()"> %if c.repo_name: - ${_('Search inside repository %(repo_name)s') % {'repo_name': c.repo_name}} + ${_('Search inside repository {repo_name}').format(repo_name=c.repo_name)} + %elif c.repo_group_name: + ${_('Search inside repository group {repo_group_name}').format(repo_group_name=c.repo_group_name)} %else: ${_('Search inside all accessible repositories')} %endif - %if c.cur_query: - » - ${c.cur_query} - %endif + <%def name="menu_bar_nav()"> %if c.repo_name: - ${self.menu_items(active='repositories')} + ${self.menu_items(active='search')} + %elif c.repo_group_name: + ${self.menu_items(active='search')} %else: ${self.menu_items(active='search')} %endif @@ -34,8 +37,26 @@ <%def name="menu_bar_subnav()"> %if c.repo_name: - ${self.repo_menu(active='options')} + ${self.repo_menu(active='search')} + %elif c.repo_group_name: + ${self.repo_group_menu(active='search')} + %endif + + +<%def name="repo_icon(db_repo)"> + %if h.is_hg(db_repo): + %endif + %if h.is_git(db_repo): + + %endif + %if h.is_svn(db_repo): + + %endif + + +<%def name="repo_group_icon()"> + <%def name="main()"> @@ -46,6 +67,12 @@ ${self.repo_page_title(c.rhodecode_db_repo)}
      ${h.form(h.route_path('search_repo',repo_name=c.repo_name),method='get')} + %elif c.repo_group_name: + +
      + ${self.repo_group_page_title(c.repo_group)} +
      + ${h.form(h.route_path('search_repo_group',repo_group_name=c.repo_group_name),method='get')} %else:
      @@ -57,12 +84,48 @@ %endif
      + ${h.text('q', c.cur_query, placeholder="Enter query...")} - ${h.select('type',c.search_type,[('content',_('File contents')), ('commit',_('Commit messages')), ('path',_('File names')),],id='id_search_type')} + ${h.select('type',c.search_type,[('content',_('Files')), ('path',_('File path')),('commit',_('Commits'))],id='id_search_type')} + ${h.hidden('max_lines', '10')} +
      +
      + + %if c.repo_name: + ${_('Global Search')} + %elif c.repo_group_name: + ${_('Global Search')} + % else: + ${_('Global Search')} + %endif + + + %if c.repo_name: + » + + ${repo_icon(c.rhodecode_db_repo)} + ${c.repo_name} + + + %elif c.repo_group_name: + » + + ${repo_group_icon()} + ${c.repo_group_name} + + %endif + + + % for search_tag in c.search_tags: +
      ${search_tag} + % endfor + +
      +
      % for error in c.errors: @@ -72,8 +135,54 @@ % endfor
      -

      ${_('Example Queries')}

      - +

      ${_('Query Langague examples')}

      +
      ${c.runtime}
      @@ -102,6 +211,12 @@ 'dropdownAutoWidth': true, 'minimumResultsForSearch': -1 }); + + $('#q').autoGrowInput({maxWidth: 920}); + + setTimeout(function() { + $('#q').keyup() + }, 1); }) diff --git a/rhodecode/templates/search/search_commit.mako b/rhodecode/templates/search/search_commit.mako --- a/rhodecode/templates/search/search_commit.mako +++ b/rhodecode/templates/search/search_commit.mako @@ -1,4 +1,7 @@ <%namespace name="base" file="/base/base.mako"/> +<%namespace name="search" file="/search/search.mako"/> + +% if c.formatted_results: @@ -17,16 +20,11 @@ %for entry in c.formatted_results: ## search results are additionally filtered, and this check is just a safe gate - % if h.HasRepoPermissionAny('repository.write','repository.read','repository.admin')(entry['repository'], 'search results commit check'): + % if c.rhodecode_user.is_admin or h.HasRepoPermissionAny('repository.write','repository.read','repository.admin')(entry['repository'], 'search results commit check'): % endif %endfor
      - %if h.get_repo_type_by_name(entry.get('repository')) == 'hg': - - %elif h.get_repo_type_by_name(entry.get('repository')) == 'git': - - %elif h.get_repo_type_by_name(entry.get('repository')) == 'svn': - - %endif + <% repo_type = entry.get('repo_type') or h.get_repo_type_by_name(entry.get('repository')) %> + ${search.repo_icon(repo_type)} ${h.link_to(entry['repository'], h.route_path('repo_summary',repo_name=entry['repository']))} @@ -50,14 +48,20 @@ - ${base.gravatar_with_user(entry['author'])} + <% + ## es6 stores this as object + author = entry['author'] + if isinstance(author, dict): + author = author['email'] + %> + ${base.gravatar_with_user(author)}
      -%if c.cur_query and c.formatted_results: +%if c.cur_query:
      ${c.formatted_results.pager('$link_previous ~2~ $link_next')}
      @@ -69,14 +73,26 @@ var cid = target_expand.data('commit-id'); if (target_expand.hasClass('open')){ - $('#c-'+cid).css({'height': '1.5em', 'white-space': 'nowrap', 'text-overflow': 'ellipsis', 'overflow':'hidden'}) - $('#t-'+cid).css({'height': 'auto', 'line-height': '.9em', 'text-overflow': 'ellipsis', 'overflow':'hidden'}) + $('#c-'+cid).css({'height': '1.5em', 'white-space': 'nowrap', 'text-overflow': 'ellipsis', 'overflow':'hidden'}); + $('#t-'+cid).css({'height': 'auto', 'line-height': '.9em', 'text-overflow': 'ellipsis', 'overflow':'hidden'}); target_expand.removeClass('open'); } else { - $('#c-'+cid).css({'height': 'auto', 'white-space': 'normal', 'text-overflow': 'initial', 'overflow':'visible'}) - $('#t-'+cid).css({'height': 'auto', 'max-height': 'none', 'text-overflow': 'initial', 'overflow':'visible'}) + $('#c-'+cid).css({'height': 'auto', 'white-space': 'normal', 'text-overflow': 'initial', 'overflow':'visible'}); + $('#t-'+cid).css({'height': 'auto', 'max-height': 'none', 'text-overflow': 'initial', 'overflow':'visible'}); target_expand.addClass('open'); } }); + + $(".message.td-description").mark( + "${c.searcher.query_to_mark(c.cur_query, 'message')}", + { + "className": 'match', + "accuracy": "complementary", + "ignorePunctuation": ":._(){}[]!'+=".split("") + } + ); + + +% endif diff --git a/rhodecode/templates/search/search_content.mako b/rhodecode/templates/search/search_content.mako --- a/rhodecode/templates/search/search_content.mako +++ b/rhodecode/templates/search/search_content.mako @@ -1,33 +1,11 @@ -<%def name="highlight_text_file(terms, text, url, line_context=3, - max_lines=10, - mimetype=None, filepath=None)"> -<% -lines = text.split('\n') -lines_of_interest = set() -matching_lines = h.get_matching_line_offsets(lines, terms) -shown_matching_lines = 0 +<%namespace name="search" file="/search/search.mako"/> -for line_number in matching_lines: - if len(lines_of_interest) < max_lines: - lines_of_interest |= set(range( - max(line_number - line_context, 0), - min(line_number + line_context, len(lines) + 1))) - shown_matching_lines += 1 - -%> -${h.code_highlight( - text, - h.get_lexer_safe( - mimetype=mimetype, - filepath=filepath, - ), - h.SearchContentCodeHtmlFormatter( - linenos=True, - cssclass="code-highlight", - url=url, - query_terms=terms, - only_line_numbers=lines_of_interest -))|n} +<%def name="highlight_text_file(has_matched_content, file_content, lexer, html_formatter, matching_lines, shown_matching_lines, url, use_hl_filter)"> +% if has_matched_content: + ${h.code_highlight(file_content, lexer, html_formatter, use_hl_filter=use_hl_filter)|n} +% else: + ${_('No content matched')}
      +% endif %if len(matching_lines) > shown_matching_lines: @@ -37,47 +15,145 @@ for line_number in matching_lines:
      +<% query_mark = c.searcher.query_to_mark(c.cur_query, 'content') %> + %for entry in c.formatted_results: + + <% + file_content = entry['content_highlight'] or entry['content'] + mimetype = entry.get('mimetype') + filepath = entry.get('path') + max_lines = h.safe_int(request.GET.get('max_lines', '10')) + line_context = h.safe_int(request.GET.get('line_contenxt', '3')) + + match_file_url=h.route_path('repo_files',repo_name=entry['repository'], commit_id=entry.get('commit_id', 'tip'),f_path=entry['f_path'], _query={"mark": query_mark}) + terms = c.cur_query + + if c.searcher.is_es_6: + # use empty terms so we default to markers usage + total_lines, matching_lines = h.get_matching_line_offsets(file_content, terms=None) + else: + total_lines, matching_lines = h.get_matching_line_offsets(file_content, terms) + + shown_matching_lines = 0 + lines_of_interest = set() + for line_number in matching_lines: + if len(lines_of_interest) < max_lines: + lines_of_interest |= set(range( + max(line_number - line_context, 0), + min(line_number + line_context, total_lines + 1))) + shown_matching_lines += 1 + lexer = h.get_lexer_safe(mimetype=mimetype, filepath=filepath) + + html_formatter = h.SearchContentCodeHtmlFormatter( + linenos=True, + cssclass="code-highlight", + url=match_file_url, + query_terms=terms, + only_line_numbers=lines_of_interest + ) + + has_matched_content = len(lines_of_interest) >= 1 + + %> ## search results are additionally filtered, and this check is just a safe gate - % if h.HasRepoPermissionAny('repository.write','repository.read','repository.admin')(entry['repository'], 'search results content check'): + % if c.rhodecode_user.is_admin or h.HasRepoPermissionAny('repository.write','repository.read','repository.admin')(entry['repository'], 'search results content check'):
      -

      - %if h.get_repo_type_by_name(entry.get('repository')) == 'hg': - - %elif h.get_repo_type_by_name(entry.get('repository')) == 'git': - - %elif h.get_repo_type_by_name(entry.get('repository')) == 'svn': - - %endif +

      + <% repo_type = entry.get('repo_type') or h.get_repo_type_by_name(entry.get('repository')) %> + ${search.repo_icon(repo_type)} ${h.link_to(entry['repository'], h.route_path('repo_summary',repo_name=entry['repository']))} -

      -
      - ${h.link_to(h.literal(entry['f_path']), h.route_path('repo_files',repo_name=entry['repository'],commit_id=entry.get('commit_id', 'tip'),f_path=entry['f_path']))} - %if entry.get('lines'): - | ${entry.get('lines', 0.)} ${_ungettext('line', 'lines', entry.get('lines', 0.))} - %endif - %if entry.get('size'): - | ${h.format_byte_size_binary(entry['size'])} - %endif - %if entry.get('mimetype'): - | ${entry.get('mimetype', "unknown mimetype")} - %endif + + ## level 1 +
      + +
      + + + + ${h.link_to(h.literal(entry['f_path']), h.route_path('repo_files',repo_name=entry['repository'],commit_id=entry.get('commit_id', 'tip'),f_path=entry['f_path']))} + + + + + +
      + +
      +
      + + ${_('Show Full History')} + + | ${h.link_to(_('Annotation'), h.route_path('repo_files:annotated', repo_name=entry.get('repository',''),commit_id=entry.get('commit_id', 'tip'),f_path=entry.get('f_path','')))} + | ${h.link_to(_('Raw'), h.route_path('repo_file_raw', repo_name=entry.get('repository',''),commit_id=entry.get('commit_id', 'tip'),f_path=entry.get('f_path','')))} + | ${h.link_to(_('Download'), h.route_path('repo_file_download',repo_name=entry.get('repository',''),commit_id=entry.get('commit_id', 'tip'),f_path=entry.get('f_path','')))} +
      +
      +
      -
      - - ${_('Show Full History')} - - | ${h.link_to(_('Annotation'), h.route_path('repo_files:annotated', repo_name=entry.get('repository',''),commit_id=entry.get('commit_id', 'tip'),f_path=entry.get('f_path','')))} - | ${h.link_to(_('Raw'), h.route_path('repo_file_raw', repo_name=entry.get('repository',''),commit_id=entry.get('commit_id', 'tip'),f_path=entry.get('f_path','')))} - | ${h.link_to(_('Download'), h.route_path('repo_file_download',repo_name=entry.get('repository',''),commit_id=entry.get('commit_id', 'tip'),f_path=entry.get('f_path','')))} + ## level 2 +
      + +
      + + %if entry.get('lines'): + ${entry.get('lines', 0.)} ${_ungettext('line', 'lines', entry.get('lines', 0.))} + (${len(matching_lines)} ${_ungettext('matched', 'matched', len(matching_lines))}) + %endif + + + + %if entry.get('size'): + | ${h.format_byte_size_binary(entry['size'])} + %endif + + + + %if entry.get('mimetype'): + | ${entry.get('mimetype', "unknown mimetype")} + %endif + +
      + +
      +
      + + <% repo_group = entry.get('repository_group')%> + ## hiden if in repo group view + % if repo_group and not c.repo_group_name: + + ${search.repo_group_icon()} + ${_('Narrow to this repository group')} + + % endif + ## hiden if in repo view + % if not c.repo_name: + + ${search.repo_icon(repo_type)} + ${_('Narrow to this repository')} + + % endif +
      +
      +
      +
      - ${highlight_text_file(c.cur_query, entry['content'], - url=h.route_path('repo_files',repo_name=entry['repository'],commit_id=entry.get('commit_id', 'tip'),f_path=entry['f_path']), - mimetype=entry.get('mimetype'), filepath=entry.get('path'))} + + ${highlight_text_file( + has_matched_content=has_matched_content, + file_content=file_content, + lexer=lexer, + html_formatter=html_formatter, + matching_lines=matching_lines, + shown_matching_lines=shown_matching_lines, + url=match_file_url, + use_hl_filter=c.searcher.is_es_6 + )}
      +
      % endif %endfor @@ -91,10 +167,14 @@ for line_number in matching_lines: %if c.cur_query: -%endif \ No newline at end of file +%endif diff --git a/rhodecode/templates/search/search_path.mako b/rhodecode/templates/search/search_path.mako --- a/rhodecode/templates/search/search_path.mako +++ b/rhodecode/templates/search/search_path.mako @@ -1,34 +1,46 @@ +<%namespace name="search" file="/search/search.mako"/> + +% if c.formatted_results: + - ##TODO: add 'Last Change' and 'Author' here + + %for entry in c.formatted_results: ## search results are additionally filtered, and this check is just a safe gate - % if h.HasRepoPermissionAny('repository.write','repository.read','repository.admin')(entry['repository'], 'search results path check'): + % if c.rhodecode_user.is_admin or h.HasRepoPermissionAny('repository.write','repository.read','repository.admin')(entry['repository'], 'search results path check'): + + % endif %endfor
      ${_('Repository')} ${_('File')}${_('Size')}${_('Lines')}
      - %if h.get_repo_type_by_name(entry.get('repository')) == 'hg': - - %elif h.get_repo_type_by_name(entry.get('repository')) == 'git': - - %elif h.get_repo_type_by_name(entry.get('repository')) == 'svn': - - %endif + <% repo_type = entry.get('repo_type') or h.get_repo_type_by_name(entry.get('repository')) %> + ${search.repo_icon(repo_type)} ${h.link_to(entry['repository'], h.route_path('repo_summary',repo_name=entry['repository']))} ${h.link_to(h.literal(entry['f_path']), h.route_path('repo_files',repo_name=entry['repository'],commit_id='tip',f_path=entry['f_path']))} + %if entry.get('size'): + ${h.format_byte_size_binary(entry['size'])} + %endif + + %if entry.get('lines'): + ${entry.get('lines', 0.)} ${_ungettext('line', 'lines', entry.get('lines', 0.))} + %endif +
      -%if c.cur_query and c.formatted_results: +%if c.cur_query:
      ${c.formatted_results.pager('$link_previous ~2~ $link_next')}
      -%endif \ No newline at end of file +%endif + +% endif diff --git a/rhodecode/templates/summary/components.mako b/rhodecode/templates/summary/components.mako --- a/rhodecode/templates/summary/components.mako +++ b/rhodecode/templates/summary/components.mako @@ -83,9 +83,9 @@
      - +