diffs: use whole chunk diff to calculate if it's oversized or not.

- This fixes an issue if a file is added that has very large number of small lines.
In this case the time to detect if the diff should be limited was very very long and CPU intensive.

configs/development.ini

@@ -1,733 +1,739 b''
 
 
 ################################################################################
 ##                 RHODECODE COMMUNITY EDITION CONFIGURATION                  ##
 # The %(here)s variable will be replaced with the parent directory of this file#
 ################################################################################
 
 [DEFAULT]
 debug = true
 
 ################################################################################
 ##                            EMAIL CONFIGURATION                             ##
 ## Uncomment and replace with the email address which should receive          ##
 ## any error reports after an application crash                               ##
 ## Additionally these settings will be used by the RhodeCode mailing system   ##
 ################################################################################
 
 ## prefix all emails subjects with given prefix, helps filtering out emails
 #email_prefix = [RhodeCode]
 
 ## email FROM address all mails will be sent
 #app_email_from = rhodecode-noreply@localhost
 
 ## Uncomment and replace with the address which should receive any error report
 ## note: using appenlight for error handling doesn't need this to be uncommented
 #email_to = admin@localhost
 
 ## in case of Application errors, sent an error email form
 #error_email_from = rhodecode_error@localhost
 
 ## additional error message to be send in case of server crash
 #error_message =
 
 
 #smtp_server = mail.server.com
 #smtp_username =
 #smtp_password =
 #smtp_port =
 #smtp_use_tls = false
 #smtp_use_ssl = true
 ## Specify available auth parameters here (e.g. LOGIN PLAIN CRAM-MD5, etc.)
 #smtp_auth =
 
 [server:main]
 ## COMMON ##
 host = 127.0.0.1
 port = 5000
 
 ##################################
 ##     WAITRESS WSGI SERVER     ##
 ## Recommended for Development  ##
 ##################################
 
 use = egg:waitress#main
 ## number of worker threads
 threads = 5
 ## MAX BODY SIZE 100GB
 max_request_body_size = 107374182400
 ## Use poll instead of select, fixes file descriptors limits problems.
 ## May not work on old windows systems.
 asyncore_use_poll = true
 
 
 ##########################
 ## GUNICORN WSGI SERVER ##
 ##########################
 ## run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
 
 #use = egg:gunicorn#main
 ## Sets the number of process workers. You must set `instance_id = *`
 ## when this option is set to more than one worker, recommended
 ## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
 ## The `instance_id = *` must be set in the [app:main] section below
 #workers = 2
 ## number of threads for each of the worker, must be set to 1 for gevent
 ## generally recommened to be at 1
 #threads = 1
 ## process name
 #proc_name = rhodecode
 ## type of worker class, one of sync, gevent
 ## recommended for bigger setup is using of of other than sync one
 #worker_class = sync
 ## The maximum number of simultaneous clients. Valid only for Gevent
 #worker_connections = 10
 ## max number of requests that worker will handle before being gracefully
 ## restarted, could prevent memory leaks
 #max_requests = 1000
 #max_requests_jitter = 30
 ## amount of time a worker can spend with handling a request before it
 ## gets killed and restarted. Set to 6hrs
 #timeout = 21600
 
 
 ## prefix middleware for RhodeCode.
 ## recommended when using proxy setup.
 ## allows to set RhodeCode under a prefix in server.
 ## eg https://server.com/custom_prefix. Enable `filter-with =` option below as well.
 ## And set your prefix like: `prefix = /custom_prefix`
 ## be sure to also set beaker.session.cookie_path = /custom_prefix if you need
 ## to make your cookies only work on prefix url
 [filter:proxy-prefix]
 use = egg:PasteDeploy#prefix
 prefix = /
 
 [app:main]
 use = egg:rhodecode-enterprise-ce
 
 ## enable proxy prefix middleware, defined above
 #filter-with = proxy-prefix
 
 # During development the we want to have the debug toolbar enabled
 pyramid.includes =
     pyramid_debugtoolbar
     rhodecode.utils.debugtoolbar
     rhodecode.lib.middleware.request_wrapper
 
 pyramid.reload_templates = true
 
 debugtoolbar.hosts = 0.0.0.0/0
 debugtoolbar.exclude_prefixes =
     /css
     /fonts
     /images
     /js
 
 ## RHODECODE PLUGINS ##
 rhodecode.includes =
     rhodecode.api
 
 
 # api prefix url
 rhodecode.api.url = /_admin/api
 
 
 ## END RHODECODE PLUGINS ##
 
 ## encryption key used to encrypt social plugin tokens,
 ## remote_urls with credentials etc, if not set it defaults to
 ## `beaker.session.secret`
 #rhodecode.encrypted_values.secret =
 
 ## decryption strict mode (enabled by default). It controls if decryption raises
 ## `SignatureVerificationError` in case of wrong key, or damaged encryption data.
 #rhodecode.encrypted_values.strict = false
 
 ## return gzipped responses from Rhodecode (static files/application)
 gzip_responses = false
 
 ## autogenerate javascript routes file on startup
 generate_js_files = false
 
 ## Optional Languages
 ## en(default), be, de, es, fr, it, ja, pl, pt, ru, zh
 lang = en
 
 ## perform a full repository scan on each server start, this should be
 ## set to false after first startup, to allow faster server restarts.
 startup.import_repos = false
 
 ## Uncomment and set this path to use archive download cache.
 ## Once enabled, generated archives will be cached at this location
 ## and served from the cache during subsequent requests for the same archive of
 ## the repository.
 #archive_cache_dir = /tmp/tarballcache
 
 ## change this to unique ID for security
 app_instance_uuid = rc-production
 
-## cut off limit for large diffs (size in bytes)
-cut_off_limit_diff = 1024000
-cut_off_limit_file = 256000
+## cut off limit for large diffs (size in bytes). If overall diff size on
+## commit, or pull request exceeds this limit this diff will be displayed
+## partially. E.g 512000 == 512Kb
+cut_off_limit_diff = 512000
+
+## cut off limit for large files inside diffs (size in bytes). Each individual
+## file inside diff which exceeds this limit will be displayed partially.
+##  E.g 128000 == 128Kb
+cut_off_limit_file = 128000
 
 ## use cache version of scm repo everywhere
 vcs_full_cache = true
 
 ## force https in RhodeCode, fixes https redirects, assumes it's always https
 ## Normally this is controlled by proper http flags sent from http server
 force_https = false
 
 ## use Strict-Transport-Security headers
 use_htsts = false
 
 ## number of commits stats will parse on each iteration
 commit_parse_limit = 25
 
 ## git rev filter option, --all is the default filter, if you need to
 ## hide all refs in changelog switch this to --branches --tags
 git_rev_filter = --branches --tags
 
 # Set to true if your repos are exposed using the dumb protocol
 git_update_server_info = false
 
 ## RSS/ATOM feed options
 rss_cut_off_limit = 256000
 rss_items_per_page = 10
 rss_include_diff = false
 
 ## gist URL alias, used to create nicer urls for gist. This should be an
 ## url that does rewrites to _admin/gists/{gistid}.
 ## example: http://gist.rhodecode.org/{gistid}. Empty means use the internal
 ## RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid}
 gist_alias_url =
 
 ## List of views (using glob pattern syntax) that AUTH TOKENS could be
 ## used for access.
 ## Adding ?auth_token=TOKEN_HASH to the url authenticates this request as if it
 ## came from the the logged in user who own this authentication token.
 ## Additionally @TOKEN syntaxt can be used to bound the view to specific
 ## authentication token. Such view would be only accessible when used together
 ## with this authentication token
 ##
 ## list of all views can be found under `/_admin/permissions/auth_token_access`
 ## The list should be "," separated and on a single line.
 ##
 ## Most common views to enable:
 #    RepoCommitsView:repo_commit_download
 #    RepoCommitsView:repo_commit_patch
 #    RepoCommitsView:repo_commit_raw
 #    RepoCommitsView:repo_commit_raw@TOKEN
 #    RepoFilesView:repo_files_diff
 #    RepoFilesView:repo_archivefile
 #    RepoFilesView:repo_file_raw
 #    GistView:*
 api_access_controllers_whitelist =
 
 ## default encoding used to convert from and to unicode
 ## can be also a comma separated list of encoding in case of mixed encodings
 default_encoding = UTF-8
 
 ## instance-id prefix
 ## a prefix key for this instance used for cache invalidation when running
 ## multiple instances of rhodecode, make sure it's globally unique for
 ## all running rhodecode instances. Leave empty if you don't use it
 instance_id =
 
 ## Fallback authentication plugin. Set this to a plugin ID to force the usage
 ## of an authentication plugin also if it is disabled by it's settings.
 ## This could be useful if you are unable to log in to the system due to broken
 ## authentication settings. Then you can enable e.g. the internal rhodecode auth
 ## module to log in again and fix the settings.
 ##
 ## Available builtin plugin IDs (hash is part of the ID):
 ## egg:rhodecode-enterprise-ce#rhodecode
 ## egg:rhodecode-enterprise-ce#pam
 ## egg:rhodecode-enterprise-ce#ldap
 ## egg:rhodecode-enterprise-ce#jasig_cas
 ## egg:rhodecode-enterprise-ce#headers
 ## egg:rhodecode-enterprise-ce#crowd
 #rhodecode.auth_plugin_fallback = egg:rhodecode-enterprise-ce#rhodecode
 
 ## alternative return HTTP header for failed authentication. Default HTTP
 ## response is 401 HTTPUnauthorized. Currently HG clients have troubles with
 ## handling that causing a series of failed authentication calls.
 ## Set this variable to 403 to return HTTPForbidden, or any other HTTP code
 ## This will be served instead of default 401 on bad authnetication
 auth_ret_code =
 
 ## use special detection method when serving auth_ret_code, instead of serving
 ## ret_code directly, use 401 initially (Which triggers credentials prompt)
 ## and then serve auth_ret_code to clients
 auth_ret_code_detection = false
 
 ## locking return code. When repository is locked return this HTTP code. 2XX
 ## codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 
 ## allows to change the repository location in settings page
 allow_repo_location_change = true
 
 ## allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
 
 ## generated license token, goto license page in RhodeCode settings to obtain
 ## new token
 license_token =
 
 ## supervisor connection uri, for managing supervisor and logs.
 supervisor.uri =
 ## supervisord group name/id we only want this RC instance to handle
 supervisor.group_id = dev
 
 ## Display extended labs settings
 labs_settings_active = true
 
 ####################################
 ###        CELERY CONFIG        ####
 ####################################
 use_celery = false
 broker.host = localhost
 broker.vhost = rabbitmqhost
 broker.port = 5672
 broker.user = rabbitmq
 broker.password = qweqwe
 
 celery.imports = rhodecode.lib.celerylib.tasks
 
 celery.result.backend = amqp
 celery.result.dburi = amqp://
 celery.result.serialier = json
 
 #celery.send.task.error.emails = true
 #celery.amqp.task.result.expires = 18000
 
 celeryd.concurrency = 2
 #celeryd.log.file = celeryd.log
 celeryd.log.level = debug
 celeryd.max.tasks.per.child = 1
 
 ## tasks will never be sent to the queue, but executed locally instead.
 celery.always.eager = false
 
 ####################################
 ###         BEAKER CACHE        ####
 ####################################
 # default cache dir for templates.  Putting this into a ramdisk
 ## can boost performance, eg. %(here)s/data_ramdisk
 cache_dir = %(here)s/data
 
 ## locking and default file storage for Beaker. Putting this into a ramdisk
 ## can boost performance, eg. %(here)s/data_ramdisk/cache/beaker_data
 beaker.cache.data_dir = %(here)s/data/cache/beaker_data
 beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
 
 beaker.cache.regions = super_short_term, short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
 
 beaker.cache.super_short_term.type = memory
 beaker.cache.super_short_term.expire = 10
 beaker.cache.super_short_term.key_length = 256
 
 beaker.cache.short_term.type = memory
 beaker.cache.short_term.expire = 60
 beaker.cache.short_term.key_length = 256
 
 beaker.cache.long_term.type = memory
 beaker.cache.long_term.expire = 36000
 beaker.cache.long_term.key_length = 256
 
 beaker.cache.sql_cache_short.type = memory
 beaker.cache.sql_cache_short.expire = 10
 beaker.cache.sql_cache_short.key_length = 256
 
 ## default is memory cache, configure only if required
 ## using multi-node or multi-worker setup
 #beaker.cache.auth_plugins.type = ext:database
 #beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
 #beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
 #beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
 #beaker.cache.auth_plugins.sa.pool_recycle = 3600
 #beaker.cache.auth_plugins.sa.pool_size = 10
 #beaker.cache.auth_plugins.sa.max_overflow = 0
 
 beaker.cache.repo_cache_long.type = memorylru_base
 beaker.cache.repo_cache_long.max_items = 4096
 beaker.cache.repo_cache_long.expire = 2592000
 
 ## default is memorylru_base cache, configure only if required
 ## using multi-node or multi-worker setup
 #beaker.cache.repo_cache_long.type = ext:memcached
 #beaker.cache.repo_cache_long.url = localhost:11211
 #beaker.cache.repo_cache_long.expire = 1209600
 #beaker.cache.repo_cache_long.key_length = 256
 
 ####################################
 ###       BEAKER SESSION        ####
 ####################################
 
 ## .session.type is type of storage options for the session, current allowed
 ## types are file, ext:memcached, ext:database, and memory (default).
 beaker.session.type = file
 beaker.session.data_dir = %(here)s/data/sessions/data
 
 ## db based session, fast, and allows easy management over logged in users
 #beaker.session.type = ext:database
 #beaker.session.table_name = db_session
 #beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode
 #beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode
 #beaker.session.sa.pool_recycle = 3600
 #beaker.session.sa.echo = false
 
 beaker.session.key = rhodecode
 beaker.session.secret = develop-rc-uytcxaz
 beaker.session.lock_dir = %(here)s/data/sessions/lock
 
 ## Secure encrypted cookie. Requires AES and AES python libraries
 ## you must disable beaker.session.secret to use this
 #beaker.session.encrypt_key = key_for_encryption
 #beaker.session.validate_key = validation_key
 
 ## sets session as invalid(also logging out user) if it haven not been
 ## accessed for given amount of time in seconds
 beaker.session.timeout = 2592000
 beaker.session.httponly = true
 ## Path to use for the cookie. Set to prefix if you use prefix middleware
 #beaker.session.cookie_path = /custom_prefix
 
 ## uncomment for https secure cookie
 beaker.session.secure = false
 
 ## auto save the session to not to use .save()
 beaker.session.auto = false
 
 ## default cookie expiration time in seconds, set to `true` to set expire
 ## at browser close
 #beaker.session.cookie_expires = 3600
 
 ###################################
 ## SEARCH INDEXING CONFIGURATION ##
 ###################################
 ## Full text search indexer is available in rhodecode-tools under
 ## `rhodecode-tools index` command
 
 ## WHOOSH Backend, doesn't require additional services to run
 ## it works good with few dozen repos
 search.module = rhodecode.lib.index.whoosh
 search.location = %(here)s/data/index
 
 ########################################
 ###    CHANNELSTREAM CONFIG         ####
 ########################################
 ## channelstream enables persistent connections and live notification
 ## in the system. It's also used by the chat system
 channelstream.enabled = false
 
 ## server address for channelstream server on the backend
 channelstream.server = 127.0.0.1:9800
 
 ## location of the channelstream server from outside world
 ## use ws:// for http or wss:// for https. This address needs to be handled
 ## by external HTTP server such as Nginx or Apache
 ## see nginx/apache configuration examples in our docs
 channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream
 channelstream.secret = secret
 channelstream.history.location = %(here)s/channelstream_history
 
 ## Internal application path that Javascript uses to connect into.
 ## If you use proxy-prefix the prefix should be added before /_channelstream
 channelstream.proxy_path = /_channelstream
 
 
 ###################################
 ##       APPENLIGHT CONFIG       ##
 ###################################
 
 ## Appenlight is tailored to work with RhodeCode, see
 ## http://appenlight.com for details how to obtain an account
 
 ## appenlight integration enabled
 appenlight = false
 
 appenlight.server_url = https://api.appenlight.com
 appenlight.api_key = YOUR_API_KEY
 #appenlight.transport_config = https://api.appenlight.com?threaded=1&timeout=5
 
 # used for JS client
 appenlight.api_public_key = YOUR_API_PUBLIC_KEY
 
 ## TWEAK AMOUNT OF INFO SENT HERE
 
 ## enables 404 error logging (default False)
 appenlight.report_404 = false
 
 ## time in seconds after request is considered being slow (default 1)
 appenlight.slow_request_time = 1
 
 ## record slow requests in application
 ## (needs to be enabled for slow datastore recording and time tracking)
 appenlight.slow_requests = true
 
 ## enable hooking to application loggers
 appenlight.logging = true
 
 ## minimum log level for log capture
 appenlight.logging.level = WARNING
 
 ## send logs only from erroneous/slow requests
 ## (saves API quota for intensive logging)
 appenlight.logging_on_error = false
 
 ## list of additonal keywords that should be grabbed from environ object
 ## can be string with comma separated list of words in lowercase
 ## (by default client will always send following info:
 ## 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that
 ## start with HTTP* this list be extended with additional keywords here
 appenlight.environ_keys_whitelist =
 
 ## list of keywords that should be blanked from request object
 ## can be string with comma separated list of words in lowercase
 ## (by default client will always blank keys that contain following words
 ## 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf'
 ## this list be extended with additional keywords set here
 appenlight.request_keys_blacklist =
 
 ## list of namespaces that should be ignores when gathering log entries
 ## can be string with comma separated list of namespaces
 ## (by default the client ignores own entries: appenlight_client.client)
 appenlight.log_namespace_blacklist =
 
 
 ################################################################################
 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 ## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 ## execute malicious code after an exception is raised.                       ##
 ################################################################################
 #set debug = false
 
 
 ##############
 ## STYLING  ##
 ##############
 debug_style = true
 
 ###########################################
 ###   MAIN RHODECODE DATABASE CONFIG    ###
 ###########################################
 #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 #sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 #sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode
 sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 
 # see sqlalchemy docs for other advanced settings
 
 ## print the sql statements to output
 sqlalchemy.db1.echo = false
 ## recycle the connections after this amount of seconds
 sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.db1.convert_unicode = true
 
 ## the number of connections to keep open inside the connection pool.
 ## 0 indicates no limit
 #sqlalchemy.db1.pool_size = 5
 
 ## the number of connections to allow in connection pool "overflow", that is
 ## connections that can be opened above and beyond the pool_size setting,
 ## which defaults to five.
 #sqlalchemy.db1.max_overflow = 10
 
 
 ##################
 ### VCS CONFIG ###
 ##################
 vcs.server.enable = true
 vcs.server = localhost:9900
 
 ## Web server connectivity protocol, responsible for web based VCS operatations
 ## Available protocols are:
 ## `http` - use http-rpc backend (default)
 vcs.server.protocol = http
 
 ## Push/Pull operations protocol, available options are:
 ## `http` - use http-rpc backend (default)
 ##
 vcs.scm_app_implementation = http
 
 ## Push/Pull operations hooks protocol, available options are:
 ## `http` - use http-rpc backend (default)
 vcs.hooks.protocol = http
 
 vcs.server.log_level = debug
 ## Start VCSServer with this instance as a subprocess, usefull for development
 vcs.start_server = true
 
 ## List of enabled VCS backends, available options are:
 ## `hg`  - mercurial
 ## `git` - git
 ## `svn` - subversion
 vcs.backends = hg, git, svn
 
 vcs.connection_timeout = 3600
 ## Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
 ## Available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
 #vcs.svn.compatible_version = pre-1.8-compatible
 
 
 ############################################################
 ### Subversion proxy support (mod_dav_svn)               ###
 ### Maps RhodeCode repo groups into SVN paths for Apache ###
 ############################################################
 ## Enable or disable the config file generation.
 svn.proxy.generate_config = false
 ## Generate config file with `SVNListParentPath` set to `On`.
 svn.proxy.list_parent_path = true
 ## Set location and file name of generated config file.
 svn.proxy.config_file_path = %(here)s/mod_dav_svn.conf
 ## Used as a prefix to the `Location` block in the generated config file.
 ## In most cases it should be set to `/`.
 svn.proxy.location_root = /
 ## Command to reload the mod dav svn configuration on change.
 ## Example: `/etc/init.d/apache2 reload`
 #svn.proxy.reload_cmd = /etc/init.d/apache2 reload
 ## If the timeout expires before the reload command finishes, the command will
 ## be killed. Setting it to zero means no timeout. Defaults to 10 seconds.
 #svn.proxy.reload_timeout = 10
 
 ############################################################
 ### SSH Support Settings                                 ###
 ############################################################
 
 ## Defines if the authorized_keys file should be written on any change of
 ## user ssh keys, setting this to false also disables posibility of adding
 ## ssh keys for users from web interface.
 ssh.generate_authorized_keyfile = false
 
 ## Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding`
 # ssh.authorized_keys_ssh_opts =
 
 ## File to generate the authorized keys together with options
 ## It is possible to have multiple key files specified in `sshd_config` e.g.
 ## AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
 ssh.authorized_keys_file_path = ~/.ssh/authorized_keys_rhodecode
 
 ## Command to execute the SSH wrapper. The binary is available in the
 ## rhodecode installation directory.
 ## e.g ~/.rccontrol/community-1/profile/bin/rcssh-wrapper
 ssh.wrapper_cmd = ~/.rccontrol/community-1/rcssh-wrapper
 
 ## Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false
 
 ## Enables logging, and detailed output send back to the client. Usefull for
 ## debugging, shouldn't be used in production.
 ssh.enable_debug_logging = false
 
 ## API KEY for user who has access to fetch other user permission information
 ## most likely an super-admin account with some IP restrictions.
 ssh.api_key =
 
 ## API Host, the server address of RhodeCode instance that the api_key will
 ## access
 ssh.api_host = http://localhost
 
 ## Paths to binary executrables, by default they are the names, but we can
 ## override them if we want to use a custom one
 ssh.executable.hg = ~/.rccontrol/vcsserver-1/profile/bin/hg
 ssh.executable.git = ~/.rccontrol/vcsserver-1/profile/bin/git
 ssh.executable.svn = ~/.rccontrol/vcsserver-1/profile/bin/svnserve
 
 
 ## Dummy marker to add new entries after.
 ## Add any custom entries below. Please don't remove.
 custom.conf = 1
 
 
 ################################
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
 keys = root, routes, rhodecode, sqlalchemy, beaker, templates, ssh_wrapper
 
 [handlers]
 keys = console, console_sql
 
 [formatters]
 keys = generic, color_formatter, color_formatter_sql
 
 #############
 ## LOGGERS ##
 #############
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_routes]
 level = DEBUG
 handlers =
 qualname = routes.middleware
 ## "level = DEBUG" logs the route matched and routing variables.
 propagate = 1
 
 [logger_beaker]
 level = DEBUG
 handlers =
 qualname = beaker.container
 propagate = 1
 
 [logger_templates]
 level = INFO
 handlers =
 qualname = pylons.templating
 propagate = 1
 
 [logger_rhodecode]
 level = DEBUG
 handlers =
 qualname = rhodecode
 propagate = 1
 
 [logger_sqlalchemy]
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 
 [logger_ssh_wrapper]
 level = DEBUG
 handlers =
 qualname = ssh_wrapper
 propagate = 1
 
 
 ##############
 ## HANDLERS ##
 ##############
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr, )
 level = DEBUG
 formatter = color_formatter
 
 [handler_console_sql]
 class = StreamHandler
 args = (sys.stderr, )
 level = DEBUG
 formatter = color_formatter_sql
 
 ################
 ## FORMATTERS ##
 ################
 
 [formatter_generic]
 class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class = rhodecode.lib.logging_formatter.ColorFormatter
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter_sql]
 class = rhodecode.lib.logging_formatter.ColorFormatterSql
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S

configs/production.ini

@@ -1,702 +1,708 b''
 
 
 ################################################################################
 ##                 RHODECODE COMMUNITY EDITION CONFIGURATION                  ##
 # The %(here)s variable will be replaced with the parent directory of this file#
 ################################################################################
 
 [DEFAULT]
 debug = true
 
 ################################################################################
 ##                            EMAIL CONFIGURATION                             ##
 ## Uncomment and replace with the email address which should receive          ##
 ## any error reports after an application crash                               ##
 ## Additionally these settings will be used by the RhodeCode mailing system   ##
 ################################################################################
 
 ## prefix all emails subjects with given prefix, helps filtering out emails
 #email_prefix = [RhodeCode]
 
 ## email FROM address all mails will be sent
 #app_email_from = rhodecode-noreply@localhost
 
 ## Uncomment and replace with the address which should receive any error report
 ## note: using appenlight for error handling doesn't need this to be uncommented
 #email_to = admin@localhost
 
 ## in case of Application errors, sent an error email form
 #error_email_from = rhodecode_error@localhost
 
 ## additional error message to be send in case of server crash
 #error_message =
 
 
 #smtp_server = mail.server.com
 #smtp_username =
 #smtp_password =
 #smtp_port =
 #smtp_use_tls = false
 #smtp_use_ssl = true
 ## Specify available auth parameters here (e.g. LOGIN PLAIN CRAM-MD5, etc.)
 #smtp_auth =
 
 [server:main]
 ## COMMON ##
 host = 127.0.0.1
 port = 5000
 
 ##################################
 ##     WAITRESS WSGI SERVER     ##
 ## Recommended for Development  ##
 ##################################
 
 #use = egg:waitress#main
 ## number of worker threads
 #threads = 5
 ## MAX BODY SIZE 100GB
 #max_request_body_size = 107374182400
 ## Use poll instead of select, fixes file descriptors limits problems.
 ## May not work on old windows systems.
 #asyncore_use_poll = true
 
 
 ##########################
 ## GUNICORN WSGI SERVER ##
 ##########################
 ## run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
 
 use = egg:gunicorn#main
 ## Sets the number of process workers. You must set `instance_id = *`
 ## when this option is set to more than one worker, recommended
 ## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
 ## The `instance_id = *` must be set in the [app:main] section below
 workers = 2
 ## number of threads for each of the worker, must be set to 1 for gevent
 ## generally recommened to be at 1
 #threads = 1
 ## process name
 proc_name = rhodecode
 ## type of worker class, one of sync, gevent
 ## recommended for bigger setup is using of of other than sync one
 worker_class = sync
 ## The maximum number of simultaneous clients. Valid only for Gevent
 #worker_connections = 10
 ## max number of requests that worker will handle before being gracefully
 ## restarted, could prevent memory leaks
 max_requests = 1000
 max_requests_jitter = 30
 ## amount of time a worker can spend with handling a request before it
 ## gets killed and restarted. Set to 6hrs
 timeout = 21600
 
 
 ## prefix middleware for RhodeCode.
 ## recommended when using proxy setup.
 ## allows to set RhodeCode under a prefix in server.
 ## eg https://server.com/custom_prefix. Enable `filter-with =` option below as well.
 ## And set your prefix like: `prefix = /custom_prefix`
 ## be sure to also set beaker.session.cookie_path = /custom_prefix if you need
 ## to make your cookies only work on prefix url
 [filter:proxy-prefix]
 use = egg:PasteDeploy#prefix
 prefix = /
 
 [app:main]
 use = egg:rhodecode-enterprise-ce
 
 ## enable proxy prefix middleware, defined above
 #filter-with = proxy-prefix
 
 ## encryption key used to encrypt social plugin tokens,
 ## remote_urls with credentials etc, if not set it defaults to
 ## `beaker.session.secret`
 #rhodecode.encrypted_values.secret =
 
 ## decryption strict mode (enabled by default). It controls if decryption raises
 ## `SignatureVerificationError` in case of wrong key, or damaged encryption data.
 #rhodecode.encrypted_values.strict = false
 
 ## return gzipped responses from Rhodecode (static files/application)
 gzip_responses = false
 
 ## autogenerate javascript routes file on startup
 generate_js_files = false
 
 ## Optional Languages
 ## en(default), be, de, es, fr, it, ja, pl, pt, ru, zh
 lang = en
 
 ## perform a full repository scan on each server start, this should be
 ## set to false after first startup, to allow faster server restarts.
 startup.import_repos = false
 
 ## Uncomment and set this path to use archive download cache.
 ## Once enabled, generated archives will be cached at this location
 ## and served from the cache during subsequent requests for the same archive of
 ## the repository.
 #archive_cache_dir = /tmp/tarballcache
 
 ## change this to unique ID for security
 app_instance_uuid = rc-production
 
-## cut off limit for large diffs (size in bytes)
-cut_off_limit_diff = 1024000
-cut_off_limit_file = 256000
+## cut off limit for large diffs (size in bytes). If overall diff size on
+## commit, or pull request exceeds this limit this diff will be displayed
+## partially. E.g 512000 == 512Kb
+cut_off_limit_diff = 512000
+
+## cut off limit for large files inside diffs (size in bytes). Each individual
+## file inside diff which exceeds this limit will be displayed partially.
+##  E.g 128000 == 128Kb
+cut_off_limit_file = 128000
 
 ## use cache version of scm repo everywhere
 vcs_full_cache = true
 
 ## force https in RhodeCode, fixes https redirects, assumes it's always https
 ## Normally this is controlled by proper http flags sent from http server
 force_https = false
 
 ## use Strict-Transport-Security headers
 use_htsts = false
 
 ## number of commits stats will parse on each iteration
 commit_parse_limit = 25
 
 ## git rev filter option, --all is the default filter, if you need to
 ## hide all refs in changelog switch this to --branches --tags
 git_rev_filter = --branches --tags
 
 # Set to true if your repos are exposed using the dumb protocol
 git_update_server_info = false
 
 ## RSS/ATOM feed options
 rss_cut_off_limit = 256000
 rss_items_per_page = 10
 rss_include_diff = false
 
 ## gist URL alias, used to create nicer urls for gist. This should be an
 ## url that does rewrites to _admin/gists/{gistid}.
 ## example: http://gist.rhodecode.org/{gistid}. Empty means use the internal
 ## RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid}
 gist_alias_url =
 
 ## List of views (using glob pattern syntax) that AUTH TOKENS could be
 ## used for access.
 ## Adding ?auth_token=TOKEN_HASH to the url authenticates this request as if it
 ## came from the the logged in user who own this authentication token.
 ## Additionally @TOKEN syntaxt can be used to bound the view to specific
 ## authentication token. Such view would be only accessible when used together
 ## with this authentication token
 ##
 ## list of all views can be found under `/_admin/permissions/auth_token_access`
 ## The list should be "," separated and on a single line.
 ##
 ## Most common views to enable:
 #    RepoCommitsView:repo_commit_download
 #    RepoCommitsView:repo_commit_patch
 #    RepoCommitsView:repo_commit_raw
 #    RepoCommitsView:repo_commit_raw@TOKEN
 #    RepoFilesView:repo_files_diff
 #    RepoFilesView:repo_archivefile
 #    RepoFilesView:repo_file_raw
 #    GistView:*
 api_access_controllers_whitelist =
 
 ## default encoding used to convert from and to unicode
 ## can be also a comma separated list of encoding in case of mixed encodings
 default_encoding = UTF-8
 
 ## instance-id prefix
 ## a prefix key for this instance used for cache invalidation when running
 ## multiple instances of rhodecode, make sure it's globally unique for
 ## all running rhodecode instances. Leave empty if you don't use it
 instance_id =
 
 ## Fallback authentication plugin. Set this to a plugin ID to force the usage
 ## of an authentication plugin also if it is disabled by it's settings.
 ## This could be useful if you are unable to log in to the system due to broken
 ## authentication settings. Then you can enable e.g. the internal rhodecode auth
 ## module to log in again and fix the settings.
 ##
 ## Available builtin plugin IDs (hash is part of the ID):
 ## egg:rhodecode-enterprise-ce#rhodecode
 ## egg:rhodecode-enterprise-ce#pam
 ## egg:rhodecode-enterprise-ce#ldap
 ## egg:rhodecode-enterprise-ce#jasig_cas
 ## egg:rhodecode-enterprise-ce#headers
 ## egg:rhodecode-enterprise-ce#crowd
 #rhodecode.auth_plugin_fallback = egg:rhodecode-enterprise-ce#rhodecode
 
 ## alternative return HTTP header for failed authentication. Default HTTP
 ## response is 401 HTTPUnauthorized. Currently HG clients have troubles with
 ## handling that causing a series of failed authentication calls.
 ## Set this variable to 403 to return HTTPForbidden, or any other HTTP code
 ## This will be served instead of default 401 on bad authnetication
 auth_ret_code =
 
 ## use special detection method when serving auth_ret_code, instead of serving
 ## ret_code directly, use 401 initially (Which triggers credentials prompt)
 ## and then serve auth_ret_code to clients
 auth_ret_code_detection = false
 
 ## locking return code. When repository is locked return this HTTP code. 2XX
 ## codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 
 ## allows to change the repository location in settings page
 allow_repo_location_change = true
 
 ## allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
 
 ## generated license token, goto license page in RhodeCode settings to obtain
 ## new token
 license_token =
 
 ## supervisor connection uri, for managing supervisor and logs.
 supervisor.uri =
 ## supervisord group name/id we only want this RC instance to handle
 supervisor.group_id = prod
 
 ## Display extended labs settings
 labs_settings_active = true
 
 ####################################
 ###        CELERY CONFIG        ####
 ####################################
 use_celery = false
 broker.host = localhost
 broker.vhost = rabbitmqhost
 broker.port = 5672
 broker.user = rabbitmq
 broker.password = qweqwe
 
 celery.imports = rhodecode.lib.celerylib.tasks
 
 celery.result.backend = amqp
 celery.result.dburi = amqp://
 celery.result.serialier = json
 
 #celery.send.task.error.emails = true
 #celery.amqp.task.result.expires = 18000
 
 celeryd.concurrency = 2
 #celeryd.log.file = celeryd.log
 celeryd.log.level = debug
 celeryd.max.tasks.per.child = 1
 
 ## tasks will never be sent to the queue, but executed locally instead.
 celery.always.eager = false
 
 ####################################
 ###         BEAKER CACHE        ####
 ####################################
 # default cache dir for templates.  Putting this into a ramdisk
 ## can boost performance, eg. %(here)s/data_ramdisk
 cache_dir = %(here)s/data
 
 ## locking and default file storage for Beaker. Putting this into a ramdisk
 ## can boost performance, eg. %(here)s/data_ramdisk/cache/beaker_data
 beaker.cache.data_dir = %(here)s/data/cache/beaker_data
 beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
 
 beaker.cache.regions = super_short_term, short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
 
 beaker.cache.super_short_term.type = memory
 beaker.cache.super_short_term.expire = 10
 beaker.cache.super_short_term.key_length = 256
 
 beaker.cache.short_term.type = memory
 beaker.cache.short_term.expire = 60
 beaker.cache.short_term.key_length = 256
 
 beaker.cache.long_term.type = memory
 beaker.cache.long_term.expire = 36000
 beaker.cache.long_term.key_length = 256
 
 beaker.cache.sql_cache_short.type = memory
 beaker.cache.sql_cache_short.expire = 10
 beaker.cache.sql_cache_short.key_length = 256
 
 ## default is memory cache, configure only if required
 ## using multi-node or multi-worker setup
 #beaker.cache.auth_plugins.type = ext:database
 #beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
 #beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
 #beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
 #beaker.cache.auth_plugins.sa.pool_recycle = 3600
 #beaker.cache.auth_plugins.sa.pool_size = 10
 #beaker.cache.auth_plugins.sa.max_overflow = 0
 
 beaker.cache.repo_cache_long.type = memorylru_base
 beaker.cache.repo_cache_long.max_items = 4096
 beaker.cache.repo_cache_long.expire = 2592000
 
 ## default is memorylru_base cache, configure only if required
 ## using multi-node or multi-worker setup
 #beaker.cache.repo_cache_long.type = ext:memcached
 #beaker.cache.repo_cache_long.url = localhost:11211
 #beaker.cache.repo_cache_long.expire = 1209600
 #beaker.cache.repo_cache_long.key_length = 256
 
 ####################################
 ###       BEAKER SESSION        ####
 ####################################
 
 ## .session.type is type of storage options for the session, current allowed
 ## types are file, ext:memcached, ext:database, and memory (default).
 beaker.session.type = file
 beaker.session.data_dir = %(here)s/data/sessions/data
 
 ## db based session, fast, and allows easy management over logged in users
 #beaker.session.type = ext:database
 #beaker.session.table_name = db_session
 #beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode
 #beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode
 #beaker.session.sa.pool_recycle = 3600
 #beaker.session.sa.echo = false
 
 beaker.session.key = rhodecode
 beaker.session.secret = production-rc-uytcxaz
 beaker.session.lock_dir = %(here)s/data/sessions/lock
 
 ## Secure encrypted cookie. Requires AES and AES python libraries
 ## you must disable beaker.session.secret to use this
 #beaker.session.encrypt_key = key_for_encryption
 #beaker.session.validate_key = validation_key
 
 ## sets session as invalid(also logging out user) if it haven not been
 ## accessed for given amount of time in seconds
 beaker.session.timeout = 2592000
 beaker.session.httponly = true
 ## Path to use for the cookie. Set to prefix if you use prefix middleware
 #beaker.session.cookie_path = /custom_prefix
 
 ## uncomment for https secure cookie
 beaker.session.secure = false
 
 ## auto save the session to not to use .save()
 beaker.session.auto = false
 
 ## default cookie expiration time in seconds, set to `true` to set expire
 ## at browser close
 #beaker.session.cookie_expires = 3600
 
 ###################################
 ## SEARCH INDEXING CONFIGURATION ##
 ###################################
 ## Full text search indexer is available in rhodecode-tools under
 ## `rhodecode-tools index` command
 
 ## WHOOSH Backend, doesn't require additional services to run
 ## it works good with few dozen repos
 search.module = rhodecode.lib.index.whoosh
 search.location = %(here)s/data/index
 
 ########################################
 ###    CHANNELSTREAM CONFIG         ####
 ########################################
 ## channelstream enables persistent connections and live notification
 ## in the system. It's also used by the chat system
 channelstream.enabled = false
 
 ## server address for channelstream server on the backend
 channelstream.server = 127.0.0.1:9800
 
 ## location of the channelstream server from outside world
 ## use ws:// for http or wss:// for https. This address needs to be handled
 ## by external HTTP server such as Nginx or Apache
 ## see nginx/apache configuration examples in our docs
 channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream
 channelstream.secret = secret
 channelstream.history.location = %(here)s/channelstream_history
 
 ## Internal application path that Javascript uses to connect into.
 ## If you use proxy-prefix the prefix should be added before /_channelstream
 channelstream.proxy_path = /_channelstream
 
 
 ###################################
 ##       APPENLIGHT CONFIG       ##
 ###################################
 
 ## Appenlight is tailored to work with RhodeCode, see
 ## http://appenlight.com for details how to obtain an account
 
 ## appenlight integration enabled
 appenlight = false
 
 appenlight.server_url = https://api.appenlight.com
 appenlight.api_key = YOUR_API_KEY
 #appenlight.transport_config = https://api.appenlight.com?threaded=1&timeout=5
 
 # used for JS client
 appenlight.api_public_key = YOUR_API_PUBLIC_KEY
 
 ## TWEAK AMOUNT OF INFO SENT HERE
 
 ## enables 404 error logging (default False)
 appenlight.report_404 = false
 
 ## time in seconds after request is considered being slow (default 1)
 appenlight.slow_request_time = 1
 
 ## record slow requests in application
 ## (needs to be enabled for slow datastore recording and time tracking)
 appenlight.slow_requests = true
 
 ## enable hooking to application loggers
 appenlight.logging = true
 
 ## minimum log level for log capture
 appenlight.logging.level = WARNING
 
 ## send logs only from erroneous/slow requests
 ## (saves API quota for intensive logging)
 appenlight.logging_on_error = false
 
 ## list of additonal keywords that should be grabbed from environ object
 ## can be string with comma separated list of words in lowercase
 ## (by default client will always send following info:
 ## 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that
 ## start with HTTP* this list be extended with additional keywords here
 appenlight.environ_keys_whitelist =
 
 ## list of keywords that should be blanked from request object
 ## can be string with comma separated list of words in lowercase
 ## (by default client will always blank keys that contain following words
 ## 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf'
 ## this list be extended with additional keywords set here
 appenlight.request_keys_blacklist =
 
 ## list of namespaces that should be ignores when gathering log entries
 ## can be string with comma separated list of namespaces
 ## (by default the client ignores own entries: appenlight_client.client)
 appenlight.log_namespace_blacklist =
 
 
 ################################################################################
 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 ## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 ## execute malicious code after an exception is raised.                       ##
 ################################################################################
 set debug = false
 
 
 ###########################################
 ###   MAIN RHODECODE DATABASE CONFIG    ###
 ###########################################
 #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 #sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 #sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode
 sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 
 # see sqlalchemy docs for other advanced settings
 
 ## print the sql statements to output
 sqlalchemy.db1.echo = false
 ## recycle the connections after this amount of seconds
 sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.db1.convert_unicode = true
 
 ## the number of connections to keep open inside the connection pool.
 ## 0 indicates no limit
 #sqlalchemy.db1.pool_size = 5
 
 ## the number of connections to allow in connection pool "overflow", that is
 ## connections that can be opened above and beyond the pool_size setting,
 ## which defaults to five.
 #sqlalchemy.db1.max_overflow = 10
 
 
 ##################
 ### VCS CONFIG ###
 ##################
 vcs.server.enable = true
 vcs.server = localhost:9900
 
 ## Web server connectivity protocol, responsible for web based VCS operatations
 ## Available protocols are:
 ## `http` - use http-rpc backend (default)
 vcs.server.protocol = http
 
 ## Push/Pull operations protocol, available options are:
 ## `http` - use http-rpc backend (default)
 ##
 vcs.scm_app_implementation = http
 
 ## Push/Pull operations hooks protocol, available options are:
 ## `http` - use http-rpc backend (default)
 vcs.hooks.protocol = http
 
 vcs.server.log_level = info
 ## Start VCSServer with this instance as a subprocess, usefull for development
 vcs.start_server = false
 
 ## List of enabled VCS backends, available options are:
 ## `hg`  - mercurial
 ## `git` - git
 ## `svn` - subversion
 vcs.backends = hg, git, svn
 
 vcs.connection_timeout = 3600
 ## Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
 ## Available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
 #vcs.svn.compatible_version = pre-1.8-compatible
 
 
 ############################################################
 ### Subversion proxy support (mod_dav_svn)               ###
 ### Maps RhodeCode repo groups into SVN paths for Apache ###
 ############################################################
 ## Enable or disable the config file generation.
 svn.proxy.generate_config = false
 ## Generate config file with `SVNListParentPath` set to `On`.
 svn.proxy.list_parent_path = true
 ## Set location and file name of generated config file.
 svn.proxy.config_file_path = %(here)s/mod_dav_svn.conf
 ## Used as a prefix to the `Location` block in the generated config file.
 ## In most cases it should be set to `/`.
 svn.proxy.location_root = /
 ## Command to reload the mod dav svn configuration on change.
 ## Example: `/etc/init.d/apache2 reload`
 #svn.proxy.reload_cmd = /etc/init.d/apache2 reload
 ## If the timeout expires before the reload command finishes, the command will
 ## be killed. Setting it to zero means no timeout. Defaults to 10 seconds.
 #svn.proxy.reload_timeout = 10
 
 ############################################################
 ### SSH Support Settings                                 ###
 ############################################################
 
 ## Defines if the authorized_keys file should be written on any change of
 ## user ssh keys, setting this to false also disables posibility of adding
 ## ssh keys for users from web interface.
 ssh.generate_authorized_keyfile = false
 
 ## Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding`
 # ssh.authorized_keys_ssh_opts =
 
 ## File to generate the authorized keys together with options
 ## It is possible to have multiple key files specified in `sshd_config` e.g.
 ## AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
 ssh.authorized_keys_file_path = ~/.ssh/authorized_keys_rhodecode
 
 ## Command to execute the SSH wrapper. The binary is available in the
 ## rhodecode installation directory.
 ## e.g ~/.rccontrol/community-1/profile/bin/rcssh-wrapper
 ssh.wrapper_cmd = ~/.rccontrol/community-1/rcssh-wrapper
 
 ## Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false
 
 ## Enables logging, and detailed output send back to the client. Usefull for
 ## debugging, shouldn't be used in production.
 ssh.enable_debug_logging = false
 
 ## API KEY for user who has access to fetch other user permission information
 ## most likely an super-admin account with some IP restrictions.
 ssh.api_key =
 
 ## API Host, the server address of RhodeCode instance that the api_key will
 ## access
 ssh.api_host = http://localhost
 
 ## Paths to binary executrables, by default they are the names, but we can
 ## override them if we want to use a custom one
 ssh.executable.hg = ~/.rccontrol/vcsserver-1/profile/bin/hg
 ssh.executable.git = ~/.rccontrol/vcsserver-1/profile/bin/git
 ssh.executable.svn = ~/.rccontrol/vcsserver-1/profile/bin/svnserve
 
 
 ## Dummy marker to add new entries after.
 ## Add any custom entries below. Please don't remove.
 custom.conf = 1
 
 
 ################################
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
 keys = root, routes, rhodecode, sqlalchemy, beaker, templates, ssh_wrapper
 
 [handlers]
 keys = console, console_sql
 
 [formatters]
 keys = generic, color_formatter, color_formatter_sql
 
 #############
 ## LOGGERS ##
 #############
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_routes]
 level = DEBUG
 handlers =
 qualname = routes.middleware
 ## "level = DEBUG" logs the route matched and routing variables.
 propagate = 1
 
 [logger_beaker]
 level = DEBUG
 handlers =
 qualname = beaker.container
 propagate = 1
 
 [logger_templates]
 level = INFO
 handlers =
 qualname = pylons.templating
 propagate = 1
 
 [logger_rhodecode]
 level = DEBUG
 handlers =
 qualname = rhodecode
 propagate = 1
 
 [logger_sqlalchemy]
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 
 [logger_ssh_wrapper]
 level = DEBUG
 handlers =
 qualname = ssh_wrapper
 propagate = 1
 
 
 ##############
 ## HANDLERS ##
 ##############
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr, )
 level = INFO
 formatter = generic
 
 [handler_console_sql]
 class = StreamHandler
 args = (sys.stderr, )
 level = WARN
 formatter = generic
 
 ################
 ## FORMATTERS ##
 ################
 
 [formatter_generic]
 class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class = rhodecode.lib.logging_formatter.ColorFormatter
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter_sql]
 class = rhodecode.lib.logging_formatter.ColorFormatterSql
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S

rhodecode/lib/diffs.py

@@ -1,1164 +1,1170 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Set of diffing helpers, previously part of vcs
 """
 
 import collections
 import re
 import difflib
 import logging
 
 from itertools import tee, imap
 
 from pylons.i18n.translation import _
 
 from rhodecode.lib.vcs.exceptions import VCSError
 from rhodecode.lib.vcs.nodes import FileNode, SubModuleNode
 from rhodecode.lib.vcs.backends.base import EmptyCommit
 from rhodecode.lib.helpers import escape
 from rhodecode.lib.utils2 import safe_unicode
 
 log = logging.getLogger(__name__)
 
 # define max context, a file with more than this numbers of lines is unusable
 # in browser anyway
 MAX_CONTEXT = 1024 * 1014
 
 
 class OPS(object):
     ADD = 'A'
     MOD = 'M'
     DEL = 'D'
 
 
 def wrap_to_table(str_):
     return '''<table class="code-difftable">
                 <tr class="line no-comment">
                 <td class="add-comment-line tooltip" title="%s"><span class="add-comment-content"></span></td>
                 <td></td>
                 <td class="lineno new"></td>
                 <td class="code no-comment"><pre>%s</pre></td>
                 </tr>
               </table>''' % (_('Click to comment'), str_)
 
 
 def wrapped_diff(filenode_old, filenode_new, diff_limit=None, file_limit=None,
                  show_full_diff=False, ignore_whitespace=True, line_context=3,
                  enable_comments=False):
     """
     returns a wrapped diff into a table, checks for cut_off_limit for file and
     whole diff and presents proper message
     """
 
     if filenode_old is None:
         filenode_old = FileNode(filenode_new.path, '', EmptyCommit())
 
     if filenode_old.is_binary or filenode_new.is_binary:
         diff = wrap_to_table(_('Binary file'))
         stats = None
         size = 0
         data = None
 
     elif diff_limit != -1 and (diff_limit is None or
         (filenode_old.size < diff_limit and filenode_new.size < diff_limit)):
 
         f_gitdiff = get_gitdiff(filenode_old, filenode_new,
                                 ignore_whitespace=ignore_whitespace,
                                 context=line_context)
         diff_processor = DiffProcessor(
             f_gitdiff, format='gitdiff', diff_limit=diff_limit,
             file_limit=file_limit, show_full_diff=show_full_diff)
         _parsed = diff_processor.prepare()
 
         diff = diff_processor.as_html(enable_comments=enable_comments)
         stats = _parsed[0]['stats'] if _parsed else None
         size = len(diff or '')
         data = _parsed[0] if _parsed else None
     else:
         diff = wrap_to_table(_('Changeset was too big and was cut off, use '
                                'diff menu to display this diff'))
         stats = None
         size = 0
         data = None
     if not diff:
         submodules = filter(lambda o: isinstance(o, SubModuleNode),
                             [filenode_new, filenode_old])
         if submodules:
             diff = wrap_to_table(escape('Submodule %r' % submodules[0]))
         else:
             diff = wrap_to_table(_('No changes detected'))
 
     cs1 = filenode_old.commit.raw_id
     cs2 = filenode_new.commit.raw_id
 
     return size, cs1, cs2, diff, stats, data
 
 
 def get_gitdiff(filenode_old, filenode_new, ignore_whitespace=True, context=3):
     """
     Returns git style diff between given ``filenode_old`` and ``filenode_new``.
 
     :param ignore_whitespace: ignore whitespaces in diff
     """
     # make sure we pass in default context
     context = context or 3
     # protect against IntOverflow when passing HUGE context
     if context > MAX_CONTEXT:
         context = MAX_CONTEXT
 
     submodules = filter(lambda o: isinstance(o, SubModuleNode),
                         [filenode_new, filenode_old])
     if submodules:
         return ''
 
     for filenode in (filenode_old, filenode_new):
         if not isinstance(filenode, FileNode):
             raise VCSError(
                 "Given object should be FileNode object, not %s"
                 % filenode.__class__)
 
     repo = filenode_new.commit.repository
     old_commit = filenode_old.commit or repo.EMPTY_COMMIT
     new_commit = filenode_new.commit
 
     vcs_gitdiff = repo.get_diff(
         old_commit, new_commit, filenode_new.path,
         ignore_whitespace, context, path1=filenode_old.path)
     return vcs_gitdiff
 
 NEW_FILENODE = 1
 DEL_FILENODE = 2
 MOD_FILENODE = 3
 RENAMED_FILENODE = 4
 COPIED_FILENODE = 5
 CHMOD_FILENODE = 6
 BIN_FILENODE = 7
 
 
 class LimitedDiffContainer(object):
 
     def __init__(self, diff_limit, cur_diff_size, diff):
         self.diff = diff
         self.diff_limit = diff_limit
         self.cur_diff_size = cur_diff_size
 
     def __getitem__(self, key):
         return self.diff.__getitem__(key)
 
     def __iter__(self):
         for l in self.diff:
             yield l
 
 
 class Action(object):
     """
     Contains constants for the action value of the lines in a parsed diff.
     """
 
     ADD = 'add'
     DELETE = 'del'
     UNMODIFIED = 'unmod'
 
     CONTEXT = 'context'
     OLD_NO_NL = 'old-no-nl'
     NEW_NO_NL = 'new-no-nl'
 
 
 class DiffProcessor(object):
     """
     Give it a unified or git diff and it returns a list of the files that were
     mentioned in the diff together with a dict of meta information that
     can be used to render it in a HTML template.
 
     .. note:: Unicode handling
 
        The original diffs are a byte sequence and can contain filenames
        in mixed encodings. This class generally returns `unicode` objects
        since the result is intended for presentation to the user.
 
     """
     _chunk_re = re.compile(r'^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)')
     _newline_marker = re.compile(r'^\\ No newline at end of file')
 
     # used for inline highlighter word split
     _token_re = re.compile(r'()(&gt;|&lt;|&amp;|\W+?)')
 
     # collapse ranges of commits over given number
     _collapse_commits_over = 5
 
     def __init__(self, diff, format='gitdiff', diff_limit=None,
                  file_limit=None, show_full_diff=True):
         """
         :param diff: A `Diff` object representing a diff from a vcs backend
         :param format: format of diff passed, `udiff` or `gitdiff`
         :param diff_limit: define the size of diff that is considered "big"
             based on that parameter cut off will be triggered, set to None
             to show full diff
         """
         self._diff = diff
         self._format = format
         self.adds = 0
         self.removes = 0
         # calculate diff size
         self.diff_limit = diff_limit
         self.file_limit = file_limit
         self.show_full_diff = show_full_diff
         self.cur_diff_size = 0
         self.parsed = False
         self.parsed_diff = []
 
+        log.debug('Initialized DiffProcessor with %s mode', format)
         if format == 'gitdiff':
             self.differ = self._highlight_line_difflib
             self._parser = self._parse_gitdiff
         else:
             self.differ = self._highlight_line_udiff
             self._parser = self._new_parse_gitdiff
 
     def _copy_iterator(self):
         """
         make a fresh copy of generator, we should not iterate thru
         an original as it's needed for repeating operations on
         this instance of DiffProcessor
         """
         self.__udiff, iterator_copy = tee(self.__udiff)
         return iterator_copy
 
     def _escaper(self, string):
         """
         Escaper for diff escapes special chars and checks the diff limit
 
         :param string:
         """
 
         self.cur_diff_size += len(string)
 
         if not self.show_full_diff and (self.cur_diff_size > self.diff_limit):
             raise DiffLimitExceeded('Diff Limit Exceeded')
 
         return safe_unicode(string)\
             .replace('&', '&amp;')\
             .replace('<', '&lt;')\
             .replace('>', '&gt;')
 
     def _line_counter(self, l):
         """
         Checks each line and bumps total adds/removes for this diff
 
         :param l:
         """
         if l.startswith('+') and not l.startswith('+++'):
             self.adds += 1
         elif l.startswith('-') and not l.startswith('---'):
             self.removes += 1
         return safe_unicode(l)
 
     def _highlight_line_difflib(self, line, next_):
         """
         Highlight inline changes in both lines.
         """
 
         if line['action'] == Action.DELETE:
             old, new = line, next_
         else:
             old, new = next_, line
 
         oldwords = self._token_re.split(old['line'])
         newwords = self._token_re.split(new['line'])
         sequence = difflib.SequenceMatcher(None, oldwords, newwords)
 
         oldfragments, newfragments = [], []
         for tag, i1, i2, j1, j2 in sequence.get_opcodes():
             oldfrag = ''.join(oldwords[i1:i2])
             newfrag = ''.join(newwords[j1:j2])
             if tag != 'equal':
                 if oldfrag:
                     oldfrag = '<del>%s</del>' % oldfrag
                 if newfrag:
                     newfrag = '<ins>%s</ins>' % newfrag
             oldfragments.append(oldfrag)
             newfragments.append(newfrag)
 
         old['line'] = "".join(oldfragments)
         new['line'] = "".join(newfragments)
 
     def _highlight_line_udiff(self, line, next_):
         """
         Highlight inline changes in both lines.
         """
         start = 0
         limit = min(len(line['line']), len(next_['line']))
         while start < limit and line['line'][start] == next_['line'][start]:
             start += 1
         end = -1
         limit -= start
         while -end <= limit and line['line'][end] == next_['line'][end]:
             end -= 1
         end += 1
         if start or end:
             def do(l):
                 last = end + len(l['line'])
                 if l['action'] == Action.ADD:
                     tag = 'ins'
                 else:
                     tag = 'del'
                 l['line'] = '%s<%s>%s</%s>%s' % (
                     l['line'][:start],
                     tag,
                     l['line'][start:last],
                     tag,
                     l['line'][last:]
                 )
             do(line)
             do(next_)
 
     def _clean_line(self, line, command):
         if command in ['+', '-', ' ']:
             # only modify the line if it's actually a diff thing
             line = line[1:]
         return line
 
     def _parse_gitdiff(self, inline_diff=True):
         _files = []
         diff_container = lambda arg: arg
 
         for chunk in self._diff.chunks():
             head = chunk.header
 
             diff = imap(self._escaper, chunk.diff.splitlines(1))
             raw_diff = chunk.raw
             limited_diff = False
             exceeds_limit = False
 
             op = None
             stats = {
                 'added': 0,
                 'deleted': 0,
                 'binary': False,
                 'ops': {},
             }
 
             if head['deleted_file_mode']:
                 op = OPS.DEL
                 stats['binary'] = True
                 stats['ops'][DEL_FILENODE] = 'deleted file'
 
             elif head['new_file_mode']:
                 op = OPS.ADD
                 stats['binary'] = True
                 stats['ops'][NEW_FILENODE] = 'new file %s' % head['new_file_mode']
             else:  # modify operation, can be copy, rename or chmod
 
                 # CHMOD
                 if head['new_mode'] and head['old_mode']:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][CHMOD_FILENODE] = (
                         'modified file chmod %s => %s' % (
                             head['old_mode'], head['new_mode']))
                 # RENAME
                 if head['rename_from'] != head['rename_to']:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][RENAMED_FILENODE] = (
                         'file renamed from %s to %s' % (
                             head['rename_from'], head['rename_to']))
                 # COPY
                 if head.get('copy_from') and head.get('copy_to'):
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][COPIED_FILENODE] = (
                         'file copied from %s to %s' % (
                             head['copy_from'], head['copy_to']))
 
                 # If our new parsed headers didn't match anything fallback to
                 # old style detection
                 if op is None:
                     if not head['a_file'] and head['b_file']:
                         op = OPS.ADD
                         stats['binary'] = True
                         stats['ops'][NEW_FILENODE] = 'new file'
 
                     elif head['a_file'] and not head['b_file']:
                         op = OPS.DEL
                         stats['binary'] = True
                         stats['ops'][DEL_FILENODE] = 'deleted file'
 
                 # it's not ADD not DELETE
                 if op is None:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][MOD_FILENODE] = 'modified file'
 
             # a real non-binary diff
             if head['a_file'] or head['b_file']:
                 try:
                     raw_diff, chunks, _stats = self._parse_lines(diff)
                     stats['binary'] = False
                     stats['added'] = _stats[0]
                     stats['deleted'] = _stats[1]
                     # explicit mark that it's a modified file
                     if op == OPS.MOD:
                         stats['ops'][MOD_FILENODE] = 'modified file'
                     exceeds_limit = len(raw_diff) > self.file_limit
 
                     # changed from _escaper function so we validate size of
                     # each file instead of the whole diff
                     # diff will hide big files but still show small ones
                     # from my tests, big files are fairly safe to be parsed
                     # but the browser is the bottleneck
                     if not self.show_full_diff and exceeds_limit:
                         raise DiffLimitExceeded('File Limit Exceeded')
 
                 except DiffLimitExceeded:
                     diff_container = lambda _diff: \
                         LimitedDiffContainer(
                             self.diff_limit, self.cur_diff_size, _diff)
 
                     exceeds_limit = len(raw_diff) > self.file_limit
                     limited_diff = True
                     chunks = []
 
             else:  # GIT format binary patch, or possibly empty diff
                 if head['bin_patch']:
                     # we have operation already extracted, but we mark simply
                     # it's a diff we wont show for binary files
                     stats['ops'][BIN_FILENODE] = 'binary diff hidden'
                 chunks = []
 
             if chunks and not self.show_full_diff and op == OPS.DEL:
                 # if not full diff mode show deleted file contents
                 # TODO: anderson: if the view is not too big, there is no way
                 # to see the content of the file
                 chunks = []
 
             chunks.insert(0, [{
                                   'old_lineno': '',
                                   'new_lineno': '',
                                   'action': Action.CONTEXT,
                                   'line': msg,
                               } for _op, msg in stats['ops'].iteritems()
                               if _op not in [MOD_FILENODE]])
 
             _files.append({
                 'filename': safe_unicode(head['b_path']),
                 'old_revision': head['a_blob_id'],
                 'new_revision': head['b_blob_id'],
                 'chunks': chunks,
                 'raw_diff': safe_unicode(raw_diff),
                 'operation': op,
                 'stats': stats,
                 'exceeds_limit': exceeds_limit,
                 'is_limited_diff': limited_diff,
             })
 
         sorter = lambda info: {OPS.ADD: 0, OPS.MOD: 1,
                                OPS.DEL: 2}.get(info['operation'])
 
         if not inline_diff:
             return diff_container(sorted(_files, key=sorter))
 
         # highlight inline changes
         for diff_data in _files:
             for chunk in diff_data['chunks']:
                 lineiter = iter(chunk)
                 try:
                     while 1:
                         line = lineiter.next()
                         if line['action'] not in (
                                 Action.UNMODIFIED, Action.CONTEXT):
                             nextline = lineiter.next()
                             if nextline['action'] in ['unmod', 'context'] or \
                                nextline['action'] == line['action']:
                                 continue
                             self.differ(line, nextline)
                 except StopIteration:
                     pass
 
         return diff_container(sorted(_files, key=sorter))
 
-
-    # FIXME: NEWDIFFS: dan: this replaces the old _escaper function
-    def _process_line(self, string):
-        """
-        Process a diff line, checks the diff limit
-
-        :param string:
-        """
-
-        self.cur_diff_size += len(string)
-
+    def _check_large_diff(self):
+        log.debug('Diff exceeds current diff_limit of %s', self.diff_limit)
         if not self.show_full_diff and (self.cur_diff_size > self.diff_limit):
-            raise DiffLimitExceeded('Diff Limit Exceeded')
-
-        return safe_unicode(string)
+            raise DiffLimitExceeded('Diff Limit `%s` Exceeded', self.diff_limit)
 
     # FIXME: NEWDIFFS: dan: this replaces _parse_gitdiff
     def _new_parse_gitdiff(self, inline_diff=True):
         _files = []
+
+        # this can be overriden later to a LimitedDiffContainer type
         diff_container = lambda arg: arg
+
         for chunk in self._diff.chunks():
             head = chunk.header
             log.debug('parsing diff %r' % head)
 
-            diff = imap(self._process_line, chunk.diff.splitlines(1))
             raw_diff = chunk.raw
             limited_diff = False
             exceeds_limit = False
-            # if 'empty_file_to_modify_and_rename' in head['a_path']:
-            #     1/0
+
             op = None
             stats = {
                 'added': 0,
                 'deleted': 0,
                 'binary': False,
                 'old_mode': None,
                 'new_mode': None,
                 'ops': {},
             }
             if head['old_mode']:
                 stats['old_mode'] = head['old_mode']
             if head['new_mode']:
                 stats['new_mode'] = head['new_mode']
             if head['b_mode']:
                 stats['new_mode'] = head['b_mode']
 
+            # delete file
             if head['deleted_file_mode']:
                 op = OPS.DEL
                 stats['binary'] = True
                 stats['ops'][DEL_FILENODE] = 'deleted file'
 
+            # new file
             elif head['new_file_mode']:
                 op = OPS.ADD
                 stats['binary'] = True
                 stats['old_mode'] = None
                 stats['new_mode'] = head['new_file_mode']
                 stats['ops'][NEW_FILENODE] = 'new file %s' % head['new_file_mode']
-            else:  # modify operation, can be copy, rename or chmod
 
+            # modify operation, can be copy, rename or chmod
+            else:
                 # CHMOD
                 if head['new_mode'] and head['old_mode']:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][CHMOD_FILENODE] = (
                         'modified file chmod %s => %s' % (
                             head['old_mode'], head['new_mode']))
 
                 # RENAME
                 if head['rename_from'] != head['rename_to']:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['renamed'] = (head['rename_from'], head['rename_to'])
                     stats['ops'][RENAMED_FILENODE] = (
                         'file renamed from %s to %s' % (
                             head['rename_from'], head['rename_to']))
                 # COPY
                 if head.get('copy_from') and head.get('copy_to'):
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['copied'] = (head['copy_from'], head['copy_to'])
                     stats['ops'][COPIED_FILENODE] = (
                         'file copied from %s to %s' % (
                             head['copy_from'], head['copy_to']))
 
                 # If our new parsed headers didn't match anything fallback to
                 # old style detection
                 if op is None:
                     if not head['a_file'] and head['b_file']:
                         op = OPS.ADD
                         stats['binary'] = True
                         stats['new_file'] = True
                         stats['ops'][NEW_FILENODE] = 'new file'
 
                     elif head['a_file'] and not head['b_file']:
                         op = OPS.DEL
                         stats['binary'] = True
                         stats['ops'][DEL_FILENODE] = 'deleted file'
 
                 # it's not ADD not DELETE
                 if op is None:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][MOD_FILENODE] = 'modified file'
 
             # a real non-binary diff
             if head['a_file'] or head['b_file']:
+                diff = iter(chunk.diff.splitlines(1))
+
+                # append each file to the diff size
+                raw_chunk_size = len(raw_diff)
+
+                exceeds_limit = raw_chunk_size > self.file_limit
+                self.cur_diff_size += raw_chunk_size
+
                 try:
+                    # Check each file instead of the whole diff.
+                    # Diff will hide big files but still show small ones.
+                    # From the tests big files are fairly safe to be parsed
+                    # but the browser is the bottleneck.
+                    if not self.show_full_diff and exceeds_limit:
+                        log.debug('File `%s` exceeds current file_limit of %s',
+                                  safe_unicode(head['b_path']), self.file_limit)
+                        raise DiffLimitExceeded(
+                            'File Limit %s Exceeded', self.file_limit)
+
+                    self._check_large_diff()
+
                     raw_diff, chunks, _stats = self._new_parse_lines(diff)
                     stats['binary'] = False
                     stats['added'] = _stats[0]
                     stats['deleted'] = _stats[1]
                     # explicit mark that it's a modified file
                     if op == OPS.MOD:
                         stats['ops'][MOD_FILENODE] = 'modified file'
-                    exceeds_limit = len(raw_diff) > self.file_limit
-
-                    # changed from _escaper function so we validate size of
-                    # each file instead of the whole diff
-                    # diff will hide big files but still show small ones
-                    # from my tests, big files are fairly safe to be parsed
-                    # but the browser is the bottleneck
-                    if not self.show_full_diff and exceeds_limit:
-                        raise DiffLimitExceeded('File Limit Exceeded')
 
                 except DiffLimitExceeded:
                     diff_container = lambda _diff: \
                         LimitedDiffContainer(
                             self.diff_limit, self.cur_diff_size, _diff)
 
-                    exceeds_limit = len(raw_diff) > self.file_limit
                     limited_diff = True
                     chunks = []
 
             else:  # GIT format binary patch, or possibly empty diff
                 if head['bin_patch']:
                     # we have operation already extracted, but we mark simply
                     # it's a diff we wont show for binary files
                     stats['ops'][BIN_FILENODE] = 'binary diff hidden'
                 chunks = []
 
+            # Hide content of deleted node by setting empty chunks
             if chunks and not self.show_full_diff and op == OPS.DEL:
                 # if not full diff mode show deleted file contents
                 # TODO: anderson: if the view is not too big, there is no way
                 # to see the content of the file
                 chunks = []
 
-            chunks.insert(0, [{
-                                  'old_lineno': '',
+            chunks.insert(
+                0, [{'old_lineno': '',
                                   'new_lineno': '',
                                   'action': Action.CONTEXT,
                                   'line': msg,
                               } for _op, msg in stats['ops'].iteritems()
                               if _op not in [MOD_FILENODE]])
 
             original_filename = safe_unicode(head['a_path'])
             _files.append({
                 'original_filename': original_filename,
                 'filename': safe_unicode(head['b_path']),
                 'old_revision': head['a_blob_id'],
                 'new_revision': head['b_blob_id'],
                 'chunks': chunks,
                 'raw_diff': safe_unicode(raw_diff),
                 'operation': op,
                 'stats': stats,
                 'exceeds_limit': exceeds_limit,
                 'is_limited_diff': limited_diff,
             })
 
-
         sorter = lambda info: {OPS.ADD: 0, OPS.MOD: 1,
                                OPS.DEL: 2}.get(info['operation'])
 
         return diff_container(sorted(_files, key=sorter))
 
     # FIXME: NEWDIFFS: dan: this gets replaced by _new_parse_lines
     def _parse_lines(self, diff):
         """
         Parse the diff an return data for the template.
         """
 
         lineiter = iter(diff)
         stats = [0, 0]
         chunks = []
         raw_diff = []
 
         try:
             line = lineiter.next()
 
             while line:
                 raw_diff.append(line)
                 lines = []
                 chunks.append(lines)
 
                 match = self._chunk_re.match(line)
 
                 if not match:
                     break
 
                 gr = match.groups()
                 (old_line, old_end,
                  new_line, new_end) = [int(x or 1) for x in gr[:-1]]
                 old_line -= 1
                 new_line -= 1
 
                 context = len(gr) == 5
                 old_end += old_line
                 new_end += new_line
 
                 if context:
                     # skip context only if it's first line
                     if int(gr[0]) > 1:
                         lines.append({
                             'old_lineno': '...',
                             'new_lineno': '...',
                             'action':     Action.CONTEXT,
                             'line':       line,
                         })
 
                 line = lineiter.next()
 
                 while old_line < old_end or new_line < new_end:
                     command = ' '
                     if line:
                         command = line[0]
 
                     affects_old = affects_new = False
 
                     # ignore those if we don't expect them
                     if command in '#@':
                         continue
                     elif command == '+':
                         affects_new = True
                         action = Action.ADD
                         stats[0] += 1
                     elif command == '-':
                         affects_old = True
                         action = Action.DELETE
                         stats[1] += 1
                     else:
                         affects_old = affects_new = True
                         action = Action.UNMODIFIED
 
                     if not self._newline_marker.match(line):
                         old_line += affects_old
                         new_line += affects_new
                         lines.append({
                             'old_lineno':   affects_old and old_line or '',
                             'new_lineno':   affects_new and new_line or '',
                             'action':       action,
                             'line':         self._clean_line(line, command)
                         })
                         raw_diff.append(line)
 
                     line = lineiter.next()
 
                     if self._newline_marker.match(line):
                         # we need to append to lines, since this is not
                         # counted in the line specs of diff
                         lines.append({
                             'old_lineno':   '...',
                             'new_lineno':   '...',
                             'action':       Action.CONTEXT,
                             'line':         self._clean_line(line, command)
                         })
 
         except StopIteration:
             pass
         return ''.join(raw_diff), chunks, stats
 
     # FIXME: NEWDIFFS: dan: this replaces _parse_lines
-    def _new_parse_lines(self, diff):
+    def _new_parse_lines(self, diff_iter):
         """
         Parse the diff an return data for the template.
         """
 
-        lineiter = iter(diff)
         stats = [0, 0]
         chunks = []
         raw_diff = []
 
+        diff_iter = imap(lambda s: safe_unicode(s), diff_iter)
+
         try:
-            line = lineiter.next()
+            line = diff_iter.next()
 
             while line:
                 raw_diff.append(line)
                 match = self._chunk_re.match(line)
 
                 if not match:
                     break
 
                 gr = match.groups()
                 (old_line, old_end,
                  new_line, new_end) = [int(x or 1) for x in gr[:-1]]
 
                 lines = []
                 hunk = {
                     'section_header': gr[-1],
                     'source_start': old_line,
                     'source_length': old_end,
                     'target_start': new_line,
                     'target_length': new_end,
                     'lines': lines,
                 }
                 chunks.append(hunk)
 
                 old_line -= 1
                 new_line -= 1
 
                 context = len(gr) == 5
                 old_end += old_line
                 new_end += new_line
 
-                line = lineiter.next()
+                line = diff_iter.next()
 
                 while old_line < old_end or new_line < new_end:
                     command = ' '
                     if line:
                         command = line[0]
 
                     affects_old = affects_new = False
 
                     # ignore those if we don't expect them
                     if command in '#@':
                         continue
                     elif command == '+':
                         affects_new = True
                         action = Action.ADD
                         stats[0] += 1
                     elif command == '-':
                         affects_old = True
                         action = Action.DELETE
                         stats[1] += 1
                     else:
                         affects_old = affects_new = True
                         action = Action.UNMODIFIED
 
                     if not self._newline_marker.match(line):
                         old_line += affects_old
                         new_line += affects_new
                         lines.append({
                             'old_lineno':   affects_old and old_line or '',
                             'new_lineno':   affects_new and new_line or '',
                             'action':       action,
                             'line':         self._clean_line(line, command)
                         })
                         raw_diff.append(line)
 
-                    line = lineiter.next()
+                    line = diff_iter.next()
 
                     if self._newline_marker.match(line):
                         # we need to append to lines, since this is not
                         # counted in the line specs of diff
                         if affects_old:
                             action = Action.OLD_NO_NL
                         elif affects_new:
                             action = Action.NEW_NO_NL
                         else:
                             raise Exception('invalid context for no newline')
 
                         lines.append({
                             'old_lineno':   None,
                             'new_lineno':   None,
                             'action':       action,
                             'line':         self._clean_line(line, command)
                         })
 
         except StopIteration:
             pass
+
         return ''.join(raw_diff), chunks, stats
 
     def _safe_id(self, idstring):
         """Make a string safe for including in an id attribute.
 
         The HTML spec says that id attributes 'must begin with
         a letter ([A-Za-z]) and may be followed by any number
         of letters, digits ([0-9]), hyphens ("-"), underscores
         ("_"), colons (":"), and periods (".")'. These regexps
         are slightly over-zealous, in that they remove colons
         and periods unnecessarily.
 
         Whitespace is transformed into underscores, and then
         anything which is not a hyphen or a character that
         matches \w (alphanumerics and underscore) is removed.
 
         """
         # Transform all whitespace to underscore
         idstring = re.sub(r'\s', "_", '%s' % idstring)
         # Remove everything that is not a hyphen or a member of \w
         idstring = re.sub(r'(?!-)\W', "", idstring).lower()
         return idstring
 
     def prepare(self, inline_diff=True):
         """
         Prepare the passed udiff for HTML rendering.
 
         :return: A list of dicts with diff information.
         """
         parsed = self._parser(inline_diff=inline_diff)
         self.parsed = True
         self.parsed_diff = parsed
         return parsed
 
     def as_raw(self, diff_lines=None):
         """
         Returns raw diff as a byte string
         """
         return self._diff.raw
 
     def as_html(self, table_class='code-difftable', line_class='line',
                 old_lineno_class='lineno old', new_lineno_class='lineno new',
                 code_class='code', enable_comments=False, parsed_lines=None):
         """
         Return given diff as html table with customized css classes
         """
         def _link_to_if(condition, label, url):
             """
             Generates a link if condition is meet or just the label if not.
             """
 
             if condition:
                 return '''<a href="%(url)s" class="tooltip"
                 title="%(title)s">%(label)s</a>''' % {
                     'title': _('Click to select line'),
                     'url': url,
                     'label': label
                 }
             else:
                 return label
         if not self.parsed:
             self.prepare()
 
         diff_lines = self.parsed_diff
         if parsed_lines:
             diff_lines = parsed_lines
 
         _html_empty = True
         _html = []
         _html.append('''<table class="%(table_class)s">\n''' % {
             'table_class': table_class
         })
 
         for diff in diff_lines:
             for line in diff['chunks']:
                 _html_empty = False
                 for change in line:
                     _html.append('''<tr class="%(lc)s %(action)s">\n''' % {
                         'lc': line_class,
                         'action': change['action']
                     })
                     anchor_old_id = ''
                     anchor_new_id = ''
                     anchor_old = "%(filename)s_o%(oldline_no)s" % {
                         'filename': self._safe_id(diff['filename']),
                         'oldline_no': change['old_lineno']
                     }
                     anchor_new = "%(filename)s_n%(oldline_no)s" % {
                         'filename': self._safe_id(diff['filename']),
                         'oldline_no': change['new_lineno']
                     }
                     cond_old = (change['old_lineno'] != '...' and
                                 change['old_lineno'])
                     cond_new = (change['new_lineno'] != '...' and
                                 change['new_lineno'])
                     if cond_old:
                         anchor_old_id = 'id="%s"' % anchor_old
                     if cond_new:
                         anchor_new_id = 'id="%s"' % anchor_new
 
                     if change['action'] != Action.CONTEXT:
                         anchor_link = True
                     else:
                         anchor_link = False
 
                     ###########################################################
                     # COMMENT ICONS
                     ###########################################################
                     _html.append('''\t<td class="add-comment-line"><span class="add-comment-content">''')
 
                     if enable_comments and change['action'] != Action.CONTEXT:
                         _html.append('''<a href="#"><span class="icon-comment-add"></span></a>''')
 
                     _html.append('''</span></td><td class="comment-toggle tooltip" title="Toggle Comment Thread"><i class="icon-comment"></i></td>\n''')
 
                     ###########################################################
                     # OLD LINE NUMBER
                     ###########################################################
                     _html.append('''\t<td %(a_id)s class="%(olc)s">''' % {
                         'a_id': anchor_old_id,
                         'olc': old_lineno_class
                     })
 
                     _html.append('''%(link)s''' % {
                         'link': _link_to_if(anchor_link, change['old_lineno'],
                                             '#%s' % anchor_old)
                     })
                     _html.append('''</td>\n''')
                     ###########################################################
                     # NEW LINE NUMBER
                     ###########################################################
 
                     _html.append('''\t<td %(a_id)s class="%(nlc)s">''' % {
                         'a_id': anchor_new_id,
                         'nlc': new_lineno_class
                     })
 
                     _html.append('''%(link)s''' % {
                         'link': _link_to_if(anchor_link, change['new_lineno'],
                                             '#%s' % anchor_new)
                     })
                     _html.append('''</td>\n''')
                     ###########################################################
                     # CODE
                     ###########################################################
                     code_classes = [code_class]
                     if (not enable_comments or
                             change['action'] == Action.CONTEXT):
                         code_classes.append('no-comment')
                     _html.append('\t<td class="%s">' % ' '.join(code_classes))
                     _html.append('''\n\t\t<pre>%(code)s</pre>\n''' % {
                         'code': change['line']
                     })
 
                     _html.append('''\t</td>''')
                     _html.append('''\n</tr>\n''')
         _html.append('''</table>''')
         if _html_empty:
             return None
         return ''.join(_html)
 
     def stat(self):
         """
         Returns tuple of added, and removed lines for this instance
         """
         return self.adds, self.removes
 
     def get_context_of_line(
             self, path, diff_line=None, context_before=3, context_after=3):
         """
         Returns the context lines for the specified diff line.
 
         :type diff_line: :class:`DiffLineNumber`
         """
         assert self.parsed, "DiffProcessor is not initialized."
 
         if None not in diff_line:
             raise ValueError(
                 "Cannot specify both line numbers: {}".format(diff_line))
 
         file_diff = self._get_file_diff(path)
         chunk, idx = self._find_chunk_line_index(file_diff, diff_line)
 
         first_line_to_include = max(idx - context_before, 0)
         first_line_after_context = idx + context_after + 1
         context_lines = chunk[first_line_to_include:first_line_after_context]
 
         line_contents = [
             _context_line(line) for line in context_lines
             if _is_diff_content(line)]
         # TODO: johbo: Interim fixup, the diff chunks drop the final newline.
         # Once they are fixed, we can drop this line here.
         if line_contents:
             line_contents[-1] = (
                 line_contents[-1][0], line_contents[-1][1].rstrip('\n') + '\n')
         return line_contents
 
     def find_context(self, path, context, offset=0):
         """
         Finds the given `context` inside of the diff.
 
         Use the parameter `offset` to specify which offset the target line has
         inside of the given `context`. This way the correct diff line will be
         returned.
 
         :param offset: Shall be used to specify the offset of the main line
             within the given `context`.
         """
         if offset < 0 or offset >= len(context):
             raise ValueError(
                 "Only positive values up to the length of the context "
                 "minus one are allowed.")
 
         matches = []
         file_diff = self._get_file_diff(path)
 
         for chunk in file_diff['chunks']:
             context_iter = iter(context)
             for line_idx, line in enumerate(chunk):
                 try:
                     if _context_line(line) == context_iter.next():
                         continue
                 except StopIteration:
                     matches.append((line_idx, chunk))
                 context_iter = iter(context)
 
         # Increment position and triger StopIteration
         # if we had a match at the end
         line_idx += 1
         try:
             context_iter.next()
         except StopIteration:
             matches.append((line_idx, chunk))
 
         effective_offset = len(context) - offset
         found_at_diff_lines = [
             _line_to_diff_line_number(chunk[idx - effective_offset])
             for idx, chunk in matches]
 
         return found_at_diff_lines
 
     def _get_file_diff(self, path):
         for file_diff in self.parsed_diff:
             if file_diff['filename'] == path:
                 break
         else:
             raise FileNotInDiffException("File {} not in diff".format(path))
         return file_diff
 
     def _find_chunk_line_index(self, file_diff, diff_line):
         for chunk in file_diff['chunks']:
             for idx, line in enumerate(chunk):
                 if line['old_lineno'] == diff_line.old:
                     return chunk, idx
                 if line['new_lineno'] == diff_line.new:
                     return chunk, idx
         raise LineNotInDiffException(
             "The line {} is not part of the diff.".format(diff_line))
 
 
 def _is_diff_content(line):
     return line['action'] in (
         Action.UNMODIFIED, Action.ADD, Action.DELETE)
 
 
 def _context_line(line):
     return (line['action'], line['line'])
 
 
 DiffLineNumber = collections.namedtuple('DiffLineNumber', ['old', 'new'])
 
 
 def _line_to_diff_line_number(line):
     new_line_no = line['new_lineno'] or None
     old_line_no = line['old_lineno'] or None
     return DiffLineNumber(old=old_line_no, new=new_line_no)
 
 
 class FileNotInDiffException(Exception):
     """
     Raised when the context for a missing file is requested.
 
     If you request the context for a line in a file which is not part of the
     given diff, then this exception is raised.
     """
 
 
 class LineNotInDiffException(Exception):
     """
     Raised when the context for a missing line is requested.
 
     If you request the context for a line in a file and this line is not
     part of the given diff, then this exception is raised.
     """
 
 
 class DiffLimitExceeded(Exception):
     pass