Show More
@@ -0,0 +1,29 b'' | |||
|
1 | # Copyright (C) 2015-2024 RhodeCode GmbH | |
|
2 | # | |
|
3 | # This program is free software: you can redistribute it and/or modify | |
|
4 | # it under the terms of the GNU Affero General Public License, version 3 | |
|
5 | # (only), as published by the Free Software Foundation. | |
|
6 | # | |
|
7 | # This program is distributed in the hope that it will be useful, | |
|
8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
|
9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
|
10 | # GNU General Public License for more details. | |
|
11 | # | |
|
12 | # You should have received a copy of the GNU Affero General Public License | |
|
13 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
|
14 | # | |
|
15 | # This program is dual-licensed. If you wish to learn more about the | |
|
16 | # RhodeCode Enterprise Edition, including its added features, Support services, | |
|
17 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |
|
18 | ||
|
19 | from .fanout_cache import get_archival_cache_store | |
|
20 | from .fanout_cache import get_archival_config | |
|
21 | ||
|
22 | from .utils import archive_iterator | |
|
23 | from .utils import ArchiveCacheLock | |
|
24 | ||
|
25 | ||
|
26 | def includeme(config): | |
|
27 | # init our cache at start | |
|
28 | settings = config.get_settings() | |
|
29 | get_archival_cache_store(settings) |
@@ -0,0 +1,60 b'' | |||
|
1 | # Copyright (C) 2015-2024 RhodeCode GmbH | |
|
2 | # | |
|
3 | # This program is free software: you can redistribute it and/or modify | |
|
4 | # it under the terms of the GNU Affero General Public License, version 3 | |
|
5 | # (only), as published by the Free Software Foundation. | |
|
6 | # | |
|
7 | # This program is distributed in the hope that it will be useful, | |
|
8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
|
9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
|
10 | # GNU General Public License for more details. | |
|
11 | # | |
|
12 | # You should have received a copy of the GNU Affero General Public License | |
|
13 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
|
14 | # | |
|
15 | # This program is dual-licensed. If you wish to learn more about the | |
|
16 | # RhodeCode Enterprise Edition, including its added features, Support services, | |
|
17 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |
|
18 | ||
|
19 | import redis | |
|
20 | from rhodecode.lib._vendor import redis_lock | |
|
21 | ||
|
22 | from .utils import ArchiveCacheLock | |
|
23 | ||
|
24 | ||
|
25 | class GenerationLock: | |
|
26 | """ | |
|
27 | Locking mechanism that detects if a lock is acquired | |
|
28 | ||
|
29 | with GenerationLock(lock_key): | |
|
30 | compute_archive() | |
|
31 | """ | |
|
32 | lock_timeout = 7200 | |
|
33 | ||
|
34 | def __init__(self, lock_key, url): | |
|
35 | self.lock_key = lock_key | |
|
36 | self._create_client(url) | |
|
37 | self.lock = self.get_lock() | |
|
38 | ||
|
39 | def _create_client(self, url): | |
|
40 | connection_pool = redis.ConnectionPool.from_url(url) | |
|
41 | self.writer_client = redis.StrictRedis( | |
|
42 | connection_pool=connection_pool | |
|
43 | ) | |
|
44 | self.reader_client = self.writer_client | |
|
45 | ||
|
46 | def get_lock(self): | |
|
47 | return redis_lock.Lock( | |
|
48 | redis_client=self.writer_client, | |
|
49 | name=self.lock_key, | |
|
50 | expire=self.lock_timeout, | |
|
51 | strict=True | |
|
52 | ) | |
|
53 | ||
|
54 | def __enter__(self): | |
|
55 | acquired = self.lock.acquire(blocking=False) | |
|
56 | if not acquired: | |
|
57 | raise ArchiveCacheLock('Failed to create a lock') | |
|
58 | ||
|
59 | def __exit__(self, exc_type, exc_val, exc_tb): | |
|
60 | self.lock.release() |
@@ -0,0 +1,30 b'' | |||
|
1 | # Copyright (C) 2015-2024 RhodeCode GmbH | |
|
2 | # | |
|
3 | # This program is free software: you can redistribute it and/or modify | |
|
4 | # it under the terms of the GNU Affero General Public License, version 3 | |
|
5 | # (only), as published by the Free Software Foundation. | |
|
6 | # | |
|
7 | # This program is distributed in the hope that it will be useful, | |
|
8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
|
9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
|
10 | # GNU General Public License for more details. | |
|
11 | # | |
|
12 | # You should have received a copy of the GNU Affero General Public License | |
|
13 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
|
14 | # | |
|
15 | # This program is dual-licensed. If you wish to learn more about the | |
|
16 | # RhodeCode Enterprise Edition, including its added features, Support services, | |
|
17 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |
|
18 | ||
|
19 | ||
|
20 | class ArchiveCacheLock(Exception): | |
|
21 | pass | |
|
22 | ||
|
23 | ||
|
24 | def archive_iterator(_reader, block_size: int = 4096 * 512): | |
|
25 | # 4096 * 64 = 64KB | |
|
26 | while 1: | |
|
27 | data = _reader.read(block_size) | |
|
28 | if not data: | |
|
29 | break | |
|
30 | yield data |
@@ -1,803 +1,813 b'' | |||
|
1 | 1 | |
|
2 | 2 | ; ######################################### |
|
3 | 3 | ; RHODECODE COMMUNITY EDITION CONFIGURATION |
|
4 | 4 | ; ######################################### |
|
5 | 5 | |
|
6 | 6 | [DEFAULT] |
|
7 | 7 | ; Debug flag sets all loggers to debug, and enables request tracking |
|
8 | 8 | debug = true |
|
9 | 9 | |
|
10 | 10 | ; ######################################################################## |
|
11 | 11 | ; EMAIL CONFIGURATION |
|
12 | 12 | ; These settings will be used by the RhodeCode mailing system |
|
13 | 13 | ; ######################################################################## |
|
14 | 14 | |
|
15 | 15 | ; prefix all emails subjects with given prefix, helps filtering out emails |
|
16 | 16 | #email_prefix = [RhodeCode] |
|
17 | 17 | |
|
18 | 18 | ; email FROM address all mails will be sent |
|
19 | 19 | #app_email_from = rhodecode-noreply@localhost |
|
20 | 20 | |
|
21 | 21 | #smtp_server = mail.server.com |
|
22 | 22 | #smtp_username = |
|
23 | 23 | #smtp_password = |
|
24 | 24 | #smtp_port = |
|
25 | 25 | #smtp_use_tls = false |
|
26 | 26 | #smtp_use_ssl = true |
|
27 | 27 | |
|
28 | 28 | [server:main] |
|
29 | 29 | ; COMMON HOST/IP CONFIG, This applies mostly to develop setup, |
|
30 | 30 | ; Host port for gunicorn are controlled by gunicorn_conf.py |
|
31 | 31 | host = 127.0.0.1 |
|
32 | 32 | port = 10020 |
|
33 | 33 | |
|
34 | 34 | |
|
35 | 35 | ; ########################### |
|
36 | 36 | ; GUNICORN APPLICATION SERVER |
|
37 | 37 | ; ########################### |
|
38 | 38 | |
|
39 | 39 | ; run with gunicorn --paste rhodecode.ini --config gunicorn_conf.py |
|
40 | 40 | |
|
41 | 41 | ; Module to use, this setting shouldn't be changed |
|
42 | 42 | use = egg:gunicorn#main |
|
43 | 43 | |
|
44 | 44 | ; Prefix middleware for RhodeCode. |
|
45 | 45 | ; recommended when using proxy setup. |
|
46 | 46 | ; allows to set RhodeCode under a prefix in server. |
|
47 | 47 | ; eg https://server.com/custom_prefix. Enable `filter-with =` option below as well. |
|
48 | 48 | ; And set your prefix like: `prefix = /custom_prefix` |
|
49 | 49 | ; be sure to also set beaker.session.cookie_path = /custom_prefix if you need |
|
50 | 50 | ; to make your cookies only work on prefix url |
|
51 | 51 | [filter:proxy-prefix] |
|
52 | 52 | use = egg:PasteDeploy#prefix |
|
53 | 53 | prefix = / |
|
54 | 54 | |
|
55 | 55 | [app:main] |
|
56 | 56 | ; The %(here)s variable will be replaced with the absolute path of parent directory |
|
57 | 57 | ; of this file |
|
58 | 58 | ; Each option in the app:main can be override by an environmental variable |
|
59 | 59 | ; |
|
60 | 60 | ;To override an option: |
|
61 | 61 | ; |
|
62 | 62 | ;RC_<KeyName> |
|
63 | 63 | ;Everything should be uppercase, . and - should be replaced by _. |
|
64 | 64 | ;For example, if you have these configuration settings: |
|
65 | 65 | ;rc_cache.repo_object.backend = foo |
|
66 | 66 | ;can be overridden by |
|
67 | 67 | ;export RC_CACHE_REPO_OBJECT_BACKEND=foo |
|
68 | 68 | |
|
69 | 69 | use = egg:rhodecode-enterprise-ce |
|
70 | 70 | |
|
71 | 71 | ; enable proxy prefix middleware, defined above |
|
72 | 72 | #filter-with = proxy-prefix |
|
73 | 73 | |
|
74 | 74 | ; ############# |
|
75 | 75 | ; DEBUG OPTIONS |
|
76 | 76 | ; ############# |
|
77 | 77 | |
|
78 | 78 | pyramid.reload_templates = true |
|
79 | 79 | |
|
80 | 80 | # During development the we want to have the debug toolbar enabled |
|
81 | 81 | pyramid.includes = |
|
82 | 82 | pyramid_debugtoolbar |
|
83 | 83 | |
|
84 | 84 | debugtoolbar.hosts = 0.0.0.0/0 |
|
85 | 85 | debugtoolbar.exclude_prefixes = |
|
86 | 86 | /css |
|
87 | 87 | /fonts |
|
88 | 88 | /images |
|
89 | 89 | /js |
|
90 | 90 | |
|
91 | 91 | ## RHODECODE PLUGINS ## |
|
92 | 92 | rhodecode.includes = |
|
93 | 93 | rhodecode.api |
|
94 | 94 | |
|
95 | 95 | |
|
96 | 96 | # api prefix url |
|
97 | 97 | rhodecode.api.url = /_admin/api |
|
98 | 98 | |
|
99 | 99 | ; enable debug style page |
|
100 | 100 | debug_style = true |
|
101 | 101 | |
|
102 | 102 | ; ################# |
|
103 | 103 | ; END DEBUG OPTIONS |
|
104 | 104 | ; ################# |
|
105 | 105 | |
|
106 | 106 | ; encryption key used to encrypt social plugin tokens, |
|
107 | 107 | ; remote_urls with credentials etc, if not set it defaults to |
|
108 | 108 | ; `beaker.session.secret` |
|
109 | 109 | #rhodecode.encrypted_values.secret = |
|
110 | 110 | |
|
111 | 111 | ; decryption strict mode (enabled by default). It controls if decryption raises |
|
112 | 112 | ; `SignatureVerificationError` in case of wrong key, or damaged encryption data. |
|
113 | 113 | #rhodecode.encrypted_values.strict = false |
|
114 | 114 | |
|
115 | 115 | ; Pick algorithm for encryption. Either fernet (more secure) or aes (default) |
|
116 | 116 | ; fernet is safer, and we strongly recommend switching to it. |
|
117 | 117 | ; Due to backward compatibility aes is used as default. |
|
118 | 118 | #rhodecode.encrypted_values.algorithm = fernet |
|
119 | 119 | |
|
120 | 120 | ; Return gzipped responses from RhodeCode (static files/application) |
|
121 | 121 | gzip_responses = false |
|
122 | 122 | |
|
123 | 123 | ; Auto-generate javascript routes file on startup |
|
124 | 124 | generate_js_files = false |
|
125 | 125 | |
|
126 | 126 | ; System global default language. |
|
127 | 127 | ; All available languages: en (default), be, de, es, fr, it, ja, pl, pt, ru, zh |
|
128 | 128 | lang = en |
|
129 | 129 | |
|
130 | 130 | ; Perform a full repository scan and import on each server start. |
|
131 | 131 | ; Settings this to true could lead to very long startup time. |
|
132 | 132 | startup.import_repos = false |
|
133 | 133 | |
|
134 | 134 | ; URL at which the application is running. This is used for Bootstrapping |
|
135 | 135 | ; requests in context when no web request is available. Used in ishell, or |
|
136 | 136 | ; SSH calls. Set this for events to receive proper url for SSH calls. |
|
137 | 137 | app.base_url = http://rhodecode.local |
|
138 | 138 | |
|
139 | 139 | ; Host at which the Service API is running. |
|
140 | 140 | app.service_api.host = http://rhodecode.local:10020 |
|
141 | 141 | |
|
142 | 142 | ; Secret for Service API authentication. |
|
143 | 143 | app.service_api.token = |
|
144 | 144 | |
|
145 | 145 | ; Unique application ID. Should be a random unique string for security. |
|
146 | 146 | app_instance_uuid = rc-production |
|
147 | 147 | |
|
148 | 148 | ; Cut off limit for large diffs (size in bytes). If overall diff size on |
|
149 | 149 | ; commit, or pull request exceeds this limit this diff will be displayed |
|
150 | 150 | ; partially. E.g 512000 == 512Kb |
|
151 | 151 | cut_off_limit_diff = 512000 |
|
152 | 152 | |
|
153 | 153 | ; Cut off limit for large files inside diffs (size in bytes). Each individual |
|
154 | 154 | ; file inside diff which exceeds this limit will be displayed partially. |
|
155 | 155 | ; E.g 128000 == 128Kb |
|
156 | 156 | cut_off_limit_file = 128000 |
|
157 | 157 | |
|
158 | 158 | ; Use cached version of vcs repositories everywhere. Recommended to be `true` |
|
159 | 159 | vcs_full_cache = true |
|
160 | 160 | |
|
161 | 161 | ; Force https in RhodeCode, fixes https redirects, assumes it's always https. |
|
162 | 162 | ; Normally this is controlled by proper flags sent from http server such as Nginx or Apache |
|
163 | 163 | force_https = false |
|
164 | 164 | |
|
165 | 165 | ; use Strict-Transport-Security headers |
|
166 | 166 | use_htsts = false |
|
167 | 167 | |
|
168 | 168 | ; Set to true if your repos are exposed using the dumb protocol |
|
169 | 169 | git_update_server_info = false |
|
170 | 170 | |
|
171 | 171 | ; RSS/ATOM feed options |
|
172 | 172 | rss_cut_off_limit = 256000 |
|
173 | 173 | rss_items_per_page = 10 |
|
174 | 174 | rss_include_diff = false |
|
175 | 175 | |
|
176 | 176 | ; gist URL alias, used to create nicer urls for gist. This should be an |
|
177 | 177 | ; url that does rewrites to _admin/gists/{gistid}. |
|
178 | 178 | ; example: http://gist.rhodecode.org/{gistid}. Empty means use the internal |
|
179 | 179 | ; RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid} |
|
180 | 180 | gist_alias_url = |
|
181 | 181 | |
|
182 | 182 | ; List of views (using glob pattern syntax) that AUTH TOKENS could be |
|
183 | 183 | ; used for access. |
|
184 | 184 | ; Adding ?auth_token=TOKEN_HASH to the url authenticates this request as if it |
|
185 | 185 | ; came from the the logged in user who own this authentication token. |
|
186 | 186 | ; Additionally @TOKEN syntax can be used to bound the view to specific |
|
187 | 187 | ; authentication token. Such view would be only accessible when used together |
|
188 | 188 | ; with this authentication token |
|
189 | 189 | ; list of all views can be found under `/_admin/permissions/auth_token_access` |
|
190 | 190 | ; The list should be "," separated and on a single line. |
|
191 | 191 | ; Most common views to enable: |
|
192 | 192 | |
|
193 | 193 | # RepoCommitsView:repo_commit_download |
|
194 | 194 | # RepoCommitsView:repo_commit_patch |
|
195 | 195 | # RepoCommitsView:repo_commit_raw |
|
196 | 196 | # RepoCommitsView:repo_commit_raw@TOKEN |
|
197 | 197 | # RepoFilesView:repo_files_diff |
|
198 | 198 | # RepoFilesView:repo_archivefile |
|
199 | 199 | # RepoFilesView:repo_file_raw |
|
200 | 200 | # GistView:* |
|
201 | 201 | api_access_controllers_whitelist = |
|
202 | 202 | |
|
203 | 203 | ; Default encoding used to convert from and to unicode |
|
204 | 204 | ; can be also a comma separated list of encoding in case of mixed encodings |
|
205 | 205 | default_encoding = UTF-8 |
|
206 | 206 | |
|
207 | 207 | ; instance-id prefix |
|
208 | 208 | ; a prefix key for this instance used for cache invalidation when running |
|
209 | 209 | ; multiple instances of RhodeCode, make sure it's globally unique for |
|
210 | 210 | ; all running RhodeCode instances. Leave empty if you don't use it |
|
211 | 211 | instance_id = |
|
212 | 212 | |
|
213 | 213 | ; Fallback authentication plugin. Set this to a plugin ID to force the usage |
|
214 | 214 | ; of an authentication plugin also if it is disabled by it's settings. |
|
215 | 215 | ; This could be useful if you are unable to log in to the system due to broken |
|
216 | 216 | ; authentication settings. Then you can enable e.g. the internal RhodeCode auth |
|
217 | 217 | ; module to log in again and fix the settings. |
|
218 | 218 | ; Available builtin plugin IDs (hash is part of the ID): |
|
219 | 219 | ; egg:rhodecode-enterprise-ce#rhodecode |
|
220 | 220 | ; egg:rhodecode-enterprise-ce#pam |
|
221 | 221 | ; egg:rhodecode-enterprise-ce#ldap |
|
222 | 222 | ; egg:rhodecode-enterprise-ce#jasig_cas |
|
223 | 223 | ; egg:rhodecode-enterprise-ce#headers |
|
224 | 224 | ; egg:rhodecode-enterprise-ce#crowd |
|
225 | 225 | |
|
226 | 226 | #rhodecode.auth_plugin_fallback = egg:rhodecode-enterprise-ce#rhodecode |
|
227 | 227 | |
|
228 | 228 | ; Flag to control loading of legacy plugins in py:/path format |
|
229 | 229 | auth_plugin.import_legacy_plugins = true |
|
230 | 230 | |
|
231 | 231 | ; alternative return HTTP header for failed authentication. Default HTTP |
|
232 | 232 | ; response is 401 HTTPUnauthorized. Currently HG clients have troubles with |
|
233 | 233 | ; handling that causing a series of failed authentication calls. |
|
234 | 234 | ; Set this variable to 403 to return HTTPForbidden, or any other HTTP code |
|
235 | 235 | ; This will be served instead of default 401 on bad authentication |
|
236 | 236 | auth_ret_code = |
|
237 | 237 | |
|
238 | 238 | ; use special detection method when serving auth_ret_code, instead of serving |
|
239 | 239 | ; ret_code directly, use 401 initially (Which triggers credentials prompt) |
|
240 | 240 | ; and then serve auth_ret_code to clients |
|
241 | 241 | auth_ret_code_detection = false |
|
242 | 242 | |
|
243 | 243 | ; locking return code. When repository is locked return this HTTP code. 2XX |
|
244 | 244 | ; codes don't break the transactions while 4XX codes do |
|
245 | 245 | lock_ret_code = 423 |
|
246 | 246 | |
|
247 | 247 | ; Filesystem location were repositories should be stored |
|
248 | 248 | repo_store.path = /var/opt/rhodecode_repo_store |
|
249 | 249 | |
|
250 | 250 | ; allows to setup custom hooks in settings page |
|
251 | 251 | allow_custom_hooks_settings = true |
|
252 | 252 | |
|
253 | 253 | ; Generated license token required for EE edition license. |
|
254 | 254 | ; New generated token value can be found in Admin > settings > license page. |
|
255 | 255 | license_token = |
|
256 | 256 | |
|
257 | 257 | ; This flag hides sensitive information on the license page such as token, and license data |
|
258 | 258 | license.hide_license_info = false |
|
259 | 259 | |
|
260 | 260 | ; supervisor connection uri, for managing supervisor and logs. |
|
261 | 261 | supervisor.uri = |
|
262 | 262 | |
|
263 | 263 | ; supervisord group name/id we only want this RC instance to handle |
|
264 | 264 | supervisor.group_id = dev |
|
265 | 265 | |
|
266 | 266 | ; Display extended labs settings |
|
267 | 267 | labs_settings_active = true |
|
268 | 268 | |
|
269 | 269 | ; Custom exception store path, defaults to TMPDIR |
|
270 | 270 | ; This is used to store exception from RhodeCode in shared directory |
|
271 | 271 | #exception_tracker.store_path = |
|
272 | 272 | |
|
273 | 273 | ; Send email with exception details when it happens |
|
274 | 274 | #exception_tracker.send_email = false |
|
275 | 275 | |
|
276 | 276 | ; Comma separated list of recipients for exception emails, |
|
277 | 277 | ; e.g admin@rhodecode.com,devops@rhodecode.com |
|
278 | 278 | ; Can be left empty, then emails will be sent to ALL super-admins |
|
279 | 279 | #exception_tracker.send_email_recipients = |
|
280 | 280 | |
|
281 | 281 | ; optional prefix to Add to email Subject |
|
282 | 282 | #exception_tracker.email_prefix = [RHODECODE ERROR] |
|
283 | 283 | |
|
284 | 284 | ; File store configuration. This is used to store and serve uploaded files |
|
285 | 285 | file_store.enabled = true |
|
286 | 286 | |
|
287 | 287 | ; Storage backend, available options are: local |
|
288 | 288 | file_store.backend = local |
|
289 | 289 | |
|
290 | 290 | ; path to store the uploaded binaries and artifacts |
|
291 | 291 | file_store.storage_path = /var/opt/rhodecode_data/file_store |
|
292 | 292 | |
|
293 | 293 | ; Uncomment and set this path to control settings for archive download cache. |
|
294 | 294 | ; Generated repo archives will be cached at this location |
|
295 | 295 | ; and served from the cache during subsequent requests for the same archive of |
|
296 | 296 | ; the repository. This path is important to be shared across filesystems and with |
|
297 | 297 | ; RhodeCode and vcsserver |
|
298 | 298 | |
|
299 | ; Redis url to acquire/check generation of archives locks | |
|
300 | archive_cache.locking.url = redis://redis:6379/1 | |
|
301 | ||
|
302 | ; Storage backend, only 'filesystem' is available now | |
|
303 | archive_cache.backend.type = filesystem | |
|
304 | ||
|
299 | 305 | ; Default is $cache_dir/archive_cache if not set |
|
300 |
archive_cache.store_dir = /var/opt/rhodecode_data/ |
|
|
306 | archive_cache.filesystem.store_dir = /var/opt/rhodecode_data/archive_cache | |
|
301 | 307 | |
|
302 | 308 | ; The limit in GB sets how much data we cache before recycling last used, defaults to 10 gb |
|
303 |
archive_cache.cache_size_gb = 1 |
|
|
309 | archive_cache.filesystem.cache_size_gb = 1 | |
|
310 | ||
|
311 | ; Eviction policy used to clear out after cache_size_gb limit is reached | |
|
312 | archive_cache.filesystem.eviction_policy = least-recently-stored | |
|
304 | 313 | |
|
305 | 314 | ; By default cache uses sharding technique, this specifies how many shards are there |
|
306 |
archive_cache.cache_shards = |
|
|
315 | archive_cache.filesystem.cache_shards = 8 | |
|
316 | ||
|
307 | 317 | |
|
308 | 318 | ; ############# |
|
309 | 319 | ; CELERY CONFIG |
|
310 | 320 | ; ############# |
|
311 | 321 | |
|
312 | 322 | ; manually run celery: /path/to/celery worker --task-events --beat --app rhodecode.lib.celerylib.loader --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler --loglevel DEBUG --ini /path/to/rhodecode.ini |
|
313 | 323 | |
|
314 | 324 | use_celery = true |
|
315 | 325 | |
|
316 | 326 | ; path to store schedule database |
|
317 | 327 | #celerybeat-schedule.path = |
|
318 | 328 | |
|
319 | 329 | ; connection url to the message broker (default redis) |
|
320 | 330 | celery.broker_url = redis://redis:6379/8 |
|
321 | 331 | |
|
322 | 332 | ; results backend to get results for (default redis) |
|
323 | 333 | celery.result_backend = redis://redis:6379/8 |
|
324 | 334 | |
|
325 | 335 | ; rabbitmq example |
|
326 | 336 | #celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost |
|
327 | 337 | |
|
328 | 338 | ; maximum tasks to execute before worker restart |
|
329 | 339 | celery.max_tasks_per_child = 20 |
|
330 | 340 | |
|
331 | 341 | ; tasks will never be sent to the queue, but executed locally instead. |
|
332 | 342 | celery.task_always_eager = false |
|
333 | 343 | |
|
334 | 344 | ; ############# |
|
335 | 345 | ; DOGPILE CACHE |
|
336 | 346 | ; ############# |
|
337 | 347 | |
|
338 | 348 | ; Default cache dir for caches. Putting this into a ramdisk can boost performance. |
|
339 | 349 | ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space |
|
340 | 350 | cache_dir = /var/opt/rhodecode_data |
|
341 | 351 | |
|
342 | 352 | ; ********************************************* |
|
343 | 353 | ; `sql_cache_short` cache for heavy SQL queries |
|
344 | 354 | ; Only supported backend is `memory_lru` |
|
345 | 355 | ; ********************************************* |
|
346 | 356 | rc_cache.sql_cache_short.backend = dogpile.cache.rc.memory_lru |
|
347 | 357 | rc_cache.sql_cache_short.expiration_time = 30 |
|
348 | 358 | |
|
349 | 359 | |
|
350 | 360 | ; ***************************************************** |
|
351 | 361 | ; `cache_repo_longterm` cache for repo object instances |
|
352 | 362 | ; Only supported backend is `memory_lru` |
|
353 | 363 | ; ***************************************************** |
|
354 | 364 | rc_cache.cache_repo_longterm.backend = dogpile.cache.rc.memory_lru |
|
355 | 365 | ; by default we use 30 Days, cache is still invalidated on push |
|
356 | 366 | rc_cache.cache_repo_longterm.expiration_time = 2592000 |
|
357 | 367 | ; max items in LRU cache, set to smaller number to save memory, and expire last used caches |
|
358 | 368 | rc_cache.cache_repo_longterm.max_size = 10000 |
|
359 | 369 | |
|
360 | 370 | |
|
361 | 371 | ; ********************************************* |
|
362 | 372 | ; `cache_general` cache for general purpose use |
|
363 | 373 | ; for simplicity use rc.file_namespace backend, |
|
364 | 374 | ; for performance and scale use rc.redis |
|
365 | 375 | ; ********************************************* |
|
366 | 376 | rc_cache.cache_general.backend = dogpile.cache.rc.file_namespace |
|
367 | 377 | rc_cache.cache_general.expiration_time = 43200 |
|
368 | 378 | ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set |
|
369 | 379 | #rc_cache.cache_general.arguments.filename = /tmp/cache_general_db |
|
370 | 380 | |
|
371 | 381 | ; alternative `cache_general` redis backend with distributed lock |
|
372 | 382 | #rc_cache.cache_general.backend = dogpile.cache.rc.redis |
|
373 | 383 | #rc_cache.cache_general.expiration_time = 300 |
|
374 | 384 | |
|
375 | 385 | ; redis_expiration_time needs to be greater then expiration_time |
|
376 | 386 | #rc_cache.cache_general.arguments.redis_expiration_time = 7200 |
|
377 | 387 | |
|
378 | 388 | #rc_cache.cache_general.arguments.host = localhost |
|
379 | 389 | #rc_cache.cache_general.arguments.port = 6379 |
|
380 | 390 | #rc_cache.cache_general.arguments.db = 0 |
|
381 | 391 | #rc_cache.cache_general.arguments.socket_timeout = 30 |
|
382 | 392 | ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends |
|
383 | 393 | #rc_cache.cache_general.arguments.distributed_lock = true |
|
384 | 394 | |
|
385 | 395 | ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen |
|
386 | 396 | #rc_cache.cache_general.arguments.lock_auto_renewal = true |
|
387 | 397 | |
|
388 | 398 | ; ************************************************* |
|
389 | 399 | ; `cache_perms` cache for permission tree, auth TTL |
|
390 | 400 | ; for simplicity use rc.file_namespace backend, |
|
391 | 401 | ; for performance and scale use rc.redis |
|
392 | 402 | ; ************************************************* |
|
393 | 403 | rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace |
|
394 | 404 | rc_cache.cache_perms.expiration_time = 3600 |
|
395 | 405 | ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set |
|
396 | 406 | #rc_cache.cache_perms.arguments.filename = /tmp/cache_perms_db |
|
397 | 407 | |
|
398 | 408 | ; alternative `cache_perms` redis backend with distributed lock |
|
399 | 409 | #rc_cache.cache_perms.backend = dogpile.cache.rc.redis |
|
400 | 410 | #rc_cache.cache_perms.expiration_time = 300 |
|
401 | 411 | |
|
402 | 412 | ; redis_expiration_time needs to be greater then expiration_time |
|
403 | 413 | #rc_cache.cache_perms.arguments.redis_expiration_time = 7200 |
|
404 | 414 | |
|
405 | 415 | #rc_cache.cache_perms.arguments.host = localhost |
|
406 | 416 | #rc_cache.cache_perms.arguments.port = 6379 |
|
407 | 417 | #rc_cache.cache_perms.arguments.db = 0 |
|
408 | 418 | #rc_cache.cache_perms.arguments.socket_timeout = 30 |
|
409 | 419 | ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends |
|
410 | 420 | #rc_cache.cache_perms.arguments.distributed_lock = true |
|
411 | 421 | |
|
412 | 422 | ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen |
|
413 | 423 | #rc_cache.cache_perms.arguments.lock_auto_renewal = true |
|
414 | 424 | |
|
415 | 425 | ; *************************************************** |
|
416 | 426 | ; `cache_repo` cache for file tree, Readme, RSS FEEDS |
|
417 | 427 | ; for simplicity use rc.file_namespace backend, |
|
418 | 428 | ; for performance and scale use rc.redis |
|
419 | 429 | ; *************************************************** |
|
420 | 430 | rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace |
|
421 | 431 | rc_cache.cache_repo.expiration_time = 2592000 |
|
422 | 432 | ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set |
|
423 | 433 | #rc_cache.cache_repo.arguments.filename = /tmp/cache_repo_db |
|
424 | 434 | |
|
425 | 435 | ; alternative `cache_repo` redis backend with distributed lock |
|
426 | 436 | #rc_cache.cache_repo.backend = dogpile.cache.rc.redis |
|
427 | 437 | #rc_cache.cache_repo.expiration_time = 2592000 |
|
428 | 438 | |
|
429 | 439 | ; redis_expiration_time needs to be greater then expiration_time |
|
430 | 440 | #rc_cache.cache_repo.arguments.redis_expiration_time = 2678400 |
|
431 | 441 | |
|
432 | 442 | #rc_cache.cache_repo.arguments.host = localhost |
|
433 | 443 | #rc_cache.cache_repo.arguments.port = 6379 |
|
434 | 444 | #rc_cache.cache_repo.arguments.db = 1 |
|
435 | 445 | #rc_cache.cache_repo.arguments.socket_timeout = 30 |
|
436 | 446 | ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends |
|
437 | 447 | #rc_cache.cache_repo.arguments.distributed_lock = true |
|
438 | 448 | |
|
439 | 449 | ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen |
|
440 | 450 | #rc_cache.cache_repo.arguments.lock_auto_renewal = true |
|
441 | 451 | |
|
442 | 452 | ; ############## |
|
443 | 453 | ; BEAKER SESSION |
|
444 | 454 | ; ############## |
|
445 | 455 | |
|
446 | 456 | ; beaker.session.type is type of storage options for the logged users sessions. Current allowed |
|
447 | 457 | ; types are file, ext:redis, ext:database, ext:memcached |
|
448 | 458 | ; Fastest ones are ext:redis and ext:database, DO NOT use memory type for session |
|
449 | 459 | #beaker.session.type = file |
|
450 | 460 | #beaker.session.data_dir = %(here)s/data/sessions |
|
451 | 461 | |
|
452 | 462 | ; Redis based sessions |
|
453 | 463 | beaker.session.type = ext:redis |
|
454 | 464 | beaker.session.url = redis://redis:6379/2 |
|
455 | 465 | |
|
456 | 466 | ; DB based session, fast, and allows easy management over logged in users |
|
457 | 467 | #beaker.session.type = ext:database |
|
458 | 468 | #beaker.session.table_name = db_session |
|
459 | 469 | #beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode |
|
460 | 470 | #beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode |
|
461 | 471 | #beaker.session.sa.pool_recycle = 3600 |
|
462 | 472 | #beaker.session.sa.echo = false |
|
463 | 473 | |
|
464 | 474 | beaker.session.key = rhodecode |
|
465 | 475 | beaker.session.secret = develop-rc-uytcxaz |
|
466 | 476 | beaker.session.lock_dir = /data_ramdisk/lock |
|
467 | 477 | |
|
468 | 478 | ; Secure encrypted cookie. Requires AES and AES python libraries |
|
469 | 479 | ; you must disable beaker.session.secret to use this |
|
470 | 480 | #beaker.session.encrypt_key = key_for_encryption |
|
471 | 481 | #beaker.session.validate_key = validation_key |
|
472 | 482 | |
|
473 | 483 | ; Sets session as invalid (also logging out user) if it haven not been |
|
474 | 484 | ; accessed for given amount of time in seconds |
|
475 | 485 | beaker.session.timeout = 2592000 |
|
476 | 486 | beaker.session.httponly = true |
|
477 | 487 | |
|
478 | 488 | ; Path to use for the cookie. Set to prefix if you use prefix middleware |
|
479 | 489 | #beaker.session.cookie_path = /custom_prefix |
|
480 | 490 | |
|
481 | 491 | ; Set https secure cookie |
|
482 | 492 | beaker.session.secure = false |
|
483 | 493 | |
|
484 | 494 | ; default cookie expiration time in seconds, set to `true` to set expire |
|
485 | 495 | ; at browser close |
|
486 | 496 | #beaker.session.cookie_expires = 3600 |
|
487 | 497 | |
|
488 | 498 | ; ############################# |
|
489 | 499 | ; SEARCH INDEXING CONFIGURATION |
|
490 | 500 | ; ############################# |
|
491 | 501 | |
|
492 | 502 | ; Full text search indexer is available in rhodecode-tools under |
|
493 | 503 | ; `rhodecode-tools index` command |
|
494 | 504 | |
|
495 | 505 | ; WHOOSH Backend, doesn't require additional services to run |
|
496 | 506 | ; it works good with few dozen repos |
|
497 | 507 | search.module = rhodecode.lib.index.whoosh |
|
498 | 508 | search.location = %(here)s/data/index |
|
499 | 509 | |
|
500 | 510 | ; #################### |
|
501 | 511 | ; CHANNELSTREAM CONFIG |
|
502 | 512 | ; #################### |
|
503 | 513 | |
|
504 | 514 | ; channelstream enables persistent connections and live notification |
|
505 | 515 | ; in the system. It's also used by the chat system |
|
506 | 516 | |
|
507 | 517 | channelstream.enabled = true |
|
508 | 518 | |
|
509 | 519 | ; server address for channelstream server on the backend |
|
510 | 520 | channelstream.server = channelstream:9800 |
|
511 | 521 | |
|
512 | 522 | ; location of the channelstream server from outside world |
|
513 | 523 | ; use ws:// for http or wss:// for https. This address needs to be handled |
|
514 | 524 | ; by external HTTP server such as Nginx or Apache |
|
515 | 525 | ; see Nginx/Apache configuration examples in our docs |
|
516 | 526 | channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream |
|
517 | 527 | channelstream.secret = ENV_GENERATED |
|
518 | 528 | channelstream.history.location = /var/opt/rhodecode_data/channelstream_history |
|
519 | 529 | |
|
520 | 530 | ; Internal application path that Javascript uses to connect into. |
|
521 | 531 | ; If you use proxy-prefix the prefix should be added before /_channelstream |
|
522 | 532 | channelstream.proxy_path = /_channelstream |
|
523 | 533 | |
|
524 | 534 | |
|
525 | 535 | ; ############################## |
|
526 | 536 | ; MAIN RHODECODE DATABASE CONFIG |
|
527 | 537 | ; ############################## |
|
528 | 538 | |
|
529 | 539 | #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30 |
|
530 | 540 | #sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode |
|
531 | 541 | #sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode?charset=utf8 |
|
532 | 542 | ; pymysql is an alternative driver for MySQL, use in case of problems with default one |
|
533 | 543 | #sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode |
|
534 | 544 | |
|
535 | 545 | sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30 |
|
536 | 546 | |
|
537 | 547 | ; see sqlalchemy docs for other advanced settings |
|
538 | 548 | ; print the sql statements to output |
|
539 | 549 | sqlalchemy.db1.echo = false |
|
540 | 550 | |
|
541 | 551 | ; recycle the connections after this amount of seconds |
|
542 | 552 | sqlalchemy.db1.pool_recycle = 3600 |
|
543 | 553 | |
|
544 | 554 | ; the number of connections to keep open inside the connection pool. |
|
545 | 555 | ; 0 indicates no limit |
|
546 | 556 | ; the general calculus with gevent is: |
|
547 | 557 | ; if your system allows 500 concurrent greenlets (max_connections) that all do database access, |
|
548 | 558 | ; then increase pool size + max overflow so that they add up to 500. |
|
549 | 559 | #sqlalchemy.db1.pool_size = 5 |
|
550 | 560 | |
|
551 | 561 | ; The number of connections to allow in connection pool "overflow", that is |
|
552 | 562 | ; connections that can be opened above and beyond the pool_size setting, |
|
553 | 563 | ; which defaults to five. |
|
554 | 564 | #sqlalchemy.db1.max_overflow = 10 |
|
555 | 565 | |
|
556 | 566 | ; Connection check ping, used to detect broken database connections |
|
557 | 567 | ; could be enabled to better handle cases if MySQL has gone away errors |
|
558 | 568 | #sqlalchemy.db1.ping_connection = true |
|
559 | 569 | |
|
560 | 570 | ; ########## |
|
561 | 571 | ; VCS CONFIG |
|
562 | 572 | ; ########## |
|
563 | 573 | vcs.server.enable = true |
|
564 | 574 | vcs.server = vcsserver:10010 |
|
565 | 575 | |
|
566 | 576 | ; Web server connectivity protocol, responsible for web based VCS operations |
|
567 | 577 | ; Available protocols are: |
|
568 | 578 | ; `http` - use http-rpc backend (default) |
|
569 | 579 | vcs.server.protocol = http |
|
570 | 580 | |
|
571 | 581 | ; Push/Pull operations protocol, available options are: |
|
572 | 582 | ; `http` - use http-rpc backend (default) |
|
573 | 583 | vcs.scm_app_implementation = http |
|
574 | 584 | |
|
575 | 585 | ; Push/Pull operations hooks protocol, available options are: |
|
576 | 586 | ; `http` - use http-rpc backend (default) |
|
577 | 587 | ; `celery` - use celery based hooks |
|
578 | 588 | vcs.hooks.protocol = http |
|
579 | 589 | |
|
580 | 590 | ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be |
|
581 | 591 | ; accessible via network. |
|
582 | 592 | ; Use vcs.hooks.host = "*" to bind to current hostname (for Docker) |
|
583 | 593 | vcs.hooks.host = * |
|
584 | 594 | |
|
585 | 595 | ; Start VCSServer with this instance as a subprocess, useful for development |
|
586 | 596 | vcs.start_server = false |
|
587 | 597 | |
|
588 | 598 | ; List of enabled VCS backends, available options are: |
|
589 | 599 | ; `hg` - mercurial |
|
590 | 600 | ; `git` - git |
|
591 | 601 | ; `svn` - subversion |
|
592 | 602 | vcs.backends = hg, git, svn |
|
593 | 603 | |
|
594 | 604 | ; Wait this number of seconds before killing connection to the vcsserver |
|
595 | 605 | vcs.connection_timeout = 3600 |
|
596 | 606 | |
|
597 | 607 | ; Cache flag to cache vcsserver remote calls locally |
|
598 | 608 | ; It uses cache_region `cache_repo` |
|
599 | 609 | vcs.methods.cache = true |
|
600 | 610 | |
|
601 | 611 | ; #################################################### |
|
602 | 612 | ; Subversion proxy support (mod_dav_svn) |
|
603 | 613 | ; Maps RhodeCode repo groups into SVN paths for Apache |
|
604 | 614 | ; #################################################### |
|
605 | 615 | |
|
606 | 616 | ; Compatibility version when creating SVN repositories. Defaults to newest version when commented out. |
|
607 | 617 | ; Set a numeric version for your current SVN e.g 1.8, or 1.12 |
|
608 | 618 | ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible |
|
609 | 619 | #vcs.svn.compatible_version = 1.8 |
|
610 | 620 | |
|
611 | 621 | ; Enable SVN proxy of requests over HTTP |
|
612 | 622 | vcs.svn.proxy.enabled = true |
|
613 | 623 | |
|
614 | 624 | ; host to connect to running SVN subsystem |
|
615 | 625 | vcs.svn.proxy.host = http://svn:8090 |
|
616 | 626 | |
|
617 | 627 | ; Enable or disable the config file generation. |
|
618 | 628 | svn.proxy.generate_config = true |
|
619 | 629 | |
|
620 | 630 | ; Generate config file with `SVNListParentPath` set to `On`. |
|
621 | 631 | svn.proxy.list_parent_path = true |
|
622 | 632 | |
|
623 | 633 | ; Set location and file name of generated config file. |
|
624 | 634 | svn.proxy.config_file_path = /etc/rhodecode/conf/svn/mod_dav_svn.conf |
|
625 | 635 | |
|
626 | 636 | ; alternative mod_dav config template. This needs to be a valid mako template |
|
627 | 637 | ; Example template can be found in the source code: |
|
628 | 638 | ; rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako |
|
629 | 639 | #svn.proxy.config_template = ~/.rccontrol/enterprise-1/custom_svn_conf.mako |
|
630 | 640 | |
|
631 | 641 | ; Used as a prefix to the `Location` block in the generated config file. |
|
632 | 642 | ; In most cases it should be set to `/`. |
|
633 | 643 | svn.proxy.location_root = / |
|
634 | 644 | |
|
635 | 645 | ; Command to reload the mod dav svn configuration on change. |
|
636 | 646 | ; Example: `/etc/init.d/apache2 reload` or /home/USER/apache_reload.sh |
|
637 | 647 | ; Make sure user who runs RhodeCode process is allowed to reload Apache |
|
638 | 648 | #svn.proxy.reload_cmd = /etc/init.d/apache2 reload |
|
639 | 649 | |
|
640 | 650 | ; If the timeout expires before the reload command finishes, the command will |
|
641 | 651 | ; be killed. Setting it to zero means no timeout. Defaults to 10 seconds. |
|
642 | 652 | #svn.proxy.reload_timeout = 10 |
|
643 | 653 | |
|
644 | 654 | ; #################### |
|
645 | 655 | ; SSH Support Settings |
|
646 | 656 | ; #################### |
|
647 | 657 | |
|
648 | 658 | ; Defines if a custom authorized_keys file should be created and written on |
|
649 | 659 | ; any change user ssh keys. Setting this to false also disables possibility |
|
650 | 660 | ; of adding SSH keys by users from web interface. Super admins can still |
|
651 | 661 | ; manage SSH Keys. |
|
652 | 662 | ssh.generate_authorized_keyfile = true |
|
653 | 663 | |
|
654 | 664 | ; Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding` |
|
655 | 665 | # ssh.authorized_keys_ssh_opts = |
|
656 | 666 | |
|
657 | 667 | ; Path to the authorized_keys file where the generate entries are placed. |
|
658 | 668 | ; It is possible to have multiple key files specified in `sshd_config` e.g. |
|
659 | 669 | ; AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode |
|
660 | 670 | ssh.authorized_keys_file_path = /etc/rhodecode/conf/ssh/authorized_keys_rhodecode |
|
661 | 671 | |
|
662 | 672 | ; Command to execute the SSH wrapper. The binary is available in the |
|
663 | 673 | ; RhodeCode installation directory. |
|
664 | 674 | ; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper |
|
665 | 675 | ; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2 |
|
666 | 676 | ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper |
|
667 | 677 | |
|
668 | 678 | ; Allow shell when executing the ssh-wrapper command |
|
669 | 679 | ssh.wrapper_cmd_allow_shell = false |
|
670 | 680 | |
|
671 | 681 | ; Enables logging, and detailed output send back to the client during SSH |
|
672 | 682 | ; operations. Useful for debugging, shouldn't be used in production. |
|
673 | 683 | ssh.enable_debug_logging = true |
|
674 | 684 | |
|
675 | 685 | ; Paths to binary executable, by default they are the names, but we can |
|
676 | 686 | ; override them if we want to use a custom one |
|
677 | 687 | ssh.executable.hg = /usr/local/bin/rhodecode_bin/vcs_bin/hg |
|
678 | 688 | ssh.executable.git = /usr/local/bin/rhodecode_bin/vcs_bin/git |
|
679 | 689 | ssh.executable.svn = /usr/local/bin/rhodecode_bin/vcs_bin/svnserve |
|
680 | 690 | |
|
681 | 691 | ; Enables SSH key generator web interface. Disabling this still allows users |
|
682 | 692 | ; to add their own keys. |
|
683 | 693 | ssh.enable_ui_key_generator = true |
|
684 | 694 | |
|
685 | 695 | ; Statsd client config, this is used to send metrics to statsd |
|
686 | 696 | ; We recommend setting statsd_exported and scrape them using Prometheus |
|
687 | 697 | #statsd.enabled = false |
|
688 | 698 | #statsd.statsd_host = 0.0.0.0 |
|
689 | 699 | #statsd.statsd_port = 8125 |
|
690 | 700 | #statsd.statsd_prefix = |
|
691 | 701 | #statsd.statsd_ipv6 = false |
|
692 | 702 | |
|
693 | 703 | ; configure logging automatically at server startup set to false |
|
694 | 704 | ; to use the below custom logging config. |
|
695 | 705 | ; RC_LOGGING_FORMATTER |
|
696 | 706 | ; RC_LOGGING_LEVEL |
|
697 | 707 | ; env variables can control the settings for logging in case of autoconfigure |
|
698 | 708 | |
|
699 | 709 | #logging.autoconfigure = true |
|
700 | 710 | |
|
701 | 711 | ; specify your own custom logging config file to configure logging |
|
702 | 712 | #logging.logging_conf_file = /path/to/custom_logging.ini |
|
703 | 713 | |
|
704 | 714 | ; Dummy marker to add new entries after. |
|
705 | 715 | ; Add any custom entries below. Please don't remove this marker. |
|
706 | 716 | custom.conf = 1 |
|
707 | 717 | |
|
708 | 718 | |
|
709 | 719 | ; ##################### |
|
710 | 720 | ; LOGGING CONFIGURATION |
|
711 | 721 | ; ##################### |
|
712 | 722 | |
|
713 | 723 | [loggers] |
|
714 | 724 | keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper |
|
715 | 725 | |
|
716 | 726 | [handlers] |
|
717 | 727 | keys = console, console_sql |
|
718 | 728 | |
|
719 | 729 | [formatters] |
|
720 | 730 | keys = generic, json, color_formatter, color_formatter_sql |
|
721 | 731 | |
|
722 | 732 | ; ####### |
|
723 | 733 | ; LOGGERS |
|
724 | 734 | ; ####### |
|
725 | 735 | [logger_root] |
|
726 | 736 | level = NOTSET |
|
727 | 737 | handlers = console |
|
728 | 738 | |
|
729 | 739 | [logger_sqlalchemy] |
|
730 | 740 | level = INFO |
|
731 | 741 | handlers = console_sql |
|
732 | 742 | qualname = sqlalchemy.engine |
|
733 | 743 | propagate = 0 |
|
734 | 744 | |
|
735 | 745 | [logger_beaker] |
|
736 | 746 | level = DEBUG |
|
737 | 747 | handlers = |
|
738 | 748 | qualname = beaker.container |
|
739 | 749 | propagate = 1 |
|
740 | 750 | |
|
741 | 751 | [logger_rhodecode] |
|
742 | 752 | level = DEBUG |
|
743 | 753 | handlers = |
|
744 | 754 | qualname = rhodecode |
|
745 | 755 | propagate = 1 |
|
746 | 756 | |
|
747 | 757 | [logger_ssh_wrapper] |
|
748 | 758 | level = DEBUG |
|
749 | 759 | handlers = |
|
750 | 760 | qualname = ssh_wrapper |
|
751 | 761 | propagate = 1 |
|
752 | 762 | |
|
753 | 763 | [logger_celery] |
|
754 | 764 | level = DEBUG |
|
755 | 765 | handlers = |
|
756 | 766 | qualname = celery |
|
757 | 767 | |
|
758 | 768 | |
|
759 | 769 | ; ######## |
|
760 | 770 | ; HANDLERS |
|
761 | 771 | ; ######## |
|
762 | 772 | |
|
763 | 773 | [handler_console] |
|
764 | 774 | class = StreamHandler |
|
765 | 775 | args = (sys.stderr, ) |
|
766 | 776 | level = DEBUG |
|
767 | 777 | ; To enable JSON formatted logs replace 'generic/color_formatter' with 'json' |
|
768 | 778 | ; This allows sending properly formatted logs to grafana loki or elasticsearch |
|
769 | 779 | formatter = color_formatter |
|
770 | 780 | |
|
771 | 781 | [handler_console_sql] |
|
772 | 782 | ; "level = DEBUG" logs SQL queries and results. |
|
773 | 783 | ; "level = INFO" logs SQL queries. |
|
774 | 784 | ; "level = WARN" logs neither. (Recommended for production systems.) |
|
775 | 785 | class = StreamHandler |
|
776 | 786 | args = (sys.stderr, ) |
|
777 | 787 | level = WARN |
|
778 | 788 | ; To enable JSON formatted logs replace 'generic/color_formatter_sql' with 'json' |
|
779 | 789 | ; This allows sending properly formatted logs to grafana loki or elasticsearch |
|
780 | 790 | formatter = color_formatter_sql |
|
781 | 791 | |
|
782 | 792 | ; ########## |
|
783 | 793 | ; FORMATTERS |
|
784 | 794 | ; ########## |
|
785 | 795 | |
|
786 | 796 | [formatter_generic] |
|
787 | 797 | class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter |
|
788 | 798 | format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s |
|
789 | 799 | datefmt = %Y-%m-%d %H:%M:%S |
|
790 | 800 | |
|
791 | 801 | [formatter_color_formatter] |
|
792 | 802 | class = rhodecode.lib.logging_formatter.ColorFormatter |
|
793 | 803 | format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s |
|
794 | 804 | datefmt = %Y-%m-%d %H:%M:%S |
|
795 | 805 | |
|
796 | 806 | [formatter_color_formatter_sql] |
|
797 | 807 | class = rhodecode.lib.logging_formatter.ColorFormatterSql |
|
798 | 808 | format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s |
|
799 | 809 | datefmt = %Y-%m-%d %H:%M:%S |
|
800 | 810 | |
|
801 | 811 | [formatter_json] |
|
802 | 812 | format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s |
|
803 | 813 | class = rhodecode.lib._vendor.jsonlogger.JsonFormatter |
@@ -1,771 +1,781 b'' | |||
|
1 | 1 | |
|
2 | 2 | ; ######################################### |
|
3 | 3 | ; RHODECODE COMMUNITY EDITION CONFIGURATION |
|
4 | 4 | ; ######################################### |
|
5 | 5 | |
|
6 | 6 | [DEFAULT] |
|
7 | 7 | ; Debug flag sets all loggers to debug, and enables request tracking |
|
8 | 8 | debug = false |
|
9 | 9 | |
|
10 | 10 | ; ######################################################################## |
|
11 | 11 | ; EMAIL CONFIGURATION |
|
12 | 12 | ; These settings will be used by the RhodeCode mailing system |
|
13 | 13 | ; ######################################################################## |
|
14 | 14 | |
|
15 | 15 | ; prefix all emails subjects with given prefix, helps filtering out emails |
|
16 | 16 | #email_prefix = [RhodeCode] |
|
17 | 17 | |
|
18 | 18 | ; email FROM address all mails will be sent |
|
19 | 19 | #app_email_from = rhodecode-noreply@localhost |
|
20 | 20 | |
|
21 | 21 | #smtp_server = mail.server.com |
|
22 | 22 | #smtp_username = |
|
23 | 23 | #smtp_password = |
|
24 | 24 | #smtp_port = |
|
25 | 25 | #smtp_use_tls = false |
|
26 | 26 | #smtp_use_ssl = true |
|
27 | 27 | |
|
28 | 28 | [server:main] |
|
29 | 29 | ; COMMON HOST/IP CONFIG, This applies mostly to develop setup, |
|
30 | 30 | ; Host port for gunicorn are controlled by gunicorn_conf.py |
|
31 | 31 | host = 127.0.0.1 |
|
32 | 32 | port = 10020 |
|
33 | 33 | |
|
34 | 34 | |
|
35 | 35 | ; ########################### |
|
36 | 36 | ; GUNICORN APPLICATION SERVER |
|
37 | 37 | ; ########################### |
|
38 | 38 | |
|
39 | 39 | ; run with gunicorn --paste rhodecode.ini --config gunicorn_conf.py |
|
40 | 40 | |
|
41 | 41 | ; Module to use, this setting shouldn't be changed |
|
42 | 42 | use = egg:gunicorn#main |
|
43 | 43 | |
|
44 | 44 | ; Prefix middleware for RhodeCode. |
|
45 | 45 | ; recommended when using proxy setup. |
|
46 | 46 | ; allows to set RhodeCode under a prefix in server. |
|
47 | 47 | ; eg https://server.com/custom_prefix. Enable `filter-with =` option below as well. |
|
48 | 48 | ; And set your prefix like: `prefix = /custom_prefix` |
|
49 | 49 | ; be sure to also set beaker.session.cookie_path = /custom_prefix if you need |
|
50 | 50 | ; to make your cookies only work on prefix url |
|
51 | 51 | [filter:proxy-prefix] |
|
52 | 52 | use = egg:PasteDeploy#prefix |
|
53 | 53 | prefix = / |
|
54 | 54 | |
|
55 | 55 | [app:main] |
|
56 | 56 | ; The %(here)s variable will be replaced with the absolute path of parent directory |
|
57 | 57 | ; of this file |
|
58 | 58 | ; Each option in the app:main can be override by an environmental variable |
|
59 | 59 | ; |
|
60 | 60 | ;To override an option: |
|
61 | 61 | ; |
|
62 | 62 | ;RC_<KeyName> |
|
63 | 63 | ;Everything should be uppercase, . and - should be replaced by _. |
|
64 | 64 | ;For example, if you have these configuration settings: |
|
65 | 65 | ;rc_cache.repo_object.backend = foo |
|
66 | 66 | ;can be overridden by |
|
67 | 67 | ;export RC_CACHE_REPO_OBJECT_BACKEND=foo |
|
68 | 68 | |
|
69 | 69 | use = egg:rhodecode-enterprise-ce |
|
70 | 70 | |
|
71 | 71 | ; enable proxy prefix middleware, defined above |
|
72 | 72 | #filter-with = proxy-prefix |
|
73 | 73 | |
|
74 | 74 | ; encryption key used to encrypt social plugin tokens, |
|
75 | 75 | ; remote_urls with credentials etc, if not set it defaults to |
|
76 | 76 | ; `beaker.session.secret` |
|
77 | 77 | #rhodecode.encrypted_values.secret = |
|
78 | 78 | |
|
79 | 79 | ; decryption strict mode (enabled by default). It controls if decryption raises |
|
80 | 80 | ; `SignatureVerificationError` in case of wrong key, or damaged encryption data. |
|
81 | 81 | #rhodecode.encrypted_values.strict = false |
|
82 | 82 | |
|
83 | 83 | ; Pick algorithm for encryption. Either fernet (more secure) or aes (default) |
|
84 | 84 | ; fernet is safer, and we strongly recommend switching to it. |
|
85 | 85 | ; Due to backward compatibility aes is used as default. |
|
86 | 86 | #rhodecode.encrypted_values.algorithm = fernet |
|
87 | 87 | |
|
88 | 88 | ; Return gzipped responses from RhodeCode (static files/application) |
|
89 | 89 | gzip_responses = false |
|
90 | 90 | |
|
91 | 91 | ; Auto-generate javascript routes file on startup |
|
92 | 92 | generate_js_files = false |
|
93 | 93 | |
|
94 | 94 | ; System global default language. |
|
95 | 95 | ; All available languages: en (default), be, de, es, fr, it, ja, pl, pt, ru, zh |
|
96 | 96 | lang = en |
|
97 | 97 | |
|
98 | 98 | ; Perform a full repository scan and import on each server start. |
|
99 | 99 | ; Settings this to true could lead to very long startup time. |
|
100 | 100 | startup.import_repos = false |
|
101 | 101 | |
|
102 | 102 | ; URL at which the application is running. This is used for Bootstrapping |
|
103 | 103 | ; requests in context when no web request is available. Used in ishell, or |
|
104 | 104 | ; SSH calls. Set this for events to receive proper url for SSH calls. |
|
105 | 105 | app.base_url = http://rhodecode.local |
|
106 | 106 | |
|
107 | 107 | ; Host at which the Service API is running. |
|
108 | 108 | app.service_api.host = http://rhodecode.local:10020 |
|
109 | 109 | |
|
110 | 110 | ; Secret for Service API authentication. |
|
111 | 111 | app.service_api.token = |
|
112 | 112 | |
|
113 | 113 | ; Unique application ID. Should be a random unique string for security. |
|
114 | 114 | app_instance_uuid = rc-production |
|
115 | 115 | |
|
116 | 116 | ; Cut off limit for large diffs (size in bytes). If overall diff size on |
|
117 | 117 | ; commit, or pull request exceeds this limit this diff will be displayed |
|
118 | 118 | ; partially. E.g 512000 == 512Kb |
|
119 | 119 | cut_off_limit_diff = 512000 |
|
120 | 120 | |
|
121 | 121 | ; Cut off limit for large files inside diffs (size in bytes). Each individual |
|
122 | 122 | ; file inside diff which exceeds this limit will be displayed partially. |
|
123 | 123 | ; E.g 128000 == 128Kb |
|
124 | 124 | cut_off_limit_file = 128000 |
|
125 | 125 | |
|
126 | 126 | ; Use cached version of vcs repositories everywhere. Recommended to be `true` |
|
127 | 127 | vcs_full_cache = true |
|
128 | 128 | |
|
129 | 129 | ; Force https in RhodeCode, fixes https redirects, assumes it's always https. |
|
130 | 130 | ; Normally this is controlled by proper flags sent from http server such as Nginx or Apache |
|
131 | 131 | force_https = false |
|
132 | 132 | |
|
133 | 133 | ; use Strict-Transport-Security headers |
|
134 | 134 | use_htsts = false |
|
135 | 135 | |
|
136 | 136 | ; Set to true if your repos are exposed using the dumb protocol |
|
137 | 137 | git_update_server_info = false |
|
138 | 138 | |
|
139 | 139 | ; RSS/ATOM feed options |
|
140 | 140 | rss_cut_off_limit = 256000 |
|
141 | 141 | rss_items_per_page = 10 |
|
142 | 142 | rss_include_diff = false |
|
143 | 143 | |
|
144 | 144 | ; gist URL alias, used to create nicer urls for gist. This should be an |
|
145 | 145 | ; url that does rewrites to _admin/gists/{gistid}. |
|
146 | 146 | ; example: http://gist.rhodecode.org/{gistid}. Empty means use the internal |
|
147 | 147 | ; RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid} |
|
148 | 148 | gist_alias_url = |
|
149 | 149 | |
|
150 | 150 | ; List of views (using glob pattern syntax) that AUTH TOKENS could be |
|
151 | 151 | ; used for access. |
|
152 | 152 | ; Adding ?auth_token=TOKEN_HASH to the url authenticates this request as if it |
|
153 | 153 | ; came from the the logged in user who own this authentication token. |
|
154 | 154 | ; Additionally @TOKEN syntax can be used to bound the view to specific |
|
155 | 155 | ; authentication token. Such view would be only accessible when used together |
|
156 | 156 | ; with this authentication token |
|
157 | 157 | ; list of all views can be found under `/_admin/permissions/auth_token_access` |
|
158 | 158 | ; The list should be "," separated and on a single line. |
|
159 | 159 | ; Most common views to enable: |
|
160 | 160 | |
|
161 | 161 | # RepoCommitsView:repo_commit_download |
|
162 | 162 | # RepoCommitsView:repo_commit_patch |
|
163 | 163 | # RepoCommitsView:repo_commit_raw |
|
164 | 164 | # RepoCommitsView:repo_commit_raw@TOKEN |
|
165 | 165 | # RepoFilesView:repo_files_diff |
|
166 | 166 | # RepoFilesView:repo_archivefile |
|
167 | 167 | # RepoFilesView:repo_file_raw |
|
168 | 168 | # GistView:* |
|
169 | 169 | api_access_controllers_whitelist = |
|
170 | 170 | |
|
171 | 171 | ; Default encoding used to convert from and to unicode |
|
172 | 172 | ; can be also a comma separated list of encoding in case of mixed encodings |
|
173 | 173 | default_encoding = UTF-8 |
|
174 | 174 | |
|
175 | 175 | ; instance-id prefix |
|
176 | 176 | ; a prefix key for this instance used for cache invalidation when running |
|
177 | 177 | ; multiple instances of RhodeCode, make sure it's globally unique for |
|
178 | 178 | ; all running RhodeCode instances. Leave empty if you don't use it |
|
179 | 179 | instance_id = |
|
180 | 180 | |
|
181 | 181 | ; Fallback authentication plugin. Set this to a plugin ID to force the usage |
|
182 | 182 | ; of an authentication plugin also if it is disabled by it's settings. |
|
183 | 183 | ; This could be useful if you are unable to log in to the system due to broken |
|
184 | 184 | ; authentication settings. Then you can enable e.g. the internal RhodeCode auth |
|
185 | 185 | ; module to log in again and fix the settings. |
|
186 | 186 | ; Available builtin plugin IDs (hash is part of the ID): |
|
187 | 187 | ; egg:rhodecode-enterprise-ce#rhodecode |
|
188 | 188 | ; egg:rhodecode-enterprise-ce#pam |
|
189 | 189 | ; egg:rhodecode-enterprise-ce#ldap |
|
190 | 190 | ; egg:rhodecode-enterprise-ce#jasig_cas |
|
191 | 191 | ; egg:rhodecode-enterprise-ce#headers |
|
192 | 192 | ; egg:rhodecode-enterprise-ce#crowd |
|
193 | 193 | |
|
194 | 194 | #rhodecode.auth_plugin_fallback = egg:rhodecode-enterprise-ce#rhodecode |
|
195 | 195 | |
|
196 | 196 | ; Flag to control loading of legacy plugins in py:/path format |
|
197 | 197 | auth_plugin.import_legacy_plugins = true |
|
198 | 198 | |
|
199 | 199 | ; alternative return HTTP header for failed authentication. Default HTTP |
|
200 | 200 | ; response is 401 HTTPUnauthorized. Currently HG clients have troubles with |
|
201 | 201 | ; handling that causing a series of failed authentication calls. |
|
202 | 202 | ; Set this variable to 403 to return HTTPForbidden, or any other HTTP code |
|
203 | 203 | ; This will be served instead of default 401 on bad authentication |
|
204 | 204 | auth_ret_code = |
|
205 | 205 | |
|
206 | 206 | ; use special detection method when serving auth_ret_code, instead of serving |
|
207 | 207 | ; ret_code directly, use 401 initially (Which triggers credentials prompt) |
|
208 | 208 | ; and then serve auth_ret_code to clients |
|
209 | 209 | auth_ret_code_detection = false |
|
210 | 210 | |
|
211 | 211 | ; locking return code. When repository is locked return this HTTP code. 2XX |
|
212 | 212 | ; codes don't break the transactions while 4XX codes do |
|
213 | 213 | lock_ret_code = 423 |
|
214 | 214 | |
|
215 | 215 | ; Filesystem location were repositories should be stored |
|
216 | 216 | repo_store.path = /var/opt/rhodecode_repo_store |
|
217 | 217 | |
|
218 | 218 | ; allows to setup custom hooks in settings page |
|
219 | 219 | allow_custom_hooks_settings = true |
|
220 | 220 | |
|
221 | 221 | ; Generated license token required for EE edition license. |
|
222 | 222 | ; New generated token value can be found in Admin > settings > license page. |
|
223 | 223 | license_token = |
|
224 | 224 | |
|
225 | 225 | ; This flag hides sensitive information on the license page such as token, and license data |
|
226 | 226 | license.hide_license_info = false |
|
227 | 227 | |
|
228 | 228 | ; supervisor connection uri, for managing supervisor and logs. |
|
229 | 229 | supervisor.uri = |
|
230 | 230 | |
|
231 | 231 | ; supervisord group name/id we only want this RC instance to handle |
|
232 | 232 | supervisor.group_id = prod |
|
233 | 233 | |
|
234 | 234 | ; Display extended labs settings |
|
235 | 235 | labs_settings_active = true |
|
236 | 236 | |
|
237 | 237 | ; Custom exception store path, defaults to TMPDIR |
|
238 | 238 | ; This is used to store exception from RhodeCode in shared directory |
|
239 | 239 | #exception_tracker.store_path = |
|
240 | 240 | |
|
241 | 241 | ; Send email with exception details when it happens |
|
242 | 242 | #exception_tracker.send_email = false |
|
243 | 243 | |
|
244 | 244 | ; Comma separated list of recipients for exception emails, |
|
245 | 245 | ; e.g admin@rhodecode.com,devops@rhodecode.com |
|
246 | 246 | ; Can be left empty, then emails will be sent to ALL super-admins |
|
247 | 247 | #exception_tracker.send_email_recipients = |
|
248 | 248 | |
|
249 | 249 | ; optional prefix to Add to email Subject |
|
250 | 250 | #exception_tracker.email_prefix = [RHODECODE ERROR] |
|
251 | 251 | |
|
252 | 252 | ; File store configuration. This is used to store and serve uploaded files |
|
253 | 253 | file_store.enabled = true |
|
254 | 254 | |
|
255 | 255 | ; Storage backend, available options are: local |
|
256 | 256 | file_store.backend = local |
|
257 | 257 | |
|
258 | 258 | ; path to store the uploaded binaries and artifacts |
|
259 | 259 | file_store.storage_path = /var/opt/rhodecode_data/file_store |
|
260 | 260 | |
|
261 | 261 | ; Uncomment and set this path to control settings for archive download cache. |
|
262 | 262 | ; Generated repo archives will be cached at this location |
|
263 | 263 | ; and served from the cache during subsequent requests for the same archive of |
|
264 | 264 | ; the repository. This path is important to be shared across filesystems and with |
|
265 | 265 | ; RhodeCode and vcsserver |
|
266 | 266 | |
|
267 | ; Redis url to acquire/check generation of archives locks | |
|
268 | archive_cache.locking.url = redis://redis:6379/1 | |
|
269 | ||
|
270 | ; Storage backend, only 'filesystem' is available now | |
|
271 | archive_cache.backend.type = filesystem | |
|
272 | ||
|
267 | 273 | ; Default is $cache_dir/archive_cache if not set |
|
268 |
archive_cache.store_dir = /var/opt/rhodecode_data/ |
|
|
274 | archive_cache.filesystem.store_dir = /var/opt/rhodecode_data/archive_cache | |
|
269 | 275 | |
|
270 | 276 | ; The limit in GB sets how much data we cache before recycling last used, defaults to 10 gb |
|
271 | archive_cache.cache_size_gb = 40 | |
|
277 | archive_cache.filesystem.cache_size_gb = 40 | |
|
278 | ||
|
279 | ; Eviction policy used to clear out after cache_size_gb limit is reached | |
|
280 | archive_cache.filesystem.eviction_policy = least-recently-stored | |
|
272 | 281 | |
|
273 | 282 | ; By default cache uses sharding technique, this specifies how many shards are there |
|
274 |
archive_cache.cache_shards = |
|
|
283 | archive_cache.filesystem.cache_shards = 8 | |
|
284 | ||
|
275 | 285 | |
|
276 | 286 | ; ############# |
|
277 | 287 | ; CELERY CONFIG |
|
278 | 288 | ; ############# |
|
279 | 289 | |
|
280 | 290 | ; manually run celery: /path/to/celery worker --task-events --beat --app rhodecode.lib.celerylib.loader --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler --loglevel DEBUG --ini /path/to/rhodecode.ini |
|
281 | 291 | |
|
282 | 292 | use_celery = true |
|
283 | 293 | |
|
284 | 294 | ; path to store schedule database |
|
285 | 295 | #celerybeat-schedule.path = |
|
286 | 296 | |
|
287 | 297 | ; connection url to the message broker (default redis) |
|
288 | 298 | celery.broker_url = redis://redis:6379/8 |
|
289 | 299 | |
|
290 | 300 | ; results backend to get results for (default redis) |
|
291 | 301 | celery.result_backend = redis://redis:6379/8 |
|
292 | 302 | |
|
293 | 303 | ; rabbitmq example |
|
294 | 304 | #celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost |
|
295 | 305 | |
|
296 | 306 | ; maximum tasks to execute before worker restart |
|
297 | 307 | celery.max_tasks_per_child = 20 |
|
298 | 308 | |
|
299 | 309 | ; tasks will never be sent to the queue, but executed locally instead. |
|
300 | 310 | celery.task_always_eager = false |
|
301 | 311 | |
|
302 | 312 | ; ############# |
|
303 | 313 | ; DOGPILE CACHE |
|
304 | 314 | ; ############# |
|
305 | 315 | |
|
306 | 316 | ; Default cache dir for caches. Putting this into a ramdisk can boost performance. |
|
307 | 317 | ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space |
|
308 | 318 | cache_dir = /var/opt/rhodecode_data |
|
309 | 319 | |
|
310 | 320 | ; ********************************************* |
|
311 | 321 | ; `sql_cache_short` cache for heavy SQL queries |
|
312 | 322 | ; Only supported backend is `memory_lru` |
|
313 | 323 | ; ********************************************* |
|
314 | 324 | rc_cache.sql_cache_short.backend = dogpile.cache.rc.memory_lru |
|
315 | 325 | rc_cache.sql_cache_short.expiration_time = 30 |
|
316 | 326 | |
|
317 | 327 | |
|
318 | 328 | ; ***************************************************** |
|
319 | 329 | ; `cache_repo_longterm` cache for repo object instances |
|
320 | 330 | ; Only supported backend is `memory_lru` |
|
321 | 331 | ; ***************************************************** |
|
322 | 332 | rc_cache.cache_repo_longterm.backend = dogpile.cache.rc.memory_lru |
|
323 | 333 | ; by default we use 30 Days, cache is still invalidated on push |
|
324 | 334 | rc_cache.cache_repo_longterm.expiration_time = 2592000 |
|
325 | 335 | ; max items in LRU cache, set to smaller number to save memory, and expire last used caches |
|
326 | 336 | rc_cache.cache_repo_longterm.max_size = 10000 |
|
327 | 337 | |
|
328 | 338 | |
|
329 | 339 | ; ********************************************* |
|
330 | 340 | ; `cache_general` cache for general purpose use |
|
331 | 341 | ; for simplicity use rc.file_namespace backend, |
|
332 | 342 | ; for performance and scale use rc.redis |
|
333 | 343 | ; ********************************************* |
|
334 | 344 | rc_cache.cache_general.backend = dogpile.cache.rc.file_namespace |
|
335 | 345 | rc_cache.cache_general.expiration_time = 43200 |
|
336 | 346 | ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set |
|
337 | 347 | #rc_cache.cache_general.arguments.filename = /tmp/cache_general_db |
|
338 | 348 | |
|
339 | 349 | ; alternative `cache_general` redis backend with distributed lock |
|
340 | 350 | #rc_cache.cache_general.backend = dogpile.cache.rc.redis |
|
341 | 351 | #rc_cache.cache_general.expiration_time = 300 |
|
342 | 352 | |
|
343 | 353 | ; redis_expiration_time needs to be greater then expiration_time |
|
344 | 354 | #rc_cache.cache_general.arguments.redis_expiration_time = 7200 |
|
345 | 355 | |
|
346 | 356 | #rc_cache.cache_general.arguments.host = localhost |
|
347 | 357 | #rc_cache.cache_general.arguments.port = 6379 |
|
348 | 358 | #rc_cache.cache_general.arguments.db = 0 |
|
349 | 359 | #rc_cache.cache_general.arguments.socket_timeout = 30 |
|
350 | 360 | ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends |
|
351 | 361 | #rc_cache.cache_general.arguments.distributed_lock = true |
|
352 | 362 | |
|
353 | 363 | ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen |
|
354 | 364 | #rc_cache.cache_general.arguments.lock_auto_renewal = true |
|
355 | 365 | |
|
356 | 366 | ; ************************************************* |
|
357 | 367 | ; `cache_perms` cache for permission tree, auth TTL |
|
358 | 368 | ; for simplicity use rc.file_namespace backend, |
|
359 | 369 | ; for performance and scale use rc.redis |
|
360 | 370 | ; ************************************************* |
|
361 | 371 | rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace |
|
362 | 372 | rc_cache.cache_perms.expiration_time = 3600 |
|
363 | 373 | ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set |
|
364 | 374 | #rc_cache.cache_perms.arguments.filename = /tmp/cache_perms_db |
|
365 | 375 | |
|
366 | 376 | ; alternative `cache_perms` redis backend with distributed lock |
|
367 | 377 | #rc_cache.cache_perms.backend = dogpile.cache.rc.redis |
|
368 | 378 | #rc_cache.cache_perms.expiration_time = 300 |
|
369 | 379 | |
|
370 | 380 | ; redis_expiration_time needs to be greater then expiration_time |
|
371 | 381 | #rc_cache.cache_perms.arguments.redis_expiration_time = 7200 |
|
372 | 382 | |
|
373 | 383 | #rc_cache.cache_perms.arguments.host = localhost |
|
374 | 384 | #rc_cache.cache_perms.arguments.port = 6379 |
|
375 | 385 | #rc_cache.cache_perms.arguments.db = 0 |
|
376 | 386 | #rc_cache.cache_perms.arguments.socket_timeout = 30 |
|
377 | 387 | ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends |
|
378 | 388 | #rc_cache.cache_perms.arguments.distributed_lock = true |
|
379 | 389 | |
|
380 | 390 | ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen |
|
381 | 391 | #rc_cache.cache_perms.arguments.lock_auto_renewal = true |
|
382 | 392 | |
|
383 | 393 | ; *************************************************** |
|
384 | 394 | ; `cache_repo` cache for file tree, Readme, RSS FEEDS |
|
385 | 395 | ; for simplicity use rc.file_namespace backend, |
|
386 | 396 | ; for performance and scale use rc.redis |
|
387 | 397 | ; *************************************************** |
|
388 | 398 | rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace |
|
389 | 399 | rc_cache.cache_repo.expiration_time = 2592000 |
|
390 | 400 | ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set |
|
391 | 401 | #rc_cache.cache_repo.arguments.filename = /tmp/cache_repo_db |
|
392 | 402 | |
|
393 | 403 | ; alternative `cache_repo` redis backend with distributed lock |
|
394 | 404 | #rc_cache.cache_repo.backend = dogpile.cache.rc.redis |
|
395 | 405 | #rc_cache.cache_repo.expiration_time = 2592000 |
|
396 | 406 | |
|
397 | 407 | ; redis_expiration_time needs to be greater then expiration_time |
|
398 | 408 | #rc_cache.cache_repo.arguments.redis_expiration_time = 2678400 |
|
399 | 409 | |
|
400 | 410 | #rc_cache.cache_repo.arguments.host = localhost |
|
401 | 411 | #rc_cache.cache_repo.arguments.port = 6379 |
|
402 | 412 | #rc_cache.cache_repo.arguments.db = 1 |
|
403 | 413 | #rc_cache.cache_repo.arguments.socket_timeout = 30 |
|
404 | 414 | ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends |
|
405 | 415 | #rc_cache.cache_repo.arguments.distributed_lock = true |
|
406 | 416 | |
|
407 | 417 | ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen |
|
408 | 418 | #rc_cache.cache_repo.arguments.lock_auto_renewal = true |
|
409 | 419 | |
|
410 | 420 | ; ############## |
|
411 | 421 | ; BEAKER SESSION |
|
412 | 422 | ; ############## |
|
413 | 423 | |
|
414 | 424 | ; beaker.session.type is type of storage options for the logged users sessions. Current allowed |
|
415 | 425 | ; types are file, ext:redis, ext:database, ext:memcached |
|
416 | 426 | ; Fastest ones are ext:redis and ext:database, DO NOT use memory type for session |
|
417 | 427 | #beaker.session.type = file |
|
418 | 428 | #beaker.session.data_dir = %(here)s/data/sessions |
|
419 | 429 | |
|
420 | 430 | ; Redis based sessions |
|
421 | 431 | beaker.session.type = ext:redis |
|
422 | 432 | beaker.session.url = redis://redis:6379/2 |
|
423 | 433 | |
|
424 | 434 | ; DB based session, fast, and allows easy management over logged in users |
|
425 | 435 | #beaker.session.type = ext:database |
|
426 | 436 | #beaker.session.table_name = db_session |
|
427 | 437 | #beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode |
|
428 | 438 | #beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode |
|
429 | 439 | #beaker.session.sa.pool_recycle = 3600 |
|
430 | 440 | #beaker.session.sa.echo = false |
|
431 | 441 | |
|
432 | 442 | beaker.session.key = rhodecode |
|
433 | 443 | beaker.session.secret = production-rc-uytcxaz |
|
434 | 444 | beaker.session.lock_dir = /data_ramdisk/lock |
|
435 | 445 | |
|
436 | 446 | ; Secure encrypted cookie. Requires AES and AES python libraries |
|
437 | 447 | ; you must disable beaker.session.secret to use this |
|
438 | 448 | #beaker.session.encrypt_key = key_for_encryption |
|
439 | 449 | #beaker.session.validate_key = validation_key |
|
440 | 450 | |
|
441 | 451 | ; Sets session as invalid (also logging out user) if it haven not been |
|
442 | 452 | ; accessed for given amount of time in seconds |
|
443 | 453 | beaker.session.timeout = 2592000 |
|
444 | 454 | beaker.session.httponly = true |
|
445 | 455 | |
|
446 | 456 | ; Path to use for the cookie. Set to prefix if you use prefix middleware |
|
447 | 457 | #beaker.session.cookie_path = /custom_prefix |
|
448 | 458 | |
|
449 | 459 | ; Set https secure cookie |
|
450 | 460 | beaker.session.secure = false |
|
451 | 461 | |
|
452 | 462 | ; default cookie expiration time in seconds, set to `true` to set expire |
|
453 | 463 | ; at browser close |
|
454 | 464 | #beaker.session.cookie_expires = 3600 |
|
455 | 465 | |
|
456 | 466 | ; ############################# |
|
457 | 467 | ; SEARCH INDEXING CONFIGURATION |
|
458 | 468 | ; ############################# |
|
459 | 469 | |
|
460 | 470 | ; Full text search indexer is available in rhodecode-tools under |
|
461 | 471 | ; `rhodecode-tools index` command |
|
462 | 472 | |
|
463 | 473 | ; WHOOSH Backend, doesn't require additional services to run |
|
464 | 474 | ; it works good with few dozen repos |
|
465 | 475 | search.module = rhodecode.lib.index.whoosh |
|
466 | 476 | search.location = %(here)s/data/index |
|
467 | 477 | |
|
468 | 478 | ; #################### |
|
469 | 479 | ; CHANNELSTREAM CONFIG |
|
470 | 480 | ; #################### |
|
471 | 481 | |
|
472 | 482 | ; channelstream enables persistent connections and live notification |
|
473 | 483 | ; in the system. It's also used by the chat system |
|
474 | 484 | |
|
475 | 485 | channelstream.enabled = true |
|
476 | 486 | |
|
477 | 487 | ; server address for channelstream server on the backend |
|
478 | 488 | channelstream.server = channelstream:9800 |
|
479 | 489 | |
|
480 | 490 | ; location of the channelstream server from outside world |
|
481 | 491 | ; use ws:// for http or wss:// for https. This address needs to be handled |
|
482 | 492 | ; by external HTTP server such as Nginx or Apache |
|
483 | 493 | ; see Nginx/Apache configuration examples in our docs |
|
484 | 494 | channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream |
|
485 | 495 | channelstream.secret = ENV_GENERATED |
|
486 | 496 | channelstream.history.location = /var/opt/rhodecode_data/channelstream_history |
|
487 | 497 | |
|
488 | 498 | ; Internal application path that Javascript uses to connect into. |
|
489 | 499 | ; If you use proxy-prefix the prefix should be added before /_channelstream |
|
490 | 500 | channelstream.proxy_path = /_channelstream |
|
491 | 501 | |
|
492 | 502 | |
|
493 | 503 | ; ############################## |
|
494 | 504 | ; MAIN RHODECODE DATABASE CONFIG |
|
495 | 505 | ; ############################## |
|
496 | 506 | |
|
497 | 507 | #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30 |
|
498 | 508 | #sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode |
|
499 | 509 | #sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode?charset=utf8 |
|
500 | 510 | ; pymysql is an alternative driver for MySQL, use in case of problems with default one |
|
501 | 511 | #sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode |
|
502 | 512 | |
|
503 | 513 | sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode |
|
504 | 514 | |
|
505 | 515 | ; see sqlalchemy docs for other advanced settings |
|
506 | 516 | ; print the sql statements to output |
|
507 | 517 | sqlalchemy.db1.echo = false |
|
508 | 518 | |
|
509 | 519 | ; recycle the connections after this amount of seconds |
|
510 | 520 | sqlalchemy.db1.pool_recycle = 3600 |
|
511 | 521 | |
|
512 | 522 | ; the number of connections to keep open inside the connection pool. |
|
513 | 523 | ; 0 indicates no limit |
|
514 | 524 | ; the general calculus with gevent is: |
|
515 | 525 | ; if your system allows 500 concurrent greenlets (max_connections) that all do database access, |
|
516 | 526 | ; then increase pool size + max overflow so that they add up to 500. |
|
517 | 527 | #sqlalchemy.db1.pool_size = 5 |
|
518 | 528 | |
|
519 | 529 | ; The number of connections to allow in connection pool "overflow", that is |
|
520 | 530 | ; connections that can be opened above and beyond the pool_size setting, |
|
521 | 531 | ; which defaults to five. |
|
522 | 532 | #sqlalchemy.db1.max_overflow = 10 |
|
523 | 533 | |
|
524 | 534 | ; Connection check ping, used to detect broken database connections |
|
525 | 535 | ; could be enabled to better handle cases if MySQL has gone away errors |
|
526 | 536 | #sqlalchemy.db1.ping_connection = true |
|
527 | 537 | |
|
528 | 538 | ; ########## |
|
529 | 539 | ; VCS CONFIG |
|
530 | 540 | ; ########## |
|
531 | 541 | vcs.server.enable = true |
|
532 | 542 | vcs.server = vcsserver:10010 |
|
533 | 543 | |
|
534 | 544 | ; Web server connectivity protocol, responsible for web based VCS operations |
|
535 | 545 | ; Available protocols are: |
|
536 | 546 | ; `http` - use http-rpc backend (default) |
|
537 | 547 | vcs.server.protocol = http |
|
538 | 548 | |
|
539 | 549 | ; Push/Pull operations protocol, available options are: |
|
540 | 550 | ; `http` - use http-rpc backend (default) |
|
541 | 551 | vcs.scm_app_implementation = http |
|
542 | 552 | |
|
543 | 553 | ; Push/Pull operations hooks protocol, available options are: |
|
544 | 554 | ; `http` - use http-rpc backend (default) |
|
545 | 555 | ; `celery` - use celery based hooks |
|
546 | 556 | vcs.hooks.protocol = http |
|
547 | 557 | |
|
548 | 558 | ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be |
|
549 | 559 | ; accessible via network. |
|
550 | 560 | ; Use vcs.hooks.host = "*" to bind to current hostname (for Docker) |
|
551 | 561 | vcs.hooks.host = * |
|
552 | 562 | |
|
553 | 563 | ; Start VCSServer with this instance as a subprocess, useful for development |
|
554 | 564 | vcs.start_server = false |
|
555 | 565 | |
|
556 | 566 | ; List of enabled VCS backends, available options are: |
|
557 | 567 | ; `hg` - mercurial |
|
558 | 568 | ; `git` - git |
|
559 | 569 | ; `svn` - subversion |
|
560 | 570 | vcs.backends = hg, git, svn |
|
561 | 571 | |
|
562 | 572 | ; Wait this number of seconds before killing connection to the vcsserver |
|
563 | 573 | vcs.connection_timeout = 3600 |
|
564 | 574 | |
|
565 | 575 | ; Cache flag to cache vcsserver remote calls locally |
|
566 | 576 | ; It uses cache_region `cache_repo` |
|
567 | 577 | vcs.methods.cache = true |
|
568 | 578 | |
|
569 | 579 | ; #################################################### |
|
570 | 580 | ; Subversion proxy support (mod_dav_svn) |
|
571 | 581 | ; Maps RhodeCode repo groups into SVN paths for Apache |
|
572 | 582 | ; #################################################### |
|
573 | 583 | |
|
574 | 584 | ; Compatibility version when creating SVN repositories. Defaults to newest version when commented out. |
|
575 | 585 | ; Set a numeric version for your current SVN e.g 1.8, or 1.12 |
|
576 | 586 | ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible |
|
577 | 587 | #vcs.svn.compatible_version = 1.8 |
|
578 | 588 | |
|
579 | 589 | ; Enable SVN proxy of requests over HTTP |
|
580 | 590 | vcs.svn.proxy.enabled = true |
|
581 | 591 | |
|
582 | 592 | ; host to connect to running SVN subsystem |
|
583 | 593 | vcs.svn.proxy.host = http://svn:8090 |
|
584 | 594 | |
|
585 | 595 | ; Enable or disable the config file generation. |
|
586 | 596 | svn.proxy.generate_config = true |
|
587 | 597 | |
|
588 | 598 | ; Generate config file with `SVNListParentPath` set to `On`. |
|
589 | 599 | svn.proxy.list_parent_path = true |
|
590 | 600 | |
|
591 | 601 | ; Set location and file name of generated config file. |
|
592 | 602 | svn.proxy.config_file_path = /etc/rhodecode/conf/svn/mod_dav_svn.conf |
|
593 | 603 | |
|
594 | 604 | ; alternative mod_dav config template. This needs to be a valid mako template |
|
595 | 605 | ; Example template can be found in the source code: |
|
596 | 606 | ; rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako |
|
597 | 607 | #svn.proxy.config_template = ~/.rccontrol/enterprise-1/custom_svn_conf.mako |
|
598 | 608 | |
|
599 | 609 | ; Used as a prefix to the `Location` block in the generated config file. |
|
600 | 610 | ; In most cases it should be set to `/`. |
|
601 | 611 | svn.proxy.location_root = / |
|
602 | 612 | |
|
603 | 613 | ; Command to reload the mod dav svn configuration on change. |
|
604 | 614 | ; Example: `/etc/init.d/apache2 reload` or /home/USER/apache_reload.sh |
|
605 | 615 | ; Make sure user who runs RhodeCode process is allowed to reload Apache |
|
606 | 616 | #svn.proxy.reload_cmd = /etc/init.d/apache2 reload |
|
607 | 617 | |
|
608 | 618 | ; If the timeout expires before the reload command finishes, the command will |
|
609 | 619 | ; be killed. Setting it to zero means no timeout. Defaults to 10 seconds. |
|
610 | 620 | #svn.proxy.reload_timeout = 10 |
|
611 | 621 | |
|
612 | 622 | ; #################### |
|
613 | 623 | ; SSH Support Settings |
|
614 | 624 | ; #################### |
|
615 | 625 | |
|
616 | 626 | ; Defines if a custom authorized_keys file should be created and written on |
|
617 | 627 | ; any change user ssh keys. Setting this to false also disables possibility |
|
618 | 628 | ; of adding SSH keys by users from web interface. Super admins can still |
|
619 | 629 | ; manage SSH Keys. |
|
620 | 630 | ssh.generate_authorized_keyfile = true |
|
621 | 631 | |
|
622 | 632 | ; Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding` |
|
623 | 633 | # ssh.authorized_keys_ssh_opts = |
|
624 | 634 | |
|
625 | 635 | ; Path to the authorized_keys file where the generate entries are placed. |
|
626 | 636 | ; It is possible to have multiple key files specified in `sshd_config` e.g. |
|
627 | 637 | ; AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode |
|
628 | 638 | ssh.authorized_keys_file_path = /etc/rhodecode/conf/ssh/authorized_keys_rhodecode |
|
629 | 639 | |
|
630 | 640 | ; Command to execute the SSH wrapper. The binary is available in the |
|
631 | 641 | ; RhodeCode installation directory. |
|
632 | 642 | ; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper |
|
633 | 643 | ; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2 |
|
634 | 644 | ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper |
|
635 | 645 | |
|
636 | 646 | ; Allow shell when executing the ssh-wrapper command |
|
637 | 647 | ssh.wrapper_cmd_allow_shell = false |
|
638 | 648 | |
|
639 | 649 | ; Enables logging, and detailed output send back to the client during SSH |
|
640 | 650 | ; operations. Useful for debugging, shouldn't be used in production. |
|
641 | 651 | ssh.enable_debug_logging = false |
|
642 | 652 | |
|
643 | 653 | ; Paths to binary executable, by default they are the names, but we can |
|
644 | 654 | ; override them if we want to use a custom one |
|
645 | 655 | ssh.executable.hg = /usr/local/bin/rhodecode_bin/vcs_bin/hg |
|
646 | 656 | ssh.executable.git = /usr/local/bin/rhodecode_bin/vcs_bin/git |
|
647 | 657 | ssh.executable.svn = /usr/local/bin/rhodecode_bin/vcs_bin/svnserve |
|
648 | 658 | |
|
649 | 659 | ; Enables SSH key generator web interface. Disabling this still allows users |
|
650 | 660 | ; to add their own keys. |
|
651 | 661 | ssh.enable_ui_key_generator = true |
|
652 | 662 | |
|
653 | 663 | ; Statsd client config, this is used to send metrics to statsd |
|
654 | 664 | ; We recommend setting statsd_exported and scrape them using Prometheus |
|
655 | 665 | #statsd.enabled = false |
|
656 | 666 | #statsd.statsd_host = 0.0.0.0 |
|
657 | 667 | #statsd.statsd_port = 8125 |
|
658 | 668 | #statsd.statsd_prefix = |
|
659 | 669 | #statsd.statsd_ipv6 = false |
|
660 | 670 | |
|
661 | 671 | ; configure logging automatically at server startup set to false |
|
662 | 672 | ; to use the below custom logging config. |
|
663 | 673 | ; RC_LOGGING_FORMATTER |
|
664 | 674 | ; RC_LOGGING_LEVEL |
|
665 | 675 | ; env variables can control the settings for logging in case of autoconfigure |
|
666 | 676 | |
|
667 | 677 | #logging.autoconfigure = true |
|
668 | 678 | |
|
669 | 679 | ; specify your own custom logging config file to configure logging |
|
670 | 680 | #logging.logging_conf_file = /path/to/custom_logging.ini |
|
671 | 681 | |
|
672 | 682 | ; Dummy marker to add new entries after. |
|
673 | 683 | ; Add any custom entries below. Please don't remove this marker. |
|
674 | 684 | custom.conf = 1 |
|
675 | 685 | |
|
676 | 686 | |
|
677 | 687 | ; ##################### |
|
678 | 688 | ; LOGGING CONFIGURATION |
|
679 | 689 | ; ##################### |
|
680 | 690 | |
|
681 | 691 | [loggers] |
|
682 | 692 | keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper |
|
683 | 693 | |
|
684 | 694 | [handlers] |
|
685 | 695 | keys = console, console_sql |
|
686 | 696 | |
|
687 | 697 | [formatters] |
|
688 | 698 | keys = generic, json, color_formatter, color_formatter_sql |
|
689 | 699 | |
|
690 | 700 | ; ####### |
|
691 | 701 | ; LOGGERS |
|
692 | 702 | ; ####### |
|
693 | 703 | [logger_root] |
|
694 | 704 | level = NOTSET |
|
695 | 705 | handlers = console |
|
696 | 706 | |
|
697 | 707 | [logger_sqlalchemy] |
|
698 | 708 | level = INFO |
|
699 | 709 | handlers = console_sql |
|
700 | 710 | qualname = sqlalchemy.engine |
|
701 | 711 | propagate = 0 |
|
702 | 712 | |
|
703 | 713 | [logger_beaker] |
|
704 | 714 | level = DEBUG |
|
705 | 715 | handlers = |
|
706 | 716 | qualname = beaker.container |
|
707 | 717 | propagate = 1 |
|
708 | 718 | |
|
709 | 719 | [logger_rhodecode] |
|
710 | 720 | level = DEBUG |
|
711 | 721 | handlers = |
|
712 | 722 | qualname = rhodecode |
|
713 | 723 | propagate = 1 |
|
714 | 724 | |
|
715 | 725 | [logger_ssh_wrapper] |
|
716 | 726 | level = DEBUG |
|
717 | 727 | handlers = |
|
718 | 728 | qualname = ssh_wrapper |
|
719 | 729 | propagate = 1 |
|
720 | 730 | |
|
721 | 731 | [logger_celery] |
|
722 | 732 | level = DEBUG |
|
723 | 733 | handlers = |
|
724 | 734 | qualname = celery |
|
725 | 735 | |
|
726 | 736 | |
|
727 | 737 | ; ######## |
|
728 | 738 | ; HANDLERS |
|
729 | 739 | ; ######## |
|
730 | 740 | |
|
731 | 741 | [handler_console] |
|
732 | 742 | class = StreamHandler |
|
733 | 743 | args = (sys.stderr, ) |
|
734 | 744 | level = INFO |
|
735 | 745 | ; To enable JSON formatted logs replace 'generic/color_formatter' with 'json' |
|
736 | 746 | ; This allows sending properly formatted logs to grafana loki or elasticsearch |
|
737 | 747 | formatter = generic |
|
738 | 748 | |
|
739 | 749 | [handler_console_sql] |
|
740 | 750 | ; "level = DEBUG" logs SQL queries and results. |
|
741 | 751 | ; "level = INFO" logs SQL queries. |
|
742 | 752 | ; "level = WARN" logs neither. (Recommended for production systems.) |
|
743 | 753 | class = StreamHandler |
|
744 | 754 | args = (sys.stderr, ) |
|
745 | 755 | level = WARN |
|
746 | 756 | ; To enable JSON formatted logs replace 'generic/color_formatter_sql' with 'json' |
|
747 | 757 | ; This allows sending properly formatted logs to grafana loki or elasticsearch |
|
748 | 758 | formatter = generic |
|
749 | 759 | |
|
750 | 760 | ; ########## |
|
751 | 761 | ; FORMATTERS |
|
752 | 762 | ; ########## |
|
753 | 763 | |
|
754 | 764 | [formatter_generic] |
|
755 | 765 | class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter |
|
756 | 766 | format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s |
|
757 | 767 | datefmt = %Y-%m-%d %H:%M:%S |
|
758 | 768 | |
|
759 | 769 | [formatter_color_formatter] |
|
760 | 770 | class = rhodecode.lib.logging_formatter.ColorFormatter |
|
761 | 771 | format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s |
|
762 | 772 | datefmt = %Y-%m-%d %H:%M:%S |
|
763 | 773 | |
|
764 | 774 | [formatter_color_formatter_sql] |
|
765 | 775 | class = rhodecode.lib.logging_formatter.ColorFormatterSql |
|
766 | 776 | format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s |
|
767 | 777 | datefmt = %Y-%m-%d %H:%M:%S |
|
768 | 778 | |
|
769 | 779 | [formatter_json] |
|
770 | 780 | format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s |
|
771 | 781 | class = rhodecode.lib._vendor.jsonlogger.JsonFormatter |
@@ -1,287 +1,286 b'' | |||
|
1 | 1 | # deps, generated via pipdeptree --exclude setuptools,wheel,pipdeptree,pip -f | tr '[:upper:]' '[:lower:]' |
|
2 | 2 | |
|
3 | 3 | alembic==1.13.1 |
|
4 | 4 | mako==1.2.4 |
|
5 | 5 | markupsafe==2.1.2 |
|
6 | 6 | sqlalchemy==1.4.52 |
|
7 | 7 | greenlet==3.0.3 |
|
8 | 8 | typing_extensions==4.9.0 |
|
9 | 9 | async-timeout==4.0.3 |
|
10 | 10 | babel==2.12.1 |
|
11 | 11 | beaker==1.12.1 |
|
12 | 12 | celery==5.3.6 |
|
13 | 13 | billiard==4.2.0 |
|
14 | 14 | click==8.1.3 |
|
15 | 15 | click-didyoumean==0.3.0 |
|
16 | 16 | click==8.1.3 |
|
17 | 17 | click-plugins==1.1.1 |
|
18 | 18 | click==8.1.3 |
|
19 | 19 | click-repl==0.2.0 |
|
20 | 20 | click==8.1.3 |
|
21 | 21 | prompt-toolkit==3.0.38 |
|
22 | 22 | wcwidth==0.2.6 |
|
23 | 23 | six==1.16.0 |
|
24 | 24 | kombu==5.3.5 |
|
25 | 25 | amqp==5.2.0 |
|
26 | 26 | vine==5.1.0 |
|
27 | 27 | vine==5.1.0 |
|
28 | 28 | python-dateutil==2.8.2 |
|
29 | 29 | six==1.16.0 |
|
30 | 30 | tzdata==2024.1 |
|
31 | 31 | vine==5.1.0 |
|
32 | 32 | channelstream==0.7.1 |
|
33 | 33 | gevent==24.2.1 |
|
34 | 34 | greenlet==3.0.3 |
|
35 | 35 | zope.event==5.0.0 |
|
36 | 36 | zope.interface==6.3.0 |
|
37 | 37 | itsdangerous==1.1.0 |
|
38 | 38 | marshmallow==2.18.0 |
|
39 | 39 | pyramid==2.0.2 |
|
40 | 40 | hupper==1.12 |
|
41 | 41 | plaster==1.1.2 |
|
42 | 42 | plaster-pastedeploy==1.0.1 |
|
43 | 43 | pastedeploy==3.1.0 |
|
44 | 44 | plaster==1.1.2 |
|
45 | 45 | translationstring==1.4 |
|
46 | 46 | venusian==3.0.0 |
|
47 | 47 | webob==1.8.7 |
|
48 | 48 | zope.deprecation==5.0.0 |
|
49 | 49 | zope.interface==6.3.0 |
|
50 | 50 | pyramid-apispec==0.3.3 |
|
51 | 51 | apispec==1.3.3 |
|
52 | 52 | pyramid-jinja2==2.10 |
|
53 | 53 | jinja2==3.1.2 |
|
54 | 54 | markupsafe==2.1.2 |
|
55 | 55 | markupsafe==2.1.2 |
|
56 | 56 | pyramid==2.0.2 |
|
57 | 57 | hupper==1.12 |
|
58 | 58 | plaster==1.1.2 |
|
59 | 59 | plaster-pastedeploy==1.0.1 |
|
60 | 60 | pastedeploy==3.1.0 |
|
61 | 61 | plaster==1.1.2 |
|
62 | 62 | translationstring==1.4 |
|
63 | 63 | venusian==3.0.0 |
|
64 | 64 | webob==1.8.7 |
|
65 | 65 | zope.deprecation==5.0.0 |
|
66 | 66 | zope.interface==6.3.0 |
|
67 | 67 | zope.deprecation==5.0.0 |
|
68 | 68 | python-dateutil==2.8.2 |
|
69 | 69 | six==1.16.0 |
|
70 | 70 | requests==2.28.2 |
|
71 | 71 | certifi==2022.12.7 |
|
72 | 72 | charset-normalizer==3.1.0 |
|
73 | 73 | idna==3.4 |
|
74 | 74 | urllib3==1.26.14 |
|
75 | 75 | ws4py==0.5.1 |
|
76 | 76 | deform==2.0.15 |
|
77 | 77 | chameleon==3.10.2 |
|
78 | 78 | colander==2.0 |
|
79 | 79 | iso8601==1.1.0 |
|
80 | 80 | translationstring==1.4 |
|
81 | 81 | iso8601==1.1.0 |
|
82 | 82 | peppercorn==0.6 |
|
83 | 83 | translationstring==1.4 |
|
84 | 84 | zope.deprecation==5.0.0 |
|
85 | diskcache==5.6.3 | |
|
86 | 85 | docutils==0.19 |
|
87 | 86 | dogpile.cache==1.3.3 |
|
88 | 87 | decorator==5.1.1 |
|
89 | 88 | stevedore==5.1.0 |
|
90 | 89 | pbr==5.11.1 |
|
91 | 90 | formencode==2.1.0 |
|
92 | 91 | six==1.16.0 |
|
93 | 92 | gunicorn==21.2.0 |
|
94 | 93 | packaging==24.0 |
|
95 | 94 | gevent==24.2.1 |
|
96 | 95 | greenlet==3.0.3 |
|
97 | 96 | zope.event==5.0.0 |
|
98 | 97 | zope.interface==6.3.0 |
|
99 | 98 | ipython==8.14.0 |
|
100 | 99 | backcall==0.2.0 |
|
101 | 100 | decorator==5.1.1 |
|
102 | 101 | jedi==0.19.0 |
|
103 | 102 | parso==0.8.3 |
|
104 | 103 | matplotlib-inline==0.1.6 |
|
105 | 104 | traitlets==5.9.0 |
|
106 | 105 | pexpect==4.8.0 |
|
107 | 106 | ptyprocess==0.7.0 |
|
108 | 107 | pickleshare==0.7.5 |
|
109 | 108 | prompt-toolkit==3.0.38 |
|
110 | 109 | wcwidth==0.2.6 |
|
111 | 110 | pygments==2.15.1 |
|
112 | 111 | stack-data==0.6.2 |
|
113 | 112 | asttokens==2.2.1 |
|
114 | 113 | six==1.16.0 |
|
115 | 114 | executing==1.2.0 |
|
116 | 115 | pure-eval==0.2.2 |
|
117 | 116 | traitlets==5.9.0 |
|
118 | 117 | markdown==3.4.3 |
|
119 | 118 | msgpack==1.0.8 |
|
120 | 119 | mysqlclient==2.1.1 |
|
121 | 120 | nbconvert==7.7.3 |
|
122 | 121 | beautifulsoup4==4.12.3 |
|
123 | 122 | soupsieve==2.5 |
|
124 | 123 | bleach==6.1.0 |
|
125 | 124 | six==1.16.0 |
|
126 | 125 | webencodings==0.5.1 |
|
127 | 126 | defusedxml==0.7.1 |
|
128 | 127 | jinja2==3.1.2 |
|
129 | 128 | markupsafe==2.1.2 |
|
130 | 129 | jupyter_core==5.3.1 |
|
131 | 130 | platformdirs==3.10.0 |
|
132 | 131 | traitlets==5.9.0 |
|
133 | 132 | jupyterlab-pygments==0.2.2 |
|
134 | 133 | markupsafe==2.1.2 |
|
135 | 134 | mistune==2.0.5 |
|
136 | 135 | nbclient==0.8.0 |
|
137 | 136 | jupyter_client==8.3.0 |
|
138 | 137 | jupyter_core==5.3.1 |
|
139 | 138 | platformdirs==3.10.0 |
|
140 | 139 | traitlets==5.9.0 |
|
141 | 140 | python-dateutil==2.8.2 |
|
142 | 141 | six==1.16.0 |
|
143 | 142 | pyzmq==25.0.0 |
|
144 | 143 | tornado==6.2 |
|
145 | 144 | traitlets==5.9.0 |
|
146 | 145 | jupyter_core==5.3.1 |
|
147 | 146 | platformdirs==3.10.0 |
|
148 | 147 | traitlets==5.9.0 |
|
149 | 148 | nbformat==5.9.2 |
|
150 | 149 | fastjsonschema==2.18.0 |
|
151 | 150 | jsonschema==4.18.6 |
|
152 | 151 | attrs==22.2.0 |
|
153 | 152 | pyrsistent==0.19.3 |
|
154 | 153 | jupyter_core==5.3.1 |
|
155 | 154 | platformdirs==3.10.0 |
|
156 | 155 | traitlets==5.9.0 |
|
157 | 156 | traitlets==5.9.0 |
|
158 | 157 | traitlets==5.9.0 |
|
159 | 158 | nbformat==5.9.2 |
|
160 | 159 | fastjsonschema==2.18.0 |
|
161 | 160 | jsonschema==4.18.6 |
|
162 | 161 | attrs==22.2.0 |
|
163 | 162 | pyrsistent==0.19.3 |
|
164 | 163 | jupyter_core==5.3.1 |
|
165 | 164 | platformdirs==3.10.0 |
|
166 | 165 | traitlets==5.9.0 |
|
167 | 166 | traitlets==5.9.0 |
|
168 | 167 | pandocfilters==1.5.0 |
|
169 | 168 | pygments==2.15.1 |
|
170 | 169 | tinycss2==1.2.1 |
|
171 | 170 | webencodings==0.5.1 |
|
172 | 171 | traitlets==5.9.0 |
|
173 | 172 | orjson==3.10.3 |
|
174 | 173 | paste==3.10.1 |
|
175 | 174 | premailer==3.10.0 |
|
176 | 175 | cachetools==5.3.3 |
|
177 | 176 | cssselect==1.2.0 |
|
178 | 177 | cssutils==2.6.0 |
|
179 | 178 | lxml==4.9.3 |
|
180 | 179 | requests==2.28.2 |
|
181 | 180 | certifi==2022.12.7 |
|
182 | 181 | charset-normalizer==3.1.0 |
|
183 | 182 | idna==3.4 |
|
184 | 183 | urllib3==1.26.14 |
|
185 | 184 | psutil==5.9.8 |
|
186 | 185 | psycopg2==2.9.9 |
|
187 | 186 | py-bcrypt==0.4 |
|
188 | 187 | pycmarkgfm==1.2.0 |
|
189 | 188 | cffi==1.16.0 |
|
190 | 189 | pycparser==2.21 |
|
191 | 190 | pycryptodome==3.17 |
|
192 | 191 | pycurl==7.45.3 |
|
193 | 192 | pymysql==1.0.3 |
|
194 | 193 | pyotp==2.8.0 |
|
195 | 194 | pyparsing==3.1.1 |
|
196 | 195 | pyramid-debugtoolbar==4.12.1 |
|
197 | 196 | pygments==2.15.1 |
|
198 | 197 | pyramid==2.0.2 |
|
199 | 198 | hupper==1.12 |
|
200 | 199 | plaster==1.1.2 |
|
201 | 200 | plaster-pastedeploy==1.0.1 |
|
202 | 201 | pastedeploy==3.1.0 |
|
203 | 202 | plaster==1.1.2 |
|
204 | 203 | translationstring==1.4 |
|
205 | 204 | venusian==3.0.0 |
|
206 | 205 | webob==1.8.7 |
|
207 | 206 | zope.deprecation==5.0.0 |
|
208 | 207 | zope.interface==6.3.0 |
|
209 | 208 | pyramid-mako==1.1.0 |
|
210 | 209 | mako==1.2.4 |
|
211 | 210 | markupsafe==2.1.2 |
|
212 | 211 | pyramid==2.0.2 |
|
213 | 212 | hupper==1.12 |
|
214 | 213 | plaster==1.1.2 |
|
215 | 214 | plaster-pastedeploy==1.0.1 |
|
216 | 215 | pastedeploy==3.1.0 |
|
217 | 216 | plaster==1.1.2 |
|
218 | 217 | translationstring==1.4 |
|
219 | 218 | venusian==3.0.0 |
|
220 | 219 | webob==1.8.7 |
|
221 | 220 | zope.deprecation==5.0.0 |
|
222 | 221 | zope.interface==6.3.0 |
|
223 | 222 | pyramid-mailer==0.15.1 |
|
224 | 223 | pyramid==2.0.2 |
|
225 | 224 | hupper==1.12 |
|
226 | 225 | plaster==1.1.2 |
|
227 | 226 | plaster-pastedeploy==1.0.1 |
|
228 | 227 | pastedeploy==3.1.0 |
|
229 | 228 | plaster==1.1.2 |
|
230 | 229 | translationstring==1.4 |
|
231 | 230 | venusian==3.0.0 |
|
232 | 231 | webob==1.8.7 |
|
233 | 232 | zope.deprecation==5.0.0 |
|
234 | 233 | zope.interface==6.3.0 |
|
235 | 234 | repoze.sendmail==4.4.1 |
|
236 | 235 | transaction==3.1.0 |
|
237 | 236 | zope.interface==6.3.0 |
|
238 | 237 | zope.interface==6.3.0 |
|
239 | 238 | transaction==3.1.0 |
|
240 | 239 | zope.interface==6.3.0 |
|
241 | 240 | python-ldap==3.4.3 |
|
242 | 241 | pyasn1==0.4.8 |
|
243 | 242 | pyasn1-modules==0.2.8 |
|
244 | 243 | pyasn1==0.4.8 |
|
245 | 244 | python-memcached==1.59 |
|
246 | 245 | six==1.16.0 |
|
247 | 246 | python-pam==2.0.2 |
|
248 | 247 | python3-saml==1.15.0 |
|
249 | 248 | isodate==0.6.1 |
|
250 | 249 | six==1.16.0 |
|
251 | 250 | lxml==4.9.3 |
|
252 | 251 | xmlsec==1.3.13 |
|
253 | 252 | lxml==4.9.3 |
|
254 | 253 | pyyaml==6.0.1 |
|
255 | 254 | redis==5.0.4 |
|
256 | 255 | async-timeout==4.0.3 |
|
257 | 256 | regex==2022.10.31 |
|
258 | 257 | routes==2.5.1 |
|
259 | 258 | repoze.lru==0.7 |
|
260 | 259 | six==1.16.0 |
|
261 | 260 | simplejson==3.19.2 |
|
262 | 261 | sshpubkeys==3.3.1 |
|
263 | 262 | cryptography==40.0.2 |
|
264 | 263 | cffi==1.16.0 |
|
265 | 264 | pycparser==2.21 |
|
266 | 265 | ecdsa==0.18.0 |
|
267 | 266 | six==1.16.0 |
|
268 | 267 | sqlalchemy==1.4.52 |
|
269 | 268 | greenlet==3.0.3 |
|
270 | 269 | typing_extensions==4.9.0 |
|
271 | 270 | supervisor==4.2.5 |
|
272 | 271 | tzlocal==4.3 |
|
273 | 272 | pytz-deprecation-shim==0.1.0.post0 |
|
274 | 273 | tzdata==2024.1 |
|
275 | 274 | tempita==0.5.2 |
|
276 | 275 | unidecode==1.3.6 |
|
277 | 276 | urlobject==2.4.3 |
|
278 | 277 | waitress==3.0.0 |
|
279 | 278 | webhelpers2==2.1 |
|
280 | 279 | markupsafe==2.1.2 |
|
281 | 280 | six==1.16.0 |
|
282 | 281 | whoosh==2.7.4 |
|
283 | 282 | zope.cachedescriptors==5.0.0 |
|
284 | 283 | qrcode==7.4.2 |
|
285 | 284 | |
|
286 | 285 | ## uncomment to add the debug libraries |
|
287 | 286 | #-r requirements_debug.txt |
@@ -1,1708 +1,1715 b'' | |||
|
1 | 1 | # Copyright (C) 2011-2023 RhodeCode GmbH |
|
2 | 2 | # |
|
3 | 3 | # This program is free software: you can redistribute it and/or modify |
|
4 | 4 | # it under the terms of the GNU Affero General Public License, version 3 |
|
5 | 5 | # (only), as published by the Free Software Foundation. |
|
6 | 6 | # |
|
7 | 7 | # This program is distributed in the hope that it will be useful, |
|
8 | 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
9 | 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
10 | 10 | # GNU General Public License for more details. |
|
11 | 11 | # |
|
12 | 12 | # You should have received a copy of the GNU Affero General Public License |
|
13 | 13 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
14 | 14 | # |
|
15 | 15 | # This program is dual-licensed. If you wish to learn more about the |
|
16 | 16 | # RhodeCode Enterprise Edition, including its added features, Support services, |
|
17 | 17 | # and proprietary license terms, please see https://rhodecode.com/licenses/ |
|
18 | 18 | |
|
19 | 19 | import itertools |
|
20 | 20 | import logging |
|
21 | 21 | import os |
|
22 | 22 | import collections |
|
23 | 23 | import urllib.request |
|
24 | 24 | import urllib.parse |
|
25 | 25 | import urllib.error |
|
26 | 26 | import pathlib |
|
27 | import time | |
|
28 | import random | |
|
27 | 29 | |
|
28 | 30 | from pyramid.httpexceptions import HTTPNotFound, HTTPBadRequest, HTTPFound |
|
29 | 31 | |
|
30 | 32 | from pyramid.renderers import render |
|
31 | 33 | from pyramid.response import Response |
|
32 | 34 | |
|
33 | 35 | import rhodecode |
|
34 | 36 | from rhodecode.apps._base import RepoAppView |
|
35 | 37 | |
|
36 | 38 | |
|
37 | 39 | from rhodecode.lib import diffs, helpers as h, rc_cache |
|
38 | 40 | from rhodecode.lib import audit_logger |
|
39 | 41 | from rhodecode.lib.hash_utils import sha1_safe |
|
40 |
from rhodecode.lib.rc_cache.archive_cache import |
|
|
42 | from rhodecode.lib.rc_cache.archive_cache import ( | |
|
43 | get_archival_cache_store, get_archival_config, ArchiveCacheLock, archive_iterator) | |
|
41 | 44 | from rhodecode.lib.str_utils import safe_bytes, convert_special_chars |
|
42 | 45 | from rhodecode.lib.view_utils import parse_path_ref |
|
43 | 46 | from rhodecode.lib.exceptions import NonRelativePathError |
|
44 | 47 | from rhodecode.lib.codeblocks import ( |
|
45 | 48 | filenode_as_lines_tokens, filenode_as_annotated_lines_tokens) |
|
46 | 49 | from rhodecode.lib.utils2 import convert_line_endings, detect_mode |
|
47 | 50 | from rhodecode.lib.type_utils import str2bool |
|
48 | 51 | from rhodecode.lib.str_utils import safe_str, safe_int |
|
49 | 52 | from rhodecode.lib.auth import ( |
|
50 | 53 | LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired) |
|
51 | 54 | from rhodecode.lib.vcs import path as vcspath |
|
52 | 55 | from rhodecode.lib.vcs.backends.base import EmptyCommit |
|
53 | 56 | from rhodecode.lib.vcs.conf import settings |
|
54 | 57 | from rhodecode.lib.vcs.nodes import FileNode |
|
55 | 58 | from rhodecode.lib.vcs.exceptions import ( |
|
56 | 59 | RepositoryError, CommitDoesNotExistError, EmptyRepositoryError, |
|
57 | 60 | ImproperArchiveTypeError, VCSError, NodeAlreadyExistsError, |
|
58 | 61 | NodeDoesNotExistError, CommitError, NodeError) |
|
59 | 62 | |
|
60 | 63 | from rhodecode.model.scm import ScmModel |
|
61 | 64 | from rhodecode.model.db import Repository |
|
62 | 65 | |
|
63 | 66 | log = logging.getLogger(__name__) |
|
64 | 67 | |
|
65 | 68 | |
|
66 | 69 | def get_archive_name(db_repo_id, db_repo_name, commit_sha, ext, subrepos=False, path_sha='', with_hash=True): |
|
67 | 70 | # original backward compat name of archive |
|
68 | 71 | clean_name = safe_str(convert_special_chars(db_repo_name).replace('/', '_')) |
|
69 | 72 | |
|
70 | 73 | # e.g vcsserver-id-abcd-sub-1-abcfdef-archive-all.zip |
|
71 | 74 | # vcsserver-id-abcd-sub-0-abcfdef-COMMIT_SHA-PATH_SHA.zip |
|
72 | 75 | id_sha = sha1_safe(str(db_repo_id))[:4] |
|
73 | 76 | sub_repo = 'sub-1' if subrepos else 'sub-0' |
|
74 | 77 | commit = commit_sha if with_hash else 'archive' |
|
75 | 78 | path_marker = (path_sha if with_hash else '') or 'all' |
|
76 | 79 | archive_name = f'{clean_name}-id-{id_sha}-{sub_repo}-{commit}-{path_marker}{ext}' |
|
77 | 80 | |
|
78 | 81 | return archive_name |
|
79 | 82 | |
|
80 | 83 | |
|
81 | 84 | def get_path_sha(at_path): |
|
82 | 85 | return safe_str(sha1_safe(at_path)[:8]) |
|
83 | 86 | |
|
84 | 87 | |
|
85 | 88 | def _get_archive_spec(fname): |
|
86 | 89 | log.debug('Detecting archive spec for: `%s`', fname) |
|
87 | 90 | |
|
88 | 91 | fileformat = None |
|
89 | 92 | ext = None |
|
90 | 93 | content_type = None |
|
91 | 94 | for a_type, content_type, extension in settings.ARCHIVE_SPECS: |
|
92 | 95 | |
|
93 | 96 | if fname.endswith(extension): |
|
94 | 97 | fileformat = a_type |
|
95 | 98 | log.debug('archive is of type: %s', fileformat) |
|
96 | 99 | ext = extension |
|
97 | 100 | break |
|
98 | 101 | |
|
99 | 102 | if not fileformat: |
|
100 | 103 | raise ValueError() |
|
101 | 104 | |
|
102 | 105 | # left over part of whole fname is the commit |
|
103 | 106 | commit_id = fname[:-len(ext)] |
|
104 | 107 | |
|
105 | 108 | return commit_id, ext, fileformat, content_type |
|
106 | 109 | |
|
107 | 110 | |
|
108 | 111 | class RepoFilesView(RepoAppView): |
|
109 | 112 | |
|
110 | 113 | @staticmethod |
|
111 | 114 | def adjust_file_path_for_svn(f_path, repo): |
|
112 | 115 | """ |
|
113 | 116 | Computes the relative path of `f_path`. |
|
114 | 117 | |
|
115 | 118 | This is mainly based on prefix matching of the recognized tags and |
|
116 | 119 | branches in the underlying repository. |
|
117 | 120 | """ |
|
118 | 121 | tags_and_branches = itertools.chain( |
|
119 | 122 | repo.branches.keys(), |
|
120 | 123 | repo.tags.keys()) |
|
121 | 124 | tags_and_branches = sorted(tags_and_branches, key=len, reverse=True) |
|
122 | 125 | |
|
123 | 126 | for name in tags_and_branches: |
|
124 | 127 | if f_path.startswith(f'{name}/'): |
|
125 | 128 | f_path = vcspath.relpath(f_path, name) |
|
126 | 129 | break |
|
127 | 130 | return f_path |
|
128 | 131 | |
|
129 | 132 | def load_default_context(self): |
|
130 | 133 | c = self._get_local_tmpl_context(include_app_defaults=True) |
|
131 | 134 | c.rhodecode_repo = self.rhodecode_vcs_repo |
|
132 | 135 | c.enable_downloads = self.db_repo.enable_downloads |
|
133 | 136 | return c |
|
134 | 137 | |
|
135 | 138 | def _ensure_not_locked(self, commit_id='tip'): |
|
136 | 139 | _ = self.request.translate |
|
137 | 140 | |
|
138 | 141 | repo = self.db_repo |
|
139 | 142 | if repo.enable_locking and repo.locked[0]: |
|
140 | 143 | h.flash(_('This repository has been locked by %s on %s') |
|
141 | 144 | % (h.person_by_id(repo.locked[0]), |
|
142 | 145 | h.format_date(h.time_to_datetime(repo.locked[1]))), |
|
143 | 146 | 'warning') |
|
144 | 147 | files_url = h.route_path( |
|
145 | 148 | 'repo_files:default_path', |
|
146 | 149 | repo_name=self.db_repo_name, commit_id=commit_id) |
|
147 | 150 | raise HTTPFound(files_url) |
|
148 | 151 | |
|
149 | 152 | def forbid_non_head(self, is_head, f_path, commit_id='tip', json_mode=False): |
|
150 | 153 | _ = self.request.translate |
|
151 | 154 | |
|
152 | 155 | if not is_head: |
|
153 | 156 | message = _('Cannot modify file. ' |
|
154 | 157 | 'Given commit `{}` is not head of a branch.').format(commit_id) |
|
155 | 158 | h.flash(message, category='warning') |
|
156 | 159 | |
|
157 | 160 | if json_mode: |
|
158 | 161 | return message |
|
159 | 162 | |
|
160 | 163 | files_url = h.route_path( |
|
161 | 164 | 'repo_files', repo_name=self.db_repo_name, commit_id=commit_id, |
|
162 | 165 | f_path=f_path) |
|
163 | 166 | raise HTTPFound(files_url) |
|
164 | 167 | |
|
165 | 168 | def check_branch_permission(self, branch_name, commit_id='tip', json_mode=False): |
|
166 | 169 | _ = self.request.translate |
|
167 | 170 | |
|
168 | 171 | rule, branch_perm = self._rhodecode_user.get_rule_and_branch_permission( |
|
169 | 172 | self.db_repo_name, branch_name) |
|
170 | 173 | if branch_perm and branch_perm not in ['branch.push', 'branch.push_force']: |
|
171 | 174 | message = _('Branch `{}` changes forbidden by rule {}.').format( |
|
172 | 175 | h.escape(branch_name), h.escape(rule)) |
|
173 | 176 | h.flash(message, 'warning') |
|
174 | 177 | |
|
175 | 178 | if json_mode: |
|
176 | 179 | return message |
|
177 | 180 | |
|
178 | 181 | files_url = h.route_path( |
|
179 | 182 | 'repo_files:default_path', repo_name=self.db_repo_name, commit_id=commit_id) |
|
180 | 183 | |
|
181 | 184 | raise HTTPFound(files_url) |
|
182 | 185 | |
|
183 | 186 | def _get_commit_and_path(self): |
|
184 | 187 | default_commit_id = self.db_repo.landing_ref_name |
|
185 | 188 | default_f_path = '/' |
|
186 | 189 | |
|
187 | 190 | commit_id = self.request.matchdict.get( |
|
188 | 191 | 'commit_id', default_commit_id) |
|
189 | 192 | f_path = self._get_f_path(self.request.matchdict, default_f_path) |
|
190 | 193 | return commit_id, f_path |
|
191 | 194 | |
|
192 | 195 | def _get_default_encoding(self, c): |
|
193 | 196 | enc_list = getattr(c, 'default_encodings', []) |
|
194 | 197 | return enc_list[0] if enc_list else 'UTF-8' |
|
195 | 198 | |
|
196 | 199 | def _get_commit_or_redirect(self, commit_id, redirect_after=True): |
|
197 | 200 | """ |
|
198 | 201 | This is a safe way to get commit. If an error occurs it redirects to |
|
199 | 202 | tip with proper message |
|
200 | 203 | |
|
201 | 204 | :param commit_id: id of commit to fetch |
|
202 | 205 | :param redirect_after: toggle redirection |
|
203 | 206 | """ |
|
204 | 207 | _ = self.request.translate |
|
205 | 208 | |
|
206 | 209 | try: |
|
207 | 210 | return self.rhodecode_vcs_repo.get_commit(commit_id) |
|
208 | 211 | except EmptyRepositoryError: |
|
209 | 212 | if not redirect_after: |
|
210 | 213 | return None |
|
211 | 214 | |
|
212 | 215 | add_new = upload_new = "" |
|
213 | 216 | if h.HasRepoPermissionAny( |
|
214 | 217 | 'repository.write', 'repository.admin')(self.db_repo_name): |
|
215 | 218 | _url = h.route_path( |
|
216 | 219 | 'repo_files_add_file', |
|
217 | 220 | repo_name=self.db_repo_name, commit_id=0, f_path='') |
|
218 | 221 | add_new = h.link_to( |
|
219 | 222 | _('add a new file'), _url, class_="alert-link") |
|
220 | 223 | |
|
221 | 224 | _url_upld = h.route_path( |
|
222 | 225 | 'repo_files_upload_file', |
|
223 | 226 | repo_name=self.db_repo_name, commit_id=0, f_path='') |
|
224 | 227 | upload_new = h.link_to( |
|
225 | 228 | _('upload a new file'), _url_upld, class_="alert-link") |
|
226 | 229 | |
|
227 | 230 | h.flash(h.literal( |
|
228 | 231 | _('There are no files yet. Click here to %s or %s.') % (add_new, upload_new)), category='warning') |
|
229 | 232 | raise HTTPFound( |
|
230 | 233 | h.route_path('repo_summary', repo_name=self.db_repo_name)) |
|
231 | 234 | |
|
232 | 235 | except (CommitDoesNotExistError, LookupError) as e: |
|
233 | 236 | msg = _('No such commit exists for this repository. Commit: {}').format(commit_id) |
|
234 | 237 | h.flash(msg, category='error') |
|
235 | 238 | raise HTTPNotFound() |
|
236 | 239 | except RepositoryError as e: |
|
237 | 240 | h.flash(h.escape(safe_str(e)), category='error') |
|
238 | 241 | raise HTTPNotFound() |
|
239 | 242 | |
|
240 | 243 | def _get_filenode_or_redirect(self, commit_obj, path, pre_load=None): |
|
241 | 244 | """ |
|
242 | 245 | Returns file_node, if error occurs or given path is directory, |
|
243 | 246 | it'll redirect to top level path |
|
244 | 247 | """ |
|
245 | 248 | _ = self.request.translate |
|
246 | 249 | |
|
247 | 250 | try: |
|
248 | 251 | file_node = commit_obj.get_node(path, pre_load=pre_load) |
|
249 | 252 | if file_node.is_dir(): |
|
250 | 253 | raise RepositoryError('The given path is a directory') |
|
251 | 254 | except CommitDoesNotExistError: |
|
252 | 255 | log.exception('No such commit exists for this repository') |
|
253 | 256 | h.flash(_('No such commit exists for this repository'), category='error') |
|
254 | 257 | raise HTTPNotFound() |
|
255 | 258 | except RepositoryError as e: |
|
256 | 259 | log.warning('Repository error while fetching filenode `%s`. Err:%s', path, e) |
|
257 | 260 | h.flash(h.escape(safe_str(e)), category='error') |
|
258 | 261 | raise HTTPNotFound() |
|
259 | 262 | |
|
260 | 263 | return file_node |
|
261 | 264 | |
|
262 | 265 | def _is_valid_head(self, commit_id, repo, landing_ref): |
|
263 | 266 | branch_name = sha_commit_id = '' |
|
264 | 267 | is_head = False |
|
265 | 268 | log.debug('Checking if commit_id `%s` is a head for %s.', commit_id, repo) |
|
266 | 269 | |
|
267 | 270 | for _branch_name, branch_commit_id in repo.branches.items(): |
|
268 | 271 | # simple case we pass in branch name, it's a HEAD |
|
269 | 272 | if commit_id == _branch_name: |
|
270 | 273 | is_head = True |
|
271 | 274 | branch_name = _branch_name |
|
272 | 275 | sha_commit_id = branch_commit_id |
|
273 | 276 | break |
|
274 | 277 | # case when we pass in full sha commit_id, which is a head |
|
275 | 278 | elif commit_id == branch_commit_id: |
|
276 | 279 | is_head = True |
|
277 | 280 | branch_name = _branch_name |
|
278 | 281 | sha_commit_id = branch_commit_id |
|
279 | 282 | break |
|
280 | 283 | |
|
281 | 284 | if h.is_svn(repo) and not repo.is_empty(): |
|
282 | 285 | # Note: Subversion only has one head. |
|
283 | 286 | if commit_id == repo.get_commit(commit_idx=-1).raw_id: |
|
284 | 287 | is_head = True |
|
285 | 288 | return branch_name, sha_commit_id, is_head |
|
286 | 289 | |
|
287 | 290 | # checked branches, means we only need to try to get the branch/commit_sha |
|
288 | 291 | if repo.is_empty(): |
|
289 | 292 | is_head = True |
|
290 | 293 | branch_name = landing_ref |
|
291 | 294 | sha_commit_id = EmptyCommit().raw_id |
|
292 | 295 | else: |
|
293 | 296 | commit = repo.get_commit(commit_id=commit_id) |
|
294 | 297 | if commit: |
|
295 | 298 | branch_name = commit.branch |
|
296 | 299 | sha_commit_id = commit.raw_id |
|
297 | 300 | |
|
298 | 301 | return branch_name, sha_commit_id, is_head |
|
299 | 302 | |
|
300 | 303 | def _get_tree_at_commit(self, c, commit_id, f_path, full_load=False, at_rev=None): |
|
301 | 304 | |
|
302 | 305 | repo_id = self.db_repo.repo_id |
|
303 | 306 | force_recache = self.get_recache_flag() |
|
304 | 307 | |
|
305 | 308 | cache_seconds = safe_int( |
|
306 | 309 | rhodecode.CONFIG.get('rc_cache.cache_repo.expiration_time')) |
|
307 | 310 | cache_on = not force_recache and cache_seconds > 0 |
|
308 | 311 | log.debug( |
|
309 | 312 | 'Computing FILE TREE for repo_id %s commit_id `%s` and path `%s`' |
|
310 | 313 | 'with caching: %s[TTL: %ss]' % ( |
|
311 | 314 | repo_id, commit_id, f_path, cache_on, cache_seconds or 0)) |
|
312 | 315 | |
|
313 | 316 | cache_namespace_uid = f'repo.{rc_cache.FILE_TREE_CACHE_VER}.{repo_id}' |
|
314 | 317 | region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid) |
|
315 | 318 | |
|
316 | 319 | @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache_on) |
|
317 | 320 | def compute_file_tree(_name_hash, _repo_id, _commit_id, _f_path, _full_load, _at_rev): |
|
318 | 321 | log.debug('Generating cached file tree at for repo_id: %s, %s, %s', |
|
319 | 322 | _repo_id, _commit_id, _f_path) |
|
320 | 323 | |
|
321 | 324 | c.full_load = _full_load |
|
322 | 325 | return render( |
|
323 | 326 | 'rhodecode:templates/files/files_browser_tree.mako', |
|
324 | 327 | self._get_template_context(c), self.request, _at_rev) |
|
325 | 328 | |
|
326 | 329 | return compute_file_tree( |
|
327 | 330 | self.db_repo.repo_name_hash, self.db_repo.repo_id, commit_id, f_path, full_load, at_rev) |
|
328 | 331 | |
|
329 | 332 | def create_pure_path(self, *parts): |
|
330 | 333 | # Split paths and sanitize them, removing any ../ etc |
|
331 | 334 | sanitized_path = [ |
|
332 | 335 | x for x in pathlib.PurePath(*parts).parts |
|
333 | 336 | if x not in ['.', '..']] |
|
334 | 337 | |
|
335 | 338 | pure_path = pathlib.PurePath(*sanitized_path) |
|
336 | 339 | return pure_path |
|
337 | 340 | |
|
338 | 341 | def _is_lf_enabled(self, target_repo): |
|
339 | 342 | lf_enabled = False |
|
340 | 343 | |
|
341 | 344 | lf_key_for_vcs_map = { |
|
342 | 345 | 'hg': 'extensions_largefiles', |
|
343 | 346 | 'git': 'vcs_git_lfs_enabled' |
|
344 | 347 | } |
|
345 | 348 | |
|
346 | 349 | lf_key_for_vcs = lf_key_for_vcs_map.get(target_repo.repo_type) |
|
347 | 350 | |
|
348 | 351 | if lf_key_for_vcs: |
|
349 | 352 | lf_enabled = self._get_repo_setting(target_repo, lf_key_for_vcs) |
|
350 | 353 | |
|
351 | 354 | return lf_enabled |
|
352 | 355 | |
|
353 | 356 | @LoginRequired() |
|
354 | 357 | @HasRepoPermissionAnyDecorator( |
|
355 | 358 | 'repository.read', 'repository.write', 'repository.admin') |
|
356 | 359 | def repo_archivefile(self): |
|
357 | 360 | # archive cache config |
|
358 | 361 | from rhodecode import CONFIG |
|
359 | 362 | _ = self.request.translate |
|
360 | 363 | self.load_default_context() |
|
361 | 364 | default_at_path = '/' |
|
362 | 365 | fname = self.request.matchdict['fname'] |
|
363 | 366 | subrepos = self.request.GET.get('subrepos') == 'true' |
|
364 | 367 | with_hash = str2bool(self.request.GET.get('with_hash', '1')) |
|
365 | 368 | at_path = self.request.GET.get('at_path') or default_at_path |
|
366 | 369 | |
|
367 | 370 | if not self.db_repo.enable_downloads: |
|
368 | 371 | return Response(_('Downloads disabled')) |
|
369 | 372 | |
|
370 | 373 | try: |
|
371 | 374 | commit_id, ext, fileformat, content_type = \ |
|
372 | 375 | _get_archive_spec(fname) |
|
373 | 376 | except ValueError: |
|
374 | 377 | return Response(_('Unknown archive type for: `{}`').format( |
|
375 | 378 | h.escape(fname))) |
|
376 | 379 | |
|
377 | 380 | try: |
|
378 | 381 | commit = self.rhodecode_vcs_repo.get_commit(commit_id) |
|
379 | 382 | except CommitDoesNotExistError: |
|
380 | 383 | return Response(_('Unknown commit_id {}').format( |
|
381 | 384 | h.escape(commit_id))) |
|
382 | 385 | except EmptyRepositoryError: |
|
383 | 386 | return Response(_('Empty repository')) |
|
384 | 387 | |
|
385 | 388 | # we used a ref, or a shorter version, lets redirect client ot use explicit hash |
|
386 | 389 | if commit_id != commit.raw_id: |
|
387 | 390 | fname=f'{commit.raw_id}{ext}' |
|
388 | 391 | raise HTTPFound(self.request.current_route_path(fname=fname)) |
|
389 | 392 | |
|
390 | 393 | try: |
|
391 | 394 | at_path = commit.get_node(at_path).path or default_at_path |
|
392 | 395 | except Exception: |
|
393 | 396 | return Response(_('No node at path {} for this repository').format(h.escape(at_path))) |
|
394 | 397 | |
|
395 | 398 | path_sha = get_path_sha(at_path) |
|
396 | 399 | |
|
397 | 400 | # used for cache etc, consistent unique archive name |
|
398 | 401 | archive_name_key = get_archive_name( |
|
399 | 402 | self.db_repo.repo_id, self.db_repo_name, commit_sha=commit.short_id, ext=ext, subrepos=subrepos, |
|
400 | 403 | path_sha=path_sha, with_hash=True) |
|
401 | 404 | |
|
402 | 405 | if not with_hash: |
|
403 | 406 | path_sha = '' |
|
404 | 407 | |
|
405 | 408 | # what end client gets served |
|
406 | 409 | response_archive_name = get_archive_name( |
|
407 | 410 | self.db_repo.repo_id, self.db_repo_name, commit_sha=commit.short_id, ext=ext, subrepos=subrepos, |
|
408 | 411 | path_sha=path_sha, with_hash=with_hash) |
|
409 | 412 | |
|
410 | 413 | # remove extension from our archive directory name |
|
411 | 414 | archive_dir_name = response_archive_name[:-len(ext)] |
|
412 | 415 | |
|
413 | 416 | archive_cache_disable = self.request.GET.get('no_cache') |
|
414 | 417 | |
|
415 | 418 | d_cache = get_archival_cache_store(config=CONFIG) |
|
416 | 419 | |
|
417 | 420 | # NOTE: we get the config to pass to a call to lazy-init the SAME type of cache on vcsserver |
|
418 | 421 | d_cache_conf = get_archival_config(config=CONFIG) |
|
419 | 422 | |
|
423 | # This is also a cache key, and lock key | |
|
420 | 424 | reentrant_lock_key = archive_name_key + '.lock' |
|
421 | with ReentrantLock(d_cache, reentrant_lock_key): | |
|
422 | # This is also a cache key | |
|
423 | use_cached_archive = False | |
|
424 | if not archive_cache_disable and archive_name_key in d_cache: | |
|
425 | reader, tag = d_cache.get(archive_name_key, read=True, tag=True, retry=True) | |
|
426 | use_cached_archive = True | |
|
427 | log.debug('Found cached archive as key=%s tag=%s, serving archive from cache reader=%s', | |
|
428 | archive_name_key, tag, reader.name) | |
|
429 | else: | |
|
430 | reader = None | |
|
431 | log.debug('Archive with key=%s is not yet cached, creating one now...', archive_name_key) | |
|
425 | ||
|
426 | use_cached_archive = False | |
|
427 | if not archive_cache_disable and archive_name_key in d_cache: | |
|
428 | reader, metadata = d_cache.fetch(archive_name_key) | |
|
432 | 429 | |
|
433 | # generate new archive, as previous was not found in the cache | |
|
434 | if not reader: | |
|
435 | ||
|
436 | try: | |
|
437 | commit.archive_repo(archive_name_key, archive_dir_name=archive_dir_name, | |
|
438 | kind=fileformat, subrepos=subrepos, | |
|
439 | archive_at_path=at_path, cache_config=d_cache_conf) | |
|
440 | except ImproperArchiveTypeError: | |
|
441 | return _('Unknown archive type') | |
|
442 | ||
|
443 | reader, tag = d_cache.get(archive_name_key, read=True, tag=True, retry=True) | |
|
430 | use_cached_archive = True | |
|
431 | log.debug('Found cached archive as key=%s tag=%s, serving archive from cache reader=%s', | |
|
432 | archive_name_key, metadata, reader.name) | |
|
433 | else: | |
|
434 | reader = None | |
|
435 | log.debug('Archive with key=%s is not yet cached, creating one now...', archive_name_key) | |
|
444 | 436 | |
|
445 | 437 | if not reader: |
|
446 | raise ValueError('archive cache reader is empty, failed to fetch file from distributed archive cache') | |
|
438 | # generate new archive, as previous was not found in the cache | |
|
439 | try: | |
|
440 | with d_cache.get_lock(reentrant_lock_key): | |
|
441 | try: | |
|
442 | commit.archive_repo(archive_name_key, archive_dir_name=archive_dir_name, | |
|
443 | kind=fileformat, subrepos=subrepos, | |
|
444 | archive_at_path=at_path, cache_config=d_cache_conf) | |
|
445 | except ImproperArchiveTypeError: | |
|
446 | return _('Unknown archive type') | |
|
447 | except ArchiveCacheLock: | |
|
448 | retry_after = round(random.uniform(0.3, 3.0), 1) | |
|
449 | time.sleep(retry_after) | |
|
447 | 450 | |
|
448 | def archive_iterator(_reader, block_size: int = 4096*512): | |
|
449 | # 4096 * 64 = 64KB | |
|
450 | while 1: | |
|
451 | data = _reader.read(block_size) | |
|
452 | if not data: | |
|
453 | break | |
|
454 | yield data | |
|
451 | location = self.request.url | |
|
452 | response = Response( | |
|
453 | f"archive {archive_name_key} generation in progress, Retry-After={retry_after}, Location={location}" | |
|
454 | ) | |
|
455 | response.headers["Retry-After"] = str(retry_after) | |
|
456 | response.status_code = 307 # temporary redirect | |
|
457 | ||
|
458 | response.location = location | |
|
459 | return response | |
|
460 | ||
|
461 | reader, metadata = d_cache.fetch(archive_name_key) | |
|
455 | 462 | |
|
456 | 463 | response = Response(app_iter=archive_iterator(reader)) |
|
457 | 464 | response.content_disposition = f'attachment; filename={response_archive_name}' |
|
458 | 465 | response.content_type = str(content_type) |
|
459 | 466 | |
|
460 | 467 | try: |
|
461 | 468 | return response |
|
462 | 469 | finally: |
|
463 | 470 | # store download action |
|
464 | 471 | audit_logger.store_web( |
|
465 | 472 | 'repo.archive.download', action_data={ |
|
466 | 473 | 'user_agent': self.request.user_agent, |
|
467 | 474 | 'archive_name': archive_name_key, |
|
468 | 475 | 'archive_spec': fname, |
|
469 | 476 | 'archive_cached': use_cached_archive}, |
|
470 | 477 | user=self._rhodecode_user, |
|
471 | 478 | repo=self.db_repo, |
|
472 | 479 | commit=True |
|
473 | 480 | ) |
|
474 | 481 | |
|
475 | 482 | def _get_file_node(self, commit_id, f_path): |
|
476 | 483 | if commit_id not in ['', None, 'None', '0' * 12, '0' * 40]: |
|
477 | 484 | commit = self.rhodecode_vcs_repo.get_commit(commit_id=commit_id) |
|
478 | 485 | try: |
|
479 | 486 | node = commit.get_node(f_path) |
|
480 | 487 | if node.is_dir(): |
|
481 | 488 | raise NodeError(f'{node} path is a {type(node)} not a file') |
|
482 | 489 | except NodeDoesNotExistError: |
|
483 | 490 | commit = EmptyCommit( |
|
484 | 491 | commit_id=commit_id, |
|
485 | 492 | idx=commit.idx, |
|
486 | 493 | repo=commit.repository, |
|
487 | 494 | alias=commit.repository.alias, |
|
488 | 495 | message=commit.message, |
|
489 | 496 | author=commit.author, |
|
490 | 497 | date=commit.date) |
|
491 | 498 | node = FileNode(safe_bytes(f_path), b'', commit=commit) |
|
492 | 499 | else: |
|
493 | 500 | commit = EmptyCommit( |
|
494 | 501 | repo=self.rhodecode_vcs_repo, |
|
495 | 502 | alias=self.rhodecode_vcs_repo.alias) |
|
496 | 503 | node = FileNode(safe_bytes(f_path), b'', commit=commit) |
|
497 | 504 | return node |
|
498 | 505 | |
|
499 | 506 | @LoginRequired() |
|
500 | 507 | @HasRepoPermissionAnyDecorator( |
|
501 | 508 | 'repository.read', 'repository.write', 'repository.admin') |
|
502 | 509 | def repo_files_diff(self): |
|
503 | 510 | c = self.load_default_context() |
|
504 | 511 | f_path = self._get_f_path(self.request.matchdict) |
|
505 | 512 | diff1 = self.request.GET.get('diff1', '') |
|
506 | 513 | diff2 = self.request.GET.get('diff2', '') |
|
507 | 514 | |
|
508 | 515 | path1, diff1 = parse_path_ref(diff1, default_path=f_path) |
|
509 | 516 | |
|
510 | 517 | ignore_whitespace = str2bool(self.request.GET.get('ignorews')) |
|
511 | 518 | line_context = self.request.GET.get('context', 3) |
|
512 | 519 | |
|
513 | 520 | if not any((diff1, diff2)): |
|
514 | 521 | h.flash( |
|
515 | 522 | 'Need query parameter "diff1" or "diff2" to generate a diff.', |
|
516 | 523 | category='error') |
|
517 | 524 | raise HTTPBadRequest() |
|
518 | 525 | |
|
519 | 526 | c.action = self.request.GET.get('diff') |
|
520 | 527 | if c.action not in ['download', 'raw']: |
|
521 | 528 | compare_url = h.route_path( |
|
522 | 529 | 'repo_compare', |
|
523 | 530 | repo_name=self.db_repo_name, |
|
524 | 531 | source_ref_type='rev', |
|
525 | 532 | source_ref=diff1, |
|
526 | 533 | target_repo=self.db_repo_name, |
|
527 | 534 | target_ref_type='rev', |
|
528 | 535 | target_ref=diff2, |
|
529 | 536 | _query=dict(f_path=f_path)) |
|
530 | 537 | # redirect to new view if we render diff |
|
531 | 538 | raise HTTPFound(compare_url) |
|
532 | 539 | |
|
533 | 540 | try: |
|
534 | 541 | node1 = self._get_file_node(diff1, path1) |
|
535 | 542 | node2 = self._get_file_node(diff2, f_path) |
|
536 | 543 | except (RepositoryError, NodeError): |
|
537 | 544 | log.exception("Exception while trying to get node from repository") |
|
538 | 545 | raise HTTPFound( |
|
539 | 546 | h.route_path('repo_files', repo_name=self.db_repo_name, |
|
540 | 547 | commit_id='tip', f_path=f_path)) |
|
541 | 548 | |
|
542 | 549 | if all(isinstance(node.commit, EmptyCommit) |
|
543 | 550 | for node in (node1, node2)): |
|
544 | 551 | raise HTTPNotFound() |
|
545 | 552 | |
|
546 | 553 | c.commit_1 = node1.commit |
|
547 | 554 | c.commit_2 = node2.commit |
|
548 | 555 | |
|
549 | 556 | if c.action == 'download': |
|
550 | 557 | _diff = diffs.get_gitdiff(node1, node2, |
|
551 | 558 | ignore_whitespace=ignore_whitespace, |
|
552 | 559 | context=line_context) |
|
553 | 560 | # NOTE: this was using diff_format='gitdiff' |
|
554 | 561 | diff = diffs.DiffProcessor(_diff, diff_format='newdiff') |
|
555 | 562 | |
|
556 | 563 | response = Response(self.path_filter.get_raw_patch(diff)) |
|
557 | 564 | response.content_type = 'text/plain' |
|
558 | 565 | response.content_disposition = ( |
|
559 | 566 | f'attachment; filename={f_path}_{diff1}_vs_{diff2}.diff' |
|
560 | 567 | ) |
|
561 | 568 | charset = self._get_default_encoding(c) |
|
562 | 569 | if charset: |
|
563 | 570 | response.charset = charset |
|
564 | 571 | return response |
|
565 | 572 | |
|
566 | 573 | elif c.action == 'raw': |
|
567 | 574 | _diff = diffs.get_gitdiff(node1, node2, |
|
568 | 575 | ignore_whitespace=ignore_whitespace, |
|
569 | 576 | context=line_context) |
|
570 | 577 | # NOTE: this was using diff_format='gitdiff' |
|
571 | 578 | diff = diffs.DiffProcessor(_diff, diff_format='newdiff') |
|
572 | 579 | |
|
573 | 580 | response = Response(self.path_filter.get_raw_patch(diff)) |
|
574 | 581 | response.content_type = 'text/plain' |
|
575 | 582 | charset = self._get_default_encoding(c) |
|
576 | 583 | if charset: |
|
577 | 584 | response.charset = charset |
|
578 | 585 | return response |
|
579 | 586 | |
|
580 | 587 | # in case we ever end up here |
|
581 | 588 | raise HTTPNotFound() |
|
582 | 589 | |
|
583 | 590 | @LoginRequired() |
|
584 | 591 | @HasRepoPermissionAnyDecorator( |
|
585 | 592 | 'repository.read', 'repository.write', 'repository.admin') |
|
586 | 593 | def repo_files_diff_2way_redirect(self): |
|
587 | 594 | """ |
|
588 | 595 | Kept only to make OLD links work |
|
589 | 596 | """ |
|
590 | 597 | f_path = self._get_f_path_unchecked(self.request.matchdict) |
|
591 | 598 | diff1 = self.request.GET.get('diff1', '') |
|
592 | 599 | diff2 = self.request.GET.get('diff2', '') |
|
593 | 600 | |
|
594 | 601 | if not any((diff1, diff2)): |
|
595 | 602 | h.flash( |
|
596 | 603 | 'Need query parameter "diff1" or "diff2" to generate a diff.', |
|
597 | 604 | category='error') |
|
598 | 605 | raise HTTPBadRequest() |
|
599 | 606 | |
|
600 | 607 | compare_url = h.route_path( |
|
601 | 608 | 'repo_compare', |
|
602 | 609 | repo_name=self.db_repo_name, |
|
603 | 610 | source_ref_type='rev', |
|
604 | 611 | source_ref=diff1, |
|
605 | 612 | target_ref_type='rev', |
|
606 | 613 | target_ref=diff2, |
|
607 | 614 | _query=dict(f_path=f_path, diffmode='sideside', |
|
608 | 615 | target_repo=self.db_repo_name,)) |
|
609 | 616 | raise HTTPFound(compare_url) |
|
610 | 617 | |
|
611 | 618 | @LoginRequired() |
|
612 | 619 | def repo_files_default_commit_redirect(self): |
|
613 | 620 | """ |
|
614 | 621 | Special page that redirects to the landing page of files based on the default |
|
615 | 622 | commit for repository |
|
616 | 623 | """ |
|
617 | 624 | c = self.load_default_context() |
|
618 | 625 | ref_name = c.rhodecode_db_repo.landing_ref_name |
|
619 | 626 | landing_url = h.repo_files_by_ref_url( |
|
620 | 627 | c.rhodecode_db_repo.repo_name, |
|
621 | 628 | c.rhodecode_db_repo.repo_type, |
|
622 | 629 | f_path='', |
|
623 | 630 | ref_name=ref_name, |
|
624 | 631 | commit_id='tip', |
|
625 | 632 | query=dict(at=ref_name) |
|
626 | 633 | ) |
|
627 | 634 | |
|
628 | 635 | raise HTTPFound(landing_url) |
|
629 | 636 | |
|
630 | 637 | @LoginRequired() |
|
631 | 638 | @HasRepoPermissionAnyDecorator( |
|
632 | 639 | 'repository.read', 'repository.write', 'repository.admin') |
|
633 | 640 | def repo_files(self): |
|
634 | 641 | c = self.load_default_context() |
|
635 | 642 | |
|
636 | 643 | view_name = getattr(self.request.matched_route, 'name', None) |
|
637 | 644 | |
|
638 | 645 | c.annotate = view_name == 'repo_files:annotated' |
|
639 | 646 | # default is false, but .rst/.md files later are auto rendered, we can |
|
640 | 647 | # overwrite auto rendering by setting this GET flag |
|
641 | 648 | c.renderer = view_name == 'repo_files:rendered' or not self.request.GET.get('no-render', False) |
|
642 | 649 | |
|
643 | 650 | commit_id, f_path = self._get_commit_and_path() |
|
644 | 651 | |
|
645 | 652 | c.commit = self._get_commit_or_redirect(commit_id) |
|
646 | 653 | c.branch = self.request.GET.get('branch', None) |
|
647 | 654 | c.f_path = f_path |
|
648 | 655 | at_rev = self.request.GET.get('at') |
|
649 | 656 | |
|
650 | 657 | # files or dirs |
|
651 | 658 | try: |
|
652 | 659 | c.file = c.commit.get_node(f_path, pre_load=['is_binary', 'size', 'data']) |
|
653 | 660 | |
|
654 | 661 | c.file_author = True |
|
655 | 662 | c.file_tree = '' |
|
656 | 663 | |
|
657 | 664 | # prev link |
|
658 | 665 | try: |
|
659 | 666 | prev_commit = c.commit.prev(c.branch) |
|
660 | 667 | c.prev_commit = prev_commit |
|
661 | 668 | c.url_prev = h.route_path( |
|
662 | 669 | 'repo_files', repo_name=self.db_repo_name, |
|
663 | 670 | commit_id=prev_commit.raw_id, f_path=f_path) |
|
664 | 671 | if c.branch: |
|
665 | 672 | c.url_prev += '?branch=%s' % c.branch |
|
666 | 673 | except (CommitDoesNotExistError, VCSError): |
|
667 | 674 | c.url_prev = '#' |
|
668 | 675 | c.prev_commit = EmptyCommit() |
|
669 | 676 | |
|
670 | 677 | # next link |
|
671 | 678 | try: |
|
672 | 679 | next_commit = c.commit.next(c.branch) |
|
673 | 680 | c.next_commit = next_commit |
|
674 | 681 | c.url_next = h.route_path( |
|
675 | 682 | 'repo_files', repo_name=self.db_repo_name, |
|
676 | 683 | commit_id=next_commit.raw_id, f_path=f_path) |
|
677 | 684 | if c.branch: |
|
678 | 685 | c.url_next += '?branch=%s' % c.branch |
|
679 | 686 | except (CommitDoesNotExistError, VCSError): |
|
680 | 687 | c.url_next = '#' |
|
681 | 688 | c.next_commit = EmptyCommit() |
|
682 | 689 | |
|
683 | 690 | # load file content |
|
684 | 691 | if c.file.is_file(): |
|
685 | 692 | |
|
686 | 693 | c.lf_node = {} |
|
687 | 694 | |
|
688 | 695 | has_lf_enabled = self._is_lf_enabled(self.db_repo) |
|
689 | 696 | if has_lf_enabled: |
|
690 | 697 | c.lf_node = c.file.get_largefile_node() |
|
691 | 698 | |
|
692 | 699 | c.file_source_page = 'true' |
|
693 | 700 | c.file_last_commit = c.file.last_commit |
|
694 | 701 | |
|
695 | 702 | c.file_size_too_big = c.file.size > c.visual.cut_off_limit_file |
|
696 | 703 | |
|
697 | 704 | if not (c.file_size_too_big or c.file.is_binary): |
|
698 | 705 | if c.annotate: # annotation has precedence over renderer |
|
699 | 706 | c.annotated_lines = filenode_as_annotated_lines_tokens( |
|
700 | 707 | c.file |
|
701 | 708 | ) |
|
702 | 709 | else: |
|
703 | 710 | c.renderer = ( |
|
704 | 711 | c.renderer and h.renderer_from_filename(c.file.path) |
|
705 | 712 | ) |
|
706 | 713 | if not c.renderer: |
|
707 | 714 | c.lines = filenode_as_lines_tokens(c.file) |
|
708 | 715 | |
|
709 | 716 | _branch_name, _sha_commit_id, is_head = \ |
|
710 | 717 | self._is_valid_head(commit_id, self.rhodecode_vcs_repo, |
|
711 | 718 | landing_ref=self.db_repo.landing_ref_name) |
|
712 | 719 | c.on_branch_head = is_head |
|
713 | 720 | |
|
714 | 721 | branch = c.commit.branch if ( |
|
715 | 722 | c.commit.branch and '/' not in c.commit.branch) else None |
|
716 | 723 | c.branch_or_raw_id = branch or c.commit.raw_id |
|
717 | 724 | c.branch_name = c.commit.branch or h.short_id(c.commit.raw_id) |
|
718 | 725 | |
|
719 | 726 | author = c.file_last_commit.author |
|
720 | 727 | c.authors = [[ |
|
721 | 728 | h.email(author), |
|
722 | 729 | h.person(author, 'username_or_name_or_email'), |
|
723 | 730 | 1 |
|
724 | 731 | ]] |
|
725 | 732 | |
|
726 | 733 | else: # load tree content at path |
|
727 | 734 | c.file_source_page = 'false' |
|
728 | 735 | c.authors = [] |
|
729 | 736 | # this loads a simple tree without metadata to speed things up |
|
730 | 737 | # later via ajax we call repo_nodetree_full and fetch whole |
|
731 | 738 | c.file_tree = self._get_tree_at_commit(c, c.commit.raw_id, f_path, at_rev=at_rev) |
|
732 | 739 | |
|
733 | 740 | c.readme_data, c.readme_file = \ |
|
734 | 741 | self._get_readme_data(self.db_repo, c.visual.default_renderer, |
|
735 | 742 | c.commit.raw_id, f_path) |
|
736 | 743 | |
|
737 | 744 | except RepositoryError as e: |
|
738 | 745 | h.flash(h.escape(safe_str(e)), category='error') |
|
739 | 746 | raise HTTPNotFound() |
|
740 | 747 | |
|
741 | 748 | if self.request.environ.get('HTTP_X_PJAX'): |
|
742 | 749 | html = render('rhodecode:templates/files/files_pjax.mako', |
|
743 | 750 | self._get_template_context(c), self.request) |
|
744 | 751 | else: |
|
745 | 752 | html = render('rhodecode:templates/files/files.mako', |
|
746 | 753 | self._get_template_context(c), self.request) |
|
747 | 754 | return Response(html) |
|
748 | 755 | |
|
749 | 756 | @HasRepoPermissionAnyDecorator( |
|
750 | 757 | 'repository.read', 'repository.write', 'repository.admin') |
|
751 | 758 | def repo_files_annotated_previous(self): |
|
752 | 759 | self.load_default_context() |
|
753 | 760 | |
|
754 | 761 | commit_id, f_path = self._get_commit_and_path() |
|
755 | 762 | commit = self._get_commit_or_redirect(commit_id) |
|
756 | 763 | prev_commit_id = commit.raw_id |
|
757 | 764 | line_anchor = self.request.GET.get('line_anchor') |
|
758 | 765 | is_file = False |
|
759 | 766 | try: |
|
760 | 767 | _file = commit.get_node(f_path) |
|
761 | 768 | is_file = _file.is_file() |
|
762 | 769 | except (NodeDoesNotExistError, CommitDoesNotExistError, VCSError): |
|
763 | 770 | pass |
|
764 | 771 | |
|
765 | 772 | if is_file: |
|
766 | 773 | history = commit.get_path_history(f_path) |
|
767 | 774 | prev_commit_id = history[1].raw_id \ |
|
768 | 775 | if len(history) > 1 else prev_commit_id |
|
769 | 776 | prev_url = h.route_path( |
|
770 | 777 | 'repo_files:annotated', repo_name=self.db_repo_name, |
|
771 | 778 | commit_id=prev_commit_id, f_path=f_path, |
|
772 | 779 | _anchor=f'L{line_anchor}') |
|
773 | 780 | |
|
774 | 781 | raise HTTPFound(prev_url) |
|
775 | 782 | |
|
776 | 783 | @LoginRequired() |
|
777 | 784 | @HasRepoPermissionAnyDecorator( |
|
778 | 785 | 'repository.read', 'repository.write', 'repository.admin') |
|
779 | 786 | def repo_nodetree_full(self): |
|
780 | 787 | """ |
|
781 | 788 | Returns rendered html of file tree that contains commit date, |
|
782 | 789 | author, commit_id for the specified combination of |
|
783 | 790 | repo, commit_id and file path |
|
784 | 791 | """ |
|
785 | 792 | c = self.load_default_context() |
|
786 | 793 | |
|
787 | 794 | commit_id, f_path = self._get_commit_and_path() |
|
788 | 795 | commit = self._get_commit_or_redirect(commit_id) |
|
789 | 796 | try: |
|
790 | 797 | dir_node = commit.get_node(f_path) |
|
791 | 798 | except RepositoryError as e: |
|
792 | 799 | return Response(f'error: {h.escape(safe_str(e))}') |
|
793 | 800 | |
|
794 | 801 | if dir_node.is_file(): |
|
795 | 802 | return Response('') |
|
796 | 803 | |
|
797 | 804 | c.file = dir_node |
|
798 | 805 | c.commit = commit |
|
799 | 806 | at_rev = self.request.GET.get('at') |
|
800 | 807 | |
|
801 | 808 | html = self._get_tree_at_commit( |
|
802 | 809 | c, commit.raw_id, dir_node.path, full_load=True, at_rev=at_rev) |
|
803 | 810 | |
|
804 | 811 | return Response(html) |
|
805 | 812 | |
|
806 | 813 | def _get_attachement_headers(self, f_path): |
|
807 | 814 | f_name = safe_str(f_path.split(Repository.NAME_SEP)[-1]) |
|
808 | 815 | safe_path = f_name.replace('"', '\\"') |
|
809 | 816 | encoded_path = urllib.parse.quote(f_name) |
|
810 | 817 | |
|
811 | 818 | headers = "attachment; " \ |
|
812 | 819 | "filename=\"{}\"; " \ |
|
813 | 820 | "filename*=UTF-8\'\'{}".format(safe_path, encoded_path) |
|
814 | 821 | |
|
815 | 822 | return safe_bytes(headers).decode('latin-1', errors='replace') |
|
816 | 823 | |
|
817 | 824 | @LoginRequired() |
|
818 | 825 | @HasRepoPermissionAnyDecorator( |
|
819 | 826 | 'repository.read', 'repository.write', 'repository.admin') |
|
820 | 827 | def repo_file_raw(self): |
|
821 | 828 | """ |
|
822 | 829 | Action for show as raw, some mimetypes are "rendered", |
|
823 | 830 | those include images, icons. |
|
824 | 831 | """ |
|
825 | 832 | c = self.load_default_context() |
|
826 | 833 | |
|
827 | 834 | commit_id, f_path = self._get_commit_and_path() |
|
828 | 835 | commit = self._get_commit_or_redirect(commit_id) |
|
829 | 836 | file_node = self._get_filenode_or_redirect(commit, f_path) |
|
830 | 837 | |
|
831 | 838 | raw_mimetype_mapping = { |
|
832 | 839 | # map original mimetype to a mimetype used for "show as raw" |
|
833 | 840 | # you can also provide a content-disposition to override the |
|
834 | 841 | # default "attachment" disposition. |
|
835 | 842 | # orig_type: (new_type, new_dispo) |
|
836 | 843 | |
|
837 | 844 | # show images inline: |
|
838 | 845 | # Do not re-add SVG: it is unsafe and permits XSS attacks. One can |
|
839 | 846 | # for example render an SVG with javascript inside or even render |
|
840 | 847 | # HTML. |
|
841 | 848 | 'image/x-icon': ('image/x-icon', 'inline'), |
|
842 | 849 | 'image/png': ('image/png', 'inline'), |
|
843 | 850 | 'image/gif': ('image/gif', 'inline'), |
|
844 | 851 | 'image/jpeg': ('image/jpeg', 'inline'), |
|
845 | 852 | 'application/pdf': ('application/pdf', 'inline'), |
|
846 | 853 | } |
|
847 | 854 | |
|
848 | 855 | mimetype = file_node.mimetype |
|
849 | 856 | try: |
|
850 | 857 | mimetype, disposition = raw_mimetype_mapping[mimetype] |
|
851 | 858 | except KeyError: |
|
852 | 859 | # we don't know anything special about this, handle it safely |
|
853 | 860 | if file_node.is_binary: |
|
854 | 861 | # do same as download raw for binary files |
|
855 | 862 | mimetype, disposition = 'application/octet-stream', 'attachment' |
|
856 | 863 | else: |
|
857 | 864 | # do not just use the original mimetype, but force text/plain, |
|
858 | 865 | # otherwise it would serve text/html and that might be unsafe. |
|
859 | 866 | # Note: underlying vcs library fakes text/plain mimetype if the |
|
860 | 867 | # mimetype can not be determined and it thinks it is not |
|
861 | 868 | # binary.This might lead to erroneous text display in some |
|
862 | 869 | # cases, but helps in other cases, like with text files |
|
863 | 870 | # without extension. |
|
864 | 871 | mimetype, disposition = 'text/plain', 'inline' |
|
865 | 872 | |
|
866 | 873 | if disposition == 'attachment': |
|
867 | 874 | disposition = self._get_attachement_headers(f_path) |
|
868 | 875 | |
|
869 | 876 | stream_content = file_node.stream_bytes() |
|
870 | 877 | |
|
871 | 878 | response = Response(app_iter=stream_content) |
|
872 | 879 | response.content_disposition = disposition |
|
873 | 880 | response.content_type = mimetype |
|
874 | 881 | |
|
875 | 882 | charset = self._get_default_encoding(c) |
|
876 | 883 | if charset: |
|
877 | 884 | response.charset = charset |
|
878 | 885 | |
|
879 | 886 | return response |
|
880 | 887 | |
|
881 | 888 | @LoginRequired() |
|
882 | 889 | @HasRepoPermissionAnyDecorator( |
|
883 | 890 | 'repository.read', 'repository.write', 'repository.admin') |
|
884 | 891 | def repo_file_download(self): |
|
885 | 892 | c = self.load_default_context() |
|
886 | 893 | |
|
887 | 894 | commit_id, f_path = self._get_commit_and_path() |
|
888 | 895 | commit = self._get_commit_or_redirect(commit_id) |
|
889 | 896 | file_node = self._get_filenode_or_redirect(commit, f_path) |
|
890 | 897 | |
|
891 | 898 | if self.request.GET.get('lf'): |
|
892 | 899 | # only if lf get flag is passed, we download this file |
|
893 | 900 | # as LFS/Largefile |
|
894 | 901 | lf_node = file_node.get_largefile_node() |
|
895 | 902 | if lf_node: |
|
896 | 903 | # overwrite our pointer with the REAL large-file |
|
897 | 904 | file_node = lf_node |
|
898 | 905 | |
|
899 | 906 | disposition = self._get_attachement_headers(f_path) |
|
900 | 907 | |
|
901 | 908 | stream_content = file_node.stream_bytes() |
|
902 | 909 | |
|
903 | 910 | response = Response(app_iter=stream_content) |
|
904 | 911 | response.content_disposition = disposition |
|
905 | 912 | response.content_type = file_node.mimetype |
|
906 | 913 | |
|
907 | 914 | charset = self._get_default_encoding(c) |
|
908 | 915 | if charset: |
|
909 | 916 | response.charset = charset |
|
910 | 917 | |
|
911 | 918 | return response |
|
912 | 919 | |
|
913 | 920 | def _get_nodelist_at_commit(self, repo_name, repo_id, commit_id, f_path): |
|
914 | 921 | |
|
915 | 922 | cache_seconds = safe_int( |
|
916 | 923 | rhodecode.CONFIG.get('rc_cache.cache_repo.expiration_time')) |
|
917 | 924 | cache_on = cache_seconds > 0 |
|
918 | 925 | log.debug( |
|
919 | 926 | 'Computing FILE SEARCH for repo_id %s commit_id `%s` and path `%s`' |
|
920 | 927 | 'with caching: %s[TTL: %ss]' % ( |
|
921 | 928 | repo_id, commit_id, f_path, cache_on, cache_seconds or 0)) |
|
922 | 929 | |
|
923 | 930 | cache_namespace_uid = f'repo.{repo_id}' |
|
924 | 931 | region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid) |
|
925 | 932 | |
|
926 | 933 | @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache_on) |
|
927 | 934 | def compute_file_search(_name_hash, _repo_id, _commit_id, _f_path): |
|
928 | 935 | log.debug('Generating cached nodelist for repo_id:%s, %s, %s', |
|
929 | 936 | _repo_id, commit_id, f_path) |
|
930 | 937 | try: |
|
931 | 938 | _d, _f = ScmModel().get_quick_filter_nodes(repo_name, _commit_id, _f_path) |
|
932 | 939 | except (RepositoryError, CommitDoesNotExistError, Exception) as e: |
|
933 | 940 | log.exception(safe_str(e)) |
|
934 | 941 | h.flash(h.escape(safe_str(e)), category='error') |
|
935 | 942 | raise HTTPFound(h.route_path( |
|
936 | 943 | 'repo_files', repo_name=self.db_repo_name, |
|
937 | 944 | commit_id='tip', f_path='/')) |
|
938 | 945 | |
|
939 | 946 | return _d + _f |
|
940 | 947 | |
|
941 | 948 | result = compute_file_search(self.db_repo.repo_name_hash, self.db_repo.repo_id, |
|
942 | 949 | commit_id, f_path) |
|
943 | 950 | return filter(lambda n: self.path_filter.path_access_allowed(n['name']), result) |
|
944 | 951 | |
|
945 | 952 | @LoginRequired() |
|
946 | 953 | @HasRepoPermissionAnyDecorator( |
|
947 | 954 | 'repository.read', 'repository.write', 'repository.admin') |
|
948 | 955 | def repo_nodelist(self): |
|
949 | 956 | self.load_default_context() |
|
950 | 957 | |
|
951 | 958 | commit_id, f_path = self._get_commit_and_path() |
|
952 | 959 | commit = self._get_commit_or_redirect(commit_id) |
|
953 | 960 | |
|
954 | 961 | metadata = self._get_nodelist_at_commit( |
|
955 | 962 | self.db_repo_name, self.db_repo.repo_id, commit.raw_id, f_path) |
|
956 | 963 | return {'nodes': [x for x in metadata]} |
|
957 | 964 | |
|
958 | 965 | def _create_references(self, branches_or_tags, symbolic_reference, f_path, ref_type): |
|
959 | 966 | items = [] |
|
960 | 967 | for name, commit_id in branches_or_tags.items(): |
|
961 | 968 | sym_ref = symbolic_reference(commit_id, name, f_path, ref_type) |
|
962 | 969 | items.append((sym_ref, name, ref_type)) |
|
963 | 970 | return items |
|
964 | 971 | |
|
965 | 972 | def _symbolic_reference(self, commit_id, name, f_path, ref_type): |
|
966 | 973 | return commit_id |
|
967 | 974 | |
|
968 | 975 | def _symbolic_reference_svn(self, commit_id, name, f_path, ref_type): |
|
969 | 976 | return commit_id |
|
970 | 977 | |
|
971 | 978 | # NOTE(dan): old code we used in "diff" mode compare |
|
972 | 979 | new_f_path = vcspath.join(name, f_path) |
|
973 | 980 | return f'{new_f_path}@{commit_id}' |
|
974 | 981 | |
|
975 | 982 | def _get_node_history(self, commit_obj, f_path, commits=None): |
|
976 | 983 | """ |
|
977 | 984 | get commit history for given node |
|
978 | 985 | |
|
979 | 986 | :param commit_obj: commit to calculate history |
|
980 | 987 | :param f_path: path for node to calculate history for |
|
981 | 988 | :param commits: if passed don't calculate history and take |
|
982 | 989 | commits defined in this list |
|
983 | 990 | """ |
|
984 | 991 | _ = self.request.translate |
|
985 | 992 | |
|
986 | 993 | # calculate history based on tip |
|
987 | 994 | tip = self.rhodecode_vcs_repo.get_commit() |
|
988 | 995 | if commits is None: |
|
989 | 996 | pre_load = ["author", "branch"] |
|
990 | 997 | try: |
|
991 | 998 | commits = tip.get_path_history(f_path, pre_load=pre_load) |
|
992 | 999 | except (NodeDoesNotExistError, CommitError): |
|
993 | 1000 | # this node is not present at tip! |
|
994 | 1001 | commits = commit_obj.get_path_history(f_path, pre_load=pre_load) |
|
995 | 1002 | |
|
996 | 1003 | history = [] |
|
997 | 1004 | commits_group = ([], _("Changesets")) |
|
998 | 1005 | for commit in commits: |
|
999 | 1006 | branch = ' (%s)' % commit.branch if commit.branch else '' |
|
1000 | 1007 | n_desc = f'r{commit.idx}:{commit.short_id}{branch}' |
|
1001 | 1008 | commits_group[0].append((commit.raw_id, n_desc, 'sha')) |
|
1002 | 1009 | history.append(commits_group) |
|
1003 | 1010 | |
|
1004 | 1011 | symbolic_reference = self._symbolic_reference |
|
1005 | 1012 | |
|
1006 | 1013 | if self.rhodecode_vcs_repo.alias == 'svn': |
|
1007 | 1014 | adjusted_f_path = RepoFilesView.adjust_file_path_for_svn( |
|
1008 | 1015 | f_path, self.rhodecode_vcs_repo) |
|
1009 | 1016 | if adjusted_f_path != f_path: |
|
1010 | 1017 | log.debug( |
|
1011 | 1018 | 'Recognized svn tag or branch in file "%s", using svn ' |
|
1012 | 1019 | 'specific symbolic references', f_path) |
|
1013 | 1020 | f_path = adjusted_f_path |
|
1014 | 1021 | symbolic_reference = self._symbolic_reference_svn |
|
1015 | 1022 | |
|
1016 | 1023 | branches = self._create_references( |
|
1017 | 1024 | self.rhodecode_vcs_repo.branches, symbolic_reference, f_path, 'branch') |
|
1018 | 1025 | branches_group = (branches, _("Branches")) |
|
1019 | 1026 | |
|
1020 | 1027 | tags = self._create_references( |
|
1021 | 1028 | self.rhodecode_vcs_repo.tags, symbolic_reference, f_path, 'tag') |
|
1022 | 1029 | tags_group = (tags, _("Tags")) |
|
1023 | 1030 | |
|
1024 | 1031 | history.append(branches_group) |
|
1025 | 1032 | history.append(tags_group) |
|
1026 | 1033 | |
|
1027 | 1034 | return history, commits |
|
1028 | 1035 | |
|
1029 | 1036 | @LoginRequired() |
|
1030 | 1037 | @HasRepoPermissionAnyDecorator( |
|
1031 | 1038 | 'repository.read', 'repository.write', 'repository.admin') |
|
1032 | 1039 | def repo_file_history(self): |
|
1033 | 1040 | self.load_default_context() |
|
1034 | 1041 | |
|
1035 | 1042 | commit_id, f_path = self._get_commit_and_path() |
|
1036 | 1043 | commit = self._get_commit_or_redirect(commit_id) |
|
1037 | 1044 | file_node = self._get_filenode_or_redirect(commit, f_path) |
|
1038 | 1045 | |
|
1039 | 1046 | if file_node.is_file(): |
|
1040 | 1047 | file_history, _hist = self._get_node_history(commit, f_path) |
|
1041 | 1048 | |
|
1042 | 1049 | res = [] |
|
1043 | 1050 | for section_items, section in file_history: |
|
1044 | 1051 | items = [] |
|
1045 | 1052 | for obj_id, obj_text, obj_type in section_items: |
|
1046 | 1053 | at_rev = '' |
|
1047 | 1054 | if obj_type in ['branch', 'bookmark', 'tag']: |
|
1048 | 1055 | at_rev = obj_text |
|
1049 | 1056 | entry = { |
|
1050 | 1057 | 'id': obj_id, |
|
1051 | 1058 | 'text': obj_text, |
|
1052 | 1059 | 'type': obj_type, |
|
1053 | 1060 | 'at_rev': at_rev |
|
1054 | 1061 | } |
|
1055 | 1062 | |
|
1056 | 1063 | items.append(entry) |
|
1057 | 1064 | |
|
1058 | 1065 | res.append({ |
|
1059 | 1066 | 'text': section, |
|
1060 | 1067 | 'children': items |
|
1061 | 1068 | }) |
|
1062 | 1069 | |
|
1063 | 1070 | data = { |
|
1064 | 1071 | 'more': False, |
|
1065 | 1072 | 'results': res |
|
1066 | 1073 | } |
|
1067 | 1074 | return data |
|
1068 | 1075 | |
|
1069 | 1076 | log.warning('Cannot fetch history for directory') |
|
1070 | 1077 | raise HTTPBadRequest() |
|
1071 | 1078 | |
|
1072 | 1079 | @LoginRequired() |
|
1073 | 1080 | @HasRepoPermissionAnyDecorator( |
|
1074 | 1081 | 'repository.read', 'repository.write', 'repository.admin') |
|
1075 | 1082 | def repo_file_authors(self): |
|
1076 | 1083 | c = self.load_default_context() |
|
1077 | 1084 | |
|
1078 | 1085 | commit_id, f_path = self._get_commit_and_path() |
|
1079 | 1086 | commit = self._get_commit_or_redirect(commit_id) |
|
1080 | 1087 | file_node = self._get_filenode_or_redirect(commit, f_path) |
|
1081 | 1088 | |
|
1082 | 1089 | if not file_node.is_file(): |
|
1083 | 1090 | raise HTTPBadRequest() |
|
1084 | 1091 | |
|
1085 | 1092 | c.file_last_commit = file_node.last_commit |
|
1086 | 1093 | if self.request.GET.get('annotate') == '1': |
|
1087 | 1094 | # use _hist from annotation if annotation mode is on |
|
1088 | 1095 | commit_ids = {x[1] for x in file_node.annotate} |
|
1089 | 1096 | _hist = ( |
|
1090 | 1097 | self.rhodecode_vcs_repo.get_commit(commit_id) |
|
1091 | 1098 | for commit_id in commit_ids) |
|
1092 | 1099 | else: |
|
1093 | 1100 | _f_history, _hist = self._get_node_history(commit, f_path) |
|
1094 | 1101 | c.file_author = False |
|
1095 | 1102 | |
|
1096 | 1103 | unique = collections.OrderedDict() |
|
1097 | 1104 | for commit in _hist: |
|
1098 | 1105 | author = commit.author |
|
1099 | 1106 | if author not in unique: |
|
1100 | 1107 | unique[commit.author] = [ |
|
1101 | 1108 | h.email(author), |
|
1102 | 1109 | h.person(author, 'username_or_name_or_email'), |
|
1103 | 1110 | 1 # counter |
|
1104 | 1111 | ] |
|
1105 | 1112 | |
|
1106 | 1113 | else: |
|
1107 | 1114 | # increase counter |
|
1108 | 1115 | unique[commit.author][2] += 1 |
|
1109 | 1116 | |
|
1110 | 1117 | c.authors = [val for val in unique.values()] |
|
1111 | 1118 | |
|
1112 | 1119 | return self._get_template_context(c) |
|
1113 | 1120 | |
|
1114 | 1121 | @LoginRequired() |
|
1115 | 1122 | @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin') |
|
1116 | 1123 | def repo_files_check_head(self): |
|
1117 | 1124 | self.load_default_context() |
|
1118 | 1125 | |
|
1119 | 1126 | commit_id, f_path = self._get_commit_and_path() |
|
1120 | 1127 | _branch_name, _sha_commit_id, is_head = \ |
|
1121 | 1128 | self._is_valid_head(commit_id, self.rhodecode_vcs_repo, |
|
1122 | 1129 | landing_ref=self.db_repo.landing_ref_name) |
|
1123 | 1130 | |
|
1124 | 1131 | new_path = self.request.POST.get('path') |
|
1125 | 1132 | operation = self.request.POST.get('operation') |
|
1126 | 1133 | path_exist = '' |
|
1127 | 1134 | |
|
1128 | 1135 | if new_path and operation in ['create', 'upload']: |
|
1129 | 1136 | new_f_path = os.path.join(f_path.lstrip('/'), new_path) |
|
1130 | 1137 | try: |
|
1131 | 1138 | commit_obj = self.rhodecode_vcs_repo.get_commit(commit_id) |
|
1132 | 1139 | # NOTE(dan): construct whole path without leading / |
|
1133 | 1140 | file_node = commit_obj.get_node(new_f_path) |
|
1134 | 1141 | if file_node is not None: |
|
1135 | 1142 | path_exist = new_f_path |
|
1136 | 1143 | except EmptyRepositoryError: |
|
1137 | 1144 | pass |
|
1138 | 1145 | except Exception: |
|
1139 | 1146 | pass |
|
1140 | 1147 | |
|
1141 | 1148 | return { |
|
1142 | 1149 | 'branch': _branch_name, |
|
1143 | 1150 | 'sha': _sha_commit_id, |
|
1144 | 1151 | 'is_head': is_head, |
|
1145 | 1152 | 'path_exists': path_exist |
|
1146 | 1153 | } |
|
1147 | 1154 | |
|
1148 | 1155 | @LoginRequired() |
|
1149 | 1156 | @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin') |
|
1150 | 1157 | def repo_files_remove_file(self): |
|
1151 | 1158 | _ = self.request.translate |
|
1152 | 1159 | c = self.load_default_context() |
|
1153 | 1160 | commit_id, f_path = self._get_commit_and_path() |
|
1154 | 1161 | |
|
1155 | 1162 | self._ensure_not_locked() |
|
1156 | 1163 | _branch_name, _sha_commit_id, is_head = \ |
|
1157 | 1164 | self._is_valid_head(commit_id, self.rhodecode_vcs_repo, |
|
1158 | 1165 | landing_ref=self.db_repo.landing_ref_name) |
|
1159 | 1166 | |
|
1160 | 1167 | self.forbid_non_head(is_head, f_path) |
|
1161 | 1168 | self.check_branch_permission(_branch_name) |
|
1162 | 1169 | |
|
1163 | 1170 | c.commit = self._get_commit_or_redirect(commit_id) |
|
1164 | 1171 | c.file = self._get_filenode_or_redirect(c.commit, f_path) |
|
1165 | 1172 | |
|
1166 | 1173 | c.default_message = _( |
|
1167 | 1174 | 'Deleted file {} via RhodeCode Enterprise').format(f_path) |
|
1168 | 1175 | c.f_path = f_path |
|
1169 | 1176 | |
|
1170 | 1177 | return self._get_template_context(c) |
|
1171 | 1178 | |
|
1172 | 1179 | @LoginRequired() |
|
1173 | 1180 | @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin') |
|
1174 | 1181 | @CSRFRequired() |
|
1175 | 1182 | def repo_files_delete_file(self): |
|
1176 | 1183 | _ = self.request.translate |
|
1177 | 1184 | |
|
1178 | 1185 | c = self.load_default_context() |
|
1179 | 1186 | commit_id, f_path = self._get_commit_and_path() |
|
1180 | 1187 | |
|
1181 | 1188 | self._ensure_not_locked() |
|
1182 | 1189 | _branch_name, _sha_commit_id, is_head = \ |
|
1183 | 1190 | self._is_valid_head(commit_id, self.rhodecode_vcs_repo, |
|
1184 | 1191 | landing_ref=self.db_repo.landing_ref_name) |
|
1185 | 1192 | |
|
1186 | 1193 | self.forbid_non_head(is_head, f_path) |
|
1187 | 1194 | self.check_branch_permission(_branch_name) |
|
1188 | 1195 | |
|
1189 | 1196 | c.commit = self._get_commit_or_redirect(commit_id) |
|
1190 | 1197 | c.file = self._get_filenode_or_redirect(c.commit, f_path) |
|
1191 | 1198 | |
|
1192 | 1199 | c.default_message = _( |
|
1193 | 1200 | 'Deleted file {} via RhodeCode Enterprise').format(f_path) |
|
1194 | 1201 | c.f_path = f_path |
|
1195 | 1202 | node_path = f_path |
|
1196 | 1203 | author = self._rhodecode_db_user.full_contact |
|
1197 | 1204 | message = self.request.POST.get('message') or c.default_message |
|
1198 | 1205 | try: |
|
1199 | 1206 | nodes = { |
|
1200 | 1207 | safe_bytes(node_path): { |
|
1201 | 1208 | 'content': b'' |
|
1202 | 1209 | } |
|
1203 | 1210 | } |
|
1204 | 1211 | ScmModel().delete_nodes( |
|
1205 | 1212 | user=self._rhodecode_db_user.user_id, repo=self.db_repo, |
|
1206 | 1213 | message=message, |
|
1207 | 1214 | nodes=nodes, |
|
1208 | 1215 | parent_commit=c.commit, |
|
1209 | 1216 | author=author, |
|
1210 | 1217 | ) |
|
1211 | 1218 | |
|
1212 | 1219 | h.flash( |
|
1213 | 1220 | _('Successfully deleted file `{}`').format( |
|
1214 | 1221 | h.escape(f_path)), category='success') |
|
1215 | 1222 | except Exception: |
|
1216 | 1223 | log.exception('Error during commit operation') |
|
1217 | 1224 | h.flash(_('Error occurred during commit'), category='error') |
|
1218 | 1225 | raise HTTPFound( |
|
1219 | 1226 | h.route_path('repo_commit', repo_name=self.db_repo_name, |
|
1220 | 1227 | commit_id='tip')) |
|
1221 | 1228 | |
|
1222 | 1229 | @LoginRequired() |
|
1223 | 1230 | @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin') |
|
1224 | 1231 | def repo_files_edit_file(self): |
|
1225 | 1232 | _ = self.request.translate |
|
1226 | 1233 | c = self.load_default_context() |
|
1227 | 1234 | commit_id, f_path = self._get_commit_and_path() |
|
1228 | 1235 | |
|
1229 | 1236 | self._ensure_not_locked() |
|
1230 | 1237 | _branch_name, _sha_commit_id, is_head = \ |
|
1231 | 1238 | self._is_valid_head(commit_id, self.rhodecode_vcs_repo, |
|
1232 | 1239 | landing_ref=self.db_repo.landing_ref_name) |
|
1233 | 1240 | |
|
1234 | 1241 | self.forbid_non_head(is_head, f_path, commit_id=commit_id) |
|
1235 | 1242 | self.check_branch_permission(_branch_name, commit_id=commit_id) |
|
1236 | 1243 | |
|
1237 | 1244 | c.commit = self._get_commit_or_redirect(commit_id) |
|
1238 | 1245 | c.file = self._get_filenode_or_redirect(c.commit, f_path) |
|
1239 | 1246 | |
|
1240 | 1247 | if c.file.is_binary: |
|
1241 | 1248 | files_url = h.route_path( |
|
1242 | 1249 | 'repo_files', |
|
1243 | 1250 | repo_name=self.db_repo_name, |
|
1244 | 1251 | commit_id=c.commit.raw_id, f_path=f_path) |
|
1245 | 1252 | raise HTTPFound(files_url) |
|
1246 | 1253 | |
|
1247 | 1254 | c.default_message = _('Edited file {} via RhodeCode Enterprise').format(f_path) |
|
1248 | 1255 | c.f_path = f_path |
|
1249 | 1256 | |
|
1250 | 1257 | return self._get_template_context(c) |
|
1251 | 1258 | |
|
1252 | 1259 | @LoginRequired() |
|
1253 | 1260 | @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin') |
|
1254 | 1261 | @CSRFRequired() |
|
1255 | 1262 | def repo_files_update_file(self): |
|
1256 | 1263 | _ = self.request.translate |
|
1257 | 1264 | c = self.load_default_context() |
|
1258 | 1265 | commit_id, f_path = self._get_commit_and_path() |
|
1259 | 1266 | |
|
1260 | 1267 | self._ensure_not_locked() |
|
1261 | 1268 | |
|
1262 | 1269 | c.commit = self._get_commit_or_redirect(commit_id) |
|
1263 | 1270 | c.file = self._get_filenode_or_redirect(c.commit, f_path) |
|
1264 | 1271 | |
|
1265 | 1272 | if c.file.is_binary: |
|
1266 | 1273 | raise HTTPFound(h.route_path('repo_files', repo_name=self.db_repo_name, |
|
1267 | 1274 | commit_id=c.commit.raw_id, f_path=f_path)) |
|
1268 | 1275 | |
|
1269 | 1276 | _branch_name, _sha_commit_id, is_head = \ |
|
1270 | 1277 | self._is_valid_head(commit_id, self.rhodecode_vcs_repo, |
|
1271 | 1278 | landing_ref=self.db_repo.landing_ref_name) |
|
1272 | 1279 | |
|
1273 | 1280 | self.forbid_non_head(is_head, f_path, commit_id=commit_id) |
|
1274 | 1281 | self.check_branch_permission(_branch_name, commit_id=commit_id) |
|
1275 | 1282 | |
|
1276 | 1283 | c.default_message = _('Edited file {} via RhodeCode Enterprise').format(f_path) |
|
1277 | 1284 | c.f_path = f_path |
|
1278 | 1285 | |
|
1279 | 1286 | old_content = c.file.str_content |
|
1280 | 1287 | sl = old_content.splitlines(1) |
|
1281 | 1288 | first_line = sl[0] if sl else '' |
|
1282 | 1289 | |
|
1283 | 1290 | r_post = self.request.POST |
|
1284 | 1291 | # line endings: 0 - Unix, 1 - Mac, 2 - DOS |
|
1285 | 1292 | line_ending_mode = detect_mode(first_line, 0) |
|
1286 | 1293 | content = convert_line_endings(r_post.get('content', ''), line_ending_mode) |
|
1287 | 1294 | |
|
1288 | 1295 | message = r_post.get('message') or c.default_message |
|
1289 | 1296 | |
|
1290 | 1297 | org_node_path = c.file.str_path |
|
1291 | 1298 | filename = r_post['filename'] |
|
1292 | 1299 | |
|
1293 | 1300 | root_path = c.file.dir_path |
|
1294 | 1301 | pure_path = self.create_pure_path(root_path, filename) |
|
1295 | 1302 | node_path = pure_path.as_posix() |
|
1296 | 1303 | |
|
1297 | 1304 | default_redirect_url = h.route_path('repo_commit', repo_name=self.db_repo_name, |
|
1298 | 1305 | commit_id=commit_id) |
|
1299 | 1306 | if content == old_content and node_path == org_node_path: |
|
1300 | 1307 | h.flash(_('No changes detected on {}').format(h.escape(org_node_path)), |
|
1301 | 1308 | category='warning') |
|
1302 | 1309 | raise HTTPFound(default_redirect_url) |
|
1303 | 1310 | |
|
1304 | 1311 | try: |
|
1305 | 1312 | mapping = { |
|
1306 | 1313 | c.file.bytes_path: { |
|
1307 | 1314 | 'org_filename': org_node_path, |
|
1308 | 1315 | 'filename': safe_bytes(node_path), |
|
1309 | 1316 | 'content': safe_bytes(content), |
|
1310 | 1317 | 'lexer': '', |
|
1311 | 1318 | 'op': 'mod', |
|
1312 | 1319 | 'mode': c.file.mode |
|
1313 | 1320 | } |
|
1314 | 1321 | } |
|
1315 | 1322 | |
|
1316 | 1323 | commit = ScmModel().update_nodes( |
|
1317 | 1324 | user=self._rhodecode_db_user.user_id, |
|
1318 | 1325 | repo=self.db_repo, |
|
1319 | 1326 | message=message, |
|
1320 | 1327 | nodes=mapping, |
|
1321 | 1328 | parent_commit=c.commit, |
|
1322 | 1329 | ) |
|
1323 | 1330 | |
|
1324 | 1331 | h.flash(_('Successfully committed changes to file `{}`').format( |
|
1325 | 1332 | h.escape(f_path)), category='success') |
|
1326 | 1333 | default_redirect_url = h.route_path( |
|
1327 | 1334 | 'repo_commit', repo_name=self.db_repo_name, commit_id=commit.raw_id) |
|
1328 | 1335 | |
|
1329 | 1336 | except Exception: |
|
1330 | 1337 | log.exception('Error occurred during commit') |
|
1331 | 1338 | h.flash(_('Error occurred during commit'), category='error') |
|
1332 | 1339 | |
|
1333 | 1340 | raise HTTPFound(default_redirect_url) |
|
1334 | 1341 | |
|
1335 | 1342 | @LoginRequired() |
|
1336 | 1343 | @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin') |
|
1337 | 1344 | def repo_files_add_file(self): |
|
1338 | 1345 | _ = self.request.translate |
|
1339 | 1346 | c = self.load_default_context() |
|
1340 | 1347 | commit_id, f_path = self._get_commit_and_path() |
|
1341 | 1348 | |
|
1342 | 1349 | self._ensure_not_locked() |
|
1343 | 1350 | |
|
1344 | 1351 | # Check if we need to use this page to upload binary |
|
1345 | 1352 | upload_binary = str2bool(self.request.params.get('upload_binary', False)) |
|
1346 | 1353 | |
|
1347 | 1354 | c.commit = self._get_commit_or_redirect(commit_id, redirect_after=False) |
|
1348 | 1355 | if c.commit is None: |
|
1349 | 1356 | c.commit = EmptyCommit(alias=self.rhodecode_vcs_repo.alias) |
|
1350 | 1357 | |
|
1351 | 1358 | if self.rhodecode_vcs_repo.is_empty(): |
|
1352 | 1359 | # for empty repository we cannot check for current branch, we rely on |
|
1353 | 1360 | # c.commit.branch instead |
|
1354 | 1361 | _branch_name, _sha_commit_id, is_head = c.commit.branch, '', True |
|
1355 | 1362 | else: |
|
1356 | 1363 | _branch_name, _sha_commit_id, is_head = \ |
|
1357 | 1364 | self._is_valid_head(commit_id, self.rhodecode_vcs_repo, |
|
1358 | 1365 | landing_ref=self.db_repo.landing_ref_name) |
|
1359 | 1366 | |
|
1360 | 1367 | self.forbid_non_head(is_head, f_path, commit_id=commit_id) |
|
1361 | 1368 | self.check_branch_permission(_branch_name, commit_id=commit_id) |
|
1362 | 1369 | |
|
1363 | 1370 | c.default_message = (_('Added file via RhodeCode Enterprise')) \ |
|
1364 | 1371 | if not upload_binary else (_('Edited file {} via RhodeCode Enterprise').format(f_path)) |
|
1365 | 1372 | c.f_path = f_path.lstrip('/') # ensure not relative path |
|
1366 | 1373 | c.replace_binary = upload_binary |
|
1367 | 1374 | |
|
1368 | 1375 | return self._get_template_context(c) |
|
1369 | 1376 | |
|
1370 | 1377 | @LoginRequired() |
|
1371 | 1378 | @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin') |
|
1372 | 1379 | @CSRFRequired() |
|
1373 | 1380 | def repo_files_create_file(self): |
|
1374 | 1381 | _ = self.request.translate |
|
1375 | 1382 | c = self.load_default_context() |
|
1376 | 1383 | commit_id, f_path = self._get_commit_and_path() |
|
1377 | 1384 | |
|
1378 | 1385 | self._ensure_not_locked() |
|
1379 | 1386 | |
|
1380 | 1387 | c.commit = self._get_commit_or_redirect(commit_id, redirect_after=False) |
|
1381 | 1388 | if c.commit is None: |
|
1382 | 1389 | c.commit = EmptyCommit(alias=self.rhodecode_vcs_repo.alias) |
|
1383 | 1390 | |
|
1384 | 1391 | # calculate redirect URL |
|
1385 | 1392 | if self.rhodecode_vcs_repo.is_empty(): |
|
1386 | 1393 | default_redirect_url = h.route_path( |
|
1387 | 1394 | 'repo_summary', repo_name=self.db_repo_name) |
|
1388 | 1395 | else: |
|
1389 | 1396 | default_redirect_url = h.route_path( |
|
1390 | 1397 | 'repo_commit', repo_name=self.db_repo_name, commit_id='tip') |
|
1391 | 1398 | |
|
1392 | 1399 | if self.rhodecode_vcs_repo.is_empty(): |
|
1393 | 1400 | # for empty repository we cannot check for current branch, we rely on |
|
1394 | 1401 | # c.commit.branch instead |
|
1395 | 1402 | _branch_name, _sha_commit_id, is_head = c.commit.branch, '', True |
|
1396 | 1403 | else: |
|
1397 | 1404 | _branch_name, _sha_commit_id, is_head = \ |
|
1398 | 1405 | self._is_valid_head(commit_id, self.rhodecode_vcs_repo, |
|
1399 | 1406 | landing_ref=self.db_repo.landing_ref_name) |
|
1400 | 1407 | |
|
1401 | 1408 | self.forbid_non_head(is_head, f_path, commit_id=commit_id) |
|
1402 | 1409 | self.check_branch_permission(_branch_name, commit_id=commit_id) |
|
1403 | 1410 | |
|
1404 | 1411 | c.default_message = (_('Added file via RhodeCode Enterprise')) |
|
1405 | 1412 | c.f_path = f_path |
|
1406 | 1413 | |
|
1407 | 1414 | r_post = self.request.POST |
|
1408 | 1415 | message = r_post.get('message') or c.default_message |
|
1409 | 1416 | filename = r_post.get('filename') |
|
1410 | 1417 | unix_mode = 0 |
|
1411 | 1418 | |
|
1412 | 1419 | if not filename: |
|
1413 | 1420 | # If there's no commit, redirect to repo summary |
|
1414 | 1421 | if type(c.commit) is EmptyCommit: |
|
1415 | 1422 | redirect_url = h.route_path( |
|
1416 | 1423 | 'repo_summary', repo_name=self.db_repo_name) |
|
1417 | 1424 | else: |
|
1418 | 1425 | redirect_url = default_redirect_url |
|
1419 | 1426 | h.flash(_('No filename specified'), category='warning') |
|
1420 | 1427 | raise HTTPFound(redirect_url) |
|
1421 | 1428 | |
|
1422 | 1429 | root_path = f_path |
|
1423 | 1430 | pure_path = self.create_pure_path(root_path, filename) |
|
1424 | 1431 | node_path = pure_path.as_posix().lstrip('/') |
|
1425 | 1432 | |
|
1426 | 1433 | author = self._rhodecode_db_user.full_contact |
|
1427 | 1434 | content = convert_line_endings(r_post.get('content', ''), unix_mode) |
|
1428 | 1435 | nodes = { |
|
1429 | 1436 | safe_bytes(node_path): { |
|
1430 | 1437 | 'content': safe_bytes(content) |
|
1431 | 1438 | } |
|
1432 | 1439 | } |
|
1433 | 1440 | |
|
1434 | 1441 | try: |
|
1435 | 1442 | |
|
1436 | 1443 | commit = ScmModel().create_nodes( |
|
1437 | 1444 | user=self._rhodecode_db_user.user_id, |
|
1438 | 1445 | repo=self.db_repo, |
|
1439 | 1446 | message=message, |
|
1440 | 1447 | nodes=nodes, |
|
1441 | 1448 | parent_commit=c.commit, |
|
1442 | 1449 | author=author, |
|
1443 | 1450 | ) |
|
1444 | 1451 | |
|
1445 | 1452 | h.flash(_('Successfully committed new file `{}`').format( |
|
1446 | 1453 | h.escape(node_path)), category='success') |
|
1447 | 1454 | |
|
1448 | 1455 | default_redirect_url = h.route_path( |
|
1449 | 1456 | 'repo_commit', repo_name=self.db_repo_name, commit_id=commit.raw_id) |
|
1450 | 1457 | |
|
1451 | 1458 | except NonRelativePathError: |
|
1452 | 1459 | log.exception('Non Relative path found') |
|
1453 | 1460 | h.flash(_('The location specified must be a relative path and must not ' |
|
1454 | 1461 | 'contain .. in the path'), category='warning') |
|
1455 | 1462 | raise HTTPFound(default_redirect_url) |
|
1456 | 1463 | except (NodeError, NodeAlreadyExistsError) as e: |
|
1457 | 1464 | h.flash(h.escape(safe_str(e)), category='error') |
|
1458 | 1465 | except Exception: |
|
1459 | 1466 | log.exception('Error occurred during commit') |
|
1460 | 1467 | h.flash(_('Error occurred during commit'), category='error') |
|
1461 | 1468 | |
|
1462 | 1469 | raise HTTPFound(default_redirect_url) |
|
1463 | 1470 | |
|
1464 | 1471 | @LoginRequired() |
|
1465 | 1472 | @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin') |
|
1466 | 1473 | @CSRFRequired() |
|
1467 | 1474 | def repo_files_upload_file(self): |
|
1468 | 1475 | _ = self.request.translate |
|
1469 | 1476 | c = self.load_default_context() |
|
1470 | 1477 | commit_id, f_path = self._get_commit_and_path() |
|
1471 | 1478 | |
|
1472 | 1479 | self._ensure_not_locked() |
|
1473 | 1480 | |
|
1474 | 1481 | c.commit = self._get_commit_or_redirect(commit_id, redirect_after=False) |
|
1475 | 1482 | if c.commit is None: |
|
1476 | 1483 | c.commit = EmptyCommit(alias=self.rhodecode_vcs_repo.alias) |
|
1477 | 1484 | |
|
1478 | 1485 | # calculate redirect URL |
|
1479 | 1486 | if self.rhodecode_vcs_repo.is_empty(): |
|
1480 | 1487 | default_redirect_url = h.route_path( |
|
1481 | 1488 | 'repo_summary', repo_name=self.db_repo_name) |
|
1482 | 1489 | else: |
|
1483 | 1490 | default_redirect_url = h.route_path( |
|
1484 | 1491 | 'repo_commit', repo_name=self.db_repo_name, commit_id='tip') |
|
1485 | 1492 | |
|
1486 | 1493 | if self.rhodecode_vcs_repo.is_empty(): |
|
1487 | 1494 | # for empty repository we cannot check for current branch, we rely on |
|
1488 | 1495 | # c.commit.branch instead |
|
1489 | 1496 | _branch_name, _sha_commit_id, is_head = c.commit.branch, '', True |
|
1490 | 1497 | else: |
|
1491 | 1498 | _branch_name, _sha_commit_id, is_head = \ |
|
1492 | 1499 | self._is_valid_head(commit_id, self.rhodecode_vcs_repo, |
|
1493 | 1500 | landing_ref=self.db_repo.landing_ref_name) |
|
1494 | 1501 | |
|
1495 | 1502 | error = self.forbid_non_head(is_head, f_path, json_mode=True) |
|
1496 | 1503 | if error: |
|
1497 | 1504 | return { |
|
1498 | 1505 | 'error': error, |
|
1499 | 1506 | 'redirect_url': default_redirect_url |
|
1500 | 1507 | } |
|
1501 | 1508 | error = self.check_branch_permission(_branch_name, json_mode=True) |
|
1502 | 1509 | if error: |
|
1503 | 1510 | return { |
|
1504 | 1511 | 'error': error, |
|
1505 | 1512 | 'redirect_url': default_redirect_url |
|
1506 | 1513 | } |
|
1507 | 1514 | |
|
1508 | 1515 | c.default_message = (_('Added file via RhodeCode Enterprise')) |
|
1509 | 1516 | c.f_path = f_path |
|
1510 | 1517 | |
|
1511 | 1518 | r_post = self.request.POST |
|
1512 | 1519 | |
|
1513 | 1520 | message = c.default_message |
|
1514 | 1521 | user_message = r_post.getall('message') |
|
1515 | 1522 | if isinstance(user_message, list) and user_message: |
|
1516 | 1523 | # we take the first from duplicated results if it's not empty |
|
1517 | 1524 | message = user_message[0] if user_message[0] else message |
|
1518 | 1525 | |
|
1519 | 1526 | nodes = {} |
|
1520 | 1527 | |
|
1521 | 1528 | for file_obj in r_post.getall('files_upload') or []: |
|
1522 | 1529 | content = file_obj.file |
|
1523 | 1530 | filename = file_obj.filename |
|
1524 | 1531 | |
|
1525 | 1532 | root_path = f_path |
|
1526 | 1533 | pure_path = self.create_pure_path(root_path, filename) |
|
1527 | 1534 | node_path = pure_path.as_posix().lstrip('/') |
|
1528 | 1535 | |
|
1529 | 1536 | nodes[safe_bytes(node_path)] = { |
|
1530 | 1537 | 'content': content |
|
1531 | 1538 | } |
|
1532 | 1539 | |
|
1533 | 1540 | if not nodes: |
|
1534 | 1541 | error = 'missing files' |
|
1535 | 1542 | return { |
|
1536 | 1543 | 'error': error, |
|
1537 | 1544 | 'redirect_url': default_redirect_url |
|
1538 | 1545 | } |
|
1539 | 1546 | |
|
1540 | 1547 | author = self._rhodecode_db_user.full_contact |
|
1541 | 1548 | |
|
1542 | 1549 | try: |
|
1543 | 1550 | commit = ScmModel().create_nodes( |
|
1544 | 1551 | user=self._rhodecode_db_user.user_id, |
|
1545 | 1552 | repo=self.db_repo, |
|
1546 | 1553 | message=message, |
|
1547 | 1554 | nodes=nodes, |
|
1548 | 1555 | parent_commit=c.commit, |
|
1549 | 1556 | author=author, |
|
1550 | 1557 | ) |
|
1551 | 1558 | if len(nodes) == 1: |
|
1552 | 1559 | flash_message = _('Successfully committed {} new files').format(len(nodes)) |
|
1553 | 1560 | else: |
|
1554 | 1561 | flash_message = _('Successfully committed 1 new file') |
|
1555 | 1562 | |
|
1556 | 1563 | h.flash(flash_message, category='success') |
|
1557 | 1564 | |
|
1558 | 1565 | default_redirect_url = h.route_path( |
|
1559 | 1566 | 'repo_commit', repo_name=self.db_repo_name, commit_id=commit.raw_id) |
|
1560 | 1567 | |
|
1561 | 1568 | except NonRelativePathError: |
|
1562 | 1569 | log.exception('Non Relative path found') |
|
1563 | 1570 | error = _('The location specified must be a relative path and must not ' |
|
1564 | 1571 | 'contain .. in the path') |
|
1565 | 1572 | h.flash(error, category='warning') |
|
1566 | 1573 | |
|
1567 | 1574 | return { |
|
1568 | 1575 | 'error': error, |
|
1569 | 1576 | 'redirect_url': default_redirect_url |
|
1570 | 1577 | } |
|
1571 | 1578 | except (NodeError, NodeAlreadyExistsError) as e: |
|
1572 | 1579 | error = h.escape(e) |
|
1573 | 1580 | h.flash(error, category='error') |
|
1574 | 1581 | |
|
1575 | 1582 | return { |
|
1576 | 1583 | 'error': error, |
|
1577 | 1584 | 'redirect_url': default_redirect_url |
|
1578 | 1585 | } |
|
1579 | 1586 | except Exception: |
|
1580 | 1587 | log.exception('Error occurred during commit') |
|
1581 | 1588 | error = _('Error occurred during commit') |
|
1582 | 1589 | h.flash(error, category='error') |
|
1583 | 1590 | return { |
|
1584 | 1591 | 'error': error, |
|
1585 | 1592 | 'redirect_url': default_redirect_url |
|
1586 | 1593 | } |
|
1587 | 1594 | |
|
1588 | 1595 | return { |
|
1589 | 1596 | 'error': None, |
|
1590 | 1597 | 'redirect_url': default_redirect_url |
|
1591 | 1598 | } |
|
1592 | 1599 | |
|
1593 | 1600 | @LoginRequired() |
|
1594 | 1601 | @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin') |
|
1595 | 1602 | @CSRFRequired() |
|
1596 | 1603 | def repo_files_replace_file(self): |
|
1597 | 1604 | _ = self.request.translate |
|
1598 | 1605 | c = self.load_default_context() |
|
1599 | 1606 | commit_id, f_path = self._get_commit_and_path() |
|
1600 | 1607 | |
|
1601 | 1608 | self._ensure_not_locked() |
|
1602 | 1609 | |
|
1603 | 1610 | c.commit = self._get_commit_or_redirect(commit_id, redirect_after=False) |
|
1604 | 1611 | if c.commit is None: |
|
1605 | 1612 | c.commit = EmptyCommit(alias=self.rhodecode_vcs_repo.alias) |
|
1606 | 1613 | |
|
1607 | 1614 | if self.rhodecode_vcs_repo.is_empty(): |
|
1608 | 1615 | default_redirect_url = h.route_path( |
|
1609 | 1616 | 'repo_summary', repo_name=self.db_repo_name) |
|
1610 | 1617 | else: |
|
1611 | 1618 | default_redirect_url = h.route_path( |
|
1612 | 1619 | 'repo_commit', repo_name=self.db_repo_name, commit_id='tip') |
|
1613 | 1620 | |
|
1614 | 1621 | if self.rhodecode_vcs_repo.is_empty(): |
|
1615 | 1622 | # for empty repository we cannot check for current branch, we rely on |
|
1616 | 1623 | # c.commit.branch instead |
|
1617 | 1624 | _branch_name, _sha_commit_id, is_head = c.commit.branch, '', True |
|
1618 | 1625 | else: |
|
1619 | 1626 | _branch_name, _sha_commit_id, is_head = \ |
|
1620 | 1627 | self._is_valid_head(commit_id, self.rhodecode_vcs_repo, |
|
1621 | 1628 | landing_ref=self.db_repo.landing_ref_name) |
|
1622 | 1629 | |
|
1623 | 1630 | error = self.forbid_non_head(is_head, f_path, json_mode=True) |
|
1624 | 1631 | if error: |
|
1625 | 1632 | return { |
|
1626 | 1633 | 'error': error, |
|
1627 | 1634 | 'redirect_url': default_redirect_url |
|
1628 | 1635 | } |
|
1629 | 1636 | error = self.check_branch_permission(_branch_name, json_mode=True) |
|
1630 | 1637 | if error: |
|
1631 | 1638 | return { |
|
1632 | 1639 | 'error': error, |
|
1633 | 1640 | 'redirect_url': default_redirect_url |
|
1634 | 1641 | } |
|
1635 | 1642 | |
|
1636 | 1643 | c.default_message = (_('Edited file {} via RhodeCode Enterprise').format(f_path)) |
|
1637 | 1644 | c.f_path = f_path |
|
1638 | 1645 | |
|
1639 | 1646 | r_post = self.request.POST |
|
1640 | 1647 | |
|
1641 | 1648 | message = c.default_message |
|
1642 | 1649 | user_message = r_post.getall('message') |
|
1643 | 1650 | if isinstance(user_message, list) and user_message: |
|
1644 | 1651 | # we take the first from duplicated results if it's not empty |
|
1645 | 1652 | message = user_message[0] if user_message[0] else message |
|
1646 | 1653 | |
|
1647 | 1654 | data_for_replacement = r_post.getall('files_upload') or [] |
|
1648 | 1655 | if (objects_count := len(data_for_replacement)) > 1: |
|
1649 | 1656 | return { |
|
1650 | 1657 | 'error': 'too many files for replacement', |
|
1651 | 1658 | 'redirect_url': default_redirect_url |
|
1652 | 1659 | } |
|
1653 | 1660 | elif not objects_count: |
|
1654 | 1661 | return { |
|
1655 | 1662 | 'error': 'missing files', |
|
1656 | 1663 | 'redirect_url': default_redirect_url |
|
1657 | 1664 | } |
|
1658 | 1665 | |
|
1659 | 1666 | content = data_for_replacement[0].file |
|
1660 | 1667 | retrieved_filename = data_for_replacement[0].filename |
|
1661 | 1668 | |
|
1662 | 1669 | if retrieved_filename.split('.')[-1] != f_path.split('.')[-1]: |
|
1663 | 1670 | return { |
|
1664 | 1671 | 'error': 'file extension of uploaded file doesn\'t match an original file\'s extension', |
|
1665 | 1672 | 'redirect_url': default_redirect_url |
|
1666 | 1673 | } |
|
1667 | 1674 | |
|
1668 | 1675 | author = self._rhodecode_db_user.full_contact |
|
1669 | 1676 | |
|
1670 | 1677 | try: |
|
1671 | 1678 | commit = ScmModel().update_binary_node( |
|
1672 | 1679 | user=self._rhodecode_db_user.user_id, |
|
1673 | 1680 | repo=self.db_repo, |
|
1674 | 1681 | message=message, |
|
1675 | 1682 | node={ |
|
1676 | 1683 | 'content': content, |
|
1677 | 1684 | 'file_path': f_path.encode(), |
|
1678 | 1685 | }, |
|
1679 | 1686 | parent_commit=c.commit, |
|
1680 | 1687 | author=author, |
|
1681 | 1688 | ) |
|
1682 | 1689 | |
|
1683 | 1690 | h.flash(_('Successfully committed 1 new file'), category='success') |
|
1684 | 1691 | |
|
1685 | 1692 | default_redirect_url = h.route_path( |
|
1686 | 1693 | 'repo_commit', repo_name=self.db_repo_name, commit_id=commit.raw_id) |
|
1687 | 1694 | |
|
1688 | 1695 | except (NodeError, NodeAlreadyExistsError) as e: |
|
1689 | 1696 | error = h.escape(e) |
|
1690 | 1697 | h.flash(error, category='error') |
|
1691 | 1698 | |
|
1692 | 1699 | return { |
|
1693 | 1700 | 'error': error, |
|
1694 | 1701 | 'redirect_url': default_redirect_url |
|
1695 | 1702 | } |
|
1696 | 1703 | except Exception: |
|
1697 | 1704 | log.exception('Error occurred during commit') |
|
1698 | 1705 | error = _('Error occurred during commit') |
|
1699 | 1706 | h.flash(error, category='error') |
|
1700 | 1707 | return { |
|
1701 | 1708 | 'error': error, |
|
1702 | 1709 | 'redirect_url': default_redirect_url |
|
1703 | 1710 | } |
|
1704 | 1711 | |
|
1705 | 1712 | return { |
|
1706 | 1713 | 'error': None, |
|
1707 | 1714 | 'redirect_url': default_redirect_url |
|
1708 | 1715 | } |
@@ -1,201 +1,205 b'' | |||
|
1 | 1 | # Copyright (C) 2010-2023 RhodeCode GmbH |
|
2 | 2 | # |
|
3 | 3 | # This program is free software: you can redistribute it and/or modify |
|
4 | 4 | # it under the terms of the GNU Affero General Public License, version 3 |
|
5 | 5 | # (only), as published by the Free Software Foundation. |
|
6 | 6 | # |
|
7 | 7 | # This program is distributed in the hope that it will be useful, |
|
8 | 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
9 | 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
10 | 10 | # GNU General Public License for more details. |
|
11 | 11 | # |
|
12 | 12 | # You should have received a copy of the GNU Affero General Public License |
|
13 | 13 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
14 | 14 | # |
|
15 | 15 | # This program is dual-licensed. If you wish to learn more about the |
|
16 | 16 | # RhodeCode Enterprise Edition, including its added features, Support services, |
|
17 | 17 | # and proprietary license terms, please see https://rhodecode.com/licenses/ |
|
18 | 18 | |
|
19 | 19 | import os |
|
20 | 20 | import tempfile |
|
21 | 21 | import logging |
|
22 | 22 | |
|
23 | 23 | from pyramid.settings import asbool |
|
24 | 24 | |
|
25 | 25 | from rhodecode.config.settings_maker import SettingsMaker |
|
26 | 26 | from rhodecode.config import utils as config_utils |
|
27 | 27 | |
|
28 | 28 | log = logging.getLogger(__name__) |
|
29 | 29 | |
|
30 | 30 | |
|
31 | 31 | def sanitize_settings_and_apply_defaults(global_config, settings): |
|
32 | 32 | """ |
|
33 | 33 | Applies settings defaults and does all type conversion. |
|
34 | 34 | |
|
35 | 35 | We would move all settings parsing and preparation into this place, so that |
|
36 | 36 | we have only one place left which deals with this part. The remaining parts |
|
37 | 37 | of the application would start to rely fully on well-prepared settings. |
|
38 | 38 | |
|
39 | 39 | This piece would later be split up per topic to avoid a big fat monster |
|
40 | 40 | function. |
|
41 | 41 | """ |
|
42 | 42 | jn = os.path.join |
|
43 | 43 | |
|
44 | 44 | global_settings_maker = SettingsMaker(global_config) |
|
45 | 45 | global_settings_maker.make_setting('debug', default=False, parser='bool') |
|
46 | 46 | debug_enabled = asbool(global_config.get('debug')) |
|
47 | 47 | |
|
48 | 48 | settings_maker = SettingsMaker(settings) |
|
49 | 49 | |
|
50 | 50 | settings_maker.make_setting( |
|
51 | 51 | 'logging.autoconfigure', |
|
52 | 52 | default=False, |
|
53 | 53 | parser='bool') |
|
54 | 54 | |
|
55 | 55 | logging_conf = jn(os.path.dirname(global_config.get('__file__')), 'logging.ini') |
|
56 | 56 | settings_maker.enable_logging(logging_conf, level='INFO' if debug_enabled else 'DEBUG') |
|
57 | 57 | |
|
58 | 58 | # Default includes, possible to change as a user |
|
59 | 59 | pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline') |
|
60 | 60 | log.debug( |
|
61 | 61 | "Using the following pyramid.includes: %s", |
|
62 | 62 | pyramid_includes) |
|
63 | 63 | |
|
64 | 64 | settings_maker.make_setting('rhodecode.edition', 'Community Edition') |
|
65 | 65 | settings_maker.make_setting('rhodecode.edition_id', 'CE') |
|
66 | 66 | |
|
67 | 67 | if 'mako.default_filters' not in settings: |
|
68 | 68 | # set custom default filters if we don't have it defined |
|
69 | 69 | settings['mako.imports'] = 'from rhodecode.lib.base import h_filter' |
|
70 | 70 | settings['mako.default_filters'] = 'h_filter' |
|
71 | 71 | |
|
72 | 72 | if 'mako.directories' not in settings: |
|
73 | 73 | mako_directories = settings.setdefault('mako.directories', [ |
|
74 | 74 | # Base templates of the original application |
|
75 | 75 | 'rhodecode:templates', |
|
76 | 76 | ]) |
|
77 | 77 | log.debug( |
|
78 | 78 | "Using the following Mako template directories: %s", |
|
79 | 79 | mako_directories) |
|
80 | 80 | |
|
81 | 81 | # NOTE(marcink): fix redis requirement for schema of connection since 3.X |
|
82 | 82 | if 'beaker.session.type' in settings and settings['beaker.session.type'] == 'ext:redis': |
|
83 | 83 | raw_url = settings['beaker.session.url'] |
|
84 | 84 | if not raw_url.startswith(('redis://', 'rediss://', 'unix://')): |
|
85 | 85 | settings['beaker.session.url'] = 'redis://' + raw_url |
|
86 | 86 | |
|
87 | 87 | settings_maker.make_setting('__file__', global_config.get('__file__')) |
|
88 | 88 | |
|
89 | 89 | # TODO: johbo: Re-think this, usually the call to config.include |
|
90 | 90 | # should allow to pass in a prefix. |
|
91 | 91 | settings_maker.make_setting('rhodecode.api.url', '/_admin/api') |
|
92 | 92 | |
|
93 | 93 | # Sanitize generic settings. |
|
94 | 94 | settings_maker.make_setting('default_encoding', 'UTF-8', parser='list') |
|
95 | 95 | settings_maker.make_setting('gzip_responses', False, parser='bool') |
|
96 | 96 | settings_maker.make_setting('startup.import_repos', 'false', parser='bool') |
|
97 | 97 | |
|
98 | 98 | # statsd |
|
99 | 99 | settings_maker.make_setting('statsd.enabled', False, parser='bool') |
|
100 | 100 | settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string') |
|
101 | 101 | settings_maker.make_setting('statsd.statsd_port', 9125, parser='int') |
|
102 | 102 | settings_maker.make_setting('statsd.statsd_prefix', '') |
|
103 | 103 | settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool') |
|
104 | 104 | |
|
105 | 105 | settings_maker.make_setting('vcs.svn.compatible_version', '') |
|
106 | 106 | settings_maker.make_setting('vcs.svn.proxy.enabled', True, parser='bool') |
|
107 | 107 | settings_maker.make_setting('vcs.svn.proxy.host', 'http://svn:8090', parser='string') |
|
108 | 108 | settings_maker.make_setting('vcs.hooks.protocol', 'http') |
|
109 | 109 | settings_maker.make_setting('vcs.hooks.host', '*') |
|
110 | 110 | settings_maker.make_setting('vcs.scm_app_implementation', 'http') |
|
111 | 111 | settings_maker.make_setting('vcs.server', '') |
|
112 | 112 | settings_maker.make_setting('vcs.server.protocol', 'http') |
|
113 | 113 | settings_maker.make_setting('vcs.server.enable', 'true', parser='bool') |
|
114 | 114 | settings_maker.make_setting('vcs.hooks.direct_calls', 'false', parser='bool') |
|
115 | 115 | settings_maker.make_setting('vcs.start_server', 'false', parser='bool') |
|
116 | 116 | settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list') |
|
117 | 117 | settings_maker.make_setting('vcs.connection_timeout', 3600, parser='int') |
|
118 | 118 | |
|
119 | 119 | settings_maker.make_setting('vcs.methods.cache', True, parser='bool') |
|
120 | 120 | |
|
121 | 121 | # repo_store path |
|
122 | 122 | settings_maker.make_setting('repo_store.path', '/var/opt/rhodecode_repo_store') |
|
123 | 123 | # Support legacy values of vcs.scm_app_implementation. Legacy |
|
124 | 124 | # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or |
|
125 | 125 | # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'. |
|
126 | 126 | scm_app_impl = settings['vcs.scm_app_implementation'] |
|
127 | 127 | if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']: |
|
128 | 128 | settings['vcs.scm_app_implementation'] = 'http' |
|
129 | 129 | |
|
130 | 130 | settings_maker.make_setting('appenlight', False, parser='bool') |
|
131 | 131 | |
|
132 | 132 | temp_store = tempfile.gettempdir() |
|
133 | 133 | tmp_cache_dir = jn(temp_store, 'rc_cache') |
|
134 | 134 | |
|
135 | 135 | # save default, cache dir, and use it for all backends later. |
|
136 | 136 | default_cache_dir = settings_maker.make_setting( |
|
137 | 137 | 'cache_dir', |
|
138 | 138 | default=tmp_cache_dir, default_when_empty=True, |
|
139 | 139 | parser='dir:ensured') |
|
140 | 140 | |
|
141 | 141 | # exception store cache |
|
142 | 142 | settings_maker.make_setting( |
|
143 | 143 | 'exception_tracker.store_path', |
|
144 | 144 | default=jn(default_cache_dir, 'exc_store'), default_when_empty=True, |
|
145 | 145 | parser='dir:ensured' |
|
146 | 146 | ) |
|
147 | 147 | |
|
148 | 148 | settings_maker.make_setting( |
|
149 | 149 | 'celerybeat-schedule.path', |
|
150 | 150 | default=jn(default_cache_dir, 'celerybeat_schedule', 'celerybeat-schedule.db'), default_when_empty=True, |
|
151 | 151 | parser='file:ensured' |
|
152 | 152 | ) |
|
153 | 153 | |
|
154 | 154 | settings_maker.make_setting('exception_tracker.send_email', False, parser='bool') |
|
155 | 155 | settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True) |
|
156 | 156 | |
|
157 | 157 | # sessions, ensure file since no-value is memory |
|
158 | 158 | settings_maker.make_setting('beaker.session.type', 'file') |
|
159 | 159 | settings_maker.make_setting('beaker.session.data_dir', jn(default_cache_dir, 'session_data')) |
|
160 | 160 | |
|
161 | 161 | # cache_general |
|
162 | 162 | settings_maker.make_setting('rc_cache.cache_general.backend', 'dogpile.cache.rc.file_namespace') |
|
163 | 163 | settings_maker.make_setting('rc_cache.cache_general.expiration_time', 60 * 60 * 12, parser='int') |
|
164 | 164 | settings_maker.make_setting('rc_cache.cache_general.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_general.db')) |
|
165 | 165 | |
|
166 | 166 | # cache_perms |
|
167 | 167 | settings_maker.make_setting('rc_cache.cache_perms.backend', 'dogpile.cache.rc.file_namespace') |
|
168 | 168 | settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60 * 60, parser='int') |
|
169 | 169 | settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_perms_db')) |
|
170 | 170 | |
|
171 | 171 | # cache_repo |
|
172 | 172 | settings_maker.make_setting('rc_cache.cache_repo.backend', 'dogpile.cache.rc.file_namespace') |
|
173 | 173 | settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60 * 60 * 24 * 30, parser='int') |
|
174 | 174 | settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_repo_db')) |
|
175 | 175 | |
|
176 | 176 | # cache_license |
|
177 | 177 | settings_maker.make_setting('rc_cache.cache_license.backend', 'dogpile.cache.rc.file_namespace') |
|
178 | 178 | settings_maker.make_setting('rc_cache.cache_license.expiration_time', 60 * 5, parser='int') |
|
179 | 179 | settings_maker.make_setting('rc_cache.cache_license.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_license_db')) |
|
180 | 180 | |
|
181 | 181 | # cache_repo_longterm memory, 96H |
|
182 | 182 | settings_maker.make_setting('rc_cache.cache_repo_longterm.backend', 'dogpile.cache.rc.memory_lru') |
|
183 | 183 | settings_maker.make_setting('rc_cache.cache_repo_longterm.expiration_time', 345600, parser='int') |
|
184 | 184 | settings_maker.make_setting('rc_cache.cache_repo_longterm.max_size', 10000, parser='int') |
|
185 | 185 | |
|
186 | 186 | # sql_cache_short |
|
187 | 187 | settings_maker.make_setting('rc_cache.sql_cache_short.backend', 'dogpile.cache.rc.memory_lru') |
|
188 | 188 | settings_maker.make_setting('rc_cache.sql_cache_short.expiration_time', 30, parser='int') |
|
189 | 189 | settings_maker.make_setting('rc_cache.sql_cache_short.max_size', 10000, parser='int') |
|
190 | 190 | |
|
191 | 191 | # archive_cache |
|
192 | settings_maker.make_setting('archive_cache.store_dir', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,) | |
|
193 |
settings_maker.make_setting('archive_cache. |
|
|
194 | settings_maker.make_setting('archive_cache.cache_shards', 10, parser='int') | |
|
192 | settings_maker.make_setting('archive_cache.locking.url', 'redis://redis:6379/1') | |
|
193 | settings_maker.make_setting('archive_cache.backend.type', 'filesystem') | |
|
194 | ||
|
195 | settings_maker.make_setting('archive_cache.filesystem.store_dir', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,) | |
|
196 | settings_maker.make_setting('archive_cache.filesystem.cache_size_gb', 10, parser='float') | |
|
197 | settings_maker.make_setting('archive_cache.filesystem.cache_shards', 8, parser='int') | |
|
198 | settings_maker.make_setting('archive_cache.filesystem.eviction_policy', 'least-recently-stored') | |
|
195 | 199 | |
|
196 | 200 | settings_maker.env_expand() |
|
197 | 201 | |
|
198 | 202 | # configure instance id |
|
199 | 203 | config_utils.set_instance_id(settings) |
|
200 | 204 | |
|
201 | 205 | return settings |
@@ -1,88 +1,264 b'' | |||
|
1 |
# Copyright (C) 2015-202 |
|
|
1 | # Copyright (C) 2015-2024 RhodeCode GmbH | |
|
2 | 2 | # |
|
3 | 3 | # This program is free software: you can redistribute it and/or modify |
|
4 | 4 | # it under the terms of the GNU Affero General Public License, version 3 |
|
5 | 5 | # (only), as published by the Free Software Foundation. |
|
6 | 6 | # |
|
7 | 7 | # This program is distributed in the hope that it will be useful, |
|
8 | 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
9 | 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
10 | 10 | # GNU General Public License for more details. |
|
11 | 11 | # |
|
12 | 12 | # You should have received a copy of the GNU Affero General Public License |
|
13 | 13 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
14 | 14 | # |
|
15 | 15 | # This program is dual-licensed. If you wish to learn more about the |
|
16 | 16 | # RhodeCode Enterprise Edition, including its added features, Support services, |
|
17 | 17 | # and proprietary license terms, please see https://rhodecode.com/licenses/ |
|
18 | 18 | |
|
19 | import logging | |
|
19 | import codecs | |
|
20 | import contextlib | |
|
21 | import functools | |
|
20 | 22 | import os |
|
21 | import diskcache | |
|
22 | from diskcache import RLock | |
|
23 | import logging | |
|
24 | import time | |
|
25 | import typing | |
|
26 | import zlib | |
|
27 | ||
|
28 | from rhodecode.lib.ext_json import json | |
|
29 | from .lock import GenerationLock | |
|
23 | 30 | |
|
24 | 31 | log = logging.getLogger(__name__) |
|
25 | 32 | |
|
26 | 33 | cache_meta = None |
|
27 | 34 | |
|
35 | UNKNOWN = -241 | |
|
36 | NO_VAL = -917 | |
|
28 | 37 | |
|
29 | class ReentrantLock(RLock): | |
|
30 | def __enter__(self): | |
|
31 | reentrant_lock_key = self._key | |
|
38 | MODE_BINARY = 'BINARY' | |
|
39 | ||
|
40 | ||
|
41 | class FileSystemCache: | |
|
42 | ||
|
43 | def __init__(self, index, directory, **settings): | |
|
44 | self._index = index | |
|
45 | self._directory = directory | |
|
46 | ||
|
47 | def _write_file(self, full_path, iterator, mode, encoding=None): | |
|
48 | full_dir, _ = os.path.split(full_path) | |
|
49 | ||
|
50 | for count in range(1, 11): | |
|
51 | with contextlib.suppress(OSError): | |
|
52 | os.makedirs(full_dir) | |
|
53 | ||
|
54 | try: | |
|
55 | # Another cache may have deleted the directory before | |
|
56 | # the file could be opened. | |
|
57 | writer = open(full_path, mode, encoding=encoding) | |
|
58 | except OSError: | |
|
59 | if count == 10: | |
|
60 | # Give up after 10 tries to open the file. | |
|
61 | raise | |
|
62 | continue | |
|
63 | ||
|
64 | with writer: | |
|
65 | size = 0 | |
|
66 | for chunk in iterator: | |
|
67 | size += len(chunk) | |
|
68 | writer.write(chunk) | |
|
69 | return size | |
|
70 | ||
|
71 | def _get_keyfile(self, key): | |
|
72 | return os.path.join(self._directory, f'{key}.key') | |
|
73 | ||
|
74 | def store(self, key, value_reader, metadata): | |
|
75 | filename, full_path = self.random_filename() | |
|
76 | key_file = self._get_keyfile(key) | |
|
77 | ||
|
78 | # STORE METADATA | |
|
79 | _metadata = { | |
|
80 | "version": "v1", | |
|
81 | "timestamp": time.time(), | |
|
82 | "filename": filename, | |
|
83 | "full_path": full_path, | |
|
84 | "key_file": key_file, | |
|
85 | } | |
|
86 | if metadata: | |
|
87 | _metadata.update(metadata) | |
|
88 | ||
|
89 | reader = functools.partial(value_reader.read, 2**22) | |
|
90 | ||
|
91 | iterator = iter(reader, b'') | |
|
92 | size = self._write_file(full_path, iterator, 'xb') | |
|
93 | ||
|
94 | # after archive is finished, we create a key to save the presence of the binary file | |
|
95 | with open(key_file, 'wb') as f: | |
|
96 | f.write(json.dumps(_metadata)) | |
|
97 | ||
|
98 | return key, size, MODE_BINARY, filename, _metadata | |
|
99 | ||
|
100 | def fetch(self, key) -> tuple[typing.BinaryIO, dict]: | |
|
101 | if key not in self: | |
|
102 | raise KeyError(key) | |
|
103 | ||
|
104 | key_file = self._get_keyfile(key) | |
|
105 | with open(key_file, 'rb') as f: | |
|
106 | metadata = json.loads(f.read()) | |
|
107 | ||
|
108 | filename = metadata['filename'] | |
|
109 | ||
|
110 | return open(os.path.join(self._directory, filename), 'rb'), metadata | |
|
111 | ||
|
112 | def random_filename(self): | |
|
113 | """Return filename and full-path tuple for file storage. | |
|
114 | ||
|
115 | Filename will be a randomly generated 28 character hexadecimal string | |
|
116 | with ".archive_cache" suffixed. Two levels of sub-directories will be used to | |
|
117 | reduce the size of directories. On older filesystems, lookups in | |
|
118 | directories with many files may be slow. | |
|
119 | """ | |
|
32 | 120 | |
|
33 | log.debug('Acquire ReentrantLock(key=%s) for archive cache generation...', reentrant_lock_key) | |
|
34 | #self.acquire() | |
|
35 | log.debug('Lock for key=%s acquired', reentrant_lock_key) | |
|
121 | hex_name = codecs.encode(os.urandom(16), 'hex').decode('utf-8') | |
|
122 | sub_dir = os.path.join(hex_name[:2], hex_name[2:4]) | |
|
123 | name = hex_name[4:] + '.archive_cache' | |
|
124 | filename = os.path.join(sub_dir, name) | |
|
125 | full_path = os.path.join(self._directory, filename) | |
|
126 | return filename, full_path | |
|
127 | ||
|
128 | def hash(self, key): | |
|
129 | """Compute portable hash for `key`. | |
|
130 | ||
|
131 | :param key: key to hash | |
|
132 | :return: hash value | |
|
133 | ||
|
134 | """ | |
|
135 | mask = 0xFFFFFFFF | |
|
136 | return zlib.adler32(key.encode('utf-8')) & mask # noqa | |
|
137 | ||
|
138 | def __contains__(self, key): | |
|
139 | """Return `True` if `key` matching item is found in cache. | |
|
140 | ||
|
141 | :param key: key matching item | |
|
142 | :return: True if key matching item | |
|
143 | ||
|
144 | """ | |
|
145 | key_file = self._get_keyfile(key) | |
|
146 | return os.path.exists(key_file) | |
|
147 | ||
|
148 | ||
|
149 | class FanoutCache: | |
|
150 | """Cache that shards keys and values.""" | |
|
151 | ||
|
152 | def __init__( | |
|
153 | self, directory=None, **settings | |
|
154 | ): | |
|
155 | """Initialize cache instance. | |
|
156 | ||
|
157 | :param str directory: cache directory | |
|
158 | :param settings: settings dict | |
|
159 | ||
|
160 | """ | |
|
161 | if directory is None: | |
|
162 | raise ValueError('directory cannot be None') | |
|
163 | ||
|
164 | directory = str(directory) | |
|
165 | directory = os.path.expanduser(directory) | |
|
166 | directory = os.path.expandvars(directory) | |
|
167 | self._directory = directory | |
|
36 | 168 | |
|
37 | def __exit__(self, *exc_info): | |
|
38 | #self.release() | |
|
39 | pass | |
|
169 | self._count = settings.pop('cache_shards') | |
|
170 | self._locking_url = settings.pop('locking_url') | |
|
171 | ||
|
172 | self._shards = tuple( | |
|
173 | FileSystemCache( | |
|
174 | index=num, | |
|
175 | directory=os.path.join(directory, 'shard_%03d' % num), | |
|
176 | **settings, | |
|
177 | ) | |
|
178 | for num in range(self._count) | |
|
179 | ) | |
|
180 | self._hash = self._shards[0].hash | |
|
181 | ||
|
182 | def get_lock(self, lock_key): | |
|
183 | return GenerationLock(lock_key, self._locking_url) | |
|
184 | ||
|
185 | def _get_shard(self, key) -> FileSystemCache: | |
|
186 | index = self._hash(key) % self._count | |
|
187 | shard = self._shards[index] | |
|
188 | return shard | |
|
189 | ||
|
190 | def store(self, key, value_reader, metadata=None): | |
|
191 | shard = self._get_shard(key) | |
|
192 | return shard.store(key, value_reader, metadata) | |
|
193 | ||
|
194 | def fetch(self, key): | |
|
195 | """Return file handle corresponding to `key` from cache. | |
|
196 | """ | |
|
197 | shard = self._get_shard(key) | |
|
198 | return shard.fetch(key) | |
|
199 | ||
|
200 | def has_key(self, key): | |
|
201 | """Return `True` if `key` matching item is found in cache. | |
|
202 | ||
|
203 | :param key: key for item | |
|
204 | :return: True if key is found | |
|
205 | ||
|
206 | """ | |
|
207 | shard = self._get_shard(key) | |
|
208 | return key in shard | |
|
209 | ||
|
210 | def __contains__(self, item): | |
|
211 | return self.has_key(item) | |
|
212 | ||
|
213 | def evict(self): | |
|
214 | """Remove old items based on the conditions""" | |
|
215 | # TODO: Implement this... | |
|
216 | return | |
|
40 | 217 | |
|
41 | 218 | |
|
42 | 219 | def get_archival_config(config): |
|
43 | 220 | |
|
44 | 221 | final_config = { |
|
45 | 'archive_cache.eviction_policy': 'least-frequently-used' | |
|
222 | ||
|
46 | 223 | } |
|
47 | 224 | |
|
48 | 225 | for k, v in config.items(): |
|
49 | 226 | if k.startswith('archive_cache'): |
|
50 | 227 | final_config[k] = v |
|
51 | 228 | |
|
52 | 229 | return final_config |
|
53 | 230 | |
|
54 | 231 | |
|
55 | 232 | def get_archival_cache_store(config): |
|
56 | 233 | |
|
57 | 234 | global cache_meta |
|
58 | 235 | if cache_meta is not None: |
|
59 | 236 | return cache_meta |
|
60 | 237 | |
|
61 | 238 | config = get_archival_config(config) |
|
239 | backend = config['archive_cache.backend.type'] | |
|
240 | if backend != 'filesystem': | |
|
241 | raise ValueError('archive_cache.backend.type only supports "filesystem"') | |
|
62 | 242 | |
|
63 |
archive_cache_ |
|
|
64 |
archive_cache_ |
|
|
65 |
archive_cache_s |
|
|
66 |
archive_cache_ |
|
|
243 | archive_cache_locking_url = config['archive_cache.locking.url'] | |
|
244 | archive_cache_dir = config['archive_cache.filesystem.store_dir'] | |
|
245 | archive_cache_size_gb = config['archive_cache.filesystem.cache_size_gb'] | |
|
246 | archive_cache_shards = config['archive_cache.filesystem.cache_shards'] | |
|
247 | archive_cache_eviction_policy = config['archive_cache.filesystem.eviction_policy'] | |
|
67 | 248 | |
|
68 | 249 | log.debug('Initializing archival cache instance under %s', archive_cache_dir) |
|
69 | 250 | |
|
70 | 251 | # check if it's ok to write, and re-create the archive cache |
|
71 | 252 | if not os.path.isdir(archive_cache_dir): |
|
72 | 253 | os.makedirs(archive_cache_dir, exist_ok=True) |
|
73 | 254 | |
|
74 |
d_cache = |
|
|
75 |
archive_cache_dir, |
|
|
76 | cull_limit=0, # manual eviction required | |
|
77 | size_limit=archive_cache_size_gb * 1024 * 1024 * 1024, | |
|
78 | eviction_policy=archive_cache_eviction_policy, | |
|
79 | timeout=30 | |
|
255 | d_cache = FanoutCache( | |
|
256 | archive_cache_dir, | |
|
257 | locking_url=archive_cache_locking_url, | |
|
258 | cache_shards=archive_cache_shards, | |
|
259 | cache_size_limit=archive_cache_size_gb * 1024 * 1024 * 1024, | |
|
260 | cache_eviction_policy=archive_cache_eviction_policy | |
|
80 | 261 | ) |
|
81 | 262 | cache_meta = d_cache |
|
82 | 263 | return cache_meta |
|
83 | 264 | |
|
84 | ||
|
85 | def includeme(config): | |
|
86 | # init our cache at start | |
|
87 | settings = config.get_settings() | |
|
88 | get_archival_cache_store(settings) |
@@ -1,844 +1,851 b'' | |||
|
1 | 1 | # Copyright (C) 2017-2023 RhodeCode GmbH |
|
2 | 2 | # |
|
3 | 3 | # This program is free software: you can redistribute it and/or modify |
|
4 | 4 | # it under the terms of the GNU Affero General Public License, version 3 |
|
5 | 5 | # (only), as published by the Free Software Foundation. |
|
6 | 6 | # |
|
7 | 7 | # This program is distributed in the hope that it will be useful, |
|
8 | 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
9 | 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
10 | 10 | # GNU General Public License for more details. |
|
11 | 11 | # |
|
12 | 12 | # You should have received a copy of the GNU Affero General Public License |
|
13 | 13 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
14 | 14 | # |
|
15 | 15 | # This program is dual-licensed. If you wish to learn more about the |
|
16 | 16 | # RhodeCode Enterprise Edition, including its added features, Support services, |
|
17 | 17 | # and proprietary license terms, please see https://rhodecode.com/licenses/ |
|
18 | 18 | |
|
19 | 19 | |
|
20 | 20 | import os |
|
21 | 21 | import sys |
|
22 | 22 | import time |
|
23 | 23 | import platform |
|
24 | 24 | import collections |
|
25 | 25 | import psutil |
|
26 | 26 | from functools import wraps |
|
27 | 27 | |
|
28 | 28 | import pkg_resources |
|
29 | 29 | import logging |
|
30 | 30 | import resource |
|
31 | 31 | |
|
32 | 32 | import configparser |
|
33 | 33 | |
|
34 | 34 | from rc_license.models import LicenseModel |
|
35 | 35 | from rhodecode.lib.str_utils import safe_str |
|
36 | 36 | |
|
37 | 37 | log = logging.getLogger(__name__) |
|
38 | 38 | |
|
39 | 39 | |
|
40 | 40 | _NA = 'NOT AVAILABLE' |
|
41 | 41 | _NA_FLOAT = 0.0 |
|
42 | 42 | |
|
43 | 43 | STATE_OK = 'ok' |
|
44 | 44 | STATE_ERR = 'error' |
|
45 | 45 | STATE_WARN = 'warning' |
|
46 | 46 | |
|
47 | 47 | STATE_OK_DEFAULT = {'message': '', 'type': STATE_OK} |
|
48 | 48 | |
|
49 | 49 | |
|
50 | 50 | registered_helpers = {} |
|
51 | 51 | |
|
52 | 52 | |
|
53 | 53 | def register_sysinfo(func): |
|
54 | 54 | """ |
|
55 | 55 | @register_helper |
|
56 | 56 | def db_check(): |
|
57 | 57 | pass |
|
58 | 58 | |
|
59 | 59 | db_check == registered_helpers['db_check'] |
|
60 | 60 | """ |
|
61 | 61 | global registered_helpers |
|
62 | 62 | registered_helpers[func.__name__] = func |
|
63 | 63 | |
|
64 | 64 | @wraps(func) |
|
65 | 65 | def _wrapper(*args, **kwargs): |
|
66 | 66 | return func(*args, **kwargs) |
|
67 | 67 | return _wrapper |
|
68 | 68 | |
|
69 | 69 | |
|
70 | 70 | # HELPERS |
|
71 | 71 | def percentage(part: (int, float), whole: (int, float)): |
|
72 | 72 | whole = float(whole) |
|
73 | 73 | if whole > 0: |
|
74 | 74 | return round(100 * float(part) / whole, 1) |
|
75 | 75 | return 0.0 |
|
76 | 76 | |
|
77 | 77 | |
|
78 | 78 | def get_storage_size(storage_path): |
|
79 | 79 | sizes = [] |
|
80 | 80 | for file_ in os.listdir(storage_path): |
|
81 | 81 | storage_file = os.path.join(storage_path, file_) |
|
82 | 82 | if os.path.isfile(storage_file): |
|
83 | 83 | try: |
|
84 | 84 | sizes.append(os.path.getsize(storage_file)) |
|
85 | 85 | except OSError: |
|
86 | 86 | log.exception('Failed to get size of storage file %s', storage_file) |
|
87 | 87 | pass |
|
88 | 88 | |
|
89 | 89 | return sum(sizes) |
|
90 | 90 | |
|
91 | 91 | |
|
92 | 92 | def get_resource(resource_type): |
|
93 | 93 | try: |
|
94 | 94 | return resource.getrlimit(resource_type) |
|
95 | 95 | except Exception: |
|
96 | 96 | return 'NOT_SUPPORTED' |
|
97 | 97 | |
|
98 | 98 | |
|
99 | 99 | def get_cert_path(ini_path): |
|
100 | 100 | default = '/etc/ssl/certs/ca-certificates.crt' |
|
101 | 101 | control_ca_bundle = os.path.join( |
|
102 | 102 | os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(ini_path)))), |
|
103 | 103 | '.rccontrol-profile/etc/ca-bundle.crt') |
|
104 | 104 | if os.path.isfile(control_ca_bundle): |
|
105 | 105 | default = control_ca_bundle |
|
106 | 106 | |
|
107 | 107 | return default |
|
108 | 108 | |
|
109 | 109 | |
|
110 | 110 | class SysInfoRes(object): |
|
111 | 111 | def __init__(self, value, state=None, human_value=None): |
|
112 | 112 | self.value = value |
|
113 | 113 | self.state = state or STATE_OK_DEFAULT |
|
114 | 114 | self.human_value = human_value or value |
|
115 | 115 | |
|
116 | 116 | def __json__(self): |
|
117 | 117 | return { |
|
118 | 118 | 'value': self.value, |
|
119 | 119 | 'state': self.state, |
|
120 | 120 | 'human_value': self.human_value, |
|
121 | 121 | } |
|
122 | 122 | |
|
123 | 123 | def get_value(self): |
|
124 | 124 | return self.__json__() |
|
125 | 125 | |
|
126 | 126 | def __str__(self): |
|
127 | 127 | return f'<SysInfoRes({self.__json__()})>' |
|
128 | 128 | |
|
129 | 129 | |
|
130 | 130 | class SysInfo(object): |
|
131 | 131 | |
|
132 | 132 | def __init__(self, func_name, **kwargs): |
|
133 | 133 | self.function_name = func_name |
|
134 | 134 | self.value = _NA |
|
135 | 135 | self.state = None |
|
136 | 136 | self.kwargs = kwargs or {} |
|
137 | 137 | |
|
138 | 138 | def __call__(self): |
|
139 | 139 | computed = self.compute(**self.kwargs) |
|
140 | 140 | if not isinstance(computed, SysInfoRes): |
|
141 | 141 | raise ValueError( |
|
142 | 142 | 'computed value for {} is not instance of ' |
|
143 | 143 | '{}, got {} instead'.format( |
|
144 | 144 | self.function_name, SysInfoRes, type(computed))) |
|
145 | 145 | return computed.__json__() |
|
146 | 146 | |
|
147 | 147 | def __str__(self): |
|
148 | 148 | return f'<SysInfo({self.function_name})>' |
|
149 | 149 | |
|
150 | 150 | def compute(self, **kwargs): |
|
151 | 151 | return self.function_name(**kwargs) |
|
152 | 152 | |
|
153 | 153 | |
|
154 | 154 | # SysInfo functions |
|
155 | 155 | @register_sysinfo |
|
156 | 156 | def python_info(): |
|
157 | 157 | value = dict(version=f'{platform.python_version()}:{platform.python_implementation()}', |
|
158 | 158 | executable=sys.executable) |
|
159 | 159 | return SysInfoRes(value=value) |
|
160 | 160 | |
|
161 | 161 | |
|
162 | 162 | @register_sysinfo |
|
163 | 163 | def py_modules(): |
|
164 | 164 | mods = dict([(p.project_name, {'version': p.version, 'location': p.location}) |
|
165 | 165 | for p in pkg_resources.working_set]) |
|
166 | 166 | |
|
167 | 167 | value = sorted(mods.items(), key=lambda k: k[0].lower()) |
|
168 | 168 | return SysInfoRes(value=value) |
|
169 | 169 | |
|
170 | 170 | |
|
171 | 171 | @register_sysinfo |
|
172 | 172 | def platform_type(): |
|
173 | 173 | from rhodecode.lib.utils import generate_platform_uuid |
|
174 | 174 | |
|
175 | 175 | value = dict( |
|
176 | 176 | name=safe_str(platform.platform()), |
|
177 | 177 | uuid=generate_platform_uuid() |
|
178 | 178 | ) |
|
179 | 179 | return SysInfoRes(value=value) |
|
180 | 180 | |
|
181 | 181 | |
|
182 | 182 | @register_sysinfo |
|
183 | 183 | def locale_info(): |
|
184 | 184 | import locale |
|
185 | 185 | |
|
186 | 186 | def safe_get_locale(locale_name): |
|
187 | 187 | try: |
|
188 | 188 | locale.getlocale(locale_name) |
|
189 | 189 | except TypeError: |
|
190 | 190 | return f'FAILED_LOCALE_GET:{locale_name}' |
|
191 | 191 | |
|
192 | 192 | value = dict( |
|
193 | 193 | locale_default=locale.getlocale(), |
|
194 | 194 | locale_lc_all=safe_get_locale(locale.LC_ALL), |
|
195 | 195 | locale_lc_ctype=safe_get_locale(locale.LC_CTYPE), |
|
196 | 196 | lang_env=os.environ.get('LANG'), |
|
197 | 197 | lc_all_env=os.environ.get('LC_ALL'), |
|
198 | 198 | local_archive_env=os.environ.get('LOCALE_ARCHIVE'), |
|
199 | 199 | ) |
|
200 | 200 | human_value = \ |
|
201 | 201 | f"LANG: {value['lang_env']}, \ |
|
202 | 202 | locale LC_ALL: {value['locale_lc_all']}, \ |
|
203 | 203 | locale LC_CTYPE: {value['locale_lc_ctype']}, \ |
|
204 | 204 | Default locales: {value['locale_default']}" |
|
205 | 205 | |
|
206 | 206 | return SysInfoRes(value=value, human_value=human_value) |
|
207 | 207 | |
|
208 | 208 | |
|
209 | 209 | @register_sysinfo |
|
210 | 210 | def ulimit_info(): |
|
211 | 211 | data = collections.OrderedDict([ |
|
212 | 212 | ('cpu time (seconds)', get_resource(resource.RLIMIT_CPU)), |
|
213 | 213 | ('file size', get_resource(resource.RLIMIT_FSIZE)), |
|
214 | 214 | ('stack size', get_resource(resource.RLIMIT_STACK)), |
|
215 | 215 | ('core file size', get_resource(resource.RLIMIT_CORE)), |
|
216 | 216 | ('address space size', get_resource(resource.RLIMIT_AS)), |
|
217 | 217 | ('locked in mem size', get_resource(resource.RLIMIT_MEMLOCK)), |
|
218 | 218 | ('heap size', get_resource(resource.RLIMIT_DATA)), |
|
219 | 219 | ('rss size', get_resource(resource.RLIMIT_RSS)), |
|
220 | 220 | ('number of processes', get_resource(resource.RLIMIT_NPROC)), |
|
221 | 221 | ('open files', get_resource(resource.RLIMIT_NOFILE)), |
|
222 | 222 | ]) |
|
223 | 223 | |
|
224 | 224 | text = ', '.join(f'{k}:{v}' for k, v in data.items()) |
|
225 | 225 | |
|
226 | 226 | value = { |
|
227 | 227 | 'limits': data, |
|
228 | 228 | 'text': text, |
|
229 | 229 | } |
|
230 | 230 | return SysInfoRes(value=value) |
|
231 | 231 | |
|
232 | 232 | |
|
233 | 233 | @register_sysinfo |
|
234 | 234 | def uptime(): |
|
235 | 235 | from rhodecode.lib.helpers import age, time_to_datetime |
|
236 | 236 | from rhodecode.translation import TranslationString |
|
237 | 237 | |
|
238 | 238 | value = dict(boot_time=0, uptime=0, text='') |
|
239 | 239 | state = STATE_OK_DEFAULT |
|
240 | 240 | |
|
241 | 241 | boot_time = psutil.boot_time() |
|
242 | 242 | value['boot_time'] = boot_time |
|
243 | 243 | value['uptime'] = time.time() - boot_time |
|
244 | 244 | |
|
245 | 245 | date_or_age = age(time_to_datetime(boot_time)) |
|
246 | 246 | if isinstance(date_or_age, TranslationString): |
|
247 | 247 | date_or_age = date_or_age.interpolate() |
|
248 | 248 | |
|
249 | 249 | human_value = value.copy() |
|
250 | 250 | human_value['boot_time'] = time_to_datetime(boot_time) |
|
251 | 251 | human_value['uptime'] = age(time_to_datetime(boot_time), show_suffix=False) |
|
252 | 252 | |
|
253 | 253 | human_value['text'] = f'Server started {date_or_age}' |
|
254 | 254 | return SysInfoRes(value=value, human_value=human_value) |
|
255 | 255 | |
|
256 | 256 | |
|
257 | 257 | @register_sysinfo |
|
258 | 258 | def memory(): |
|
259 | 259 | from rhodecode.lib.helpers import format_byte_size_binary |
|
260 | 260 | value = dict(available=0, used=0, used_real=0, cached=0, percent=0, |
|
261 | 261 | percent_used=0, free=0, inactive=0, active=0, shared=0, |
|
262 | 262 | total=0, buffers=0, text='') |
|
263 | 263 | |
|
264 | 264 | state = STATE_OK_DEFAULT |
|
265 | 265 | |
|
266 | 266 | value.update(dict(psutil.virtual_memory()._asdict())) |
|
267 | 267 | value['used_real'] = value['total'] - value['available'] |
|
268 | 268 | value['percent_used'] = psutil._common.usage_percent(value['used_real'], value['total'], 1) |
|
269 | 269 | |
|
270 | 270 | human_value = value.copy() |
|
271 | 271 | human_value['text'] = '{}/{}, {}% used'.format( |
|
272 | 272 | format_byte_size_binary(value['used_real']), |
|
273 | 273 | format_byte_size_binary(value['total']), |
|
274 | 274 | value['percent_used']) |
|
275 | 275 | |
|
276 | 276 | keys = list(value.keys())[::] |
|
277 | 277 | keys.pop(keys.index('percent')) |
|
278 | 278 | keys.pop(keys.index('percent_used')) |
|
279 | 279 | keys.pop(keys.index('text')) |
|
280 | 280 | for k in keys: |
|
281 | 281 | human_value[k] = format_byte_size_binary(value[k]) |
|
282 | 282 | |
|
283 | 283 | if state['type'] == STATE_OK and value['percent_used'] > 90: |
|
284 | 284 | msg = 'Critical: your available RAM memory is very low.' |
|
285 | 285 | state = {'message': msg, 'type': STATE_ERR} |
|
286 | 286 | |
|
287 | 287 | elif state['type'] == STATE_OK and value['percent_used'] > 70: |
|
288 | 288 | msg = 'Warning: your available RAM memory is running low.' |
|
289 | 289 | state = {'message': msg, 'type': STATE_WARN} |
|
290 | 290 | |
|
291 | 291 | return SysInfoRes(value=value, state=state, human_value=human_value) |
|
292 | 292 | |
|
293 | 293 | |
|
294 | 294 | @register_sysinfo |
|
295 | 295 | def machine_load(): |
|
296 | 296 | value = {'1_min': _NA_FLOAT, '5_min': _NA_FLOAT, '15_min': _NA_FLOAT, 'text': ''} |
|
297 | 297 | state = STATE_OK_DEFAULT |
|
298 | 298 | |
|
299 | 299 | # load averages |
|
300 | 300 | if hasattr(psutil.os, 'getloadavg'): |
|
301 | 301 | value.update(dict( |
|
302 | 302 | list(zip(['1_min', '5_min', '15_min'], psutil.os.getloadavg())) |
|
303 | 303 | )) |
|
304 | 304 | |
|
305 | 305 | human_value = value.copy() |
|
306 | 306 | human_value['text'] = '1min: {}, 5min: {}, 15min: {}'.format( |
|
307 | 307 | value['1_min'], value['5_min'], value['15_min']) |
|
308 | 308 | |
|
309 | 309 | if state['type'] == STATE_OK and value['15_min'] > 5.0: |
|
310 | 310 | msg = 'Warning: your machine load is very high.' |
|
311 | 311 | state = {'message': msg, 'type': STATE_WARN} |
|
312 | 312 | |
|
313 | 313 | return SysInfoRes(value=value, state=state, human_value=human_value) |
|
314 | 314 | |
|
315 | 315 | |
|
316 | 316 | @register_sysinfo |
|
317 | 317 | def cpu(): |
|
318 | 318 | value = {'cpu': 0, 'cpu_count': 0, 'cpu_usage': []} |
|
319 | 319 | state = STATE_OK_DEFAULT |
|
320 | 320 | |
|
321 | 321 | value['cpu'] = psutil.cpu_percent(0.5) |
|
322 | 322 | value['cpu_usage'] = psutil.cpu_percent(0.5, percpu=True) |
|
323 | 323 | value['cpu_count'] = psutil.cpu_count() |
|
324 | 324 | |
|
325 | 325 | human_value = value.copy() |
|
326 | 326 | human_value['text'] = '{} cores at {} %'.format(value['cpu_count'], value['cpu']) |
|
327 | 327 | |
|
328 | 328 | return SysInfoRes(value=value, state=state, human_value=human_value) |
|
329 | 329 | |
|
330 | 330 | |
|
331 | 331 | @register_sysinfo |
|
332 | 332 | def storage(): |
|
333 | 333 | from rhodecode.lib.helpers import format_byte_size_binary |
|
334 | 334 | from rhodecode.lib.utils import get_rhodecode_repo_store_path |
|
335 | 335 | path = get_rhodecode_repo_store_path() |
|
336 | 336 | |
|
337 | 337 | value = dict(percent=0, used=0, total=0, path=path, text='') |
|
338 | 338 | state = STATE_OK_DEFAULT |
|
339 | 339 | |
|
340 | 340 | try: |
|
341 | 341 | value.update(dict(psutil.disk_usage(path)._asdict())) |
|
342 | 342 | except Exception as e: |
|
343 | 343 | log.exception('Failed to fetch disk info') |
|
344 | 344 | state = {'message': str(e), 'type': STATE_ERR} |
|
345 | 345 | |
|
346 | 346 | human_value = value.copy() |
|
347 | 347 | human_value['used'] = format_byte_size_binary(value['used']) |
|
348 | 348 | human_value['total'] = format_byte_size_binary(value['total']) |
|
349 | 349 | human_value['text'] = "{}/{}, {}% used".format( |
|
350 | 350 | format_byte_size_binary(value['used']), |
|
351 | 351 | format_byte_size_binary(value['total']), |
|
352 | 352 | value['percent']) |
|
353 | 353 | |
|
354 | 354 | if state['type'] == STATE_OK and value['percent'] > 90: |
|
355 | 355 | msg = 'Critical: your disk space is very low.' |
|
356 | 356 | state = {'message': msg, 'type': STATE_ERR} |
|
357 | 357 | |
|
358 | 358 | elif state['type'] == STATE_OK and value['percent'] > 70: |
|
359 | 359 | msg = 'Warning: your disk space is running low.' |
|
360 | 360 | state = {'message': msg, 'type': STATE_WARN} |
|
361 | 361 | |
|
362 | 362 | return SysInfoRes(value=value, state=state, human_value=human_value) |
|
363 | 363 | |
|
364 | 364 | |
|
365 | 365 | @register_sysinfo |
|
366 | 366 | def storage_inodes(): |
|
367 | 367 | from rhodecode.lib.utils import get_rhodecode_repo_store_path |
|
368 | 368 | path = get_rhodecode_repo_store_path() |
|
369 | 369 | |
|
370 | 370 | value = dict(percent=0.0, free=0, used=0, total=0, path=path, text='') |
|
371 | 371 | state = STATE_OK_DEFAULT |
|
372 | 372 | |
|
373 | 373 | try: |
|
374 | 374 | i_stat = os.statvfs(path) |
|
375 | 375 | value['free'] = i_stat.f_ffree |
|
376 | 376 | value['used'] = i_stat.f_files-i_stat.f_favail |
|
377 | 377 | value['total'] = i_stat.f_files |
|
378 | 378 | value['percent'] = percentage(value['used'], value['total']) |
|
379 | 379 | except Exception as e: |
|
380 | 380 | log.exception('Failed to fetch disk inodes info') |
|
381 | 381 | state = {'message': str(e), 'type': STATE_ERR} |
|
382 | 382 | |
|
383 | 383 | human_value = value.copy() |
|
384 | 384 | human_value['text'] = "{}/{}, {}% used".format( |
|
385 | 385 | value['used'], value['total'], value['percent']) |
|
386 | 386 | |
|
387 | 387 | if state['type'] == STATE_OK and value['percent'] > 90: |
|
388 | 388 | msg = 'Critical: your disk free inodes are very low.' |
|
389 | 389 | state = {'message': msg, 'type': STATE_ERR} |
|
390 | 390 | |
|
391 | 391 | elif state['type'] == STATE_OK and value['percent'] > 70: |
|
392 | 392 | msg = 'Warning: your disk free inodes are running low.' |
|
393 | 393 | state = {'message': msg, 'type': STATE_WARN} |
|
394 | 394 | |
|
395 | 395 | return SysInfoRes(value=value, state=state, human_value=human_value) |
|
396 | 396 | |
|
397 | 397 | |
|
398 | 398 | @register_sysinfo |
|
399 | 399 | def storage_archives(): |
|
400 | 400 | import rhodecode |
|
401 | 401 | from rhodecode.lib.utils import safe_str |
|
402 | 402 | from rhodecode.lib.helpers import format_byte_size_binary |
|
403 | 403 | |
|
404 | msg = 'Archive cache storage is controlled by ' \ | |
|
405 | 'archive_cache.store_dir=/path/to/cache option in the .ini file' | |
|
406 | path = safe_str(rhodecode.CONFIG.get('archive_cache.store_dir', msg)) | |
|
404 | storage_type = rhodecode.ConfigGet().get_str('archive_cache.backend.type') | |
|
405 | storage_key = 'archive_cache.filesystem.store_dir' | |
|
406 | ||
|
407 | default_msg = 'Archive cache storage is controlled by '\ | |
|
408 | f'{storage_key}=/path/to/cache option in the .ini file' | |
|
409 | path = rhodecode.ConfigGet().get_str(storage_key, missing=default_msg) | |
|
407 | 410 | |
|
408 | 411 | value = dict(percent=0, used=0, total=0, items=0, path=path, text='') |
|
409 | 412 | state = STATE_OK_DEFAULT |
|
410 | 413 | try: |
|
414 | if storage_type != 'filesystem': | |
|
415 | # raise Exc to stop reporting on different type | |
|
416 | raise ValueError('Storage type must be "filesystem"') | |
|
417 | ||
|
411 | 418 | items_count = 0 |
|
412 | 419 | used = 0 |
|
413 | 420 | for root, dirs, files in os.walk(path): |
|
414 | 421 | if root == path: |
|
415 | 422 | items_count = len(dirs) |
|
416 | 423 | |
|
417 | 424 | for f in files: |
|
418 | 425 | try: |
|
419 | 426 | used += os.path.getsize(os.path.join(root, f)) |
|
420 | 427 | except OSError: |
|
421 | 428 | pass |
|
422 | 429 | value.update({ |
|
423 | 430 | 'percent': 100, |
|
424 | 431 | 'used': used, |
|
425 | 432 | 'total': used, |
|
426 | 433 | 'items': items_count |
|
427 | 434 | }) |
|
428 | 435 | |
|
429 | 436 | except Exception as e: |
|
430 | 437 | log.exception('failed to fetch archive cache storage') |
|
431 | 438 | state = {'message': str(e), 'type': STATE_ERR} |
|
432 | 439 | |
|
433 | 440 | human_value = value.copy() |
|
434 | 441 | human_value['used'] = format_byte_size_binary(value['used']) |
|
435 | 442 | human_value['total'] = format_byte_size_binary(value['total']) |
|
436 | 443 | human_value['text'] = "{} ({} items)".format( |
|
437 | 444 | human_value['used'], value['items']) |
|
438 | 445 | |
|
439 | 446 | return SysInfoRes(value=value, state=state, human_value=human_value) |
|
440 | 447 | |
|
441 | 448 | |
|
442 | 449 | @register_sysinfo |
|
443 | 450 | def storage_gist(): |
|
444 | 451 | from rhodecode.model.gist import GIST_STORE_LOC |
|
445 | 452 | from rhodecode.lib.utils import safe_str, get_rhodecode_repo_store_path |
|
446 | 453 | from rhodecode.lib.helpers import format_byte_size_binary |
|
447 | 454 | path = safe_str(os.path.join( |
|
448 | 455 | get_rhodecode_repo_store_path(), GIST_STORE_LOC)) |
|
449 | 456 | |
|
450 | 457 | # gist storage |
|
451 | 458 | value = dict(percent=0, used=0, total=0, items=0, path=path, text='') |
|
452 | 459 | state = STATE_OK_DEFAULT |
|
453 | 460 | |
|
454 | 461 | try: |
|
455 | 462 | items_count = 0 |
|
456 | 463 | used = 0 |
|
457 | 464 | for root, dirs, files in os.walk(path): |
|
458 | 465 | if root == path: |
|
459 | 466 | items_count = len(dirs) |
|
460 | 467 | |
|
461 | 468 | for f in files: |
|
462 | 469 | try: |
|
463 | 470 | used += os.path.getsize(os.path.join(root, f)) |
|
464 | 471 | except OSError: |
|
465 | 472 | pass |
|
466 | 473 | value.update({ |
|
467 | 474 | 'percent': 100, |
|
468 | 475 | 'used': used, |
|
469 | 476 | 'total': used, |
|
470 | 477 | 'items': items_count |
|
471 | 478 | }) |
|
472 | 479 | except Exception as e: |
|
473 | 480 | log.exception('failed to fetch gist storage items') |
|
474 | 481 | state = {'message': str(e), 'type': STATE_ERR} |
|
475 | 482 | |
|
476 | 483 | human_value = value.copy() |
|
477 | 484 | human_value['used'] = format_byte_size_binary(value['used']) |
|
478 | 485 | human_value['total'] = format_byte_size_binary(value['total']) |
|
479 | 486 | human_value['text'] = "{} ({} items)".format( |
|
480 | 487 | human_value['used'], value['items']) |
|
481 | 488 | |
|
482 | 489 | return SysInfoRes(value=value, state=state, human_value=human_value) |
|
483 | 490 | |
|
484 | 491 | |
|
485 | 492 | @register_sysinfo |
|
486 | 493 | def storage_temp(): |
|
487 | 494 | import tempfile |
|
488 | 495 | from rhodecode.lib.helpers import format_byte_size_binary |
|
489 | 496 | |
|
490 | 497 | path = tempfile.gettempdir() |
|
491 | 498 | value = dict(percent=0, used=0, total=0, items=0, path=path, text='') |
|
492 | 499 | state = STATE_OK_DEFAULT |
|
493 | 500 | |
|
494 | 501 | if not psutil: |
|
495 | 502 | return SysInfoRes(value=value, state=state) |
|
496 | 503 | |
|
497 | 504 | try: |
|
498 | 505 | value.update(dict(psutil.disk_usage(path)._asdict())) |
|
499 | 506 | except Exception as e: |
|
500 | 507 | log.exception('Failed to fetch temp dir info') |
|
501 | 508 | state = {'message': str(e), 'type': STATE_ERR} |
|
502 | 509 | |
|
503 | 510 | human_value = value.copy() |
|
504 | 511 | human_value['used'] = format_byte_size_binary(value['used']) |
|
505 | 512 | human_value['total'] = format_byte_size_binary(value['total']) |
|
506 | 513 | human_value['text'] = "{}/{}, {}% used".format( |
|
507 | 514 | format_byte_size_binary(value['used']), |
|
508 | 515 | format_byte_size_binary(value['total']), |
|
509 | 516 | value['percent']) |
|
510 | 517 | |
|
511 | 518 | return SysInfoRes(value=value, state=state, human_value=human_value) |
|
512 | 519 | |
|
513 | 520 | |
|
514 | 521 | @register_sysinfo |
|
515 | 522 | def search_info(): |
|
516 | 523 | import rhodecode |
|
517 | 524 | from rhodecode.lib.index import searcher_from_config |
|
518 | 525 | |
|
519 | 526 | backend = rhodecode.CONFIG.get('search.module', '') |
|
520 | 527 | location = rhodecode.CONFIG.get('search.location', '') |
|
521 | 528 | |
|
522 | 529 | try: |
|
523 | 530 | searcher = searcher_from_config(rhodecode.CONFIG) |
|
524 | 531 | searcher = searcher.__class__.__name__ |
|
525 | 532 | except Exception: |
|
526 | 533 | searcher = None |
|
527 | 534 | |
|
528 | 535 | value = dict( |
|
529 | 536 | backend=backend, searcher=searcher, location=location, text='') |
|
530 | 537 | state = STATE_OK_DEFAULT |
|
531 | 538 | |
|
532 | 539 | human_value = value.copy() |
|
533 | 540 | human_value['text'] = "backend:`{}`".format(human_value['backend']) |
|
534 | 541 | |
|
535 | 542 | return SysInfoRes(value=value, state=state, human_value=human_value) |
|
536 | 543 | |
|
537 | 544 | |
|
538 | 545 | @register_sysinfo |
|
539 | 546 | def git_info(): |
|
540 | 547 | from rhodecode.lib.vcs.backends import git |
|
541 | 548 | state = STATE_OK_DEFAULT |
|
542 | 549 | value = human_value = '' |
|
543 | 550 | try: |
|
544 | 551 | value = git.discover_git_version(raise_on_exc=True) |
|
545 | 552 | human_value = f'version reported from VCSServer: {value}' |
|
546 | 553 | except Exception as e: |
|
547 | 554 | state = {'message': str(e), 'type': STATE_ERR} |
|
548 | 555 | |
|
549 | 556 | return SysInfoRes(value=value, state=state, human_value=human_value) |
|
550 | 557 | |
|
551 | 558 | |
|
552 | 559 | @register_sysinfo |
|
553 | 560 | def hg_info(): |
|
554 | 561 | from rhodecode.lib.vcs.backends import hg |
|
555 | 562 | state = STATE_OK_DEFAULT |
|
556 | 563 | value = human_value = '' |
|
557 | 564 | try: |
|
558 | 565 | value = hg.discover_hg_version(raise_on_exc=True) |
|
559 | 566 | human_value = f'version reported from VCSServer: {value}' |
|
560 | 567 | except Exception as e: |
|
561 | 568 | state = {'message': str(e), 'type': STATE_ERR} |
|
562 | 569 | return SysInfoRes(value=value, state=state, human_value=human_value) |
|
563 | 570 | |
|
564 | 571 | |
|
565 | 572 | @register_sysinfo |
|
566 | 573 | def svn_info(): |
|
567 | 574 | from rhodecode.lib.vcs.backends import svn |
|
568 | 575 | state = STATE_OK_DEFAULT |
|
569 | 576 | value = human_value = '' |
|
570 | 577 | try: |
|
571 | 578 | value = svn.discover_svn_version(raise_on_exc=True) |
|
572 | 579 | human_value = f'version reported from VCSServer: {value}' |
|
573 | 580 | except Exception as e: |
|
574 | 581 | state = {'message': str(e), 'type': STATE_ERR} |
|
575 | 582 | return SysInfoRes(value=value, state=state, human_value=human_value) |
|
576 | 583 | |
|
577 | 584 | |
|
578 | 585 | @register_sysinfo |
|
579 | 586 | def vcs_backends(): |
|
580 | 587 | import rhodecode |
|
581 | 588 | value = rhodecode.CONFIG.get('vcs.backends') |
|
582 | 589 | human_value = 'Enabled backends in order: {}'.format(','.join(value)) |
|
583 | 590 | return SysInfoRes(value=value, human_value=human_value) |
|
584 | 591 | |
|
585 | 592 | |
|
586 | 593 | @register_sysinfo |
|
587 | 594 | def vcs_server(): |
|
588 | 595 | import rhodecode |
|
589 | 596 | from rhodecode.lib.vcs.backends import get_vcsserver_service_data |
|
590 | 597 | |
|
591 | 598 | server_url = rhodecode.CONFIG.get('vcs.server') |
|
592 | 599 | enabled = rhodecode.CONFIG.get('vcs.server.enable') |
|
593 | 600 | protocol = rhodecode.CONFIG.get('vcs.server.protocol') or 'http' |
|
594 | 601 | state = STATE_OK_DEFAULT |
|
595 | 602 | version = None |
|
596 | 603 | workers = 0 |
|
597 | 604 | |
|
598 | 605 | try: |
|
599 | 606 | data = get_vcsserver_service_data() |
|
600 | 607 | if data and 'version' in data: |
|
601 | 608 | version = data['version'] |
|
602 | 609 | |
|
603 | 610 | if data and 'config' in data: |
|
604 | 611 | conf = data['config'] |
|
605 | 612 | workers = conf.get('workers', 'NOT AVAILABLE') |
|
606 | 613 | |
|
607 | 614 | connection = 'connected' |
|
608 | 615 | except Exception as e: |
|
609 | 616 | connection = 'failed' |
|
610 | 617 | state = {'message': str(e), 'type': STATE_ERR} |
|
611 | 618 | |
|
612 | 619 | value = dict( |
|
613 | 620 | url=server_url, |
|
614 | 621 | enabled=enabled, |
|
615 | 622 | protocol=protocol, |
|
616 | 623 | connection=connection, |
|
617 | 624 | version=version, |
|
618 | 625 | text='', |
|
619 | 626 | ) |
|
620 | 627 | |
|
621 | 628 | human_value = value.copy() |
|
622 | 629 | human_value['text'] = \ |
|
623 | 630 | '{url}@ver:{ver} via {mode} mode[workers:{workers}], connection:{conn}'.format( |
|
624 | 631 | url=server_url, ver=version, workers=workers, mode=protocol, |
|
625 | 632 | conn=connection) |
|
626 | 633 | |
|
627 | 634 | return SysInfoRes(value=value, state=state, human_value=human_value) |
|
628 | 635 | |
|
629 | 636 | |
|
630 | 637 | @register_sysinfo |
|
631 | 638 | def vcs_server_config(): |
|
632 | 639 | from rhodecode.lib.vcs.backends import get_vcsserver_service_data |
|
633 | 640 | state = STATE_OK_DEFAULT |
|
634 | 641 | |
|
635 | 642 | value = {} |
|
636 | 643 | try: |
|
637 | 644 | data = get_vcsserver_service_data() |
|
638 | 645 | value = data['app_config'] |
|
639 | 646 | except Exception as e: |
|
640 | 647 | state = {'message': str(e), 'type': STATE_ERR} |
|
641 | 648 | |
|
642 | 649 | human_value = value.copy() |
|
643 | 650 | human_value['text'] = 'VCS Server config' |
|
644 | 651 | |
|
645 | 652 | return SysInfoRes(value=value, state=state, human_value=human_value) |
|
646 | 653 | |
|
647 | 654 | |
|
648 | 655 | @register_sysinfo |
|
649 | 656 | def rhodecode_app_info(): |
|
650 | 657 | import rhodecode |
|
651 | 658 | edition = rhodecode.CONFIG.get('rhodecode.edition') |
|
652 | 659 | |
|
653 | 660 | value = dict( |
|
654 | 661 | rhodecode_version=rhodecode.__version__, |
|
655 | 662 | rhodecode_lib_path=os.path.abspath(rhodecode.__file__), |
|
656 | 663 | text='' |
|
657 | 664 | ) |
|
658 | 665 | human_value = value.copy() |
|
659 | 666 | human_value['text'] = 'RhodeCode {edition}, version {ver}'.format( |
|
660 | 667 | edition=edition, ver=value['rhodecode_version'] |
|
661 | 668 | ) |
|
662 | 669 | return SysInfoRes(value=value, human_value=human_value) |
|
663 | 670 | |
|
664 | 671 | |
|
665 | 672 | @register_sysinfo |
|
666 | 673 | def rhodecode_config(): |
|
667 | 674 | import rhodecode |
|
668 | 675 | path = rhodecode.CONFIG.get('__file__') |
|
669 | 676 | rhodecode_ini_safe = rhodecode.CONFIG.copy() |
|
670 | 677 | cert_path = get_cert_path(path) |
|
671 | 678 | |
|
672 | 679 | try: |
|
673 | 680 | config = configparser.ConfigParser() |
|
674 | 681 | config.read(path) |
|
675 | 682 | parsed_ini = config |
|
676 | 683 | if parsed_ini.has_section('server:main'): |
|
677 | 684 | parsed_ini = dict(parsed_ini.items('server:main')) |
|
678 | 685 | except Exception: |
|
679 | 686 | log.exception('Failed to read .ini file for display') |
|
680 | 687 | parsed_ini = {} |
|
681 | 688 | |
|
682 | 689 | rhodecode_ini_safe['server:main'] = parsed_ini |
|
683 | 690 | |
|
684 | 691 | blacklist = [ |
|
685 | 692 | f'rhodecode_{LicenseModel.LICENSE_DB_KEY}', |
|
686 | 693 | 'routes.map', |
|
687 | 694 | 'sqlalchemy.db1.url', |
|
688 | 695 | 'channelstream.secret', |
|
689 | 696 | 'beaker.session.secret', |
|
690 | 697 | 'rhodecode.encrypted_values.secret', |
|
691 | 698 | 'rhodecode_auth_github_consumer_key', |
|
692 | 699 | 'rhodecode_auth_github_consumer_secret', |
|
693 | 700 | 'rhodecode_auth_google_consumer_key', |
|
694 | 701 | 'rhodecode_auth_google_consumer_secret', |
|
695 | 702 | 'rhodecode_auth_bitbucket_consumer_secret', |
|
696 | 703 | 'rhodecode_auth_bitbucket_consumer_key', |
|
697 | 704 | 'rhodecode_auth_twitter_consumer_secret', |
|
698 | 705 | 'rhodecode_auth_twitter_consumer_key', |
|
699 | 706 | |
|
700 | 707 | 'rhodecode_auth_twitter_secret', |
|
701 | 708 | 'rhodecode_auth_github_secret', |
|
702 | 709 | 'rhodecode_auth_google_secret', |
|
703 | 710 | 'rhodecode_auth_bitbucket_secret', |
|
704 | 711 | |
|
705 | 712 | 'appenlight.api_key', |
|
706 | 713 | ('app_conf', 'sqlalchemy.db1.url') |
|
707 | 714 | ] |
|
708 | 715 | for k in blacklist: |
|
709 | 716 | if isinstance(k, tuple): |
|
710 | 717 | section, key = k |
|
711 | 718 | if section in rhodecode_ini_safe: |
|
712 | 719 | rhodecode_ini_safe[section] = '**OBFUSCATED**' |
|
713 | 720 | else: |
|
714 | 721 | rhodecode_ini_safe.pop(k, None) |
|
715 | 722 | |
|
716 | 723 | # TODO: maybe put some CONFIG checks here ? |
|
717 | 724 | return SysInfoRes(value={'config': rhodecode_ini_safe, |
|
718 | 725 | 'path': path, 'cert_path': cert_path}) |
|
719 | 726 | |
|
720 | 727 | |
|
721 | 728 | @register_sysinfo |
|
722 | 729 | def database_info(): |
|
723 | 730 | import rhodecode |
|
724 | 731 | from sqlalchemy.engine import url as engine_url |
|
725 | 732 | from rhodecode.model import meta |
|
726 | 733 | from rhodecode.model.meta import Session |
|
727 | 734 | from rhodecode.model.db import DbMigrateVersion |
|
728 | 735 | |
|
729 | 736 | state = STATE_OK_DEFAULT |
|
730 | 737 | |
|
731 | 738 | db_migrate = DbMigrateVersion.query().filter( |
|
732 | 739 | DbMigrateVersion.repository_id == 'rhodecode_db_migrations').one() |
|
733 | 740 | |
|
734 | 741 | db_url_obj = engine_url.make_url(rhodecode.CONFIG['sqlalchemy.db1.url']) |
|
735 | 742 | |
|
736 | 743 | try: |
|
737 | 744 | engine = meta.get_engine() |
|
738 | 745 | db_server_info = engine.dialect._get_server_version_info( |
|
739 | 746 | Session.connection(bind=engine)) |
|
740 | 747 | db_version = '.'.join(map(str, db_server_info)) |
|
741 | 748 | except Exception: |
|
742 | 749 | log.exception('failed to fetch db version') |
|
743 | 750 | db_version = 'UNKNOWN' |
|
744 | 751 | |
|
745 | 752 | db_info = dict( |
|
746 | 753 | migrate_version=db_migrate.version, |
|
747 | 754 | type=db_url_obj.get_backend_name(), |
|
748 | 755 | version=db_version, |
|
749 | 756 | url=repr(db_url_obj) |
|
750 | 757 | ) |
|
751 | 758 | current_version = db_migrate.version |
|
752 | 759 | expected_version = rhodecode.__dbversion__ |
|
753 | 760 | if state['type'] == STATE_OK and current_version != expected_version: |
|
754 | 761 | msg = 'Critical: database schema mismatch, ' \ |
|
755 | 762 | 'expected version {}, got {}. ' \ |
|
756 | 763 | 'Please run migrations on your database.'.format( |
|
757 | 764 | expected_version, current_version) |
|
758 | 765 | state = {'message': msg, 'type': STATE_ERR} |
|
759 | 766 | |
|
760 | 767 | human_value = db_info.copy() |
|
761 | 768 | human_value['url'] = "{} @ migration version: {}".format( |
|
762 | 769 | db_info['url'], db_info['migrate_version']) |
|
763 | 770 | human_value['version'] = "{} {}".format(db_info['type'], db_info['version']) |
|
764 | 771 | return SysInfoRes(value=db_info, state=state, human_value=human_value) |
|
765 | 772 | |
|
766 | 773 | |
|
767 | 774 | @register_sysinfo |
|
768 | 775 | def server_info(environ): |
|
769 | 776 | import rhodecode |
|
770 | 777 | from rhodecode.lib.base import get_server_ip_addr, get_server_port |
|
771 | 778 | |
|
772 | 779 | value = { |
|
773 | 780 | 'server_ip': '{}:{}'.format( |
|
774 | 781 | get_server_ip_addr(environ, log_errors=False), |
|
775 | 782 | get_server_port(environ) |
|
776 | 783 | ), |
|
777 | 784 | 'server_id': rhodecode.CONFIG.get('instance_id'), |
|
778 | 785 | } |
|
779 | 786 | return SysInfoRes(value=value) |
|
780 | 787 | |
|
781 | 788 | |
|
782 | 789 | @register_sysinfo |
|
783 | 790 | def usage_info(): |
|
784 | 791 | from rhodecode.model.db import User, Repository, true |
|
785 | 792 | value = { |
|
786 | 793 | 'users': User.query().count(), |
|
787 | 794 | 'users_active': User.query().filter(User.active == true()).count(), |
|
788 | 795 | 'repositories': Repository.query().count(), |
|
789 | 796 | 'repository_types': { |
|
790 | 797 | 'hg': Repository.query().filter( |
|
791 | 798 | Repository.repo_type == 'hg').count(), |
|
792 | 799 | 'git': Repository.query().filter( |
|
793 | 800 | Repository.repo_type == 'git').count(), |
|
794 | 801 | 'svn': Repository.query().filter( |
|
795 | 802 | Repository.repo_type == 'svn').count(), |
|
796 | 803 | }, |
|
797 | 804 | } |
|
798 | 805 | return SysInfoRes(value=value) |
|
799 | 806 | |
|
800 | 807 | |
|
801 | 808 | def get_system_info(environ): |
|
802 | 809 | environ = environ or {} |
|
803 | 810 | return { |
|
804 | 811 | 'rhodecode_app': SysInfo(rhodecode_app_info)(), |
|
805 | 812 | 'rhodecode_config': SysInfo(rhodecode_config)(), |
|
806 | 813 | 'rhodecode_usage': SysInfo(usage_info)(), |
|
807 | 814 | 'python': SysInfo(python_info)(), |
|
808 | 815 | 'py_modules': SysInfo(py_modules)(), |
|
809 | 816 | |
|
810 | 817 | 'platform': SysInfo(platform_type)(), |
|
811 | 818 | 'locale': SysInfo(locale_info)(), |
|
812 | 819 | 'server': SysInfo(server_info, environ=environ)(), |
|
813 | 820 | 'database': SysInfo(database_info)(), |
|
814 | 821 | 'ulimit': SysInfo(ulimit_info)(), |
|
815 | 822 | 'storage': SysInfo(storage)(), |
|
816 | 823 | 'storage_inodes': SysInfo(storage_inodes)(), |
|
817 | 824 | 'storage_archive': SysInfo(storage_archives)(), |
|
818 | 825 | 'storage_gist': SysInfo(storage_gist)(), |
|
819 | 826 | 'storage_temp': SysInfo(storage_temp)(), |
|
820 | 827 | |
|
821 | 828 | 'search': SysInfo(search_info)(), |
|
822 | 829 | |
|
823 | 830 | 'uptime': SysInfo(uptime)(), |
|
824 | 831 | 'load': SysInfo(machine_load)(), |
|
825 | 832 | 'cpu': SysInfo(cpu)(), |
|
826 | 833 | 'memory': SysInfo(memory)(), |
|
827 | 834 | |
|
828 | 835 | 'vcs_backends': SysInfo(vcs_backends)(), |
|
829 | 836 | 'vcs_server': SysInfo(vcs_server)(), |
|
830 | 837 | |
|
831 | 838 | 'vcs_server_config': SysInfo(vcs_server_config)(), |
|
832 | 839 | |
|
833 | 840 | 'git': SysInfo(git_info)(), |
|
834 | 841 | 'hg': SysInfo(hg_info)(), |
|
835 | 842 | 'svn': SysInfo(svn_info)(), |
|
836 | 843 | } |
|
837 | 844 | |
|
838 | 845 | |
|
839 | 846 | def load_system_info(key): |
|
840 | 847 | """ |
|
841 | 848 | get_sys_info('vcs_server') |
|
842 | 849 | get_sys_info('database') |
|
843 | 850 | """ |
|
844 | 851 | return SysInfo(registered_helpers[key])() |
General Comments 0
You need to be logged in to leave comments.
Login now