##// END OF EJS Templates
fix(jupiter): sanitizing rendering (HTML) of jupyter notebooks, upgraded bleach version. Fixes RCCE-15, RCCE-14
ilin.s -
r5252:1ecdda64 default
parent child Browse files
Show More
@@ -1,295 +1,295 b''
1 1 # deps, generated via pipdeptree --exclude setuptools,wheel,pipdeptree,pip -f | tr '[:upper:]' '[:lower:]'
2 2
3 3 alembic==1.12.1
4 4 mako==1.2.4
5 5 markupsafe==2.1.2
6 6 sqlalchemy==1.4.50
7 7 greenlet==3.0.3
8 8 typing_extensions==4.8.0
9 9 async-timeout==4.0.3
10 10 babel==2.12.1
11 11 beaker==1.12.1
12 12 celery==5.3.4
13 13 billiard==4.1.0
14 14 click==8.1.3
15 15 click-didyoumean==0.3.0
16 16 click==8.1.3
17 17 click-plugins==1.1.1
18 18 click==8.1.3
19 19 click-repl==0.2.0
20 20 click==8.1.3
21 21 prompt-toolkit==3.0.38
22 22 wcwidth==0.2.6
23 23 six==1.16.0
24 24 kombu==5.3.2
25 25 amqp==5.1.1
26 26 vine==5.0.0
27 27 vine==5.0.0
28 28 python-dateutil==2.8.2
29 29 six==1.16.0
30 30 tzdata==2023.3
31 31 vine==5.0.0
32 32 channelstream==0.7.1
33 33 gevent==23.9.1
34 34 greenlet==3.0.3
35 35 zope.event==5.0.0
36 36 zope.interface==6.1.0
37 37 itsdangerous==1.1.0
38 38 marshmallow==2.18.0
39 39 pyramid==2.0.2
40 40 hupper==1.12
41 41 plaster==1.1.2
42 42 plaster-pastedeploy==1.0.1
43 43 pastedeploy==3.1.0
44 44 plaster==1.1.2
45 45 translationstring==1.4
46 46 venusian==3.0.0
47 47 webob==1.8.7
48 48 zope.deprecation==5.0.0
49 49 zope.interface==6.1.0
50 50 pyramid-apispec==0.3.3
51 51 apispec==1.3.3
52 52 pyramid-jinja2==2.10
53 53 jinja2==3.1.2
54 54 markupsafe==2.1.2
55 55 markupsafe==2.1.2
56 56 pyramid==2.0.2
57 57 hupper==1.12
58 58 plaster==1.1.2
59 59 plaster-pastedeploy==1.0.1
60 60 pastedeploy==3.1.0
61 61 plaster==1.1.2
62 62 translationstring==1.4
63 63 venusian==3.0.0
64 64 webob==1.8.7
65 65 zope.deprecation==5.0.0
66 66 zope.interface==6.1.0
67 67 zope.deprecation==5.0.0
68 68 python-dateutil==2.8.2
69 69 six==1.16.0
70 70 requests==2.28.2
71 71 certifi==2022.12.7
72 72 charset-normalizer==3.1.0
73 73 idna==3.4
74 74 urllib3==1.26.14
75 75 ws4py==0.5.1
76 76 deform==2.0.15
77 77 chameleon==3.10.2
78 78 colander==2.0
79 79 iso8601==1.1.0
80 80 translationstring==1.4
81 81 iso8601==1.1.0
82 82 peppercorn==0.6
83 83 translationstring==1.4
84 84 zope.deprecation==5.0.0
85 85 diskcache==5.6.3
86 86 docutils==0.19
87 87 dogpile.cache==1.3.0
88 88 decorator==5.1.1
89 89 stevedore==5.1.0
90 90 pbr==5.11.1
91 91 formencode==2.1.0
92 92 six==1.16.0
93 93 gunicorn==21.2.0
94 94 packaging==23.1
95 95 gevent==23.9.1
96 96 greenlet==3.0.3
97 97 zope.event==5.0.0
98 98 zope.interface==6.1.0
99 99 ipython==8.14.0
100 100 backcall==0.2.0
101 101 decorator==5.1.1
102 102 jedi==0.19.0
103 103 parso==0.8.3
104 104 matplotlib-inline==0.1.6
105 105 traitlets==5.9.0
106 106 pexpect==4.8.0
107 107 ptyprocess==0.7.0
108 108 pickleshare==0.7.5
109 109 prompt-toolkit==3.0.38
110 110 wcwidth==0.2.6
111 111 pygments==2.15.1
112 112 stack-data==0.6.2
113 113 asttokens==2.2.1
114 114 six==1.16.0
115 115 executing==1.2.0
116 116 pure-eval==0.2.2
117 117 traitlets==5.9.0
118 118 markdown==3.4.3
119 119 msgpack==1.0.7
120 120 mysqlclient==2.1.1
121 121 nbconvert==7.7.3
122 122 beautifulsoup4==4.11.2
123 123 soupsieve==2.4
124 bleach==6.0.0
124 bleach==6.1.0
125 125 six==1.16.0
126 126 webencodings==0.5.1
127 127 defusedxml==0.7.1
128 128 jinja2==3.1.2
129 129 markupsafe==2.1.2
130 130 jupyter_core==5.3.1
131 131 platformdirs==3.10.0
132 132 traitlets==5.9.0
133 133 jupyterlab-pygments==0.2.2
134 134 markupsafe==2.1.2
135 135 mistune==2.0.5
136 136 nbclient==0.8.0
137 137 jupyter_client==8.3.0
138 138 jupyter_core==5.3.1
139 139 platformdirs==3.10.0
140 140 traitlets==5.9.0
141 141 python-dateutil==2.8.2
142 142 six==1.16.0
143 143 pyzmq==25.0.0
144 144 tornado==6.2
145 145 traitlets==5.9.0
146 146 jupyter_core==5.3.1
147 147 platformdirs==3.10.0
148 148 traitlets==5.9.0
149 149 nbformat==5.9.2
150 150 fastjsonschema==2.18.0
151 151 jsonschema==4.18.6
152 152 attrs==22.2.0
153 153 pyrsistent==0.19.3
154 154 jupyter_core==5.3.1
155 155 platformdirs==3.10.0
156 156 traitlets==5.9.0
157 157 traitlets==5.9.0
158 158 traitlets==5.9.0
159 159 nbformat==5.9.2
160 160 fastjsonschema==2.18.0
161 161 jsonschema==4.18.6
162 162 attrs==22.2.0
163 163 pyrsistent==0.19.3
164 164 jupyter_core==5.3.1
165 165 platformdirs==3.10.0
166 166 traitlets==5.9.0
167 167 traitlets==5.9.0
168 168 packaging==23.1
169 169 pandocfilters==1.5.0
170 170 pygments==2.15.1
171 171 tinycss2==1.2.1
172 172 webencodings==0.5.1
173 173 traitlets==5.9.0
174 174 orjson==3.9.10
175 175 pastescript==3.3.0
176 176 paste==3.7.1
177 177 six==1.16.0
178 178 pastedeploy==3.1.0
179 179 six==1.16.0
180 180 premailer==3.10.0
181 181 cachetools==5.3.1
182 182 cssselect==1.2.0
183 183 cssutils==2.6.0
184 184 lxml==4.9.3
185 185 requests==2.28.2
186 186 certifi==2022.12.7
187 187 charset-normalizer==3.1.0
188 188 idna==3.4
189 189 urllib3==1.26.14
190 190 psutil==5.9.5
191 191 psycopg2==2.9.9
192 192 py-bcrypt==0.4
193 193 pycmarkgfm==1.2.0
194 194 cffi==1.16.0
195 195 pycparser==2.21
196 196 pycryptodome==3.17
197 197 pycurl==7.45.2
198 198 pymysql==1.0.3
199 199 pyotp==2.8.0
200 200 pyparsing==3.1.1
201 201 pyramid-debugtoolbar==4.10
202 202 pygments==2.15.1
203 203 pyramid==2.0.2
204 204 hupper==1.12
205 205 plaster==1.1.2
206 206 plaster-pastedeploy==1.0.1
207 207 pastedeploy==3.1.0
208 208 plaster==1.1.2
209 209 translationstring==1.4
210 210 venusian==3.0.0
211 211 webob==1.8.7
212 212 zope.deprecation==5.0.0
213 213 zope.interface==6.1.0
214 214 pyramid-mako==1.1.0
215 215 mako==1.2.4
216 216 markupsafe==2.1.2
217 217 pyramid==2.0.2
218 218 hupper==1.12
219 219 plaster==1.1.2
220 220 plaster-pastedeploy==1.0.1
221 221 pastedeploy==3.1.0
222 222 plaster==1.1.2
223 223 translationstring==1.4
224 224 venusian==3.0.0
225 225 webob==1.8.7
226 226 zope.deprecation==5.0.0
227 227 zope.interface==6.1.0
228 228 pyramid-mailer==0.15.1
229 229 pyramid==2.0.2
230 230 hupper==1.12
231 231 plaster==1.1.2
232 232 plaster-pastedeploy==1.0.1
233 233 pastedeploy==3.1.0
234 234 plaster==1.1.2
235 235 translationstring==1.4
236 236 venusian==3.0.0
237 237 webob==1.8.7
238 238 zope.deprecation==5.0.0
239 239 zope.interface==6.1.0
240 240 repoze.sendmail==4.4.1
241 241 transaction==3.1.0
242 242 zope.interface==6.1.0
243 243 zope.interface==6.1.0
244 244 transaction==3.1.0
245 245 zope.interface==6.1.0
246 246 python-ldap==3.4.3
247 247 pyasn1==0.4.8
248 248 pyasn1-modules==0.2.8
249 249 pyasn1==0.4.8
250 250 python-memcached==1.59
251 251 six==1.16.0
252 252 python-pam==2.0.2
253 253 python3-saml==1.15.0
254 254 isodate==0.6.1
255 255 six==1.16.0
256 256 lxml==4.9.3
257 257 xmlsec==1.3.13
258 258 lxml==4.9.3
259 259 pyyaml==6.0.1
260 260 redis==5.0.1
261 261 regex==2022.10.31
262 262 routes==2.5.1
263 263 repoze.lru==0.7
264 264 six==1.16.0
265 265 simplejson==3.19.1
266 266 sshpubkeys==3.3.1
267 267 cryptography==40.0.2
268 268 cffi==1.16.0
269 269 pycparser==2.21
270 270 ecdsa==0.18.0
271 271 six==1.16.0
272 272 sqlalchemy==1.4.50
273 273 greenlet==3.0.3
274 274 typing_extensions==4.8.0
275 275 supervisor==4.2.5
276 276 tzlocal==4.3
277 277 pytz-deprecation-shim==0.1.0.post0
278 278 tzdata==2023.3
279 279 unidecode==1.3.6
280 280 urlobject==2.4.3
281 281 waitress==2.1.2
282 282 weberror==0.13.1
283 283 paste==3.7.1
284 284 six==1.16.0
285 285 pygments==2.15.1
286 286 tempita==0.5.2
287 287 webob==1.8.7
288 288 webhelpers2==2.0
289 289 markupsafe==2.1.2
290 290 six==1.16.0
291 291 whoosh==2.7.4
292 292 zope.cachedescriptors==5.0.0
293 293
294 294 ## uncomment to add the debug libraries
295 295 #-r requirements_debug.txt
@@ -1,543 +1,562 b''
1 1
2 2
3 3 # Copyright (C) 2011-2023 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21
22 22 """
23 23 Renderer for markup languages with ability to parse using rst or markdown
24 24 """
25 25
26 26 import re
27 27 import os
28 28 import lxml
29 29 import logging
30 30 import urllib.parse
31 31 import pycmarkgfm
32 32
33 33 from mako.lookup import TemplateLookup
34 34 from mako.template import Template as MakoTemplate
35 35
36 36 from docutils.core import publish_parts
37 37 from docutils.parsers.rst import directives
38 38 from docutils import writers
39 39 from docutils.writers import html4css1
40 40 import markdown
41 41
42 42 from rhodecode.lib.utils2 import safe_str, MENTIONS_REGEX
43 43
44 44 log = logging.getLogger(__name__)
45 45
46 46 # default renderer used to generate automated comments
47 47 DEFAULT_COMMENTS_RENDERER = 'rst'
48 48
49 49 try:
50 50 from lxml.html import fromstring
51 51 from lxml.html import tostring
52 52 except ImportError:
53 53 log.exception('Failed to import lxml')
54 54 fromstring = None
55 55 tostring = None
56 56
57 57
58 58 class CustomHTMLTranslator(writers.html4css1.HTMLTranslator):
59 59 """
60 60 Custom HTML Translator used for sandboxing potential
61 61 JS injections in ref links
62 62 """
63 63 def visit_literal_block(self, node):
64 64 self.body.append(self.starttag(node, 'pre', CLASS='codehilite literal-block'))
65 65
66 66 def visit_reference(self, node):
67 67 if 'refuri' in node.attributes:
68 68 refuri = node['refuri']
69 69 if ':' in refuri:
70 70 prefix, link = refuri.lstrip().split(':', 1)
71 71 prefix = prefix or ''
72 72
73 73 if prefix.lower() == 'javascript':
74 74 # we don't allow javascript type of refs...
75 75 node['refuri'] = 'javascript:alert("SandBoxedJavascript")'
76 76
77 77 # old style class requires this...
78 78 return html4css1.HTMLTranslator.visit_reference(self, node)
79 79
80 80
81 81 class RhodeCodeWriter(writers.html4css1.Writer):
82 82 def __init__(self):
83 83 super(RhodeCodeWriter, self).__init__()
84 84 self.translator_class = CustomHTMLTranslator
85 85
86 86
87 87 def relative_links(html_source, server_paths):
88 88 if not html_source:
89 89 return html_source
90 90
91 91 if not fromstring and tostring:
92 92 return html_source
93 93
94 94 try:
95 95 doc = lxml.html.fromstring(html_source)
96 96 except Exception:
97 97 return html_source
98 98
99 99 for el in doc.cssselect('img, video'):
100 100 src = el.attrib.get('src')
101 101 if src:
102 102 el.attrib['src'] = relative_path(src, server_paths['raw'])
103 103
104 104 for el in doc.cssselect('a:not(.gfm)'):
105 105 src = el.attrib.get('href')
106 106 if src:
107 107 raw_mode = el.attrib['href'].endswith('?raw=1')
108 108 if raw_mode:
109 109 el.attrib['href'] = relative_path(src, server_paths['raw'])
110 110 else:
111 111 el.attrib['href'] = relative_path(src, server_paths['standard'])
112 112
113 113 return lxml.html.tostring(doc, encoding='unicode')
114 114
115 115
116 116 def relative_path(path, request_path, is_repo_file=None):
117 117 """
118 118 relative link support, path is a rel path, and request_path is current
119 119 server path (not absolute)
120 120
121 121 e.g.
122 122
123 123 path = '../logo.png'
124 124 request_path= '/repo/files/path/file.md'
125 125 produces: '/repo/files/logo.png'
126 126 """
127 127 # TODO(marcink): unicode/str support ?
128 128 # maybe=> safe_str(urllib.quote(safe_str(final_path), '/:'))
129 129
130 130 def dummy_check(p):
131 131 return True # assume default is a valid file path
132 132
133 133 is_repo_file = is_repo_file or dummy_check
134 134 if not path:
135 135 return request_path
136 136
137 137 path = safe_str(path)
138 138 request_path = safe_str(request_path)
139 139
140 140 if path.startswith(('data:', 'javascript:', '#', ':')):
141 141 # skip data, anchor, invalid links
142 142 return path
143 143
144 144 is_absolute = bool(urllib.parse.urlparse(path).netloc)
145 145 if is_absolute:
146 146 return path
147 147
148 148 if not request_path:
149 149 return path
150 150
151 151 if path.startswith('/'):
152 152 path = path[1:]
153 153
154 154 if path.startswith('./'):
155 155 path = path[2:]
156 156
157 157 parts = request_path.split('/')
158 158 # compute how deep we need to traverse the request_path
159 159 depth = 0
160 160
161 161 if is_repo_file(request_path):
162 162 # if request path is a VALID file, we use a relative path with
163 163 # one level up
164 164 depth += 1
165 165
166 166 while path.startswith('../'):
167 167 depth += 1
168 168 path = path[3:]
169 169
170 170 if depth > 0:
171 171 parts = parts[:-depth]
172 172
173 173 parts.append(path)
174 174 final_path = '/'.join(parts).lstrip('/')
175 175
176 176 return '/' + final_path
177 177
178 178
179 179 _cached_markdown_renderer = None
180 180
181 181
182 182 def get_markdown_renderer(extensions, output_format):
183 183 global _cached_markdown_renderer
184 184
185 185 if _cached_markdown_renderer is None:
186 186 _cached_markdown_renderer = markdown.Markdown(
187 187 extensions=extensions + ['legacy_attrs'],
188 188 output_format=output_format)
189 189 return _cached_markdown_renderer
190 190
191 191
192 192 def get_markdown_renderer_flavored(extensions, output_format):
193 193 """
194 194 Dummy wrapper to mimic markdown API and render github HTML rendered
195 195
196 196 """
197 197 md = get_markdown_renderer(extensions, output_format)
198 198
199 199 class GFM(object):
200 200 def convert(self, source):
201 201 with pycmarkgfm.parse_gfm(source, options=pycmarkgfm.options.hardbreaks) as document:
202 202 parsed_md = document.to_commonmark()
203 203 return md.convert(parsed_md)
204 204
205 205 return GFM()
206 206
207 207
208 208 class MarkupRenderer(object):
209 209 RESTRUCTUREDTEXT_DISALLOWED_DIRECTIVES = ['include', 'meta', 'raw']
210 210
211 211 MARKDOWN_PAT = re.compile(r'\.(md|mkdn?|mdown|markdown)$', re.IGNORECASE)
212 212 RST_PAT = re.compile(r'\.re?st$', re.IGNORECASE)
213 213 JUPYTER_PAT = re.compile(r'\.(ipynb)$', re.IGNORECASE)
214 214 PLAIN_PAT = re.compile(r'^readme$', re.IGNORECASE)
215 215
216 216 URL_PAT = re.compile(r'(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]'
217 217 r'|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)')
218 218
219 219 MENTION_PAT = re.compile(MENTIONS_REGEX)
220 220
221 221 extensions = ['markdown.extensions.codehilite', 'markdown.extensions.extra',
222 222 'markdown.extensions.def_list', 'markdown.extensions.sane_lists']
223 223
224 224 output_format = 'html4'
225 225
226 226 # extension together with weights. Lower is first means we control how
227 227 # extensions are attached to readme names with those.
228 228 PLAIN_EXTS = [
229 229 # prefer no extension
230 230 ('', 0), # special case that renders READMES names without extension
231 231 ('.text', 2), ('.TEXT', 2),
232 232 ('.txt', 3), ('.TXT', 3)
233 233 ]
234 234
235 235 RST_EXTS = [
236 236 ('.rst', 1), ('.rest', 1),
237 237 ('.RST', 2), ('.REST', 2)
238 238 ]
239 239
240 240 MARKDOWN_EXTS = [
241 241 ('.md', 1), ('.MD', 1),
242 242 ('.mkdn', 2), ('.MKDN', 2),
243 243 ('.mdown', 3), ('.MDOWN', 3),
244 244 ('.markdown', 4), ('.MARKDOWN', 4)
245 245 ]
246 246
247 247 def _detect_renderer(self, source, filename=None):
248 248 """
249 249 runs detection of what renderer should be used for generating html
250 250 from a markup language
251 251
252 252 filename can be also explicitly a renderer name
253 253
254 254 :param source:
255 255 :param filename:
256 256 """
257 257
258 258 if MarkupRenderer.MARKDOWN_PAT.findall(filename):
259 259 detected_renderer = 'markdown'
260 260 elif MarkupRenderer.RST_PAT.findall(filename):
261 261 detected_renderer = 'rst'
262 262 elif MarkupRenderer.JUPYTER_PAT.findall(filename):
263 263 detected_renderer = 'jupyter'
264 264 elif MarkupRenderer.PLAIN_PAT.findall(filename):
265 265 detected_renderer = 'plain'
266 266 else:
267 267 detected_renderer = 'plain'
268 268
269 269 return getattr(MarkupRenderer, detected_renderer)
270 270
271 271 @classmethod
272 272 def sanitize_html(cls, text):
273 273 from .html_filters import sanitize_html
274 274 return sanitize_html(text, markdown=True)
275 275
276 276 @classmethod
277 277 def renderer_from_filename(cls, filename, exclude):
278 278 """
279 279 Detect renderer markdown/rst from filename and optionally use exclude
280 280 list to remove some options. This is mostly used in helpers.
281 281 Returns None when no renderer can be detected.
282 282 """
283 283 def _filter(elements):
284 284 if isinstance(exclude, (list, tuple)):
285 285 return [x for x in elements if x not in exclude]
286 286 return elements
287 287
288 288 if filename.endswith(
289 289 tuple(_filter([x[0] for x in cls.MARKDOWN_EXTS if x[0]]))):
290 290 return 'markdown'
291 291 if filename.endswith(tuple(_filter([x[0] for x in cls.RST_EXTS if x[0]]))):
292 292 return 'rst'
293 293
294 294 return None
295 295
296 296 def render(self, source, filename=None):
297 297 """
298 298 Renders a given filename using detected renderer
299 299 it detects renderers based on file extension or mimetype.
300 300 At last it will just do a simple html replacing new lines with <br/>
301 301 """
302 302
303 303 renderer = self._detect_renderer(source, filename)
304 304 readme_data = renderer(source)
305 305 return readme_data
306 306
307 307 @classmethod
308 308 def urlify_text(cls, text):
309 309 def url_func(match_obj):
310 310 url_full = match_obj.groups()[0]
311 311 return f'<a href="{url_full}">{url_full}</a>'
312 312
313 313 return cls.URL_PAT.sub(url_func, text)
314 314
315 315 @classmethod
316 316 def convert_mentions(cls, text, mode):
317 317 mention_pat = cls.MENTION_PAT
318 318
319 319 def wrapp(match_obj):
320 320 uname = match_obj.groups()[0]
321 321 hovercard_url = "pyroutes.url('hovercard_username', {'username': '%s'});" % uname
322 322
323 323 if mode == 'markdown':
324 324 tmpl = '<strong class="tooltip-hovercard" data-hovercard-alt="{uname}" data-hovercard-url="{hovercard_url}">@{uname}</strong>'
325 325 elif mode == 'rst':
326 326 tmpl = ' **@{uname}** '
327 327 else:
328 328 raise ValueError('mode must be rst or markdown')
329 329
330 330 return tmpl.format(**{'uname': uname,
331 331 'hovercard_url': hovercard_url})
332 332
333 333 return mention_pat.sub(wrapp, text).strip()
334 334
335 335 @classmethod
336 336 def plain(cls, source, universal_newline=True, leading_newline=True):
337 337 source = safe_str(source)
338 338 if universal_newline:
339 339 newline = '\n'
340 340 source = newline.join(source.splitlines())
341 341
342 342 rendered_source = cls.urlify_text(source)
343 343 source = ''
344 344 if leading_newline:
345 345 source += '<br />'
346 346 source += rendered_source.replace("\n", '<br />')
347 347
348 348 rendered = cls.sanitize_html(source)
349 349 return rendered
350 350
351 351 @classmethod
352 352 def markdown(cls, source, safe=True, flavored=True, mentions=False,
353 353 clean_html=True):
354 354 """
355 355 returns markdown rendered code cleaned by the bleach library
356 356 """
357 357
358 358 if flavored:
359 359 markdown_renderer = get_markdown_renderer_flavored(
360 360 cls.extensions, cls.output_format)
361 361 else:
362 362 markdown_renderer = get_markdown_renderer(
363 363 cls.extensions, cls.output_format)
364 364
365 365 if mentions:
366 366 mention_hl = cls.convert_mentions(source, mode='markdown')
367 367 # we extracted mentions render with this using Mentions false
368 368 return cls.markdown(mention_hl, safe=safe, flavored=flavored,
369 369 mentions=False)
370 370
371 371 try:
372 372 rendered = markdown_renderer.convert(source)
373 373
374 374 except Exception:
375 375 log.exception('Error when rendering Markdown')
376 376 if safe:
377 377 log.debug('Fallback to render in plain mode')
378 378 rendered = cls.plain(source)
379 379 else:
380 380 raise
381 381
382 382 if clean_html:
383 383 rendered = cls.sanitize_html(rendered)
384 384 return rendered
385 385
386 386 @classmethod
387 387 def rst(cls, source, safe=True, mentions=False, clean_html=False):
388 388
389 389 if mentions:
390 390 mention_hl = cls.convert_mentions(source, mode='rst')
391 391 # we extracted mentions render with this using Mentions false
392 392 return cls.rst(mention_hl, safe=safe, mentions=False)
393 393
394 394 source = safe_str(source)
395 395 try:
396 396 docutils_settings = dict(
397 397 [(alias, None) for alias in
398 398 cls.RESTRUCTUREDTEXT_DISALLOWED_DIRECTIVES])
399 399
400 400 docutils_settings.update({
401 401 'input_encoding': 'unicode',
402 402 'report_level': 4,
403 403 'syntax_highlight': 'short',
404 404 })
405 405
406 406 for k, v in list(docutils_settings.items()):
407 407 directives.register_directive(k, v)
408 408
409 409 parts = publish_parts(source=source,
410 410 writer=RhodeCodeWriter(),
411 411 settings_overrides=docutils_settings)
412 412 rendered = parts["fragment"]
413 413 if clean_html:
414 414 rendered = cls.sanitize_html(rendered)
415 415 return parts['html_title'] + rendered
416 416 except Exception:
417 417 log.exception('Error when rendering RST')
418 418 if safe:
419 419 log.debug('Fallback to render in plain mode')
420 420 return cls.plain(source)
421 421 else:
422 422 raise
423 423
424 424 @classmethod
425 425 def jupyter(cls, source, safe=True):
426 426 from rhodecode.lib import helpers
427 from .html_sanitizer_defs import markdown_attrs, all_tags, all_styles
427 428
428 429 from traitlets import default, config
429 430 import nbformat
430 431 from nbconvert import HTMLExporter
431 432 from nbconvert.preprocessors import Preprocessor
433 from nbconvert.preprocessors.sanitize import SanitizeHTML
432 434
433 435 class CustomHTMLExporter(HTMLExporter):
434 436
435 437 @default("template_file")
436 438 def _template_file_default(self):
437 439 if self.template_extension:
438 440 return "basic/index" + self.template_extension
439 441
440 442 class Sandbox(Preprocessor):
441 443
442 def preprocess(self, nb, resources):
444 def preprocess_cell(self, cell, resources, cell_index):
445 if not safe:
446 return cell, resources
443 447 sandbox_text = 'SandBoxed(IPython.core.display.Javascript object)'
444 for cell in nb['cells']:
445 if not safe:
446 continue
448 if cell.cell_type == "markdown":
449 cell.source = cls.sanitize_html(cell.source)
450 return cell, resources
447 451
448 if 'outputs' in cell:
449 for cell_output in cell['outputs']:
450 if 'data' in cell_output:
451 if 'application/javascript' in cell_output['data']:
452 cell_output['data']['text/plain'] = sandbox_text
453 cell_output['data'].pop('application/javascript', None)
454
455 if 'source' in cell and cell['cell_type'] == 'markdown':
456 # sanitize similar like in markdown
457 cell['source'] = cls.sanitize_html(cell['source'])
458
459 return nb, resources
452 for cell_output in cell.outputs:
453 if 'data' in cell_output:
454 if 'application/javascript' in cell_output['data']:
455 cell_output['data']['text/plain'] = sandbox_text
456 cell_output['data'].pop('application/javascript', None)
457 return cell, resources
460 458
461 459 def _sanitize_resources(input_resources):
462 460 """
463 461 Skip/sanitize some of the CSS generated and included in jupyter
464 462 so it doesn't mess up UI so much
465 463 """
466 464
467 465 # TODO(marcink): probably we should replace this with whole custom
468 466 # CSS set that doesn't screw up, but jupyter generated html has some
469 467 # special markers, so it requires Custom HTML exporter template with
470 468 # _default_template_path_default, to achieve that
471 469
472 470 # strip the reset CSS
473 471 input_resources[0] = input_resources[0][input_resources[0].find('/*! Source'):]
474 472 return input_resources
475 473
476 474 def as_html(notebook):
477 475 conf = config.Config()
478 conf.CustomHTMLExporter.default_preprocessors = [Sandbox]
476 # TODO: Keep an eye on the order of preprocessors
477 conf.CustomHTMLExporter.default_preprocessors = [Sandbox, SanitizeHTML]
479 478 conf.Sandbox.enabled = True
479 conf.SanitizeHTML.enabled = True
480 conf.SanitizeHTML.attributes = markdown_attrs
481 conf.SanitizeHTML.tags = all_tags
482 conf.SanitizeHTML.styles = all_styles
483 conf.SanitizeHTML.sanitized_output_types = {
484 "text/html",
485 "text/markdown",
486 }
487 conf.SanitizeHTML.safe_output_keys = {
488 "metadata",
489 "text/plain",
490 "text/latex",
491 "application/json",
492 "image/png",
493 "image/jpg"
494 "image/jpeg",
495 "image/svg",
496 "image/svg+xml"
497 }
498
480 499 html_exporter = CustomHTMLExporter(config=conf)
481 500
482 501 (body, resources) = html_exporter.from_notebook_node(notebook)
483 502
484 503 header = '<!-- ## IPYTHON NOTEBOOK RENDERING ## -->'
485 504 js = MakoTemplate(r'''
486 505 <!-- MathJax configuration -->
487 506 <script type="text/x-mathjax-config">
488 507 MathJax.Hub.Config({
489 508 jax: ["input/TeX","output/HTML-CSS", "output/PreviewHTML"],
490 509 extensions: ["tex2jax.js","MathMenu.js","MathZoom.js", "fast-preview.js", "AssistiveMML.js", "[Contrib]/a11y/accessibility-menu.js"],
491 510 TeX: {
492 511 extensions: ["AMSmath.js","AMSsymbols.js","noErrors.js","noUndefined.js"]
493 512 },
494 513 tex2jax: {
495 514 inlineMath: [ ['$','$'], ["\\(","\\)"] ],
496 515 displayMath: [ ['$$','$$'], ["\\[","\\]"] ],
497 516 processEscapes: true,
498 517 processEnvironments: true
499 518 },
500 519 // Center justify equations in code and markdown cells. Elsewhere
501 520 // we use CSS to left justify single line equations in code cells.
502 521 displayAlign: 'center',
503 522 "HTML-CSS": {
504 523 styles: {'.MathJax_Display': {"margin": 0}},
505 524 linebreaks: { automatic: true },
506 525 availableFonts: ["STIX", "TeX"]
507 526 },
508 527 showMathMenu: false
509 528 });
510 529 </script>
511 530 <!-- End of MathJax configuration -->
512 531 <script src="${h.asset('js/src/math_jax/MathJax.js')}"></script>
513 532 ''').render(h=helpers)
514 533
515 534 css = MakoTemplate(r'''
516 535 <link rel="stylesheet" type="text/css" href="${h.asset('css/style-ipython.css', ver=ver)}" media="screen"/>
517 536 ''').render(h=helpers, ver='ver1')
518 537
519 538 body = '\n'.join([header, css, js, body])
520 539 return body, resources
521 540
522 541 # TODO: In the event of a newer jupyter notebook version, consider increasing the as_version parameter
523 542 notebook = nbformat.reads(source, as_version=4)
524 543 (body, resources) = as_html(notebook)
525 544 return body
526 545
527 546
528 547 class RstTemplateRenderer(object):
529 548
530 549 def __init__(self):
531 550 base = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
532 551 rst_template_dirs = [os.path.join(base, 'templates', 'rst_templates')]
533 552 self.template_store = TemplateLookup(
534 553 directories=rst_template_dirs,
535 554 input_encoding='utf-8',
536 555 imports=['from rhodecode.lib import helpers as h'])
537 556
538 557 def _get_template(self, templatename):
539 558 return self.template_store.get_template(templatename)
540 559
541 560 def render(self, template_name, **kwargs):
542 561 template = self._get_template(template_name)
543 562 return template.render(**kwargs)
@@ -1,785 +1,897 b''
1 1
2 2 # Copyright (C) 2010-2023 RhodeCode GmbH
3 3 #
4 4 # This program is free software: you can redistribute it and/or modify
5 5 # it under the terms of the GNU Affero General Public License, version 3
6 6 # (only), as published by the Free Software Foundation.
7 7 #
8 8 # This program is distributed in the hope that it will be useful,
9 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 11 # GNU General Public License for more details.
12 12 #
13 13 # You should have received a copy of the GNU Affero General Public License
14 14 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 15 #
16 16 # This program is dual-licensed. If you wish to learn more about the
17 17 # RhodeCode Enterprise Edition, including its added features, Support services,
18 18 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 19
20 import mock
20 21 import pytest
21 22
22 23 from rhodecode.lib.markup_renderer import (
23 24 MarkupRenderer, RstTemplateRenderer, relative_path, relative_links)
24 25
25 26
26 27 @pytest.mark.parametrize(
27 28 "filename, expected_renderer",
28 29 [
29 30 ('readme.md', 'markdown'),
30 31 ('readme.Md', 'markdown'),
31 32 ('readme.MdoWn', 'markdown'),
32 33 ('readme.rst', 'rst'),
33 34 ('readme.Rst', 'rst'),
34 35 ('readme.rest', 'rst'),
35 36 ('readme.rest', 'rst'),
36 37
37 38 ('markdown.xml', 'plain'),
38 39 ('rest.xml', 'plain'),
39 40 ('readme.xml', 'plain'),
40 41
41 42 ('readme', 'plain'),
42 43 ('README', 'plain'),
43 44 ('readme.mdx', 'plain'),
44 45 ('readme.rstx', 'plain'),
45 46 ('readmex', 'plain'),
46 47 ])
47 48 def test_detect_renderer(filename, expected_renderer):
48 49 detected_renderer = MarkupRenderer()._detect_renderer(
49 50 '', filename=filename).__name__
50 51 assert expected_renderer == detected_renderer
51 52
52 53
53 54 def test_markdown_xss_link():
54 55 xss_md = "[link](javascript:alert('XSS: pwned!'))"
55 56 rendered_html = MarkupRenderer.markdown(xss_md)
56 57 assert 'href="javascript:alert(\'XSS: pwned!\')"' not in rendered_html
57 58
58 59
59 60 def test_markdown_xss_inline_html():
60 61 xss_md = '\n'.join([
61 62 '> <a name="n"',
62 63 '> href="javascript:alert(\'XSS: pwned!\')">link</a>'])
63 64 rendered_html = MarkupRenderer.markdown(xss_md)
64 65 assert 'href="javascript:alert(\'XSS: pwned!\')">' not in rendered_html
65 66
66 67
67 68 def test_markdown_inline_html():
68 69 xss_md = '\n'.join(['> <a name="n"',
69 70 '> onload="javascript:alert()" href="https://rhodecode.com">link</a>'])
70 71 rendered_html = MarkupRenderer.markdown(xss_md)
71 72 assert '<a name="n" href="https://rhodecode.com">link</a>' in rendered_html
72 73
73 74
74 75 def test_markdown_bleach_renders_correct():
75 76 test_md = """
76 77 This is intended as a quick reference and showcase. For more complete info, see [John Gruber's original spec](http://daringfireball.net/projects/markdown/) and the [Github-flavored Markdown info page](http://github.github.com/github-flavored-markdown/).
77 78
78 79 Note that there is also a [Cheatsheet specific to Markdown Here](./Markdown-Here-Cheatsheet) if that's what you're looking for. You can also check out [more Markdown tools](./Other-Markdown-Tools).
79 80
80 81 ##### Table of Contents
81 82 [Headers](#headers)
82 83 [Emphasis](#emphasis)
83 84 [Lists](#lists)
84 85 [Links](#links)
85 86 [Images](#images)
86 87 [Code and Syntax Highlighting](#code)
87 88 [Tables](#tables)
88 89 [Blockquotes](#blockquotes)
89 90 [Inline HTML](#html)
90 91 [Horizontal Rule](#hr)
91 92 [Line Breaks](#lines)
92 93 [Youtube videos](#videos)
93 94
94 95
95 96 ## Headers
96 97
97 98 ```no-highlight
98 99 # H1
99 100 ## H2
100 101 ### H3
101 102 #### H4
102 103 ##### H5
103 104 ###### H6
104 105
105 106 Alternatively, for H1 and H2, an underline-ish style:
106 107
107 108 Alt-H1
108 109 ======
109 110
110 111 Alt-H2
111 112 ------
112 113 ```
113 114
114 115 # H1
115 116 ## H2
116 117 ### H3
117 118 #### H4
118 119 ##### H5
119 120 ###### H6
120 121
121 122 Alternatively, for H1 and H2, an underline-ish style:
122 123
123 124 Alt-H1
124 125 ======
125 126
126 127 Alt-H2
127 128 ------
128 129
129 130 ## Emphasis
130 131
131 132 ```no-highlight
132 133 Emphasis, aka italics, with *asterisks* or _underscores_.
133 134
134 135 Strong emphasis, aka bold, with **asterisks** or __underscores__.
135 136
136 137 Combined emphasis with **asterisks and _underscores_**.
137 138
138 139 Strikethrough uses two tildes. ~~Scratch this.~~
139 140 ```
140 141
141 142 Emphasis, aka italics, with *asterisks* or _underscores_.
142 143
143 144 Strong emphasis, aka bold, with **asterisks** or __underscores__.
144 145
145 146 Combined emphasis with **asterisks and _underscores_**.
146 147
147 148 Strikethrough uses two tildes. ~~Scratch this.~~
148 149
149 150
150 151 ## Lists
151 152
152 153 (In this example, leading and trailing spaces are shown with with dots: β‹…)
153 154
154 155 ```no-highlight
155 156 1. First ordered list item
156 157 2. Another item
157 158 β‹…β‹…* Unordered sub-list.
158 159 1. Actual numbers don't matter, just that it's a number
159 160 β‹…β‹…1. Ordered sub-list
160 161 4. And another item.
161 162
162 163 β‹…β‹…β‹…You can have properly indented paragraphs within list items. Notice the blank line above, and the leading spaces (at least one, but we'll use three here to also align the raw Markdown).
163 164
164 165 β‹…β‹…β‹…To have a line break without a paragraph, you will need to use two trailing spaces.β‹…β‹…
165 166 β‹…β‹…β‹…Note that this line is separate, but within the same paragraph.β‹…β‹…
166 167 β‹…β‹…β‹…(This is contrary to the typical GFM line break behaviour, where trailing spaces are not required.)
167 168
168 169 * Unordered list can use asterisks
169 170 - Or minuses
170 171 + Or pluses
171 172 ```
172 173
173 174 1. First ordered list item
174 175 2. Another item
175 176 * Unordered sub-list.
176 177 1. Actual numbers don't matter, just that it's a number
177 178 1. Ordered sub-list
178 179 4. And another item.
179 180
180 181 You can have properly indented paragraphs within list items. Notice the blank line above, and the leading spaces (at least one, but we'll use three here to also align the raw Markdown).
181 182
182 183 To have a line break without a paragraph, you will need to use two trailing spaces.
183 184 Note that this line is separate, but within the same paragraph.
184 185 (This is contrary to the typical GFM line break behaviour, where trailing spaces are not required.)
185 186
186 187 * Unordered list can use asterisks
187 188 - Or minuses
188 189 + Or pluses
189 190
190 191
191 192 ## Links
192 193
193 194 There are two ways to create links.
194 195
195 196 ```no-highlight
196 197 [I'm an inline-style link](https://www.google.com)
197 198
198 199 [I'm an inline-style link with title](https://www.google.com "Google's Homepage")
199 200
200 201 [I'm a reference-style link][Arbitrary case-insensitive reference text]
201 202
202 203 [I'm a relative reference to a repository file (LICENSE)](./LICENSE)
203 204
204 205 [I'm a relative reference to a repository file (IMAGE)](./img/logo.png)
205 206
206 207 [I'm a relative reference to a repository file (IMAGE2)](img/logo.png)
207 208
208 209 [You can use numbers for reference-style link definitions][1]
209 210
210 211 Or leave it empty and use the [link text itself].
211 212
212 213 URLs and URLs in angle brackets will automatically get turned into links.
213 214 http://www.example.com or <http://www.example.com> and sometimes
214 215 example.com (but not on Github, for example).
215 216
216 217 Some text to show that the reference links can follow later.
217 218
218 219 [arbitrary case-insensitive reference text]: https://www.mozilla.org
219 220 [1]: http://slashdot.org
220 221 [link text itself]: http://www.reddit.com
221 222 ```
222 223
223 224 [I'm an inline-style link](https://www.google.com)
224 225
225 226 [I'm an inline-style link with title](https://www.google.com "Google's Homepage")
226 227
227 228 [I'm a reference-style link][Arbitrary case-insensitive reference text]
228 229
229 230 [I'm a relative reference to a repository file (LICENSE)](./LICENSE)
230 231
231 232 [I'm a relative reference to a repository file (IMAGE)](./img/logo.png)
232 233
233 234 [I'm a relative reference to a repository file (IMAGE2)](img/logo.png)
234 235
235 236 [You can use numbers for reference-style link definitions][1]
236 237
237 238 Or leave it empty and use the [link text itself].
238 239
239 240 URLs and URLs in angle brackets will automatically get turned into links.
240 241 http://www.example.com or <http://www.example.com> and sometimes
241 242 example.com (but not on Github, for example).
242 243
243 244 Some text to show that the reference links can follow later.
244 245
245 246 [arbitrary case-insensitive reference text]: https://www.mozilla.org
246 247 [1]: http://slashdot.org
247 248 [link text itself]: http://www.reddit.com
248 249
249 250
250 251 ## Images
251 252
252 253 ```no-highlight
253 254 Here's our logo (hover to see the title text):
254 255
255 256 Inline-style:
256 257 ![alt text](https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png "Logo Title Text 1")
257 258
258 259 relative-src-style:
259 260 ![alt text](img/logo.png)
260 261
261 262 Reference-style:
262 263 ![alt text][logo]
263 264
264 265 [logo]: https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png "Logo Title Text 2"
265 266 ```
266 267
267 268 Here's our logo (hover to see the title text):
268 269
269 270 Inline-style:
270 271 ![alt text](https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png "Logo Title Text 1")
271 272
272 273 relative-src-style:
273 274 ![alt text](img/logo.png)
274 275
275 276 relative-src-style:
276 277 ![alt text](./img/logo.png)
277 278
278 279 Reference-style:
279 280 ![alt text][logo]
280 281
281 282 [logo]: https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png "Logo Title Text 2"
282 283
283 284
284 285 ## Code and Syntax Highlighting
285 286
286 287 Code blocks are part of the Markdown spec, but syntax highlighting isn't. However, many renderers -- like Github's and *Markdown Here* -- support syntax highlighting. Which languages are supported and how those language names should be written will vary from renderer to renderer. *Markdown Here* supports highlighting for dozens of languages (and not-really-languages, like diffs and HTTP headers); to see the complete list, and how to write the language names, see the [highlight.js demo page](http://softwaremaniacs.org/media/soft/highlight/test.html).
287 288
288 289 ```no-highlight
289 290 Inline `code` has `back-ticks around` it.
290 291 ```
291 292
292 293 Inline `code` has `back-ticks around` it.
293 294
294 295 Blocks of code are either fenced by lines with three back-ticks <code>```</code>, or are indented with four spaces. I recommend only using the fenced code blocks -- they're easier and only they support syntax highlighting.
295 296
296 297 ```javascript
297 298 var s = "JavaScript syntax highlighting";
298 299 console.log(s);
299 300 ```
300 301
301 302 ```python
302 303 s = "Python syntax highlighting"
303 304 print(s)
304 305
305 306 class Orm(object):
306 307 pass
307 308 ```
308 309
309 310 ```
310 311 No language indicated, so no syntax highlighting.
311 312 But let's throw in a &lt;b&gt;tag&lt;/b&gt;.
312 313 ```
313 314
314 315
315 316 ```javascript
316 317 var s = "JavaScript syntax highlighting";
317 318 alert(s);
318 319 ```
319 320
320 321 ```python
321 322 s = "Python syntax highlighting"
322 323 print(s)
323 324
324 325 class Orm(object):
325 326 pass
326 327 ```
327 328
328 329 ```
329 330 No language indicated, so no syntax highlighting in Markdown Here (varies on Github).
330 331 But let's throw in a <b>tag</b>.
331 332 ```
332 333
333 334
334 335 ## Tables
335 336
336 337 Tables aren't part of the core Markdown spec, but they are part of GFM and *Markdown Here* supports them. They are an easy way of adding tables to your email -- a task that would otherwise require copy-pasting from another application.
337 338
338 339 ```no-highlight
339 340 Colons can be used to align columns.
340 341
341 342 | Tables | Are | Cool |
342 343 | ------------- |:-------------:| -----:|
343 344 | col 3 is | right-aligned | $1600 |
344 345 | col 2 is | centered | $12 |
345 346 | zebra stripes | are neat | $1 |
346 347
347 348 There must be at least 3 dashes separating each header cell.
348 349 The outer pipes (|) are optional, and you don't need to make the
349 350 raw Markdown line up prettily. You can also use inline Markdown.
350 351
351 352 Markdown | Less | Pretty
352 353 --- | --- | ---
353 354 *Still* | `renders` | **nicely**
354 355 1 | 2 | 3
355 356 ```
356 357
357 358 Colons can be used to align columns.
358 359
359 360 | Tables | Are | Cool |
360 361 | ------------- |:-------------:| -----:|
361 362 | col 3 is | right-aligned | $1600 |
362 363 | col 2 is | centered | $12 |
363 364 | zebra stripes | are neat | $1 |
364 365
365 366 There must be at least 3 dashes separating each header cell. The outer pipes (|) are optional, and you don't need to make the raw Markdown line up prettily. You can also use inline Markdown.
366 367
367 368 Markdown | Less | Pretty
368 369 --- | --- | ---
369 370 *Still* | `renders` | **nicely**
370 371 1 | 2 | 3
371 372
372 373
373 374 ## Blockquotes
374 375
375 376 ```no-highlight
376 377 > Blockquotes are very handy in email to emulate reply text.
377 378 > This line is part of the same quote.
378 379
379 380 Quote break.
380 381
381 382 > This is a very long line that will still be quoted properly when it wraps. Oh boy let's keep writing to make sure this is long enough to actually wrap for everyone. Oh, you can *put* **Markdown** into a blockquote.
382 383 ```
383 384
384 385 > Blockquotes are very handy in email to emulate reply text.
385 386 > This line is part of the same quote.
386 387
387 388 Quote break.
388 389
389 390 > This is a very long line that will still be quoted properly when it wraps. Oh boy let's keep writing to make sure this is long enough to actually wrap for everyone. Oh, you can *put* **Markdown** into a blockquote.
390 391
391 392
392 393 ## Inline HTML
393 394
394 395 You can also use raw HTML in your Markdown, and it'll mostly work pretty well.
395 396
396 397 ```no-highlight
397 398 <dl>
398 399 <dt>Definition list</dt>
399 400 <dd>Is something people use sometimes.</dd>
400 401
401 402 <dt>Markdown in HTML</dt>
402 403 <dd>Does *not* work **very** well. Use HTML <em>tags</em>.</dd>
403 404 </dl>
404 405 ```
405 406
406 407 <dl>
407 408 <dt>Definition list</dt>
408 409 <dd>Is something people use sometimes.</dd>
409 410
410 411 <dt>Markdown in HTML</dt>
411 412 <dd>Does *not* work **very** well. Use HTML <em>tags</em>.</dd>
412 413 </dl>
413 414
414 415
415 416 ## Horizontal Rule
416 417
417 418 ```
418 419 Three or more...
419 420
420 421 ---
421 422
422 423 Hyphens
423 424
424 425 ***
425 426
426 427 Asterisks
427 428
428 429 ___
429 430
430 431 Underscores
431 432 ```
432 433
433 434 Three or more...
434 435
435 436 ---
436 437
437 438 Hyphens
438 439
439 440 ***
440 441
441 442 Asterisks
442 443
443 444 ___
444 445
445 446 Underscores
446 447
447 448
448 449 ## Line Breaks
449 450
450 451 My basic recommendation for learning how line breaks work is to experiment and discover -- hit &lt;Enter&gt; once (i.e., insert one newline), then hit it twice (i.e., insert two newlines), see what happens. You'll soon learn to get what you want. "Markdown Toggle" is your friend.
451 452
452 453 Here are some things to try out:
453 454
454 455 ```
455 456 Here's a line for us to start with.
456 457
457 458 This line is separated from the one above by two newlines, so it will be a *separate paragraph*.
458 459
459 460 This line is also a separate paragraph, but...
460 461 This line is only separated by a single newline, so it's a separate line in the *same paragraph*.
461 462 ```
462 463
463 464 Here's a line for us to start with.
464 465
465 466 This line is separated from the one above by two newlines, so it will be a *separate paragraph*.
466 467
467 468 This line is also begins a separate paragraph, but...
468 469 This line is only separated by a single newline, so it's a separate line in the *same paragraph*.
469 470
470 471 (Technical note: *Markdown Here* uses GFM line breaks, so there's no need to use MD's two-space line breaks.)
471 472
472 473
473 474 ## Youtube videos
474 475
475 476 They can't be added directly but you can add an image with a link to the video like this:
476 477
477 478 ```no-highlight
478 479 <a href="http://www.youtube.com/watch?feature=player_embedded&v=YOUTUBE_VIDEO_ID_HERE
479 480 " target="_blank"><img src="http://img.youtube.com/vi/YOUTUBE_VIDEO_ID_HERE/0.jpg"
480 481 alt="IMAGE ALT TEXT HERE" width="240" height="180" border="10" /></a>
481 482 ```
482 483
483 484 Or, in pure Markdown, but losing the image sizing and border:
484 485
485 486 ```no-highlight
486 487 [![IMAGE ALT TEXT HERE](http://img.youtube.com/vi/YOUTUBE_VIDEO_ID_HERE/0.jpg)](http://www.youtube.com/watch?v=YOUTUBE_VIDEO_ID_HERE)
487 488 ```
488 489
489 490 Referencing a bug by #bugID in your git commit links it to the slip. For example #1.
490 491
491 492 ---
492 493
493 494 License: [CC-BY](https://creativecommons.org/licenses/by/3.0/)
494 495 """
495 496 raw_rendered_html = MarkupRenderer.markdown(test_md, clean_html=False)
496 497 bleached_rendered_html = MarkupRenderer.markdown(test_md, clean_html=True)
497 498 assert raw_rendered_html == bleached_rendered_html
498 499
499 500
500 501 def test_rst_xss_link():
501 502 xss_rst = "`Link<javascript:alert('XSS: pwned!')>`_"
502 503 rendered_html = MarkupRenderer.rst(xss_rst)
503 504 assert "href=javascript:alert('XSS: pwned!')" not in rendered_html
504 505
505 506
506 507 @pytest.mark.xfail(reason='Bug in docutils. Waiting answer from the author')
507 508 def test_rst_xss_inline_html():
508 509 xss_rst = '<a href="javascript:alert(\'XSS: pwned!\')">link</a>'
509 510 rendered_html = MarkupRenderer.rst(xss_rst)
510 511 assert 'href="javascript:alert(' not in rendered_html
511 512
512 513
513 514 def test_rst_xss_raw_directive():
514 515 xss_rst = '\n'.join([
515 516 '.. raw:: html',
516 517 '',
517 518 ' <a href="javascript:alert(\'XSS: pwned!\')">link</a>'])
518 519 rendered_html = MarkupRenderer.rst(xss_rst)
519 520 assert 'href="javascript:alert(' not in rendered_html
520 521
521 522
522 523 def test_render_rst_template_without_files():
523 524 expected = u'''\
524 525 Pull request updated. Auto status change to |under_review|
525 526
526 527 .. role:: added
527 528 .. role:: removed
528 529 .. parsed-literal::
529 530
530 531 Changed commits:
531 532 * :added:`2 added`
532 533 * :removed:`3 removed`
533 534
534 535 No file changes found
535 536
536 537 .. |under_review| replace:: *"NEW STATUS"*'''
537 538
538 539 params = {
539 540 'under_review_label': 'NEW STATUS',
540 541 'added_commits': ['a', 'b'],
541 542 'removed_commits': ['a', 'b', 'c'],
542 543 'changed_files': [],
543 544 'added_files': [],
544 545 'modified_files': [],
545 546 'removed_files': [],
546 547 'ancestor_commit_id': 'aaabbbcccdddeee',
547 548 }
548 549 renderer = RstTemplateRenderer()
549 550 rendered = renderer.render('pull_request_update.mako', **params)
550 551 assert expected == rendered
551 552
552 553
553 554 def test_render_rst_template_with_files():
554 555 expected = u'''\
555 556 Pull request updated. Auto status change to |under_review|
556 557
557 558 .. role:: added
558 559 .. role:: removed
559 560 .. parsed-literal::
560 561
561 562 Changed commits:
562 563 * :added:`1 added`
563 564 * :removed:`3 removed`
564 565
565 566 Changed files:
566 567 * `A /path/a.py <#a_c-aaabbbcccddd-68ed34923b68>`_
567 568 * `A /path/b.js <#a_c-aaabbbcccddd-64f90608b607>`_
568 569 * `M /path/d.js <#a_c-aaabbbcccddd-85842bf30c6e>`_
569 570 * `M /path/Δ™.py <#a_c-aaabbbcccddd-d713adf009cd>`_
570 571 * `R /path/ΕΊ.py`
571 572
572 573 .. |under_review| replace:: *"NEW STATUS"*'''
573 574
574 575 added = ['/path/a.py', '/path/b.js']
575 576 modified = ['/path/d.js', u'/path/Δ™.py']
576 577 removed = [u'/path/ΕΊ.py']
577 578
578 579 params = {
579 580 'under_review_label': 'NEW STATUS',
580 581 'added_commits': ['a'],
581 582 'removed_commits': ['a', 'b', 'c'],
582 583 'changed_files': added + modified + removed,
583 584 'added_files': added,
584 585 'modified_files': modified,
585 586 'removed_files': removed,
586 587 'ancestor_commit_id': 'aaabbbcccdddeee',
587 588 }
588 589 renderer = RstTemplateRenderer()
589 590 rendered = renderer.render('pull_request_update.mako', **params)
590 591
591 592 assert expected == rendered
592 593
593 594
594 595 def test_render_rst_auto_status_template():
595 596 expected = u'''\
596 597 Auto status change to |new_status|
597 598
598 599 .. |new_status| replace:: *"NEW STATUS"*'''
599 600
600 601 params = {
601 602 'new_status_label': 'NEW STATUS',
602 603 'pull_request': None,
603 604 'commit_id': None,
604 605 }
605 606 renderer = RstTemplateRenderer()
606 607 rendered = renderer.render('auto_status_change.mako', **params)
607 608 assert expected == rendered
608 609
609 610
610 611 @pytest.mark.parametrize(
611 612 "src_path, server_path, is_path, expected",
612 613 [
613 614 ('source.png', '/repo/files/path', lambda p: False,
614 615 '/repo/files/path/source.png'),
615 616
616 617 ('source.png', 'mk/git/blob/master/README.md', lambda p: True,
617 618 '/mk/git/blob/master/source.png'),
618 619
619 620 ('./source.png', 'mk/git/blob/master/README.md', lambda p: True,
620 621 '/mk/git/blob/master/source.png'),
621 622
622 623 ('/source.png', 'mk/git/blob/master/README.md', lambda p: True,
623 624 '/mk/git/blob/master/source.png'),
624 625
625 626 ('./source.png', 'repo/files/path/source.md', lambda p: True,
626 627 '/repo/files/path/source.png'),
627 628
628 629 ('./source.png', '/repo/files/path/file.md', lambda p: True,
629 630 '/repo/files/path/source.png'),
630 631
631 632 ('../source.png', '/repo/files/path/file.md', lambda p: True,
632 633 '/repo/files/source.png'),
633 634
634 635 ('./../source.png', '/repo/files/path/file.md', lambda p: True,
635 636 '/repo/files/source.png'),
636 637
637 638 ('./source.png', '/repo/files/path/file.md', lambda p: True,
638 639 '/repo/files/path/source.png'),
639 640
640 641 ('../../../source.png', 'path/file.md', lambda p: True,
641 642 '/source.png'),
642 643
643 644 ('../../../../../source.png', '/path/file.md', None,
644 645 '/source.png'),
645 646
646 647 ('../../../../../source.png', 'files/path/file.md', None,
647 648 '/source.png'),
648 649
649 650 ('../../../../../https://google.com/image.png', 'files/path/file.md', None,
650 651 '/https://google.com/image.png'),
651 652
652 653 ('https://google.com/image.png', 'files/path/file.md', None,
653 654 'https://google.com/image.png'),
654 655
655 656 ('://foo', '/files/path/file.md', None,
656 657 '://foo'),
657 658
658 659 (u'ν•œκΈ€.png', '/files/path/file.md', None,
659 660 u'/files/path/ν•œκΈ€.png'),
660 661
661 662 ('my custom image.png', '/files/path/file.md', None,
662 663 '/files/path/my custom image.png'),
663 664 ])
664 665 def test_relative_path(src_path, server_path, is_path, expected):
665 666 path = relative_path(src_path, server_path, is_path)
666 667 assert path == expected
667 668
668 669
669 670 @pytest.mark.parametrize(
670 671 "src_html, expected_html",
671 672 [
672 673 ('<div></div>', '<div></div>'),
673 674 ('<img src="/file.png"></img>', '<img src="/path/raw/file.png">'),
674 675 ('<img src="data:abcd"/>', '<img src="data:abcd">'),
675 676 ('<a href="/file.png?raw=1"></a>', '<a href="/path/raw/file.png?raw=1"></a>'),
676 677 ('<a href="/file.png"></a>', '<a href="/path/file.png"></a>'),
677 678 ('<a href="#anchor"></a>', '<a href="#anchor"></a>'),
678 679 ('<a href="./README.md?raw=1"></a>', '<a href="/path/raw/README.md?raw=1"></a>'),
679 680 ('<a href="./README.md"></a>', '<a href="/path/README.md"></a>'),
680 681 ('<a href="../README.md"></a>', '<a href="/README.md"></a>'),
681 682
682 683 ])
683 684 def test_relative_links(src_html, expected_html):
684 685 server_paths = {'raw': '/path/raw/file.md', 'standard': '/path/file.md'}
685 686 assert relative_links(src_html, server_paths=server_paths) == expected_html
686 687
687 688
688 689 @pytest.mark.parametrize("notebook_source, expected_output", [
689 690 ("""
690 691 {
691 692 "nbformat": 3,
692 693 "nbformat_minor": 0,
693 694 "worksheets": [
694 695 {
695 696 "cells": [
696 697 {
697 698 "cell_type": "code",
698 699 "execution_count": 1,
699 700 "metadata": {},
700 701 "outputs": [
701 702 {
702 703 "name": "stdout",
703 704 "output_type": "stream",
704 705 "text": [
705 706 "Hello, World!\\n"
706 707 ]
707 708 }
708 709 ],
709 710 "input": "print('Hello, World!')"
710 711 }
711 712 ]
712 713 }
713 714 ],
714 715 "metadata": {
715 716 "kernelspec": {
716 717 "display_name": "Python 3",
717 718 "language": "python",
718 719 "name": "python3"
719 720 },
720 721 "language_info": {
721 722 "codemirror_mode": {
722 723 "name": "ipython",
723 724 "version": 3
724 725 },
725 726 "file_extension": ".py",
726 727 "mimetype": "text/x-python",
727 728 "name": "python",
728 729 "nbconvert_exporter": "python",
729 730 "pygments_lexer": "ipython3",
730 731 "version": "3.8.5"
731 732 }
732 733 }
733 734 }
734 735 """, "Hello, World!"),
735 736 ("""
736 737 {
737 738 "nbformat": 4,
738 739 "nbformat_minor": 1,
739 740 "cells": [
740 741 {
741 742 "cell_type": "code",
742 743 "execution_count": 1,
743 744 "metadata": {},
744 745 "outputs": [
745 746 {
746 747 "name": "stdout",
747 748 "output_type": "stream",
748 749 "text": [
749 750 "Hello, World!\\n"
750 751 ]
751 752 }
752 753 ],
753 754 "source": [
754 755 "print('Hello, World!')"
755 756 ]
756 757 }
757 758 ],
758 759 "metadata": {
759 760 "kernelspec": {
760 761 "display_name": "Python 3",
761 762 "language": "python",
762 763 "name": "python3"
763 764 },
764 765 "language_info": {
765 766 "codemirror_mode": {
766 767 "name": "ipython",
767 768 "version": 4
768 769 },
769 770 "file_extension": ".py",
770 771 "mimetype": "text/x-python",
771 772 "name": "python",
772 773 "nbconvert_exporter": "python",
773 774 "pygments_lexer": "ipython3",
774 775 "version": "3.9.1"
775 776 }
776 777 }
777 778 }
778 779 """, "Hello, World!")
779 780 ])
780 781 def test_jp_notebook_html_generation(notebook_source, expected_output):
781 import mock
782 782 with mock.patch('rhodecode.lib.helpers.asset'):
783 783 body = MarkupRenderer.jupyter(notebook_source)
784 784 assert "<!-- ## IPYTHON NOTEBOOK RENDERING ## -->" in body
785 785 assert expected_output in body
786
787
788 @pytest.mark.parametrize("notebook_source, expected_output", [
789 ({"cells": [
790 {
791 "cell_type": "code",
792 "execution_count": 0,
793 "metadata": {},
794 "outputs": [
795 {
796 "data": {
797 "text/html": [
798 "<select><iframe></select><img src=x: onerror=alert('xss')>\n"
799 ],
800 "text/plain": []
801 },
802 "metadata": {},
803 "output_type": "display_data"
804 }
805 ],
806 "source": [
807 ""
808 ]
809 }
810 ],
811 "metadata": {
812 "kernelspec": {
813 "display_name": "Python 3",
814 "language": "python",
815 "name": "python3"
816 },
817 "language_info": {
818 "codemirror_mode": {
819 "name": "ipython",
820 "version": 3
821 },
822 "file_extension": ".py",
823 "mimetype": "text/x-python",
824 "name": "python",
825 "nbconvert_exporter": "python",
826 "pygments_lexer": "ipython3",
827 "version": "3.9.6"
828 }
829 },
830 "nbformat": 4,
831 "nbformat_minor": 5
832 }, '<select></select><img alt="No description has been provided for this image"/>'),
833 ({
834 "cells": [
835 {
836 "cell_type": "markdown",
837 "metadata": {},
838 "source": [
839 "<label for=buttonid style=\"cursor: text\">not safe to click here</label>\n"
840 ]
841 },
842 {
843 "cell_type": "markdown",
844 "metadata": {
845 "highlighter": "codemirror"
846 },
847 "source": "<div class=\"jp-InputArea-editor\"><div class=\"CodeMirror cm-s-jupyter\"><div class=\"CodeMirror-scroll\"><div class=\"CodeMirror-sizer\"><div style=\"top:0px; position:relative\"><div class=\"CodeMirror-lines\"><div style=\"outline:none; position:relative\"><div class=\"CodeMirror-code\"><div style=\"position: relative\"><label for=buttonid style=\"cursor: text\"><pre class=\"CodeMirror-line\" style=\"background:transparent\"><span style=\"padding-right: 0.1px\"><span class=\"cm-builtin\">print</span>(<span class=\"cm-string\">&quot;Also not safe to click here&quot;</span>)</span></pre></label></div></div></div></div></div></div></div></div></div>"
848 },
849 {
850 "cell_type": "code",
851 "execution_count": 0,
852 "metadata": {
853 "xrender": True
854 },
855 "outputs": [
856 {
857 "data": {
858 "text/html": [
859 "<form id=jp-mk-edit action='javascript:alert(1)' style='display:none'><button type=submit id=buttonid></form>\n"
860 ],
861 "text/plain": []
862 },
863 "metadata": {},
864 "output_type": "display_data"
865 }
866 ],
867 "source": ""
868 }
869 ],
870 "metadata": {
871 "kernelspec": {
872 "display_name": "Python 3",
873 "language": "python",
874 "name": "python3"
875 },
876 "language_info": {
877 "codemirror_mode": {
878 "name": "ipython",
879 "version": 3
880 },
881 "file_extension": ".py",
882 "mimetype": "text/x-python",
883 "name": "python",
884 "nbconvert_exporter": "python",
885 "pygments_lexer": "ipython3",
886 "version": "3.9.6"
887 }
888 },
889 "nbformat": 4,
890 "nbformat_minor": 5
891 }, '<form style="display:none;">')
892 ])
893 def test_jp_notebook_against_xss(notebook_source, expected_output):
894 import json
895 with mock.patch('rhodecode.lib.helpers.asset'):
896 body = MarkupRenderer.jupyter(json.dumps(notebook_source))
897 assert expected_output in body
General Comments 0
You need to be logged in to leave comments. Login now