##// END OF EJS Templates
styling: use sandboxed style for ipython so it doesn't affect style of application anymore.
marcink -
r3784:cc55d3ca default
parent child Browse files
Show More

The requested changes are too big and content was truncated. Show full diff

1 NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff
@@ -1,296 +1,296 b''
1 1 # Nix environment for the community edition
2 2 #
3 3 # This shall be as lean as possible, just producing the enterprise-ce
4 4 # derivation. For advanced tweaks to pimp up the development environment we use
5 5 # "shell.nix" so that it does not have to clutter this file.
6 6 #
7 7 # Configuration, set values in "~/.nixpkgs/config.nix".
8 8 # example
9 9 # {
10 10 # # Thoughts on how to configure the dev environment
11 11 # rc = {
12 12 # codeInternalUrl = "https://usr:token@code.rhodecode.com/internal";
13 13 # sources = {
14 14 # rhodecode-vcsserver = "/home/user/work/rhodecode-vcsserver";
15 15 # rhodecode-enterprise-ce = "/home/user/work/rhodecode-enterprise-ce";
16 16 # rhodecode-enterprise-ee = "/home/user/work/rhodecode-enterprise-ee";
17 17 # };
18 18 # };
19 19 # }
20 20
21 21 args@
22 22 { system ? builtins.currentSystem
23 23 , pythonPackages ? "python27Packages"
24 24 , pythonExternalOverrides ? self: super: {}
25 25 , doCheck ? false
26 26 , ...
27 27 }:
28 28
29 29 let
30 30 pkgs_ = args.pkgs or (import <nixpkgs> { inherit system; });
31 31 in
32 32
33 33 let
34 34 pkgs = import <nixpkgs> {
35 35 overlays = [
36 36 (import ./pkgs/overlays.nix)
37 37 ];
38 38 inherit
39 39 (pkgs_)
40 40 system;
41 41 };
42 42
43 43 # Works with the new python-packages, still can fallback to the old
44 44 # variant.
45 45 basePythonPackagesUnfix = basePythonPackages.__unfix__ or (
46 46 self: basePythonPackages.override (a: { inherit self; }));
47 47
48 48 # Evaluates to the last segment of a file system path.
49 49 basename = path: with pkgs.lib; last (splitString "/" path);
50 50
51 51 # source code filter used as arugment to builtins.filterSource.
52 52 src-filter = path: type: with pkgs.lib;
53 53 let
54 54 ext = last (splitString "." path);
55 55 in
56 56 !builtins.elem (basename path) [
57 57 ".git" ".hg" "__pycache__" ".eggs" ".idea" ".dev"
58 58 "node_modules" "node_binaries"
59 59 "build" "data" "result" "tmp"] &&
60 60 !builtins.elem ext ["egg-info" "pyc"] &&
61 61 # TODO: johbo: This check is wrong, since "path" contains an absolute path,
62 62 # it would still be good to restore it since we want to ignore "result-*".
63 63 !hasPrefix "result" path;
64 64
65 65 sources =
66 66 let
67 67 inherit
68 68 (pkgs.lib)
69 69 all
70 70 isString
71 71 attrValues;
72 72 sourcesConfig = pkgs.config.rc.sources or {};
73 73 in
74 74 # Ensure that sources are configured as strings. Using a path
75 75 # would result in a copy into the nix store.
76 76 assert all isString (attrValues sourcesConfig);
77 77 sourcesConfig;
78 78
79 79 version = builtins.readFile "${rhodecode-enterprise-ce-src}/rhodecode/VERSION";
80 80 rhodecode-enterprise-ce-src = builtins.filterSource src-filter ./.;
81 81
82 82 nodeEnv = import ./pkgs/node-default.nix {
83 83 inherit
84 84 pkgs
85 85 system;
86 86 };
87 87 nodeDependencies = nodeEnv.shell.nodeDependencies;
88 88
89 89 rhodecode-testdata-src = sources.rhodecode-testdata or (
90 90 pkgs.fetchhg {
91 91 url = "https://code.rhodecode.com/upstream/rc_testdata";
92 92 rev = "v0.10.0";
93 93 sha256 = "0zn9swwvx4vgw4qn8q3ri26vvzgrxn15x6xnjrysi1bwmz01qjl0";
94 94 });
95 95
96 96 rhodecode-testdata = import "${rhodecode-testdata-src}/default.nix" {
97 97 inherit
98 98 doCheck
99 99 pkgs
100 100 pythonPackages;
101 101 };
102 102
103 103 pythonLocalOverrides = self: super: {
104 104 rhodecode-enterprise-ce =
105 105 let
106 106 linkNodePackages = ''
107 107 export RHODECODE_CE_PATH=${rhodecode-enterprise-ce-src}
108 108
109 109 echo "[BEGIN]: Link node packages and binaries"
110 110 # johbo: Linking individual packages allows us to run "npm install"
111 111 # inside of a shell to try things out. Re-entering the shell will
112 112 # restore a clean environment.
113 113 rm -fr node_modules
114 114 mkdir node_modules
115 115 ln -s ${nodeDependencies}/lib/node_modules/* node_modules/
116 116 export NODE_PATH=./node_modules
117 117
118 118 rm -fr node_binaries
119 119 mkdir node_binaries
120 120 ln -s ${nodeDependencies}/bin/* node_binaries/
121 121 echo "[DONE ]: Link node packages and binaries"
122 122 '';
123 123
124 124 releaseName = "RhodeCodeEnterpriseCE-${version}";
125 125 in super.rhodecode-enterprise-ce.override (attrs: {
126 126 inherit
127 127 doCheck
128 128 version;
129 129
130 130 name = "rhodecode-enterprise-ce-${version}";
131 131 releaseName = releaseName;
132 132 src = rhodecode-enterprise-ce-src;
133 133 dontStrip = true; # prevent strip, we don't need it.
134 134
135 135 # expose following attributed outside
136 136 passthru = {
137 137 inherit
138 138 rhodecode-testdata
139 139 linkNodePackages
140 140 myPythonPackagesUnfix
141 141 pythonLocalOverrides
142 142 pythonCommunityOverrides;
143 143
144 144 pythonPackages = self;
145 145 };
146 146
147 147 buildInputs =
148 148 attrs.buildInputs or [] ++ [
149 149 rhodecode-testdata
150 150 ];
151 151
152 152 #NOTE: option to inject additional propagatedBuildInputs
153 153 propagatedBuildInputs =
154 154 attrs.propagatedBuildInputs or [] ++ [
155 155
156 156 ];
157 157
158 158 LC_ALL = "en_US.UTF-8";
159 159 LOCALE_ARCHIVE =
160 160 if pkgs.stdenv.isLinux
161 161 then "${pkgs.glibcLocales}/lib/locale/locale-archive"
162 162 else "";
163 163
164 164 # Add bin directory to path so that tests can find 'rhodecode'.
165 165 preCheck = ''
166 166 export PATH="$out/bin:$PATH"
167 167 '';
168 168
169 169 # custom check phase for testing
170 170 checkPhase = ''
171 171 runHook preCheck
172 172 PYTHONHASHSEED=random py.test -vv -p no:sugar -r xw --cov-config=.coveragerc --cov=rhodecode --cov-report=term-missing rhodecode
173 173 runHook postCheck
174 174 '';
175 175
176 176 postCheck = ''
177 177 echo "Cleanup of rhodecode/tests"
178 178 rm -rf $out/lib/${self.python.libPrefix}/site-packages/rhodecode/tests
179 179 '';
180 180
181 181 preBuild = ''
182 182 echo "[BEGIN]: Building frontend assets"
183 183 ${linkNodePackages}
184 184 make web-build
185 185 rm -fr node_modules
186 186 rm -fr node_binaries
187 187 echo "[DONE ]: Building frontend assets"
188 188 '';
189 189
190 190 postInstall = ''
191 191 # check required files
192 192 STATIC_CHECK="/robots.txt /502.html
193 193 /js/scripts.js /js/rhodecode-components.js
194 /css/style.css /css/style-polymer.css"
194 /css/style.css /css/style-polymer.css /css/style-ipython.css"
195 195
196 196 for file in $STATIC_CHECK;
197 197 do
198 198 if [ ! -f rhodecode/public/$file ]; then
199 199 echo "Missing $file"
200 200 exit 1
201 201 fi
202 202 done
203 203
204 204 echo "Writing enterprise-ce meta information for rccontrol to nix-support/rccontrol"
205 205 mkdir -p $out/nix-support/rccontrol
206 206 cp -v rhodecode/VERSION $out/nix-support/rccontrol/version
207 207 echo "[DONE ]: enterprise-ce meta information for rccontrol written"
208 208
209 209 mkdir -p $out/etc
210 210 cp configs/production.ini $out/etc
211 211 echo "[DONE ]: saved enterprise-ce production.ini into $out/etc"
212 212
213 213 cp -Rf rhodecode/config/rcextensions $out/etc/rcextensions.tmpl
214 214 echo "[DONE ]: saved enterprise-ce rcextensions into $out/etc/rcextensions.tmpl"
215 215
216 216 # python based programs need to be wrapped
217 217 mkdir -p $out/bin
218 218
219 219 # required binaries from dependencies
220 220 ln -s ${self.supervisor}/bin/supervisorctl $out/bin/
221 221 ln -s ${self.supervisor}/bin/supervisord $out/bin/
222 222 ln -s ${self.pastescript}/bin/paster $out/bin/
223 223 ln -s ${self.channelstream}/bin/channelstream $out/bin/
224 224 ln -s ${self.celery}/bin/celery $out/bin/
225 225 ln -s ${self.gunicorn}/bin/gunicorn $out/bin/
226 226 ln -s ${self.pyramid}/bin/prequest $out/bin/
227 227 ln -s ${self.pyramid}/bin/pserve $out/bin/
228 228
229 229 echo "[DONE ]: created symlinks into $out/bin"
230 230 DEPS="$out/bin/supervisorctl \
231 231 $out/bin/supervisord \
232 232 $out/bin/paster \
233 233 $out/bin/channelstream \
234 234 $out/bin/celery \
235 235 $out/bin/gunicorn \
236 236 $out/bin/prequest \
237 237 $out/bin/pserve"
238 238
239 239 # wrap only dependency scripts, they require to have full PYTHONPATH set
240 240 # to be able to import all packages
241 241 for file in $DEPS;
242 242 do
243 243 wrapProgram $file \
244 244 --prefix PATH : $PATH \
245 245 --prefix PYTHONPATH : $PYTHONPATH \
246 246 --set PYTHONHASHSEED random
247 247 done
248 248
249 249 echo "[DONE ]: enterprise-ce binary wrapping"
250 250
251 251 # rhodecode-tools don't need wrapping
252 252 ln -s ${self.rhodecode-tools}/bin/rhodecode-* $out/bin/
253 253
254 254 # expose sources of CE
255 255 ln -s $out $out/etc/rhodecode_enterprise_ce_source
256 256
257 257 # expose static files folder
258 258 cp -Rf $out/lib/${self.python.libPrefix}/site-packages/rhodecode/public/ $out/etc/static
259 259 chmod 755 -R $out/etc/static
260 260
261 261 '';
262 262 });
263 263
264 264 };
265 265
266 266 basePythonPackages = with builtins;
267 267 if isAttrs pythonPackages then
268 268 pythonPackages
269 269 else
270 270 getAttr pythonPackages pkgs;
271 271
272 272 pythonGeneratedPackages = import ./pkgs/python-packages.nix {
273 273 inherit
274 274 pkgs;
275 275 inherit
276 276 (pkgs)
277 277 fetchurl
278 278 fetchgit
279 279 fetchhg;
280 280 };
281 281
282 282 pythonCommunityOverrides = import ./pkgs/python-packages-overrides.nix {
283 283 inherit pkgs basePythonPackages;
284 284 };
285 285
286 286 # Apply all overrides and fix the final package set
287 287 myPythonPackagesUnfix = with pkgs.lib;
288 288 (extends pythonExternalOverrides
289 289 (extends pythonLocalOverrides
290 290 (extends pythonCommunityOverrides
291 291 (extends pythonGeneratedPackages
292 292 basePythonPackagesUnfix))));
293 293
294 294 myPythonPackages = (pkgs.lib.fix myPythonPackagesUnfix);
295 295
296 296 in myPythonPackages.rhodecode-enterprise-ce
@@ -1,175 +1,177 b''
1 1 {
2 2 "dirs": {
3 3 "css": {
4 4 "src": "rhodecode/public/css",
5 5 "dest": "rhodecode/public/css"
6 6 },
7 7 "js": {
8 8 "src": "rhodecode/public/js/src",
9 9 "src_rc": "rhodecode/public/js/rhodecode",
10 10 "dest": "rhodecode/public/js",
11 11 "node_modules": "node_modules"
12 12 }
13 13 },
14 14 "copy": {
15 15 "main": {
16 16 "files": [
17 17 {
18 18 "expand": true,
19 19 "cwd": "node_modules/@webcomponents",
20 20 "src": "webcomponentsjs/*.*",
21 21 "dest": "<%= dirs.js.dest %>/vendors"
22 22 },
23 23 {
24 24 "src": "<%= dirs.css.src %>/style-polymer.css",
25 25 "dest": "<%= dirs.js.dest %>/src/components/style-polymer.css"
26 26 }
27 27 ]
28 28 }
29 29 },
30 30 "concat": {
31 31 "dist": {
32 32 "src": [
33 33 "<%= dirs.js.node_modules %>/jquery/dist/jquery.min.js",
34 34 "<%= dirs.js.node_modules %>/mousetrap/mousetrap.min.js",
35 35 "<%= dirs.js.node_modules %>/moment/min/moment.min.js",
36 36 "<%= dirs.js.node_modules %>/clipboard/dist/clipboard.min.js",
37 37 "<%= dirs.js.node_modules %>/favico.js/favico-0.3.10.min.js",
38 38 "<%= dirs.js.node_modules %>/dropzone/dist/dropzone.js",
39 39 "<%= dirs.js.node_modules %>/sticky-sidebar/dist/sticky-sidebar.min.js",
40 40 "<%= dirs.js.node_modules %>/sticky-sidebar/dist/jquery.sticky-sidebar.min.js",
41 41 "<%= dirs.js.node_modules %>/waypoints/lib/noframework.waypoints.min.js",
42 42 "<%= dirs.js.node_modules %>/waypoints/lib/jquery.waypoints.min.js",
43 43 "<%= dirs.js.node_modules %>/appenlight-client/appenlight-client.min.js",
44 44 "<%= dirs.js.src %>/logging.js",
45 45 "<%= dirs.js.src %>/bootstrap.js",
46 46 "<%= dirs.js.src %>/i18n_utils.js",
47 47 "<%= dirs.js.src %>/deform.js",
48 48 "<%= dirs.js.src %>/ejs.js",
49 49 "<%= dirs.js.src %>/ejs_templates/utils.js",
50 50 "<%= dirs.js.src %>/plugins/jquery.pjax.js",
51 51 "<%= dirs.js.src %>/plugins/jquery.dataTables.js",
52 52 "<%= dirs.js.src %>/plugins/flavoured_checkbox.js",
53 53 "<%= dirs.js.src %>/plugins/jquery.auto-grow-input.js",
54 54 "<%= dirs.js.src %>/plugins/jquery.autocomplete.js",
55 55 "<%= dirs.js.src %>/plugins/jquery.debounce.js",
56 56 "<%= dirs.js.node_modules %>/mark.js/dist/jquery.mark.min.js",
57 57 "<%= dirs.js.src %>/plugins/jquery.timeago.js",
58 58 "<%= dirs.js.src %>/plugins/jquery.timeago-extension.js",
59 59 "<%= dirs.js.src %>/select2/select2.js",
60 60 "<%= dirs.js.src %>/codemirror/codemirror.js",
61 61 "<%= dirs.js.src %>/codemirror/codemirror_loadmode.js",
62 62 "<%= dirs.js.src %>/codemirror/codemirror_hint.js",
63 63 "<%= dirs.js.src %>/codemirror/codemirror_overlay.js",
64 64 "<%= dirs.js.src %>/codemirror/codemirror_placeholder.js",
65 65 "<%= dirs.js.src %>/codemirror/codemirror_simplemode.js",
66 66 "<%= dirs.js.dest %>/mode/meta.js",
67 67 "<%= dirs.js.dest %>/mode/meta_ext.js",
68 68 "<%= dirs.js.src_rc %>/i18n/select2/translations.js",
69 69 "<%= dirs.js.src %>/rhodecode/utils/array.js",
70 70 "<%= dirs.js.src %>/rhodecode/utils/string.js",
71 71 "<%= dirs.js.src %>/rhodecode/utils/pyroutes.js",
72 72 "<%= dirs.js.src %>/rhodecode/utils/ajax.js",
73 73 "<%= dirs.js.src %>/rhodecode/utils/autocomplete.js",
74 74 "<%= dirs.js.src %>/rhodecode/utils/colorgenerator.js",
75 75 "<%= dirs.js.src %>/rhodecode/utils/ie.js",
76 76 "<%= dirs.js.src %>/rhodecode/utils/os.js",
77 77 "<%= dirs.js.src %>/rhodecode/utils/topics.js",
78 78 "<%= dirs.js.src %>/rhodecode/init.js",
79 79 "<%= dirs.js.src %>/rhodecode/changelog.js",
80 80 "<%= dirs.js.src %>/rhodecode/codemirror.js",
81 81 "<%= dirs.js.src %>/rhodecode/comments.js",
82 82 "<%= dirs.js.src %>/rhodecode/constants.js",
83 83 "<%= dirs.js.src %>/rhodecode/files.js",
84 84 "<%= dirs.js.src %>/rhodecode/followers.js",
85 85 "<%= dirs.js.src %>/rhodecode/menus.js",
86 86 "<%= dirs.js.src %>/rhodecode/notifications.js",
87 87 "<%= dirs.js.src %>/rhodecode/permissions.js",
88 88 "<%= dirs.js.src %>/rhodecode/pjax.js",
89 89 "<%= dirs.js.src %>/rhodecode/pullrequests.js",
90 90 "<%= dirs.js.src %>/rhodecode/settings.js",
91 91 "<%= dirs.js.src %>/rhodecode/select2_widgets.js",
92 92 "<%= dirs.js.src %>/rhodecode/tooltips.js",
93 93 "<%= dirs.js.src %>/rhodecode/users.js",
94 94 "<%= dirs.js.src %>/rhodecode/appenlight.js",
95 95 "<%= dirs.js.src %>/rhodecode.js",
96 96 "<%= dirs.js.dest %>/rhodecode-components.js"
97 97 ],
98 98 "dest": "<%= dirs.js.dest %>/scripts.js",
99 99 "nonull": true
100 100 }
101 101 },
102 102 "less": {
103 103 "development": {
104 104 "options": {
105 105 "compress": false,
106 106 "yuicompress": false,
107 107 "optimization": 0
108 108 },
109 109 "files": {
110 110 "<%= dirs.css.dest %>/style.css": "<%= dirs.css.src %>/main.less",
111 "<%= dirs.css.dest %>/style-polymer.css": "<%= dirs.css.src %>/polymer.less"
111 "<%= dirs.css.dest %>/style-polymer.css": "<%= dirs.css.src %>/polymer.less",
112 "<%= dirs.css.dest %>/style-ipython.css": "<%= dirs.css.src %>/ipython.less"
112 113 }
113 114 },
114 115 "production": {
115 116 "options": {
116 117 "compress": true,
117 118 "yuicompress": true,
118 119 "optimization": 2
119 120 },
120 121 "files": {
121 122 "<%= dirs.css.dest %>/style.css": "<%= dirs.css.src %>/main.less",
122 "<%= dirs.css.dest %>/style-polymer.css": "<%= dirs.css.src %>/polymer.less"
123 "<%= dirs.css.dest %>/style-polymer.css": "<%= dirs.css.src %>/polymer.less",
124 "<%= dirs.css.dest %>/style-ipython.css": "<%= dirs.css.src %>/ipython.less"
123 125 }
124 126 },
125 127 "components": {
126 128 "files": [
127 129 {
128 130 "cwd": "<%= dirs.js.src %>/components/",
129 131 "dest": "<%= dirs.js.src %>/components/",
130 132 "src": [
131 133 "**/*.less"
132 134 ],
133 135 "expand": true,
134 136 "ext": ".css"
135 137 }
136 138 ]
137 139 }
138 140 },
139 141 "watch": {
140 142 "less": {
141 143 "files": [
142 144 "<%= dirs.css.src %>/**/*.less",
143 145 "<%= dirs.js.src %>/components/**/*.less"
144 146 ],
145 147 "tasks": [
146 148 "less:development",
147 149 "less:components",
148 150 "concat:polymercss",
149 151 "webpack",
150 152 "concat:dist"
151 153 ]
152 154 },
153 155 "js": {
154 156 "files": [
155 157 "!<%= dirs.js.src %>/components/root-styles.gen.html",
156 158 "<%= dirs.js.src %>/**/*.js",
157 159 "<%= dirs.js.src %>/components/**/*.html"
158 160 ],
159 161 "tasks": [
160 162 "less:components",
161 163 "concat:polymercss",
162 164 "webpack",
163 165 "concat:dist"
164 166 ]
165 167 }
166 168 },
167 169 "jshint": {
168 170 "rhodecode": {
169 171 "src": "<%= dirs.js.src %>/rhodecode/**/*.js",
170 172 "options": {
171 173 "jshintrc": ".jshintrc"
172 174 }
173 175 }
174 176 }
175 177 }
@@ -1,559 +1,559 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2011-2019 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21
22 22 """
23 23 Renderer for markup languages with ability to parse using rst or markdown
24 24 """
25 25
26 26 import re
27 27 import os
28 28 import lxml
29 29 import logging
30 30 import urlparse
31 31 import bleach
32 32
33 33 from mako.lookup import TemplateLookup
34 34 from mako.template import Template as MakoTemplate
35 35
36 36 from docutils.core import publish_parts
37 37 from docutils.parsers.rst import directives
38 38 from docutils import writers
39 39 from docutils.writers import html4css1
40 40 import markdown
41 41
42 42 from rhodecode.lib.markdown_ext import GithubFlavoredMarkdownExtension
43 43 from rhodecode.lib.utils2 import (safe_unicode, md5_safe, MENTIONS_REGEX)
44 44
45 45 log = logging.getLogger(__name__)
46 46
47 47 # default renderer used to generate automated comments
48 48 DEFAULT_COMMENTS_RENDERER = 'rst'
49 49
50 50
51 51 class CustomHTMLTranslator(writers.html4css1.HTMLTranslator):
52 52 """
53 53 Custom HTML Translator used for sandboxing potential
54 54 JS injections in ref links
55 55 """
56 56
57 57 def visit_reference(self, node):
58 58 if 'refuri' in node.attributes:
59 59 refuri = node['refuri']
60 60 if ':' in refuri:
61 61 prefix, link = refuri.lstrip().split(':', 1)
62 62 prefix = prefix or ''
63 63
64 64 if prefix.lower() == 'javascript':
65 65 # we don't allow javascript type of refs...
66 66 node['refuri'] = 'javascript:alert("SandBoxedJavascript")'
67 67
68 68 # old style class requires this...
69 69 return html4css1.HTMLTranslator.visit_reference(self, node)
70 70
71 71
72 72 class RhodeCodeWriter(writers.html4css1.Writer):
73 73 def __init__(self):
74 74 writers.Writer.__init__(self)
75 75 self.translator_class = CustomHTMLTranslator
76 76
77 77
78 78 def relative_links(html_source, server_paths):
79 79 if not html_source:
80 80 return html_source
81 81
82 82 try:
83 83 from lxml.html import fromstring
84 84 from lxml.html import tostring
85 85 except ImportError:
86 86 log.exception('Failed to import lxml')
87 87 return html_source
88 88
89 89 try:
90 90 doc = lxml.html.fromstring(html_source)
91 91 except Exception:
92 92 return html_source
93 93
94 94 for el in doc.cssselect('img, video'):
95 95 src = el.attrib.get('src')
96 96 if src:
97 97 el.attrib['src'] = relative_path(src, server_paths['raw'])
98 98
99 99 for el in doc.cssselect('a:not(.gfm)'):
100 100 src = el.attrib.get('href')
101 101 if src:
102 102 raw_mode = el.attrib['href'].endswith('?raw=1')
103 103 if raw_mode:
104 104 el.attrib['href'] = relative_path(src, server_paths['raw'])
105 105 else:
106 106 el.attrib['href'] = relative_path(src, server_paths['standard'])
107 107
108 108 return lxml.html.tostring(doc)
109 109
110 110
111 111 def relative_path(path, request_path, is_repo_file=None):
112 112 """
113 113 relative link support, path is a rel path, and request_path is current
114 114 server path (not absolute)
115 115
116 116 e.g.
117 117
118 118 path = '../logo.png'
119 119 request_path= '/repo/files/path/file.md'
120 120 produces: '/repo/files/logo.png'
121 121 """
122 122 # TODO(marcink): unicode/str support ?
123 123 # maybe=> safe_unicode(urllib.quote(safe_str(final_path), '/:'))
124 124
125 125 def dummy_check(p):
126 126 return True # assume default is a valid file path
127 127
128 128 is_repo_file = is_repo_file or dummy_check
129 129 if not path:
130 130 return request_path
131 131
132 132 path = safe_unicode(path)
133 133 request_path = safe_unicode(request_path)
134 134
135 135 if path.startswith((u'data:', u'javascript:', u'#', u':')):
136 136 # skip data, anchor, invalid links
137 137 return path
138 138
139 139 is_absolute = bool(urlparse.urlparse(path).netloc)
140 140 if is_absolute:
141 141 return path
142 142
143 143 if not request_path:
144 144 return path
145 145
146 146 if path.startswith(u'/'):
147 147 path = path[1:]
148 148
149 149 if path.startswith(u'./'):
150 150 path = path[2:]
151 151
152 152 parts = request_path.split('/')
153 153 # compute how deep we need to traverse the request_path
154 154 depth = 0
155 155
156 156 if is_repo_file(request_path):
157 157 # if request path is a VALID file, we use a relative path with
158 158 # one level up
159 159 depth += 1
160 160
161 161 while path.startswith(u'../'):
162 162 depth += 1
163 163 path = path[3:]
164 164
165 165 if depth > 0:
166 166 parts = parts[:-depth]
167 167
168 168 parts.append(path)
169 169 final_path = u'/'.join(parts).lstrip(u'/')
170 170
171 171 return u'/' + final_path
172 172
173 173
174 174 _cached_markdown_renderer = None
175 175
176 176
177 177 def get_markdown_renderer(extensions, output_format):
178 178 global _cached_markdown_renderer
179 179
180 180 if _cached_markdown_renderer is None:
181 181 _cached_markdown_renderer = markdown.Markdown(
182 182 extensions=extensions,
183 183 enable_attributes=False, output_format=output_format)
184 184 return _cached_markdown_renderer
185 185
186 186
187 187 _cached_markdown_renderer_flavored = None
188 188
189 189
190 190 def get_markdown_renderer_flavored(extensions, output_format):
191 191 global _cached_markdown_renderer_flavored
192 192
193 193 if _cached_markdown_renderer_flavored is None:
194 194 _cached_markdown_renderer_flavored = markdown.Markdown(
195 195 extensions=extensions + [GithubFlavoredMarkdownExtension()],
196 196 enable_attributes=False, output_format=output_format)
197 197 return _cached_markdown_renderer_flavored
198 198
199 199
200 200 class MarkupRenderer(object):
201 201 RESTRUCTUREDTEXT_DISALLOWED_DIRECTIVES = ['include', 'meta', 'raw']
202 202
203 203 MARKDOWN_PAT = re.compile(r'\.(md|mkdn?|mdown|markdown)$', re.IGNORECASE)
204 204 RST_PAT = re.compile(r'\.re?st$', re.IGNORECASE)
205 205 JUPYTER_PAT = re.compile(r'\.(ipynb)$', re.IGNORECASE)
206 206 PLAIN_PAT = re.compile(r'^readme$', re.IGNORECASE)
207 207
208 208 URL_PAT = re.compile(r'(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]'
209 209 r'|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)')
210 210
211 211 extensions = ['markdown.extensions.codehilite', 'markdown.extensions.extra',
212 212 'markdown.extensions.def_list', 'markdown.extensions.sane_lists']
213 213
214 214 output_format = 'html4'
215 215
216 216 # extension together with weights. Lower is first means we control how
217 217 # extensions are attached to readme names with those.
218 218 PLAIN_EXTS = [
219 219 # prefer no extension
220 220 ('', 0), # special case that renders READMES names without extension
221 221 ('.text', 2), ('.TEXT', 2),
222 222 ('.txt', 3), ('.TXT', 3)
223 223 ]
224 224
225 225 RST_EXTS = [
226 226 ('.rst', 1), ('.rest', 1),
227 227 ('.RST', 2), ('.REST', 2)
228 228 ]
229 229
230 230 MARKDOWN_EXTS = [
231 231 ('.md', 1), ('.MD', 1),
232 232 ('.mkdn', 2), ('.MKDN', 2),
233 233 ('.mdown', 3), ('.MDOWN', 3),
234 234 ('.markdown', 4), ('.MARKDOWN', 4)
235 235 ]
236 236
237 237 def _detect_renderer(self, source, filename=None):
238 238 """
239 239 runs detection of what renderer should be used for generating html
240 240 from a markup language
241 241
242 242 filename can be also explicitly a renderer name
243 243
244 244 :param source:
245 245 :param filename:
246 246 """
247 247
248 248 if MarkupRenderer.MARKDOWN_PAT.findall(filename):
249 249 detected_renderer = 'markdown'
250 250 elif MarkupRenderer.RST_PAT.findall(filename):
251 251 detected_renderer = 'rst'
252 252 elif MarkupRenderer.JUPYTER_PAT.findall(filename):
253 253 detected_renderer = 'jupyter'
254 254 elif MarkupRenderer.PLAIN_PAT.findall(filename):
255 255 detected_renderer = 'plain'
256 256 else:
257 257 detected_renderer = 'plain'
258 258
259 259 return getattr(MarkupRenderer, detected_renderer)
260 260
261 261 @classmethod
262 262 def bleach_clean(cls, text):
263 263 from .bleach_whitelist import markdown_attrs, markdown_tags
264 264 allowed_tags = markdown_tags
265 265 allowed_attrs = markdown_attrs
266 266
267 267 try:
268 268 return bleach.clean(text, tags=allowed_tags, attributes=allowed_attrs)
269 269 except Exception:
270 270 return 'UNPARSEABLE TEXT'
271 271
272 272 @classmethod
273 273 def renderer_from_filename(cls, filename, exclude):
274 274 """
275 275 Detect renderer markdown/rst from filename and optionally use exclude
276 276 list to remove some options. This is mostly used in helpers.
277 277 Returns None when no renderer can be detected.
278 278 """
279 279 def _filter(elements):
280 280 if isinstance(exclude, (list, tuple)):
281 281 return [x for x in elements if x not in exclude]
282 282 return elements
283 283
284 284 if filename.endswith(
285 285 tuple(_filter([x[0] for x in cls.MARKDOWN_EXTS if x[0]]))):
286 286 return 'markdown'
287 287 if filename.endswith(tuple(_filter([x[0] for x in cls.RST_EXTS if x[0]]))):
288 288 return 'rst'
289 289
290 290 return None
291 291
292 292 def render(self, source, filename=None):
293 293 """
294 294 Renders a given filename using detected renderer
295 295 it detects renderers based on file extension or mimetype.
296 296 At last it will just do a simple html replacing new lines with <br/>
297 297
298 298 :param file_name:
299 299 :param source:
300 300 """
301 301
302 302 renderer = self._detect_renderer(source, filename)
303 303 readme_data = renderer(source)
304 304 return readme_data
305 305
306 306 @classmethod
307 307 def _flavored_markdown(cls, text):
308 308 """
309 309 Github style flavored markdown
310 310
311 311 :param text:
312 312 """
313 313
314 314 # Extract pre blocks.
315 315 extractions = {}
316 316
317 317 def pre_extraction_callback(matchobj):
318 318 digest = md5_safe(matchobj.group(0))
319 319 extractions[digest] = matchobj.group(0)
320 320 return "{gfm-extraction-%s}" % digest
321 321 pattern = re.compile(r'<pre>.*?</pre>', re.MULTILINE | re.DOTALL)
322 322 text = re.sub(pattern, pre_extraction_callback, text)
323 323
324 324 # Prevent foo_bar_baz from ending up with an italic word in the middle.
325 325 def italic_callback(matchobj):
326 326 s = matchobj.group(0)
327 327 if list(s).count('_') >= 2:
328 328 return s.replace('_', r'\_')
329 329 return s
330 330 text = re.sub(r'^(?! {4}|\t)\w+_\w+_\w[\w_]*', italic_callback, text)
331 331
332 332 # Insert pre block extractions.
333 333 def pre_insert_callback(matchobj):
334 334 return '\n\n' + extractions[matchobj.group(1)]
335 335 text = re.sub(r'\{gfm-extraction-([0-9a-f]{32})\}',
336 336 pre_insert_callback, text)
337 337
338 338 return text
339 339
340 340 @classmethod
341 341 def urlify_text(cls, text):
342 342 def url_func(match_obj):
343 343 url_full = match_obj.groups()[0]
344 344 return '<a href="%(url)s">%(url)s</a>' % ({'url': url_full})
345 345
346 346 return cls.URL_PAT.sub(url_func, text)
347 347
348 348 @classmethod
349 349 def plain(cls, source, universal_newline=True, leading_newline=True):
350 350 source = safe_unicode(source)
351 351 if universal_newline:
352 352 newline = '\n'
353 353 source = newline.join(source.splitlines())
354 354
355 355 rendered_source = cls.urlify_text(source)
356 356 source = ''
357 357 if leading_newline:
358 358 source += '<br />'
359 359 source += rendered_source.replace("\n", '<br />')
360 360
361 361 rendered = cls.bleach_clean(source)
362 362 return rendered
363 363
364 364 @classmethod
365 365 def markdown(cls, source, safe=True, flavored=True, mentions=False,
366 366 clean_html=True):
367 367 """
368 368 returns markdown rendered code cleaned by the bleach library
369 369 """
370 370
371 371 if flavored:
372 372 markdown_renderer = get_markdown_renderer_flavored(
373 373 cls.extensions, cls.output_format)
374 374 else:
375 375 markdown_renderer = get_markdown_renderer(
376 376 cls.extensions, cls.output_format)
377 377
378 378 if mentions:
379 379 mention_pat = re.compile(MENTIONS_REGEX)
380 380
381 381 def wrapp(match_obj):
382 382 uname = match_obj.groups()[0]
383 383 return ' **@%(uname)s** ' % {'uname': uname}
384 384 mention_hl = mention_pat.sub(wrapp, source).strip()
385 385 # we extracted mentions render with this using Mentions false
386 386 return cls.markdown(mention_hl, safe=safe, flavored=flavored,
387 387 mentions=False)
388 388
389 389 source = safe_unicode(source)
390 390
391 391 try:
392 392 if flavored:
393 393 source = cls._flavored_markdown(source)
394 394 rendered = markdown_renderer.convert(source)
395 395 except Exception:
396 396 log.exception('Error when rendering Markdown')
397 397 if safe:
398 398 log.debug('Fallback to render in plain mode')
399 399 rendered = cls.plain(source)
400 400 else:
401 401 raise
402 402
403 403 if clean_html:
404 404 rendered = cls.bleach_clean(rendered)
405 405 return rendered
406 406
407 407 @classmethod
408 408 def rst(cls, source, safe=True, mentions=False, clean_html=False):
409 409 if mentions:
410 410 mention_pat = re.compile(MENTIONS_REGEX)
411 411
412 412 def wrapp(match_obj):
413 413 uname = match_obj.groups()[0]
414 414 return ' **@%(uname)s** ' % {'uname': uname}
415 415 mention_hl = mention_pat.sub(wrapp, source).strip()
416 416 # we extracted mentions render with this using Mentions false
417 417 return cls.rst(mention_hl, safe=safe, mentions=False)
418 418
419 419 source = safe_unicode(source)
420 420 try:
421 421 docutils_settings = dict(
422 422 [(alias, None) for alias in
423 423 cls.RESTRUCTUREDTEXT_DISALLOWED_DIRECTIVES])
424 424
425 425 docutils_settings.update({
426 426 'input_encoding': 'unicode', 'report_level': 4})
427 427
428 428 for k, v in docutils_settings.iteritems():
429 429 directives.register_directive(k, v)
430 430
431 431 parts = publish_parts(source=source,
432 432 writer=RhodeCodeWriter(),
433 433 settings_overrides=docutils_settings)
434 434 rendered = parts["fragment"]
435 435 if clean_html:
436 436 rendered = cls.bleach_clean(rendered)
437 437 return parts['html_title'] + rendered
438 438 except Exception:
439 439 log.exception('Error when rendering RST')
440 440 if safe:
441 441 log.debug('Fallbacking to render in plain mode')
442 442 return cls.plain(source)
443 443 else:
444 444 raise
445 445
446 446 @classmethod
447 447 def jupyter(cls, source, safe=True):
448 448 from rhodecode.lib import helpers
449 449
450 450 from traitlets.config import Config
451 451 import nbformat
452 452 from nbconvert import HTMLExporter
453 453 from nbconvert.preprocessors import Preprocessor
454 454
455 455 class CustomHTMLExporter(HTMLExporter):
456 456 def _template_file_default(self):
457 457 return 'basic'
458 458
459 459 class Sandbox(Preprocessor):
460 460
461 461 def preprocess(self, nb, resources):
462 462 sandbox_text = 'SandBoxed(IPython.core.display.Javascript object)'
463 463 for cell in nb['cells']:
464 464 if not safe:
465 465 continue
466 466
467 467 if 'outputs' in cell:
468 468 for cell_output in cell['outputs']:
469 469 if 'data' in cell_output:
470 470 if 'application/javascript' in cell_output['data']:
471 471 cell_output['data']['text/plain'] = sandbox_text
472 472 cell_output['data'].pop('application/javascript', None)
473 473
474 474 if 'source' in cell and cell['cell_type'] == 'markdown':
475 475 # sanitize similar like in markdown
476 476 cell['source'] = cls.bleach_clean(cell['source'])
477 477
478 478 return nb, resources
479 479
480 480 def _sanitize_resources(input_resources):
481 481 """
482 482 Skip/sanitize some of the CSS generated and included in jupyter
483 483 so it doesn't messes up UI so much
484 484 """
485 485
486 486 # TODO(marcink): probably we should replace this with whole custom
487 487 # CSS set that doesn't screw up, but jupyter generated html has some
488 488 # special markers, so it requires Custom HTML exporter template with
489 489 # _default_template_path_default, to achieve that
490 490
491 491 # strip the reset CSS
492 492 input_resources[0] = input_resources[0][input_resources[0].find('/*! Source'):]
493 493 return input_resources
494 494
495 495 def as_html(notebook):
496 496 conf = Config()
497 497 conf.CustomHTMLExporter.preprocessors = [Sandbox]
498 498 html_exporter = CustomHTMLExporter(config=conf)
499 499
500 500 (body, resources) = html_exporter.from_notebook_node(notebook)
501 501 header = '<!-- ## IPYTHON NOTEBOOK RENDERING ## -->'
502 502 js = MakoTemplate(r'''
503 <!-- Load mathjax -->
504 <!-- MathJax configuration -->
505 <script type="text/x-mathjax-config">
506 MathJax.Hub.Config({
507 jax: ["input/TeX","output/HTML-CSS", "output/PreviewHTML"],
508 extensions: ["tex2jax.js","MathMenu.js","MathZoom.js", "fast-preview.js", "AssistiveMML.js", "[Contrib]/a11y/accessibility-menu.js"],
509 TeX: {
510 extensions: ["AMSmath.js","AMSsymbols.js","noErrors.js","noUndefined.js"]
511 },
512 tex2jax: {
513 inlineMath: [ ['$','$'], ["\\(","\\)"] ],
514 displayMath: [ ['$$','$$'], ["\\[","\\]"] ],
515 processEscapes: true,
516 processEnvironments: true
517 },
518 // Center justify equations in code and markdown cells. Elsewhere
519 // we use CSS to left justify single line equations in code cells.
520 displayAlign: 'center',
521 "HTML-CSS": {
522 styles: {'.MathJax_Display': {"margin": 0}},
523 linebreaks: { automatic: true },
524 availableFonts: ["STIX", "TeX"]
525 },
526 showMathMenu: false
527 });
528 </script>
529 <!-- End of mathjax configuration -->
530 <script src="${h.asset('js/src/math_jax/MathJax.js')}"></script>
503 <!-- MathJax configuration -->
504 <script type="text/x-mathjax-config">
505 MathJax.Hub.Config({
506 jax: ["input/TeX","output/HTML-CSS", "output/PreviewHTML"],
507 extensions: ["tex2jax.js","MathMenu.js","MathZoom.js", "fast-preview.js", "AssistiveMML.js", "[Contrib]/a11y/accessibility-menu.js"],
508 TeX: {
509 extensions: ["AMSmath.js","AMSsymbols.js","noErrors.js","noUndefined.js"]
510 },
511 tex2jax: {
512 inlineMath: [ ['$','$'], ["\\(","\\)"] ],
513 displayMath: [ ['$$','$$'], ["\\[","\\]"] ],
514 processEscapes: true,
515 processEnvironments: true
516 },
517 // Center justify equations in code and markdown cells. Elsewhere
518 // we use CSS to left justify single line equations in code cells.
519 displayAlign: 'center',
520 "HTML-CSS": {
521 styles: {'.MathJax_Display': {"margin": 0}},
522 linebreaks: { automatic: true },
523 availableFonts: ["STIX", "TeX"]
524 },
525 showMathMenu: false
526 });
527 </script>
528 <!-- End of MathJax configuration -->
529 <script src="${h.asset('js/src/math_jax/MathJax.js')}"></script>
531 530 ''').render(h=helpers)
532 531
533 css = '<style>{}</style>'.format(
534 ''.join(_sanitize_resources(resources['inlining']['css'])))
532 css = MakoTemplate(r'''
533 <link rel="stylesheet" type="text/css" href="${h.asset('css/style-ipython.css', ver=ver)}" media="screen"/>
534 ''').render(h=helpers, ver='ver1')
535 535
536 536 body = '\n'.join([header, css, js, body])
537 537 return body, resources
538 538
539 539 notebook = nbformat.reads(source, as_version=4)
540 540 (body, resources) = as_html(notebook)
541 541 return body
542 542
543 543
544 544 class RstTemplateRenderer(object):
545 545
546 546 def __init__(self):
547 547 base = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
548 548 rst_template_dirs = [os.path.join(base, 'templates', 'rst_templates')]
549 549 self.template_store = TemplateLookup(
550 550 directories=rst_template_dirs,
551 551 input_encoding='utf-8',
552 552 imports=['from rhodecode.lib import helpers as h'])
553 553
554 554 def _get_template(self, templatename):
555 555 return self.template_store.get_template(templatename)
556 556
557 557 def render(self, template_name, **kwargs):
558 558 template = self._get_template(template_name)
559 559 return template.render(**kwargs)
General Comments 0
You need to be logged in to leave comments. Login now