diff --git a/rhodecode/controllers/summary.py b/rhodecode/controllers/summary.py
--- a/rhodecode/controllers/summary.py
+++ b/rhodecode/controllers/summary.py
@@ -38,7 +38,7 @@ from rhodecode.lib.utils2 import safe_st
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAnyDecorator, NotAnonymous, XHRRequired)
 from rhodecode.lib.base import BaseRepoController, render
-from rhodecode.lib.markup_renderer import MarkupRenderer
+from rhodecode.lib.markup_renderer import MarkupRenderer, relative_links
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.vcs.backends.base import EmptyCommit
 from rhodecode.lib.vcs.exceptions import (
@@ -70,7 +70,12 @@ class SummaryController(BaseRepoControll
                 log.debug("Searching for a README file.")
                 readme_node = ReadmeFinder(default_renderer).search(commit)
             if readme_node:
-                readme_data = self._render_readme_or_none(commit, readme_node)
+                relative_url = h.url('files_raw_home',
+                                     repo_name=repo_name,
+                                     revision=commit.raw_id,
+                                     f_path=readme_node.path)
+                readme_data = self._render_readme_or_none(
+                    commit, readme_node, relative_url)
                 readme_filename = readme_node.path
             return readme_data, readme_filename
 
@@ -95,13 +100,16 @@ class SummaryController(BaseRepoControll
             log.exception(
                 "Problem getting commit when trying to render the README.")
 
-    def _render_readme_or_none(self, commit, readme_node):
+    def _render_readme_or_none(self, commit, readme_node, relative_url):
         log.debug(
             'Found README file `%s` rendering...', readme_node.path)
         renderer = MarkupRenderer()
         try:
-            return renderer.render(
+            html_source = renderer.render(
                 readme_node.content, filename=readme_node.path)
+            if relative_url:
+                return relative_links(html_source, relative_url)
+            return html_source
         except Exception:
             log.exception(
                 "Exception while trying to render the README")
diff --git a/rhodecode/lib/helpers.py b/rhodecode/lib/helpers.py
--- a/rhodecode/lib/helpers.py
+++ b/rhodecode/lib/helpers.py
@@ -75,7 +75,7 @@ from rhodecode.lib.utils import repo_nam
 from rhodecode.lib.utils2 import str2bool, safe_unicode, safe_str, \
     get_commit_safe, datetime_to_time, time_to_datetime, time_to_utcdatetime, \
     AttributeDict, safe_int, md5, md5_safe
-from rhodecode.lib.markup_renderer import MarkupRenderer
+from rhodecode.lib.markup_renderer import MarkupRenderer, relative_links
 from rhodecode.lib.vcs.exceptions import CommitDoesNotExistError
 from rhodecode.lib.vcs.backends.base import BaseChangeset, EmptyCommit
 from rhodecode.config.conf import DATE_FORMAT, DATETIME_FORMAT
@@ -1828,19 +1828,29 @@ def renderer_from_filename(filename, exc
     return None
 
 
-def render(source, renderer='rst', mentions=False):
+def render(source, renderer='rst', mentions=False, relative_url=None):
+
+    def maybe_convert_relative_links(html_source):
+        if relative_url:
+            return relative_links(html_source, relative_url)
+        return html_source
+
     if renderer == 'rst':
         return literal(
             '<div class="rst-block">%s</div>' %
-            MarkupRenderer.rst(source, mentions=mentions))
+            maybe_convert_relative_links(
+                MarkupRenderer.rst(source, mentions=mentions)))
     elif renderer == 'markdown':
         return literal(
             '<div class="markdown-block">%s</div>' %
-            MarkupRenderer.markdown(source, flavored=True, mentions=mentions))
+            maybe_convert_relative_links(
+                MarkupRenderer.markdown(source, flavored=True,
+                                        mentions=mentions)))
     elif renderer == 'jupyter':
         return literal(
             '<div class="ipynb">%s</div>' %
-            MarkupRenderer.jupyter(source))
+            maybe_convert_relative_links(
+                MarkupRenderer.jupyter(source)))
 
     # None means just show the file-source
     return None
diff --git a/rhodecode/lib/markup_renderer.py b/rhodecode/lib/markup_renderer.py
--- a/rhodecode/lib/markup_renderer.py
+++ b/rhodecode/lib/markup_renderer.py
@@ -25,8 +25,10 @@ Renderer for markup languages with abili
 
 import re
 import os
+import lxml
 import logging
-import itertools
+import urlparse
+import urllib
 
 from mako.lookup import TemplateLookup
 from mako.template import Template as MakoTemplate
@@ -35,9 +37,9 @@ from docutils.core import publish_parts
 from docutils.parsers.rst import directives
 import markdown
 
-from rhodecode.lib.markdown_ext import (
-    UrlizeExtension, GithubFlavoredMarkdownExtension)
-from rhodecode.lib.utils2 import safe_unicode, md5_safe, MENTIONS_REGEX
+from rhodecode.lib.markdown_ext import GithubFlavoredMarkdownExtension
+from rhodecode.lib.utils2 import (
+    safe_str, safe_unicode, md5_safe, MENTIONS_REGEX)
 
 log = logging.getLogger(__name__)
 
@@ -45,6 +47,84 @@ log = logging.getLogger(__name__)
 DEFAULT_COMMENTS_RENDERER = 'rst'
 
 
+def relative_links(html_source, server_path):
+    doc = lxml.html.fromstring(html_source)
+    for el in doc.cssselect('img, video'):
+        src = el.attrib['src']
+        if src:
+            el.attrib['src'] = relative_path(src, server_path)
+
+    for el in doc.cssselect('a:not(.gfm)'):
+        src = el.attrib['href']
+        if src:
+            el.attrib['href'] = relative_path(src, server_path)
+
+    return lxml.html.tostring(doc)
+
+
+def relative_path(path, request_path, is_repo_file=None):
+    """
+    relative link support, path is a rel path, and request_path is current
+    server path (not absolute)
+
+    e.g.
+
+    path = '../logo.png'
+    request_path= '/repo/files/path/file.md'
+    produces: '/repo/files/logo.png'
+    """
+    # TODO(marcink): unicode/str support ?
+    # maybe=> safe_unicode(urllib.quote(safe_str(final_path), '/:'))
+
+    def dummy_check(p):
+        return True  # assume default is a valid file path
+
+    is_repo_file = is_repo_file or dummy_check
+    if not path:
+        return request_path
+
+    path = safe_unicode(path)
+    request_path = safe_unicode(request_path)
+
+    if path.startswith((u'data:', u'#', u':')):
+        # skip data, anchor, invalid links
+        return path
+
+    is_absolute = bool(urlparse.urlparse(path).netloc)
+    if is_absolute:
+        return path
+
+    if not request_path:
+        return path
+
+    if path.startswith(u'/'):
+        path = path[1:]
+
+    if path.startswith(u'./'):
+        path = path[2:]
+
+    parts = request_path.split('/')
+    # compute how deep we need to traverse the request_path
+    depth = 0
+
+    if is_repo_file(request_path):
+        # if request path is a VALID file, we use a relative path with
+        # one level up
+        depth += 1
+
+    while path.startswith(u'../'):
+        depth += 1
+        path = path[3:]
+
+    if depth > 0:
+        parts = parts[:-depth]
+
+    parts.append(path)
+    final_path = u'/'.join(parts).lstrip(u'/')
+
+    return u'/' + final_path
+
+
 class MarkupRenderer(object):
     RESTRUCTUREDTEXT_DISALLOWED_DIRECTIVES = ['include', 'meta', 'raw']
 
diff --git a/rhodecode/templates/files/files_source.mako b/rhodecode/templates/files/files_source.mako
--- a/rhodecode/templates/files/files_source.mako
+++ b/rhodecode/templates/files/files_source.mako
@@ -53,7 +53,7 @@
        %else:
         % if c.file.size < c.cut_off_limit:
             %if c.renderer and not c.annotate:
-                ${h.render(c.file.content, renderer=c.renderer)}
+                ${h.render(c.file.content, renderer=c.renderer, relative_url=h.url('files_raw_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path))}
             %else:
                 <table class="cb codehilite">
                 %if c.annotate:
diff --git a/rhodecode/tests/lib/test_markup_renderer.py b/rhodecode/tests/lib/test_markup_renderer.py
--- a/rhodecode/tests/lib/test_markup_renderer.py
+++ b/rhodecode/tests/lib/test_markup_renderer.py
@@ -20,7 +20,8 @@
 
 import pytest
 
-from rhodecode.lib.markup_renderer import MarkupRenderer, RstTemplateRenderer
+from rhodecode.lib.markup_renderer import (
+    MarkupRenderer, RstTemplateRenderer, relative_path, relative_links)
 
 
 @pytest.mark.parametrize(
@@ -177,3 +178,78 @@ Auto status change to |new_status|
     renderer = RstTemplateRenderer()
     rendered = renderer.render('auto_status_change.mako', **params)
     assert expected == rendered
+
+
+@pytest.mark.parametrize(
+    "src_path, server_path, is_path, expected",
+    [
+        ('source.png', '/repo/files/path', lambda p: False,
+         '/repo/files/path/source.png'),
+
+        ('source.png', 'mk/git/blob/master/README.md', lambda p: True,
+         '/mk/git/blob/master/source.png'),
+
+        ('./source.png', 'mk/git/blob/master/README.md', lambda p: True,
+         '/mk/git/blob/master/source.png'),
+
+        ('/source.png', 'mk/git/blob/master/README.md', lambda p: True,
+         '/mk/git/blob/master/source.png'),
+
+        ('./source.png', 'repo/files/path/source.md', lambda p: True,
+         '/repo/files/path/source.png'),
+
+        ('./source.png', '/repo/files/path/file.md', lambda p: True,
+         '/repo/files/path/source.png'),
+
+        ('../source.png', '/repo/files/path/file.md', lambda p: True,
+         '/repo/files/source.png'),
+
+        ('./../source.png', '/repo/files/path/file.md', lambda p: True,
+         '/repo/files/source.png'),
+
+        ('./source.png', '/repo/files/path/file.md', lambda p: True,
+         '/repo/files/path/source.png'),
+
+        ('../../../source.png', 'path/file.md', lambda p: True,
+         '/source.png'),
+
+        ('../../../../../source.png', '/path/file.md', None,
+         '/source.png'),
+
+        ('../../../../../source.png', 'files/path/file.md', None,
+         '/source.png'),
+
+        ('../../../../../https://google.com/image.png', 'files/path/file.md', None,
+         '/https://google.com/image.png'),
+
+        ('https://google.com/image.png', 'files/path/file.md', None,
+         'https://google.com/image.png'),
+
+        ('://foo', '/files/path/file.md', None,
+         '://foo'),
+
+        (u'한글.png', '/files/path/file.md', None,
+         u'/files/path/한글.png'),
+
+        ('my custom image.png', '/files/path/file.md', None,
+         '/files/path/my custom image.png'),
+    ])
+def test_relative_path(src_path, server_path, is_path, expected):
+    path = relative_path(src_path, server_path, is_path)
+    assert path == expected
+
+
+@pytest.mark.parametrize(
+    "src_html, expected_html",
+    [
+        ('<div></div>', '<div></div>'),
+        ('<img src="/file.png"></img>', '<img src="/path/raw/file.png">'),
+        ('<img src="data:abcd"/>', '<img src="data:abcd">'),
+        ('<a href="/file.png"></a>', '<a href="/path/raw/file.png"></a>'),
+        ('<a href="#anchor"></a>', '<a href="#anchor"></a>'),
+        ('<a href="./README.md"></a>', '<a href="/path/raw/README.md"></a>'),
+        ('<a href="../README.md"></a>', '<a href="/path/README.md"></a>'),
+
+    ])
+def test_relative_links(src_html, expected_html):
+    assert relative_links(src_html, '/path/raw/file.md') == expected_html