diff --git a/mercurial/hgweb/__init__.py b/mercurial/hgweb/__init__.py
--- a/mercurial/hgweb/__init__.py
+++ b/mercurial/hgweb/__init__.py
@@ -96,6 +96,16 @@ def createservice(ui, repo, opts):
         alluis.update([repo.baseui, repo.ui])
     else:
         baseui = ui
+    webconf = opts.get('web_conf') or opts.get('webdir_conf')
+    if webconf:
+        # load server settings (e.g. web.port) to "copied" ui, which allows
+        # hgwebdir to reload webconf cleanly
+        servui = ui.copy()
+        servui.readconfig(webconf, sections=['web'])
+        alluis.add(servui)
+    else:
+        servui = ui
+
     optlist = ("name templates style address port prefix ipv6"
                " accesslog errorlog certificate encoding")
     for o in optlist.split():
@@ -105,7 +115,6 @@ def createservice(ui, repo, opts):
         for u in alluis:
             u.setconfig("web", o, val, 'serve')
 
-    webconf = opts.get('web_conf') or opts.get('webdir_conf')
     if webconf:
         app = hgwebdir_mod.hgwebdir(webconf, baseui=baseui)
     else:
@@ -113,4 +122,4 @@ def createservice(ui, repo, opts):
             raise error.RepoError(_("there is no Mercurial repository"
                                     " here (.hg not found)"))
         app = hgweb_mod.hgweb(repo, baseui=baseui)
-    return httpservice(ui, app, opts)
+    return httpservice(servui, app, opts)
diff --git a/tests/test-hgwebdir.t b/tests/test-hgwebdir.t
--- a/tests/test-hgwebdir.t
+++ b/tests/test-hgwebdir.t
@@ -1181,6 +1181,38 @@ test inexistent and inaccessible repo sh
   </body>
   </html>
   
+
+test listening address/port specified by web-conf (issue4699):
+
+  $ killdaemons.py
+  $ cat >> paths.conf <<EOF
+  > [web]
+  > address = localhost
+  > port = $HGPORT1
+  > EOF
+  $ hg serve -d --pid-file=hg.pid --web-conf paths.conf \
+  >     -A access-paths.log -E error-paths-9.log
+  listening at http://*:$HGPORT1/ (bound to 127.0.0.1:$HGPORT1) (glob)
+  $ cat hg.pid >> $DAEMON_PIDS
+  $ get-with-headers.py localhost:$HGPORT1 '?style=raw'
+  200 Script output follows
+  
+  
+  
+test --port option overrides web.port:
+
+  $ killdaemons.py
+  $ hg serve -p $HGPORT2 -d -v --pid-file=hg.pid --web-conf paths.conf \
+  >     -A access-paths.log -E error-paths-10.log
+  listening at http://*:$HGPORT2/ (bound to 127.0.0.1:$HGPORT2) (glob)
+  $ cat hg.pid >> $DAEMON_PIDS
+  $ get-with-headers.py localhost:$HGPORT2 '?style=raw'
+  200 Script output follows
+  
+  
+  
+
+  $ killdaemons.py
   $ cat > collections.conf <<EOF
   > [collections]
   > $root=$root
@@ -1338,6 +1370,14 @@ paths errors 8
 
   $ cat error-paths-8.log
 
+paths errors 9
+
+  $ cat error-paths-9.log
+
+paths errors 10
+
+  $ cat error-paths-10.log
+
 collections errors
 
   $ cat error-collections.log