process-management: use safer way to extract data for processes.

- prevents from unknown PID errors.

rhodecode/apps/admin/views/process_management.py

@@ -1,133 +1,163 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 
 import psutil
 import signal
 from pyramid.view import view_config
 
 from rhodecode.apps._base import BaseAppView
 from rhodecode.apps.admin.navigation import navigation_list
 from rhodecode.lib.auth import (
     LoginRequired, HasPermissionAllDecorator, CSRFRequired)
-from rhodecode.lib.utils2 import safe_int
+from rhodecode.lib.utils2 import safe_int, StrictAttributeDict
 
 log = logging.getLogger(__name__)
 
 
 class AdminProcessManagementView(BaseAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context()
+        return c
 
-        return c
+    def _format_proc(self, proc, with_children=False):
+        try:
+            mem = proc.memory_info()
+            proc_formatted = StrictAttributeDict({
+                'pid': proc.pid,
+                'name': proc.name(),
+                'mem_rss': mem.rss,
+                'mem_vms': mem.vms,
+                'cpu_percent': proc.cpu_percent(),
+                'create_time': proc.create_time(),
+                'cmd': ' '.join(proc.cmdline()),
+            })
+
+            if with_children:
+                proc_formatted.update({
+                    'children': [self._format_proc(x)
+                                 for x in proc.children(recursive=True)]
+                })
+        except Exception:
+            log.exception('Failed to load proc')
+            proc_formatted = None
+        return proc_formatted
+
+    def get_processes(self):
+        proc_list = []
+        for p in psutil.process_iter():
+            if 'gunicorn' in p.name():
+                proc = self._format_proc(p, with_children=True)
+                if proc:
+                    proc_list.append(proc)
+
+        return proc_list
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='admin_settings_process_management', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     def process_management(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         c.active = 'process_management'
         c.navlist = navigation_list(self.request)
-        c.gunicorn_processes = (
-            p for p in psutil.process_iter() if 'gunicorn' in p.name())
+        c.gunicorn_processes = self.get_processes()
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='admin_settings_process_management_data', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings_process_management_data.mako')
     def process_management_data(self):
         _ = self.request.translate
         c = self.load_default_context()
-        c.gunicorn_processes = (
-            p for p in psutil.process_iter() if 'gunicorn' in p.name())
+        c.gunicorn_processes = self.get_processes()
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='admin_settings_process_management_signal',
         request_method='POST', renderer='json_ext')
     def process_management_signal(self):
         pids = self.request.json.get('pids', [])
         result = []
         def on_terminate(proc):
             msg = "process `PID:{}` terminated with exit code {}".format(
                 proc.pid, proc.returncode)
             result.append(msg)
 
         procs = []
         for pid in pids:
             pid = safe_int(pid)
             if pid:
                 try:
                     proc = psutil.Process(pid)
                 except psutil.NoSuchProcess:
                     continue
 
                 children = proc.children(recursive=True)
                 if children:
                     print('Wont kill Master Process')
                 else:
                     procs.append(proc)
 
         for p in procs:
             p.terminate()
         gone, alive = psutil.wait_procs(procs, timeout=10, callback=on_terminate)
         for p in alive:
             p.kill()
 
         return {'result': result}
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='admin_settings_process_management_master_signal',
         request_method='POST', renderer='json_ext')
     def process_management_master_signal(self):
         pid_data = self.request.json.get('pid_data', {})
         pid = safe_int(pid_data['pid'])
         action = pid_data['action']
         if pid:
             try:
                 proc = psutil.Process(pid)
             except psutil.NoSuchProcess:
                 return {'result': 'failure_no_such_process'}
 
             children = proc.children(recursive=True)
             if children:
                 # master process
                 if action == '+' and len(children) <= 20:
                     proc.send_signal(signal.SIGTTIN)
                 elif action == '-' and len(children) >= 2:
                     proc.send_signal(signal.SIGTTOU)
                 else:
                     return {'result': 'failure_wrong_action'}
                 return {'result': 'success'}
 
         return {'result': 'failure_not_master'}

rhodecode/templates/admin/settings/settings_process_management_data.mako

@@ -1,95 +1,91 b''
 
 <table id="procList">
     <%
         def get_name(proc):
-            cmd = ' '.join(proc.cmdline())
-            if 'vcsserver.ini' in cmd:
+            if 'vcsserver.ini' in proc.cmd:
                 return 'VCSServer'
-            elif 'rhodecode.ini' in cmd:
+            elif 'rhodecode.ini' in proc.cmd:
                 return 'RhodeCode'
-            return proc.name()
+            return proc.name
     %>
     <tr>
         <td colspan="8">
             <span id="processTimeStamp">${h.format_date(h.datetime.now())}</span>
         </td>
     </tr>
     % for proc in c.gunicorn_processes:
-        <% mem = proc.memory_info()%>
-        <% children = proc.children(recursive=True) %>
-        % if children:
 
+        % if proc.children:
         <tr>
             <td>
                 <code>
                 ${proc.pid} - ${get_name(proc)}
                 </code>
             </td>
             <td>
                 <a href="#showCommand" onclick="$('#pid'+${proc.pid}).toggle();return false"> command </a>
                 <code id="pid${proc.pid}" style="display: none">
-                ${' '.join(proc.cmdline())}
+                ${proc.cmd}
                 </code>
             </td>
             <td></td>
             <td>
-                RSS:${h.format_byte_size_binary(mem.rss)}
+                RSS:${h.format_byte_size_binary(proc.mem_rss)}
             </td>
             <td>
-                VMS:${h.format_byte_size_binary(mem.vms)}
+                VMS:${h.format_byte_size_binary(proc.mem_vms)}
             </td>
             <td>
-                AGE: ${h.age_component(h.time_to_utcdatetime(proc.create_time()))}
+                AGE: ${h.age_component(h.time_to_utcdatetime(proc.create_time))}
             </td>
             <td>
                 MASTER
                 % if request.GET.get('dev'):
                     | <a href="#addWorker" onclick="addWorker(${proc.pid}); return false">ADD</a> | <a href="#removeWorker" onclick="removeWorker(${proc.pid}); return false">REMOVE</a>
                 % endif
             </td>
         </tr>
         <% mem_sum = 0 %>
-        % for proc_child in children:
-            <% mem = proc_child.memory_info()%>
+        % for proc_child in proc.children:
             <tr>
                 <td>
                     <code>
                       | ${proc_child.pid} - ${get_name(proc_child)}
                     </code>
                 </td>
                 <td>
                     <a href="#showCommand" onclick="$('#pid'+${proc_child.pid}).toggle();return false"> command </a>
                     <code id="pid${proc_child.pid}" style="display: none">
-                    ${' '.join(proc_child.cmdline())}
+                    ${proc_child.cmd}
                     </code>
                 </td>
                 <td>
-                    CPU: ${proc_child.cpu_percent()} %
+                    CPU: ${proc_child.cpu_percent} %
                 </td>
                 <td>
-                    RSS:${h.format_byte_size_binary(mem.rss)}
-                    <% mem_sum += mem.rss %>
+                    RSS:${h.format_byte_size_binary(proc_child.mem_rss)}
+                    <% mem_sum += proc_child.mem_rss %>
                 </td>
                 <td>
-                    VMS:${h.format_byte_size_binary(mem.vms)}
+                    VMS:${h.format_byte_size_binary(proc_child.mem_vms)}
                 </td>
                 <td>
-                    AGE: ${h.age_component(h.time_to_utcdatetime(proc_child.create_time()))}
+                    AGE: ${h.age_component(h.time_to_utcdatetime(proc_child.create_time))}
                 </td>
                 <td>
                     <a href="#restartProcess" onclick="restart(this, ${proc_child.pid});return false">
                         restart
                     </a>
                 </td>
             </tr>
         % endfor
         <tr>
-            <td colspan="2"><code>| total processes: ${len(children)}</code></td>
+            <td colspan="2"><code>| total processes: ${len(proc.children)}</code></td>
             <td></td>
             <td><strong>RSS:${h.format_byte_size_binary(mem_sum)}</strong></td>
             <td></td>
         </tr>
         <tr><td> <code> -- </code> </td></tr>
         % endif
     % endfor
 </table>