code docs, updates

rhodecode/lib/celerylib/__init__.py

@@ -1,106 +1,106
 # -*- coding: utf-8 -*-
 """
-    package.rhodecode.lib.celerylib.__init__
-    ~~~~~~~~~~~~~~
+    rhodecode.lib.celerylib.__init__
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     celery libs for RhodeCode
     
     :created_on: Nov 27, 2010
     :author: marcink
     :copyright: (C) 2009-2011 Marcin Kuzminski <[e-mail]>    
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
 import os
 import sys
 import socket
 import traceback
 import logging
 
 from hashlib import md5
 from decorator import decorator
 from vcs.utils.lazy import LazyProperty
 
 from rhodecode.lib.pidlock import DaemonLock, LockHeld
 
 from pylons import  config
 
 log = logging.getLogger(__name__)
 
 def str2bool(v):
     return v.lower() in ["yes", "true", "t", "1"] if v else None
 
 try:
     CELERY_ON = str2bool(config['app_conf'].get('use_celery'))
 except KeyError:
     CELERY_ON = False
 
 class ResultWrapper(object):
     def __init__(self, task):
         self.task = task
 
     @LazyProperty
     def result(self):
         return self.task
 
 def run_task(task, *args, **kwargs):
     if CELERY_ON:
         try:
             t = task.delay(*args, **kwargs)
             log.info('running task %s:%s', t.task_id, task)
             return t
         except socket.error, e:
             if  e.errno == 111:
                 log.debug('Unable to connect to celeryd. Sync execution')
             else:
                 log.error(traceback.format_exc())
         except KeyError, e:
                 log.debug('Unable to connect to celeryd. Sync execution')
         except Exception, e:
             log.error(traceback.format_exc())
 
     log.debug('executing task %s in sync mode', task)
     return ResultWrapper(task(*args, **kwargs))
 
 
 def locked_task(func):
     def __wrapper(func, *fargs, **fkwargs):
         params = list(fargs)
         params.extend(['%s-%s' % ar for ar in fkwargs.items()])
 
         lockkey = 'task_%s' % \
             md5(str(func.__name__) + '-' + \
                 '-'.join(map(str, params))).hexdigest()
         log.info('running task with lockkey %s', lockkey)
         try:
             l = DaemonLock(lockkey)
             ret = func(*fargs, **fkwargs)
             l.release()
             return ret
         except LockHeld:
             log.info('LockHeld')
             return 'Task with key %s already running' % lockkey
 
     return decorator(__wrapper, func)
 
 
 
 
 
 
 
 

rhodecode/lib/celerylib/tasks.py

@@ -1,359 +1,386
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.lib.celerylib.tasks
+    ~~~~~~~~~~~~~~
+
+    RhodeCode task modules, containing all task that suppose to be run
+    by celery daemon
+    
+    :created_on: Oct 6, 2010
+    :author: marcink
+    :copyright: (C) 2009-2011 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your opinion) any later version of the license.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
 from celery.decorators import task
 
 import os
 import traceback
 from time import mktime
 from operator import itemgetter
 
 from pylons import config
 from pylons.i18n.translation import _
 
 from rhodecode.lib.celerylib import run_task, locked_task, str2bool
 from rhodecode.lib.helpers import person
 from rhodecode.lib.smtp_mailer import SmtpMailer
 from rhodecode.lib.utils import OrderedDict, add_cache
 from rhodecode.model import init_model
 from rhodecode.model import meta
 from rhodecode.model.db import RhodeCodeUi
 
 from vcs.backends import get_repo
 
 from sqlalchemy import engine_from_config
 
 add_cache(config)
 
 try:
     import json
 except ImportError:
     #python 2.5 compatibility
     import simplejson as json
 
 __all__ = ['whoosh_index', 'get_commits_stats',
            'reset_user_password', 'send_email']
 
 CELERY_ON = str2bool(config['app_conf'].get('use_celery'))
 
 def get_session():
     if CELERY_ON:
         engine = engine_from_config(config, 'sqlalchemy.db1.')
         init_model(engine)
     sa = meta.Session()
     return sa
 
 def get_repos_path():
     sa = get_session()
     q = sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
     return q.ui_value
 
 @task
 @locked_task
 def whoosh_index(repo_location, full_index):
     log = whoosh_index.get_logger()
     from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
     index_location = config['index_dir']
     WhooshIndexingDaemon(index_location=index_location,
                          repo_location=repo_location, sa=get_session())\
                          .run(full_index=full_index)
 
 @task
 @locked_task
 def get_commits_stats(repo_name, ts_min_y, ts_max_y):
     from rhodecode.model.db import Statistics, Repository
     log = get_commits_stats.get_logger()
 
     #for js data compatibilty
     author_key_cleaner = lambda k: person(k).replace('"', "")
 
     commits_by_day_author_aggregate = {}
     commits_by_day_aggregate = {}
     repos_path = get_repos_path()
     p = os.path.join(repos_path, repo_name)
     repo = get_repo(p)
 
     skip_date_limit = True
     parse_limit = 250 #limit for single task changeset parsing optimal for
     last_rev = 0
     last_cs = None
     timegetter = itemgetter('time')
 
     sa = get_session()
 
     dbrepo = sa.query(Repository)\
         .filter(Repository.repo_name == repo_name).scalar()
     cur_stats = sa.query(Statistics)\
         .filter(Statistics.repository == dbrepo).scalar()
     if cur_stats:
         last_rev = cur_stats.stat_on_revision
     if not repo.revisions:
         return True
 
     if last_rev == repo.revisions[-1] and len(repo.revisions) > 1:
         #pass silently without any work if we're not on first revision or 
         #current state of parsing revision(from db marker) is the last revision
         return True
 
     if cur_stats:
         commits_by_day_aggregate = OrderedDict(
                                        json.loads(
                                         cur_stats.commit_activity_combined))
         commits_by_day_author_aggregate = json.loads(cur_stats.commit_activity)
 
     log.debug('starting parsing %s', parse_limit)
     lmktime = mktime
 
     last_rev = last_rev + 1 if last_rev > 0 else last_rev
     for rev in repo.revisions[last_rev:last_rev + parse_limit]:
         last_cs = cs = repo.get_changeset(rev)
         k = lmktime([cs.date.timetuple()[0], cs.date.timetuple()[1],
                       cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0])
 
         if commits_by_day_author_aggregate.has_key(author_key_cleaner(cs.author)):
             try:
                 l = [timegetter(x) for x in commits_by_day_author_aggregate\
                         [author_key_cleaner(cs.author)]['data']]
                 time_pos = l.index(k)
             except ValueError:
                 time_pos = False
 
             if time_pos >= 0 and time_pos is not False:
 
                 datadict = commits_by_day_author_aggregate\
                     [author_key_cleaner(cs.author)]['data'][time_pos]
 
                 datadict["commits"] += 1
                 datadict["added"] += len(cs.added)
                 datadict["changed"] += len(cs.changed)
                 datadict["removed"] += len(cs.removed)
 
             else:
                 if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
 
                     datadict = {"time":k,
                                 "commits":1,
                                 "added":len(cs.added),
                                 "changed":len(cs.changed),
                                 "removed":len(cs.removed),
                                }
                     commits_by_day_author_aggregate\
                         [author_key_cleaner(cs.author)]['data'].append(datadict)
 
         else:
             if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
                 commits_by_day_author_aggregate[author_key_cleaner(cs.author)] = {
                                     "label":author_key_cleaner(cs.author),
                                     "data":[{"time":k,
                                              "commits":1,
                                              "added":len(cs.added),
                                              "changed":len(cs.changed),
                                              "removed":len(cs.removed),
                                              }],
                                     "schema":["commits"],
                                     }
 
         #gather all data by day
         if commits_by_day_aggregate.has_key(k):
             commits_by_day_aggregate[k] += 1
         else:
             commits_by_day_aggregate[k] = 1
 
     overview_data = sorted(commits_by_day_aggregate.items(), key=itemgetter(0))
     if not commits_by_day_author_aggregate:
         commits_by_day_author_aggregate[author_key_cleaner(repo.contact)] = {
             "label":author_key_cleaner(repo.contact),
             "data":[0, 1],
             "schema":["commits"],
         }
 
     stats = cur_stats if cur_stats else Statistics()
     stats.commit_activity = json.dumps(commits_by_day_author_aggregate)
     stats.commit_activity_combined = json.dumps(overview_data)
 
     log.debug('last revison %s', last_rev)
     leftovers = len(repo.revisions[last_rev:])
     log.debug('revisions to parse %s', leftovers)
 
     if last_rev == 0 or leftovers < parse_limit:
         log.debug('getting code trending stats')
         stats.languages = json.dumps(__get_codes_stats(repo_name))
 
     stats.repository = dbrepo
     stats.stat_on_revision = last_cs.revision
 
     try:
         sa.add(stats)
         sa.commit()
     except:
         log.error(traceback.format_exc())
         sa.rollback()
         return False
     if len(repo.revisions) > 1:
         run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y)
 
     return True
 
 @task
 def reset_user_password(user_email):
     log = reset_user_password.get_logger()
     from rhodecode.lib import auth
     from rhodecode.model.db import User
 
     try:
         try:
             sa = get_session()
             user = sa.query(User).filter(User.email == user_email).scalar()
             new_passwd = auth.PasswordGenerator().gen_password(8,
                              auth.PasswordGenerator.ALPHABETS_BIG_SMALL)
             if user:
                 user.password = auth.get_crypt_password(new_passwd)
                 sa.add(user)
                 sa.commit()
                 log.info('change password for %s', user_email)
             if new_passwd is None:
                 raise Exception('unable to generate new password')
 
         except:
             log.error(traceback.format_exc())
             sa.rollback()
 
         run_task(send_email, user_email,
                  "Your new rhodecode password",
                  'Your new rhodecode password:%s' % (new_passwd))
         log.info('send new password mail to %s', user_email)
 
 
     except:
         log.error('Failed to update user password')
         log.error(traceback.format_exc())
 
     return True
 
 @task
 def send_email(recipients, subject, body):
     """
     Sends an email with defined parameters from the .ini files.
     
     
     :param recipients: list of recipients, it this is empty the defined email
         address from field 'email_to' is used instead
     :param subject: subject of the mail
     :param body: body of the mail
     """
     log = send_email.get_logger()
     email_config = config
 
     if not recipients:
         recipients = [email_config.get('email_to')]
 
     mail_from = email_config.get('app_email_from')
     user = email_config.get('smtp_username')
     passwd = email_config.get('smtp_password')
     mail_server = email_config.get('smtp_server')
     mail_port = email_config.get('smtp_port')
     tls = str2bool(email_config.get('smtp_use_tls'))
     ssl = str2bool(email_config.get('smtp_use_ssl'))
 
     try:
         m = SmtpMailer(mail_from, user, passwd, mail_server,
                        mail_port, ssl, tls)
         m.send(recipients, subject, body)
     except:
         log.error('Mail sending failed')
         log.error(traceback.format_exc())
         return False
     return True
 
 @task
 def create_repo_fork(form_data, cur_user):
     from rhodecode.model.repo import RepoModel
     from vcs import get_backend
     log = create_repo_fork.get_logger()
     repo_model = RepoModel(get_session())
     repo_model.create(form_data, cur_user, just_db=True, fork=True)
     repo_name = form_data['repo_name']
     repos_path = get_repos_path()
     repo_path = os.path.join(repos_path, repo_name)
     repo_fork_path = os.path.join(repos_path, form_data['fork_name'])
     alias = form_data['repo_type']
 
     log.info('creating repo fork %s as %s', repo_name, repo_path)
     backend = get_backend(alias)
     backend(str(repo_fork_path), create=True, src_url=str(repo_path))
 
 def __get_codes_stats(repo_name):
     LANGUAGES_EXTENSIONS_MAP = {'scm': 'Scheme', 'asmx': 'VbNetAspx', 'Rout':
     'RConsole', 'rest': 'Rst', 'abap': 'ABAP', 'go': 'Go', 'phtml': 'HtmlPhp',
     'ns2': 'Newspeak', 'xml': 'EvoqueXml', 'sh-session': 'BashSession', 'ads':
     'Ada', 'clj': 'Clojure', 'll': 'Llvm', 'ebuild': 'Bash', 'adb': 'Ada',
     'ada': 'Ada', 'c++-objdump': 'CppObjdump', 'aspx':
     'VbNetAspx', 'ksh': 'Bash', 'coffee': 'CoffeeScript', 'vert': 'GLShader',
     'Makefile.*': 'Makefile', 'di': 'D', 'dpatch': 'DarcsPatch', 'rake':
     'Ruby', 'moo': 'MOOCode', 'erl-sh': 'ErlangShell', 'geo': 'GLShader',
     'pov': 'Povray', 'bas': 'VbNet', 'bat': 'Batch', 'd': 'D', 'lisp':
     'CommonLisp', 'h': 'C', 'rbx': 'Ruby', 'tcl': 'Tcl', 'c++': 'Cpp', 'md':
     'MiniD', '.vimrc': 'Vim', 'xsd': 'Xml', 'ml': 'Ocaml', 'el': 'CommonLisp',
     'befunge': 'Befunge', 'xsl': 'Xslt', 'pyx': 'Cython', 'cfm':
     'ColdfusionHtml', 'evoque': 'Evoque', 'cfg': 'Ini', 'htm': 'Html',
     'Makefile': 'Makefile', 'cfc': 'ColdfusionHtml', 'tex': 'Tex', 'cs':
     'CSharp', 'mxml': 'Mxml', 'patch': 'Diff', 'apache.conf': 'ApacheConf',
     'scala': 'Scala', 'applescript': 'AppleScript', 'GNUmakefile': 'Makefile',
     'c-objdump': 'CObjdump', 'lua': 'Lua', 'apache2.conf': 'ApacheConf', 'rb':
     'Ruby', 'gemspec': 'Ruby', 'rl': 'RagelObjectiveC', 'vala': 'Vala', 'tmpl':
     'Cheetah', 'bf': 'Brainfuck', 'plt': 'Gnuplot', 'G': 'AntlrRuby', 'xslt':
     'Xslt', 'flxh': 'Felix', 'asax': 'VbNetAspx', 'Rakefile': 'Ruby', 'S': 'S',
     'wsdl': 'Xml', 'js': 'Javascript', 'autodelegate': 'Myghty', 'properties':
     'Ini', 'bash': 'Bash', 'c': 'C', 'g': 'AntlrRuby', 'r3': 'Rebol', 's':
     'Gas', 'ashx': 'VbNetAspx', 'cxx': 'Cpp', 'boo': 'Boo', 'prolog': 'Prolog',
     'sqlite3-console': 'SqliteConsole', 'cl': 'CommonLisp', 'cc': 'Cpp', 'pot':
     'Gettext', 'vim': 'Vim', 'pxi': 'Cython', 'yaml': 'Yaml', 'SConstruct':
     'Python', 'diff': 'Diff', 'txt': 'Text', 'cw': 'Redcode', 'pxd': 'Cython',
     'plot': 'Gnuplot', 'java': 'Java', 'hrl': 'Erlang', 'py': 'Python',
     'makefile': 'Makefile', 'squid.conf': 'SquidConf', 'asm': 'Nasm', 'toc':
     'Tex', 'kid': 'Genshi', 'rhtml': 'Rhtml', 'po': 'Gettext', 'pl': 'Prolog',
     'pm': 'Perl', 'hx': 'Haxe', 'ascx': 'VbNetAspx', 'ooc': 'Ooc', 'asy':
     'Asymptote', 'hs': 'Haskell', 'SConscript': 'Python', 'pytb':
     'PythonTraceback', 'myt': 'Myghty', 'hh': 'Cpp', 'R': 'S', 'aux': 'Tex',
     'rst': 'Rst', 'cpp-objdump': 'CppObjdump', 'lgt': 'Logtalk', 'rss': 'Xml',
     'flx': 'Felix', 'b': 'Brainfuck', 'f': 'Fortran', 'rbw': 'Ruby',
     '.htaccess': 'ApacheConf', 'cxx-objdump': 'CppObjdump', 'j': 'ObjectiveJ',
     'mll': 'Ocaml', 'yml': 'Yaml', 'mu': 'MuPAD', 'r': 'Rebol', 'ASM': 'Nasm',
     'erl': 'Erlang', 'mly': 'Ocaml', 'mo': 'Modelica', 'def': 'Modula2', 'ini':
     'Ini', 'control': 'DebianControl', 'vb': 'VbNet', 'vapi': 'Vala', 'pro':
     'Prolog', 'spt': 'Cheetah', 'mli': 'Ocaml', 'as': 'ActionScript3', 'cmd':
     'Batch', 'cpp': 'Cpp', 'io': 'Io', 'tac': 'Python', 'haml': 'Haml', 'rkt':
     'Racket', 'st':'Smalltalk', 'inc': 'Povray', 'pas': 'Delphi', 'cmake':
     'CMake', 'csh':'Tcsh', 'hpp': 'Cpp', 'feature': 'Gherkin', 'html': 'Html',
     'php':'Php', 'php3':'Php', 'php4':'Php', 'php5':'Php', 'xhtml': 'Html',
     'hxx': 'Cpp', 'eclass': 'Bash', 'css': 'Css',
     'frag': 'GLShader', 'd-objdump': 'DObjdump', 'weechatlog': 'IrcLogs',
     'tcsh': 'Tcsh', 'objdump': 'Objdump', 'pyw': 'Python', 'h++': 'Cpp',
     'py3tb': 'Python3Traceback', 'jsp': 'Jsp', 'sql': 'Sql', 'mak': 'Makefile',
     'php': 'Php', 'mao': 'Mako', 'man': 'Groff', 'dylan': 'Dylan', 'sass':
     'Sass', 'cfml': 'ColdfusionHtml', 'darcspatch': 'DarcsPatch', 'tpl':
     'Smarty', 'm': 'ObjectiveC', 'f90': 'Fortran', 'mod': 'Modula2', 'sh':
     'Bash', 'lhs': 'LiterateHaskell', 'sources.list': 'SourcesList', 'axd':
     'VbNetAspx', 'sc': 'Python'}
 
     repos_path = get_repos_path()
     p = os.path.join(repos_path, repo_name)
     repo = get_repo(p)
     tip = repo.get_changeset()
     code_stats = {}
 
     def aggregate(cs):
         for f in cs[2]:
             ext = f.extension
             key = LANGUAGES_EXTENSIONS_MAP.get(ext, ext)
             key = key or ext
             if ext in LANGUAGES_EXTENSIONS_MAP.keys() and not f.is_binary:
                 if code_stats.has_key(key):
                     code_stats[key] += 1
                 else:
                     code_stats[key] = 1
 
     map(aggregate, tip.walk('/'))
 
     return code_stats or {}
 
 
 
 

rhodecode/lib/indexers/__init__.py

@@ -1,203 +1,229
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.lib.indexers.__init__
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Whoosh indexing module for RhodeCode
+    
+    :created_on: Aug 17, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your opinion) any later version of the license.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
 import os
 import sys
 import traceback
 from os.path import dirname as dn, join as jn
 
 #to get the rhodecode import
 sys.path.append(dn(dn(dn(os.path.realpath(__file__)))))
 
 from string import strip
 
 from rhodecode.model import init_model
 from rhodecode.model.scm import ScmModel
 from rhodecode.config.environment import load_environment
 from rhodecode.lib.utils import BasePasterCommand, Command, add_cache
 
 from shutil import rmtree
 from webhelpers.html.builder import escape
 from vcs.utils.lazy import LazyProperty
 
 from sqlalchemy import engine_from_config
 
 from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter
 from whoosh.fields import TEXT, ID, STORED, Schema, FieldType
 from whoosh.index import create_in, open_dir
 from whoosh.formats import Characters
 from whoosh.highlight import highlight, SimpleFragmenter, HtmlFormatter
 
 
 #EXTENSIONS WE WANT TO INDEX CONTENT OFF
 INDEX_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c',
                     'cfg', 'cfm', 'cpp', 'cs', 'css', 'diff', 'do', 'el', 'erl',
                     'h', 'htm', 'html', 'ini', 'java', 'js', 'jsp', 'jspx', 'lisp',
                     'lua', 'm', 'mako', 'ml', 'pas', 'patch', 'php', 'php3',
                     'php4', 'phtml', 'pm', 'py', 'rb', 'rst', 's', 'sh', 'sql',
                     'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt',
                     'yaws']
 
 #CUSTOM ANALYZER wordsplit + lowercase filter
 ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()
 
 
 #INDEX SCHEMA DEFINITION
 SCHEMA = Schema(owner=TEXT(),
                 repository=TEXT(stored=True),
                 path=TEXT(stored=True),
                 content=FieldType(format=Characters(ANALYZER),
                              scorable=True, stored=True),
                 modtime=STORED(), extension=TEXT(stored=True))
 
 
 IDX_NAME = 'HG_INDEX'
 FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n')
 FRAGMENTER = SimpleFragmenter(200)
 
 
 class MakeIndex(BasePasterCommand):
 
     max_args = 1
     min_args = 1
 
     usage = "CONFIG_FILE"
     summary = "Creates index for full text search given configuration file"
     group_name = "RhodeCode"
     takes_config_file = -1
     parser = Command.standard_parser(verbose=True)
 
     def command(self):
 
         from pylons import config
         add_cache(config)
         engine = engine_from_config(config, 'sqlalchemy.db1.')
         init_model(engine)
 
         index_location = config['index_dir']
         repo_location = self.options.repo_location
         repo_list = map(strip, self.options.repo_list.split(','))
 
         #======================================================================
         # WHOOSH DAEMON
         #======================================================================
         from rhodecode.lib.pidlock import LockHeld, DaemonLock
         from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
         try:
             l = DaemonLock()
             WhooshIndexingDaemon(index_location=index_location,
                                  repo_location=repo_location,
                                  repo_list=repo_list)\
                 .run(full_index=self.options.full_index)
             l.release()
         except LockHeld:
             sys.exit(1)
 
     def update_parser(self):
         self.parser.add_option('--repo-location',
                           action='store',
                           dest='repo_location',
                           help="Specifies repositories location to index REQUIRED",
                           )
         self.parser.add_option('--index-only',
                           action='store',
                           dest='repo_list',
                           help="Specifies a comma separated list of repositores "
                                 "to build index on OPTIONAL",
                           )
         self.parser.add_option('-f',
                           action='store_true',
                           dest='full_index',
                           help="Specifies that index should be made full i.e"
                                 " destroy old and build from scratch",
                           default=False)
 
 class ResultWrapper(object):
     def __init__(self, search_type, searcher, matcher, highlight_items):
         self.search_type = search_type
         self.searcher = searcher
         self.matcher = matcher
         self.highlight_items = highlight_items
         self.fragment_size = 200 / 2
 
     @LazyProperty
     def doc_ids(self):
         docs_id = []
         while self.matcher.is_active():
             docnum = self.matcher.id()
             chunks = [offsets for offsets in self.get_chunks()]
             docs_id.append([docnum, chunks])
             self.matcher.next()
         return docs_id
 
     def __str__(self):
         return '<%s at %s>' % (self.__class__.__name__, len(self.doc_ids))
 
     def __repr__(self):
         return self.__str__()
 
     def __len__(self):
         return len(self.doc_ids)
 
     def __iter__(self):
         """
         Allows Iteration over results,and lazy generate content
 
         *Requires* implementation of ``__getitem__`` method.
         """
         for docid in self.doc_ids:
             yield self.get_full_content(docid)
 
     def __getslice__(self, i, j):
         """
         Slicing of resultWrapper
         """
         slice = []
         for docid in self.doc_ids[i:j]:
             slice.append(self.get_full_content(docid))
         return slice
 
 
     def get_full_content(self, docid):
         res = self.searcher.stored_fields(docid[0])
         f_path = res['path'][res['path'].find(res['repository']) \
                              + len(res['repository']):].lstrip('/')
 
         content_short = self.get_short_content(res, docid[1])
         res.update({'content_short':content_short,
                     'content_short_hl':self.highlight(content_short),
                     'f_path':f_path})
 
         return res
 
     def get_short_content(self, res, chunks):
 
         return ''.join([res['content'][chunk[0]:chunk[1]] for chunk in chunks])
 
     def get_chunks(self):
         """
         Smart function that implements chunking the content
         but not overlap chunks so it doesn't highlight the same
         close occurrences twice.
         @param matcher:
         @param size:
         """
         memory = [(0, 0)]
         for span in self.matcher.spans():
             start = span.startchar or 0
             end = span.endchar or 0
             start_offseted = max(0, start - self.fragment_size)
             end_offseted = end + self.fragment_size
 
             if start_offseted < memory[-1][1]:
                 start_offseted = memory[-1][1]
             memory.append((start_offseted, end_offseted,))
             yield (start_offseted, end_offseted,)
 
     def highlight(self, content, top=5):
         if self.search_type != 'content':
             return ''
         hl = highlight(escape(content),
                  self.highlight_items,
                  analyzer=ANALYZER,
                  fragmenter=FRAGMENTER,
                  formatter=FORMATTER,
                  top=top)
         return hl

rhodecode/lib/middleware/https_fixup.py

@@ -1,47 +1,48
-#!/usr/bin/env python
-# encoding: utf-8
-# middleware to handle https correctly
-# Copyright (C) 2009-2011 Marcin Kuzminski <[e-mail]>
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.lib.middleware.https_fixup
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
+    middleware to handle https correctly
+    
+    :created_on: May 23, 2010
+    :author: marcink
+    :copyright: (C) 2009-2011 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-"""
-Created on May 23, 2010
-
-@author: marcink
-"""
-
 class HttpsFixup(object):
     def __init__(self, app):
         self.application = app
     
     def __call__(self, environ, start_response):
         self.__fixup(environ)
         return self.application(environ, start_response)
     
     
     def __fixup(self, environ):
         """Function to fixup the environ as needed. In order to use this
         middleware you should set this header inside your 
         proxy ie. nginx, apache etc.
         """
         proto = environ.get('HTTP_X_URL_SCHEME')
             
         if proto == 'https':
             environ['wsgi.url_scheme'] = proto
         else:
             environ['wsgi.url_scheme'] = 'http'
         return None

rhodecode/lib/middleware/simplegit.py

@@ -1,222 +1,226
-#!/usr/bin/env python
-# encoding: utf-8
-# middleware to handle git api calls
-# Copyright (C) 2009-2011 Marcin Kuzminski <[e-mail]>
-#
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.lib.middleware.simplegit
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    SimpleGit middleware for handling git protocol request (push/clone etc.)
+    It's implemented with basic auth function    
+    
+    :created_on: Apr 28, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
-"""
-Created on 2010-04-28
 
-@author: marcink
-SimpleGit middleware for handling git protocol request (push/clone etc.)
-It's implemented with basic auth function
-"""
+import os
+import logging
+import traceback
 
 from dulwich import server as dulserver
 
 class SimpleGitUploadPackHandler(dulserver.UploadPackHandler):
 
     def handle(self):
         write = lambda x: self.proto.write_sideband(1, x)
 
         graph_walker = dulserver.ProtocolGraphWalker(self, self.repo.object_store,
             self.repo.get_peeled)
         objects_iter = self.repo.fetch_objects(
           graph_walker.determine_wants, graph_walker, self.progress,
           get_tagged=self.get_tagged)
 
         # Do they want any objects?
         if len(objects_iter) == 0:
             return
 
         self.progress("counting objects: %d, done.\n" % len(objects_iter))
         dulserver.write_pack_data(dulserver.ProtocolFile(None, write), objects_iter,
                         len(objects_iter))
         messages = []
         messages.append('thank you for using rhodecode')
 
         for msg in messages:
             self.progress(msg + "\n")
         # we are done
         self.proto.write("0000")
 
 dulserver.DEFAULT_HANDLERS = {
   'git-upload-pack': SimpleGitUploadPackHandler,
   'git-receive-pack': dulserver.ReceivePackHandler,
 }
 
 from dulwich.repo import Repo
 from dulwich.web import HTTPGitApplication
+
 from paste.auth.basic import AuthBasicAuthenticator
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
+
 from rhodecode.lib.auth import authfunc, HasPermissionAnyMiddleware
 from rhodecode.lib.utils import invalidate_cache, check_repo_fast
 from rhodecode.model.user import UserModel
+
 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError
-import logging
-import os
-import traceback
 
 log = logging.getLogger(__name__)
 
 def is_git(environ):
-    """
-    Returns True if request's target is git server. ``HTTP_USER_AGENT`` would
+    """Returns True if request's target is git server. ``HTTP_USER_AGENT`` would
     then have git client version given.
     
     :param environ:
     """
     http_user_agent = environ.get('HTTP_USER_AGENT')
     if http_user_agent and http_user_agent.startswith('git'):
         return True
     return False
 
 class SimpleGit(object):
 
     def __init__(self, application, config):
         self.application = application
         self.config = config
         #authenticate this git request using 
         self.authenticate = AuthBasicAuthenticator('', authfunc)
         self.ipaddr = '0.0.0.0'
         self.repository = None
         self.username = None
         self.action = None
 
     def __call__(self, environ, start_response):
         if not is_git(environ):
             return self.application(environ, start_response)
 
         proxy_key = 'HTTP_X_REAL_IP'
         def_key = 'REMOTE_ADDR'
         self.ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))
         # skip passing error to error controller
         environ['pylons.status_code_redirect'] = True
         #===================================================================
         # AUTHENTICATE THIS GIT REQUEST
         #===================================================================
         username = REMOTE_USER(environ)
         if not username:
             self.authenticate.realm = self.config['rhodecode_realm']
             result = self.authenticate(environ)
             if isinstance(result, str):
                 AUTH_TYPE.update(environ, 'basic')
                 REMOTE_USER.update(environ, result)
             else:
                 return result.wsgi_application(environ, start_response)
 
         #=======================================================================
         # GET REPOSITORY
         #=======================================================================
         try:
             repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])
             if repo_name.endswith('/'):
                 repo_name = repo_name.rstrip('/')
             self.repository = repo_name
         except:
             log.error(traceback.format_exc())
             return HTTPInternalServerError()(environ, start_response)
 
         #===================================================================
         # CHECK PERMISSIONS FOR THIS REQUEST
         #===================================================================
         self.action = self.__get_action(environ)
         if self.action:
             username = self.__get_environ_user(environ)
             try:
                 user = self.__get_user(username)
                 self.username = user.username
             except:
                 log.error(traceback.format_exc())
                 return HTTPInternalServerError()(environ, start_response)
 
             #check permissions for this repository
             if self.action == 'push':
                 if not HasPermissionAnyMiddleware('repository.write',
                                                   'repository.admin')\
                                                     (user, repo_name):
                     return HTTPForbidden()(environ, start_response)
 
             else:
                 #any other action need at least read permission
                 if not HasPermissionAnyMiddleware('repository.read',
                                                   'repository.write',
                                                   'repository.admin')\
                                                     (user, repo_name):
                     return HTTPForbidden()(environ, start_response)
 
         self.extras = {'ip':self.ipaddr,
                        'username':self.username,
                        'action':self.action,
                        'repository':self.repository}
 
         #===================================================================
         # GIT REQUEST HANDLING
         #===================================================================
         self.basepath = self.config['base_path']
         self.repo_path = os.path.join(self.basepath, self.repo_name)
         #quick check if that dir exists...
         if check_repo_fast(self.repo_name, self.basepath):
             return HTTPNotFound()(environ, start_response)
         try:
             app = self.__make_app()
         except:
             log.error(traceback.format_exc())
             return HTTPInternalServerError()(environ, start_response)
 
         #invalidate cache on push
         if self.action == 'push':
             self.__invalidate_cache(self.repo_name)
             messages = []
             messages.append('thank you for using rhodecode')
             return app(environ, start_response)
         else:
             return app(environ, start_response)
 
 
     def __make_app(self):
         backend = dulserver.DictBackend({'/' + self.repo_name: Repo(self.repo_path)})
         gitserve = HTTPGitApplication(backend)
 
         return gitserve
 
     def __get_environ_user(self, environ):
         return environ.get('REMOTE_USER')
 
     def __get_user(self, username):
         return UserModel().get_by_username(username, cache=True)
 
     def __get_action(self, environ):
-        """
-        Maps git request commands into a pull or push command.
+        """Maps git request commands into a pull or push command.
+        
         :param environ:
         """
         service = environ['QUERY_STRING'].split('=')
         if len(service) > 1:
             service_cmd = service[1]
             mapping = {'git-receive-pack': 'push',
                        'git-upload-pack': 'pull',
                        }
 
             return mapping.get(service_cmd, service_cmd if service_cmd else 'other')
         else:
             return 'other'
 
     def __invalidate_cache(self, repo_name):
         """we know that some change was made to repositories and we should
         invalidate the cache to see the changes right away but only for
         push requests"""
         invalidate_cache('get_repo_cached_%s' % repo_name)

rhodecode/lib/middleware/simplehg.py

@@ -1,230 +1,216
-#!/usr/bin/env python
-# encoding: utf-8
-# middleware to handle mercurial api calls
-# Copyright (C) 2009-2011 Marcin Kuzminski <[e-mail]>
-#
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.lib.middleware.simplehg
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    SimpleHG middleware for handling mercurial protocol request 
+    (push/clone etc.). It's implemented with basic auth function
+    
+    :created_on: Apr 28, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
-"""
-Created on 2010-04-28
 
-@author: marcink
-SimpleHG middleware for handling mercurial protocol request (push/clone etc.)
-It's implemented with basic auth function
-"""
 from mercurial.error import RepoError
 from mercurial.hgweb import hgweb
 from mercurial.hgweb.request import wsgiapplication
 from paste.auth.basic import AuthBasicAuthenticator
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 from rhodecode.lib.auth import authfunc, HasPermissionAnyMiddleware
 from rhodecode.lib.utils import make_ui, invalidate_cache, \
     check_repo_fast, ui_sections
 from rhodecode.model.user import UserModel
 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError
 import logging
 import os
 import traceback
 
 log = logging.getLogger(__name__)
 
 def is_mercurial(environ):
-    """
-    Returns True if request's target is mercurial server - header
+    """Returns True if request's target is mercurial server - header
     ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
     """
     http_accept = environ.get('HTTP_ACCEPT')
     if http_accept and http_accept.startswith('application/mercurial'):
         return True
     return False
 
 class SimpleHg(object):
 
     def __init__(self, application, config):
         self.application = application
         self.config = config
         #authenticate this mercurial request using authfunc
         self.authenticate = AuthBasicAuthenticator('', authfunc)
         self.ipaddr = '0.0.0.0'
         self.repository = None
         self.username = None
         self.action = None
 
     def __call__(self, environ, start_response):
         if not is_mercurial(environ):
             return self.application(environ, start_response)
 
         proxy_key = 'HTTP_X_REAL_IP'
         def_key = 'REMOTE_ADDR'
         self.ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))
         # skip passing error to error controller
         environ['pylons.status_code_redirect'] = True
         #===================================================================
         # AUTHENTICATE THIS MERCURIAL REQUEST
         #===================================================================
         username = REMOTE_USER(environ)
 
         if not username:
             self.authenticate.realm = self.config['rhodecode_realm']
             result = self.authenticate(environ)
             if isinstance(result, str):
                 AUTH_TYPE.update(environ, 'basic')
                 REMOTE_USER.update(environ, result)
             else:
                 return result.wsgi_application(environ, start_response)
 
         #=======================================================================
         # GET REPOSITORY
         #=======================================================================
         try:
             repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])
             if repo_name.endswith('/'):
                 repo_name = repo_name.rstrip('/')
             self.repository = repo_name
         except:
             log.error(traceback.format_exc())
             return HTTPInternalServerError()(environ, start_response)
 
         #===================================================================
         # CHECK PERMISSIONS FOR THIS REQUEST
         #===================================================================
         self.action = self.__get_action(environ)
         if self.action:
             username = self.__get_environ_user(environ)
             try:
                 user = self.__get_user(username)
                 self.username = user.username
             except:
                 log.error(traceback.format_exc())
                 return HTTPInternalServerError()(environ, start_response)
 
             #check permissions for this repository
             if self.action == 'push':
                 if not HasPermissionAnyMiddleware('repository.write',
                                                   'repository.admin')\
                                                     (user, repo_name):
                     return HTTPForbidden()(environ, start_response)
 
             else:
                 #any other action need at least read permission
                 if not HasPermissionAnyMiddleware('repository.read',
                                                   'repository.write',
                                                   'repository.admin')\
                                                     (user, repo_name):
                     return HTTPForbidden()(environ, start_response)
 
         self.extras = {'ip':self.ipaddr,
                        'username':self.username,
                        'action':self.action,
                        'repository':self.repository}
 
         #===================================================================
         # MERCURIAL REQUEST HANDLING
         #===================================================================
         environ['PATH_INFO'] = '/'#since we wrap into hgweb, reset the path
         self.baseui = make_ui('db')
         self.basepath = self.config['base_path']
         self.repo_path = os.path.join(self.basepath, repo_name)
 
         #quick check if that dir exists...
         if check_repo_fast(repo_name, self.basepath):
             return HTTPNotFound()(environ, start_response)
         try:
             app = wsgiapplication(self.__make_app)
         except RepoError, e:
             if str(e).find('not found') != -1:
                 return HTTPNotFound()(environ, start_response)
         except Exception:
             log.error(traceback.format_exc())
             return HTTPInternalServerError()(environ, start_response)
 
         #invalidate cache on push
         if self.action == 'push':
             self.__invalidate_cache(repo_name)
 
         return app(environ, start_response)
 
 
     def __make_app(self):
         hgserve = hgweb(str(self.repo_path), baseui=self.baseui)
         return  self.__load_web_settings(hgserve, self.extras)
 
     def __get_environ_user(self, environ):
         return environ.get('REMOTE_USER')
 
     def __get_user(self, username):
         return UserModel().get_by_username(username, cache=True)
 
     def __get_action(self, environ):
-        """
-        Maps mercurial request commands into a clone,pull or push command.
+        """Maps mercurial request commands into a clone,pull or push command.
         This should always return a valid command string
         :param environ:
         """
         mapping = {'changegroup': 'pull',
                    'changegroupsubset': 'pull',
                    'stream_out': 'pull',
                    'listkeys': 'pull',
                    'unbundle': 'push',
                    'pushkey': 'push', }
         for qry in environ['QUERY_STRING'].split('&'):
             if qry.startswith('cmd'):
                 cmd = qry.split('=')[-1]
                 if mapping.has_key(cmd):
                     return mapping[cmd]
                 else:
                     return cmd
 
     def __invalidate_cache(self, repo_name):
         """we know that some change was made to repositories and we should
         invalidate the cache to see the changes right away but only for
         push requests"""
         invalidate_cache('get_repo_cached_%s' % repo_name)
 
 
     def __load_web_settings(self, hgserve, extras={}):
         #set the global ui for hgserve instance passed
         hgserve.repo.ui = self.baseui
 
         hgrc = os.path.join(self.repo_path, '.hg', 'hgrc')
 
         #inject some additional parameters that will be available in ui
         #for hooks
         for k, v in extras.items():
             hgserve.repo.ui.setconfig('rhodecode_extras', k, v)
 
         repoui = make_ui('file', hgrc, False)
 
         if repoui:
             #overwrite our ui instance with the section from hgrc file
             for section in ui_sections:
                 for k, v in repoui.configitems(section):
                     hgserve.repo.ui.setconfig(section, k, v)
 
         return hgserve
-
-
-
-
-
-
-
-
-
-
-
-
-
-