cached-commits: updated logic on cached commit updates....
dan -
r4162:5cadc4f9 default
Not Reviewed
Show More
Add another comment
TODOs: 0 unresolved 0 Resolved
COMMENTS: 0 General 0 Inline
@@ -55,7 +55,7
55 from webhelpers2.text import remove_formatting
55 from webhelpers2.text import remove_formatting
56
56
57 from rhodecode.translation import _
57 from rhodecode.translation import _
58 from rhodecode.lib.vcs import get_vcs_instance
58 from rhodecode.lib.vcs import get_vcs_instance, VCSError
59 from rhodecode.lib.vcs.backends.base import EmptyCommit, Reference
59 from rhodecode.lib.vcs.backends.base import EmptyCommit, Reference
60 from rhodecode.lib.utils2 import (
60 from rhodecode.lib.utils2 import (
61 str2bool, safe_str, get_commit_safe, safe_unicode, sha1_safe,
61 str2bool, safe_str, get_commit_safe, safe_unicode, sha1_safe,
@@ -2359,7 +2359,8
2359
2359
2360 def update_commit_cache(self, cs_cache=None, config=None):
2360 def update_commit_cache(self, cs_cache=None, config=None):
2361 """
2361 """
2362 Update cache of last commit for repository, keys should be::
2362 Update cache of last commit for repository
2363 cache_keys should be::
2363
2364
2364 source_repo_id
2365 source_repo_id
2365 short_id
2366 short_id
@@ -2373,11 +2374,17
2373
2374
2374 """
2375 """
2375 from rhodecode.lib.vcs.backends.base import BaseChangeset
2376 from rhodecode.lib.vcs.backends.base import BaseChangeset
2377 from rhodecode.lib.vcs.utils.helpers import parse_datetime
2378 empty_date = datetime.datetime.fromtimestamp(0)
2379
2376 if cs_cache is None:
2380 if cs_cache is None:
2377 # use no-cache version here
2381 # use no-cache version here
2378 scm_repo = self.scm_instance(cache=False, config=config)
2382 try:
2379
2383 scm_repo = self.scm_instance(cache=False, config=config)
2384 except VCSError:
2385 scm_repo = None
2380 empty = scm_repo is None or scm_repo.is_empty()
2386 empty = scm_repo is None or scm_repo.is_empty()
2387
2381 if not empty:
2388 if not empty:
2382 cs_cache = scm_repo.get_commit(
2389 cs_cache = scm_repo.get_commit(
2383 pre_load=["author", "date", "message", "parents", "branch"])
2390 pre_load=["author", "date", "message", "parents", "branch"])
@@ -2395,33 +2402,39
2395
2402
2396 # check if we have maybe already latest cached revision
2403 # check if we have maybe already latest cached revision
2397 if is_outdated(cs_cache) or not self.changeset_cache:
2404 if is_outdated(cs_cache) or not self.changeset_cache:
2398 _default = datetime.datetime.utcnow()
2405 _current_datetime = datetime.datetime.utcnow()
2399 last_change = cs_cache.get('date') or _default
2406 last_change = cs_cache.get('date') or _current_datetime
2400 # we check if last update is newer than the new value
2407 # we check if last update is newer than the new value
2401 # if yes, we use the current timestamp instead. Imagine you get
2408 # if yes, we use the current timestamp instead. Imagine you get
2402 # old commit pushed 1y ago, we'd set last update 1y to ago.
2409 # old commit pushed 1y ago, we'd set last update 1y to ago.
2403 last_change_timestamp = datetime_to_time(last_change)
2410 last_change_timestamp = datetime_to_time(last_change)
2404 current_timestamp = datetime_to_time(last_change)
2411 current_timestamp = datetime_to_time(last_change)
2405 if last_change_timestamp > current_timestamp:
2412 if last_change_timestamp > current_timestamp and not empty:
2406 cs_cache['date'] = _default
2413 cs_cache['date'] = _current_datetime
2407
2414
2415 _date_latest = parse_datetime(cs_cache.get('date') or empty_date)
2408 cs_cache['updated_on'] = time.time()
2416 cs_cache['updated_on'] = time.time()
2409 self.changeset_cache = cs_cache
2417 self.changeset_cache = cs_cache
2410 self.updated_on = last_change
2418 self.updated_on = last_change
2411 Session().add(self)
2419 Session().add(self)
2412 Session().commit()
2420 Session().commit()
2413
2421
2414 log.debug('updated repo `%s` with new commit cache %s',
2415 self.repo_name, cs_cache)
2416 else:
2422 else:
2417 cs_cache = self.changeset_cache
2423 if empty:
2424 cs_cache = EmptyCommit().__json__()
2425 else:
2426 cs_cache = self.changeset_cache
2427
2428 _date_latest = parse_datetime(cs_cache.get('date') or empty_date)
2429
2418 cs_cache['updated_on'] = time.time()
2430 cs_cache['updated_on'] = time.time()
2419 self.changeset_cache = cs_cache
2431 self.changeset_cache = cs_cache
2432 self.updated_on = _date_latest
2420 Session().add(self)
2433 Session().add(self)
2421 Session().commit()
2434 Session().commit()
2422
2435
2423 log.debug('Skipping update_commit_cache for repo:`%s` '
2436 log.debug('updated repo `%s` with new commit cache %s, and last update_date: %s',
2424 'commit already with latest changes', self.repo_name)
2437 self.repo_name, cs_cache, _date_latest)
2425
2438
2426 @property
2439 @property
2427 def tip(self):
2440 def tip(self):
@@ -2886,7 +2899,8
2886
2899
2887 def update_commit_cache(self, config=None):
2900 def update_commit_cache(self, config=None):
2888 """
2901 """
2889 Update cache of last changeset for newest repository inside this group, keys should be::
2902 Update cache of last commit for newest repository inside this repository group.
2903 cache_keys should be::
2890
2904
2891 source_repo_id
2905 source_repo_id
2892 short_id
2906 short_id
@@ -2899,50 +2913,37
2899
2913
2900 """
2914 """
2901 from rhodecode.lib.vcs.utils.helpers import parse_datetime
2915 from rhodecode.lib.vcs.utils.helpers import parse_datetime
2902
2903 def repo_groups_and_repos():
2904 all_entries = OrderedDefaultDict(list)
2905
2906 def _get_members(root_gr, pos=0):
2907
2908 for repo in root_gr.repositories:
2909 all_entries[root_gr].append(repo)
2910
2911 # fill in all parent positions
2912 for parent_group in root_gr.parents:
2913 all_entries[parent_group].extend(all_entries[root_gr])
2914
2915 children_groups = root_gr.children.all()
2916 if children_groups:
2917 for cnt, gr in enumerate(children_groups, 1):
2918 _get_members(gr, pos=pos+cnt)
2919
2920 _get_members(root_gr=self)
2921 return all_entries
2922
2923 empty_date = datetime.datetime.fromtimestamp(0)
2916 empty_date = datetime.datetime.fromtimestamp(0)
2924 for repo_group, repos in repo_groups_and_repos().items():
2917
2925
2918 def repo_groups_and_repos(root_gr):
2926 latest_repo_cs_cache = {}
2919 for _repo in root_gr.repositories:
2927 _date_latest = empty_date
2920 yield _repo
2928 for repo in repos:
2921 for child_group in root_gr.children.all():
2929 repo_cs_cache = repo.changeset_cache
2922 yield child_group
2930 date_latest = latest_repo_cs_cache.get('date', empty_date)
2923
2931 date_current = repo_cs_cache.get('date', empty_date)
2924 latest_repo_cs_cache = {}
2932 current_timestamp = datetime_to_time(parse_datetime(date_latest))
2925 for obj in repo_groups_and_repos(self):
2933 if current_timestamp < datetime_to_time(parse_datetime(date_current)):
2926 repo_cs_cache = obj.changeset_cache
2934 latest_repo_cs_cache = repo_cs_cache
2927 date_latest = latest_repo_cs_cache.get('date', empty_date)
2935 latest_repo_cs_cache['source_repo_id'] = repo.repo_id
2928 date_current = repo_cs_cache.get('date', empty_date)
2936 _date_latest = parse_datetime(latest_repo_cs_cache['date'])
2929 current_timestamp = datetime_to_time(parse_datetime(date_latest))
2937
2930 if current_timestamp < datetime_to_time(parse_datetime(date_current)):
2938 latest_repo_cs_cache['updated_on'] = time.time()
2931 latest_repo_cs_cache = repo_cs_cache
2939 repo_group.changeset_cache = latest_repo_cs_cache
2932 if hasattr(obj, 'repo_id'):
2940 repo_group.updated_on = _date_latest
2933 latest_repo_cs_cache['source_repo_id'] = obj.repo_id
2941 Session().add(repo_group)
2934 else:
2942 Session().commit()
2935 latest_repo_cs_cache['source_repo_id'] = repo_cs_cache.get('source_repo_id')
2943
2936
2944 log.debug('updated repo group `%s` with new commit cache %s',
2937 _date_latest = parse_datetime(latest_repo_cs_cache.get('date') or empty_date)
2945 repo_group.group_name, latest_repo_cs_cache)
2938
2939 latest_repo_cs_cache['updated_on'] = time.time()
2940 self.changeset_cache = latest_repo_cs_cache
2941 self.updated_on = _date_latest
2942 Session().add(self)
2943 Session().commit()
2944
2945 log.debug('updated repo group `%s` with new commit cache %s, and last update_date: %s',
2946 self.group_name, latest_repo_cs_cache, _date_latest)
2946
2947
2947 def permissions(self, with_admins=True, with_owner=True,
2948 def permissions(self, with_admins=True, with_owner=True,
2948 expand_from_user_groups=False):
2949 expand_from_user_groups=False):
Comments 0
You need to be logged in to leave comments. Login now