fix(LFS): fixed LFSOidStorage, enbled LFS fetching on repo fetch, fixed tests and python3 transition related leftovers. Fixes: RCCE-8

vcsserver/echo_stub/remote_wsgi.py

@@ -44,4 +44,4 b' def _assert_valid_config(config):'
     config = config.copy()
 
     # This is what git needs from config at this stage
-    config.pop(b'git_update_server_info')
+    config.pop('git_update_server_info')

vcsserver/git_lfs/lib.py

@@ -118,7 +118,9 b' class LFSOidStore:'
     def __init__(self, oid, repo, store_location=None):
         self.oid = oid
         self.repo = repo
-        self.store_path = store_location or self.get_default_store()
+        defined_store_path = store_location or self.get_default_store()
+        self.store_suffix = f"/objects/{oid[:2]}/{oid[2:4]}"
+        self.store_path = f"{defined_store_path.rstrip('/')}{self.store_suffix}"
         self.tmp_oid_path = os.path.join(self.store_path, oid + '.tmp')
         self.oid_path = os.path.join(self.store_path, oid)
         self.fd = None

vcsserver/git_lfs/tests/test_lfs_app.py

@@ -22,6 +22,7 b' from webtest.app import TestApp as WebOb'
 from vcsserver.lib.rc_json import json
 from vcsserver.str_utils import safe_bytes
 from vcsserver.git_lfs.app import create_app
+from vcsserver.git_lfs.lib import LFSOidStore
 
 
 @pytest.fixture(scope='function')
@@ -118,7 +119,7 b' class TestLFSApplication:'
 
     def test_app_batch_api_download(self, git_lfs_app, http_auth):
         oid = '456'
-        oid_path = os.path.join(git_lfs_app._store, oid)
+        oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
         if not os.path.isdir(os.path.dirname(oid_path)):
             os.makedirs(os.path.dirname(oid_path))
         with open(oid_path, 'wb') as f:
@@ -209,7 +210,7 b' class TestLFSApplication:'
 
     def test_app_verify_api_size_mismatch(self, git_lfs_app):
         oid = 'existing'
-        oid_path = os.path.join(git_lfs_app._store, oid)
+        oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
         if not os.path.isdir(os.path.dirname(oid_path)):
             os.makedirs(os.path.dirname(oid_path))
         with open(oid_path, 'wb') as f:
@@ -225,7 +226,7 b' class TestLFSApplication:'
 
     def test_app_verify_api(self, git_lfs_app):
         oid = 'existing'
-        oid_path = os.path.join(git_lfs_app._store, oid)
+        oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
         if not os.path.isdir(os.path.dirname(oid_path)):
             os.makedirs(os.path.dirname(oid_path))
         with open(oid_path, 'wb') as f:
@@ -249,7 +250,7 b' class TestLFSApplication:'
 
     def test_app_download_api(self, git_lfs_app):
         oid = 'existing'
-        oid_path = os.path.join(git_lfs_app._store, oid)
+        oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
         if not os.path.isdir(os.path.dirname(oid_path)):
             os.makedirs(os.path.dirname(oid_path))
         with open(oid_path, 'wb') as f:
@@ -268,6 +269,6 b' class TestLFSApplication:'
         assert json.loads(response.text) == {'upload': 'ok'}
 
         # verify that we actually wrote that OID
-        oid_path = os.path.join(git_lfs_app._store, oid)
+        oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
         assert os.path.isfile(oid_path)
         assert 'CONTENT' == open(oid_path).read()

vcsserver/remote/git_remote.py

@@ -178,9 +178,13 b' class GitRemote(RemoteBase):'
         params = [
             '-c', 'core.askpass=""',
         ]
-        ssl_cert_dir = config.get('vcs_ssl_dir')
-        if ssl_cert_dir:
-            params.extend(['-c', f'http.sslCAinfo={ssl_cert_dir}'])
+        config_attrs = {
+            'vcs_ssl_dir': 'http.sslCAinfo={}',
+            'vcs_git_lfs_store_location': 'lfs.storage={}'
+        }
+        for key, param in config_attrs.items():
+            if value := config.get(key):
+                params.extend(['-c', param.format(value)])
         return params
 
     @reraise_safe_exceptions
@@ -757,7 +761,7 b' class GitRemote(RemoteBase):'
         return remote_refs
 
     @reraise_safe_exceptions
-    def sync_fetch(self, wire, url, refs=None, all_refs=False):
+    def sync_fetch(self, wire, url, refs=None, all_refs=False, **kwargs):
         self._factory.repo(wire)
         if refs and not isinstance(refs, (list, tuple)):
             refs = [refs]
@@ -807,6 +811,12 b' class GitRemote(RemoteBase):'
                     fail_on_stderr=False,
                     _copts=self._remote_conf(config),
                     extra_env={'GIT_TERMINAL_PROMPT': '0'})
+            if kwargs.get('sync_large_objects'):
+                self.run_git_command(
+                    wire, ['lfs', 'fetch', url, '--all'],
+                    fail_on_stderr=False,
+                    _copts=self._remote_conf(config),
+                )
 
         return remote_refs
 

vcsserver/scm_app.py

@@ -245,10 +245,10 b' class GitLFSHandler:'
 
 def create_git_lfs_wsgi_app(repo_path, repo_name, config):
     git_path = settings.GIT_EXECUTABLE
-    update_server_info = config.pop(b'git_update_server_info')
-    git_lfs_enabled = config.pop(b'git_lfs_enabled')
-    git_lfs_store_path = config.pop(b'git_lfs_store_path')
-    git_lfs_http_scheme = config.pop(b'git_lfs_http_scheme', 'http')
+    update_server_info = config.pop('git_update_server_info')
+    git_lfs_enabled = config.pop('git_lfs_enabled')
+    git_lfs_store_path = config.pop('git_lfs_store_path')
+    git_lfs_http_scheme = config.pop('git_lfs_http_scheme', 'http')
     app = GitLFSHandler(
         repo_path, repo_name, git_path, update_server_info, config)