Show More
@@ -1,248 +1,253 b'' | |||||
1 | # lfs - hash-preserving large file support using Git-LFS protocol |
|
1 | # lfs - hash-preserving large file support using Git-LFS protocol | |
2 | # |
|
2 | # | |
3 | # Copyright 2017 Facebook, Inc. |
|
3 | # Copyright 2017 Facebook, Inc. | |
4 | # |
|
4 | # | |
5 | # This software may be used and distributed according to the terms of the |
|
5 | # This software may be used and distributed according to the terms of the | |
6 | # GNU General Public License version 2 or any later version. |
|
6 | # GNU General Public License version 2 or any later version. | |
7 |
|
7 | |||
8 | """lfs - large file support (EXPERIMENTAL) |
|
8 | """lfs - large file support (EXPERIMENTAL) | |
9 |
|
9 | |||
10 | Configs:: |
|
10 | Configs:: | |
11 |
|
11 | |||
12 | [lfs] |
|
12 | [lfs] | |
13 | # Remote endpoint. Multiple protocols are supported: |
|
13 | # Remote endpoint. Multiple protocols are supported: | |
14 | # - http(s)://user:pass@example.com/path |
|
14 | # - http(s)://user:pass@example.com/path | |
15 | # git-lfs endpoint |
|
15 | # git-lfs endpoint | |
16 | # - file:///tmp/path |
|
16 | # - file:///tmp/path | |
17 | # local filesystem, usually for testing |
|
17 | # local filesystem, usually for testing | |
18 | # if unset, lfs will prompt setting this when it must use this value. |
|
18 | # if unset, lfs will prompt setting this when it must use this value. | |
19 | # (default: unset) |
|
19 | # (default: unset) | |
20 | url = https://example.com/lfs |
|
20 | url = https://example.com/lfs | |
21 |
|
21 | |||
22 | # Which files to track in LFS. Path tests are "**.extname" for file |
|
22 | # Which files to track in LFS. Path tests are "**.extname" for file | |
23 | # extensions, and "path:under/some/directory" for path prefix. Both |
|
23 | # extensions, and "path:under/some/directory" for path prefix. Both | |
24 | # are relative to the repository root, and the latter must be quoted. |
|
24 | # are relative to the repository root, and the latter must be quoted. | |
25 | # File size can be tested with the "size()" fileset, and tests can be |
|
25 | # File size can be tested with the "size()" fileset, and tests can be | |
26 | # joined with fileset operators. (See "hg help filesets.operators".) |
|
26 | # joined with fileset operators. (See "hg help filesets.operators".) | |
27 | # |
|
27 | # | |
28 | # Some examples: |
|
28 | # Some examples: | |
29 | # - all() # everything |
|
29 | # - all() # everything | |
30 | # - none() # nothing |
|
30 | # - none() # nothing | |
31 | # - size(">20MB") # larger than 20MB |
|
31 | # - size(">20MB") # larger than 20MB | |
32 | # - !**.txt # anything not a *.txt file |
|
32 | # - !**.txt # anything not a *.txt file | |
33 | # - **.zip | **.tar.gz | **.7z # some types of compressed files |
|
33 | # - **.zip | **.tar.gz | **.7z # some types of compressed files | |
34 | # - "path:bin" # files under "bin" in the project root |
|
34 | # - "path:bin" # files under "bin" in the project root | |
35 | # - (**.php & size(">2MB")) | (**.js & size(">5MB")) | **.tar.gz |
|
35 | # - (**.php & size(">2MB")) | (**.js & size(">5MB")) | **.tar.gz | |
36 | # | ("path:bin" & !"path:/bin/README") | size(">1GB") |
|
36 | # | ("path:bin" & !"path:/bin/README") | size(">1GB") | |
37 | # (default: none()) |
|
37 | # (default: none()) | |
38 | track = size(">10M") |
|
38 | track = size(">10M") | |
39 |
|
39 | |||
40 | # how many times to retry before giving up on transferring an object |
|
40 | # how many times to retry before giving up on transferring an object | |
41 | retry = 5 |
|
41 | retry = 5 | |
42 |
|
42 | |||
43 | # the local directory to store lfs files for sharing across local clones. |
|
43 | # the local directory to store lfs files for sharing across local clones. | |
44 | # If not set, the cache is located in an OS specific cache location. |
|
44 | # If not set, the cache is located in an OS specific cache location. | |
45 | usercache = /path/to/global/cache |
|
45 | usercache = /path/to/global/cache | |
46 | """ |
|
46 | """ | |
47 |
|
47 | |||
48 | from __future__ import absolute_import |
|
48 | from __future__ import absolute_import | |
49 |
|
49 | |||
50 | from mercurial.i18n import _ |
|
50 | from mercurial.i18n import _ | |
51 |
|
51 | |||
52 | from mercurial import ( |
|
52 | from mercurial import ( | |
53 | bundle2, |
|
53 | bundle2, | |
54 | changegroup, |
|
54 | changegroup, | |
55 | cmdutil, |
|
55 | cmdutil, | |
56 | context, |
|
56 | context, | |
57 | exchange, |
|
57 | exchange, | |
58 | extensions, |
|
58 | extensions, | |
59 | filelog, |
|
59 | filelog, | |
60 | fileset, |
|
60 | fileset, | |
61 | hg, |
|
61 | hg, | |
62 | localrepo, |
|
62 | localrepo, | |
63 | minifileset, |
|
63 | minifileset, | |
64 | node, |
|
64 | node, | |
65 | pycompat, |
|
65 | pycompat, | |
66 | registrar, |
|
66 | registrar, | |
67 | revlog, |
|
67 | revlog, | |
68 | scmutil, |
|
68 | scmutil, | |
69 | templatekw, |
|
69 | templatekw, | |
70 | upgrade, |
|
70 | upgrade, | |
71 | vfs as vfsmod, |
|
71 | vfs as vfsmod, | |
72 | wireproto, |
|
72 | wireproto, | |
73 | ) |
|
73 | ) | |
74 |
|
74 | |||
75 | from . import ( |
|
75 | from . import ( | |
76 | blobstore, |
|
76 | blobstore, | |
77 | wrapper, |
|
77 | wrapper, | |
78 | ) |
|
78 | ) | |
79 |
|
79 | |||
80 | # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for |
|
80 | # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for | |
81 | # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should |
|
81 | # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should | |
82 | # be specifying the version(s) of Mercurial they are tested with, or |
|
82 | # be specifying the version(s) of Mercurial they are tested with, or | |
83 | # leave the attribute unspecified. |
|
83 | # leave the attribute unspecified. | |
84 | testedwith = 'ships-with-hg-core' |
|
84 | testedwith = 'ships-with-hg-core' | |
85 |
|
85 | |||
86 | configtable = {} |
|
86 | configtable = {} | |
87 | configitem = registrar.configitem(configtable) |
|
87 | configitem = registrar.configitem(configtable) | |
88 |
|
88 | |||
89 | configitem('experimental', 'lfs.user-agent', |
|
89 | configitem('experimental', 'lfs.user-agent', | |
90 | default=None, |
|
90 | default=None, | |
91 | ) |
|
91 | ) | |
92 |
|
92 | |||
93 | configitem('lfs', 'url', |
|
93 | configitem('lfs', 'url', | |
94 | default=None, |
|
94 | default=None, | |
95 | ) |
|
95 | ) | |
96 | configitem('lfs', 'usercache', |
|
96 | configitem('lfs', 'usercache', | |
97 | default=None, |
|
97 | default=None, | |
98 | ) |
|
98 | ) | |
99 | # Deprecated |
|
99 | # Deprecated | |
100 | configitem('lfs', 'threshold', |
|
100 | configitem('lfs', 'threshold', | |
101 | default=None, |
|
101 | default=None, | |
102 | ) |
|
102 | ) | |
103 | configitem('lfs', 'track', |
|
103 | configitem('lfs', 'track', | |
104 | default='none()', |
|
104 | default='none()', | |
105 | ) |
|
105 | ) | |
106 | configitem('lfs', 'retry', |
|
106 | configitem('lfs', 'retry', | |
107 | default=5, |
|
107 | default=5, | |
108 | ) |
|
108 | ) | |
109 |
|
109 | |||
110 | cmdtable = {} |
|
110 | cmdtable = {} | |
111 | command = registrar.command(cmdtable) |
|
111 | command = registrar.command(cmdtable) | |
112 |
|
112 | |||
113 | templatekeyword = registrar.templatekeyword() |
|
113 | templatekeyword = registrar.templatekeyword() | |
114 |
|
114 | |||
115 | def featuresetup(ui, supported): |
|
115 | def featuresetup(ui, supported): | |
116 | # don't die on seeing a repo with the lfs requirement |
|
116 | # don't die on seeing a repo with the lfs requirement | |
117 | supported |= {'lfs'} |
|
117 | supported |= {'lfs'} | |
118 |
|
118 | |||
119 | def uisetup(ui): |
|
119 | def uisetup(ui): | |
120 | localrepo.localrepository.featuresetupfuncs.add(featuresetup) |
|
120 | localrepo.localrepository.featuresetupfuncs.add(featuresetup) | |
121 |
|
121 | |||
122 | def reposetup(ui, repo): |
|
122 | def reposetup(ui, repo): | |
123 | # Nothing to do with a remote repo |
|
123 | # Nothing to do with a remote repo | |
124 | if not repo.local(): |
|
124 | if not repo.local(): | |
125 | return |
|
125 | return | |
126 |
|
126 | |||
127 | trackspec = repo.ui.config('lfs', 'track') |
|
127 | repo.svfs.options['lfstrack'] = _trackedmatcher(repo) | |
128 |
|
||||
129 | # deprecated config: lfs.threshold |
|
|||
130 | threshold = repo.ui.configbytes('lfs', 'threshold') |
|
|||
131 | if threshold: |
|
|||
132 | fileset.parse(trackspec) # make sure syntax errors are confined |
|
|||
133 | trackspec = "(%s) | size('>%d')" % (trackspec, threshold) |
|
|||
134 |
|
||||
135 | repo.svfs.options['lfstrack'] = minifileset.compile(trackspec) |
|
|||
136 | repo.svfs.lfslocalblobstore = blobstore.local(repo) |
|
128 | repo.svfs.lfslocalblobstore = blobstore.local(repo) | |
137 | repo.svfs.lfsremoteblobstore = blobstore.remote(repo) |
|
129 | repo.svfs.lfsremoteblobstore = blobstore.remote(repo) | |
138 |
|
130 | |||
139 | # Push hook |
|
131 | # Push hook | |
140 | repo.prepushoutgoinghooks.add('lfs', wrapper.prepush) |
|
132 | repo.prepushoutgoinghooks.add('lfs', wrapper.prepush) | |
141 |
|
133 | |||
142 | if 'lfs' not in repo.requirements: |
|
134 | if 'lfs' not in repo.requirements: | |
143 | def checkrequireslfs(ui, repo, **kwargs): |
|
135 | def checkrequireslfs(ui, repo, **kwargs): | |
144 | if 'lfs' not in repo.requirements: |
|
136 | if 'lfs' not in repo.requirements: | |
145 | last = kwargs.get('node_last') |
|
137 | last = kwargs.get('node_last') | |
146 | _bin = node.bin |
|
138 | _bin = node.bin | |
147 | if last: |
|
139 | if last: | |
148 | s = repo.set('%n:%n', _bin(kwargs['node']), _bin(last)) |
|
140 | s = repo.set('%n:%n', _bin(kwargs['node']), _bin(last)) | |
149 | else: |
|
141 | else: | |
150 | s = repo.set('%n', _bin(kwargs['node'])) |
|
142 | s = repo.set('%n', _bin(kwargs['node'])) | |
151 | for ctx in s: |
|
143 | for ctx in s: | |
152 | # TODO: is there a way to just walk the files in the commit? |
|
144 | # TODO: is there a way to just walk the files in the commit? | |
153 | if any(ctx[f].islfs() for f in ctx.files() if f in ctx): |
|
145 | if any(ctx[f].islfs() for f in ctx.files() if f in ctx): | |
154 | repo.requirements.add('lfs') |
|
146 | repo.requirements.add('lfs') | |
155 | repo._writerequirements() |
|
147 | repo._writerequirements() | |
156 | break |
|
148 | break | |
157 |
|
149 | |||
158 | ui.setconfig('hooks', 'commit.lfs', checkrequireslfs, 'lfs') |
|
150 | ui.setconfig('hooks', 'commit.lfs', checkrequireslfs, 'lfs') | |
159 | ui.setconfig('hooks', 'pretxnchangegroup.lfs', checkrequireslfs, 'lfs') |
|
151 | ui.setconfig('hooks', 'pretxnchangegroup.lfs', checkrequireslfs, 'lfs') | |
160 |
|
152 | |||
|
153 | def _trackedmatcher(repo): | |||
|
154 | """Return a function (path, size) -> bool indicating whether or not to | |||
|
155 | track a given file with lfs.""" | |||
|
156 | trackspec = repo.ui.config('lfs', 'track') | |||
|
157 | ||||
|
158 | # deprecated config: lfs.threshold | |||
|
159 | threshold = repo.ui.configbytes('lfs', 'threshold') | |||
|
160 | if threshold: | |||
|
161 | fileset.parse(trackspec) # make sure syntax errors are confined | |||
|
162 | trackspec = "(%s) | size('>%d')" % (trackspec, threshold) | |||
|
163 | ||||
|
164 | return minifileset.compile(trackspec) | |||
|
165 | ||||
161 | def wrapfilelog(filelog): |
|
166 | def wrapfilelog(filelog): | |
162 | wrapfunction = extensions.wrapfunction |
|
167 | wrapfunction = extensions.wrapfunction | |
163 |
|
168 | |||
164 | wrapfunction(filelog, 'addrevision', wrapper.filelogaddrevision) |
|
169 | wrapfunction(filelog, 'addrevision', wrapper.filelogaddrevision) | |
165 | wrapfunction(filelog, 'renamed', wrapper.filelogrenamed) |
|
170 | wrapfunction(filelog, 'renamed', wrapper.filelogrenamed) | |
166 | wrapfunction(filelog, 'size', wrapper.filelogsize) |
|
171 | wrapfunction(filelog, 'size', wrapper.filelogsize) | |
167 |
|
172 | |||
168 | def extsetup(ui): |
|
173 | def extsetup(ui): | |
169 | wrapfilelog(filelog.filelog) |
|
174 | wrapfilelog(filelog.filelog) | |
170 |
|
175 | |||
171 | wrapfunction = extensions.wrapfunction |
|
176 | wrapfunction = extensions.wrapfunction | |
172 |
|
177 | |||
173 | wrapfunction(cmdutil, '_updatecatformatter', wrapper._updatecatformatter) |
|
178 | wrapfunction(cmdutil, '_updatecatformatter', wrapper._updatecatformatter) | |
174 | wrapfunction(scmutil, 'wrapconvertsink', wrapper.convertsink) |
|
179 | wrapfunction(scmutil, 'wrapconvertsink', wrapper.convertsink) | |
175 |
|
180 | |||
176 | wrapfunction(upgrade, '_finishdatamigration', |
|
181 | wrapfunction(upgrade, '_finishdatamigration', | |
177 | wrapper.upgradefinishdatamigration) |
|
182 | wrapper.upgradefinishdatamigration) | |
178 |
|
183 | |||
179 | wrapfunction(upgrade, 'preservedrequirements', |
|
184 | wrapfunction(upgrade, 'preservedrequirements', | |
180 | wrapper.upgraderequirements) |
|
185 | wrapper.upgraderequirements) | |
181 |
|
186 | |||
182 | wrapfunction(upgrade, 'supporteddestrequirements', |
|
187 | wrapfunction(upgrade, 'supporteddestrequirements', | |
183 | wrapper.upgraderequirements) |
|
188 | wrapper.upgraderequirements) | |
184 |
|
189 | |||
185 | wrapfunction(changegroup, |
|
190 | wrapfunction(changegroup, | |
186 | 'supportedoutgoingversions', |
|
191 | 'supportedoutgoingversions', | |
187 | wrapper.supportedoutgoingversions) |
|
192 | wrapper.supportedoutgoingversions) | |
188 | wrapfunction(changegroup, |
|
193 | wrapfunction(changegroup, | |
189 | 'allsupportedversions', |
|
194 | 'allsupportedversions', | |
190 | wrapper.allsupportedversions) |
|
195 | wrapper.allsupportedversions) | |
191 |
|
196 | |||
192 | wrapfunction(exchange, 'push', wrapper.push) |
|
197 | wrapfunction(exchange, 'push', wrapper.push) | |
193 | wrapfunction(wireproto, '_capabilities', wrapper._capabilities) |
|
198 | wrapfunction(wireproto, '_capabilities', wrapper._capabilities) | |
194 |
|
199 | |||
195 | wrapfunction(context.basefilectx, 'cmp', wrapper.filectxcmp) |
|
200 | wrapfunction(context.basefilectx, 'cmp', wrapper.filectxcmp) | |
196 | wrapfunction(context.basefilectx, 'isbinary', wrapper.filectxisbinary) |
|
201 | wrapfunction(context.basefilectx, 'isbinary', wrapper.filectxisbinary) | |
197 | context.basefilectx.islfs = wrapper.filectxislfs |
|
202 | context.basefilectx.islfs = wrapper.filectxislfs | |
198 |
|
203 | |||
199 | revlog.addflagprocessor( |
|
204 | revlog.addflagprocessor( | |
200 | revlog.REVIDX_EXTSTORED, |
|
205 | revlog.REVIDX_EXTSTORED, | |
201 | ( |
|
206 | ( | |
202 | wrapper.readfromstore, |
|
207 | wrapper.readfromstore, | |
203 | wrapper.writetostore, |
|
208 | wrapper.writetostore, | |
204 | wrapper.bypasscheckhash, |
|
209 | wrapper.bypasscheckhash, | |
205 | ), |
|
210 | ), | |
206 | ) |
|
211 | ) | |
207 |
|
212 | |||
208 | wrapfunction(hg, 'clone', wrapper.hgclone) |
|
213 | wrapfunction(hg, 'clone', wrapper.hgclone) | |
209 | wrapfunction(hg, 'postshare', wrapper.hgpostshare) |
|
214 | wrapfunction(hg, 'postshare', wrapper.hgpostshare) | |
210 |
|
215 | |||
211 | # Make bundle choose changegroup3 instead of changegroup2. This affects |
|
216 | # Make bundle choose changegroup3 instead of changegroup2. This affects | |
212 | # "hg bundle" command. Note: it does not cover all bundle formats like |
|
217 | # "hg bundle" command. Note: it does not cover all bundle formats like | |
213 | # "packed1". Using "packed1" with lfs will likely cause trouble. |
|
218 | # "packed1". Using "packed1" with lfs will likely cause trouble. | |
214 | names = [k for k, v in exchange._bundlespeccgversions.items() if v == '02'] |
|
219 | names = [k for k, v in exchange._bundlespeccgversions.items() if v == '02'] | |
215 | for k in names: |
|
220 | for k in names: | |
216 | exchange._bundlespeccgversions[k] = '03' |
|
221 | exchange._bundlespeccgversions[k] = '03' | |
217 |
|
222 | |||
218 | # bundlerepo uses "vfsmod.readonlyvfs(othervfs)", we need to make sure lfs |
|
223 | # bundlerepo uses "vfsmod.readonlyvfs(othervfs)", we need to make sure lfs | |
219 | # options and blob stores are passed from othervfs to the new readonlyvfs. |
|
224 | # options and blob stores are passed from othervfs to the new readonlyvfs. | |
220 | wrapfunction(vfsmod.readonlyvfs, '__init__', wrapper.vfsinit) |
|
225 | wrapfunction(vfsmod.readonlyvfs, '__init__', wrapper.vfsinit) | |
221 |
|
226 | |||
222 | # when writing a bundle via "hg bundle" command, upload related LFS blobs |
|
227 | # when writing a bundle via "hg bundle" command, upload related LFS blobs | |
223 | wrapfunction(bundle2, 'writenewbundle', wrapper.writenewbundle) |
|
228 | wrapfunction(bundle2, 'writenewbundle', wrapper.writenewbundle) | |
224 |
|
229 | |||
225 | @templatekeyword('lfs_files') |
|
230 | @templatekeyword('lfs_files') | |
226 | def lfsfiles(repo, ctx, **args): |
|
231 | def lfsfiles(repo, ctx, **args): | |
227 | """List of strings. LFS files added or modified by the changeset.""" |
|
232 | """List of strings. LFS files added or modified by the changeset.""" | |
228 | args = pycompat.byteskwargs(args) |
|
233 | args = pycompat.byteskwargs(args) | |
229 |
|
234 | |||
230 | pointers = wrapper.pointersfromctx(ctx) # {path: pointer} |
|
235 | pointers = wrapper.pointersfromctx(ctx) # {path: pointer} | |
231 | files = sorted(pointers.keys()) |
|
236 | files = sorted(pointers.keys()) | |
232 |
|
237 | |||
233 | makemap = lambda v: { |
|
238 | makemap = lambda v: { | |
234 | 'file': v, |
|
239 | 'file': v, | |
235 | 'oid': pointers[v].oid(), |
|
240 | 'oid': pointers[v].oid(), | |
236 | } |
|
241 | } | |
237 |
|
242 | |||
238 | # TODO: make the separator ', '? |
|
243 | # TODO: make the separator ', '? | |
239 | f = templatekw._showlist('lfs_file', files, args) |
|
244 | f = templatekw._showlist('lfs_file', files, args) | |
240 | return templatekw._hybrid(f, files, makemap, pycompat.identity) |
|
245 | return templatekw._hybrid(f, files, makemap, pycompat.identity) | |
241 |
|
246 | |||
242 | @command('debuglfsupload', |
|
247 | @command('debuglfsupload', | |
243 | [('r', 'rev', [], _('upload large files introduced by REV'))]) |
|
248 | [('r', 'rev', [], _('upload large files introduced by REV'))]) | |
244 | def debuglfsupload(ui, repo, **opts): |
|
249 | def debuglfsupload(ui, repo, **opts): | |
245 | """upload lfs blobs added by the working copy parent or given revisions""" |
|
250 | """upload lfs blobs added by the working copy parent or given revisions""" | |
246 | revs = opts.get('rev', []) |
|
251 | revs = opts.get('rev', []) | |
247 | pointers = wrapper.extractpointers(repo, scmutil.revrange(repo, revs)) |
|
252 | pointers = wrapper.extractpointers(repo, scmutil.revrange(repo, revs)) | |
248 | wrapper.uploadblobs(repo, pointers) |
|
253 | wrapper.uploadblobs(repo, pointers) |
General Comments 0
You need to be logged in to leave comments.
Login now