Dulwich.io dulwich / 8ce0bc9
Merge durin42/missing-object-finder. This fixes #211. Jelmer Vernooij 4 years ago
7 changed file(s) with 110 addition(s) and 30 deletion(s). Raw diff Collapse all Expand all
1818
1919 * Resolve delta refs when pulling into a MemoryRepo.
2020 (Max Shawabkeh, #256)
21
22 * Fix handling of tags of non-commits in missing object finder.
23 (Augie Fackler, #211)
2124
2225 IMPROVEMENTS
2326
910910
911911
912912 def _split_commits_and_tags(obj_store, lst, ignore_unknown=False):
913 """Split object id list into two list with commit SHA1s and tag SHA1s.
913 """Split object id list into three lists with commit, tag, and other SHAs.
914914
915915 Commits referenced by tags are included into commits
916916 list as well. Only SHA1s known in this repository will get
921921 :param lst: Collection of commit and tag SHAs
922922 :param ignore_unknown: True to skip SHA1 missing in the repository
923923 silently.
924 :return: A tuple of (commits, tags) SHA1s
924 :return: A tuple of (commits, tags, others) SHA1s
925925 """
926926 commits = set()
927927 tags = set()
928 others = set()
928929 for e in lst:
929930 try:
930931 o = obj_store[e]
936937 commits.add(e)
937938 elif isinstance(o, Tag):
938939 tags.add(e)
939 commits.add(o.object[1])
940 tagged = o.object[1]
941 c, t, o = _split_commits_and_tags(
942 obj_store, [tagged], ignore_unknown=ignore_unknown)
943 commits |= c
944 tags |= t
945 others |= o
940946 else:
941 raise KeyError('Not a commit or a tag: %s' % e)
942 return (commits, tags)
947 others.add(e)
948 return (commits, tags, others)
943949
944950
945951 class MissingObjectFinder(object):
965971 # and such SHAs would get filtered out by _split_commits_and_tags,
966972 # wants shall list only known SHAs, and otherwise
967973 # _split_commits_and_tags fails with KeyError
968 have_commits, have_tags = (
974 have_commits, have_tags, have_others = (
969975 _split_commits_and_tags(object_store, haves, True))
970 want_commits, want_tags = (
976 want_commits, want_tags, want_others = (
971977 _split_commits_and_tags(object_store, wants, False))
972978 # all_ancestors is a set of commits that shall not be sent
973979 # (complete repository up to 'haves')
992998 self.sha_done.add(t)
993999
9941000 missing_tags = want_tags.difference(have_tags)
995 # in fact, what we 'want' is commits and tags
1001 missing_others = want_others.difference(have_others)
1002 # in fact, what we 'want' is commits, tags, and others
9961003 # we've found missing
9971004 wants = missing_commits.union(missing_tags)
1005 wants = wants.union(missing_others)
9981006
9991007 self.objects_to_send = set([(w, None, False) for w in wants])
10001008
2424 from dulwich.tests import TestCase
2525 from dulwich.tests.utils import (
2626 make_object,
27 make_tag,
2728 build_commit_graph,
2829 skipIfPY3,
2930 )
9091 self.assertMissingMatch([self.cmt(1).id], [self.cmt(3).id],
9192 self.missing_1_3)
9293
93 def test_bogus_haves_failure(self):
94 """Ensure non-existent SHA in haves are not tolerated"""
94 def test_bogus_haves(self):
95 """Ensure non-existent SHA in haves are tolerated"""
9596 bogus_sha = self.cmt(2).id[::-1]
9697 haves = [self.cmt(1).id, bogus_sha]
9798 wants = [self.cmt(3).id]
98 self.assertRaises(KeyError, self.store.find_missing_objects,
99 self.store, haves, wants)
99 self.assertMissingMatch(haves, wants, self.missing_1_3)
100100
101101 def test_bogus_wants_failure(self):
102102 """Ensure non-existent SHA in wants are not tolerated"""
104104 haves = [self.cmt(1).id]
105105 wants = [self.cmt(3).id, bogus_sha]
106106 self.assertRaises(KeyError, self.store.find_missing_objects,
107 self.store, haves, wants)
107 haves, wants)
108108
109109 def test_no_changes(self):
110110 self.assertMissingMatch([self.cmt(3).id], [self.cmt(3).id], [])
194194 self.cmt(7).id, self.cmt(6).id, self.cmt(4).id,
195195 self.cmt(7).tree, self.cmt(6).tree, self.cmt(4).tree,
196196 self.f1_4_id])
197
198
199 class MOFTagsTest(MissingObjectFinderTest):
200 def setUp(self):
201 super(MOFTagsTest, self).setUp()
202 f1_1 = make_object(Blob, data='f1')
203 commit_spec = [[1]]
204 trees = {1: [('f1', f1_1)]}
205 self.commits = build_commit_graph(self.store, commit_spec, trees)
206
207 self._normal_tag = make_tag(self.cmt(1))
208 self.store.add_object(self._normal_tag)
209
210 self._tag_of_tag = make_tag(self._normal_tag)
211 self.store.add_object(self._tag_of_tag)
212
213 self._tag_of_tree = make_tag(self.store[self.cmt(1).tree])
214 self.store.add_object(self._tag_of_tree)
215
216 self._tag_of_blob = make_tag(f1_1)
217 self.store.add_object(self._tag_of_blob)
218
219 self._tag_of_tag_of_blob = make_tag(self._tag_of_blob)
220 self.store.add_object(self._tag_of_tag_of_blob)
221
222 self.f1_1_id = f1_1.id
223
224 def test_tagged_commit(self):
225 # The user already has the tagged commit, all they want is the tag,
226 # so send them only the tag object.
227 self.assertMissingMatch([self.cmt(1).id], [self._normal_tag.id],
228 [self._normal_tag.id])
229
230 # The remaining cases are unusual, but do happen in the wild.
231 def test_tagged_tag(self):
232 # User already has tagged tag, send only tag of tag
233 self.assertMissingMatch([self._normal_tag.id], [self._tag_of_tag.id],
234 [self._tag_of_tag.id])
235 # User needs both tags, but already has commit
236 self.assertMissingMatch([self.cmt(1).id], [self._tag_of_tag.id],
237 [self._normal_tag.id, self._tag_of_tag.id])
238
239 def test_tagged_tree(self):
240 self.assertMissingMatch(
241 [], [self._tag_of_tree.id],
242 [self._tag_of_tree.id, self.cmt(1).tree, self.f1_1_id])
243
244 def test_tagged_blob(self):
245 self.assertMissingMatch([], [self._tag_of_blob.id],
246 [self._tag_of_blob.id, self.f1_1_id])
247
248 def test_tagged_tagged_blob(self):
249 self.assertMissingMatch([], [self._tag_of_tag_of_blob.id],
250 [self._tag_of_tag_of_blob.id,
251 self._tag_of_blob.id, self.f1_1_id])
5252 )
5353 from dulwich.tests.utils import (
5454 make_object,
55 make_tag,
5556 build_pack,
5657 skipIfPY3,
5758 )
8384 self.store.add_objects([])
8485
8586 def test_add_commit(self):
86 # TODO: Argh, no way to construct Git commit objects without
87 # TODO: Argh, no way to construct Git commit objects without
8788 # access to a serialized form.
8889 self.store.add_objects([])
8990
168169 self.assertEqual(expected, list(actual))
169170
170171 def make_tag(self, name, obj):
171 tag = make_object(Tag, name=name, message='',
172 tag_time=12345, tag_timezone=0,
173 tagger='Test Tagger <test@example.com>',
174 object=(object_class(obj.type_name), obj.id))
172 tag = make_tag(obj, name=name)
175173 self.store.add_object(tag)
176174 return tag
177175
399397
400398 def test_lookup_not_tree(self):
401399 self.assertRaises(NotTreeError, tree_lookup_path, self.get_object, self.tree_id, 'ad/b/j')
402
403 # TODO: MissingObjectFinderTests
404400
405401 @skipIfPY3
406402 class ObjectStoreGraphWalkerTests(TestCase):
5858 from dulwich.tests.utils import (
5959 make_commit,
6060 make_object,
61 make_tag,
6162 skipIfPY3,
6263 )
6364 from dulwich.protocol import (
279280
280281 def test_tag(self):
281282 c1, c2 = self.make_linear_commits(2)
282 tag = make_object(Tag, name='tag', message='',
283 tagger='Tagger <test@example.com>',
284 tag_time=12345, tag_timezone=0,
285 object=(Commit, c2.id))
283 tag = make_tag(c2, name='tag')
286284 self._store.add_object(tag)
287285
288286 self.assertEqual((set([c1.id]), set([c2.id])),
6060
6161 from dulwich.tests.utils import (
6262 make_object,
63 make_tag,
6364 skipIfPY3,
6465 )
6566
228229 blob2 = make_object(Blob, data='2')
229230 blob3 = make_object(Blob, data='3')
230231
231 tag1 = make_object(Tag, name='tag-tag',
232 tagger='Test <test@example.com>',
233 tag_time=12345,
234 tag_timezone=0,
235 message='message',
236 object=(Blob, blob2.id))
232 tag1 = make_tag(blob2, name='tag-tag')
237233
238234 objects = [blob1, blob2, blob3, tag1]
239235 refs = {
3535 from dulwich.objects import (
3636 FixedSha,
3737 Commit,
38 Tag,
39 object_class,
3840 )
3941 from dulwich.pack import (
4042 OFS_DELTA,
99101 __dict__ instead of __slots__.
100102 """
101103 pass
104 TestObject.__name__ = 'TestObject_' + cls.__name__
102105
103106 obj = TestObject()
104107 for name, value in attrs.items():
129132 'tree': b'0' * 40}
130133 all_attrs.update(attrs)
131134 return make_object(Commit, **all_attrs)
135
136
137 def make_tag(target, **attrs):
138 """Make a Tag object with a default set of values.
139
140 :param target: object to be tagged (Commit, Blob, Tree, etc)
141 :param attrs: dict of attributes to overwrite from the default values.
142 :return: A newly initialized Tag object.
143 """
144 target_id = target.id
145 target_type = object_class(target.type_name)
146 default_time = int(time.mktime(datetime.datetime(2010, 1, 1).timetuple()))
147 all_attrs = {'tagger': 'Test Author <test@nodomain.com>',
148 'tag_time': default_time,
149 'tag_timezone': 0,
150 'message': 'Test message.',
151 'object': (target_type, target_id),
152 'name': 'Test Tag',
153 }
154 all_attrs.update(attrs)
155 return make_object(Tag, **all_attrs)
132156
133157
134158 def functest_builder(method, func):