Dulwich.io dulwich / 3345d77
Merge commit 'de801168537280a7665e3d57873483914733a24d' into tempmaster Jelmer Vernooij 5 months ago
6 changed file(s) with 165 addition(s) and 76 deletion(s). Raw diff Collapse all Expand all
00 0.19.11 UNRELEASED
11
22 IMPROVEMENTS
3
4 * Use fullname from gecos field, if available.
5 (Jelmer Vernooij)
6
7 * Support ``GIT_AUTHOR_NAME`` / ``GIT_AUTHOR_EMAIL``.
8 (Jelmer Vernooij)
39
410 * Add support for short ids in parse_commit. (Jelmer Vernooij)
511
612 * Add support for ``prune`` and ``prune_tags`` arguments
713 to ``porcelain.fetch``. (Jelmer Vernooij, #681)
14
15 BUG FIXES
16
17 * Fix handling of race conditions when new packs appear.
18 (Jelmer Vernooij)
819
920 0.19.10 2018-01-15
1021
6161 committer = b"Mark Mikofski <mark.mikofski@sunpowercorp.com>"
6262 test_tags = [b'v0.1a', b'v0.1']
6363 tag_test_data = {
64 test_tags[0]: [1484788003, b'0' * 40, None],
64 test_tags[0]: [1484788003, b'3' * 40, None],
6565 test_tags[1]: [1484788314, b'1' * 40, (1484788401, b'2' * 40)]
6666 }
6767
2323
2424 from io import BytesIO
2525 import errno
26 from itertools import chain
2726 import os
2827 import stat
2928 import sys
3029 import tempfile
31 import time
3230
3331 from dulwich.diff_tree import (
3432 tree_changes,
5452 Pack,
5553 PackData,
5654 PackInflater,
55 PackFileDisappeared,
5756 iter_sha1,
5857 pack_objects_to_data,
5958 write_pack_header,
309308 This does not check alternates.
310309 """
311310 for pack in self.packs:
312 if sha in pack:
313 return True
311 try:
312 if sha in pack:
313 return True
314 except PackFileDisappeared:
315 pass
314316 return False
315317
316318 def __contains__(self, sha):
325327 return True
326328 return False
327329
328 def _pack_cache_stale(self):
329 """Check whether the pack cache is stale."""
330 raise NotImplementedError(self._pack_cache_stale)
331
332 def _add_known_pack(self, base_name, pack):
330 def _add_cached_pack(self, base_name, pack):
333331 """Add a newly appeared pack to the cache by path.
334332
335333 """
339337 if prev_pack:
340338 prev_pack.close()
341339
342 def _flush_pack_cache(self):
340 def _clear_cached_packs(self):
343341 pack_cache = self._pack_cache
344342 self._pack_cache = {}
345343 while pack_cache:
346344 (name, pack) = pack_cache.popitem()
347345 pack.close()
348346
347 def _iter_cached_packs(self):
348 return self._pack_cache.values()
349
350 def _update_pack_cache(self):
351 raise NotImplementedError(self._update_pack_cache)
352
349353 def close(self):
350 self._flush_pack_cache()
354 self._clear_cached_packs()
351355
352356 @property
353357 def packs(self):
354358 """List with pack objects."""
355 if self._pack_cache is None or self._pack_cache_stale():
356 self._update_pack_cache()
357
358 return self._pack_cache.values()
359 return (
360 list(self._iter_cached_packs()) + list(self._update_pack_cache()))
359361
360362 def _iter_alternate_objects(self):
361363 """Iterate over the SHAs of all the objects in alternate stores."""
402404 old_packs = {p.name(): p for p in self.packs}
403405 for name, pack in old_packs.items():
404406 objects.update((obj, None) for obj in pack.iterobjects())
405 self._flush_pack_cache()
407 self._clear_cached_packs()
406408
407409 # The name of the consolidated pack might match the name of a
408410 # pre-existing pack. Take care not to remove the newly created
420422
421423 def __iter__(self):
422424 """Iterate over the SHAs that are present in this store."""
423 iterables = (list(self.packs) + [self._iter_loose_objects()] +
424 [self._iter_alternate_objects()])
425 return chain(*iterables)
425 self._update_pack_cache()
426 for pack in self._iter_cached_packs():
427 try:
428 for sha in pack:
429 yield sha
430 except PackFileDisappeared:
431 pass
432 for sha in self._iter_loose_objects():
433 yield sha
434 for sha in self._iter_alternate_objects():
435 yield sha
426436
427437 def contains_loose(self, sha):
428438 """Check if a particular object is present by SHA1 and is loose.
437447 :param name: sha for the object.
438448 :return: tuple with numeric type and object contents.
439449 """
450 if name == ZERO_SHA:
451 raise KeyError(name)
440452 if len(name) == 40:
441453 sha = hex_to_sha(name)
442454 hexsha = name
445457 hexsha = None
446458 else:
447459 raise AssertionError("Invalid object name %r" % name)
448 for pack in self.packs:
460 for pack in self._iter_cached_packs():
449461 try:
450462 return pack.get_raw(sha)
451 except KeyError:
463 except (KeyError, PackFileDisappeared):
452464 pass
453465 if hexsha is None:
454466 hexsha = sha_to_hex(name)
455467 ret = self._get_loose_object(hexsha)
456468 if ret is not None:
457469 return ret.type_num, ret.as_raw_string()
470 # Maybe something else has added a pack with the object
471 # in the mean time?
472 for pack in self._update_pack_cache():
473 try:
474 return pack.get_raw(sha)
475 except KeyError:
476 pass
458477 for alternate in self.alternates:
459478 try:
460479 return alternate.get_raw(hexsha)
485504 super(DiskObjectStore, self).__init__()
486505 self.path = path
487506 self.pack_dir = os.path.join(self.path, PACKDIR)
488 self._pack_cache_time = 0
489 self._pack_cache = {}
490507 self._alternates = None
491508
492509 def __repr__(self):
544561 self.alternates.append(DiskObjectStore(path))
545562
546563 def _update_pack_cache(self):
564 """Read and iterate over new pack files and cache them."""
547565 try:
548566 pack_dir_contents = os.listdir(self.pack_dir)
549567 except OSError as e:
550568 if e.errno == errno.ENOENT:
551 self._pack_cache_time = 0
552569 self.close()
553 return
570 return []
554571 raise
555 self._pack_cache_time = max(
556 os.stat(self.pack_dir).st_mtime, time.time())
557572 pack_files = set()
558573 for name in pack_dir_contents:
559574 if name.startswith("pack-") and name.endswith(".pack"):
565580 pack_files.add(pack_name)
566581
567582 # Open newly appeared pack files
583 new_packs = []
568584 for f in pack_files:
569585 if f not in self._pack_cache:
570 self._pack_cache[f] = Pack(os.path.join(self.pack_dir, f))
586 pack = Pack(os.path.join(self.pack_dir, f))
587 new_packs.append(pack)
588 self._pack_cache[f] = pack
571589 # Remove disappeared pack files
572590 for f in set(self._pack_cache) - pack_files:
573591 self._pack_cache.pop(f).close()
574
575 def _pack_cache_stale(self):
576 try:
577 return os.stat(self.pack_dir).st_mtime >= self._pack_cache_time
578 except OSError as e:
579 if e.errno == errno.ENOENT:
580 return True
581 raise
592 return new_packs
582593
583594 def _get_shafile_path(self, sha):
584595 # Check from object dir
675686 # Add the pack to the store and return it.
676687 final_pack = Pack(pack_base_name)
677688 final_pack.check_length_and_checksum()
678 self._add_known_pack(pack_base_name, final_pack)
689 self._add_cached_pack(pack_base_name, final_pack)
679690 return final_pack
680691
681692 def add_thin_pack(self, read_all, read_some):
713724 basename = self._get_pack_basepath(entries)
714725 with GitFile(basename+".idx", "wb") as f:
715726 write_pack_index_v2(f, entries, p.get_stored_checksum())
716 if self._pack_cache is None or self._pack_cache_stale():
717 self._update_pack_cache()
718 try:
719 return self._pack_cache[basename]
720 except KeyError:
721 pass
727 for pack in self.packs:
728 if pack._basename == basename:
729 return pack
722730 target_pack = basename + '.pack'
723731 if sys.platform == 'win32':
724732 # Windows might have the target pack file lingering. Attempt
730738 raise
731739 os.rename(path, target_pack)
732740 final_pack = Pack(basename)
733 self._add_known_pack(basename, final_pack)
741 self._add_cached_pack(basename, final_pack)
734742 return final_pack
735743
736744 def add_pack(self):
966974
967975 :param needle: SHA1 of the object to check for
968976 """
977 if needle == ZERO_SHA:
978 return False
969979 return needle in self.store
970980
971981 def __getitem__(self, key):
110110 return ret, crc32
111111
112112
113 class PackFileDisappeared(Exception):
114
115 def __init__(self, obj):
116 self.obj = obj
117
118
113119 class UnpackedObject(object):
114120 """Class encapsulating an object unpacked from a pack file.
115121
390396 """
391397 if len(sha) == 40:
392398 sha = hex_to_sha(sha)
393 return self._object_index(sha)
399 try:
400 return self._object_index(sha)
401 except ValueError:
402 closed = getattr(self._contents, 'closed', None)
403 if closed in (None, True):
404 raise PackFileDisappeared(self)
405 raise
394406
395407 def object_sha1(self, index):
396408 """Return the SHA1 corresponding to the index in the pack file.
119119 self.identity = identity
120120
121121
122 def _get_default_identity():
123 import getpass
124 import socket
125 username = getpass.getuser()
126 try:
127 import pwd
128 except ImportError:
129 fullname = None
130 else:
131 try:
132 gecos = pwd.getpwnam(username).pw_gecos
133 except KeyError:
134 fullname = None
135 else:
136 fullname = gecos.split(',')[0].decode('utf-8')
137 if not fullname:
138 fullname = username
139 email = os.environ.get('EMAIL')
140 if email is None:
141 email = "{}@{}".format(username, socket.gethostname())
142 return (fullname, email)
143
144
145 def get_user_identity(config, kind=None):
146 """Determine the identity to use for new commits.
147 """
148 if kind:
149 user = os.environ.get("GIT_" + kind + "_NAME")
150 if user is not None:
151 user = user.encode('utf-8')
152 email = os.environ.get("GIT_" + kind + "_EMAIL")
153 if email is not None:
154 email = email.encode('utf-8')
155 else:
156 user = None
157 email = None
158 if user is None:
159 try:
160 user = config.get(("user", ), "name")
161 except KeyError:
162 user = None
163 if email is None:
164 try:
165 email = config.get(("user", ), "email")
166 except KeyError:
167 email = None
168 default_user, default_email = _get_default_identity()
169 if user is None:
170 user = default_user.encode('utf-8')
171 if email is None:
172 email = default_email.encode('utf-8')
173 return (user + b" <" + email + b">")
174
175
122176 def check_user_identity(identity):
123177 """Verify that a user identity is formatted correctly.
124178
613667 else:
614668 raise ValueError(name)
615669
616 def _get_user_identity(self, config):
670 def _get_user_identity(self, config, kind=None):
617671 """Determine the identity to use for new commits.
618672 """
619 user = os.environ.get("GIT_COMMITTER_NAME")
620 email = os.environ.get("GIT_COMMITTER_EMAIL")
621 if user:
622 user = user.encode(sys.getdefaultencoding())
623 if email:
624 email = email.encode(sys.getdefaultencoding())
625 if user is None:
626 try:
627 user = config.get(("user", ), "name")
628 except KeyError:
629 user = None
630 if email is None:
631 try:
632 email = config.get(("user", ), "email")
633 except KeyError:
634 email = None
635 if user is None:
636 import getpass
637 user = getpass.getuser().encode(sys.getdefaultencoding())
638 if email is None:
639 import getpass
640 import socket
641 email = ("{}@{}".format(getpass.getuser(), socket.gethostname())
642 .encode(sys.getdefaultencoding()))
643 return (user + b" <" + email + b">")
673 # TODO(jelmer): Deprecate this function in favor of get_user_identity
674 return get_user_identity(config)
644675
645676 def _add_graftpoints(self, updated_graftpoints):
646677 """Add or modify graftpoints
714745 if merge_heads is None:
715746 merge_heads = self._read_heads('MERGE_HEADS')
716747 if committer is None:
717 committer = self._get_user_identity(config)
748 committer = get_user_identity(config, kind='COMMITTER')
718749 check_user_identity(committer)
719750 c.committer = committer
720751 if commit_timestamp is None:
726757 commit_timezone = 0
727758 c.commit_timezone = commit_timezone
728759 if author is None:
729 # FIXME: Support GIT_AUTHOR_NAME/GIT_AUTHOR_EMAIL environment
730 # variables
731 author = committer
760 author = get_user_identity(config, kind='AUTHOR')
732761 c.author = author
733762 check_user_identity(author)
734763 if author_timestamp is None:
862862 b"Jelmer <jelmer@apache.org>",
863863 r[commit_sha].committer)
864864
865 def overrideEnv(self, name, value):
866 def restore():
867 if oldval is not None:
868 os.environ[name] = oldval
869 else:
870 del os.environ[name]
871 oldval = os.environ.get(name)
872 os.environ[name] = value
873 self.addCleanup(restore)
874
875 def test_commit_config_identity_from_env(self):
876 # commit falls back to the users' identity if it wasn't specified
877 self.overrideEnv('GIT_COMMITTER_NAME', 'joe')
878 self.overrideEnv('GIT_COMMITTER_EMAIL', 'joe@example.com')
879 r = self._repo
880 c = r.get_config()
881 c.set((b"user", ), b"name", b"Jelmer")
882 c.set((b"user", ), b"email", b"jelmer@apache.org")
883 c.write_to_path()
884 commit_sha = r.do_commit(b'message')
885 self.assertEqual(
886 b"Jelmer <jelmer@apache.org>",
887 r[commit_sha].author)
888 self.assertEqual(
889 b"joe <joe@example.com>",
890 r[commit_sha].committer)
891
865892 def test_commit_fail_ref(self):
866893 r = self._repo
867894