From ad491366de6c883cd04539cb86db31049201dfbd Mon Sep 17 00:00:00 2001 From: Nguyễn Thái Ngọc Duy Date: Thu, 5 Dec 2013 20:02:32 +0700 Subject: make the sender advertise shallow commits to the receiver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If either receive-pack or upload-pack is called on a shallow repository, shallow commits (*) will be sent after the ref advertisement (but before the packet flush), so that the receiver has the full "shape" of the sender's commit graph. This will be needed for the receiver to update its .git/shallow if necessary. This breaks the protocol for all clients trying to push to a shallow repo, or fetch from one. Which is basically the same end result as today's "is_repository_shallow() && die()" in receive-pack and upload-pack. New clients will be made aware of shallow upstream and can make use of this information. The sender must send all shallow commits that are sent in the following pack. It may send more shallow commits than necessary. upload-pack for example may choose to advertise no shallow commits if it knows in advance that the pack it's going to send contains no shallow commits. But upload-pack is the server, so we choose the cheaper way, send full .git/shallow and let the client deal with it. Smart HTTP is not affected by this patch. Shallow support on smart-http comes later separately. (*) A shallow commit is a commit that terminates the revision walker. It is usually put in .git/shallow in order to keep the revision walker from going out of bound because there is no guarantee that objects behind this commit is available. Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- commit.h | 1 + 1 file changed, 1 insertion(+) (limited to 'commit.h') diff --git a/commit.h b/commit.h index bd841f4d0c..a8795263b6 100644 --- a/commit.h +++ b/commit.h @@ -205,6 +205,7 @@ extern int write_shallow_commits(struct strbuf *out, int use_pack_protocol); extern void setup_alternate_shallow(struct lock_file *shallow_lock, const char **alternate_shallow_file); extern char *setup_temporary_shallow(void); +extern void advertise_shallow_grafts(int); int is_descendant_of(struct commit *, struct commit_list *); int in_merge_bases(struct commit *, struct commit *); -- cgit v1.2.3 From 1a30f5a2f2a3d5d9b3cf6e126ac19deb40324515 Mon Sep 17 00:00:00 2001 From: Nguyễn Thái Ngọc Duy Date: Thu, 5 Dec 2013 20:02:34 +0700 Subject: shallow.c: extend setup_*_shallow() to accept extra shallow commits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- commit.h | 8 +++++--- fetch-pack.c | 5 +++-- shallow.c | 20 +++++++++++++++----- upload-pack.c | 2 +- 4 files changed, 24 insertions(+), 11 deletions(-) (limited to 'commit.h') diff --git a/commit.h b/commit.h index a8795263b6..1faf717212 100644 --- a/commit.h +++ b/commit.h @@ -201,10 +201,12 @@ extern struct commit_list *get_shallow_commits(struct object_array *heads, int depth, int shallow_flag, int not_shallow_flag); extern void check_shallow_file_for_update(void); extern void set_alternate_shallow_file(const char *path); -extern int write_shallow_commits(struct strbuf *out, int use_pack_protocol); +extern int write_shallow_commits(struct strbuf *out, int use_pack_protocol, + const struct sha1_array *extra); extern void setup_alternate_shallow(struct lock_file *shallow_lock, - const char **alternate_shallow_file); -extern char *setup_temporary_shallow(void); + const char **alternate_shallow_file, + const struct sha1_array *extra); +extern char *setup_temporary_shallow(const struct sha1_array *extra); extern void advertise_shallow_grafts(int); int is_descendant_of(struct commit *, struct commit_list *); diff --git a/fetch-pack.c b/fetch-pack.c index 1042448fa0..0e7483e1fe 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -311,7 +311,7 @@ static int find_common(struct fetch_pack_args *args, } if (is_repository_shallow()) - write_shallow_commits(&req_buf, 1); + write_shallow_commits(&req_buf, 1, NULL); if (args->depth > 0) packet_buf_write(&req_buf, "deepen %d", args->depth); packet_buf_flush(&req_buf); @@ -850,7 +850,8 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, if (args->stateless_rpc) packet_flush(fd[1]); if (args->depth > 0) - setup_alternate_shallow(&shallow_lock, &alternate_shallow_file); + setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, + NULL); else alternate_shallow_file = NULL; if (get_pack(args, fd, pack_lockfile)) diff --git a/shallow.c b/shallow.c index f2c04b28dc..822c626600 100644 --- a/shallow.c +++ b/shallow.c @@ -165,22 +165,31 @@ static int write_one_shallow(const struct commit_graft *graft, void *cb_data) return 0; } -int write_shallow_commits(struct strbuf *out, int use_pack_protocol) +int write_shallow_commits(struct strbuf *out, int use_pack_protocol, + const struct sha1_array *extra) { struct write_shallow_data data; + int i; data.out = out; data.use_pack_protocol = use_pack_protocol; data.count = 0; for_each_commit_graft(write_one_shallow, &data); + if (!extra) + return data.count; + for (i = 0; i < extra->nr; i++) { + strbuf_addstr(out, sha1_to_hex(extra->sha1[i])); + strbuf_addch(out, '\n'); + data.count++; + } return data.count; } -char *setup_temporary_shallow(void) +char *setup_temporary_shallow(const struct sha1_array *extra) { struct strbuf sb = STRBUF_INIT; int fd; - if (write_shallow_commits(&sb, 0)) { + if (write_shallow_commits(&sb, 0, extra)) { struct strbuf path = STRBUF_INIT; strbuf_addstr(&path, git_path("shallow_XXXXXX")); fd = xmkstemp(path.buf); @@ -199,7 +208,8 @@ char *setup_temporary_shallow(void) } void setup_alternate_shallow(struct lock_file *shallow_lock, - const char **alternate_shallow_file) + const char **alternate_shallow_file, + const struct sha1_array *extra) { struct strbuf sb = STRBUF_INIT; int fd; @@ -207,7 +217,7 @@ void setup_alternate_shallow(struct lock_file *shallow_lock, check_shallow_file_for_update(); fd = hold_lock_file_for_update(shallow_lock, git_path("shallow"), LOCK_DIE_ON_ERROR); - if (write_shallow_commits(&sb, 0)) { + if (write_shallow_commits(&sb, 0, extra)) { if (write_in_full(fd, sb.buf, sb.len) != sb.len) die_errno("failed to write to %s", shallow_lock->filename); diff --git a/upload-pack.c b/upload-pack.c index 38b2a29110..f082f069ce 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -84,7 +84,7 @@ static void create_pack_file(void) char *shallow_file = NULL; if (shallow_nr) { - shallow_file = setup_temporary_shallow(); + shallow_file = setup_temporary_shallow(NULL); argv[arg++] = "--shallow-file"; argv[arg++] = shallow_file; } -- cgit v1.2.3 From 58babfffdeeecaa4d6edecaac1fb0c595218b801 Mon Sep 17 00:00:00 2001 From: Nguyễn Thái Ngọc Duy Date: Thu, 5 Dec 2013 20:02:35 +0700 Subject: shallow.c: the 8 steps to select new commits for .git/shallow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Suppose a fetch or push is requested between two shallow repositories (with no history deepening or shortening). A pack that contains necessary objects is transferred over together with .git/shallow of the sender. The receiver has to determine whether it needs to update .git/shallow if new refs needs new shallow comits. The rule here is avoid updating .git/shallow by default. But we don't want to waste the received pack. If the pack contains two refs, one needs new shallow commits installed in .git/shallow and one does not, we keep the latter and reject/warn about the former. Even if .git/shallow update is allowed, we only add shallow commits strictly necessary for the former ref (remember the sender can send more shallow commits than necessary) and pay attention not to accidentally cut the receiver history short (no history shortening is asked for) So the steps to figure out what ref need what new shallow commits are: 1. Split the sender shallow commit list into "ours" and "theirs" list by has_sha1_file. Those that exist in current repo in "ours", the remaining in "theirs". 2. Check the receiver .git/shallow, remove from "ours" the ones that also exist in .git/shallow. 3. Fetch the new pack. Either install or unpack it. 4. Do has_sha1_file on "theirs" list again. Drop the ones that fail has_sha1_file. Obviously the new pack does not need them. 5. If the pack is kept, remove from "ours" the ones that do not exist in the new pack. 6. Walk the new refs to answer the question "what shallow commits, both ours and theirs, are required in .git/shallow in order to add this ref?". Shallow commits not associated to any refs are removed from their respective list. 7. (*) Check reachability (from the current refs) of all remaining commits in "ours". Those reachable are removed. We do not want to cut any part of our (reachable) history. We only check up commits. True reachability test is done by check_everything_connected() at the end as usual. 8. Combine the final "ours" and "theirs" and add them all to .git/shallow. Install new refs. The case where some hook rejects some refs on a push is explained in more detail in the push patches. Of these steps, #6 and #7 are expensive. Both require walking through some commits, or in the worst case all commits. And we rather avoid them in at least common case, where the transferred pack does not contain any shallow commits that the sender advertises. Let's look at each scenario: 1) the sender has longer history than the receiver All shallow commits from the sender will be put into "theirs" list at step 1 because none of them exists in current repo. In the common case, "theirs" becomes empty at step 4 and exit early. 2) the sender has shorter history than the receiver All shallow commits from the sender are likely in "ours" list at step 1. In the common case, if the new pack is kept, we could empty "ours" and exit early at step 5. If the pack is not kept, we hit the expensive step 6 then exit after "ours" is emptied. There'll be only a handful of objects to walk in fast-forward case. If it's forced update, we may need to walk to the bottom. 3) the sender has same .git/shallow as the receiver This is similar to case 2 except that "ours" should be emptied at step 2 and exit early. A fetch after "clone --depth=X" is case 1. A fetch after "clone" (from a shallow repo) is case 3. Luckily they're cheap for the common case. A push from "clone --depth=X" falls into case 2, which is expensive. Some more work may be done at the sender/client side to avoid more work on the server side: if the transferred pack does not contain any shallow commits, send-pack should not send any shallow commits to the receive-pack, effectively turning it into a normal push and avoid all steps. This patch implements all steps except #3, already handled by fetch-pack and receive-pack, #6 and #7, which has their own patch due to their size. (*) in previous versions step 7 was put before step 3. I reorder it so that the common case that keeps the pack does not need to walk commits at all. In future if we implement faster commit reachability check (maybe with the help of pack bitmaps or commit cache), step 7 could become cheap and be moved up before 6 again. Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- cache.h | 2 ++ commit.h | 15 +++++++++++++ shallow.c | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ trace.c | 2 +- 4 files changed, 90 insertions(+), 1 deletion(-) (limited to 'commit.h') diff --git a/cache.h b/cache.h index ce377e1354..55dd4e3c8e 100644 --- a/cache.h +++ b/cache.h @@ -1236,6 +1236,8 @@ __attribute__((format (printf, 2, 3))) extern void trace_argv_printf(const char **argv, const char *format, ...); extern void trace_repo_setup(const char *prefix); extern int trace_want(const char *key); +__attribute__((format (printf, 2, 3))) +extern void trace_printf_key(const char *key, const char *fmt, ...); extern void trace_strbuf(const char *key, const struct strbuf *buf); void packet_trace_identity(const char *prog); diff --git a/commit.h b/commit.h index 1faf717212..9ead93bffb 100644 --- a/commit.h +++ b/commit.h @@ -193,6 +193,8 @@ extern struct commit_list *get_octopus_merge_bases(struct commit_list *in); /* largest positive number a signed 32-bit integer can contain */ #define INFINITE_DEPTH 0x7fffffff +struct sha1_array; +struct ref; extern int register_shallow(const unsigned char *sha1); extern int unregister_shallow(const unsigned char *sha1); extern int for_each_commit_graft(each_commit_graft_fn, void *); @@ -209,6 +211,19 @@ extern void setup_alternate_shallow(struct lock_file *shallow_lock, extern char *setup_temporary_shallow(const struct sha1_array *extra); extern void advertise_shallow_grafts(int); +struct shallow_info { + struct sha1_array *shallow; + int *ours, nr_ours; + int *theirs, nr_theirs; + struct sha1_array *ref; +}; + +extern void prepare_shallow_info(struct shallow_info *, struct sha1_array *); +extern void clear_shallow_info(struct shallow_info *); +extern void remove_nonexistent_theirs_shallow(struct shallow_info *); +extern void remove_nonexistent_ours_in_pack(struct shallow_info *, + struct packed_git *); + int is_descendant_of(struct commit *, struct commit_list *); int in_merge_bases(struct commit *, struct commit *); int in_merge_bases_many(struct commit *, int, struct commit **); diff --git a/shallow.c b/shallow.c index 822c626600..ecd950fd04 100644 --- a/shallow.c +++ b/shallow.c @@ -2,6 +2,12 @@ #include "commit.h" #include "tag.h" #include "pkt-line.h" +#include "remote.h" +#include "refs.h" +#include "sha1-array.h" +#include "diff.h" +#include "revision.h" +#include "commit-slab.h" static int is_shallow = -1; static struct stat shallow_stat; @@ -245,3 +251,69 @@ void advertise_shallow_grafts(int fd) return; for_each_commit_graft(advertise_shallow_grafts_cb, &fd); } + +#define TRACE_KEY "GIT_TRACE_SHALLOW" + +/* + * Step 1, split sender shallow commits into "ours" and "theirs" + * Step 2, clean "ours" based on .git/shallow + */ +void prepare_shallow_info(struct shallow_info *info, struct sha1_array *sa) +{ + int i; + trace_printf_key(TRACE_KEY, "shallow: prepare_shallow_info\n"); + memset(info, 0, sizeof(*info)); + info->shallow = sa; + if (!sa) + return; + info->ours = xmalloc(sizeof(*info->ours) * sa->nr); + info->theirs = xmalloc(sizeof(*info->theirs) * sa->nr); + for (i = 0; i < sa->nr; i++) { + if (has_sha1_file(sa->sha1[i])) { + struct commit_graft *graft; + graft = lookup_commit_graft(sa->sha1[i]); + if (graft && graft->nr_parent < 0) + continue; + info->ours[info->nr_ours++] = i; + } else + info->theirs[info->nr_theirs++] = i; + } +} + +void clear_shallow_info(struct shallow_info *info) +{ + free(info->ours); + free(info->theirs); +} + +/* Step 4, remove non-existent ones in "theirs" after getting the pack */ + +void remove_nonexistent_theirs_shallow(struct shallow_info *info) +{ + unsigned char (*sha1)[20] = info->shallow->sha1; + int i, dst; + trace_printf_key(TRACE_KEY, "shallow: remove_nonexistent_theirs_shallow\n"); + for (i = dst = 0; i < info->nr_theirs; i++) { + if (i != dst) + info->theirs[dst] = info->theirs[i]; + if (has_sha1_file(sha1[info->theirs[i]])) + dst++; + } + info->nr_theirs = dst; +} + +/* Step 5, remove non-existent ones in "ours" in the pack */ +void remove_nonexistent_ours_in_pack(struct shallow_info *info, + struct packed_git *p) +{ + unsigned char (*sha1)[20] = info->shallow->sha1; + int i, dst; + trace_printf_key(TRACE_KEY, "shallow: remove_nonexistent_ours_in_pack\n"); + for (i = dst = 0; i < info->nr_ours; i++) { + if (i != dst) + info->ours[dst] = info->ours[i]; + if (find_pack_entry_one(sha1[info->ours[i]], p)) + dst++; + } + info->nr_ours = dst; +} diff --git a/trace.c b/trace.c index 3d744d1d4d..08180a90bc 100644 --- a/trace.c +++ b/trace.c @@ -76,7 +76,7 @@ static void trace_vprintf(const char *key, const char *fmt, va_list ap) } __attribute__((format (printf, 2, 3))) -static void trace_printf_key(const char *key, const char *fmt, ...) +void trace_printf_key(const char *key, const char *fmt, ...) { va_list ap; va_start(ap, fmt); -- cgit v1.2.3 From 8e277383e0902551c0d5d3ef9591196de5a7078c Mon Sep 17 00:00:00 2001 From: Nguyễn Thái Ngọc Duy Date: Thu, 5 Dec 2013 20:02:36 +0700 Subject: shallow.c: steps 6 and 7 to select new commits for .git/shallow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- commit.h | 3 + shallow.c | 294 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 297 insertions(+) (limited to 'commit.h') diff --git a/commit.h b/commit.h index 9ead93bffb..69bca3e4be 100644 --- a/commit.h +++ b/commit.h @@ -223,6 +223,9 @@ extern void clear_shallow_info(struct shallow_info *); extern void remove_nonexistent_theirs_shallow(struct shallow_info *); extern void remove_nonexistent_ours_in_pack(struct shallow_info *, struct packed_git *); +extern void assign_shallow_commits_to_refs(struct shallow_info *info, + uint32_t **used, + int *ref_status); int is_descendant_of(struct commit *, struct commit_list *); int in_merge_bases(struct commit *, struct commit *); diff --git a/shallow.c b/shallow.c index ecd950fd04..fb6069ba0c 100644 --- a/shallow.c +++ b/shallow.c @@ -317,3 +317,297 @@ void remove_nonexistent_ours_in_pack(struct shallow_info *info, } info->nr_ours = dst; } + +define_commit_slab(ref_bitmap, uint32_t *); + +struct paint_info { + struct ref_bitmap ref_bitmap; + unsigned nr_bits; + char **slab; + char *free, *end; + unsigned slab_count; +}; + +static uint32_t *paint_alloc(struct paint_info *info) +{ + unsigned nr = (info->nr_bits + 31) / 32; + unsigned size = nr * sizeof(uint32_t); + void *p; + if (!info->slab_count || info->free + size > info->end) { + info->slab_count++; + info->slab = xrealloc(info->slab, + info->slab_count * sizeof(*info->slab)); + info->free = xmalloc(COMMIT_SLAB_SIZE); + info->slab[info->slab_count - 1] = info->free; + info->end = info->free + COMMIT_SLAB_SIZE; + } + p = info->free; + info->free += size; + return p; +} + +/* + * Given a commit SHA-1, walk down to parents until either SEEN, + * UNINTERESTING or BOTTOM is hit. Set the id-th bit in ref_bitmap for + * all walked commits. + */ +static void paint_down(struct paint_info *info, const unsigned char *sha1, + int id) +{ + unsigned int i, nr; + struct commit_list *head = NULL; + int bitmap_nr = (info->nr_bits + 31) / 32; + int bitmap_size = bitmap_nr * sizeof(uint32_t); + uint32_t *tmp = xmalloc(bitmap_size); /* to be freed before return */ + uint32_t *bitmap = paint_alloc(info); + struct commit *c = lookup_commit_reference_gently(sha1, 1); + if (!c) + return; + memset(bitmap, 0, bitmap_size); + bitmap[id / 32] |= (1 << (id % 32)); + commit_list_insert(c, &head); + while (head) { + struct commit_list *p; + struct commit *c = head->item; + uint32_t **refs = ref_bitmap_at(&info->ref_bitmap, c); + + p = head; + head = head->next; + free(p); + + /* XXX check "UNINTERESTING" from pack bitmaps if available */ + if (c->object.flags & (SEEN | UNINTERESTING)) + continue; + else + c->object.flags |= SEEN; + + if (*refs == NULL) + *refs = bitmap; + else { + memcpy(tmp, *refs, bitmap_size); + for (i = 0; i < bitmap_nr; i++) + tmp[i] |= bitmap[i]; + if (memcmp(tmp, *refs, bitmap_size)) { + *refs = paint_alloc(info); + memcpy(*refs, tmp, bitmap_size); + } + } + + if (c->object.flags & BOTTOM) + continue; + + if (parse_commit(c)) + die("unable to parse commit %s", + sha1_to_hex(c->object.sha1)); + + for (p = c->parents; p; p = p->next) { + uint32_t **p_refs = ref_bitmap_at(&info->ref_bitmap, + p->item); + if (p->item->object.flags & SEEN) + continue; + if (*p_refs == NULL || *p_refs == *refs) + *p_refs = *refs; + commit_list_insert(p->item, &head); + } + } + + nr = get_max_object_index(); + for (i = 0; i < nr; i++) { + struct object *o = get_indexed_object(i); + if (o && o->type == OBJ_COMMIT) + o->flags &= ~SEEN; + } + + free(tmp); +} + +static int mark_uninteresting(const char *refname, + const unsigned char *sha1, + int flags, void *cb_data) +{ + struct commit *commit = lookup_commit_reference_gently(sha1, 1); + if (!commit) + return 0; + commit->object.flags |= UNINTERESTING; + mark_parents_uninteresting(commit); + return 0; +} + +static void post_assign_shallow(struct shallow_info *info, + struct ref_bitmap *ref_bitmap, + int *ref_status); +/* + * Step 6(+7), associate shallow commits with new refs + * + * info->ref must be initialized before calling this function. + * + * If used is not NULL, it's an array of info->shallow->nr + * bitmaps. The n-th bit set in the m-th bitmap if ref[n] needs the + * m-th shallow commit from info->shallow. + * + * If used is NULL, "ours" and "theirs" are updated. And if ref_status + * is not NULL it's an array of ref->nr ints. ref_status[i] is true if + * the ref needs some shallow commits from either info->ours or + * info->theirs. + */ +void assign_shallow_commits_to_refs(struct shallow_info *info, + uint32_t **used, int *ref_status) +{ + unsigned char (*sha1)[20] = info->shallow->sha1; + struct sha1_array *ref = info->ref; + unsigned int i, nr; + int *shallow, nr_shallow = 0; + struct paint_info pi; + + trace_printf_key(TRACE_KEY, "shallow: assign_shallow_commits_to_refs\n"); + shallow = xmalloc(sizeof(*shallow) * (info->nr_ours + info->nr_theirs)); + for (i = 0; i < info->nr_ours; i++) + shallow[nr_shallow++] = info->ours[i]; + for (i = 0; i < info->nr_theirs; i++) + shallow[nr_shallow++] = info->theirs[i]; + + /* + * Prepare the commit graph to track what refs can reach what + * (new) shallow commits. + */ + nr = get_max_object_index(); + for (i = 0; i < nr; i++) { + struct object *o = get_indexed_object(i); + if (!o || o->type != OBJ_COMMIT) + continue; + + o->flags &= ~(UNINTERESTING | BOTTOM | SEEN); + } + + memset(&pi, 0, sizeof(pi)); + init_ref_bitmap(&pi.ref_bitmap); + pi.nr_bits = ref->nr; + + /* + * "--not --all" to cut short the traversal if new refs + * connect to old refs. If not (e.g. force ref updates) it'll + * have to go down to the current shallow commits. + */ + head_ref(mark_uninteresting, NULL); + for_each_ref(mark_uninteresting, NULL); + + /* Mark potential bottoms so we won't go out of bound */ + for (i = 0; i < nr_shallow; i++) { + struct commit *c = lookup_commit(sha1[shallow[i]]); + c->object.flags |= BOTTOM; + } + + for (i = 0; i < ref->nr; i++) + paint_down(&pi, ref->sha1[i], i); + + if (used) { + int bitmap_size = ((pi.nr_bits + 31) / 32) * sizeof(uint32_t); + memset(used, 0, sizeof(*used) * info->shallow->nr); + for (i = 0; i < nr_shallow; i++) { + const struct commit *c = lookup_commit(sha1[shallow[i]]); + uint32_t **map = ref_bitmap_at(&pi.ref_bitmap, c); + if (*map) + used[shallow[i]] = xmemdupz(*map, bitmap_size); + } + /* + * unreachable shallow commits are not removed from + * "ours" and "theirs". The user is supposed to run + * step 7 on every ref separately and not trust "ours" + * and "theirs" any more. + */ + } else + post_assign_shallow(info, &pi.ref_bitmap, ref_status); + + clear_ref_bitmap(&pi.ref_bitmap); + for (i = 0; i < pi.slab_count; i++) + free(pi.slab[i]); + free(pi.slab); + free(shallow); +} + +struct commit_array { + struct commit **commits; + int nr, alloc; +}; + +static int add_ref(const char *refname, + const unsigned char *sha1, int flags, void *cb_data) +{ + struct commit_array *ca = cb_data; + ALLOC_GROW(ca->commits, ca->nr + 1, ca->alloc); + ca->commits[ca->nr] = lookup_commit_reference_gently(sha1, 1); + if (ca->commits[ca->nr]) + ca->nr++; + return 0; +} + +static void update_refstatus(int *ref_status, int nr, uint32_t *bitmap) +{ + int i; + if (!ref_status) + return; + for (i = 0; i < nr; i++) + if (bitmap[i / 32] & (1 << (i % 32))) + ref_status[i]++; +} + +/* + * Step 7, reachability test on "ours" at commit level + */ +static void post_assign_shallow(struct shallow_info *info, + struct ref_bitmap *ref_bitmap, + int *ref_status) +{ + unsigned char (*sha1)[20] = info->shallow->sha1; + struct commit *c; + uint32_t **bitmap; + int dst, i, j; + int bitmap_nr = (info->ref->nr + 31) / 32; + struct commit_array ca; + + trace_printf_key(TRACE_KEY, "shallow: post_assign_shallow\n"); + if (ref_status) + memset(ref_status, 0, sizeof(*ref_status) * info->ref->nr); + + /* Remove unreachable shallow commits from "theirs" */ + for (i = dst = 0; i < info->nr_theirs; i++) { + if (i != dst) + info->theirs[dst] = info->theirs[i]; + c = lookup_commit(sha1[info->theirs[i]]); + bitmap = ref_bitmap_at(ref_bitmap, c); + if (!*bitmap) + continue; + for (j = 0; j < bitmap_nr; j++) + if (bitmap[0][j]) { + update_refstatus(ref_status, info->ref->nr, *bitmap); + dst++; + break; + } + } + info->nr_theirs = dst; + + memset(&ca, 0, sizeof(ca)); + head_ref(add_ref, &ca); + for_each_ref(add_ref, &ca); + + /* Remove unreachable shallow commits from "ours" */ + for (i = dst = 0; i < info->nr_ours; i++) { + if (i != dst) + info->ours[dst] = info->ours[i]; + c = lookup_commit(sha1[info->ours[i]]); + bitmap = ref_bitmap_at(ref_bitmap, c); + if (!*bitmap) + continue; + for (j = 0; j < bitmap_nr; j++) + if (bitmap[0][j] && + /* Step 7, reachability test at commit level */ + !in_merge_bases_many(c, ca.nr, ca.commits)) { + update_refstatus(ref_status, info->ref->nr, *bitmap); + dst++; + break; + } + } + info->nr_ours = dst; + + free(ca.commits); +} -- cgit v1.2.3 From 069c053222bfc62a6522430a137e9b2c7ff36e4c Mon Sep 17 00:00:00 2001 From: Nguyễn Thái Ngọc Duy Date: Thu, 5 Dec 2013 20:02:45 +0700 Subject: add GIT_SHALLOW_FILE to propagate --shallow-file to subprocesses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This may be needed when a hook is run after a new shallow pack is received, but .git/shallow is not settled yet. A temporary shallow file to plug all loose ends should be used instead. GIT_SHALLOW_FILE is overriden by --shallow-file. --shallow-file does not work in this case because the hook may spawn many git subprocesses and the launch commands do not have --shallow-file as it's a recent addition. Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- cache.h | 1 + commit.h | 2 +- environment.c | 6 ++++++ git.c | 2 +- shallow.c | 4 +++- 5 files changed, 12 insertions(+), 3 deletions(-) (limited to 'commit.h') diff --git a/cache.h b/cache.h index 55dd4e3c8e..8b132878ce 100644 --- a/cache.h +++ b/cache.h @@ -354,6 +354,7 @@ static inline enum object_type object_type(unsigned int mode) #define DB_ENVIRONMENT "GIT_OBJECT_DIRECTORY" #define INDEX_ENVIRONMENT "GIT_INDEX_FILE" #define GRAFT_ENVIRONMENT "GIT_GRAFT_FILE" +#define GIT_SHALLOW_FILE_ENVIRONMENT "GIT_SHALLOW_FILE" #define TEMPLATE_DIR_ENVIRONMENT "GIT_TEMPLATE_DIR" #define CONFIG_ENVIRONMENT "GIT_CONFIG" #define CONFIG_DATA_ENVIRONMENT "GIT_CONFIG_PARAMETERS" diff --git a/commit.h b/commit.h index 69bca3e4be..79649efc7c 100644 --- a/commit.h +++ b/commit.h @@ -202,7 +202,7 @@ extern int is_repository_shallow(void); extern struct commit_list *get_shallow_commits(struct object_array *heads, int depth, int shallow_flag, int not_shallow_flag); extern void check_shallow_file_for_update(void); -extern void set_alternate_shallow_file(const char *path); +extern void set_alternate_shallow_file(const char *path, int override); extern int write_shallow_commits(struct strbuf *out, int use_pack_protocol, const struct sha1_array *extra); extern void setup_alternate_shallow(struct lock_file *shallow_lock, diff --git a/environment.c b/environment.c index 0a15349cfe..b73b39d72f 100644 --- a/environment.c +++ b/environment.c @@ -10,6 +10,7 @@ #include "cache.h" #include "refs.h" #include "fmt-merge-msg.h" +#include "commit.h" int trust_executable_bit = 1; int trust_ctime = 1; @@ -97,6 +98,7 @@ const char * const local_repo_env[] = { INDEX_ENVIRONMENT, NO_REPLACE_OBJECTS_ENVIRONMENT, GIT_PREFIX_ENVIRONMENT, + GIT_SHALLOW_FILE_ENVIRONMENT, NULL }; @@ -124,6 +126,7 @@ static char *expand_namespace(const char *raw_namespace) static void setup_git_env(void) { const char *gitfile; + const char *shallow_file; git_dir = getenv(GIT_DIR_ENVIRONMENT); if (!git_dir) @@ -147,6 +150,9 @@ static void setup_git_env(void) read_replace_refs = 0; namespace = expand_namespace(getenv(GIT_NAMESPACE_ENVIRONMENT)); namespace_len = strlen(namespace); + shallow_file = getenv(GIT_SHALLOW_FILE_ENVIRONMENT); + if (shallow_file) + set_alternate_shallow_file(shallow_file, 0); } int is_bare_repository(void) diff --git a/git.c b/git.c index cb5208de6a..179c4f6ff8 100644 --- a/git.c +++ b/git.c @@ -162,7 +162,7 @@ static int handle_options(const char ***argv, int *argc, int *envchanged) } else if (!strcmp(cmd, "--shallow-file")) { (*argv)++; (*argc)--; - set_alternate_shallow_file((*argv)[0]); + set_alternate_shallow_file((*argv)[0], 1); if (envchanged) *envchanged = 1; } else if (!strcmp(cmd, "-C")) { diff --git a/shallow.c b/shallow.c index 52268544fd..ec9179480f 100644 --- a/shallow.c +++ b/shallow.c @@ -13,10 +13,12 @@ static int is_shallow = -1; static struct stat shallow_stat; static char *alternate_shallow_file; -void set_alternate_shallow_file(const char *path) +void set_alternate_shallow_file(const char *path, int override) { if (is_shallow != -1) die("BUG: is_repository_shallow must not be called before set_alternate_shallow_file"); + if (alternate_shallow_file && !override) + return; free(alternate_shallow_file); alternate_shallow_file = path ? xstrdup(path) : NULL; } -- cgit v1.2.3 From 0a1bc12b6e401825f009ac8bb14fc438f77e2d9f Mon Sep 17 00:00:00 2001 From: Nguyễn Thái Ngọc Duy Date: Thu, 5 Dec 2013 20:02:47 +0700 Subject: receive-pack: allow pushes that update .git/shallow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The basic 8 steps to update .git/shallow does not fully apply here because the user may choose to accept just a few refs (while fetch always accepts all refs). The steps are modified a bit. 1-6. same as before. After calling assign_shallow_commits_to_refs at step 6, each shallow commit has a bitmap that marks all refs that require it. 7. mark all "ours" shallow commits that are reachable from any refs. We will need to do the original step 7 on them later. 8. go over all shallow commit bitmaps, mark refs that require new shallow commits. 9. setup a strict temporary shallow file to plug all the holes, even if it may cut some of our history short. This file is used by all hooks. The hooks could use --shallow-file=$GIT_DIR/shallow to overcome this and reach everything in current repo. 10. go over the new refs one by one. For each ref, do the reachability test if it needs a shallow commit on the list from step 7. Remove it if it's reachable from our refs. Gather all required shallow commits, run check_everything_connected() with the new ref, then install them to .git/shallow. This mode is disabled by default and can be turned on with receive.shallowupdate Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- Documentation/config.txt | 4 ++ builtin/receive-pack.c | 163 +++++++++++++++++++++++++++++++++++++++++++---- commit.h | 9 +++ shallow.c | 23 +++++++ t/t5538-push-shallow.sh | 15 +++++ 5 files changed, 201 insertions(+), 13 deletions(-) (limited to 'commit.h') diff --git a/Documentation/config.txt b/Documentation/config.txt index ab26963d61..1a0bd0d4ed 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -2026,6 +2026,10 @@ receive.updateserverinfo:: If set to true, git-receive-pack will run git-update-server-info after receiving data from git-push and updating refs. +receive.shallowupdate:: + If set to true, .git/shallow can be updated when new refs + require new shallow roots. Otherwise those refs are rejected. + remote.pushdefault:: The remote to push to by default. Overrides `branch..remote` for all branches, and is overridden by diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index b9de2e8ff6..5c85bb4b49 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -44,6 +44,7 @@ static int fix_thin = 1; static const char *head_name; static void *head_name_to_free; static int sent_capabilities; +static int shallow_update; static const char *alt_shallow_file; static enum deny_action parse_deny_action(const char *var, const char *value) @@ -123,6 +124,11 @@ static int receive_pack_config(const char *var, const char *value, void *cb) return 0; } + if (strcmp(var, "receive.shallowupdate") == 0) { + shallow_update = git_config_bool(var, value); + return 0; + } + return git_default_config(var, value, cb); } @@ -423,7 +429,46 @@ static void refuse_unconfigured_deny_delete_current(void) rp_error("%s", refuse_unconfigured_deny_delete_current_msg[i]); } -static const char *update(struct command *cmd) +static int command_singleton_iterator(void *cb_data, unsigned char sha1[20]); +static int update_shallow_ref(struct command *cmd, struct shallow_info *si) +{ + static struct lock_file shallow_lock; + struct sha1_array extra = SHA1_ARRAY_INIT; + const char *alt_file; + uint32_t mask = 1 << (cmd->index % 32); + int i; + + trace_printf_key("GIT_TRACE_SHALLOW", + "shallow: update_shallow_ref %s\n", cmd->ref_name); + for (i = 0; i < si->shallow->nr; i++) + if (si->used_shallow[i] && + (si->used_shallow[i][cmd->index / 32] & mask) && + !delayed_reachability_test(si, i)) + sha1_array_append(&extra, si->shallow->sha1[i]); + + setup_alternate_shallow(&shallow_lock, &alt_file, &extra); + if (check_shallow_connected(command_singleton_iterator, + 0, cmd, alt_file)) { + rollback_lock_file(&shallow_lock); + sha1_array_clear(&extra); + return -1; + } + + commit_lock_file(&shallow_lock); + + /* + * Make sure setup_alternate_shallow() for the next ref does + * not lose these new roots.. + */ + for (i = 0; i < extra.nr; i++) + register_shallow(extra.sha1[i]); + + si->shallow_ref[cmd->index] = 0; + sha1_array_clear(&extra); + return 0; +} + +static const char *update(struct command *cmd, struct shallow_info *si) { const char *name = cmd->ref_name; struct strbuf namespaced_name_buf = STRBUF_INIT; @@ -531,6 +576,10 @@ static const char *update(struct command *cmd) return NULL; /* good */ } else { + if (shallow_update && si->shallow_ref[cmd->index] && + update_shallow_ref(cmd, si)) + return "shallow error"; + lock = lock_any_ref_for_update(namespaced_name, old_sha1, 0, NULL); if (!lock) { @@ -671,12 +720,16 @@ static int command_singleton_iterator(void *cb_data, unsigned char sha1[20]) return 0; } -static void set_connectivity_errors(struct command *commands) +static void set_connectivity_errors(struct command *commands, + struct shallow_info *si) { struct command *cmd; for (cmd = commands; cmd; cmd = cmd->next) { struct command *singleton = cmd; + if (shallow_update && si->shallow_ref[cmd->index]) + /* to be checked in update_shallow_ref() */ + continue; if (!check_everything_connected(command_singleton_iterator, 0, &singleton)) continue; @@ -684,18 +737,26 @@ static void set_connectivity_errors(struct command *commands) } } +struct iterate_data { + struct command *cmds; + struct shallow_info *si; +}; + static int iterate_receive_command_list(void *cb_data, unsigned char sha1[20]) { - struct command **cmd_list = cb_data; + struct iterate_data *data = cb_data; + struct command **cmd_list = &data->cmds; struct command *cmd = *cmd_list; - while (cmd) { + for (; cmd; cmd = cmd->next) { + if (shallow_update && data->si->shallow_ref[cmd->index]) + /* to be checked in update_shallow_ref() */ + continue; if (!is_null_sha1(cmd->new_sha1) && !cmd->skip_update) { hashcpy(sha1, cmd->new_sha1); *cmd_list = cmd->next; return 0; } - cmd = cmd->next; } *cmd_list = NULL; return -1; /* end of list */ @@ -715,10 +776,14 @@ static void reject_updates_to_hidden(struct command *commands) } } -static void execute_commands(struct command *commands, const char *unpacker_error) +static void execute_commands(struct command *commands, + const char *unpacker_error, + struct shallow_info *si) { + int checked_connectivity; struct command *cmd; unsigned char sha1[20]; + struct iterate_data data; if (unpacker_error) { for (cmd = commands; cmd; cmd = cmd->next) @@ -726,10 +791,10 @@ static void execute_commands(struct command *commands, const char *unpacker_erro return; } - cmd = commands; - if (check_everything_connected(iterate_receive_command_list, - 0, &cmd)) - set_connectivity_errors(commands); + data.cmds = commands; + data.si = si; + if (check_everything_connected(iterate_receive_command_list, 0, &data)) + set_connectivity_errors(commands, si); reject_updates_to_hidden(commands); @@ -746,6 +811,7 @@ static void execute_commands(struct command *commands, const char *unpacker_erro free(head_name_to_free); head_name = head_name_to_free = resolve_refdup("HEAD", sha1, 0, NULL); + checked_connectivity = 1; for (cmd = commands; cmd; cmd = cmd->next) { if (cmd->error_string) continue; @@ -753,7 +819,22 @@ static void execute_commands(struct command *commands, const char *unpacker_erro if (cmd->skip_update) continue; - cmd->error_string = update(cmd); + cmd->error_string = update(cmd, si); + if (shallow_update && !cmd->error_string && + si->shallow_ref[cmd->index]) { + error("BUG: connectivity check has not been run on ref %s", + cmd->ref_name); + checked_connectivity = 0; + } + } + + if (shallow_update) { + if (!checked_connectivity) + error("BUG: run 'git fsck' for safety.\n" + "If there are errors, try to remove " + "the reported refs above"); + if (alt_shallow_file && *alt_shallow_file) + unlink(alt_shallow_file); } } @@ -924,6 +1005,53 @@ static const char *unpack_with_sideband(struct shallow_info *si) return ret; } +static void prepare_shallow_update(struct command *commands, + struct shallow_info *si) +{ + int i, j, k, bitmap_size = (si->ref->nr + 31) / 32; + + si->used_shallow = xmalloc(sizeof(*si->used_shallow) * + si->shallow->nr); + assign_shallow_commits_to_refs(si, si->used_shallow, NULL); + + si->need_reachability_test = + xcalloc(si->shallow->nr, sizeof(*si->need_reachability_test)); + si->reachable = + xcalloc(si->shallow->nr, sizeof(*si->reachable)); + si->shallow_ref = xcalloc(si->ref->nr, sizeof(*si->shallow_ref)); + + for (i = 0; i < si->nr_ours; i++) + si->need_reachability_test[si->ours[i]] = 1; + + for (i = 0; i < si->shallow->nr; i++) { + if (!si->used_shallow[i]) + continue; + for (j = 0; j < bitmap_size; j++) { + if (!si->used_shallow[i][j]) + continue; + si->need_reachability_test[i]++; + for (k = 0; k < 32; k++) + if (si->used_shallow[i][j] & (1 << k)) + si->shallow_ref[j * 32 + k]++; + } + + /* + * true for those associated with some refs and belong + * in "ours" list aka "step 7 not done yet" + */ + si->need_reachability_test[i] = + si->need_reachability_test[i] > 1; + } + + /* + * keep hooks happy by forcing a temporary shallow file via + * env variable because we can't add --shallow-file to every + * command. check_everything_connected() will be done with + * true .git/shallow though. + */ + setenv(GIT_SHALLOW_FILE_ENVIRONMENT, alt_shallow_file, 1); +} + static void update_shallow_info(struct command *commands, struct shallow_info *si, struct sha1_array *ref) @@ -932,8 +1060,10 @@ static void update_shallow_info(struct command *commands, int *ref_status; remove_nonexistent_theirs_shallow(si); /* XXX remove_nonexistent_ours_in_pack() */ - if (!si->nr_ours && !si->nr_theirs) + if (!si->nr_ours && !si->nr_theirs) { + shallow_update = 0; return; + } for (cmd = commands; cmd; cmd = cmd->next) { if (is_null_sha1(cmd->new_sha1)) @@ -943,6 +1073,11 @@ static void update_shallow_info(struct command *commands, } si->ref = ref; + if (shallow_update) { + prepare_shallow_update(commands, si); + return; + } + ref_status = xmalloc(sizeof(*ref_status) * ref->nr); assign_shallow_commits_to_refs(si, NULL, ref_status); for (cmd = commands; cmd; cmd = cmd->next) { @@ -1064,11 +1199,13 @@ int cmd_receive_pack(int argc, const char **argv, const char *prefix) const char *unpack_status = NULL; prepare_shallow_info(&si, &shallow); + if (!si.nr_ours && !si.nr_theirs) + shallow_update = 0; if (!delete_only(commands)) { unpack_status = unpack_with_sideband(&si); update_shallow_info(commands, &si, &ref); } - execute_commands(commands, unpack_status); + execute_commands(commands, unpack_status, &si); if (pack_lockfile) unlink_or_warn(pack_lockfile); if (report_status) diff --git a/commit.h b/commit.h index 79649efc7c..a1f2d49433 100644 --- a/commit.h +++ b/commit.h @@ -216,6 +216,14 @@ struct shallow_info { int *ours, nr_ours; int *theirs, nr_theirs; struct sha1_array *ref; + + /* for receive-pack */ + uint32_t **used_shallow; + int *need_reachability_test; + int *reachable; + int *shallow_ref; + struct commit **commits; + int nr_commits; }; extern void prepare_shallow_info(struct shallow_info *, struct sha1_array *); @@ -226,6 +234,7 @@ extern void remove_nonexistent_ours_in_pack(struct shallow_info *, extern void assign_shallow_commits_to_refs(struct shallow_info *info, uint32_t **used, int *ref_status); +extern int delayed_reachability_test(struct shallow_info *si, int c); int is_descendant_of(struct commit *, struct commit_list *); int in_merge_bases(struct commit *, struct commit *); diff --git a/shallow.c b/shallow.c index ec9179480f..3c36dd82bc 100644 --- a/shallow.c +++ b/shallow.c @@ -617,3 +617,26 @@ static void post_assign_shallow(struct shallow_info *info, free(ca.commits); } + +/* (Delayed) step 7, reachability test at commit level */ +int delayed_reachability_test(struct shallow_info *si, int c) +{ + if (si->need_reachability_test[c]) { + struct commit *commit = lookup_commit(si->shallow->sha1[c]); + + if (!si->commits) { + struct commit_array ca; + memset(&ca, 0, sizeof(ca)); + head_ref(add_ref, &ca); + for_each_ref(add_ref, &ca); + si->commits = ca.commits; + si->nr_commits = ca.nr; + } + + si->reachable[c] = in_merge_bases_many(commit, + si->nr_commits, + si->commits); + si->need_reachability_test[c] = 0; + } + return si->reachable[c]; +} diff --git a/t/t5538-push-shallow.sh b/t/t5538-push-shallow.sh index 650c31a888..ff5eb5bcf5 100755 --- a/t/t5538-push-shallow.sh +++ b/t/t5538-push-shallow.sh @@ -67,4 +67,19 @@ test_expect_success 'push from shallow clone, with grafted roots' ' git fsck ' +test_expect_success 'add new shallow root with receive.updateshallow on' ' + test_config receive.shallowupdate true && + ( + cd shallow2 && + git push ../.git +master:refs/remotes/shallow2/master + ) && + git log --format=%s shallow2/master >actual && + git fsck && + cat <expect && +c +b +EOF + test_cmp expect actual +' + test_done -- cgit v1.2.3 From eab3296c7e5c99f559818357e70eeae09c24ac99 Mon Sep 17 00:00:00 2001 From: Nguyễn Thái Ngọc Duy Date: Thu, 5 Dec 2013 20:02:54 +0700 Subject: prune: clean .git/shallow after pruning objects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch teaches "prune" to remove shallow roots that are no longer reachable from any refs (e.g. when the relevant refs are removed). Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- Documentation/git-prune.txt | 2 ++ builtin/gc.c | 1 + builtin/prune.c | 4 ++++ commit.h | 1 + shallow.c | 55 +++++++++++++++++++++++++++++++++++++++++++-- t/t5304-prune.sh | 10 +++++++++ 6 files changed, 71 insertions(+), 2 deletions(-) (limited to 'commit.h') diff --git a/Documentation/git-prune.txt b/Documentation/git-prune.txt index bf824108c1..058ac0dc85 100644 --- a/Documentation/git-prune.txt +++ b/Documentation/git-prune.txt @@ -24,6 +24,8 @@ objects unreachable from any of these head objects from the object database. In addition, it prunes the unpacked objects that are also found in packs by running 'git prune-packed'. +It also removes entries from .git/shallow that are not reachable by +any ref. Note that unreachable, packed objects will remain. If this is not desired, see linkgit:git-repack[1]. diff --git a/builtin/gc.c b/builtin/gc.c index c14190f840..cec8ecd754 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -16,6 +16,7 @@ #include "run-command.h" #include "sigchain.h" #include "argv-array.h" +#include "commit.h" #define FAILED_RUN "failed to run %s" diff --git a/builtin/prune.c b/builtin/prune.c index 6366917c6d..2214040349 100644 --- a/builtin/prune.c +++ b/builtin/prune.c @@ -170,5 +170,9 @@ int cmd_prune(int argc, const char **argv, const char *prefix) s = mkpathdup("%s/pack", get_object_directory()); remove_temporary_files(s); free(s); + + if (is_repository_shallow()) + prune_shallow(show_only); + return 0; } diff --git a/commit.h b/commit.h index a1f2d49433..affe210337 100644 --- a/commit.h +++ b/commit.h @@ -235,6 +235,7 @@ extern void assign_shallow_commits_to_refs(struct shallow_info *info, uint32_t **used, int *ref_status); extern int delayed_reachability_test(struct shallow_info *si, int c); +extern void prune_shallow(int show_only); int is_descendant_of(struct commit *, struct commit_list *); int in_merge_bases(struct commit *, struct commit *); diff --git a/shallow.c b/shallow.c index 3c36dd82bc..c766fc3012 100644 --- a/shallow.c +++ b/shallow.c @@ -155,10 +155,14 @@ void check_shallow_file_for_update(void) die("shallow file was changed during fetch"); } +#define SEEN_ONLY 1 +#define VERBOSE 2 + struct write_shallow_data { struct strbuf *out; int use_pack_protocol; int count; + unsigned flags; }; static int write_one_shallow(const struct commit_graft *graft, void *cb_data) @@ -167,6 +171,15 @@ static int write_one_shallow(const struct commit_graft *graft, void *cb_data) const char *hex = sha1_to_hex(graft->sha1); if (graft->nr_parent != -1) return 0; + if (data->flags & SEEN_ONLY) { + struct commit *c = lookup_commit(graft->sha1); + if (!c || !(c->object.flags & SEEN)) { + if (data->flags & VERBOSE) + printf("Removing %s from .git/shallow\n", + sha1_to_hex(c->object.sha1)); + return 0; + } + } data->count++; if (data->use_pack_protocol) packet_buf_write(data->out, "shallow %s", hex); @@ -177,14 +190,16 @@ static int write_one_shallow(const struct commit_graft *graft, void *cb_data) return 0; } -int write_shallow_commits(struct strbuf *out, int use_pack_protocol, - const struct sha1_array *extra) +static int write_shallow_commits_1(struct strbuf *out, int use_pack_protocol, + const struct sha1_array *extra, + unsigned flags) { struct write_shallow_data data; int i; data.out = out; data.use_pack_protocol = use_pack_protocol; data.count = 0; + data.flags = flags; for_each_commit_graft(write_one_shallow, &data); if (!extra) return data.count; @@ -196,6 +211,12 @@ int write_shallow_commits(struct strbuf *out, int use_pack_protocol, return data.count; } +int write_shallow_commits(struct strbuf *out, int use_pack_protocol, + const struct sha1_array *extra) +{ + return write_shallow_commits_1(out, use_pack_protocol, extra, 0); +} + char *setup_temporary_shallow(const struct sha1_array *extra) { struct strbuf sb = STRBUF_INIT; @@ -258,6 +279,36 @@ void advertise_shallow_grafts(int fd) for_each_commit_graft(advertise_shallow_grafts_cb, &fd); } +/* + * mark_reachable_objects() should have been run prior to this and all + * reachable commits marked as "SEEN". + */ +void prune_shallow(int show_only) +{ + static struct lock_file shallow_lock; + struct strbuf sb = STRBUF_INIT; + int fd; + + if (show_only) { + write_shallow_commits_1(&sb, 0, NULL, SEEN_ONLY | VERBOSE); + strbuf_release(&sb); + return; + } + check_shallow_file_for_update(); + fd = hold_lock_file_for_update(&shallow_lock, git_path("shallow"), + LOCK_DIE_ON_ERROR); + if (write_shallow_commits_1(&sb, 0, NULL, SEEN_ONLY)) { + if (write_in_full(fd, sb.buf, sb.len) != sb.len) + die_errno("failed to write to %s", + shallow_lock.filename); + commit_lock_file(&shallow_lock); + } else { + unlink(git_path("shallow")); + rollback_lock_file(&shallow_lock); + } + strbuf_release(&sb); +} + #define TRACE_KEY "GIT_TRACE_SHALLOW" /* diff --git a/t/t5304-prune.sh b/t/t5304-prune.sh index e4bb3a1457..66c9a41739 100755 --- a/t/t5304-prune.sh +++ b/t/t5304-prune.sh @@ -221,4 +221,14 @@ EOF test_cmp expected actual ' +test_expect_success 'prune .git/shallow' ' + SHA1=`echo hi|git commit-tree HEAD^{tree}` && + echo $SHA1 >.git/shallow && + git prune --dry-run >out && + grep $SHA1 .git/shallow && + grep $SHA1 out && + git prune && + ! test -f .git/shallow +' + test_done -- cgit v1.2.3 From feefdf62c107fd63056becf547ea324dbb730e30 Mon Sep 17 00:00:00 2001 From: Ramsay Jones Date: Mon, 6 Jan 2014 00:00:58 +0000 Subject: shallow: remove unused code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit 58babfff ("shallow.c: the 8 steps to select new commits for .git/shallow", 05-12-2013) added a function to implement step 5 of the quoted eight steps, namely 'remove_nonexistent_ours_in_pack()'. This function implements an optional optimization step in the new shallow commit selection algorithm. However, this function has no callers. (The commented out call sites would need to change, in order to provide information required by the function.) Signed-off-by: Ramsay Jones Acked-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- builtin/receive-pack.c | 1 - commit.h | 2 -- fetch-pack.c | 1 - shallow.c | 16 ---------------- 4 files changed, 20 deletions(-) (limited to 'commit.h') diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index 78fe8ee62f..bc4f5dc463 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -1059,7 +1059,6 @@ static void update_shallow_info(struct command *commands, struct command *cmd; int *ref_status; remove_nonexistent_theirs_shallow(si); - /* XXX remove_nonexistent_ours_in_pack() */ if (!si->nr_ours && !si->nr_theirs) { shallow_update = 0; return; diff --git a/commit.h b/commit.h index affe210337..2a20b10d39 100644 --- a/commit.h +++ b/commit.h @@ -229,8 +229,6 @@ struct shallow_info { extern void prepare_shallow_info(struct shallow_info *, struct sha1_array *); extern void clear_shallow_info(struct shallow_info *); extern void remove_nonexistent_theirs_shallow(struct shallow_info *); -extern void remove_nonexistent_ours_in_pack(struct shallow_info *, - struct packed_git *); extern void assign_shallow_commits_to_refs(struct shallow_info *info, uint32_t **used, int *ref_status); diff --git a/fetch-pack.c b/fetch-pack.c index a2d1b4ab28..9bc29cfce7 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -986,7 +986,6 @@ static void update_shallow(struct fetch_pack_args *args, return; remove_nonexistent_theirs_shallow(si); - /* XXX remove_nonexistent_ours_in_pack() */ if (!si->nr_ours && !si->nr_theirs) return; for (i = 0; i < nr_sought; i++) diff --git a/shallow.c b/shallow.c index c766fc3012..e483780d49 100644 --- a/shallow.c +++ b/shallow.c @@ -359,22 +359,6 @@ void remove_nonexistent_theirs_shallow(struct shallow_info *info) info->nr_theirs = dst; } -/* Step 5, remove non-existent ones in "ours" in the pack */ -void remove_nonexistent_ours_in_pack(struct shallow_info *info, - struct packed_git *p) -{ - unsigned char (*sha1)[20] = info->shallow->sha1; - int i, dst; - trace_printf_key(TRACE_KEY, "shallow: remove_nonexistent_ours_in_pack\n"); - for (i = dst = 0; i < info->nr_ours; i++) { - if (i != dst) - info->ours[dst] = info->ours[i]; - if (find_pack_entry_one(sha1[info->ours[i]], p)) - dst++; - } - info->nr_ours = dst; -} - define_commit_slab(ref_bitmap, uint32_t *); struct paint_info { -- cgit v1.2.3