Skip to content

Commit dc89c34

Browse files
gitstergitster
committed
Merge branch 'ds/sparse-index-ignored-files'
In cone mode, the sparse-index code path learned to remove ignored files (like build artifacts) outside the sparse cone, allowing the entire directory outside the sparse cone to be removed, which is especially useful when the sparse patterns change. * ds/sparse-index-ignored-files: sparse-checkout: clear tracked sparse dirs sparse-index: add SPARSE_INDEX_MEMORY_ONLY flag attr: be careful about sparse directories sparse-checkout: create helper methods sparse-index: use WRITE_TREE_MISSING_OK sparse-index: silently return when cache tree fails unpack-trees: fix nested sparse-dir search sparse-index: silently return when not using cone-mode patterns t7519: rewrite sparse index test
2 parents e78db9d + 55dfcf9 commit dc89c34

File tree

12 files changed

+312
-62
lines changed

12 files changed

+312
-62
lines changed

‎Documentation/git-sparse-checkout.txt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,16 @@ case-insensitive check. This corrects for case mismatched filenames in the
210210
'git sparse-checkout set' command to reflect the expected cone in the working
211211
directory.
212212

213+
When changing the sparse-checkout patterns in cone mode, Git will inspect each
214+
tracked directory that is not within the sparse-checkout cone to see if it
215+
contains any untracked files. If all of those files are ignored due to the
216+
`.gitignore` patterns, then the directory will be deleted. If any of the
217+
untracked files within that directory is not ignored, then no deletions will
218+
occur within that directory and a warning message will appear. If these files
219+
are important, then reset your sparse-checkout definition so they are included,
220+
use `git add` and `git commit` to store them, then remove any remaining files
221+
manually to ensure Git can behave optimally.
222+
213223

214224
SUBMODULES
215225
----------

‎attr.c‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include "utf8.h"
1515
#include "quote.h"
1616
#include "thread-utils.h"
17+
#include "dir.h"
1718

1819
const char git_attr__true[] = "(builtin)true";
1920
const char git_attr__false[] = "\0(builtin)false";
@@ -744,6 +745,20 @@ static struct attr_stack *read_attr_from_index(struct index_state *istate,
744745
if (!istate)
745746
return NULL;
746747

748+
/*
749+
* The .gitattributes file only applies to files within its
750+
* parent directory. In the case of cone-mode sparse-checkout,
751+
* the .gitattributes file is sparse if and only if all paths
752+
* within that directory are also sparse. Thus, don't load the
753+
* .gitattributes file since it will not matter.
754+
*
755+
* In the case of a sparse index, it is critical that we don't go
756+
* looking for a .gitattributes file, as doing so would cause the
757+
* index to expand.
758+
*/
759+
if (!path_in_cone_mode_sparse_checkout(path, istate))
760+
return NULL;
761+
747762
buf = read_blob_data_from_index(istate, path, NULL);
748763
if (!buf)
749764
return NULL;

‎builtin/add.c‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -190,21 +190,16 @@ static int refresh(int verbose, const struct pathspec *pathspec)
190190
struct string_list only_match_skip_worktree = STRING_LIST_INIT_NODUP;
191191
int flags = REFRESH_IGNORE_SKIP_WORKTREE |
192192
(verbose ? REFRESH_IN_PORCELAIN : REFRESH_QUIET);
193-
struct pattern_list pl = { 0 };
194-
int sparse_checkout_enabled = !get_sparse_checkout_patterns(&pl);
195193

196194
seen = xcalloc(pathspec->nr, 1);
197195
refresh_index(&the_index, flags, pathspec, seen,
198196
_("Unstaged changes after refreshing the index:"));
199197
for (i = 0; i < pathspec->nr; i++) {
200198
if (!seen[i]) {
201199
const char *path = pathspec->items[i].original;
202-
int dtype = DT_REG;
203200

204201
if (matches_skip_worktree(pathspec, i, &skip_worktree_seen) ||
205-
(sparse_checkout_enabled &&
206-
!path_matches_pattern_list(path, strlen(path), NULL,
207-
&dtype, &pl, &the_index))) {
202+
!path_in_sparse_checkout(path, &the_index)) {
208203
string_list_append(&only_match_skip_worktree,
209204
pathspec->items[i].original);
210205
} else {

‎builtin/sparse-checkout.c‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,98 @@ static int sparse_checkout_list(int argc, const char **argv)
100100
return 0;
101101
}
102102

103+
static void clean_tracked_sparse_directories(struct repository *r)
104+
{
105+
int i, was_full = 0;
106+
struct strbuf path = STRBUF_INIT;
107+
size_t pathlen;
108+
struct string_list_item *item;
109+
struct string_list sparse_dirs = STRING_LIST_INIT_DUP;
110+
111+
/*
112+
* If we are not using cone mode patterns, then we cannot
113+
* delete directories outside of the sparse cone.
114+
*/
115+
if (!r || !r->index || !r->worktree)
116+
return;
117+
if (init_sparse_checkout_patterns(r->index) ||
118+
!r->index->sparse_checkout_patterns->use_cone_patterns)
119+
return;
120+
121+
/*
122+
* Use the sparse index as a data structure to assist finding
123+
* directories that are safe to delete. This conversion to a
124+
* sparse index will not delete directories that contain
125+
* conflicted entries or submodules.
126+
*/
127+
if (!r->index->sparse_index) {
128+
/*
129+
* If something, such as a merge conflict or other concern,
130+
* prevents us from converting to a sparse index, then do
131+
* not try deleting files.
132+
*/
133+
if (convert_to_sparse(r->index, SPARSE_INDEX_MEMORY_ONLY))
134+
return;
135+
was_full = 1;
136+
}
137+
138+
strbuf_addstr(&path, r->worktree);
139+
strbuf_complete(&path, '/');
140+
pathlen = path.len;
141+
142+
/*
143+
* Collect directories that have gone out of scope but also
144+
* exist on disk, so there is some work to be done. We need to
145+
* store the entries in a list before exploring, since that might
146+
* expand the sparse-index again.
147+
*/
148+
for (i = 0; i < r->index->cache_nr; i++) {
149+
struct cache_entry *ce = r->index->cache[i];
150+
151+
if (S_ISSPARSEDIR(ce->ce_mode) &&
152+
repo_file_exists(r, ce->name))
153+
string_list_append(&sparse_dirs, ce->name);
154+
}
155+
156+
for_each_string_list_item(item, &sparse_dirs) {
157+
struct dir_struct dir = DIR_INIT;
158+
struct pathspec p = { 0 };
159+
struct strvec s = STRVEC_INIT;
160+
161+
strbuf_setlen(&path, pathlen);
162+
strbuf_addstr(&path, item->string);
163+
164+
dir.flags |= DIR_SHOW_IGNORED_TOO;
165+
166+
setup_standard_excludes(&dir);
167+
strvec_push(&s, path.buf);
168+
169+
parse_pathspec(&p, PATHSPEC_GLOB, 0, NULL, s.v);
170+
fill_directory(&dir, r->index, &p);
171+
172+
if (dir.nr) {
173+
warning(_("directory '%s' contains untracked files,"
174+
" but is not in the sparse-checkout cone"),
175+
item->string);
176+
} else if (remove_dir_recursively(&path, 0)) {
177+
/*
178+
* Removal is "best effort". If something blocks
179+
* the deletion, then continue with a warning.
180+
*/
181+
warning(_("failed to remove directory '%s'"),
182+
item->string);
183+
}
184+
185+
dir_clear(&dir);
186+
}
187+
188+
string_list_clear(&sparse_dirs, 0);
189+
strbuf_release(&path);
190+
191+
if (was_full)
192+
ensure_full_index(r->index);
193+
}
194+
103195
static int update_working_directory(struct pattern_list *pl)
104196
{
105197
enum update_sparsity_result result;
@@ -141,6 +233,8 @@ static int update_working_directory(struct pattern_list *pl)
141233
else
142234
rollback_lock_file(&lock_file);
143235

236+
clean_tracked_sparse_directories(r);
237+
144238
r->index->sparse_checkout_patterns = NULL;
145239
return result;
146240
}

‎dir.c‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1439,6 +1439,58 @@ enum pattern_match_result path_matches_pattern_list(
14391439
return result;
14401440
}
14411441

1442+
int init_sparse_checkout_patterns(struct index_state *istate)
1443+
{
1444+
if (!core_apply_sparse_checkout)
1445+
return 1;
1446+
if (istate->sparse_checkout_patterns)
1447+
return 0;
1448+
1449+
CALLOC_ARRAY(istate->sparse_checkout_patterns, 1);
1450+
1451+
if (get_sparse_checkout_patterns(istate->sparse_checkout_patterns) < 0) {
1452+
FREE_AND_NULL(istate->sparse_checkout_patterns);
1453+
return -1;
1454+
}
1455+
1456+
return 0;
1457+
}
1458+
1459+
static int path_in_sparse_checkout_1(const char *path,
1460+
struct index_state *istate,
1461+
int require_cone_mode)
1462+
{
1463+
const char *base;
1464+
int dtype = DT_REG;
1465+
1466+
/*
1467+
* We default to accepting a path if there are no patterns or
1468+
* they are of the wrong type.
1469+
*/
1470+
if (init_sparse_checkout_patterns(istate) ||
1471+
(require_cone_mode &&
1472+
!istate->sparse_checkout_patterns->use_cone_patterns))
1473+
return 1;
1474+
1475+
base = strrchr(path, '/');
1476+
return path_matches_pattern_list(path, strlen(path), base ? base + 1 : path,
1477+
&dtype,
1478+
istate->sparse_checkout_patterns,
1479+
istate) > 0;
1480+
}
1481+
1482+
int path_in_sparse_checkout(const char *path,
1483+
struct index_state *istate)
1484+
{
1485+
return path_in_sparse_checkout_1(path, istate, 0);
1486+
}
1487+
1488+
int path_in_cone_mode_sparse_checkout(const char *path,
1489+
struct index_state *istate)
1490+
{
1491+
return path_in_sparse_checkout_1(path, istate, 1);
1492+
}
1493+
14421494
static struct path_pattern *last_matching_pattern_from_lists(
14431495
struct dir_struct *dir, struct index_state *istate,
14441496
const char *pathname, int pathlen,

‎dir.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -394,6 +394,14 @@ enum pattern_match_result path_matches_pattern_list(const char *pathname,
394394
const char *basename, int *dtype,
395395
struct pattern_list *pl,
396396
struct index_state *istate);
397+
398+
int init_sparse_checkout_patterns(struct index_state *state);
399+
400+
int path_in_sparse_checkout(const char *path,
401+
struct index_state *istate);
402+
int path_in_cone_mode_sparse_checkout(const char *path,
403+
struct index_state *istate);
404+
397405
struct dir_entry *dir_add_ignored(struct dir_struct *dir,
398406
struct index_state *istate,
399407
const char *pathname, int len);

‎read-cache.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3069,7 +3069,7 @@ static int do_write_locked_index(struct index_state *istate, struct lock_file *l
30693069
int ret;
30703070
int was_full = !istate->sparse_index;
30713071

3072-
ret = convert_to_sparse(istate);
3072+
ret = convert_to_sparse(istate, 0);
30733073

30743074
if (ret) {
30753075
warning(_("failed to convert to a sparse-index"));
@@ -3182,7 +3182,7 @@ static int write_shared_index(struct index_state *istate,
31823182
int ret, was_full = !istate->sparse_index;
31833183

31843184
move_cache_to_base_index(istate);
3185-
convert_to_sparse(istate);
3185+
convert_to_sparse(istate, 0);
31863186

31873187
trace2_region_enter_printf("index", "shared/do_write_index",
31883188
the_repository, "%s", get_tempfile_path(*temp));

‎sparse-index.c‎

Lines changed: 43 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -33,19 +33,14 @@ static int convert_to_sparse_rec(struct index_state *istate,
3333
{
3434
int i, can_convert = 1;
3535
int start_converted = num_converted;
36-
enum pattern_match_result match;
37-
int dtype = DT_UNKNOWN;
3836
struct strbuf child_path = STRBUF_INIT;
39-
struct pattern_list *pl = istate->sparse_checkout_patterns;
4037

4138
/*
4239
* Is the current path outside of the sparse cone?
4340
* Then check if the region can be replaced by a sparse
4441
* directory entry (everything is sparse and merged).
4542
*/
46-
match = path_matches_pattern_list(ct_path, ct_pathlen,
47-
NULL, &dtype, pl, istate);
48-
if (match != NOT_MATCHED)
43+
if (path_in_sparse_checkout(ct_path, istate))
4944
can_convert = 0;
5045

5146
for (i = start; can_convert && i < end; i++) {
@@ -127,41 +122,51 @@ static int index_has_unmerged_entries(struct index_state *istate)
127122
return 0;
128123
}
129124

130-
int convert_to_sparse(struct index_state *istate)
125+
int convert_to_sparse(struct index_state *istate, int flags)
131126
{
132127
int test_env;
133-
if (istate->split_index || istate->sparse_index ||
128+
if (istate->sparse_index || !istate->cache_nr ||
134129
!core_apply_sparse_checkout || !core_sparse_checkout_cone)
135130
return 0;
136131

137132
if (!istate->repo)
138133
istate->repo = the_repository;
139134

140-
/*
141-
* The GIT_TEST_SPARSE_INDEX environment variable triggers the
142-
* index.sparse config variable to be on.
143-
*/
144-
test_env = git_env_bool("GIT_TEST_SPARSE_INDEX", -1);
145-
if (test_env >= 0)
146-
set_sparse_index_config(istate->repo, test_env);
147-
148-
/*
149-
* Only convert to sparse if index.sparse is set.
150-
*/
151-
prepare_repo_settings(istate->repo);
152-
if (!istate->repo->settings.sparse_index)
153-
return 0;
135+
if (!(flags & SPARSE_INDEX_MEMORY_ONLY)) {
136+
/*
137+
* The sparse index is not (yet) integrated with a split index.
138+
*/
139+
if (istate->split_index)
140+
return 0;
141+
/*
142+
* The GIT_TEST_SPARSE_INDEX environment variable triggers the
143+
* index.sparse config variable to be on.
144+
*/
145+
test_env = git_env_bool("GIT_TEST_SPARSE_INDEX", -1);
146+
if (test_env >= 0)
147+
set_sparse_index_config(istate->repo, test_env);
154148

155-
if (!istate->sparse_checkout_patterns) {
156-
istate->sparse_checkout_patterns = xcalloc(1, sizeof(struct pattern_list));
157-
if (get_sparse_checkout_patterns(istate->sparse_checkout_patterns) < 0)
149+
/*
150+
* Only convert to sparse if index.sparse is set.
151+
*/
152+
prepare_repo_settings(istate->repo);
153+
if (!istate->repo->settings.sparse_index)
158154
return 0;
159155
}
160156

161-
if (!istate->sparse_checkout_patterns->use_cone_patterns) {
162-
warning(_("attempting to use sparse-index without cone mode"));
163-
return -1;
164-
}
157+
if (init_sparse_checkout_patterns(istate))
158+
return 0;
159+
160+
/*
161+
* We need cone-mode patterns to use sparse-index. If a user edits
162+
* their sparse-checkout file manually, then we can detect during
163+
* parsing that they are not actually using cone-mode patterns and
164+
* hence we need to abort this conversion _without error_. Warnings
165+
* already exist in the pattern parsing to inform the user of their
166+
* bad patterns.
167+
*/
168+
if (!istate->sparse_checkout_patterns->use_cone_patterns)
169+
return 0;
165170

166171
/*
167172
* NEEDSWORK: If we have unmerged entries, then stay full.
@@ -172,10 +177,15 @@ int convert_to_sparse(struct index_state *istate)
172177

173178
/* Clear and recompute the cache-tree */
174179
cache_tree_free(&istate->cache_tree);
175-
if (cache_tree_update(istate, 0)) {
176-
warning(_("unable to update cache-tree, staying full"));
177-
return -1;
178-
}
180+
/*
181+
* Silently return if there is a problem with the cache tree update,
182+
* which might just be due to a conflict state in some entry.
183+
*
184+
* This might create new tree objects, so be sure to use
185+
* WRITE_TREE_MISSING_OK.
186+
*/
187+
if (cache_tree_update(istate, WRITE_TREE_MISSING_OK))
188+
return 0;
179189

180190
remove_fsmonitor(istate);
181191

‎sparse-index.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22
#define SPARSE_INDEX_H__
33

44
struct index_state;
5-
int convert_to_sparse(struct index_state *istate);
5+
#define SPARSE_INDEX_MEMORY_ONLY (1 << 0)
6+
int convert_to_sparse(struct index_state *istate, int flags);
67

78
/*
89
* Some places in the codebase expect to search for a specific path.

0 commit comments

Comments
 (0)