Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 67 additions & 6 deletions pre_commit/commands/gc.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
from __future__ import annotations

import collections
import heapq
import os.path
import sqlite3
from typing import Any

import pre_commit.constants as C
Expand Down Expand Up @@ -58,12 +61,45 @@ def _mark_used_repos(
))


def _gc(store: Store) -> int:
def _next_gc_gen(db: sqlite3.Connection) -> int:
gen, = db.execute(
'SELECT COALESCE(MAX(gc_gen), 0) FROM repos_used',
).fetchone()
return gen + 1


def _keep_unused(
keep: int,
used_repos: set[tuple[str, str]],
unused_repos: set[tuple[str, str]],
last_used: dict[tuple[str, str], tuple[int, int]],
) -> set[tuple[str, str]]:
n_used = collections.Counter(repo for repo, _ in used_repos)
by_repo: dict[str, list[tuple[str, str]]] = collections.defaultdict(list)
for key in unused_repos:
by_repo[key[0]].append(key)

ret = set()
for repo, keys in by_repo.items():
n = keep - n_used[repo]
if n > 0:
ret.update(heapq.nlargest(n, keys, key=last_used.__getitem__))
return ret


def _gc(store: Store, keep: int) -> int:
with store.exclusive_lock(), store.connect() as db:
store._create_configs_table(db)

repos = db.execute('SELECT repo, ref, path FROM repos').fetchall()
all_repos = {(repo, ref): path for repo, ref, path in repos}
store._create_repos_used_table(db)

repos = db.execute(
'SELECT repos.repo, repos.ref, path, repos.rowid, '
' COALESCE(gc_gen, 0) '
'FROM repos LEFT JOIN repos_used USING (repo, ref)',
).fetchall()
all_repos = {
(repo, ref): path for repo, ref, path, _, _ in repos
}
unused_repos = set(all_repos)

configs_rows = db.execute('SELECT path FROM configs').fetchall()
Expand All @@ -83,16 +119,41 @@ def _gc(store: Store) -> int:
paths = [(path,) for path in dead_configs]
db.executemany('DELETE FROM configs WHERE path = ?', paths)

# track recency across `gc` runs
used_repos = set(all_repos) - unused_repos
gc_gen = _next_gc_gen(db)
db.executemany(
'INSERT OR REPLACE INTO repos_used (repo, ref, gc_gen) '
'VALUES (?, ?, ?)',
[(repo, ref, gc_gen) for repo, ref in sorted(used_repos)],
)

if keep:
last_used = {
(repo, ref): (gc_gen, rowid)
for repo, ref, _, rowid, gc_gen in repos
}
unused_repos -= _keep_unused(
keep, used_repos, unused_repos, last_used,
)

db.executemany(
'DELETE FROM repos WHERE repo = ? and ref = ?',
sorted(unused_repos),
)
db.execute(
'DELETE FROM repos_used WHERE NOT EXISTS ('
' SELECT 1 FROM repos'
' WHERE repos.repo = repos_used.repo'
' AND repos.ref = repos_used.ref'
')',
)
for k in unused_repos:
rmtree(all_repos[k])

return len(unused_repos)


def gc(store: Store) -> int:
output.write_line(f'{_gc(store)} repo(s) removed.')
def gc(store: Store, keep: int = 0) -> int:
output.write_line(f'{_gc(store, keep)} repo(s) removed.')
return 0
18 changes: 16 additions & 2 deletions pre_commit/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,13 @@
}


def _nonnegative_int(s: str) -> int:
ret = int(s)
if ret < 0:
raise argparse.ArgumentTypeError('expected a non-negative integer')
return ret


def _add_config_option(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
'-c', '--config', default=C.CONFIG_FILE,
Expand Down Expand Up @@ -244,7 +251,14 @@ def _add_cmd(name: str, *, help: str) -> argparse.ArgumentParser:

_add_cmd('clean', help='Clean out pre-commit files.')

_add_cmd('gc', help='Clean unused cached repos.')
gc_parser = _add_cmd('gc', help='Clean unused cached repos.')
gc_parser.add_argument(
'--keep', type=_nonnegative_int, default=0, metavar='N',
help=(
'Keep up to N recently used revisions of each repo. '
'(default %(default)s).'
),
)

hazmat_parser = _add_cmd(
'hazmat', help='Composable tools for rare use in hook `entry`.',
Expand Down Expand Up @@ -394,7 +408,7 @@ def _add_cmd(name: str, *, help: str) -> argparse.ArgumentParser:
elif args.command == 'clean':
return clean(store)
elif args.command == 'gc':
return gc(store)
return gc(store, args.keep)
elif args.command == 'hazmat':
return hazmat.impl(args)
elif args.command == 'hook-impl':
Expand Down
11 changes: 11 additions & 0 deletions pre_commit/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ def __init__(self, directory: str | None = None) -> None:
');',
)
self._create_configs_table(db)
self._create_repos_used_table(db)

# Atomic file move
os.replace(tmpfile, self.db_path)
Expand Down Expand Up @@ -222,6 +223,16 @@ def _create_configs_table(self, db: sqlite3.Connection) -> None:
');',
)

def _create_repos_used_table(self, db: sqlite3.Connection) -> None:
db.executescript(
'CREATE TABLE IF NOT EXISTS repos_used ('
' repo TEXT NOT NULL,'
' ref TEXT NOT NULL,'
' gc_gen INTEGER NOT NULL,'
' PRIMARY KEY (repo, ref)'
');',
)

def mark_config_used(self, path: str) -> None:
if self.readonly: # pragma: win32 no cover
return
Expand Down
124 changes: 124 additions & 0 deletions tests/commands/gc_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,33 @@ def _config_count(store):
return db.execute('SELECT COUNT(1) FROM configs').fetchone()[0]


def _repo_refs(store):
with store.connect() as db:
return {ref for ref, in db.execute('SELECT ref FROM repos')}


def _repos_used_count(store):
with store.connect() as db:
return db.execute('SELECT COUNT(1) FROM repos_used').fetchone()[0]


def _install_revs(tempdir_factory, store, n):
path = make_repo(tempdir_factory, 'script_hooks_repo')
revs = [git.head_rev(path)]
for _ in range(n - 1):
git_commit(cwd=path)
revs.append(git.head_rev(path))

write_config('.', make_config_from_repo(path, rev=revs[0]))
store.mark_config_used(C.CONFIG_FILE)
assert not install_hooks(C.CONFIG_FILE, store)
for rev in revs[1:]:
with modify_config() as config:
config['repos'][0]['rev'] = rev
assert not install_hooks(C.CONFIG_FILE, store)
return revs


def _remove_config_assert_cleared(store, cap_out):
os.remove(C.CONFIG_FILE)
assert not gc(store)
Expand Down Expand Up @@ -173,3 +200,100 @@ def test_gc_pre_1_14_roll_forward(store, cap_out):

assert not gc(store)
assert cap_out.get() == '0 repo(s) removed.\n'


def test_gc_keep_retains_unreferenced_repos(
tempdir_factory, store, in_git_dir, cap_out,
):
old_rev, new_rev = _install_revs(tempdir_factory, store, 2)

assert _repo_count(store) == 2
# keep the unreferenced revision
assert not gc(store, keep=2)
assert _repo_refs(store) == {old_rev, new_rev}
assert cap_out.get().splitlines()[-1] == '0 repo(s) removed.'

# revisiting the old revision keeps the cache warm
with modify_config() as config:
config['repos'][0]['rev'] = old_rev
assert not gc(store, keep=2)
assert _repo_refs(store) == {old_rev, new_rev}

# lowering the limit removes the unreferenced revision
assert not gc(store, keep=1)
assert _repo_refs(store) == {old_rev}
assert _repos_used_count(store) == 1
assert cap_out.get().splitlines()[-1] == '1 repo(s) removed.'


def test_gc_keep_evicts_least_recently_used(
tempdir_factory, store, in_git_dir, cap_out,
):
revs = _install_revs(tempdir_factory, store, 3)

# mark revs[1], then revs[0] as used
for rev in reversed(revs[:-1]):
assert not gc(store, keep=3)
with modify_config() as config:
config['repos'][0]['rev'] = rev

assert _repo_count(store) == 3
assert not gc(store, keep=2)
# revs[2] was cloned last but used least recently
assert _repo_refs(store) == {revs[0], revs[1]}
assert cap_out.get().splitlines()[-1] == '1 repo(s) removed.'


def test_gc_keep_falls_back_to_clone_order(
tempdir_factory, store, in_git_dir, cap_out,
):
revs = _install_revs(tempdir_factory, store, 3)

assert _repo_count(store) == 3
assert not gc(store, keep=2)
assert _repo_refs(store) == {revs[1], revs[2]}
assert cap_out.get().splitlines()[-1] == '1 repo(s) removed.'


def test_gc_keep_does_not_evict_referenced_repos(
tempdir_factory, store, in_git_dir, cap_out,
):
path = make_repo(tempdir_factory, 'script_hooks_repo')
old_rev = git.head_rev(path)
git_commit(cwd=path)
new_rev = git.head_rev(path)

# reference both revisions
os.mkdir('other')
write_config('.', make_config_from_repo(path, rev=old_rev))
write_config('other', make_config_from_repo(path, rev=new_rev))
store.mark_config_used(C.CONFIG_FILE)
store.mark_config_used(os.path.join('other', C.CONFIG_FILE))
assert not install_hooks(C.CONFIG_FILE, store)
assert not install_hooks(os.path.join('other', C.CONFIG_FILE), store)

assert _repo_count(store) == 2
# referenced repos can exceed `keep`
assert not gc(store, keep=1)
assert _repo_refs(store) == {old_rev, new_rev}
assert cap_out.get().splitlines()[-1] == '0 repo(s) removed.'


def test_gc_keep_removes_stale_usage_rows(store, cap_out):
with store.connect() as db:
db.execute(
'INSERT INTO repos_used (repo, ref, gc_gen) VALUES (?, ?, ?)',
('repo', 'ref', 1),
)

assert not gc(store, keep=2)
assert _repos_used_count(store) == 0
assert cap_out.get() == '0 repo(s) removed.\n'


def test_gc_roll_forward_no_repos_used_table(store, cap_out):
with store.connect() as db: # simulate a store from an older version
db.executescript('DROP TABLE repos_used')

assert not gc(store, keep=2)
assert cap_out.get() == '0 repo(s) removed.\n'
11 changes: 11 additions & 0 deletions tests/main_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,17 @@ def test_all_cmds(command, mock_commands, mock_store_dir):
assert_only_one_mock_called(mock_commands)


def test_gc_keep_must_be_nonnegative(mock_commands):
with pytest.raises(SystemExit):
main.main(('gc', '--keep', '-1'))


def test_gc_keep(mock_commands, mock_store_dir):
main.main(('gc', '--keep', '1'))

assert mock_commands.gc.call_args.args[1] == 1


def test_hazmat(mock_store_dir):
with mock.patch.object(hazmat, 'impl') as mck:
main.main(('hazmat', 'cd', 'subdir', '--', 'cmd', '--', 'f1', 'f2'))
Expand Down
Loading