Skip to content

Commit 28f66d8

Browse files
committed
move to our nicer solution with abstractions
1 parent c9e6370 commit 28f66d8

2 files changed

Lines changed: 84 additions & 57 deletions

File tree

‎sync.py‎

Lines changed: 38 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -5,35 +5,21 @@
55

66

77
def sync(source, dest):
8-
# Walk the source folder and build a dict of filenames and their hashes
9-
source_hashes = {}
10-
for folder, _, files in os.walk(source):
11-
for fn in files:
12-
source_hashes[hash_file(Path(folder) / fn)] = fn
13-
14-
seen = set() # Keep track of the files we've found in the target
15-
16-
# Walk the target folder and get the filenames and hashes
17-
for folder, _, files in os.walk(dest):
18-
for fn in files:
19-
dest_path = Path(folder) / fn
20-
dest_hash = hash_file(dest_path)
21-
seen.add(dest_hash)
22-
23-
# if there's a file in target that's not in source, delete it
24-
if dest_hash not in source_hashes:
25-
dest_path.remove()
8+
# imperative shell step 1, gather inputs
9+
source_hashes = read_paths_and_hashes(source)
10+
dest_hashes = read_paths_and_hashes(dest)
2611

27-
# if there's a file in target that has a different path in source,
28-
# move it to the correct path
29-
elif dest_hash in source_hashes and fn != source_hashes[dest_hash]:
30-
shutil.move(dest_path, Path(folder) / source_hashes[dest_hash])
12+
# step 2: call functional core
13+
actions = determine_actions(source_hashes, dest_hashes, source, dest)
3114

32-
# for every file that appears in source but not target, copy the file to
33-
# the target
34-
for source_hash, fn in source_hashes.items():
35-
if source_hash not in seen:
36-
shutil.copy(Path(source) / fn, Path(dest) / fn)
15+
# imperative shell step 3, apply outputs
16+
for action, *paths in actions:
17+
if action == "COPY":
18+
shutil.copyfile(*paths)
19+
if action == "MOVE":
20+
shutil.move(*paths)
21+
if action == "DELETE":
22+
os.remove(paths[0])
3723

3824

3925
BLOCKSIZE = 65536
@@ -47,3 +33,28 @@ def hash_file(path):
4733
hasher.update(buf)
4834
buf = file.read(BLOCKSIZE)
4935
return hasher.hexdigest()
36+
37+
38+
def read_paths_and_hashes(root):
39+
hashes = {}
40+
for folder, _, files in os.walk(root):
41+
for fn in files:
42+
hashes[hash_file(Path(folder) / fn)] = fn
43+
return hashes
44+
45+
46+
def determine_actions(source_hashes, dest_hashes, source_folder, dest_folder):
47+
for sha, filename in source_hashes.items():
48+
if sha not in dest_hashes:
49+
sourcepath = Path(source_folder) / filename
50+
destpath = Path(dest_folder) / filename
51+
yield "COPY", sourcepath, destpath
52+
53+
elif dest_hashes[sha] != filename:
54+
olddestpath = Path(dest_folder) / dest_hashes[sha]
55+
newdestpath = Path(dest_folder) / filename
56+
yield "MOVE", olddestpath, newdestpath
57+
58+
for sha, filename in dest_hashes.items():
59+
if sha not in source_hashes:
60+
yield "DELETE", dest_folder / filename

‎test_sync.py‎

Lines changed: 46 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,45 +1,61 @@
11
import tempfile
22
from pathlib import Path
33
import shutil
4-
from sync import sync
4+
from sync import sync, determine_actions
55

66

7-
def test_when_a_file_exists_in_the_source_but_not_the_destination():
8-
try:
9-
source = tempfile.mkdtemp()
10-
dest = tempfile.mkdtemp()
7+
class TestE2E:
8+
@staticmethod
9+
def test_when_a_file_exists_in_the_source_but_not_the_destination():
10+
try:
11+
source = tempfile.mkdtemp()
12+
dest = tempfile.mkdtemp()
1113

12-
content = "I am a very useful file"
13-
(Path(source) / "my-file").write_text(content)
14+
content = "I am a very useful file"
15+
(Path(source) / "my-file").write_text(content)
1416

15-
sync(source, dest)
17+
sync(source, dest)
1618

17-
expected_path = Path(dest) / "my-file"
18-
assert expected_path.exists()
19-
assert expected_path.read_text() == content
19+
expected_path = Path(dest) / "my-file"
20+
assert expected_path.exists()
21+
assert expected_path.read_text() == content
2022

21-
finally:
22-
shutil.rmtree(source)
23-
shutil.rmtree(dest)
23+
finally:
24+
shutil.rmtree(source)
25+
shutil.rmtree(dest)
2426

27+
@staticmethod
28+
def test_when_a_file_has_been_renamed_in_the_source():
29+
try:
30+
source = tempfile.mkdtemp()
31+
dest = tempfile.mkdtemp()
2532

26-
def test_when_a_file_has_been_renamed_in_the_source():
27-
try:
28-
source = tempfile.mkdtemp()
29-
dest = tempfile.mkdtemp()
33+
content = "I am a file that was renamed"
34+
source_path = Path(source) / "source-filename"
35+
old_dest_path = Path(dest) / "dest-filename"
36+
expected_dest_path = Path(dest) / "source-filename"
37+
source_path.write_text(content)
38+
old_dest_path.write_text(content)
3039

31-
content = "I am a file that was renamed"
32-
source_path = Path(source) / "source-filename"
33-
old_dest_path = Path(dest) / "dest-filename"
34-
expected_dest_path = Path(dest) / "source-filename"
35-
source_path.write_text(content)
36-
old_dest_path.write_text(content)
40+
sync(source, dest)
3741

38-
sync(source, dest)
42+
assert old_dest_path.exists() is False
43+
assert expected_dest_path.read_text() == content
3944

40-
assert old_dest_path.exists() is False
41-
assert expected_dest_path.read_text() == content
45+
finally:
46+
shutil.rmtree(source)
47+
shutil.rmtree(dest)
4248

43-
finally:
44-
shutil.rmtree(source)
45-
shutil.rmtree(dest)
49+
50+
def test_when_a_file_exists_in_the_source_but_not_the_destination():
51+
source_hashes = {"hash1": "fn1"}
52+
dest_hashes = {}
53+
actions = determine_actions(source_hashes, dest_hashes, Path("/src"), Path("/dst"))
54+
assert list(actions) == [("COPY", Path("/src/fn1"), Path("/dst/fn1"))]
55+
56+
57+
def test_when_a_file_has_been_renamed_in_the_source():
58+
source_hashes = {"hash1": "fn1"}
59+
dest_hashes = {"hash1": "fn2"}
60+
actions = determine_actions(source_hashes, dest_hashes, Path("/src"), Path("/dst"))
61+
assert list(actions) == [("MOVE", Path("/dst/fn2"), Path("/dst/fn1"))]

0 commit comments

Comments
 (0)