forked from leejet/stable-diffusion.cpp
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathremove_utf8_bom.py
More file actions
234 lines (203 loc) · 6.17 KB
/
Copy pathremove_utf8_bom.py
File metadata and controls
234 lines (203 loc) · 6.17 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
#!/usr/bin/env python3
"""Remove UTF-8 BOMs from files under a directory.
By default this scans the current working directory recursively and skips
repository areas that should not be touched by ordinary maintenance scripts.
Only files whose first three bytes are the UTF-8 BOM are rewritten.
"""
import argparse
import os
import shutil
import sys
import tempfile
from pathlib import Path
UTF8_BOM = b"\xef\xbb\xbf"
DEFAULT_EXCLUDED_DIR_NAMES = {
".git",
".hg",
".svn",
".mypy_cache",
".pytest_cache",
"__pycache__",
"test",
}
DEFAULT_EXCLUDED_DIR_PREFIXES = {
"build",
}
DEFAULT_EXCLUDED_REL_DIRS = {
"examples/server/frontend",
"ggml",
"models",
"src/vocab",
"thirdparty",
}
def rel_posix(path: Path, root: Path) -> str:
try:
return path.relative_to(root).as_posix()
except ValueError:
return path.as_posix()
def should_skip_dir(
path: Path,
root: Path,
excluded_rel_dirs: set[str],
excluded_names: set[str],
excluded_prefixes: set[str],
) -> bool:
rel = rel_posix(path, root)
return (
path.name in excluded_names
or rel in excluded_rel_dirs
or any(path.name.startswith(prefix) for prefix in excluded_prefixes)
)
def iter_files(
root: Path,
recursive: bool,
excluded_rel_dirs: set[str],
excluded_names: set[str],
excluded_prefixes: set[str],
follow_symlinks: bool,
):
if recursive:
for dirpath, dirnames, filenames in os.walk(root, followlinks=follow_symlinks):
current_dir = Path(dirpath)
dirnames[:] = [
name
for name in dirnames
if not should_skip_dir(
current_dir / name,
root,
excluded_rel_dirs,
excluded_names,
excluded_prefixes,
)
]
for filename in filenames:
path = current_dir / filename
if path.is_symlink() and not follow_symlinks:
continue
yield path
else:
for path in root.iterdir():
if path.is_file() and (follow_symlinks or not path.is_symlink()):
yield path
def has_utf8_bom(path: Path) -> bool:
with path.open("rb") as f:
return f.read(len(UTF8_BOM)) == UTF8_BOM
def strip_utf8_bom(path: Path) -> None:
tmp_path = None
try:
with path.open("rb") as src:
if src.read(len(UTF8_BOM)) != UTF8_BOM:
return
fd, tmp_name = tempfile.mkstemp(
prefix=f".{path.name}.",
suffix=".tmp",
dir=str(path.parent),
)
tmp_path = Path(tmp_name)
with os.fdopen(fd, "wb") as dst:
shutil.copyfileobj(src, dst, length=1024 * 1024)
shutil.copystat(path, tmp_path, follow_symlinks=False)
os.replace(tmp_path, path)
tmp_path = None
finally:
if tmp_path is not None:
try:
tmp_path.unlink()
except FileNotFoundError:
pass
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Scan files and convert UTF-8 BOM files to UTF-8 without BOM.",
)
parser.add_argument(
"root",
nargs="?",
default=".",
help="Directory to scan. Defaults to the current directory.",
)
parser.add_argument(
"-n",
"--dry-run",
action="store_true",
help="Only list files that would be converted.",
)
parser.add_argument(
"--no-recursive",
action="store_true",
help="Only scan files directly under root.",
)
parser.add_argument(
"--include-repo-excluded",
action="store_true",
help="Do not skip default repository excluded directories.",
)
parser.add_argument(
"--exclude-dir",
action="append",
default=[],
metavar="DIR",
help="Additional directory name or root-relative path to skip. Can be used multiple times.",
)
parser.add_argument(
"--follow-symlinks",
action="store_true",
help="Follow symlinked directories and files.",
)
parser.add_argument(
"-q",
"--quiet",
action="store_true",
help="Only print the final summary.",
)
return parser.parse_args()
def main() -> int:
args = parse_args()
root = Path(args.root).resolve()
if not root.is_dir():
print(f"error: not a directory: {root}", file=sys.stderr)
return 2
excluded_names = set()
excluded_rel_dirs = set()
excluded_prefixes = set()
if not args.include_repo_excluded:
excluded_names.update(DEFAULT_EXCLUDED_DIR_NAMES)
excluded_rel_dirs.update(DEFAULT_EXCLUDED_REL_DIRS)
excluded_prefixes.update(DEFAULT_EXCLUDED_DIR_PREFIXES)
for item in args.exclude_dir:
normalized = Path(item).as_posix().strip("/")
if "/" in normalized:
excluded_rel_dirs.add(normalized)
else:
excluded_names.add(normalized)
scanned = 0
converted = 0
errors = 0
for path in iter_files(
root=root,
recursive=not args.no_recursive,
excluded_rel_dirs=excluded_rel_dirs,
excluded_names=excluded_names,
excluded_prefixes=excluded_prefixes,
follow_symlinks=args.follow_symlinks,
):
scanned += 1
try:
if not has_utf8_bom(path):
continue
converted += 1
rel = rel_posix(path, root)
if args.dry_run:
if not args.quiet:
print(f"would convert: {rel}")
else:
strip_utf8_bom(path)
if not args.quiet:
print(f"converted: {rel}")
except OSError as exc:
errors += 1
print(f"error: {rel_posix(path, root)}: {exc}", file=sys.stderr)
action = "would convert" if args.dry_run else "converted"
print(f"scanned {scanned} file(s), {action} {converted}, errors {errors}")
return 1 if errors else 0
if __name__ == "__main__":
raise SystemExit(main())