Mercurial > p > roundup > code
changeset 8088:1045425c23b2
refactor!: replace os.listdir() with os.scandir()
In many places we did a listdir() then a stat to see if it's a file or
directory. This change removes the need for the stat call. Also for
larger directories, scandir() is an iterator, so less memory use.
There is one remnant of listdir used in an error handler. That
requires a stat on each element in the directory, so there is no
benefit to using scandir() other than a slight memory saving on a
rarely used piece of code.
BREAKING CHANGE:
Python 2 requires installation of scandir pip package after this
commit.
| author | John Rouillard <rouilj@ieee.org> |
|---|---|
| date | Tue, 16 Jul 2024 01:05:49 -0400 |
| parents | 0cb81ee2e572 |
| children | f2b049b49fca |
| files | roundup/admin.py roundup/backends/blobfiles.py roundup/cgi/templating.py roundup/init.py roundup/instance.py roundup/scripts/roundup_gettext.py |
| diffstat | 6 files changed, 22 insertions(+), 19 deletions(-) [+] |
line wrap: on
line diff
--- a/roundup/admin.py Sun Jul 14 13:36:06 2024 -0400 +++ b/roundup/admin.py Tue Jul 16 01:05:49 2024 -0400 @@ -1226,8 +1226,9 @@ delimiter = ':' # import all the files - for file in os.listdir(import_dir): - classname, ext = os.path.splitext(file) + for dir_entry in os.scandir(import_dir): + filename = dir_entry.name + classname, ext = os.path.splitext(filename) # we only care about CSV files if ext != '.csv' or classname.endswith('-journals'): continue @@ -1235,7 +1236,7 @@ cl = self.get_class(classname) # ensure that the properties and the CSV file headings match - with open(os.path.join(import_dir, file), 'r') as f: + with open(os.path.join(import_dir, filename), 'r') as f: reader = csv.reader(f, colon_separated, lineterminator='\n') file_props = None maxid = 1
--- a/roundup/backends/blobfiles.py Sun Jul 14 13:36:06 2024 -0400 +++ b/roundup/backends/blobfiles.py Tue Jul 16 01:05:49 2024 -0400 @@ -27,12 +27,11 @@ if not os.path.exists(dir): return 0 num_files = 0 - for dir_entry in os.listdir(dir): - full_filename = os.path.join(dir, dir_entry) - if os.path.isfile(full_filename): + for dir_entry in os.scandir(dir): + if dir_entry.is_file(): num_files = num_files + 1 - elif os.path.isdir(full_filename): - num_files = num_files + files_in_dir(full_filename) + elif dir_entry.is_dir(): + num_files = num_files + files_in_dir(dir_entry.path) return num_files
--- a/roundup/cgi/templating.py Sun Jul 14 13:36:06 2024 -0400 +++ b/roundup/cgi/templating.py Tue Jul 16 01:05:49 2024 -0400 @@ -334,9 +334,10 @@ def precompile(self): """ Precompile templates in load directory by loading them """ - for filename in os.listdir(self.template_dir): + for dir_entry in os.scandir(self.template_dir): + filename = dir_entry.name # skip subdirs - if os.path.isdir(filename): + if dir_entry.is_dir(): continue # skip files without ".html" or ".xml" extension - .css, .js etc.
--- a/roundup/init.py Sun Jul 14 13:36:06 2024 -0400 +++ b/roundup/init.py Tue Jul 16 01:05:49 2024 -0400 @@ -44,7 +44,7 @@ """ # Prevent 'hidden' files (those starting with '.') from being considered. - names = [f for f in os.listdir(src) if not f.startswith('.')] + names = [f.name for f in os.scandir(src) if not f.name.startswith('.')] try: os.mkdir(dst) except OSError as error: @@ -111,7 +111,7 @@ config.save(config_ini_file) -def listTemplates(dir): +def listTemplates(directory): ''' List all the Roundup template directories in a given directory. Find all the dirs that contain a TEMPLATE-INFO.txt and parse it. @@ -119,9 +119,8 @@ Return a list of dicts of info about the templates. ''' ret = {} - for idir in os.listdir(dir): - idir = os.path.join(dir, idir) - ti = loadTemplateInfo(idir) + for dir_entry in os.scandir(directory): + ti = loadTemplateInfo(dir_entry.path) if ti: ret[ti['name']] = ti return ret
--- a/roundup/instance.py Sun Jul 14 13:36:06 2024 -0400 +++ b/roundup/instance.py Tue Jul 16 01:05:49 2024 -0400 @@ -188,7 +188,8 @@ dirpath = os.path.join(self.tracker_home, dirname) if os.path.isdir(dirpath): sys.path.insert(1, dirpath) - for name in os.listdir(dirpath): + for dir_entry in os.scandir(dirpath): + name = dir_entry.name if not name.endswith('.py'): continue env = {}
--- a/roundup/scripts/roundup_gettext.py Sun Jul 14 13:36:06 2024 -0400 +++ b/roundup/scripts/roundup_gettext.py Tue Jul 16 01:05:49 2024 -0400 @@ -64,6 +64,8 @@ TEMPLATE_FILE = "messages.pot" +# FIXME: html directory can be specified in config.ini. Should be +# a parameter, or config.ini should be loaded. def run(): # return unless command line arguments contain single directory path if (len(sys.argv) != 2) or (sys.argv[1] in ("-h", "--help")): @@ -76,9 +78,9 @@ if os.path.isdir(htmldir): # glob is not used because i want to match file names # without case sensitivity, and that is easier done this way. - htmlfiles = [filename for filename in os.listdir(htmldir) - if os.path.isfile(os.path.join(htmldir, filename)) - and filename.lower().endswith(".html")] + htmlfiles = [e.name for e in os.scandir(htmldir) + if e.is_file() + and e.name.lower().endswith(".html")] else: htmlfiles = [] # return if no html files found
