changeset 8088:1045425c23b2

refactor!: replace os.listdir() with os.scandir() In many places we did a listdir() then a stat to see if it's a file or directory. This change removes the need for the stat call. Also for larger directories, scandir() is an iterator, so less memory use. There is one remnant of listdir used in an error handler. That requires a stat on each element in the directory, so there is no benefit to using scandir() other than a slight memory saving on a rarely used piece of code. BREAKING CHANGE: Python 2 requires installation of scandir pip package after this commit.
author John Rouillard <rouilj@ieee.org>
date Tue, 16 Jul 2024 01:05:49 -0400
parents 0cb81ee2e572
children f2b049b49fca
files roundup/admin.py roundup/backends/blobfiles.py roundup/cgi/templating.py roundup/init.py roundup/instance.py roundup/scripts/roundup_gettext.py
diffstat 6 files changed, 22 insertions(+), 19 deletions(-) [+]
line wrap: on
line diff
--- a/roundup/admin.py	Sun Jul 14 13:36:06 2024 -0400
+++ b/roundup/admin.py	Tue Jul 16 01:05:49 2024 -0400
@@ -1226,8 +1226,9 @@
             delimiter = ':'
 
         # import all the files
-        for file in os.listdir(import_dir):
-            classname, ext = os.path.splitext(file)
+        for dir_entry in os.scandir(import_dir):
+            filename = dir_entry.name
+            classname, ext = os.path.splitext(filename)
             # we only care about CSV files
             if ext != '.csv' or classname.endswith('-journals'):
                 continue
@@ -1235,7 +1236,7 @@
             cl = self.get_class(classname)
 
             # ensure that the properties and the CSV file headings match
-            with open(os.path.join(import_dir, file), 'r') as f:
+            with open(os.path.join(import_dir, filename), 'r') as f:
                 reader = csv.reader(f, colon_separated, lineterminator='\n')
                 file_props = None
                 maxid = 1
--- a/roundup/backends/blobfiles.py	Sun Jul 14 13:36:06 2024 -0400
+++ b/roundup/backends/blobfiles.py	Tue Jul 16 01:05:49 2024 -0400
@@ -27,12 +27,11 @@
     if not os.path.exists(dir):
         return 0
     num_files = 0
-    for dir_entry in os.listdir(dir):
-        full_filename = os.path.join(dir, dir_entry)
-        if os.path.isfile(full_filename):
+    for dir_entry in os.scandir(dir):
+        if dir_entry.is_file():
             num_files = num_files + 1
-        elif os.path.isdir(full_filename):
-            num_files = num_files + files_in_dir(full_filename)
+        elif dir_entry.is_dir():
+            num_files = num_files + files_in_dir(dir_entry.path)
     return num_files
 
 
--- a/roundup/cgi/templating.py	Sun Jul 14 13:36:06 2024 -0400
+++ b/roundup/cgi/templating.py	Tue Jul 16 01:05:49 2024 -0400
@@ -334,9 +334,10 @@
 
     def precompile(self):
         """ Precompile templates in load directory by loading them """
-        for filename in os.listdir(self.template_dir):
+        for dir_entry in os.scandir(self.template_dir):
+            filename = dir_entry.name
             # skip subdirs
-            if os.path.isdir(filename):
+            if dir_entry.is_dir():
                 continue
 
             # skip files without ".html" or ".xml" extension - .css, .js etc.
--- a/roundup/init.py	Sun Jul 14 13:36:06 2024 -0400
+++ b/roundup/init.py	Tue Jul 16 01:05:49 2024 -0400
@@ -44,7 +44,7 @@
     """
 
     # Prevent 'hidden' files (those starting with '.') from being considered.
-    names = [f for f in os.listdir(src) if not f.startswith('.')]
+    names = [f.name for f in os.scandir(src) if not f.name.startswith('.')]
     try:
         os.mkdir(dst)
     except OSError as error:
@@ -111,7 +111,7 @@
         config.save(config_ini_file)
 
 
-def listTemplates(dir):
+def listTemplates(directory):
     ''' List all the Roundup template directories in a given directory.
 
         Find all the dirs that contain a TEMPLATE-INFO.txt and parse it.
@@ -119,9 +119,8 @@
         Return a list of dicts of info about the templates.
     '''
     ret = {}
-    for idir in os.listdir(dir):
-        idir = os.path.join(dir, idir)
-        ti = loadTemplateInfo(idir)
+    for dir_entry in os.scandir(directory):
+        ti = loadTemplateInfo(dir_entry.path)
         if ti:
             ret[ti['name']] = ti
     return ret
--- a/roundup/instance.py	Sun Jul 14 13:36:06 2024 -0400
+++ b/roundup/instance.py	Tue Jul 16 01:05:49 2024 -0400
@@ -188,7 +188,8 @@
         dirpath = os.path.join(self.tracker_home, dirname)
         if os.path.isdir(dirpath):
             sys.path.insert(1, dirpath)
-            for name in os.listdir(dirpath):
+            for dir_entry in os.scandir(dirpath):
+                name = dir_entry.name
                 if not name.endswith('.py'):
                     continue
                 env = {}
--- a/roundup/scripts/roundup_gettext.py	Sun Jul 14 13:36:06 2024 -0400
+++ b/roundup/scripts/roundup_gettext.py	Tue Jul 16 01:05:49 2024 -0400
@@ -64,6 +64,8 @@
 TEMPLATE_FILE = "messages.pot"
 
 
+# FIXME: html directory can be specified in config.ini. Should be
+#        a parameter, or config.ini should be loaded.
 def run():
     # return unless command line arguments contain single directory path
     if (len(sys.argv) != 2) or (sys.argv[1] in ("-h", "--help")):
@@ -76,9 +78,9 @@
     if os.path.isdir(htmldir):
         # glob is not used because i want to match file names
         # without case sensitivity, and that is easier done this way.
-        htmlfiles = [filename for filename in os.listdir(htmldir)
-                     if os.path.isfile(os.path.join(htmldir, filename))
-                     and filename.lower().endswith(".html")]
+        htmlfiles = [e.name for e in os.scandir(htmldir)
+                     if e.is_file()
+                     and e.name.lower().endswith(".html")]
     else:
         htmlfiles = []
     # return if no html files found

Roundup Issue Tracker: http://roundup-tracker.org/