Mercurial > p > roundup > code
changeset 8224:5913ec1673c2
refactor: extract code as method from do_import
This started by trying to fix a resource error where the os.scandir
iterator was not closed for some reason (it should have been exhausted
and close). While the ResourceWarning is fixed with:
with os.scandir() as ...:
indented existing code
it breaks under python2. Trying a python2 friendly option:
dirscan = os.scandir()
...
if hasattr(dirscan,'close'):
dirscan.close()
doesn't fix the resource warning.
Using the 'with os.scandir()...' indented a nested block too far so I
extracted it into another method.
It is only this method that survives 8-(.
| author | John Rouillard <rouilj@ieee.org> |
|---|---|
| date | Wed, 18 Dec 2024 21:49:48 -0500 |
| parents | cfd8eb705c2f |
| children | 63a016f4b562 |
| files | roundup/admin.py |
| diffstat | 1 files changed, 42 insertions(+), 23 deletions(-) [+] |
line wrap: on
line diff
--- a/roundup/admin.py Wed Dec 18 20:25:35 2024 -0500 +++ b/roundup/admin.py Wed Dec 18 21:49:48 2024 -0500 @@ -1245,29 +1245,8 @@ cl = self.get_class(classname) - # ensure that the properties and the CSV file headings match - with open(os.path.join(import_dir, filename), 'r') as f: - reader = csv.reader(f, colon_separated, lineterminator='\n') - file_props = None - maxid = 1 - # loop through the file and create a node for each entry - for n, r in enumerate(reader): - if file_props is None: - file_props = r - continue - - if self.verbose: - sys.stdout.write('\rImporting %s - %s' % (classname, n)) - sys.stdout.flush() - - # do the import and figure the current highest nodeid - nodeid = cl.import_list(file_props, r) - if hasattr(cl, 'import_files') and import_files: - cl.import_files(import_dir, nodeid) - maxid = max(maxid, int(nodeid)) - - # (print to sys.stdout here to allow tests to squash it .. ugh) - print(file=sys.stdout) + maxid = self.import_class(dir_entry.path, colon_separated, cl, + import_dir, import_files) # import the journals with open(os.path.join(import_dir, classname + '-journals.csv'), 'r') as f: @@ -1283,6 +1262,46 @@ self.db_uncommitted = True return 0 + def import_class(self, filepath, csv_format_class, hyperdb_class, + import_dir, import_files): + """Import class given csv class filepath, csv_format_class and + hyperdb_class, directory for import, and boolean to import + files. + + Optionally import files as well if import_files is True + otherwise just import database data. + + Returns: maxid seen in csv file + """ + + maxid = 1 + + # ensure that the properties and the CSV file headings match + with open(filepath, 'r') as f: + reader = csv.reader(f, csv_format_class, lineterminator='\n') + file_props = None + # loop through the file and create a node for each entry + for n, r in enumerate(reader): + if file_props is None: + file_props = r + continue + + if self.verbose: + sys.stdout.write('\rImporting %s - %s' % ( + hyperdb_class.classname, n)) + sys.stdout.flush() + + # do the import and figure the current highest nodeid + nodeid = hyperdb_class.import_list(file_props, r) + if hasattr(hyperdb_class, 'import_files') and import_files: + hyperdb_class.import_files(import_dir, nodeid) + maxid = max(maxid, int(nodeid)) + + # (print to sys.stdout here to allow tests to squash it .. ugh) + print(file=sys.stdout) + + return maxid + def do_importtables(self, args): ''"""Usage: importtables export_dir This imports the database tables exported using exporttables.
