Mercurial > p > roundup > code
comparison roundup/admin.py @ 8224:5913ec1673c2
refactor: extract code as method from do_import
This started by trying to fix a resource error where the os.scandir
iterator was not closed for some reason (it should have been exhausted
and close). While the ResourceWarning is fixed with:
with os.scandir() as ...:
indented existing code
it breaks under python2. Trying a python2 friendly option:
dirscan = os.scandir()
...
if hasattr(dirscan,'close'):
dirscan.close()
doesn't fix the resource warning.
Using the 'with os.scandir()...' indented a nested block too far so I
extracted it into another method.
It is only this method that survives 8-(.
| author | John Rouillard <rouilj@ieee.org> |
|---|---|
| date | Wed, 18 Dec 2024 21:49:48 -0500 |
| parents | c12377fb4144 |
| children | 1189c742e4b3 |
comparison
equal
deleted
inserted
replaced
| 8223:cfd8eb705c2f | 8224:5913ec1673c2 |
|---|---|
| 1243 if ext != '.csv' or classname.endswith('-journals'): | 1243 if ext != '.csv' or classname.endswith('-journals'): |
| 1244 continue | 1244 continue |
| 1245 | 1245 |
| 1246 cl = self.get_class(classname) | 1246 cl = self.get_class(classname) |
| 1247 | 1247 |
| 1248 # ensure that the properties and the CSV file headings match | 1248 maxid = self.import_class(dir_entry.path, colon_separated, cl, |
| 1249 with open(os.path.join(import_dir, filename), 'r') as f: | 1249 import_dir, import_files) |
| 1250 reader = csv.reader(f, colon_separated, lineterminator='\n') | |
| 1251 file_props = None | |
| 1252 maxid = 1 | |
| 1253 # loop through the file and create a node for each entry | |
| 1254 for n, r in enumerate(reader): | |
| 1255 if file_props is None: | |
| 1256 file_props = r | |
| 1257 continue | |
| 1258 | |
| 1259 if self.verbose: | |
| 1260 sys.stdout.write('\rImporting %s - %s' % (classname, n)) | |
| 1261 sys.stdout.flush() | |
| 1262 | |
| 1263 # do the import and figure the current highest nodeid | |
| 1264 nodeid = cl.import_list(file_props, r) | |
| 1265 if hasattr(cl, 'import_files') and import_files: | |
| 1266 cl.import_files(import_dir, nodeid) | |
| 1267 maxid = max(maxid, int(nodeid)) | |
| 1268 | |
| 1269 # (print to sys.stdout here to allow tests to squash it .. ugh) | |
| 1270 print(file=sys.stdout) | |
| 1271 | 1250 |
| 1272 # import the journals | 1251 # import the journals |
| 1273 with open(os.path.join(import_dir, classname + '-journals.csv'), 'r') as f: | 1252 with open(os.path.join(import_dir, classname + '-journals.csv'), 'r') as f: |
| 1274 reader = csv.reader(f, colon_separated, lineterminator='\n') | 1253 reader = csv.reader(f, colon_separated, lineterminator='\n') |
| 1275 cl.import_journals(reader) | 1254 cl.import_journals(reader) |
| 1280 # set the id counter | 1259 # set the id counter |
| 1281 self.db.setid(classname, str(maxid + 1)) | 1260 self.db.setid(classname, str(maxid + 1)) |
| 1282 | 1261 |
| 1283 self.db_uncommitted = True | 1262 self.db_uncommitted = True |
| 1284 return 0 | 1263 return 0 |
| 1264 | |
| 1265 def import_class(self, filepath, csv_format_class, hyperdb_class, | |
| 1266 import_dir, import_files): | |
| 1267 """Import class given csv class filepath, csv_format_class and | |
| 1268 hyperdb_class, directory for import, and boolean to import | |
| 1269 files. | |
| 1270 | |
| 1271 Optionally import files as well if import_files is True | |
| 1272 otherwise just import database data. | |
| 1273 | |
| 1274 Returns: maxid seen in csv file | |
| 1275 """ | |
| 1276 | |
| 1277 maxid = 1 | |
| 1278 | |
| 1279 # ensure that the properties and the CSV file headings match | |
| 1280 with open(filepath, 'r') as f: | |
| 1281 reader = csv.reader(f, csv_format_class, lineterminator='\n') | |
| 1282 file_props = None | |
| 1283 # loop through the file and create a node for each entry | |
| 1284 for n, r in enumerate(reader): | |
| 1285 if file_props is None: | |
| 1286 file_props = r | |
| 1287 continue | |
| 1288 | |
| 1289 if self.verbose: | |
| 1290 sys.stdout.write('\rImporting %s - %s' % ( | |
| 1291 hyperdb_class.classname, n)) | |
| 1292 sys.stdout.flush() | |
| 1293 | |
| 1294 # do the import and figure the current highest nodeid | |
| 1295 nodeid = hyperdb_class.import_list(file_props, r) | |
| 1296 if hasattr(hyperdb_class, 'import_files') and import_files: | |
| 1297 hyperdb_class.import_files(import_dir, nodeid) | |
| 1298 maxid = max(maxid, int(nodeid)) | |
| 1299 | |
| 1300 # (print to sys.stdout here to allow tests to squash it .. ugh) | |
| 1301 print(file=sys.stdout) | |
| 1302 | |
| 1303 return maxid | |
| 1285 | 1304 |
| 1286 def do_importtables(self, args): | 1305 def do_importtables(self, args): |
| 1287 ''"""Usage: importtables export_dir | 1306 ''"""Usage: importtables export_dir |
| 1288 This imports the database tables exported using exporttables. | 1307 This imports the database tables exported using exporttables. |
| 1289 | 1308 |