Skip to content

Commit ea7e1cf

Browse files
JohnVolkJohnVolk
committed
add and improve util delete functions, improve optimizationresult table method, pending fix for issue 22
1 parent 885f313 commit ea7e1cf

3 files changed

Lines changed: 87 additions & 28 deletions

File tree

‎prms_python/optimizer.py‎

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -599,35 +599,61 @@ def result_table(self, freq='daily', top_n=5, latex=False):
599599
sim_names = [path.split(os.sep)[-1] for path in sim_dirs]
600600
meas_var = self._get_measured(self.stage)
601601
statvar_name = self._get_statvar_name(self.stage)
602+
orig_statvar = load_statvar(OPJ(self.input_dir,'statvar.dat'))\
603+
[statvar_name]
604+
602605
result_df = pd.DataFrame(columns=\
603606
['NSE','RMSE','PBIAS','COEF_DET','ABS(PBIAS)'])
607+
orig_results = pd.DataFrame(index=['orig_params'],\
608+
columns=['NSE','RMSE','PBIAS','COEF_DET'])
609+
# get datetime indices that overlap from measured and simulated
610+
sim_out = load_statvar(OPJ(sim_dirs[0], 'outputs', 'statvar.dat'))\
611+
[statvar_name]
612+
idx = meas_var.index.intersection(sim_out.index)
613+
meas_var = copy(meas_var[idx])
614+
#sim_out = sim_out[idx]
615+
orig_statvar = orig_statvar[idx]
616+
617+
if freq == 'monthly':
618+
meas_mo = meas_var.groupby(meas_var.index.month).mean()
619+
orig_mo = orig_statvar.groupby(orig_statvar.index.month).mean()
620+
604621
for i, sim in enumerate(sim_dirs):
605622
sim_out = load_statvar(OPJ(sim, 'outputs', 'statvar.dat'))\
606623
['{}'.format(statvar_name)]
607-
idx = meas_var.index.intersection(sim_out.index)
608-
meas_var = copy(meas_var[idx])
609624
sim_out = sim_out[idx]
610625
if freq == 'daily':
611626
result_df.loc[sim_names[i]] = [\
612627
nash_sutcliffe(meas_var, sim_out),\
613628
rmse(meas_var, sim_out),\
614629
percent_bias(meas_var,sim_out),\
615630
meas_var.corr(sim_out)**2,\
616-
np.abs(percent_bias(meas_var, sim_out)) ]
631+
np.abs(percent_bias(meas_var, sim_out)) ]
632+
orig_results.loc['orig_params'] = [\
633+
nash_sutcliffe(orig_statvar,meas_var),\
634+
rmse(orig_statvar,meas_var),\
635+
percent_bias(orig_statvar,meas_var),\
636+
orig_statvar.corr(meas_var)**2]
637+
617638
elif freq == 'monthly':
618-
meas_mo = meas_var.groupby(meas_var.index.month).mean()
619639
sim_out = sim_out.groupby(sim_out.index.month).mean()
620640
result_df.loc[sim_names[i]] = [\
621641
nash_sutcliffe(meas_mo, sim_out),\
622642
rmse(meas_mo, sim_out),\
623643
percent_bias(meas_mo, sim_out),\
624644
meas_mo.corr(sim_out)**2,\
625645
np.abs(percent_bias(meas_mo, sim_out)) ]
626-
646+
orig_results.loc['orig_params'] = [\
647+
nash_sutcliffe(orig_mo,meas_mo),\
648+
rmse(orig_mo,meas_mo),\
649+
percent_bias(orig_mo,meas_mo),\
650+
orig_mo.corr(meas_mo)**2]
651+
627652
sorted_result = result_df.sort_values(by=['NSE','RMSE','ABS(PBIAS)',\
628653
'COEF_DET'], ascending=[False,True,True,False])
629654
sorted_result.columns.name = '{} parameters'.format(self.stage)
630655
sorted_result = sorted_result[['NSE','RMSE','PBIAS','COEF_DET']]
656+
sorted_result = pd.concat([orig_results,sorted_result])
631657

632658
if latex: return sorted_result[:top_n].to_latex(escape=False)
633659
else: return sorted_result[:top_n]

‎prms_python/simulation.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ def run(self, prms_exec='prms', nproc=None):
3131

3232
pool = mp.Pool(processes=nproc)
3333
pool.map(_simulation_runner, self.series)
34+
pool.close()
3435

3536
return self
3637

@@ -48,7 +49,7 @@ def outputs_iter(self):
4849
4950
Would return something like
5051
51-
{'simulation_dir': 'path/to/sim/', 'statvar': <pandas.DataFrame>,
52+
{'simulation_dir': 'path/to/sim/', 'statvar': 'path/to/statvar',
5253
'data': <data.Data>, 'parameters': <parameters.Parameters>}
5354
5455
Returns:
@@ -61,8 +62,8 @@ def outputs_iter(self):
6162
{
6263
'simulation_dir': d,
6364
'statvar': OPJ(d, 'outputs', 'statvar.dat'),
64-
'data': Data(OPJ(d, 'inputs', 'data')),
65-
'parameters': Parameters(OPJ(d, 'inputs', 'parameters'))
65+
'data': OPJ(d, 'inputs', 'data'),
66+
'parameters': OPJ(d, 'inputs', 'parameters')
6667
}
6768
for d in dirs
6869
)
@@ -211,6 +212,9 @@ def run(self, prms_exec='prms'):
211212
prms_finished = poll >= 0
212213

213214
self.has_run = True
215+
# avoid too many files open error
216+
p.stdout.close()
217+
p.stderr.close()
214218

215219
if self.simulation_dir:
216220
os.mkdir('inputs')

‎prms_python/util.py‎

Lines changed: 49 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,53 @@
22
Utilities for working with PRMS data or other functionality that aren't
33
appropriate to put elsewhere at this time.
44
"""
5-
import os
5+
import os, shutil, json
66
import numpy as np
77
import pandas as pd
88

9-
def delete_out_files(work_directory, file_name=''):
9+
from .optimizer import OptimizationResult
10+
11+
def remove_all_optimization_sims_of_other_stage(work_directory,stage):
12+
"""
13+
14+
"""
15+
result = OptimizationResult(work_directory,stage=stage)
16+
17+
tracked_dirs = []
18+
19+
for f in result.metadata_json_paths[stage]:
20+
with open(f) as fh:
21+
json_data = json.load(fh)
22+
tracked_dirs.extend(json_data.get('sim_dirs'))
23+
24+
# track number of simulation directories not tracked by certain stage
25+
# and recursively delete them and their contents
26+
count = 0
27+
for d in os.listdir(result.working_dir):
28+
path = os.path.join(result.working_dir, d)
29+
if path in tracked_dirs:
30+
continue
31+
elif os.path.isdir(path):
32+
count+=1
33+
for dirpath, dirnames, filenames in os.walk(path, topdown=False):
34+
shutil.rmtree(dirpath, ignore_errors=True)
35+
36+
print('deleted {} simulations that were either not tracked by a JSON file'\
37+
.format(count) + ' or were not part of {} optimization stage'\
38+
.format(stage))
39+
40+
def delete_files(work_directory, file_name=''):
1041
"""
11-
Delete all output files of a certain name from PRMS simulations,
12-
can be useful since files can be large and may not be being used.
42+
Recursively delete all files of a certain name from PRMS simulations.
43+
Can be useful because files can be large and may not be being used.
1344
For example initial condition output files are often large and not
14-
always used, alternatively animation files may no longer be needed.
45+
always used, similarly animation, data, control, ... files may
46+
no longer be needed.
1547
1648
Arguments:
17-
work_directory (str): path to directory with simulation outputs
18-
two directories above where the actual prms_ic.out files exist.
19-
file_name (str) = Name of the PRMS output file(s) to be removed,
20-
default='' empty string- nothing will be deleted.
49+
work_directory (str): path to directory with simulations.
50+
file_name (str) = Name of the PRMS input or output file(s) to be
51+
removed, default='' empty string- nothing will be deleted.
2152
2253
e.g. if you have several simulation directories:
2354
@@ -27,23 +58,21 @@ def delete_out_files(work_directory, file_name=''):
2758
"test/results/intcp:-35.39_slope:0.39",
2859
"test/results/intcp:-20.91_slope:0.41"
2960
30-
each of these contains an '/outputs' folder with a prms_ic.out
31-
file that you would like to delete. In this case, delete all ic
32-
files like so:
61+
each of these contains an '/inputs' folder with a duplicate data
62+
file that you would like to delete. In this case, delete all
63+
data files like so:
3364
3465
>>> work_dir = 'test/results/'
35-
>>> delete_ic_files(work_dir, file_name='prms_ic.out')
66+
>>> delete_ic_files(work_dir, file_name='data')
3667
3768
Returns:
3869
None
3970
"""
40-
for fd in os.listdir(work_directory):
41-
if os.path.isdir(os.path.join(work_directory,fd)):
42-
try:
43-
os.remove(os.path.join(work_directory, fd, 'outputs', file_name))
44-
except: # file might not exist
45-
continue
46-
71+
for dirpath, dirnames, filenames in os.walk(work_directory, topdown=False):
72+
paths = (os.path.join(dirpath, filename) for filename in filenames\
73+
if filename == file_name)
74+
for path in paths:
75+
os.remove(path)
4776

4877
def load_statvar(statvar_file):
4978
"""

0 commit comments

Comments
 (0)