|
6 | 6 | import numpy as np |
7 | 7 | import pandas as pd |
8 | 8 |
|
| 9 | +def calc_emp_CDF(data): |
| 10 | + """ |
| 11 | + Create empirical CDF of arbitrary data |
| 12 | + |
| 13 | + Arguments: |
| 14 | + data (array_like) : array to calculate CDF on |
| 15 | + Returns: |
| 16 | + X (numpy.ndarray) : array of x values of CDF (sorted data) |
| 17 | + |
| 18 | + F (numpy.ndarray) : array of CDF values for each X value or cumulative |
| 19 | + exceedence probability, in [0,1]. |
| 20 | + """ |
| 21 | + n_bins = len(data) |
| 22 | + X = np.sort(data) |
| 23 | + F = np.array(range(n_bins))/float(n_bins) |
| 24 | + return X,F |
| 25 | + |
| 26 | +def Kolmogorov_Smirnov(uncond, cond, n_bins=10000): |
| 27 | + """ |
| 28 | + Calculate the Kolmogorov-Smirnov statistic between two datasets by first |
| 29 | + computing their empirical CDFs |
| 30 | + |
| 31 | + Arguments: |
| 32 | + uncond (array_like) : data for creating the unconditional CDF. |
| 33 | + cond (array_like) : data for creating the conditional CDF |
| 34 | + n_bins (int) : number of bins for both CDFs, note if n_bins > length |
| 35 | + of either dataset then CDF values are interpolated by numpy |
| 36 | + Returns: |
| 37 | + KS (float) : Kolmogorov-Smirnov statistic, i.e. absolute max distance |
| 38 | + between uncond and cond CDFs |
| 39 | + """ |
| 40 | + # create unconditional CDF (F_Uc) |
| 41 | + H,X = np.histogram(uncond, bins=n_bins, normed=True) |
| 42 | + dx = X[1] - X[0] |
| 43 | + F_Uc = np.cumsum(H)*dx |
| 44 | + # create conditional CDF (F_C) |
| 45 | + H,X = np.histogram(cond, bins=n_bins, normed=True) |
| 46 | + dx = X[1] - X[0] |
| 47 | + F_C = np.cumsum(H)*dx |
| 48 | + # Calc max absolulte divergence |
| 49 | + KS = np.max(np.abs(F_Uc - F_C)) |
| 50 | + return KS |
| 51 | + |
9 | 52 | def remove_all_optimization_sims_of_other_stage(work_directory, stage): |
10 | 53 | """ |
11 | 54 | Track number of simulation directories not tracked by a specific stage |
|
0 commit comments