Skip to content

Commit b3e3eff

Browse files
murd0murd0
committed
Datetime support, now pd.Series rather than np
1 parent e30d700 commit b3e3eff

2 files changed

Lines changed: 84 additions & 32 deletions

File tree

‎codonPython/tests/tolerance_test.py‎

Lines changed: 51 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,22 @@
44
import pandas.util.testing as pdt
55
import pytest
66

7-
7+
## TODO migrate from numpy arrays to pandas series/dataframes
88
testdata = [
9-
np.array([1234, 1235, 1236, 1237, 1238, 1239, 1240, 1241, 1242]),
10-
np.array([1, 2, 3, 4, 5, 5.5, 6, 6.5, 7]),
9+
pd.Series([1234, 1235, 1236, 1237, 1238, 1239, 1240, 1241, 1242]),
10+
pd.Series([1, 2, 3, 4, 5, 5.5, 6, 6.5, 7]),
1111
]
1212

1313

14-
@pytest.mark.parametrize("t, y, to_exclude, poly_features, alpha, expected", [
14+
@pytest.mark.parametrize("t, y, to_exclude, poly_features, alpha, parse_dates, expected", [
1515
(
1616
*testdata,
1717
2,
1818
[1, 2],
1919
0.05,
20+
False,
2021
pd.DataFrame({
22+
"t" : [1241, 1242, 1241, 1242],
2123
'yhat_u': [
2224
8.11380197739608,
2325
9.051653693670929,
@@ -38,14 +40,51 @@
3840
5.907230032835563,
3941
],
4042
'polynomial': [1, 1, 2, 2]
41-
})
43+
}),
4244
),
4345
(
4446
*testdata,
4547
2,
4648
[3],
4749
0.05,
50+
False,
51+
pd.DataFrame({
52+
"t" : [1241, 1242],
53+
'yhat_u': [
54+
6.753927165005773,
55+
7.214574732953706,
56+
],
57+
'yobs': [6.5, 7.0],
58+
'yhat': [
59+
6.0000000000000036,
60+
5.571428571428576,
61+
],
62+
'yhat_l': [
63+
5.2460728349942345,
64+
3.928282409903445,
65+
],
66+
'polynomial': [3, 3]
67+
}),
68+
),
69+
(
70+
pd.Series([ # Check dates
71+
"2012-05-16",
72+
"2012-05-17",
73+
"2012-05-18",
74+
"2012-05-19",
75+
"2012-05-20",
76+
"2012-05-21",
77+
"2012-05-22",
78+
"2012-05-23",
79+
"2012-05-24",
80+
]),
81+
pd.Series([1, 2, 3, 4, 5, 5.5, 6, 6.5, 7]),
82+
2,
83+
[3],
84+
0.05,
85+
True,
4886
pd.DataFrame({
87+
"t" : ["2012-05-23", "2012-05-24"],
4988
'yhat_u': [
5089
6.753927165005773,
5190
7.214574732953706,
@@ -60,16 +99,17 @@
6099
3.928282409903445,
61100
],
62101
'polynomial': [3, 3]
63-
})
102+
}),
64103
),
65104
])
66-
def test_tolerance_checking_BAU(t, y, to_exclude, poly_features, alpha, expected):
105+
def test_tolerance_checking_BAU(t, y, to_exclude, poly_features, alpha, parse_dates, expected):
67106
obtained = check_tolerance(
68107
t,
69108
y,
70109
to_exclude=to_exclude,
71110
poly_features=poly_features,
72111
alpha=alpha,
112+
parse_dates=parse_dates,
73113
)
74114
pdt.assert_frame_equal(expected, obtained)
75115

@@ -120,16 +160,16 @@ def test_ValueErrors(t, y, to_exclude, poly_features, alpha):
120160
0.05,
121161
),
122162
(
123-
np.array([1234, 1235, 1236, 1237, 1238, 1239,
163+
pd.Series([1234, 1235, 1236, 1237, 1238, 1239,
124164
1240, 1241, np.nan]), # Missing t value
125-
np.array([1, 2, 3, 4, 5, 5.5, 6, 6.5, 7]),
165+
pd.Series([1, 2, 3, 4, 5, 5.5, 6, 6.5, 7]),
126166
2,
127167
[2],
128168
0.05,
129169
),
130170
(
131-
np.array([1234, 1235, 1236, 1237, 1238, 1239, 1240, 1241, 1242]),
132-
np.array([1, 2, 3, 4, 5, 5.5, 6, 6.5, np.nan]), # Missing y value
171+
pd.Series([1234, 1235, 1236, 1237, 1238, 1239, 1240, 1241, 1242]),
172+
pd.Series([1, 2, 3, 4, 5, 5.5, 6, 6.5, np.nan]), # Missing y value
133173
2,
134174
[2],
135175
0.05,

‎codonPython/tolerance.py‎

Lines changed: 33 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,21 @@
11
import numpy as np
22
import pandas as pd
3+
from datetime import datetime
34
from sklearn.preprocessing import StandardScaler, PolynomialFeatures
45
from sklearn.pipeline import make_pipeline
56
import statsmodels.api as sm
67
from statsmodels.sandbox.regression.predstd import wls_prediction_std
78

89

9-
def check_tolerance(t, y, to_exclude: int = 1, poly_features: list = [1, 2], alpha: float = 0.05, predict_all: bool = False) -> pd.DataFrame:
10+
def check_tolerance(t, y, to_exclude: int = 1, poly_features: list = [1, 2], alpha: float = 0.05, parse_dates: bool = False, predict_all: bool = False) -> pd.DataFrame:
1011
"""
1112
Check that some future values are within a weighted least squares confidence interval.
1213
1314
Parameters
1415
----------
15-
t : np.array
16+
t : pd.Series
1617
N explanatory time points of shape (N, 1).
17-
y : np.array
18+
y : pd.Series
1819
The corresponding response variable values to X, of shape (N, 1).
1920
to_exclude : int, default = 1
2021
How many of the last y values will have their tolerances checked.
@@ -24,6 +25,8 @@ def check_tolerance(t, y, to_exclude: int = 1, poly_features: list = [1, 2], alp
2425
a second degree polynomial to the data and return both sets of results.
2526
alpha : float, default = 0.05
2627
Alpha parameter for the weighted least squares confidence interval.
28+
parse_dates : bool, default = True
29+
Set to true to parse string dates in t
2730
predict_all : bool, default = False
2831
Set to true to show predictions for all points of the dataset.
2932
@@ -32,6 +35,7 @@ def check_tolerance(t, y, to_exclude: int = 1, poly_features: list = [1, 2], alp
3235
-------
3336
pd.DataFrame
3437
DataFrame containing:
38+
"t" : Value for t
3539
"yhat_u" : Upper condfidence interval for y
3640
"yobs" : Observed value for y
3741
"yhat" : Predicted value for y
@@ -42,15 +46,15 @@ def check_tolerance(t, y, to_exclude: int = 1, poly_features: list = [1, 2], alp
4246
Examples
4347
--------
4448
>>> check_tolerance(
45-
... t = np.array([1001,1002,1003,1004,1005,1006]),
46-
... y = np.array([2,3,4,4.5,5,5.1]),
49+
... t = pd.Series([1001,1002,1003,1004,1005,1006]),
50+
... y = pd.Series([2,3,4,4.5,5,5.1]),
4751
... to_exclude = 2,
4852
... )
49-
yhat_u yobs yhat yhat_l polynomial
50-
0 6.817413 5.0 5.500 4.182587 1
51-
1 7.952702 5.1 6.350 4.747298 1
52-
2 9.077182 5.0 4.875 0.672818 2
53-
3 13.252339 5.1 4.975 -3.302339 2
53+
t yhat_u yobs yhat yhat_l polynomial
54+
0 1005 6.817413 5.0 5.500 4.182587 1
55+
1 1006 7.952702 5.1 6.350 4.747298 1
56+
2 1005 9.077182 5.0 4.875 0.672818 2
57+
3 1006 13.252339 5.1 4.975 -3.302339 2
5458
"""
5559

5660
if not isinstance(poly_features, list):
@@ -66,21 +70,27 @@ def check_tolerance(t, y, to_exclude: int = 1, poly_features: list = [1, 2], alp
6670
"""The sample size for your model is smaller than 4. This will not produce a good
6771
model. Either reduce to_exclude or increase your sample size to continue."""
6872
)
69-
assert np.isfinite(y).all(), (
73+
assert y.notna().all(), (
7074
f"""Your sample contains missing or infinite values for y at locations
7175
{list(map(tuple, np.where(np.isnan(y))))}. Exclude these values to continue."""
7276
)
73-
assert np.isfinite(t).all(), (
77+
assert t.notna().all(), (
7478
f"""Your sample contains missing or infinite values for t at locations
7579
{list(map(tuple, np.where(np.isnan(t))))}. Exclude these values to continue."""
7680
)
7781

82+
# Convert date strings to numeric variables for the model
83+
if parse_dates:
84+
t_numeric = pd.to_datetime(t)
85+
t_numeric = (t_numeric - datetime(1970, 1, 1)) \
86+
.apply(lambda x: x.days)
7887

79-
# Sort data by X increasing
80-
idx = np.argsort(t)
88+
# Sort data by t increasing. t_ is for internal use.
89+
idx = np.argsort(t_numeric.values) if parse_dates else np.argsort(t.values)
90+
t_ = t_numeric[idx] if parse_dates else t[idx]
8191
t = t[idx]
8292
y = y[idx]
83-
93+
8494
results = pd.DataFrame()
8595
for degree in poly_features:
8696
transforms = make_pipeline(
@@ -89,13 +99,14 @@ def check_tolerance(t, y, to_exclude: int = 1, poly_features: list = [1, 2], alp
8999
)
90100

91101
# Fit transforms to training data only, apply to all data.
92-
fitted_transforms = transforms.fit(t[:-to_exclude].reshape(-1, 1))
93-
_t = fitted_transforms.transform(t.reshape(-1, 1))
102+
fitted_transforms = transforms.fit(t_[:-to_exclude].values.reshape(-1, 1))
103+
t_scaled = fitted_transforms.transform(t_.values.reshape(-1, 1))
94104

95-
t_train, y_train = _t[:-to_exclude, :], y[:-to_exclude]
96-
t_predict, y_predict = (
97-
_t if predict_all else _t[-to_exclude:, :],
98-
y if predict_all else y[-to_exclude:]
105+
t_train, y_train = t_scaled[:-to_exclude, :], y[:-to_exclude]
106+
t_predict, y_predict, t_orig = (
107+
t_scaled if predict_all else t_scaled[-to_exclude:, :],
108+
y if predict_all else y[-to_exclude:],
109+
t if predict_all else t[-to_exclude:],
99110
)
100111

101112
# Fit ordinary least squares model to the training data, then predict for the
@@ -109,6 +120,7 @@ def check_tolerance(t, y, to_exclude: int = 1, poly_features: list = [1, 2], alp
109120
# Store model results in master frame
110121
results = results.append(
111122
pd.DataFrame({
123+
"t" : t_orig,
112124
"yhat_u" : yhat_u,
113125
"yobs" : y_predict,
114126
"yhat" : yhat,

0 commit comments

Comments
 (0)