-
-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathfile_cache.py
More file actions
147 lines (118 loc) · 3.98 KB
/
Copy pathfile_cache.py
File metadata and controls
147 lines (118 loc) · 3.98 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
# coding=utf-8
import datetime
import hashlib
import os
import pickle
from collections import OrderedDict
from functools import wraps
__refresh__ = False
DEFAULT = './'
format_dict = {'Y': '%Y', 'm': "%Y-%m", 'd': "%Y-%m-%d",
'H': '%Y-%m-%d %H', 'M': '%Y-%m-%d %H:%M', 'S': '%Y-%m-%d %H:%M:%S'}
def get_cache_path(enable_cache: bool = False):
dt = datetime.datetime.today().strftime('%Y%m%d')
# __cache_path__ == f"{default}"
cache_path = os.path.join(DEFAULT, dt)
if not os.path.exists(cache_path) and enable_cache:
os.mkdir(cache_path)
return cache_path
def date_format(granularity: str):
if granularity in format_dict.keys():
return format_dict.get(granularity)
else:
raise ValueError(f'date_format not support: {granularity}')
pass
def prepare_args(func, arg, kwargs: dict, granularity: str = 'H', exploit_func_name: bool = True,
enable_cache: bool = False):
"""
:param func:
:param arg:
:param kwargs:
:param granularity:
:param exploit_func_name: cache file name whether include func name
:param enable_cache:
:return:
"""
time_format_dimension = date_format(granularity)
dt_str = datetime.datetime.now().strftime(time_format_dimension)
kwargs = OrderedDict(sorted(kwargs.items(), key=lambda t: t[0])) # sort kwargs to fix hashcode if sample input
func_name = func.__name__.__str__()
cls_obj = func.__qualname__ != func_name
if cls_obj:
obj = arg[0]
obj = obj.__name__ if hasattr(obj, '__name__') else obj.__class__.__name__
arg_tuple = tuple([obj] + list(map(str, arg[1:])))
else:
arg_tuple = arg
key = pickle.dumps([func_name, arg_tuple, kwargs, dt_str]) # get the unique key for the same input
if exploit_func_name:
name = f"{func_name}_{hashlib.sha1(key).hexdigest()}_{dt_str}" # create cache file name
else:
name = hashlib.sha1(key).hexdigest() # create cache file name
file_path = get_cache_path(enable_cache=enable_cache)
return file_path, name
def write(fg, res):
with open(fg, 'wb') as f:
pickle.dump(res, f)
def read(fg):
with open(fg, 'rb') as f:
res = pickle.load(f)
return res
def _cache(func, arg, kwargs, granularity='H', enable_cache: bool = False, exploit_func=True):
"""
:param func:
:param arg:
:param kwargs:
:param granularity:
:param enable_cache:
:param exploit_func: cache file name whether include func name
:return:
"""
if enable_cache:
file_path, name = prepare_args(func, arg, kwargs, granularity=granularity, exploit_func_name=exploit_func,
enable_cache=enable_cache)
fg = os.path.join(file_path, name)
if os.path.exists(fg):
return read(fg)
else:
res = func(*arg, **kwargs)
write(fg, res)
return res
else:
res = func(*arg, **kwargs)
return res
def file_cache(**deco_arg_dict):
# if callable(deco_arg_dict):
# @wraps(deco_arg_dict)
# def wrapped(*args, **kwargs):
# return _cache(deco_arg_dict, args, kwargs, granularity='d', enable_cache=False)
#
# return wrapped
# else:
def _deco(func):
@wraps(func)
def __deco(*args, **kwargs):
return _cache(func, args, kwargs, **deco_arg_dict)
return __deco
return _deco
if __name__ == '__main__':
@file_cache(enable_cache=True)
def test(a, b=2):
return a, b
class YGH(object):
@staticmethod
@file_cache(enable_cache=True)
def test(a, b=2):
return a, b
@classmethod
@file_cache(enable_cache=True)
def test2(cls, a, b=3):
return a, b
@file_cache(enable_cache=True)
def test3(self, a, b=3):
return a, b
print(test(1, b=3))
print(YGH.test(1, b=3))
print(YGH.test2(1, b=3))
print(YGH().test3(1, b=3))
pass