forked from adamlaska/datatracker
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathserializer.py
More file actions
265 lines (240 loc) · 12.1 KB
/
serializer.py
File metadata and controls
265 lines (240 loc) · 12.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
# Copyright The IETF Trust 2018-2020, All Rights Reserved
# -*- coding: utf-8 -*-
import hashlib
import json
from django.core.cache import cache
from django.core.exceptions import ObjectDoesNotExist, FieldError
from django.core.serializers.json import Serializer
from django.http import HttpResponse
from django.utils.encoding import smart_text
from django.db.models import Field
from django.db.models.query import QuerySet
from django.db.models.signals import post_save, post_delete, m2m_changed
import debug # pyflakes:ignore
def filter_from_queryargs(request):
#@debug.trace
def fix_ranges(d):
for k,v in d.items():
if v.startswith("[") and v.endswith("]"):
d[k] = [ s for s in v[1:-1].split(",") if s ]
elif "," in v:
d[k] = [ s for s in v.split(",") if s ]
if k.endswith('__in') and not isinstance(d[k], list):
d[k] = [ d[k] ]
return d
def is_ascii(s):
return all(ord(c) < 128 for c in s)
# limit parameter keys to ascii.
params = dict( (k,v) for (k,v) in list(request.GET.items()) if is_ascii(k) )
filter = fix_ranges(dict([(k,params[k]) for k in list(params.keys()) if not k.startswith("not__")]))
exclude = fix_ranges(dict([(k[5:],params[k]) for k in list(params.keys()) if k.startswith("not__")]))
return filter, exclude
def unique_obj_name(obj):
"""Return a unique string representation for an object, based on app, class and ID
"""
app = obj._meta.app_label
model = obj.__class__.__name__.lower()
id = obj.pk
return "%s.%s[%s]" % (app,model,id)
def cached_get(key, calculate_value, timeout=None):
"""Try to get value from cache using key. If no value exists calculate
it by calling calculate_value. Timeout is defined in seconds."""
value = cache.get(key)
if value is None:
value = calculate_value()
cache.set(key, value, timeout)
return value
def model_top_level_cache_key(model):
return model.__module__ + '.' + model._meta.model.__name__
def clear_top_level_cache(sender, instance, *args, **kwargs):
cache.delete(model_top_level_cache_key(instance))
def clear_top_level_cache_m2m(sender, instance, action, reverse, model, *args, **kwargs):
# Purge cache for both models affected and the potentially custom 'through' model
cache.delete_many((
model_top_level_cache_key(instance),
model_top_level_cache_key(model),
model_top_level_cache_key(sender),
))
post_save.connect(clear_top_level_cache, dispatch_uid='clear_top_level_cache')
post_delete.connect(clear_top_level_cache, dispatch_uid='clear_top_level_cache')
m2m_changed.connect(clear_top_level_cache_m2m, dispatch_uid='clear_top_level_cache')
class AdminJsonSerializer(Serializer):
"""
Serializes a QuerySet to Json, with selectable object expansion.
The representation is different from that of the builtin Json
serializer in that there is no separate "model", "pk" and "fields"
entries for each object, instead only the "fields" dictionary is
serialized, and the model is the key of a top-level dictionary
entry which encloses the table serialization:
{
"app.model": {
"1": {
"foo": "1",
"bar": 42,
}
}
}
"""
internal_use_only = False
use_natural_keys = False
def serialize(self, queryset, **options):
qi = options.get('query_info', '').encode('utf-8')
if len(list(queryset)) == 1:
obj = queryset[0]
key = 'json:%s:%s' % (hashlib.md5(qi).hexdigest(), unique_obj_name(obj))
is_cached = cache.get(model_top_level_cache_key(obj)) is True
if is_cached:
value = cached_get(key, lambda: super(AdminJsonSerializer, self).serialize(queryset, **options))
else:
value = super(AdminJsonSerializer, self).serialize(queryset, **options)
cache.set(key, value)
cache.set(model_top_level_cache_key(obj), True)
return value
else:
return super(AdminJsonSerializer, self).serialize(queryset, **options)
def start_serialization(self):
super(AdminJsonSerializer, self).start_serialization()
self.json_kwargs.pop("expand", None)
self.json_kwargs.pop("query_info", None)
def get_dump_object(self, obj):
return self._current
def end_object(self, obj):
expansions = [ n.split("__")[0] for n in self.options.get('expand', []) if n ]
for name in expansions:
try:
field = getattr(obj, name)
#self._current["_"+name] = smart_text(field)
if not isinstance(field, Field):
options = self.options.copy()
options["expand"] = [ v[len(name)+2:] for v in options["expand"] if v.startswith(name+"__") ]
if hasattr(field, "all"):
if options["expand"]:
# If the following code (doing qs.select_related() is commented out it
# is because it has the unfortunate side effect of changing the json
# rendering of booleans, from 'true/false' to '1/0', but only for the
# models pulled in by select_related(). If that's acceptable, we can
# comment this in again later. (The problem is known, captured in
# Django issue #15040: https://code.djangoproject.com/ticket/15040
self._current[name] = dict([ (rel.pk, self.expand_related(rel, name)) for rel in field.all().select_related() ])
# self._current[name] = dict([ (rel.pk, self.expand_related(rel, name)) for rel in field.all() ])
else:
self._current[name] = dict([ (rel.pk, self.expand_related(rel, name)) for rel in field.all() ])
else:
if callable(field):
try:
field_value = field()
except Exception:
field_value = None
else:
field_value = field
if isinstance(field_value, QuerySet) or isinstance(field_value, list):
self._current[name] = dict([ (rel.pk, self.expand_related(rel, name)) for rel in field_value ])
else:
if hasattr(field_value, "_meta"):
self._current[name] = self.expand_related(field_value, name)
else:
self._current[name] = str(field_value)
except ObjectDoesNotExist:
pass
except AttributeError:
names = [f.name for f in obj._meta.get_fields()]
if name in names and hasattr(obj, '%s_set' % name):
related_objects = getattr(obj, '%s_set' % name).all()
if self.options["expand"]:
self._current[name] = dict([(rel.pk, self.expand_related(rel, name)) for rel in related_objects.select_related()])
else:
self._current[name] = dict([(rel.pk, self.expand_related(rel, name)) for rel in related_objects])
else:
raise FieldError("Cannot resolve keyword '%s' into field. "
"Choices are: %s" % (name, ", ".join(names)))
super(AdminJsonSerializer, self).end_object(obj)
def expand_related(self, related, name):
options = self.options.copy()
options["expand"] = [ v[len(name)+2:] for v in options["expand"] if v.startswith(name+"__") ]
bytes = self.__class__().serialize([ related ], **options)
data = json.loads(bytes)[0]
if 'password' in data:
del data['password']
return data
def handle_fk_field(self, obj, field):
try:
related = getattr(obj, field.name)
except ObjectDoesNotExist:
related = None
if related is not None:
if field.name in self.options.get('expand', []):
related = self.expand_related(related, field.name)
elif self.use_natural_keys and hasattr(related, 'natural_key'):
related = related.natural_key()
elif field.remote_field.field_name == related._meta.pk.name:
# Related to remote object via primary key
related = smart_text(related._get_pk_val(), strings_only=True)
else:
# Related to remote object via other field
related = smart_text(getattr(related, field.remote_field.field_name), strings_only=True)
self._current[field.name] = related
def handle_m2m_field(self, obj, field):
if field.remote_field.through._meta.auto_created:
if field.name in self.options.get('expand', []):
m2m_value = lambda value: self.expand_related(value, field.name)
elif self.use_natural_keys and hasattr(field.remote_field.to, 'natural_key'):
m2m_value = lambda value: value.natural_key()
else:
m2m_value = lambda value: smart_text(value._get_pk_val(), strings_only=True)
self._current[field.name] = [m2m_value(related)
for related in getattr(obj, field.name).iterator()]
class JsonExportMixin(object):
"""
Adds JSON export to a DetailView
"""
# def json_object(self, request, object_id, extra_context=None):
# "The json view for an object of this model."
# try:
# obj = self.get_queryset().get(pk=unquote(object_id))
# except self.model.DoesNotExist:
# # Don't raise Http404 just yet, because we haven't checked
# # permissions yet. We don't want an unauthenticated user to be able
# # to determine whether a given object exists.
# obj = None
#
# if obj is None:
# raise Http404(_('%(name)s object with primary key %(key)r does not exist.') % {'name': force_text(self.model._meta.verbose_name), 'key': escape(object_id)})
#
# content_type = 'application/json'
# return HttpResponse(serialize([ obj ], sort_keys=True, indent=3)[2:-2], content_type=content_type)
def json_view(self, request, filter={}, expand=[]):
qfilter, exclude = filter_from_queryargs(request)
for k in list(qfilter.keys()):
if k.startswith("_"):
del qfilter[k]
# discard a possible apikey, rather than using it as a queryset argument
if 'apikey' in qfilter:
del qfilter['apikey']
qfilter.update(filter)
filter = qfilter
key = request.GET.get("_key", "pk")
exp = [ e for e in request.GET.get("_expand", "").split(",") if e ]
for e in exp:
while True:
expand.append(e)
if not "__" in e:
break
e = e.rsplit("__", 1)[0]
#
expand = set(expand)
content_type = 'application/json'
query_info = "%s?%s" % (request.META["PATH_INFO"], request.META["QUERY_STRING"])
try:
qs = self.get_queryset().filter(**filter).exclude(**exclude)
except (FieldError, ValueError) as e:
return HttpResponse(json.dumps({"error": str(e)}, sort_keys=True, indent=3), content_type=content_type)
try:
if expand:
qs = qs.select_related()
serializer = AdminJsonSerializer()
items = [(getattr(o, key), serializer.serialize([o], expand=expand, query_info=query_info) ) for o in qs ]
qd = dict( ( k, json.loads(v)[0] ) for k,v in items )
except (FieldError, ValueError) as e:
return HttpResponse(json.dumps({"error": str(e)}, sort_keys=True, indent=3), content_type=content_type)
text = json.dumps({smart_text(self.model._meta): qd}, sort_keys=True, indent=3)
return HttpResponse(text, content_type=content_type)