-
Notifications
You must be signed in to change notification settings - Fork 25
Expand file tree
/
Copy pathidsync.py
More file actions
311 lines (253 loc) · 13.1 KB
/
Copy pathidsync.py
File metadata and controls
311 lines (253 loc) · 13.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
import random
from typing import *
from objectbox.logger import logger
from objectbox.model.model import Model
from objectbox.model.entity import _Entity
from objectbox.model.properties import Property, Index, HnswIndex
from objectbox.model.iduid import IdUid
MODEL_PARSER_VERSION = 5
class IdSync:
"""
Synchronizes a model with the IDs from model JSON file.
After syncing, the model will have all IDs assigned.
The JSON file is written (from scratch) based on the model.
"""
def __init__(self, model: Model, model_json_filepath: str):
self.model = model
if len(model.entities) == 0:
raise ValueError("A valid model must have at least one entity")
self.model_filepath = model_json_filepath
self.model_json = None
self._assigned_uids: Set[int] = set()
self._load_model_json()
def _load_model_json(self):
import json
from os import path
if not path.exists(self.model_filepath):
logger.debug(f"Model file not found: {self.model_filepath}")
return
with open(self.model_filepath, "rt") as model_file:
self.model_json = json.load(model_file)
logger.debug(f"Syncing model with model file: {self.model_filepath}")
self._load_assigned_uids()
def _load_assigned_uids(self):
for entity_json in self.model_json["entities"]:
entity_uid = IdUid.from_str(entity_json["id"]).uid
if entity_uid in self._assigned_uids:
raise ValueError(f"An entity's UID {entity_uid} has already been used elsewhere")
self._assigned_uids.add(entity_uid)
for prop_json in entity_json["properties"]:
prop_uid = IdUid.from_str(prop_json["id"]).uid
if prop_uid in self._assigned_uids:
raise ValueError(f"A property's UID {prop_uid} has already been used elsewhere")
self._assigned_uids.add(prop_uid)
if "indexId" in prop_json:
index_uid = IdUid.from_str(prop_json["indexId"]).uid
if index_uid in self._assigned_uids:
raise ValueError(f"An index's UID {index_uid} has already been used elsewhere")
self._assigned_uids.add(index_uid)
def _save_model_json(self):
""" Replaces model JSON with the serialized model whose ID/UIDs are assigned. """
# model.validate_ids_assigned()
model_json = {
"_note1": "KEEP THIS FILE! Check it into a version control system (VCS) like git.",
"_note2": "ObjectBox manages crucial IDs for your object model. See docs for details.",
"_note3": "If you have VCS merge conflicts, you must resolve them according to ObjectBox docs.",
"modelVersionParserMinimum": MODEL_PARSER_VERSION,
"entities": [],
"lastEntityId": str(self.model.last_entity_iduid),
"lastIndexId": str(self.model.last_index_iduid)
}
# TODO lastRelationId
# TODO modelVersion
# TODO retiredEntityUids
# TODO retiredIndexUids
# TODO retiredPropertyUids
# TODO retiredRelationUids
# TODO version
for entity in self.model.entities:
entity_json = {
"id": str(entity._iduid),
"name": entity._name,
"lastPropertyId": str(entity._last_property_iduid),
"properties": []
}
for prop in entity._properties:
prop_json = {
"id": str(prop.iduid),
"name": prop.name,
"type": prop._ob_type,
}
if prop._flags != 0:
prop_json["flags"] = prop._flags
if prop.index is not None:
prop_json["indexId"] = str(prop.index.iduid)
entity_json["properties"].append(prop_json)
model_json["entities"].append(entity_json)
import json
with open(self.model_filepath, "w") as model_file:
model_file.write(json.dumps(model_json, indent=2)) # Pretty
# *** Sync ***
def _find_entity_json_by_uid(self, uid: int) -> Optional[Dict[str, Any]]:
""" Finds entity JSON by UID. """
if self.model_json is None:
return None
# TODO put entities in a dict (e.g. while/after loading) for faster lookup
for entity_json in self.model_json["entities"]:
if IdUid.from_str(entity_json["id"]).uid == uid:
return entity_json
return None
def _find_entity_json_by_name(self, entity_name: str) -> Optional[Dict[str, Any]]:
""" Finds entity JSON by name. """
if self.model_json is None:
return None
# TODO put entities in a dict (e.g. while/after loading) for faster lookup
for entity_json in self.model_json["entities"]:
if entity_json["name"] == entity_name:
return entity_json
return None
def _find_property_json_by_uid(self, entity_json: Dict[str, Any], uid: int) -> Optional[Dict[str, Any]]:
""" Finds entity property JSON by property UID. """
# TODO put properties in a multi-dict (e.g. while/after loading) for faster lookup
for prop_json in entity_json["properties"]:
if IdUid.from_str(prop_json["id"]).uid == uid:
return prop_json
return None
def _find_property_json_by_name(self, entity_json: Dict[str, Any], prop_name: str) -> Optional[Dict[str, Any]]:
""" Finds entity property JSON by property name. """
# TODO put properties in a multi-dict (e.g. while/after loading) for faster lookup
for prop_json in entity_json["properties"]:
if prop_json["name"] == prop_name:
return prop_json
return None
def _generate_uid(self) -> int:
while True:
generated_uid = random.getrandbits(63) + 1 # 0 would be invalid
if generated_uid not in self._assigned_uids:
break
self._assigned_uids.add(generated_uid)
return generated_uid
def _validate_uid_unassigned(self, uid: int):
""" Validates that a user supplied UID is not assigned for any other entity/property/index.
Raises a ValueError if the UID is already assigned elsewhere.
"""
if uid in self._assigned_uids:
raise ValueError(f"User supplied UID {uid} is already assigned elsewhere")
def _validate_matching_prop(self, entity: _Entity, prop: Property, prop_json: Dict[str, Any]):
""" Validates that the given property matches the JSON property. """
try:
# Don't check name equality as the property could be matched by UID (rename)
# if validate_name and prop.name != prop_json["name"]:
# raise ValueError(f"name {prop.name} != name {prop_json['name']} (in JSON)")
if prop._ob_type != prop_json["type"]:
raise ValueError(f"OBX type {prop._ob_type} != OBX type {prop_json['type']} (in JSON)")
json_flags = prop_json.get("flags", 0)
if prop._flags != json_flags:
raise ValueError(f"flags {prop._flags} != flags {json_flags} (in JSON)")
if prop.index is None and "indexId" in prop_json:
raise ValueError("property hasn't index, but index found in JSON")
elif prop.index is not None and "indexId" not in prop_json:
raise ValueError("property has index, but index not found in JSON")
except ValueError as error:
raise ValueError(f"Property {entity._name}.{prop.name} mismatches property found in JSON file: {error}")
def _sync_index(self, entity: _Entity, prop: Property, prop_json: Optional[Dict[str, Any]]) -> bool:
assert prop.index is not None
index = prop.index
write_json = False
# Fetch index ID/UID from JSON file
iduid_json = None
if (prop_json is not None) and ("indexId" in prop_json):
iduid_json = IdUid.from_str(prop_json["indexId"])
# User provided a UID not matching index's, make sure it's not assigned elsewhere
if index.has_uid() and (iduid_json is not None) and (index.uid != iduid_json.uid):
self._validate_uid_unassigned(index.uid)
# Generate UID only if not supplied by the user, and index isn't found in JSON
if not index.has_uid() and iduid_json is None:
index.iduid.uid = self._generate_uid()
if (iduid_json is not None) and (not index.has_uid() or index.iduid.uid == iduid_json.uid): # Load
index.iduid = IdUid.from_str(prop_json["indexId"])
else: # Assign new ID to new index
index.iduid = IdUid(self.model.last_index_iduid.id + 1, index.uid)
self.model.last_index_iduid = index.iduid
write_json = True
return write_json
def _sync_property(self, entity: _Entity, prop: Property, entity_json: Optional[Dict[str, Any]]) -> bool:
write_json = False
prop_json = None
if prop.has_uid():
if entity_json is not None:
prop_json = self._find_property_json_by_uid(entity_json, prop.uid)
if prop_json is None:
# User provided a UID not matching any property (within the entity), make sure it's not assigned
# elsewhere
self._validate_uid_unassigned(prop.uid)
else:
write_json = prop.name != prop_json["name"] # If renaming we shall update the JSON
else:
if entity_json is not None:
prop_json = self._find_property_json_by_name(entity_json, prop.name)
if prop_json is not None: # Load existing IDs from JSON
# Property was matched with a JSON property (either by UID or by name), make sure they're equal
self._validate_matching_prop(entity, prop, prop_json)
prop.iduid = IdUid.from_str(prop_json["id"])
else: # Assign new ID to new property
if not prop.has_uid():
prop.iduid.uid = self._generate_uid()
prop.iduid = IdUid(entity._last_property_iduid.id + 1, prop.iduid.uid)
entity._last_property_iduid = prop.iduid
write_json = True
if prop.index is not None:
write_json |= self._sync_index(entity, prop, prop_json)
return write_json
def _sync_entity(self, entity: _Entity) -> bool:
write_json = False
# entity_json = None
if entity._has_uid():
entity_json = self._find_entity_json_by_uid(entity._uid)
if entity_json is None:
# User provided a UID not matching any entity, make sure it's not assigned elsewhere
self._validate_uid_unassigned(entity._uid)
else:
write_json = entity._name != entity_json["name"] # If renaming we shall update the JSON
else:
entity_json = self._find_entity_json_by_name(entity._name)
# Write JSON if the number of properties differs (to handle removed property)
if entity_json is not None:
write_json |= len(entity._properties) != len(entity_json["properties"])
if entity_json is not None: # Load existing IDs from JSON
entity._iduid = IdUid.from_str(entity_json["id"])
entity._last_property_iduid = IdUid.from_str(entity_json["lastPropertyId"])
else: # Assign new ID to new entity
if not entity._has_uid():
entity._iduid.uid = self._generate_uid()
entity._iduid = IdUid(self.model.last_entity_iduid.id + 1, entity._iduid.uid)
self.model.last_entity_iduid = entity._iduid
entity._last_property_iduid = IdUid(0, 0)
write_json = True
# Load properties
for prop in entity._properties:
write_json |= self._sync_property(entity, prop, entity_json)
return write_json
def sync(self) -> bool:
""" Syncs the provided model with the model JSON file.
Returns True if the model JSON was written. """
if self.model_json is not None:
self.model.last_entity_iduid = IdUid.from_str(self.model_json["lastEntityId"])
self.model.last_index_iduid = IdUid.from_str(self.model_json["lastIndexId"])
# self.model.last_relation_iduid =
write_json = False
# Write JSON if the number of entities differs (to handle removed entity)
if self.model_json is not None:
write_json |= len(self.model_json["entities"]) != len(self.model.entities)
for entity in self.model.entities:
write_json |= self._sync_entity(entity)
if write_json:
logger.info(f"Model changed, writing model.json: {self.model_filepath}")
self._save_model_json()
self.model.on_sync() # Notify model synced
return write_json
def sync_model(model: Model, model_filepath: str = "objectbox-model.json") -> bool:
""" Syncs the provided model with the model JSON file.
Returns True if changes were made and the model JSON was written. """
id_sync = IdSync(model, model_filepath)
return id_sync.sync()