Coverage for /Users/davegaeddert/Development/dropseed/plain/plain-models/plain/models/deletion.py: 15%

244 statements  

« prev     ^ index     » next       coverage.py v7.6.1, created at 2024-10-16 22:03 -0500

1from collections import Counter, defaultdict 

2from functools import partial, reduce 

3from itertools import chain 

4from operator import attrgetter, or_ 

5 

6from plain.models import ( 

7 query_utils, 

8 signals, 

9 sql, 

10 transaction, 

11) 

12from plain.models.db import IntegrityError, connections 

13from plain.models.query import QuerySet 

14 

15 

16class ProtectedError(IntegrityError): 

17 def __init__(self, msg, protected_objects): 

18 self.protected_objects = protected_objects 

19 super().__init__(msg, protected_objects) 

20 

21 

22class RestrictedError(IntegrityError): 

23 def __init__(self, msg, restricted_objects): 

24 self.restricted_objects = restricted_objects 

25 super().__init__(msg, restricted_objects) 

26 

27 

28def CASCADE(collector, field, sub_objs, using): 

29 collector.collect( 

30 sub_objs, 

31 source=field.remote_field.model, 

32 source_attr=field.name, 

33 nullable=field.null, 

34 fail_on_restricted=False, 

35 ) 

36 if field.null and not connections[using].features.can_defer_constraint_checks: 

37 collector.add_field_update(field, None, sub_objs) 

38 

39 

40def PROTECT(collector, field, sub_objs, using): 

41 raise ProtectedError( 

42 "Cannot delete some instances of model '{}' because they are " 

43 "referenced through a protected foreign key: '{}.{}'".format( 

44 field.remote_field.model.__name__, 

45 sub_objs[0].__class__.__name__, 

46 field.name, 

47 ), 

48 sub_objs, 

49 ) 

50 

51 

52def RESTRICT(collector, field, sub_objs, using): 

53 collector.add_restricted_objects(field, sub_objs) 

54 collector.add_dependency(field.remote_field.model, field.model) 

55 

56 

57def SET(value): 

58 if callable(value): 

59 

60 def set_on_delete(collector, field, sub_objs, using): 

61 collector.add_field_update(field, value(), sub_objs) 

62 

63 else: 

64 

65 def set_on_delete(collector, field, sub_objs, using): 

66 collector.add_field_update(field, value, sub_objs) 

67 

68 set_on_delete.deconstruct = lambda: ("plain.models.SET", (value,), {}) 

69 set_on_delete.lazy_sub_objs = True 

70 return set_on_delete 

71 

72 

73def SET_NULL(collector, field, sub_objs, using): 

74 collector.add_field_update(field, None, sub_objs) 

75 

76 

77SET_NULL.lazy_sub_objs = True 

78 

79 

80def SET_DEFAULT(collector, field, sub_objs, using): 

81 collector.add_field_update(field, field.get_default(), sub_objs) 

82 

83 

84SET_DEFAULT.lazy_sub_objs = True 

85 

86 

87def DO_NOTHING(collector, field, sub_objs, using): 

88 pass 

89 

90 

91def get_candidate_relations_to_delete(opts): 

92 # The candidate relations are the ones that come from N-1 and 1-1 relations. 

93 # N-N (i.e., many-to-many) relations aren't candidates for deletion. 

94 return ( 

95 f 

96 for f in opts.get_fields(include_hidden=True) 

97 if f.auto_created and not f.concrete and (f.one_to_one or f.one_to_many) 

98 ) 

99 

100 

101class Collector: 

102 def __init__(self, using, origin=None): 

103 self.using = using 

104 # A Model or QuerySet object. 

105 self.origin = origin 

106 # Initially, {model: {instances}}, later values become lists. 

107 self.data = defaultdict(set) 

108 # {(field, value): [instances, …]} 

109 self.field_updates = defaultdict(list) 

110 # {model: {field: {instances}}} 

111 self.restricted_objects = defaultdict(partial(defaultdict, set)) 

112 # fast_deletes is a list of queryset-likes that can be deleted without 

113 # fetching the objects into memory. 

114 self.fast_deletes = [] 

115 

116 # Tracks deletion-order dependency for databases without transactions 

117 # or ability to defer constraint checks. Only concrete model classes 

118 # should be included, as the dependencies exist only between actual 

119 # database tables. 

120 self.dependencies = defaultdict(set) # {model: {models}} 

121 

122 def add(self, objs, source=None, nullable=False, reverse_dependency=False): 

123 """ 

124 Add 'objs' to the collection of objects to be deleted. If the call is 

125 the result of a cascade, 'source' should be the model that caused it, 

126 and 'nullable' should be set to True if the relation can be null. 

127 

128 Return a list of all objects that were not already collected. 

129 """ 

130 if not objs: 

131 return [] 

132 new_objs = [] 

133 model = objs[0].__class__ 

134 instances = self.data[model] 

135 for obj in objs: 

136 if obj not in instances: 

137 new_objs.append(obj) 

138 instances.update(new_objs) 

139 # Nullable relationships can be ignored -- they are nulled out before 

140 # deleting, and therefore do not affect the order in which objects have 

141 # to be deleted. 

142 if source is not None and not nullable: 

143 self.add_dependency(source, model, reverse_dependency=reverse_dependency) 

144 return new_objs 

145 

146 def add_dependency(self, model, dependency, reverse_dependency=False): 

147 if reverse_dependency: 

148 model, dependency = dependency, model 

149 self.dependencies[model._meta.concrete_model].add( 

150 dependency._meta.concrete_model 

151 ) 

152 self.data.setdefault(dependency, self.data.default_factory()) 

153 

154 def add_field_update(self, field, value, objs): 

155 """ 

156 Schedule a field update. 'objs' must be a homogeneous iterable 

157 collection of model instances (e.g. a QuerySet). 

158 """ 

159 self.field_updates[field, value].append(objs) 

160 

161 def add_restricted_objects(self, field, objs): 

162 if objs: 

163 model = objs[0].__class__ 

164 self.restricted_objects[model][field].update(objs) 

165 

166 def clear_restricted_objects_from_set(self, model, objs): 

167 if model in self.restricted_objects: 

168 self.restricted_objects[model] = { 

169 field: items - objs 

170 for field, items in self.restricted_objects[model].items() 

171 } 

172 

173 def clear_restricted_objects_from_queryset(self, model, qs): 

174 if model in self.restricted_objects: 

175 objs = set( 

176 qs.filter( 

177 pk__in=[ 

178 obj.pk 

179 for objs in self.restricted_objects[model].values() 

180 for obj in objs 

181 ] 

182 ) 

183 ) 

184 self.clear_restricted_objects_from_set(model, objs) 

185 

186 def _has_signal_listeners(self, model): 

187 return signals.pre_delete.has_listeners( 

188 model 

189 ) or signals.post_delete.has_listeners(model) 

190 

191 def can_fast_delete(self, objs, from_field=None): 

192 """ 

193 Determine if the objects in the given queryset-like or single object 

194 can be fast-deleted. This can be done if there are no cascades, no 

195 parents and no signal listeners for the object class. 

196 

197 The 'from_field' tells where we are coming from - we need this to 

198 determine if the objects are in fact to be deleted. Allow also 

199 skipping parent -> child -> parent chain preventing fast delete of 

200 the child. 

201 """ 

202 if from_field and from_field.remote_field.on_delete is not CASCADE: 

203 return False 

204 if hasattr(objs, "_meta"): 

205 model = objs._meta.model 

206 elif hasattr(objs, "model") and hasattr(objs, "_raw_delete"): 

207 model = objs.model 

208 else: 

209 return False 

210 if self._has_signal_listeners(model): 

211 return False 

212 # The use of from_field comes from the need to avoid cascade back to 

213 # parent when parent delete is cascading to child. 

214 opts = model._meta 

215 return ( 

216 all( 

217 link == from_field 

218 for link in opts.concrete_model._meta.parents.values() 

219 ) 

220 and 

221 # Foreign keys pointing to this model. 

222 all( 

223 related.field.remote_field.on_delete is DO_NOTHING 

224 for related in get_candidate_relations_to_delete(opts) 

225 ) 

226 and ( 

227 # Something like generic foreign key. 

228 not any( 

229 hasattr(field, "bulk_related_objects") 

230 for field in opts.private_fields 

231 ) 

232 ) 

233 ) 

234 

235 def get_del_batches(self, objs, fields): 

236 """ 

237 Return the objs in suitably sized batches for the used connection. 

238 """ 

239 field_names = [field.name for field in fields] 

240 conn_batch_size = max( 

241 connections[self.using].ops.bulk_batch_size(field_names, objs), 1 

242 ) 

243 if len(objs) > conn_batch_size: 

244 return [ 

245 objs[i : i + conn_batch_size] 

246 for i in range(0, len(objs), conn_batch_size) 

247 ] 

248 else: 

249 return [objs] 

250 

251 def collect( 

252 self, 

253 objs, 

254 source=None, 

255 nullable=False, 

256 collect_related=True, 

257 source_attr=None, 

258 reverse_dependency=False, 

259 keep_parents=False, 

260 fail_on_restricted=True, 

261 ): 

262 """ 

263 Add 'objs' to the collection of objects to be deleted as well as all 

264 parent instances. 'objs' must be a homogeneous iterable collection of 

265 model instances (e.g. a QuerySet). If 'collect_related' is True, 

266 related objects will be handled by their respective on_delete handler. 

267 

268 If the call is the result of a cascade, 'source' should be the model 

269 that caused it and 'nullable' should be set to True, if the relation 

270 can be null. 

271 

272 If 'reverse_dependency' is True, 'source' will be deleted before the 

273 current model, rather than after. (Needed for cascading to parent 

274 models, the one case in which the cascade follows the forwards 

275 direction of an FK rather than the reverse direction.) 

276 

277 If 'keep_parents' is True, data of parent model's will be not deleted. 

278 

279 If 'fail_on_restricted' is False, error won't be raised even if it's 

280 prohibited to delete such objects due to RESTRICT, that defers 

281 restricted object checking in recursive calls where the top-level call 

282 may need to collect more objects to determine whether restricted ones 

283 can be deleted. 

284 """ 

285 if self.can_fast_delete(objs): 

286 self.fast_deletes.append(objs) 

287 return 

288 new_objs = self.add( 

289 objs, source, nullable, reverse_dependency=reverse_dependency 

290 ) 

291 if not new_objs: 

292 return 

293 

294 model = new_objs[0].__class__ 

295 

296 if not keep_parents: 

297 # Recursively collect concrete model's parent models, but not their 

298 # related objects. These will be found by meta.get_fields() 

299 concrete_model = model._meta.concrete_model 

300 for ptr in concrete_model._meta.parents.values(): 

301 if ptr: 

302 parent_objs = [getattr(obj, ptr.name) for obj in new_objs] 

303 self.collect( 

304 parent_objs, 

305 source=model, 

306 source_attr=ptr.remote_field.related_name, 

307 collect_related=False, 

308 reverse_dependency=True, 

309 fail_on_restricted=False, 

310 ) 

311 if not collect_related: 

312 return 

313 

314 if keep_parents: 

315 parents = set(model._meta.get_parent_list()) 

316 model_fast_deletes = defaultdict(list) 

317 protected_objects = defaultdict(list) 

318 for related in get_candidate_relations_to_delete(model._meta): 

319 # Preserve parent reverse relationships if keep_parents=True. 

320 if keep_parents and related.model in parents: 

321 continue 

322 field = related.field 

323 on_delete = field.remote_field.on_delete 

324 if on_delete == DO_NOTHING: 

325 continue 

326 related_model = related.related_model 

327 if self.can_fast_delete(related_model, from_field=field): 

328 model_fast_deletes[related_model].append(field) 

329 continue 

330 batches = self.get_del_batches(new_objs, [field]) 

331 for batch in batches: 

332 sub_objs = self.related_objects(related_model, [field], batch) 

333 # Non-referenced fields can be deferred if no signal receivers 

334 # are connected for the related model as they'll never be 

335 # exposed to the user. Skip field deferring when some 

336 # relationships are select_related as interactions between both 

337 # features are hard to get right. This should only happen in 

338 # the rare cases where .related_objects is overridden anyway. 

339 if not ( 

340 sub_objs.query.select_related 

341 or self._has_signal_listeners(related_model) 

342 ): 

343 referenced_fields = set( 

344 chain.from_iterable( 

345 (rf.attname for rf in rel.field.foreign_related_fields) 

346 for rel in get_candidate_relations_to_delete( 

347 related_model._meta 

348 ) 

349 ) 

350 ) 

351 sub_objs = sub_objs.only(*tuple(referenced_fields)) 

352 if getattr(on_delete, "lazy_sub_objs", False) or sub_objs: 

353 try: 

354 on_delete(self, field, sub_objs, self.using) 

355 except ProtectedError as error: 

356 key = f"'{field.model.__name__}.{field.name}'" 

357 protected_objects[key] += error.protected_objects 

358 if protected_objects: 

359 raise ProtectedError( 

360 "Cannot delete some instances of model {!r} because they are " 

361 "referenced through protected foreign keys: {}.".format( 

362 model.__name__, 

363 ", ".join(protected_objects), 

364 ), 

365 set(chain.from_iterable(protected_objects.values())), 

366 ) 

367 for related_model, related_fields in model_fast_deletes.items(): 

368 batches = self.get_del_batches(new_objs, related_fields) 

369 for batch in batches: 

370 sub_objs = self.related_objects(related_model, related_fields, batch) 

371 self.fast_deletes.append(sub_objs) 

372 for field in model._meta.private_fields: 

373 if hasattr(field, "bulk_related_objects"): 

374 # It's something like generic foreign key. 

375 sub_objs = field.bulk_related_objects(new_objs, self.using) 

376 self.collect( 

377 sub_objs, source=model, nullable=True, fail_on_restricted=False 

378 ) 

379 

380 if fail_on_restricted: 

381 # Raise an error if collected restricted objects (RESTRICT) aren't 

382 # candidates for deletion also collected via CASCADE. 

383 for related_model, instances in self.data.items(): 

384 self.clear_restricted_objects_from_set(related_model, instances) 

385 for qs in self.fast_deletes: 

386 self.clear_restricted_objects_from_queryset(qs.model, qs) 

387 if self.restricted_objects.values(): 

388 restricted_objects = defaultdict(list) 

389 for related_model, fields in self.restricted_objects.items(): 

390 for field, objs in fields.items(): 

391 if objs: 

392 key = f"'{related_model.__name__}.{field.name}'" 

393 restricted_objects[key] += objs 

394 if restricted_objects: 

395 raise RestrictedError( 

396 "Cannot delete some instances of model {!r} because " 

397 "they are referenced through restricted foreign keys: " 

398 "{}.".format( 

399 model.__name__, 

400 ", ".join(restricted_objects), 

401 ), 

402 set(chain.from_iterable(restricted_objects.values())), 

403 ) 

404 

405 def related_objects(self, related_model, related_fields, objs): 

406 """ 

407 Get a QuerySet of the related model to objs via related fields. 

408 """ 

409 predicate = query_utils.Q.create( 

410 [(f"{related_field.name}__in", objs) for related_field in related_fields], 

411 connector=query_utils.Q.OR, 

412 ) 

413 return related_model._base_manager.using(self.using).filter(predicate) 

414 

415 def instances_with_model(self): 

416 for model, instances in self.data.items(): 

417 for obj in instances: 

418 yield model, obj 

419 

420 def sort(self): 

421 sorted_models = [] 

422 concrete_models = set() 

423 models = list(self.data) 

424 while len(sorted_models) < len(models): 

425 found = False 

426 for model in models: 

427 if model in sorted_models: 

428 continue 

429 dependencies = self.dependencies.get(model._meta.concrete_model) 

430 if not (dependencies and dependencies.difference(concrete_models)): 

431 sorted_models.append(model) 

432 concrete_models.add(model._meta.concrete_model) 

433 found = True 

434 if not found: 

435 return 

436 self.data = {model: self.data[model] for model in sorted_models} 

437 

438 def delete(self): 

439 # sort instance collections 

440 for model, instances in self.data.items(): 

441 self.data[model] = sorted(instances, key=attrgetter("pk")) 

442 

443 # if possible, bring the models in an order suitable for databases that 

444 # don't support transactions or cannot defer constraint checks until the 

445 # end of a transaction. 

446 self.sort() 

447 # number of objects deleted for each model label 

448 deleted_counter = Counter() 

449 

450 # Optimize for the case with a single obj and no dependencies 

451 if len(self.data) == 1 and len(instances) == 1: 

452 instance = list(instances)[0] 

453 if self.can_fast_delete(instance): 

454 with transaction.mark_for_rollback_on_error(self.using): 

455 count = sql.DeleteQuery(model).delete_batch( 

456 [instance.pk], self.using 

457 ) 

458 setattr(instance, model._meta.pk.attname, None) 

459 return count, {model._meta.label: count} 

460 

461 with transaction.atomic(using=self.using, savepoint=False): 

462 # send pre_delete signals 

463 for model, obj in self.instances_with_model(): 

464 if not model._meta.auto_created: 

465 signals.pre_delete.send( 

466 sender=model, 

467 instance=obj, 

468 using=self.using, 

469 origin=self.origin, 

470 ) 

471 

472 # fast deletes 

473 for qs in self.fast_deletes: 

474 count = qs._raw_delete(using=self.using) 

475 if count: 

476 deleted_counter[qs.model._meta.label] += count 

477 

478 # update fields 

479 for (field, value), instances_list in self.field_updates.items(): 

480 updates = [] 

481 objs = [] 

482 for instances in instances_list: 

483 if ( 

484 isinstance(instances, QuerySet) 

485 and instances._result_cache is None 

486 ): 

487 updates.append(instances) 

488 else: 

489 objs.extend(instances) 

490 if updates: 

491 combined_updates = reduce(or_, updates) 

492 combined_updates.update(**{field.name: value}) 

493 if objs: 

494 model = objs[0].__class__ 

495 query = sql.UpdateQuery(model) 

496 query.update_batch( 

497 list({obj.pk for obj in objs}), {field.name: value}, self.using 

498 ) 

499 

500 # reverse instance collections 

501 for instances in self.data.values(): 

502 instances.reverse() 

503 

504 # delete instances 

505 for model, instances in self.data.items(): 

506 query = sql.DeleteQuery(model) 

507 pk_list = [obj.pk for obj in instances] 

508 count = query.delete_batch(pk_list, self.using) 

509 if count: 

510 deleted_counter[model._meta.label] += count 

511 

512 if not model._meta.auto_created: 

513 for obj in instances: 

514 signals.post_delete.send( 

515 sender=model, 

516 instance=obj, 

517 using=self.using, 

518 origin=self.origin, 

519 ) 

520 

521 for model, instances in self.data.items(): 

522 for instance in instances: 

523 setattr(instance, model._meta.pk.attname, None) 

524 return sum(deleted_counter.values()), dict(deleted_counter)