Coverage for src/extratools_core/jsontools.py: 54%

61 statements  

« prev     ^ index     » next       coverage.py v7.8.1, created at 2025-05-22 04:16 -0700

1import json 

2from csv import DictWriter 

3from io import StringIO 

4from pathlib import Path 

5from typing import Any, TypedDict 

6 

7type JsonDict = dict[str, Any] 

8 

9type DictOfJsonDicts = dict[str, JsonDict] 

10type ListOfJsonDicts = list[JsonDict] 

11 

12 

13class DictOfJsonDictsDiffUpdate(TypedDict): 

14 old: JsonDict 

15 new: JsonDict 

16 

17 

18class DictOfJsonDictsDiff(TypedDict): 

19 deletes: dict[str, JsonDict] 

20 inserts: dict[str, JsonDict] 

21 updates: dict[str, DictOfJsonDictsDiffUpdate] 

22 

23 

24class ListOfJsonDictsDiff(TypedDict): 

25 deletes: list[JsonDict] 

26 inserts: list[JsonDict] 

27 

28 

29def flatten(data: Any) -> Any: 

30 def flatten_rec(data: Any, path: str) -> None: 

31 if isinstance(data, dict): 

32 for k, v in data.items(): 

33 flatten_rec(v, path + (f".{k}" if path else k)) 

34 elif isinstance(data, list): 

35 for i, v in enumerate(data): 

36 flatten_rec(v, path + f"[{i}]") 

37 else: 

38 flatten_dict[path or "."] = data 

39 

40 flatten_dict: JsonDict = {} 

41 flatten_rec(data, "") 

42 return flatten_dict 

43 

44 

45def json_to_csv( 

46 data: DictOfJsonDicts | ListOfJsonDicts, 

47 /, 

48 csv_path: Path | str | None = None, 

49 *, 

50 key_field_name: str = "_key", 

51) -> str: 

52 if isinstance(data, dict): 

53 data = [ 

54 { 

55 # In case there is already a key field in each record, 

56 # the new key field will be overwritten. 

57 # It is okay though as the existing key field is likely 

58 # serving the purpose of containing keys. 

59 key_field_name: key, 

60 **value, 

61 } 

62 for key, value in data.items() 

63 ] 

64 

65 fields: set[str] = set() 

66 for record in data: 

67 fields.update(record.keys()) 

68 

69 sio = StringIO() 

70 

71 writer = DictWriter(sio, fieldnames=fields) 

72 writer.writeheader() 

73 writer.writerows(data) 

74 

75 csv_str: str = sio.getvalue() 

76 

77 if csv_path: 

78 Path(csv_path).write_text(csv_str) 

79 

80 return csv_str 

81 

82 

83def dict_of_json_dicts_diff( 

84 old: DictOfJsonDicts, 

85 new: DictOfJsonDicts, 

86) -> DictOfJsonDictsDiff: 

87 inserts: dict[str, JsonDict] = {} 

88 updates: dict[str, DictOfJsonDictsDiffUpdate] = {} 

89 

90 for new_key, new_value in new.items(): 

91 old_value: dict[str, Any] | None = old.get(new_key, None) 

92 if old_value is None: 

93 inserts[new_key] = new_value 

94 elif json.dumps(old_value) != json.dumps(new_value): 

95 updates[new_key] = { 

96 "old": old_value, 

97 "new": new_value, 

98 } 

99 

100 deletes: dict[str, JsonDict] = { 

101 old_key: old_value 

102 for old_key, old_value in old.items() 

103 if old_key not in new 

104 } 

105 

106 return { 

107 "deletes": deletes, 

108 "inserts": inserts, 

109 "updates": updates, 

110 } 

111 

112 

113def list_of_json_dicts_diff( 

114 old: ListOfJsonDicts, 

115 new: ListOfJsonDicts, 

116) -> ListOfJsonDictsDiff: 

117 old_dict: DictOfJsonDicts = { 

118 json.dumps(d): d 

119 for d in old 

120 } 

121 new_dict: DictOfJsonDicts = { 

122 json.dumps(d): d 

123 for d in new 

124 } 

125 

126 inserts: list[JsonDict] = [ 

127 new_value 

128 for new_key, new_value in new_dict.items() 

129 if new_key not in old_dict 

130 ] 

131 deletes: list[JsonDict] = [ 

132 old_value 

133 for old_key, old_value in old_dict.items() 

134 if old_key not in new_dict 

135 ] 

136 

137 return { 

138 "deletes": deletes, 

139 "inserts": inserts, 

140 }