muutils.json_serialize.json_serialize
provides the basic framework for json serialization of objects
notably:
SerializerHandler
defines how to serialize a specific type of objectJsonSerializer
handles configuration for which handlers to usejson_serialize
provides the default configuration if you don't care -- call it on any object!
1"""provides the basic framework for json serialization of objects 2 3notably: 4 5- `SerializerHandler` defines how to serialize a specific type of object 6- `JsonSerializer` handles configuration for which handlers to use 7- `json_serialize` provides the default configuration if you don't care -- call it on any object! 8 9""" 10 11from __future__ import annotations 12 13import inspect 14import warnings 15from dataclasses import dataclass, is_dataclass 16from pathlib import Path 17from typing import Any, Callable, Iterable, Mapping, Set, Union 18 19from muutils.errormode import ErrorMode 20 21try: 22 from muutils.json_serialize.array import ArrayMode, serialize_array 23except ImportError as e: 24 ArrayMode = str # type: ignore[misc] 25 serialize_array = lambda *args, **kwargs: None # noqa: E731 26 warnings.warn( 27 f"muutils.json_serialize.array could not be imported probably because missing numpy, array serialization will not work: \n{e}", 28 ImportWarning, 29 ) 30 31from muutils.json_serialize.util import ( 32 _FORMAT_KEY, 33 Hashableitem, 34 JSONitem, 35 MonoTuple, 36 SerializationException, 37 _recursive_hashify, 38 isinstance_namedtuple, 39 safe_getsource, 40 string_as_lines, 41 try_catch, 42) 43 44# pylint: disable=protected-access 45 46SERIALIZER_SPECIAL_KEYS: MonoTuple[str] = ( 47 "__name__", 48 "__doc__", 49 "__module__", 50 "__class__", 51 "__dict__", 52 "__annotations__", 53) 54 55SERIALIZER_SPECIAL_FUNCS: dict[str, Callable] = { 56 "str": str, 57 "dir": dir, 58 "type": try_catch(lambda x: str(type(x).__name__)), 59 "repr": try_catch(lambda x: repr(x)), 60 "code": try_catch(lambda x: inspect.getsource(x)), 61 "sourcefile": try_catch(lambda x: inspect.getsourcefile(x)), 62} 63 64SERIALIZE_DIRECT_AS_STR: Set[str] = { 65 "<class 'torch.device'>", 66 "<class 'torch.dtype'>", 67} 68 69ObjectPath = MonoTuple[Union[str, int]] 70 71 72@dataclass 73class SerializerHandler: 74 """a handler for a specific type of object 75 76 # Parameters: 77 - `check : Callable[[JsonSerializer, Any], bool]` takes a JsonSerializer and an object, returns whether to use this handler 78 - `serialize : Callable[[JsonSerializer, Any, ObjectPath], JSONitem]` takes a JsonSerializer, an object, and the current path, returns the serialized object 79 - `desc : str` description of the handler (optional) 80 """ 81 82 # (self_config, object) -> whether to use this handler 83 check: Callable[["JsonSerializer", Any, ObjectPath], bool] 84 # (self_config, object, path) -> serialized object 85 serialize_func: Callable[["JsonSerializer", Any, ObjectPath], JSONitem] 86 # unique identifier for the handler 87 uid: str 88 # description of this serializer 89 desc: str 90 91 def serialize(self) -> dict: 92 """serialize the handler info""" 93 return { 94 # get the code and doc of the check function 95 "check": { 96 "code": safe_getsource(self.check), 97 "doc": string_as_lines(self.check.__doc__), 98 }, 99 # get the code and doc of the load function 100 "serialize_func": { 101 "code": safe_getsource(self.serialize_func), 102 "doc": string_as_lines(self.serialize_func.__doc__), 103 }, 104 # get the uid, source_pckg, priority, and desc 105 "uid": str(self.uid), 106 "source_pckg": getattr(self.serialize_func, "source_pckg", None), 107 "__module__": getattr(self.serialize_func, "__module__", None), 108 "desc": str(self.desc), 109 } 110 111 112BASE_HANDLERS: MonoTuple[SerializerHandler] = ( 113 SerializerHandler( 114 check=lambda self, obj, path: isinstance( 115 obj, (bool, int, float, str, type(None)) 116 ), 117 serialize_func=lambda self, obj, path: obj, 118 uid="base types", 119 desc="base types (bool, int, float, str, None)", 120 ), 121 SerializerHandler( 122 check=lambda self, obj, path: isinstance(obj, Mapping), 123 serialize_func=lambda self, obj, path: { 124 str(k): self.json_serialize(v, tuple(path) + (k,)) for k, v in obj.items() 125 }, 126 uid="dictionaries", 127 desc="dictionaries", 128 ), 129 SerializerHandler( 130 check=lambda self, obj, path: isinstance(obj, (list, tuple)), 131 serialize_func=lambda self, obj, path: [ 132 self.json_serialize(x, tuple(path) + (i,)) for i, x in enumerate(obj) 133 ], 134 uid="(list, tuple) -> list", 135 desc="lists and tuples as lists", 136 ), 137) 138 139 140def _serialize_override_serialize_func( 141 self: "JsonSerializer", obj: Any, path: ObjectPath 142) -> JSONitem: 143 # obj_cls: type = type(obj) 144 # if hasattr(obj_cls, "_register_self") and callable(obj_cls._register_self): 145 # obj_cls._register_self() 146 147 # get the serialized object 148 return obj.serialize() 149 150 151DEFAULT_HANDLERS: MonoTuple[SerializerHandler] = tuple(BASE_HANDLERS) + ( 152 SerializerHandler( 153 # TODO: allow for custom serialization handler name 154 check=lambda self, obj, path: hasattr(obj, "serialize") 155 and callable(obj.serialize), 156 serialize_func=_serialize_override_serialize_func, 157 uid=".serialize override", 158 desc="objects with .serialize method", 159 ), 160 SerializerHandler( 161 check=lambda self, obj, path: isinstance_namedtuple(obj), 162 serialize_func=lambda self, obj, path: self.json_serialize(dict(obj._asdict())), 163 uid="namedtuple -> dict", 164 desc="namedtuples as dicts", 165 ), 166 SerializerHandler( 167 check=lambda self, obj, path: is_dataclass(obj), 168 serialize_func=lambda self, obj, path: { 169 k: self.json_serialize(getattr(obj, k), tuple(path) + (k,)) 170 for k in obj.__dataclass_fields__ 171 }, 172 uid="dataclass -> dict", 173 desc="dataclasses as dicts", 174 ), 175 SerializerHandler( 176 check=lambda self, obj, path: isinstance(obj, Path), 177 serialize_func=lambda self, obj, path: obj.as_posix(), 178 uid="path -> str", 179 desc="Path objects as posix strings", 180 ), 181 SerializerHandler( 182 check=lambda self, obj, path: str(type(obj)) in SERIALIZE_DIRECT_AS_STR, 183 serialize_func=lambda self, obj, path: str(obj), 184 uid="obj -> str(obj)", 185 desc="directly serialize objects in `SERIALIZE_DIRECT_AS_STR` to strings", 186 ), 187 SerializerHandler( 188 check=lambda self, obj, path: str(type(obj)) == "<class 'numpy.ndarray'>", 189 serialize_func=lambda self, obj, path: serialize_array(self, obj, path=path), 190 uid="numpy.ndarray", 191 desc="numpy arrays", 192 ), 193 SerializerHandler( 194 check=lambda self, obj, path: str(type(obj)) == "<class 'torch.Tensor'>", 195 serialize_func=lambda self, obj, path: serialize_array( 196 self, obj.detach().cpu(), path=path 197 ), 198 uid="torch.Tensor", 199 desc="pytorch tensors", 200 ), 201 SerializerHandler( 202 check=lambda self, obj, path: ( 203 str(type(obj)) == "<class 'pandas.core.frame.DataFrame'>" 204 ), 205 serialize_func=lambda self, obj, path: { 206 _FORMAT_KEY: "pandas.DataFrame", 207 "columns": obj.columns.tolist(), 208 "data": obj.to_dict(orient="records"), 209 "path": path, # type: ignore 210 }, 211 uid="pandas.DataFrame", 212 desc="pandas DataFrames", 213 ), 214 SerializerHandler( 215 check=lambda self, obj, path: isinstance(obj, (set, list, tuple)) 216 or isinstance(obj, Iterable), 217 serialize_func=lambda self, obj, path: [ 218 self.json_serialize(x, tuple(path) + (i,)) for i, x in enumerate(obj) 219 ], 220 uid="(set, list, tuple, Iterable) -> list", 221 desc="sets, lists, tuples, and Iterables as lists", 222 ), 223 SerializerHandler( 224 check=lambda self, obj, path: True, 225 serialize_func=lambda self, obj, path: { 226 **{k: str(getattr(obj, k, None)) for k in SERIALIZER_SPECIAL_KEYS}, 227 **{k: f(obj) for k, f in SERIALIZER_SPECIAL_FUNCS.items()}, 228 }, 229 uid="fallback", 230 desc="fallback handler -- serialize object attributes and special functions as strings", 231 ), 232) 233 234 235class JsonSerializer: 236 """Json serialization class (holds configs) 237 238 # Parameters: 239 - `array_mode : ArrayMode` 240 how to write arrays 241 (defaults to `"array_list_meta"`) 242 - `error_mode : ErrorMode` 243 what to do when we can't serialize an object (will use repr as fallback if "ignore" or "warn") 244 (defaults to `"except"`) 245 - `handlers_pre : MonoTuple[SerializerHandler]` 246 handlers to use before the default handlers 247 (defaults to `tuple()`) 248 - `handlers_default : MonoTuple[SerializerHandler]` 249 default handlers to use 250 (defaults to `DEFAULT_HANDLERS`) 251 - `write_only_format : bool` 252 changes _FORMAT_KEY keys in output to "__write_format__" (when you want to serialize something in a way that zanj won't try to recover the object when loading) 253 (defaults to `False`) 254 255 # Raises: 256 - `ValueError`: on init, if `args` is not empty 257 - `SerializationException`: on `json_serialize()`, if any error occurs when trying to serialize an object and `error_mode` is set to `ErrorMode.EXCEPT"` 258 259 """ 260 261 def __init__( 262 self, 263 *args, 264 array_mode: ArrayMode = "array_list_meta", 265 error_mode: ErrorMode = ErrorMode.EXCEPT, 266 handlers_pre: MonoTuple[SerializerHandler] = tuple(), 267 handlers_default: MonoTuple[SerializerHandler] = DEFAULT_HANDLERS, 268 write_only_format: bool = False, 269 ): 270 if len(args) > 0: 271 raise ValueError( 272 f"JsonSerializer takes no positional arguments!\n{args = }" 273 ) 274 275 self.array_mode: ArrayMode = array_mode 276 self.error_mode: ErrorMode = ErrorMode.from_any(error_mode) 277 self.write_only_format: bool = write_only_format 278 # join up the handlers 279 self.handlers: MonoTuple[SerializerHandler] = tuple(handlers_pre) + tuple( 280 handlers_default 281 ) 282 283 def json_serialize( 284 self, 285 obj: Any, 286 path: ObjectPath = tuple(), 287 ) -> JSONitem: 288 try: 289 for handler in self.handlers: 290 if handler.check(self, obj, path): 291 output: JSONitem = handler.serialize_func(self, obj, path) 292 if self.write_only_format: 293 if isinstance(output, dict) and _FORMAT_KEY in output: 294 new_fmt: JSONitem = output.pop(_FORMAT_KEY) 295 output["__write_format__"] = new_fmt 296 return output 297 298 raise ValueError(f"no handler found for object with {type(obj) = }") 299 300 except Exception as e: 301 if self.error_mode == "except": 302 obj_str: str = repr(obj) 303 if len(obj_str) > 1000: 304 obj_str = obj_str[:1000] + "..." 305 raise SerializationException( 306 f"error serializing at {path = } with last handler: '{handler.uid}'\nfrom: {e}\nobj: {obj_str}" 307 ) from e 308 elif self.error_mode == "warn": 309 warnings.warn( 310 f"error serializing at {path = }, will return as string\n{obj = }\nexception = {e}" 311 ) 312 313 return repr(obj) 314 315 def hashify( 316 self, 317 obj: Any, 318 path: ObjectPath = tuple(), 319 force: bool = True, 320 ) -> Hashableitem: 321 """try to turn any object into something hashable""" 322 data = self.json_serialize(obj, path=path) 323 324 # recursive hashify, turning dicts and lists into tuples 325 return _recursive_hashify(data, force=force) 326 327 328GLOBAL_JSON_SERIALIZER: JsonSerializer = JsonSerializer() 329 330 331def json_serialize(obj: Any, path: ObjectPath = tuple()) -> JSONitem: 332 """serialize object to json-serializable object with default config""" 333 return GLOBAL_JSON_SERIALIZER.json_serialize(obj, path=path)
SERIALIZER_SPECIAL_KEYS: None =
('__name__', '__doc__', '__module__', '__class__', '__dict__', '__annotations__')
SERIALIZER_SPECIAL_FUNCS: dict[str, typing.Callable] =
{'str': <class 'str'>, 'dir': <built-in function dir>, 'type': <function <lambda>>, 'repr': <function <lambda>>, 'code': <function <lambda>>, 'sourcefile': <function <lambda>>}
SERIALIZE_DIRECT_AS_STR: Set[str] =
{"<class 'torch.dtype'>", "<class 'torch.device'>"}
ObjectPath =
tuple[typing.Union[str, int], ...]
@dataclass
class
SerializerHandler:
73@dataclass 74class SerializerHandler: 75 """a handler for a specific type of object 76 77 # Parameters: 78 - `check : Callable[[JsonSerializer, Any], bool]` takes a JsonSerializer and an object, returns whether to use this handler 79 - `serialize : Callable[[JsonSerializer, Any, ObjectPath], JSONitem]` takes a JsonSerializer, an object, and the current path, returns the serialized object 80 - `desc : str` description of the handler (optional) 81 """ 82 83 # (self_config, object) -> whether to use this handler 84 check: Callable[["JsonSerializer", Any, ObjectPath], bool] 85 # (self_config, object, path) -> serialized object 86 serialize_func: Callable[["JsonSerializer", Any, ObjectPath], JSONitem] 87 # unique identifier for the handler 88 uid: str 89 # description of this serializer 90 desc: str 91 92 def serialize(self) -> dict: 93 """serialize the handler info""" 94 return { 95 # get the code and doc of the check function 96 "check": { 97 "code": safe_getsource(self.check), 98 "doc": string_as_lines(self.check.__doc__), 99 }, 100 # get the code and doc of the load function 101 "serialize_func": { 102 "code": safe_getsource(self.serialize_func), 103 "doc": string_as_lines(self.serialize_func.__doc__), 104 }, 105 # get the uid, source_pckg, priority, and desc 106 "uid": str(self.uid), 107 "source_pckg": getattr(self.serialize_func, "source_pckg", None), 108 "__module__": getattr(self.serialize_func, "__module__", None), 109 "desc": str(self.desc), 110 }
a handler for a specific type of object
Parameters:
- `check : Callable[[JsonSerializer, Any], bool]` takes a JsonSerializer and an object, returns whether to use this handler
- `serialize : Callable[[JsonSerializer, Any, ObjectPath], JSONitem]` takes a JsonSerializer, an object, and the current path, returns the serialized object
- `desc : str` description of the handler (optional)
SerializerHandler( check: Callable[[JsonSerializer, Any, tuple[Union[str, int], ...]], bool], serialize_func: Callable[[JsonSerializer, Any, tuple[Union[str, int], ...]], Union[bool, int, float, str, NoneType, List[Union[bool, int, float, str, NoneType, List[Any], Dict[str, Any]]], Dict[str, Union[bool, int, float, str, NoneType, List[Any], Dict[str, Any]]]]], uid: str, desc: str)
check: Callable[[JsonSerializer, Any, tuple[Union[str, int], ...]], bool]
serialize_func: Callable[[JsonSerializer, Any, tuple[Union[str, int], ...]], Union[bool, int, float, str, NoneType, List[Union[bool, int, float, str, NoneType, List[Any], Dict[str, Any]]], Dict[str, Union[bool, int, float, str, NoneType, List[Any], Dict[str, Any]]]]]
def
serialize(self) -> dict:
92 def serialize(self) -> dict: 93 """serialize the handler info""" 94 return { 95 # get the code and doc of the check function 96 "check": { 97 "code": safe_getsource(self.check), 98 "doc": string_as_lines(self.check.__doc__), 99 }, 100 # get the code and doc of the load function 101 "serialize_func": { 102 "code": safe_getsource(self.serialize_func), 103 "doc": string_as_lines(self.serialize_func.__doc__), 104 }, 105 # get the uid, source_pckg, priority, and desc 106 "uid": str(self.uid), 107 "source_pckg": getattr(self.serialize_func, "source_pckg", None), 108 "__module__": getattr(self.serialize_func, "__module__", None), 109 "desc": str(self.desc), 110 }
serialize the handler info
BASE_HANDLERS: None =
(SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='base types', desc='base types (bool, int, float, str, None)'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='dictionaries', desc='dictionaries'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='(list, tuple) -> list', desc='lists and tuples as lists'))
DEFAULT_HANDLERS: None =
(SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='base types', desc='base types (bool, int, float, str, None)'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='dictionaries', desc='dictionaries'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='(list, tuple) -> list', desc='lists and tuples as lists'), SerializerHandler(check=<function <lambda>>, serialize_func=<function _serialize_override_serialize_func>, uid='.serialize override', desc='objects with .serialize method'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='namedtuple -> dict', desc='namedtuples as dicts'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='dataclass -> dict', desc='dataclasses as dicts'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='path -> str', desc='Path objects as posix strings'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='obj -> str(obj)', desc='directly serialize objects in `SERIALIZE_DIRECT_AS_STR` to strings'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='numpy.ndarray', desc='numpy arrays'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='torch.Tensor', desc='pytorch tensors'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='pandas.DataFrame', desc='pandas DataFrames'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='(set, list, tuple, Iterable) -> list', desc='sets, lists, tuples, and Iterables as lists'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='fallback', desc='fallback handler -- serialize object attributes and special functions as strings'))
class
JsonSerializer:
236class JsonSerializer: 237 """Json serialization class (holds configs) 238 239 # Parameters: 240 - `array_mode : ArrayMode` 241 how to write arrays 242 (defaults to `"array_list_meta"`) 243 - `error_mode : ErrorMode` 244 what to do when we can't serialize an object (will use repr as fallback if "ignore" or "warn") 245 (defaults to `"except"`) 246 - `handlers_pre : MonoTuple[SerializerHandler]` 247 handlers to use before the default handlers 248 (defaults to `tuple()`) 249 - `handlers_default : MonoTuple[SerializerHandler]` 250 default handlers to use 251 (defaults to `DEFAULT_HANDLERS`) 252 - `write_only_format : bool` 253 changes _FORMAT_KEY keys in output to "__write_format__" (when you want to serialize something in a way that zanj won't try to recover the object when loading) 254 (defaults to `False`) 255 256 # Raises: 257 - `ValueError`: on init, if `args` is not empty 258 - `SerializationException`: on `json_serialize()`, if any error occurs when trying to serialize an object and `error_mode` is set to `ErrorMode.EXCEPT"` 259 260 """ 261 262 def __init__( 263 self, 264 *args, 265 array_mode: ArrayMode = "array_list_meta", 266 error_mode: ErrorMode = ErrorMode.EXCEPT, 267 handlers_pre: MonoTuple[SerializerHandler] = tuple(), 268 handlers_default: MonoTuple[SerializerHandler] = DEFAULT_HANDLERS, 269 write_only_format: bool = False, 270 ): 271 if len(args) > 0: 272 raise ValueError( 273 f"JsonSerializer takes no positional arguments!\n{args = }" 274 ) 275 276 self.array_mode: ArrayMode = array_mode 277 self.error_mode: ErrorMode = ErrorMode.from_any(error_mode) 278 self.write_only_format: bool = write_only_format 279 # join up the handlers 280 self.handlers: MonoTuple[SerializerHandler] = tuple(handlers_pre) + tuple( 281 handlers_default 282 ) 283 284 def json_serialize( 285 self, 286 obj: Any, 287 path: ObjectPath = tuple(), 288 ) -> JSONitem: 289 try: 290 for handler in self.handlers: 291 if handler.check(self, obj, path): 292 output: JSONitem = handler.serialize_func(self, obj, path) 293 if self.write_only_format: 294 if isinstance(output, dict) and _FORMAT_KEY in output: 295 new_fmt: JSONitem = output.pop(_FORMAT_KEY) 296 output["__write_format__"] = new_fmt 297 return output 298 299 raise ValueError(f"no handler found for object with {type(obj) = }") 300 301 except Exception as e: 302 if self.error_mode == "except": 303 obj_str: str = repr(obj) 304 if len(obj_str) > 1000: 305 obj_str = obj_str[:1000] + "..." 306 raise SerializationException( 307 f"error serializing at {path = } with last handler: '{handler.uid}'\nfrom: {e}\nobj: {obj_str}" 308 ) from e 309 elif self.error_mode == "warn": 310 warnings.warn( 311 f"error serializing at {path = }, will return as string\n{obj = }\nexception = {e}" 312 ) 313 314 return repr(obj) 315 316 def hashify( 317 self, 318 obj: Any, 319 path: ObjectPath = tuple(), 320 force: bool = True, 321 ) -> Hashableitem: 322 """try to turn any object into something hashable""" 323 data = self.json_serialize(obj, path=path) 324 325 # recursive hashify, turning dicts and lists into tuples 326 return _recursive_hashify(data, force=force)
Json serialization class (holds configs)
Parameters:
array_mode : ArrayMode
how to write arrays (defaults to"array_list_meta"
)error_mode : ErrorMode
what to do when we can't serialize an object (will use repr as fallback if "ignore" or "warn") (defaults to"except"
)handlers_pre : MonoTuple[SerializerHandler]
handlers to use before the default handlers (defaults totuple()
)handlers_default : MonoTuple[SerializerHandler]
default handlers to use (defaults toDEFAULT_HANDLERS
)write_only_format : bool
changes _FORMAT_KEY keys in output to "__write_format__" (when you want to serialize something in a way that zanj won't try to recover the object when loading) (defaults toFalse
)
Raises:
ValueError
: on init, ifargs
is not emptySerializationException
: onjson_serialize()
, if any error occurs when trying to serialize an object anderror_mode
is set toErrorMode.EXCEPT"
JsonSerializer( *args, array_mode: Literal['list', 'array_list_meta', 'array_hex_meta', 'array_b64_meta', 'external', 'zero_dim'] = 'array_list_meta', error_mode: muutils.errormode.ErrorMode = ErrorMode.Except, handlers_pre: None = (), handlers_default: None = (SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='base types', desc='base types (bool, int, float, str, None)'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='dictionaries', desc='dictionaries'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='(list, tuple) -> list', desc='lists and tuples as lists'), SerializerHandler(check=<function <lambda>>, serialize_func=<function _serialize_override_serialize_func>, uid='.serialize override', desc='objects with .serialize method'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='namedtuple -> dict', desc='namedtuples as dicts'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='dataclass -> dict', desc='dataclasses as dicts'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='path -> str', desc='Path objects as posix strings'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='obj -> str(obj)', desc='directly serialize objects in `SERIALIZE_DIRECT_AS_STR` to strings'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='numpy.ndarray', desc='numpy arrays'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='torch.Tensor', desc='pytorch tensors'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='pandas.DataFrame', desc='pandas DataFrames'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='(set, list, tuple, Iterable) -> list', desc='sets, lists, tuples, and Iterables as lists'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='fallback', desc='fallback handler -- serialize object attributes and special functions as strings')), write_only_format: bool = False)
262 def __init__( 263 self, 264 *args, 265 array_mode: ArrayMode = "array_list_meta", 266 error_mode: ErrorMode = ErrorMode.EXCEPT, 267 handlers_pre: MonoTuple[SerializerHandler] = tuple(), 268 handlers_default: MonoTuple[SerializerHandler] = DEFAULT_HANDLERS, 269 write_only_format: bool = False, 270 ): 271 if len(args) > 0: 272 raise ValueError( 273 f"JsonSerializer takes no positional arguments!\n{args = }" 274 ) 275 276 self.array_mode: ArrayMode = array_mode 277 self.error_mode: ErrorMode = ErrorMode.from_any(error_mode) 278 self.write_only_format: bool = write_only_format 279 # join up the handlers 280 self.handlers: MonoTuple[SerializerHandler] = tuple(handlers_pre) + tuple( 281 handlers_default 282 )
array_mode: Literal['list', 'array_list_meta', 'array_hex_meta', 'array_b64_meta', 'external', 'zero_dim']
error_mode: muutils.errormode.ErrorMode
def
json_serialize( self, obj: Any, path: tuple[typing.Union[str, int], ...] = ()) -> Union[bool, int, float, str, NoneType, List[Union[bool, int, float, str, NoneType, List[Any], Dict[str, Any]]], Dict[str, Union[bool, int, float, str, NoneType, List[Any], Dict[str, Any]]]]:
284 def json_serialize( 285 self, 286 obj: Any, 287 path: ObjectPath = tuple(), 288 ) -> JSONitem: 289 try: 290 for handler in self.handlers: 291 if handler.check(self, obj, path): 292 output: JSONitem = handler.serialize_func(self, obj, path) 293 if self.write_only_format: 294 if isinstance(output, dict) and _FORMAT_KEY in output: 295 new_fmt: JSONitem = output.pop(_FORMAT_KEY) 296 output["__write_format__"] = new_fmt 297 return output 298 299 raise ValueError(f"no handler found for object with {type(obj) = }") 300 301 except Exception as e: 302 if self.error_mode == "except": 303 obj_str: str = repr(obj) 304 if len(obj_str) > 1000: 305 obj_str = obj_str[:1000] + "..." 306 raise SerializationException( 307 f"error serializing at {path = } with last handler: '{handler.uid}'\nfrom: {e}\nobj: {obj_str}" 308 ) from e 309 elif self.error_mode == "warn": 310 warnings.warn( 311 f"error serializing at {path = }, will return as string\n{obj = }\nexception = {e}" 312 ) 313 314 return repr(obj)
def
hashify( self, obj: Any, path: tuple[typing.Union[str, int], ...] = (), force: bool = True) -> Union[bool, int, float, str, tuple]:
316 def hashify( 317 self, 318 obj: Any, 319 path: ObjectPath = tuple(), 320 force: bool = True, 321 ) -> Hashableitem: 322 """try to turn any object into something hashable""" 323 data = self.json_serialize(obj, path=path) 324 325 # recursive hashify, turning dicts and lists into tuples 326 return _recursive_hashify(data, force=force)
try to turn any object into something hashable
def
json_serialize( obj: Any, path: tuple[typing.Union[str, int], ...] = ()) -> Union[bool, int, float, str, NoneType, List[Union[bool, int, float, str, NoneType, List[Any], Dict[str, Any]]], Dict[str, Union[bool, int, float, str, NoneType, List[Any], Dict[str, Any]]]]:
332def json_serialize(obj: Any, path: ObjectPath = tuple()) -> JSONitem: 333 """serialize object to json-serializable object with default config""" 334 return GLOBAL_JSON_SERIALIZER.json_serialize(obj, path=path)
serialize object to json-serializable object with default config