Package shaystack
Implementation of Haystack project https://www.project-haystack.org/ Propose API :
- to read or write Haystack file (Zinc, JSon, CSV)
- to manipulate ontology in memory (Grid class)
- to implement REST API (https://project-haystack.org/doc/docHaystack/HttpApi)
- to implement GraphQL API
With some sample provider:
- Import ontology on S3 bucket
- Import ontology on SQLite or Postgres
- and expose the data via Flask or AWS Lambda
Expand source code
# -*- coding: utf-8 -*-
# Haystack module
# See the accompanying LICENSE file.
# (C) 2016 VRT Systems
# (C) 2021 Engie Digital
#
# vim: set ts=4 sts=4 et tw=78 sw=4 si:
"""
Implementation of Haystack project https://www.project-haystack.org/
Propose API :
- to read or write Haystack file (Zinc, JSon, CSV)
- to manipulate ontology in memory (Grid class)
- to implement REST API (https://project-haystack.org/doc/docHaystack/HttpApi)
- to implement GraphQL API
With some sample provider:
- Import ontology on S3 bucket
- Import ontology on SQLite or Postgres
- and expose the data via Flask or AWS Lambda
"""
from .datatypes import Quantity, Coordinate, Uri, Bin, MARKER, NA, \
REMOVE, Ref, XStr
from .dumper import dump, dump_scalar
from .grid import Grid
from .grid_filter import parse_filter, parse_hs_datetime_format
from .metadata import MetadataObject
from .ops import *
from .parser import parse, parse_scalar, MODE, MODE_JSON, MODE_TRIO, MODE_ZINC, MODE_CSV, \
suffix_to_mode, mode_to_suffix
from .pintutil import unit_reg
from .providers import HaystackInterface
from .type import HaystackType, Entity
from .version import Version, VER_2_0, VER_3_0, LATEST_VER
__all__ = ['Grid', 'dump', 'parse', 'dump_scalar', 'parse_scalar', 'parse_filter',
'MetadataObject', 'unit_reg', 'zoneinfo',
'HaystackType', 'Entity',
'Coordinate', 'Uri', 'Bin', 'XStr', 'Quantity', 'MARKER', 'NA', 'REMOVE', 'Ref',
'MODE', 'MODE_JSON', 'MODE_ZINC', 'MODE_TRIO', 'MODE_CSV', 'suffix_to_mode', 'mode_to_suffix',
'parse_hs_datetime_format',
'VER_2_0', 'VER_3_0', 'LATEST_VER', 'Version',
"HaystackInterface",
"about",
"ops",
"formats",
"read",
"nav",
"watch_sub",
"watch_unsub",
"watch_poll",
"point_write",
"his_read",
"his_write",
"invoke_action",
]
__pdoc__ = {
"csvdumper": False,
"csvparser": False,
"datatypes": False,
"dumper": False,
"filter_ast": False,
"grid": False,
"grid_diff": False,
"grid_filter": False,
"jsondumper": False,
"jsonparser": False,
"metadata": False,
"ops": False,
"parser": False,
"pintutil": False,
"sortabledict": False,
"triodumper": False,
"trioparser": False,
"version": False,
"zincdumper": False,
"zincparser": False,
"zoneinfo": False,
}
__author__ = 'Engie Digital, VRT Systems'
__copyright__ = 'Copyright 2016-2020, Engie Digital & VRT System'
__credits__ = ['See AUTHORS']
__license__ = 'BSD'
__maintainer__ = 'Philippe PRADOS'
__email__ = 'shaystack@prados.fr'
Sub-modules
shaystack.empty_grid
-
Read-only Empty grid
shaystack.exception
-
Specific exceptions for Haystack API
shaystack.providers
-
Implementation of Haystack API
shaystack.tools
-
Tools for all parser and dumper
shaystack.type
-
The typing for Haystack
Functions
def MODE(x)
-
Expand source code
def new_type(x): return x
def about(envs: Dict[str, str], request: shaystack.ops.HaystackHttpRequest, stage: str) ‑> shaystack.ops.HaystackHttpResponse
-
Implement Haystack about.
Args
envs
- The environments variables
request
- The HTTP Request
stage
- The current stage (
prod
,dev
, etc.)
Returns
The HTTP Response
Expand source code
def about(envs: Dict[str, str], request: HaystackHttpRequest, stage: str) -> HaystackHttpResponse: """ Implement Haystack about. Args: envs: The environments variables request: The HTTP Request stage: The current stage (`prod`, `dev`, etc.) Returns: The HTTP Response """ headers = request.headers log.debug("HAYSTACK_PROVIDER=%s", envs.get("HAYSTACK_PROVIDER", None)) log.debug("HAYSTACK_DB=%s", envs.get("HAYSTACK_DB", None)) try: provider = get_singleton_provider(envs) if headers["Host"].startswith("localhost:"): home = "http://" + headers["Host"] + "/" else: home = "https://" + headers["Host"] + "/" + stage grid_response = provider.about(home) assert grid_response is not None return _format_response(headers, grid_response, 200, "OK") except Exception as ex: # pylint: disable=broad-except response = _manage_exception(headers, ex, stage) return response
def dump(grid: shaystack.grid.Grid, mode: Mode = 'text/zinc') ‑> str
-
Dump a single grid in the specified over-the-wire format.
Args
grid
- The grid to dump.
mode
- The format. Must be MODE_ZINC, MODE_CSV or MODE_JSON
Expand source code
def dump(grid: Grid, mode: MODE = MODE_ZINC) -> str: """ Dump a single grid in the specified over-the-wire format. Args: grid: The grid to dump. mode: The format. Must be MODE_ZINC, MODE_CSV or MODE_JSON """ if mode == MODE_ZINC: return dump_zinc_grid(grid) if mode == MODE_TRIO: return dump_trio_grid(grid) if mode == MODE_JSON: return dump_json_grid(grid) if mode == MODE_CSV: return dump_csv_grid(grid) raise NotImplementedError('Format not implemented: %s' % mode)
def dump_scalar(scalar: Any, mode: Mode = 'text/zinc', version: shaystack.version.Version = <shaystack.version.Version object>) ‑> Union[str, NoneType]
-
Dump a scalar value in the specified over-the-wire format and version.
Args
scalar
- The value to dump
mode
- The format. Must be MODE_ZINC, MODE_CSV or MODE_JSON
version
- The Haystack version to apply
Expand source code
def dump_scalar(scalar: Any, mode: MODE = MODE_ZINC, version: Version = LATEST_VER) -> Optional[str]: """ Dump a scalar value in the specified over-the-wire format and version. Args: scalar: The value to dump mode: The format. Must be MODE_ZINC, MODE_CSV or MODE_JSON version: The Haystack version to apply """ if mode == MODE_ZINC: return dump_zinc_scalar(scalar, version=version) if mode == MODE_TRIO: return dump_trio_scalar(scalar, version=version) if mode == MODE_JSON: return dump_json_scalar(scalar, version=version) if mode == MODE_CSV: return dump_csv_scalar(scalar, version=version) raise NotImplementedError('Format not implemented: %s' % mode)
def formats(envs: Dict[str, str], request: shaystack.ops.HaystackHttpRequest, stage: str) ‑> shaystack.ops.HaystackHttpResponse
-
Implement Haystack 'formats'.
Args
envs
- The environments variables
request
- The HTTP Request
stage
- The current stage (
prod
,dev
, etc.)
Returns
The HTTP Response
Expand source code
def formats(envs: Dict[str, str], request: HaystackHttpRequest, stage: str) -> HaystackHttpResponse: """ Implement Haystack 'formats'. Args: envs: The environments variables request: The HTTP Request stage: The current stage (`prod`, `dev`, etc.) Returns: The HTTP Response """ headers = request.headers try: provider = get_singleton_provider(envs) grid_response = provider.formats() if grid_response is None: grid_response = Grid( version=_DEFAULT_VERSION, columns={ "mime": {}, "receive": {}, "send": {}, }, ) grid_response.extend( [ { "mime": MODE_ZINC, "receive": MARKER, "send": MARKER, }, { "mime": MODE_TRIO, "receive": MARKER, "send": MARKER, }, { "mime": MODE_JSON, "receive": MARKER, "send": MARKER, }, { "mime": MODE_CSV, "receive": MARKER, "send": MARKER, }, ] ) response = _format_response(headers, grid_response, 200, "OK") except Exception as ex: # pylint: disable=broad-except response = _manage_exception(headers, ex, stage) return response
def his_read(envs: Dict[str, str], request: shaystack.ops.HaystackHttpRequest, stage: str) ‑> shaystack.ops.HaystackHttpResponse
-
Implement Haystack 'hisRead'.
Args
envs
- The environments variables
request
- The HTTP Request
stage
- The current stage (
prod
,dev
, etc.)
Returns
The HTTP Response
Expand source code
def his_read(envs: Dict[str, str], request: HaystackHttpRequest, stage: str) -> HaystackHttpResponse: """ Implement Haystack 'hisRead'. Args: envs: The environments variables request: The HTTP Request stage: The current stage (`prod`, `dev`, etc.) Returns: The HTTP Response """ headers, args = (request.headers, request.args) try: provider = get_singleton_provider(envs) grid_request = _parse_body(request) entity_id = date_version = None date_range = None default_tz = provider.get_tz() if grid_request: if "id" in grid_request.column: entity_id = grid_request[0].get("id", "") if "range" in grid_request.column: date_range = grid_request[0].get("range", "") date_version = ( grid_request[0].get("version", None) if grid_request else None ) # Priority of query string if args: if "id" in args: entity_id = Ref(args["id"][1:]) if "range" in args: date_range = args["range"] if "version" in args: date_version = parse_hs_datetime_format(args["version"], default_tz) grid_date_range = parse_date_range(date_range, provider.get_tz()) log.debug( "id=%s range=%s, date_version=%s", entity_id, grid_date_range, date_version ) if date_version: if isinstance(date_version, date): date_version = datetime.combine(date_version, datetime.max.time()) \ .replace(tzinfo=provider.get_tz()) if grid_date_range[1] > date_version: grid_date_range = (grid_date_range[0], date_version) grid_response = provider.his_read(entity_id, grid_date_range, date_version) assert grid_response is not None response = _format_response(headers, grid_response, 200, "OK") except Exception as ex: # pylint: disable=broad-except response = _manage_exception(headers, ex, stage) return response
def his_write(envs: Dict[str, str], request: shaystack.ops.HaystackHttpRequest, stage: str) ‑> shaystack.ops.HaystackHttpResponse
-
Implement Haystack 'hisWrite'.
Args
envs
- The environments variables
request
- The HTTP Request
stage
- The current stage (
prod
,dev
, etc.)
Returns
The HTTP Response
Expand source code
def his_write(envs: Dict[str, str], request: HaystackHttpRequest, stage: str) -> HaystackHttpResponse: """ Implement Haystack 'hisWrite'. Args: envs: The environments variables request: The HTTP Request stage: The current stage (`prod`, `dev`, etc.) Returns: The HTTP Response """ headers, args = (request.headers, request.args) try: provider = get_singleton_provider(envs) grid_request = _parse_body(request) entity_id = grid_request.metadata.get("id") date_version = grid_request.metadata.get("version") time_serie_grid = grid_request default_tz = provider.get_tz() # Priority of query string if args: if "id" in args: entity_id = Ref(args["id"][1:]) if "ts" in args: # Array of tuple time_serie_grid = Grid(version=VER_3_0, columns=["date", "val"]) time_serie_grid.extend( [ {"date": parse_hs_datetime_format(d, default_tz), "val": v} for d, v in literal_eval(args["ts"]) ] ) if "version" in args: date_version = parse_hs_datetime_format(args["version"], default_tz) grid_response = provider.his_write(entity_id, time_serie_grid, date_version) assert grid_response is not None response = _format_response(headers, grid_response, 200, "OK") except Exception as ex: # pylint: disable=broad-except response = _manage_exception(headers, ex, stage) return response
def invoke_action(envs: Dict[str, str], request: shaystack.ops.HaystackHttpRequest, stage: str) ‑> shaystack.ops.HaystackHttpResponse
-
Implement Haystack 'invokeAction'.
Args
envs
- The environments variables
request
- The HTTP Request
stage
- The current stage (
prod
,dev
, etc.)
Returns
The HTTP Response
Expand source code
def invoke_action(envs: Dict[str, str], request: HaystackHttpRequest, stage: str) -> HaystackHttpResponse: """ Implement Haystack 'invokeAction'. Args: envs: The environments variables request: The HTTP Request stage: The current stage (`prod`, `dev`, etc.) Returns: The HTTP Response """ headers, args = (request.headers, request.args) try: provider = get_singleton_provider(envs) grid_request = _parse_body(request) entity_id = grid_request.metadata.get("id") action = grid_request.metadata.get("action") # Priority of query string if args: if "id" in args: entity_id = Ref(args["id"][1:]) if "action" in args: action = args["action"] params = grid_request[0] if grid_request else {} grid_response = provider.invoke_action(entity_id, action, params) assert grid_response is not None response = _format_response(headers, grid_response, 200, "OK") except Exception as ex: # pylint: disable=broad-except response = _manage_exception(headers, ex, stage) return response
def mode_to_suffix(mode: Mode) ‑> Union[str, NoneType]
-
Convert haystack mode to file suffix
Args
mode
- The haystack mode (
MODE_…
)
Returns
The file suffix (
.zinc
,.json
,.trio
or.csv
)Expand source code
def mode_to_suffix(mode: MODE) -> Optional[str]: """Convert haystack mode to file suffix Args: mode: The haystack mode (`MODE_...`) Returns: The file suffix (`.zinc`, `.json`, `.trio` or `.csv`) """ return _mode_to_suffix.get(mode, None)
-
Implement Haystack 'nav'.
Args
envs
- The environments variables
request
- The HTTP Request
stage
- The current stage (
prod
,dev
, etc.)
Returns
The HTTP Response
Expand source code
def nav(envs: Dict[str, str], request: HaystackHttpRequest, stage: str) -> HaystackHttpResponse: """ Implement Haystack 'nav'. Args: envs: The environments variables request: The HTTP Request stage: The current stage (`prod`, `dev`, etc.) Returns: The HTTP Response """ headers, args = (request.headers, request.args) try: provider = get_singleton_provider(envs) grid_request = _parse_body(request) nav_id = None if grid_request and "navId" in grid_request.column: nav_id = grid_request[0]["navId"] if args and "navId" in args: nav_id = args["navId"] grid_response = provider.nav(nav_id=nav_id) assert grid_response is not None response = _format_response(headers, grid_response, 200, "OK") except Exception as ex: # pylint: disable=broad-except response = _manage_exception(headers, ex, stage) return response
def parse(grid_str: str, mode: Mode = 'text/zinc') ‑> shaystack.grid.Grid
-
Parse a grid.
Args
grid_str
- The string to parse
mode
- The format (
MODE_…
)
Returns
a grid
Expand source code
def parse(grid_str: str, mode: MODE = MODE_ZINC) -> Grid: # Decode incoming text """ Parse a grid. Args: grid_str: The string to parse mode: The format (`MODE_...`) Returns: a grid """ if isinstance(grid_str, bytes): # pragma: no cover # No coverage here, because it *should* be handled above unless the user # is preempting us by calling `parse_grid` directly. if grid_str[:2] == b'\xef\xbb': grid_str = grid_str.decode(encoding="utf-8-sig") else: grid_str = grid_str.decode(encoding="utf-8") if grid_str and grid_str[-1] not in ['\n', '\r']: grid_str += '\n' if mode == MODE_ZINC: return parse_zinc_grid(grid_str) if mode == MODE_TRIO: return parse_trio_grid(grid_str) if mode == MODE_JSON: return parse_json_grid(grid_str) if mode == MODE_CSV: return parse_csv_grid(grid_str) raise NotImplementedError('Format not implemented: %s' % mode)
def parse_filter(grid_filter: str) ‑> shaystack.filter_ast.FilterAST
-
Return an AST tree of filter. Can be used to generate other language (Python, SQL, etc.)
Args
grid_filter
- A filter request
Returns
A
FilterAST
Expand source code
def parse_filter(grid_filter: str) -> FilterAST: """Return an AST tree of filter. Can be used to generate other language (Python, SQL, etc.) Args: grid_filter: A filter request Returns: A `FilterAST` """ with pyparser_lock: return FilterAST(hs_filter.parseString(grid_filter, parseAll=True)[0])
def parse_hs_datetime_format(datetime_str: str, timezone: datetime.tzinfo) ‑> datetime.datetime
-
Parse the haystack date time (for filter).
Args
datetime_str
- The string to parse
timezone
- Time zone info
Returns
the corresponding
datetime
Raises
pyparsing.ParseException
if the string does not conformExpand source code
def parse_hs_datetime_format(datetime_str: str, timezone: tzinfo) -> datetime: """ Parse the haystack date time (for filter). Args: datetime_str: The string to parse timezone: Time zone info Returns: the corresponding `datetime` Raises: `pyparsing.ParseException` if the string does not conform """ if datetime_str == "today": return datetime.combine(date.today(), datetime.min.time()) \ .replace(tzinfo=timezone) if datetime_str == "yesterday": return datetime.combine(date.today() - timedelta(days=1), datetime.min.time()) \ .replace(tzinfo=timezone) if datetime_str == "today": return datetime.combine(date.today(), datetime.min.time()) \ .replace(tzinfo=timezone) return hs_all_date.parseString(datetime_str, parseAll=True)[0]
def parse_scalar(scalar: Union[bytes, str], mode: Mode = 'text/zinc', version: Union[shaystack.version.Version, str] = <shaystack.version.Version object>) ‑> Any
-
Parse a scalar value
Args
scalar
- The scalar data to parse
mode
- The haystack mode
version
- The haystack version
Returns
a value
Expand source code
def parse_scalar(scalar: Union[bytes, str], mode: MODE = MODE_ZINC, version: Union[Version, str] = LATEST_VER) -> Any: # Decode version string """ Parse a scalar value Args: scalar: The scalar data to parse mode: The haystack mode version: The haystack version Returns: a value """ charset = 'utf-8' if not isinstance(version, Version): version = Version(version) # Decode incoming text if isinstance(scalar, bytes): scalar = scalar.decode(encoding=charset) if mode == MODE_ZINC: return parse_zinc_scalar(scalar, version=version) if mode == MODE_TRIO: return parse_trio_scalar(scalar, version=version) if mode == MODE_JSON: return parse_json_scalar(scalar, version=version) if mode == MODE_CSV: return parse_csv_scalar(scalar, version=version) raise NotImplementedError('Format not implemented: %s' % mode)
def point_write(envs: Dict[str, str], request: shaystack.ops.HaystackHttpRequest, stage: str) ‑> shaystack.ops.HaystackHttpResponse
-
Implement Haystack 'pointWrite'.
Args
envs
- The environments variables
request
- The HTTP Request
stage
- The current stage (
prod
,dev
, etc.)
Returns
The HTTP Response
Expand source code
def point_write(envs: Dict[str, str], request: HaystackHttpRequest, stage: str) -> HaystackHttpResponse: """ Implement Haystack 'pointWrite'. Args: envs: The environments variables request: The HTTP Request stage: The current stage (`prod`, `dev`, etc.) Returns: The HTTP Response """ headers, args = (request.headers, request.args) try: provider = get_singleton_provider(envs) grid_request = _parse_body(request) date_version = None level = 17 val = who = duration = None entity_id = None default_tz = provider.get_tz() if grid_request: entity_id = grid_request[0]["id"] date_version = grid_request[0].get("version", None) if "level" in grid_request[0]: level = int(grid_request[0]["level"]) val = grid_request[0].get("val") who = grid_request[0].get("who") duration = grid_request[0].get("duration") # Must be quantity if "id" in args: entity_id = Ref(args["id"][1:]) if "level" in args: level = int(args["level"]) if "val" in args: val = parse_scalar( args["val"], mode=MODE_ZINC, ) if "who" in args: val = args["who"] if "duration" in args: duration = parse_scalar(args["duration"]) assert isinstance(duration, Quantity) if "version" in args: date_version = parse_hs_datetime_format(args["version"], default_tz) if entity_id is None: raise ValueError("'id' must be set") if val is not None: provider.point_write_write( entity_id, level, val, who, duration, date_version ) grid_response = EmptyGrid else: grid_response = provider.point_write_read(entity_id, date_version) assert grid_response is not None assert "level" in grid_response.column assert "levelDis" in grid_response.column assert "val" in grid_response.column assert "who" in grid_response.column response = _format_response(headers, grid_response, 200, "OK") except Exception as ex: # pylint: disable=broad-except response = _manage_exception(headers, ex, stage) return response
def read(envs: Dict[str, str], request: shaystack.ops.HaystackHttpRequest, stage: str) ‑> shaystack.ops.HaystackHttpResponse
-
Implement Haystack
read()
Args
envs
- The environments variables
request
- The HTTP Request
stage
- The current stage (
prod
,dev
, etc.)
Returns
The HTTP Response
Expand source code
def read(envs: Dict[str, str], request: HaystackHttpRequest, stage: str) -> HaystackHttpResponse: """ Implement Haystack `read` Args: envs: The environments variables request: The HTTP Request stage: The current stage (`prod`, `dev`, etc.) Returns: The HTTP Response """ headers, args = (request.headers, request.args) try: provider = get_singleton_provider(envs) grid_request = _parse_body(request) read_ids: Optional[List[Ref]] = None select = read_filter = date_version = None limit = 0 default_tz = provider.get_tz() if grid_request: if "id" in grid_request.column: read_ids = [row["id"] for row in grid_request] else: if "filter" in grid_request.column: read_filter = grid_request[0].get("filter", "") else: read_filter = "" if "limit" in grid_request.column: limit = int(grid_request[0].get("limit", 0)) if "select" in grid_request.column: select = grid_request[0].get("select", "*") date_version = ( grid_request[0].get("version", None) if grid_request else None ) # Priority of query string if args: if "id" in args: read_ids = [Ref(entity_id) for entity_id in args["id"].split(",")] else: if "filter" in args: read_filter = args["filter"] if "limit" in args: limit = int(args["limit"]) if "select" in args: select = args["select"] if "version" in args: date_version = parse_hs_datetime_format(args["version"], default_tz) if read_filter is None: read_filter = "" if read_ids is None and read_filter is None: raise ValueError("'id' or 'filter' must be set") log.debug( "id=%s select='%s' filter='%s' limit=%s, date_version=%s", read_ids, select, read_filter, limit, date_version, ) grid_response = provider.read(limit, select, read_ids, read_filter, date_version) assert grid_response is not None response = _format_response(headers, grid_response, 200, "OK") except Exception as ex: # pylint: disable=broad-except response = _manage_exception(headers, ex, stage) return response
def suffix_to_mode(ext: str) ‑> Union[Mode, NoneType]
-
Convert a file suffix to Haystack mode
Args
ext
- The file suffix (
.zinc
,.json
,.trio
or.csv
)
Returns
The corresponding haystack mode (
MODE_…
)Expand source code
def suffix_to_mode(ext: str) -> Optional[MODE]: """Convert a file suffix to Haystack mode Args: ext: The file suffix (`.zinc`, `.json`, `.trio` or `.csv`) Returns: The corresponding haystack mode (`MODE_...`) """ return cast(Optional[MODE], _suffix_to_mode.get(ext, None))
def watch_poll(envs: Dict[str, str], request: shaystack.ops.HaystackHttpRequest, stage: str) ‑> shaystack.ops.HaystackHttpResponse
-
Implement Haystack 'watchPoll'.
Args
envs
- The environments variables
request
- The HTTP Request
stage
- The current stage (
prod
,dev
, etc.)
Returns
The HTTP Response
Expand source code
def watch_poll(envs: Dict[str, str], request: HaystackHttpRequest, stage: str) -> HaystackHttpResponse: """ Implement Haystack 'watchPoll'. Args: envs: The environments variables request: The HTTP Request stage: The current stage (`prod`, `dev`, etc.) Returns: The HTTP Response """ headers, args = (request.headers, request.args) try: provider = get_singleton_provider(envs) grid_request = _parse_body(request) watch_id = None refresh = False if grid_request: if "watchId" in grid_request.metadata: watch_id = grid_request.metadata["watchId"] if "refresh" in grid_request.metadata: refresh = True if args: if "watchId" in args: watch_id = args["watchId"] if "refresh" in args: refresh = True grid_response = provider.watch_poll(watch_id, refresh) assert grid_response is not None response = _format_response(headers, grid_response, 200, "OK") except Exception as ex: # pylint: disable=broad-except response = _manage_exception(headers, ex, stage) return response
def watch_sub(envs: Dict[str, str], request: shaystack.ops.HaystackHttpRequest, stage: str) ‑> shaystack.ops.HaystackHttpResponse
-
Implement Haystack 'watchSub'.
Args
envs
- The environments variables
request
- The HTTP Request
stage
- The current stage (
prod
,dev
, etc.)
Returns
The HTTP Response
Expand source code
def watch_sub(envs: Dict[str, str], request: HaystackHttpRequest, stage: str) -> HaystackHttpResponse: """ Implement Haystack 'watchSub'. Args: envs: The environments variables request: The HTTP Request stage: The current stage (`prod`, `dev`, etc.) Returns: The HTTP Response """ headers, args = (request.headers, request.args) try: provider = get_singleton_provider(envs) grid_request = _parse_body(request) watch_dis = watch_id = lease = None ids = [] if grid_request: watch_dis = grid_request.metadata["watchDis"] watch_id = grid_request.metadata.get("watchId", None) if "lease" in grid_request.metadata: lease = int(grid_request.metadata["lease"]) ids = [row["id"] for row in grid_request] if args: if "watchDis" in args: watch_dis = args["watchDis"] if "watchId" in args: watch_id = args["watchId"] if "lease" in args: lease = int(args["lease"]) if "ids" in args: # Use list of str ids = [Ref(x[1:]) for x in literal_eval(args["ids"])] if not watch_dis: raise ValueError("'watchDis' and 'watchId' must be setted") grid_response = provider.watch_sub(watch_dis, watch_id, ids, lease) assert grid_response is not None assert "watchId" in grid_response.metadata assert "lease" in grid_response.metadata response = _format_response(headers, grid_response, 200, "OK") except Exception as ex: # pylint: disable=broad-except response = _manage_exception(headers, ex, stage) return response
def watch_unsub(envs: Dict[str, str], request: shaystack.ops.HaystackHttpRequest, stage: str) ‑> shaystack.ops.HaystackHttpResponse
-
Implement Haystack 'watchUnsub'.
Args
envs
- The environments variables
request
- The HTTP Request
stage
- The current stage (
prod
,dev
, etc.)
Returns
The HTTP Response
Expand source code
def watch_unsub(envs: Dict[str, str], request: HaystackHttpRequest, stage: str) -> HaystackHttpResponse: """ Implement Haystack 'watchUnsub'. Args: envs: The environments variables request: The HTTP Request stage: The current stage (`prod`, `dev`, etc.) Returns: The HTTP Response """ headers, args = (request.headers, request.args) try: provider = get_singleton_provider(envs) grid_request = _parse_body(request) close = False watch_id = False ids = [] if grid_request: if "watchId" in grid_request.metadata: watch_id = grid_request.metadata["watchId"] if "close" in grid_request.metadata: close = bool(grid_request.metadata["close"]) ids = [row["id"] for row in grid_request] if args: if "watchId" in args: watch_id = args["watchId"] if "close" in args: close = bool(args["close"]) if "ids" in args: # Use list of str ids = {Ref(x[1:]) for x in literal_eval(args["ids"])} if not watch_id: raise ValueError("'watchId' must be set") provider.watch_unsub(watch_id, ids, close) grid_response = EmptyGrid response = _format_response(headers, grid_response, 200, "OK") except Exception as ex: # pylint: disable=broad-except response = _manage_exception(headers, ex, stage) return response
Classes
class Bin (...)
-
A convenience class to allow identification of a Bin from other string types.
Expand source code
class Bin(str): """A convenience class to allow identification of a Bin from other string types. """ def __repr__(self) -> str: return '%s(%s)' % (self.__class__.__name__, super().__repr__()) def __eq__(self, other: 'Bin') -> bool: assert isinstance(other, Bin) return super().__eq__(other)
Ancestors
- builtins.str
class Coordinate (latitude: float, longitude: float)
-
A 2D co-ordinate in degrees latitude and longitude.
Args
latitude
- the latitude
longitude
- the longitude
Expand source code
class Coordinate: """A 2D co-ordinate in degrees latitude and longitude. Args: latitude: the latitude longitude: the longitude """ __slots__ = "latitude", "longitude" def __init__(self, latitude: float, longitude: float): self.latitude = latitude self.longitude = longitude def __repr__(self) -> str: return '%s(%r, %r)' % ( self.__class__.__name__, self.latitude, self.longitude ) def __str__(self) -> str: return ('%f\N{DEGREE SIGN} lat %f\N{DEGREE SIGN} long' % ( round(self.latitude, ndigits=6), round(self.longitude, ndigits=6) )) def __eq__(self, other: 'Coordinate') -> bool: if not isinstance(other, Coordinate): return False return (self.latitude == other.latitude) and \ (self.longitude == other.longitude) def __ne__(self, other: 'Coordinate') -> bool: if not isinstance(other, Coordinate): return True return not self == other def __hash__(self) -> int: return hash(self.latitude) ^ hash(self.longitude)
Instance variables
var latitude
-
Return an attribute of instance, which is of type owner.
var longitude
-
Return an attribute of instance, which is of type owner.
class Grid (version: Union[str, shaystack.version.Version, NoneType] = None, metadata: Union[NoneType, Dict[str, Union[str, int, float, bool, datetime.date, datetime.time, datetime.datetime, shaystack.datatypes.Ref, shaystack.datatypes.Quantity, shaystack.datatypes.Coordinate, shaystack.datatypes.Uri, shaystack.datatypes.Bin, shaystack.datatypes.XStr, shaystack.datatypes._MarkerType, shaystack.datatypes._NAType, shaystack.datatypes._RemoveType, List[Any], Dict[str, Any], NoneType]], shaystack.metadata.MetadataObject, shaystack.sortabledict.SortableDict] = None, columns: Union[shaystack.sortabledict.SortableDict, Dict[str, Union[str, int, float, bool, datetime.date, datetime.time, datetime.datetime, shaystack.datatypes.Ref, shaystack.datatypes.Quantity, shaystack.datatypes.Coordinate, shaystack.datatypes.Uri, shaystack.datatypes.Bin, shaystack.datatypes.XStr, shaystack.datatypes._MarkerType, shaystack.datatypes._NAType, shaystack.datatypes._RemoveType, List[Any], Dict[str, Any], NoneType]], Iterable[Union[Tuple[str, Any], str]], NoneType] = None)
-
A grid is basically a series of tabular records. The grid has a header which describes some metadata about the grid and its columns. This is followed by zero or more rows.
The grid propose different standard operator.
- It's possible to access an entity with the position in the grid:
grid[1]
- or if the entity has an
id
, with thisid
:grid[Ref("abc")]
- To extract a portion of grid, use a slice:
grid[10:12]
- To check if a specific
id
is in the grid:Ref("abc") in grid
- To calculate the difference between grid:
grid_v2 - grid_v1
- To merge two grid:
grid_a + grid_b
- To compare a grid:
grid_v1 + (grid_v2 - grid_v1) == grid_v2
- Size of grid:
len(grid)
- Replace an entity with
id
:grid[Ref("abc")] = new_entity
- Replace a group of entities:
grid[2:3] = [new_entity1,new_entity2]
- Delete an entity with
id
:del grid[Ref("abc")]
Args
version
- The haystack version (See VERSION_…)
metadata
- A dictionary with the metadata associated with the grid.
columns
- A list of columns, or a dictionary with columns name and corresponding metadata
Expand source code
class Grid(MutableSequence): # pytlint: disable=too-many-ancestors """A grid is basically a series of tabular records. The grid has a header which describes some metadata about the grid and its columns. This is followed by zero or more rows. The grid propose different standard operator. - It's possible to access an entity with the position in the grid: `grid[1]` - or if the entity has an `id`, with this `id`: `grid[Ref("abc")]` - To extract a portion of grid, use a slice: `grid[10:12]` - To check if a specific `id` is in the grid: `Ref("abc") in grid` - To calculate the difference between grid: `grid_v2 - grid_v1` - To merge two grid: `grid_a + grid_b` - To compare a grid: `grid_v1 + (grid_v2 - grid_v1) == grid_v2` - Size of grid: `len(grid)` - Replace an entity with `id`: `grid[Ref("abc")] = new_entity` - Replace a group of entities: `grid[2:3] = [new_entity1,new_entity2]` - Delete an entity with `id`: `del grid[Ref("abc")]` Args: version: The haystack version (See VERSION_...) metadata: A dictionary with the metadata associated with the grid. columns: A list of columns, or a dictionary with columns name and corresponding metadata """ __slots__ = "_version", "_version_given", "metadata", "column", "_row", "_index" def __init__(self, version: Union[str, Version, None] = None, metadata: Union[None, Entity, MetadataObject, SortableDict] = None, columns: Union[SortableDict, Entity, Iterable[Union[Tuple[str, Any], str]], None] = None): version_given = version is not None if version_given: version = Version(version) else: version = VER_3_0 self._version = version self._version_given = version_given # Metadata self.metadata = MetadataObject(validate_fn=self._detect_or_validate) # The columns self.column = SortableDict() # Rows self._row: List[Entity] = [] # Internal index self._index: Optional[Dict[Ref, Entity]] = None if metadata is not None: self.metadata.update(metadata.items()) if columns is not None: if isinstance(columns, (dict, SortableDict)): columns = list(columns.items()) elif isinstance(columns, Sequence) and columns and not isinstance(columns[0], tuple): columns = list(zip(columns, [{}] * len(columns))) for col_id, col_meta in columns: # Convert sorted lists and dicts back to a list of items. if isinstance(col_meta, (dict, SortableDict)): col_meta = list(col_meta.items()) metadata_object = MetadataObject(validate_fn=self._detect_or_validate) metadata_object.extend(col_meta) self.column.add_item(col_id, metadata_object) # noinspection PyArgumentList @staticmethod def _approx_check(version_1: Any, version_2: Any) -> bool: """ Compare two values with a tolerance Args: version_1: value one version_2: value two Returns: true if the values are approximately identical """ if isinstance(version_1, numbers.Number) and isinstance(version_2, numbers.Number): # noinspection PyUnresolvedReferences return abs(version_1 - version_2) < 0.000001 # pylint: disable=C0123 if type(version_1) != type(version_2) and \ not (isinstance(version_1, str) and isinstance(version_2, str)): return False # pylint: enable=C0123 if isinstance(version_1, datetime.time) and isinstance(version_2, datetime.time): # noinspection PyArgumentList return version_1.replace(microsecond=0) == version_2.replace(microsecond=0) if isinstance(version_1, datetime.datetime) and isinstance(version_2, datetime.datetime): dt1, dt2 = version_1.replace(tzinfo=pytz.UTC), version_2.replace(tzinfo=pytz.UTC) return dt1.date() == dt2.date() and Grid._approx_check(dt1.time(), dt2.time()) if isinstance(version_1, Quantity) and isinstance(version_2, Quantity): return version_1.units == version_2.units and \ Grid._approx_check(version_1.m, version_2.m) if isinstance(version_1, Coordinate) and isinstance(version_2, Coordinate): return Grid._approx_check(version_1.latitude, version_2.latitude) and \ Grid._approx_check(version_1.longitude, version_2.longitude) if isinstance(version_1, dict) and isinstance(version_2, dict): for key, val in version_1.items(): if not Grid._approx_check(val, version_2.get(key, None)): return False for key, val in version_2.items(): if key not in version_1 and not Grid._approx_check(version_1.get(key, None), val): return False return True return version_1 == version_2 def __eq__(self, other: 'Grid') -> bool: """ Campare two grid with tolerance. Args: other: Other grid Returns: true if equals """ if not isinstance(other, Grid): return False if set(self.metadata.keys()) != set(other.metadata.keys()): return False for key in self.metadata.keys(): if not Grid._approx_check(self.metadata[key], other.metadata[key]): return False # Check column matches if set(self.column.keys()) != set(other.column.keys()): return False for col_name in self.column.keys(): if col_name not in other.column or \ len(self.column[col_name]) != len(other.column[col_name]): return False for key in self.column[col_name].keys(): if not Grid._approx_check(self.column[col_name][key], other.column[col_name][key]): return False # Check row matches if len(self) != len(other): return False pending_right_row = [id(row) for row in other if 'id' not in row] for left in self._row: # Search record in other with same values find = False if 'id' in left: if left['id'] in other and self._approx_check(left, other[left['id']]): find = True else: for right in other._row: if id(right) not in pending_right_row: continue if self._approx_check(left, right): find = True pending_right_row.remove(id(right)) break if not find: return False return True def __sub__(self, other: 'Grid') -> 'Grid': """Calculate the difference between two grid. The result is a grid with only the attributs to update (change value, delete, etc) If a row with id must be removed, - if the row has an id, the result add a row with this id, and a tag 'remove_' - if the row has not an id, the result add a row with all values of the original row, and a tag 'remove_' It's possible to update all metadatas, the order of columns, add, remove or update some rows It's possible to apply the result in a grid, with the add operator. At all time, with gridA and gridB, gridA + (gridB - gridA) == gridB Args: other: grid to substract Returns: Only the difference between grid. """ assert isinstance(other, Grid) from .grid_diff import grid_diff # pylint: disable: import-outside-toplevel return grid_diff(other, self) def __add__(self, other: 'Grid') -> 'Grid': """Merge two grids. The metadata can be modified with the values from other. Some attributs can be removed if the `other` attributs is REMOVE. If a row have a 'remove_' tag, the corresponding row was removed. The result of __sub__() can be used to patch the current grid. At all time, with `gridA` and `gridB`, `gridA + (gridB - gridA) == gridB` Args: other: List of entities to updates Returns: A new grid """ assert isinstance(other, Grid) from .grid_diff import grid_merge # pylint: disable: import-outside-toplevel if 'diff_' in self.metadata: return grid_merge(other.copy(), self) return grid_merge(self.copy(), other) def __repr__(self) -> str: # pragma: no cover """Return a representation of this grid.""" parts = ['\tVersion: %s' % str(self.version)] if bool(self.metadata): parts.append('\tMetadata: %s' % self.metadata) column_meta = [] for col_name, col_meta in self.column.items(): if bool(col_meta): column_meta.append('\t\t%s: %s' % (col_name, col_meta)) else: column_meta.append('\t\t%s' % col_name) if bool(column_meta): parts.append('\tColumns:\n%s' % '\n'.join(column_meta)) elif self.column: parts.append('\tColumns: %s' % ', '.join(self.column.keys())) else: parts.append('\tNo columns') if bool(self): parts.extend([ '\t---- Row %4d:\n\t%s' % (row, '\n\t'.join([ (('%s=%r' % (col_name, data[col_name])) if col_name in data else ('%s absent' % col_name)) for col_name in self.column.keys()])) for (row, data) in enumerate(self) ]) else: parts.append('\tNo rows') class_name = self.__class__.__name__ return '%s\n%s\n' % ( class_name, '\n'.join(parts) ) def __getitem__(self, key: Union[int, Ref, slice]) -> Union[Entity, 'Grid']: """Retrieve the row at index. :param key: index, Haystack reference or slice Args: key: the position, the Reference or a slice Returns: The entity of an new grid with a portion of entities, with the same metadata and columns """ if isinstance(key, int): return cast(Entity, self._row[key]) if isinstance(key, slice): result = Grid(version=self.version, metadata=self.metadata, columns=self.column) result._row = self._row[key] result._index = None return result assert isinstance(key, Ref), "The 'key' must be a Ref or int" if not self._index: self.reindex() return cast(Entity, self._index[key]) def __contains__(self, key: Union[int, Ref]) -> bool: """Return an entity with the corresponding id. Args: key: The position of id of entity Returns: 'True' if the referenced entity is present """ if isinstance(key, int): return 0 <= key < len(self._row) if not self._index: self.reindex() return key in self._index def __len__(self) -> int: """Return the number of rows in the grid.""" return len(self._row) def __setitem__(self, index: Union[int, Ref, slice], value: Union[Entity, List[Entity]]) -> 'Grid': """Replace the row at index. Args: index: position of reference, reference of slice value: the new entity Returns: `self` """ if isinstance(value, dict): for val in value.values(): self._detect_or_validate(val) if isinstance(index, int): if not isinstance(value, dict): raise TypeError('value must be a dict') if "id" in self._row[index]: self._index.pop(self._row[index]['id'], None) self._row[index] = value if "id" in value: self._index[value["id"]] = value elif isinstance(index, slice): if isinstance(value, dict): raise TypeError('value must be iterable, not a dict') self._index = None self._row[index] = value for row in value: for val in row.values(): self._detect_or_validate(val) else: if not isinstance(value, dict): raise TypeError('value must be a dict') if not self._index: self.reindex() idx = list.index(self._row, self._index[index]) if "id" in self._row[idx]: self._index.pop(self._row[idx]['id'], None) self._row[idx] = value if "id" in value: self._index[value["id"]] = value return self def __delitem__(self, key: Union[int, Ref]) -> Optional[Entity]: """Delete the row at index. Args: key: The key to find the corresponding entity Returns: The entity or `None` """ return self.pop(key) def clear(self): self._row = [] self._index = None @property def version(self) -> Version: # pragma: no cover """ The haystack version of grid """ return self._version @property def nearest_version(self) -> Version: # pragma: no cover """ The nearest haystack version of grid """ return Version.nearest(self._version) def get(self, index: Ref, default: Optional[Entity] = None) -> Entity: """Return an entity with the corresponding id. Args: index: The id of entity default: The default value if the entity is not found Returns: The entity with the id == index or the default value """ if not self._index: self.reindex() return cast(Entity, self._index.get(index, default)) def keys(self) -> KeysView[Ref]: """ Return the list of ids of entities with `id` Returns: The list of ids of entities with `id` """ if not self._index: self.reindex() return self._index.keys() def pop(self, *index: Union[int, Ref]) -> Optional[Entity]: """Delete the row at index or with specified Ref id. If multiple index/key was specified, all row was removed. Return the old value of the first deleted item. Args: *index: A list of index (position or reference) """ ret_value = None for key in sorted(index, reverse=True): # Remove index at the end if isinstance(key, int): if not 0 <= key < len(self._row): ret_value = None else: if "id" in self._row[key] and self._index: del self._index[self._row[key]['id']] ret_value = self._row[key] del self._row[key] else: if not self._index: self.reindex() if key not in self._index: ret_value = None else: self._row.remove(self._index[key]) ret_value = self._index.pop(key) return cast(Optional[Entity], ret_value) def insert(self, index: int, value: Entity) -> 'Grid': """Insert a new entity before the index position. Args: index: The position where to insert value: The new entity to add Returns `self` """ if not isinstance(value, dict): raise TypeError('value must be a dict') for val in value.values(): self._detect_or_validate(val) self._row.insert(index, value) if "id" in value: if not self._index: self.reindex() self._index[value["id"]] = value return self def reindex(self) -> 'Grid': """Reindex the grid if the user, update directly an id of a row. Returns `self` """ self._index = {} for item in self._row: if "id" in item: assert isinstance(item["id"], Ref), "The 'id' tag must be a reference" self._index[item["id"]] = item return self def pack_columns(self) -> 'Grid': """ Remove unused columns. Returns: `self` """ using_columns = set() columns_keys = self.column.keys() for row in self._row: for col_name in columns_keys: if col_name in row: using_columns.add(col_name) if len(using_columns) == len(columns_keys): # All columns was found return self self.column = {k: self.column[k] for k in using_columns} return self def extends_columns(self) -> 'Grid': """ Add missing columns Returns: `self` """ new_cols = self.column.copy() for row in self._row: for k in row.keys(): if k not in new_cols: new_cols[k] = {} self.column = new_cols return self def extend(self, values: Iterable[Entity]) -> 'Grid': """ Add a list of entities inside the grid Args: values: The list of entities to insert. Returns: `self` """ super().extend(values) for item in self._row: if "id" in item: self._index[item["id"]] = item return self def sort(self, tag: str) -> 'Grid': """ Sort the entity by a specific tag Args: tag: The tag to use to sort the entity Returns: `self` """ self._row = sorted(self._row, key=lambda row: row[tag]) return self def copy(self) -> 'Grid': """ Create a copy of current grid. The corresponding entities were duplicate Returns: A copy of the current grid. """ a_copy = copy.deepcopy(self) a_copy._index = None # Remove index pylint: disable=protected-access return a_copy def filter(self, grid_filter: str, limit: int = 0) -> 'Grid': """Return a filter version of this grid. The entities were share between original grid and the result. Use a `grid.filter(...).deepcopy()` if you not want to share metadata, columns and rows Args: grid_filter: The filter expression (see specification) limit: The maximum number of result Returns: A new grid with only the selected entities. """ assert limit >= 0 from .grid_filter import filter_function # pylint: disable: import-outside-toplevel if grid_filter is None or grid_filter.strip() == '': if limit == 0: return self result = Grid(version=self.version, metadata=self.metadata, columns=self.column) result.extend(self.__getitem__(slice(0, limit))) return result result = Grid(version=self.version, metadata=self.metadata, columns=self.column) a_filter = filter_function(grid_filter) for row in self._row: if a_filter(self, row): result.append(row) if limit and len(result) == limit: break return result def select(self, select: Optional[str]) -> 'Grid': """ Select only some tags in the grid. Args: select: A list a tags (accept operator ! to exclude some columns) Returns: A new grid with only selected columns. Use grid.purge_grid() to update the entities. """ if select: select = select.strip() if select not in ["*", '']: if '!' in select: new_cols = copy.deepcopy(self.column) new_grid = Grid(version=self.version, metadata=self.metadata, columns=new_cols) for col in re.split('[, ]', select): col = col.strip() if not col.startswith('!'): raise ValueError("Impossible to merge positive and negative selection") if col[1:] in new_cols: del new_cols[col[1:]] new_grid.column = new_cols return new_grid new_cols = SortableDict() new_grid = cast(Grid, self[:]) for col in re.split('[, ]', select): col = col.strip() if col.startswith('!'): raise ValueError("Impossible to merge positive and negative selection") if col in self.column: new_cols[col] = self.column[col] else: new_cols[col] = {} new_grid.column = new_cols return cast(Grid, new_grid) return self.copy() def purge(self) -> 'Grid': """ Remove all tags in all entities, not in columns Returns: A new grid with entities compatible with the columns """ cols = self.column new_grid = Grid(version=self.version, metadata=self.metadata, columns=cols) for row in self: new_grid.append({key: val for key, val in row.items() if key in cols}) return new_grid def _detect_or_validate(self, val: Any) -> bool: """Detect the version used from the row content, or validate against the version if given. """ if (val is NA) \ or isinstance(val, (list, dict, SortableDict, Grid)): # Project Haystack 3.0 type. self._assert_version(VER_3_0) return True def _assert_version(self, version: Version) -> None: """Assert that the grid version is equal to or above the given value. If no version is set, set the version. """ if self.nearest_version < version: if self._version_given: raise ValueError( 'Data type requires version %s' % version) self._version = version
Ancestors
- collections.abc.MutableSequence
- collections.abc.Sequence
- collections.abc.Reversible
- collections.abc.Collection
- collections.abc.Sized
- collections.abc.Iterable
- collections.abc.Container
Subclasses
- shaystack.empty_grid._ImmuableGrid
Instance variables
var column
-
Return an attribute of instance, which is of type owner.
var metadata
-
Return an attribute of instance, which is of type owner.
var nearest_version : shaystack.version.Version
-
The nearest haystack version of grid
Expand source code
@property def nearest_version(self) -> Version: # pragma: no cover """ The nearest haystack version of grid """ return Version.nearest(self._version)
var version : shaystack.version.Version
-
The haystack version of grid
Expand source code
@property def version(self) -> Version: # pragma: no cover """ The haystack version of grid """ return self._version
Methods
def clear(self)
-
S.clear() -> None – remove all items from S
Expand source code
def clear(self): self._row = [] self._index = None
def copy(self) ‑> shaystack.grid.Grid
-
Create a copy of current grid.
The corresponding entities were duplicate
Returns
A copy of the current grid.
Expand source code
def copy(self) -> 'Grid': """ Create a copy of current grid. The corresponding entities were duplicate Returns: A copy of the current grid. """ a_copy = copy.deepcopy(self) a_copy._index = None # Remove index pylint: disable=protected-access return a_copy
def extend(self, values: Iterable[Dict[str, Union[str, int, float, bool, datetime.date, datetime.time, datetime.datetime, shaystack.datatypes.Ref, shaystack.datatypes.Quantity, shaystack.datatypes.Coordinate, shaystack.datatypes.Uri, shaystack.datatypes.Bin, shaystack.datatypes.XStr, shaystack.datatypes._MarkerType, shaystack.datatypes._NAType, shaystack.datatypes._RemoveType, List[Any], Dict[str, Any], NoneType]]]) ‑> shaystack.grid.Grid
-
Add a list of entities inside the grid
Args
values
- The list of entities to insert.
Returns
self
Expand source code
def extend(self, values: Iterable[Entity]) -> 'Grid': """ Add a list of entities inside the grid Args: values: The list of entities to insert. Returns: `self` """ super().extend(values) for item in self._row: if "id" in item: self._index[item["id"]] = item return self
def extends_columns(self) ‑> shaystack.grid.Grid
-
Add missing columns
Returns
self
Expand source code
def extends_columns(self) -> 'Grid': """ Add missing columns Returns: `self` """ new_cols = self.column.copy() for row in self._row: for k in row.keys(): if k not in new_cols: new_cols[k] = {} self.column = new_cols return self
def filter(self, grid_filter: str, limit: int = 0) ‑> shaystack.grid.Grid
-
Return a filter version of this grid.
The entities were share between original grid and the result.
Use a
grid.filter(…).deepcopy()
if you not want to share metadata, columns and rowsArgs
grid_filter
- The filter expression (see specification)
limit
- The maximum number of result
Returns
A new grid with only the selected entities.
Expand source code
def filter(self, grid_filter: str, limit: int = 0) -> 'Grid': """Return a filter version of this grid. The entities were share between original grid and the result. Use a `grid.filter(...).deepcopy()` if you not want to share metadata, columns and rows Args: grid_filter: The filter expression (see specification) limit: The maximum number of result Returns: A new grid with only the selected entities. """ assert limit >= 0 from .grid_filter import filter_function # pylint: disable: import-outside-toplevel if grid_filter is None or grid_filter.strip() == '': if limit == 0: return self result = Grid(version=self.version, metadata=self.metadata, columns=self.column) result.extend(self.__getitem__(slice(0, limit))) return result result = Grid(version=self.version, metadata=self.metadata, columns=self.column) a_filter = filter_function(grid_filter) for row in self._row: if a_filter(self, row): result.append(row) if limit and len(result) == limit: break return result
def get(self, index: shaystack.datatypes.Ref, default: Union[Dict[str, Union[str, int, float, bool, datetime.date, datetime.time, datetime.datetime, shaystack.datatypes.Ref, shaystack.datatypes.Quantity, shaystack.datatypes.Coordinate, shaystack.datatypes.Uri, shaystack.datatypes.Bin, shaystack.datatypes.XStr, shaystack.datatypes._MarkerType, shaystack.datatypes._NAType, shaystack.datatypes._RemoveType, List[Any], Dict[str, Any], NoneType]], NoneType] = None) ‑> Dict[str, Union[str, int, float, bool, datetime.date, datetime.time, datetime.datetime, shaystack.datatypes.Ref, shaystack.datatypes.Quantity, shaystack.datatypes.Coordinate, shaystack.datatypes.Uri, shaystack.datatypes.Bin, shaystack.datatypes.XStr, shaystack.datatypes._MarkerType, shaystack.datatypes._NAType, shaystack.datatypes._RemoveType, List[Any], Dict[str, Any], NoneType]]
-
Return an entity with the corresponding id.
Args
index
- The id of entity
default
- The default value if the entity is not found
Returns
The entity with the id == index or the default value
Expand source code
def get(self, index: Ref, default: Optional[Entity] = None) -> Entity: """Return an entity with the corresponding id. Args: index: The id of entity default: The default value if the entity is not found Returns: The entity with the id == index or the default value """ if not self._index: self.reindex() return cast(Entity, self._index.get(index, default))
def insert(self, index: int, value: Dict[str, Union[str, int, float, bool, datetime.date, datetime.time, datetime.datetime, shaystack.datatypes.Ref, shaystack.datatypes.Quantity, shaystack.datatypes.Coordinate, shaystack.datatypes.Uri, shaystack.datatypes.Bin, shaystack.datatypes.XStr, shaystack.datatypes._MarkerType, shaystack.datatypes._NAType, shaystack.datatypes._RemoveType, List[Any], Dict[str, Any], NoneType]]) ‑> shaystack.grid.Grid
-
Insert a new entity before the index position.
Args
index
- The position where to insert
value
- The new entity to add
Returns
self
Expand source code
def insert(self, index: int, value: Entity) -> 'Grid': """Insert a new entity before the index position. Args: index: The position where to insert value: The new entity to add Returns `self` """ if not isinstance(value, dict): raise TypeError('value must be a dict') for val in value.values(): self._detect_or_validate(val) self._row.insert(index, value) if "id" in value: if not self._index: self.reindex() self._index[value["id"]] = value return self
def keys(self) ‑> KeysView[shaystack.datatypes.Ref]
-
Return the list of ids of entities with
id
Returns
The list of ids of entities with
id
Expand source code
def keys(self) -> KeysView[Ref]: """ Return the list of ids of entities with `id` Returns: The list of ids of entities with `id` """ if not self._index: self.reindex() return self._index.keys()
def pack_columns(self) ‑> shaystack.grid.Grid
-
Remove unused columns.
Returns
self
Expand source code
def pack_columns(self) -> 'Grid': """ Remove unused columns. Returns: `self` """ using_columns = set() columns_keys = self.column.keys() for row in self._row: for col_name in columns_keys: if col_name in row: using_columns.add(col_name) if len(using_columns) == len(columns_keys): # All columns was found return self self.column = {k: self.column[k] for k in using_columns} return self
def pop(self, *index: Union[int, shaystack.datatypes.Ref]) ‑> Union[Dict[str, Union[str, int, float, bool, datetime.date, datetime.time, datetime.datetime, shaystack.datatypes.Ref, shaystack.datatypes.Quantity, shaystack.datatypes.Coordinate, shaystack.datatypes.Uri, shaystack.datatypes.Bin, shaystack.datatypes.XStr, shaystack.datatypes._MarkerType, shaystack.datatypes._NAType, shaystack.datatypes._RemoveType, List[Any], Dict[str, Any], NoneType]], NoneType]
-
Delete the row at index or with specified Ref id. If multiple index/key was specified, all row was removed. Return the old value of the first deleted item.
Args
*index
- A list of index (position or reference)
Expand source code
def pop(self, *index: Union[int, Ref]) -> Optional[Entity]: """Delete the row at index or with specified Ref id. If multiple index/key was specified, all row was removed. Return the old value of the first deleted item. Args: *index: A list of index (position or reference) """ ret_value = None for key in sorted(index, reverse=True): # Remove index at the end if isinstance(key, int): if not 0 <= key < len(self._row): ret_value = None else: if "id" in self._row[key] and self._index: del self._index[self._row[key]['id']] ret_value = self._row[key] del self._row[key] else: if not self._index: self.reindex() if key not in self._index: ret_value = None else: self._row.remove(self._index[key]) ret_value = self._index.pop(key) return cast(Optional[Entity], ret_value)
def purge(self) ‑> shaystack.grid.Grid
-
Remove all tags in all entities, not in columns
Returns
A new grid with entities compatible with the columns
Expand source code
def purge(self) -> 'Grid': """ Remove all tags in all entities, not in columns Returns: A new grid with entities compatible with the columns """ cols = self.column new_grid = Grid(version=self.version, metadata=self.metadata, columns=cols) for row in self: new_grid.append({key: val for key, val in row.items() if key in cols}) return new_grid
def reindex(self) ‑> shaystack.grid.Grid
-
Reindex the grid if the user, update directly an id of a row. Returns
self
Expand source code
def reindex(self) -> 'Grid': """Reindex the grid if the user, update directly an id of a row. Returns `self` """ self._index = {} for item in self._row: if "id" in item: assert isinstance(item["id"], Ref), "The 'id' tag must be a reference" self._index[item["id"]] = item return self
def select(self, select: Union[str, NoneType]) ‑> shaystack.grid.Grid
-
Select only some tags in the grid.
Args
select
- A list a tags (accept operator ! to exclude some columns)
Returns
A new grid with only selected columns. Use grid.purge_grid() to update the entities.
Expand source code
def select(self, select: Optional[str]) -> 'Grid': """ Select only some tags in the grid. Args: select: A list a tags (accept operator ! to exclude some columns) Returns: A new grid with only selected columns. Use grid.purge_grid() to update the entities. """ if select: select = select.strip() if select not in ["*", '']: if '!' in select: new_cols = copy.deepcopy(self.column) new_grid = Grid(version=self.version, metadata=self.metadata, columns=new_cols) for col in re.split('[, ]', select): col = col.strip() if not col.startswith('!'): raise ValueError("Impossible to merge positive and negative selection") if col[1:] in new_cols: del new_cols[col[1:]] new_grid.column = new_cols return new_grid new_cols = SortableDict() new_grid = cast(Grid, self[:]) for col in re.split('[, ]', select): col = col.strip() if col.startswith('!'): raise ValueError("Impossible to merge positive and negative selection") if col in self.column: new_cols[col] = self.column[col] else: new_cols[col] = {} new_grid.column = new_cols return cast(Grid, new_grid) return self.copy()
def sort(self, tag: str) ‑> shaystack.grid.Grid
-
Sort the entity by a specific tag
Args
tag
- The tag to use to sort the entity
Returns
self
Expand source code
def sort(self, tag: str) -> 'Grid': """ Sort the entity by a specific tag Args: tag: The tag to use to sort the entity Returns: `self` """ self._row = sorted(self._row, key=lambda row: row[tag]) return self
- It's possible to access an entity with the position in the grid:
class HaystackInterface (envs: Dict[str, str])
-
Interface to implement to be compatible with Haystack REST protocol. The subclasses may be abstract (implemented only a part of methods), the 'ops' code detect that, and can calculate the set of implemented operations.
Expand source code
class HaystackInterface(ABC): """ Interface to implement to be compatible with Haystack REST protocol. The subclasses may be abstract (implemented only a part of methods), the 'ops' code detect that, and can calculate the set of implemented operations. """ __slots__ = ['_envs'] def __init__(self, envs: Dict[str, str]): assert envs is not None self._envs = envs @property @abstractmethod def name(self) -> str: """ The name of the provider. """ raise NotImplementedError() def __repr__(self) -> str: return self.name def __str__(self) -> str: return self.__repr__() def __enter__(self): return self def __exit__(self, exc_type, exc_value, exc_traceback): pass # noinspection PyMethodMayBeStatic def get_tz(self) -> BaseTzInfo: # pylint: disable=no-self-use """ Return server time zone. """ return get_localzone() def get_customer_id(self) -> str: # pylint: disable=no-self-use """ Override this for multi-tenant. May be, extract the customer id from the current `Principal`. """ return '' def values_for_tag(self, tag: str, date_version: Optional[datetime] = None) -> List[Any]: """Get all values for a given tag. Args: tag: The tag to analyse. date_version: version date of the ontology. Returns: All unique values for a specific tag """ raise NotImplementedError() def versions(self) -> List[datetime]: # pylint: disable=no-self-use """ Return a list of versions fot the current ontology. Returns: datetime for each version or empty array if unknown """ return [] @abstractmethod def about(self, home: str) -> Grid: """Implement the Haystack 'about' ops. The subclasse must complet the result with "productUri", "productVersion", "moduleName" and "moduleVersion" Args: home: Home url of the API Returns: The default 'about' grid. """ grid = Grid( version=VER_3_0, columns=[ "haystackVersion", # Str version of REST implementation "tz", # Str of server's default timezone "serverName", # Str name of the server or project database "serverTime", "serverBootTime", "productName", # Str name of the server software product "productUri", "productVersion", # module which implements Haystack server protocol "moduleName", # if its a plug-in to the product "moduleVersion" # Str version of moduleName ], ) grid.append( { "haystackVersion": str(VER_3_0), "tz": str(self.get_tz()), "serverName": "haystack_" + self._envs.get("AWS_REGION", "local"), "serverTime": datetime.now(tz=self.get_tz()).replace(microsecond=0), "serverBootTime": datetime.now(tz=self.get_tz()).replace( microsecond=0 ), "productName": "Haystack Provider", "productUri": Uri(home), "productVersion": "0.1", "moduleName": "AbstractProvider", "moduleVersion": "0.1", } ) return grid # noinspection PyUnresolvedReferences def ops(self) -> Grid: """ Implement the Haystack 'ops' ops. Notes: Automatically calculate the implemented version. Returns: A Grid containing 'ops' name operations and its related description """ grid = Grid( version=VER_3_0, columns={ "name": {}, "summary": {}, }, ) all_haystack_ops = { "about": "Summary information for server", "ops": "Operations supported by this server", "formats": "Grid data formats supported by this server", "read": "The read op is used to read a set of entity records either by their unique " "identifier or using a filter.", "nav": "The nav op is used navigate a project for learning and discovery", "watch_sub": "The watch_sub operation is used to create new watches " "or add entities to an existing watch.", "watch_unsub": "The watch_unsub operation is used to close a watch entirely " "or remove entities from a watch.", "watch_poll": "The watch_poll operation is used to poll a watch for " "changes to the subscribed entity records.", "point_write": "The point_write_read op is used to: read the current status of a " "writable point's priority array " "or write to a given level", "his_read": "The his_read op is used to read a time-series data " "from historized point.", "his_write": "The his_write op is used to post new time-series " "data to a historized point.", "invoke_action": "The invoke_action op is used to invoke a " "user action on a target record.", } # Remove abstract method # noinspection PyUnresolvedReferences for abstract_method in self.__class__.__base__.__abstractmethods__: all_haystack_ops.pop(abstract_method, None) if ( "point_write_read" in self.__class__.__base__.__abstractmethods__ or "point_write_write" in self.__class__.__base__.__abstractmethods__ ): all_haystack_ops.pop("point_write", None) all_haystack_ops = {_to_camel(k): v for k, v in all_haystack_ops.items()} grid.extend( [ {"name": name, "summary": summary} for name, summary in all_haystack_ops.items() ] ) return grid def formats(self) -> Optional[Grid]: # pylint: disable=no-self-use """ Implement the Haystack 'formats' ops. Notes: Implement this method, only if you want to limit the format negotiation Returns: The grid format or None. If None, the API accept all formats ZINC, TRIO, JSON and CSV. """ return None # type: ignore @abstractmethod def read( self, limit: int, select: Optional[str], entity_ids: Optional[List[Ref]], grid_filter: Optional[str], date_version: Optional[datetime], ) -> Grid: # pylint: disable=no-self-use """ Implement the Haystack 'read' ops. Args: limit: The number of record to return or zero select: The selected tag separated with comma, else '' or '*' entity_ids: A list en ids. If set, grid_filter and limit are ignored. grid_filter: A filter to apply. Ignored if entity_ids is set. date_version: The date of the ontology version. Returns: The requested Grid """ raise NotImplementedError() @abstractmethod def nav(self, nav_id: str) -> Any: # pylint: disable=no-self-use """ Implement the Haystack 'nav' ops. This operation allows servers to expose the database in a human-friendly tree (or graph) that can be explored Args: nav_id: The string for nav id column """ raise NotImplementedError() @abstractmethod def watch_sub( self, watch_dis: str, watch_id: Optional[str], ids: List[Ref], lease: Optional[int], ) -> Grid: # pylint: disable=no-self-use """ Implement the Haystack 'watchSub' ops. Args: watch_dis: Watch description watch_id: The user watch_id to update or None. ids: The list of ids to watch. lease: Lease to apply. Returns: A Grid """ raise NotImplementedError() @abstractmethod def watch_unsub( self, watch_id: str, ids: List[Ref], close: bool ) -> Grid: # pylint: disable=no-self-use """ Implement the Haystack 'watchUnsub' ops. Args: watch_id: The user watch_id to update or None ids: The list of ids to watch close: Set to True to close Returns: A Grid """ raise NotImplementedError() @abstractmethod def watch_poll( self, watch_id: str, refresh: bool ) -> Grid: # pylint: disable=no-self-use """ Implement the Haystack 'watchPoll' ops. Args: watch_id: The user watch_id to update or None refresh: Set to True for refreshing the data Returns: A Grid where each row corresponds to a watched entity. """ raise NotImplementedError() @abstractmethod def point_write_read( self, entity_id: Ref, date_version: Optional[datetime] ) -> Grid: # pylint: disable=no-self-use """ Implement the Haystack 'pointWrite' ops. Args: entity_id: The entity to update date_version: The optional date version to update Returns: A Grid """ raise NotImplementedError() @abstractmethod def point_write_write( self, entity_id: Ref, level: int, val: Optional[Any], duration: Quantity, who: Optional[str], date_version: Optional[datetime] = None, ) -> None: # pylint: disable=no-self-use """ Implement the Haystack 'pointWrite' ops. Args: entity_id: The entity to update level: Number from 1-17 for level to write val: Value to write or null to auto the level duration: Number with duration unit if setting level 8 who: Optional username performing the write, otherwise user dis is used date_version: The optional date version to update Returns: None """ raise NotImplementedError() # Date dates_range must be: # "today" # "yesterday" # "{date}" # "{date},{date}" # "{dateTime},{dateTime}" # "{dateTime}" @abstractmethod def his_read( self, entity_id: Ref, dates_range: Tuple[datetime, datetime], date_version: Optional[datetime] = None, ) -> Grid: # pylint: disable=no-self-use """ Implement the Haystack 'hisRead' ops. Args: entity_id: The entity to read dates_range: May be "today", "yesterday", {date}, ({date},{date}), ({datetime},{datetime}), {dateTime} date_version: The optional date version to read Returns: A grid """ raise NotImplementedError() @abstractmethod def his_write( self, entity_id: Ref, time_series: Grid, date_version: Optional[datetime] = None ) -> Grid: # pylint: disable=no-self-use """ Implement the Haystack 'hisWrite' ops. Args: entity_id: The entity to read time_series: A grid with a time series date_version: The optional date version to update Returns: A grid """ raise NotImplementedError() @abstractmethod def invoke_action( self, entity_id: Ref, action: str, params: Dict[str, Any], date_version: Optional[datetime] = None ) -> Grid: # pylint: disable=no-self-use """ Implement the Haystack 'invokeAction' ops. Args: entity_id: The entity to read action: The action string params: A dictionary with parameters date_version: The optional date version to update Returns: A grid """ raise NotImplementedError()
Ancestors
- abc.ABC
Subclasses
Instance variables
var name : str
-
The name of the provider.
Expand source code
@property @abstractmethod def name(self) -> str: """ The name of the provider. """ raise NotImplementedError()
Methods
def about(self, home: str) ‑> shaystack.grid.Grid
-
Implement the Haystack 'about' ops.
The subclasse must complet the result with "productUri", "productVersion", "moduleName" and "moduleVersion"
Args
home
- Home url of the API
Returns
The default 'about' grid.
Expand source code
@abstractmethod def about(self, home: str) -> Grid: """Implement the Haystack 'about' ops. The subclasse must complet the result with "productUri", "productVersion", "moduleName" and "moduleVersion" Args: home: Home url of the API Returns: The default 'about' grid. """ grid = Grid( version=VER_3_0, columns=[ "haystackVersion", # Str version of REST implementation "tz", # Str of server's default timezone "serverName", # Str name of the server or project database "serverTime", "serverBootTime", "productName", # Str name of the server software product "productUri", "productVersion", # module which implements Haystack server protocol "moduleName", # if its a plug-in to the product "moduleVersion" # Str version of moduleName ], ) grid.append( { "haystackVersion": str(VER_3_0), "tz": str(self.get_tz()), "serverName": "haystack_" + self._envs.get("AWS_REGION", "local"), "serverTime": datetime.now(tz=self.get_tz()).replace(microsecond=0), "serverBootTime": datetime.now(tz=self.get_tz()).replace( microsecond=0 ), "productName": "Haystack Provider", "productUri": Uri(home), "productVersion": "0.1", "moduleName": "AbstractProvider", "moduleVersion": "0.1", } ) return grid
def formats(self) ‑> Union[shaystack.grid.Grid, NoneType]
-
Implement the Haystack 'formats' ops.
Notes
Implement this method, only if you want to limit the format negotiation
Returns
The grid format or None. If None, the API accept all formats ZINC, TRIO, JSON and CSV.
Expand source code
def formats(self) -> Optional[Grid]: # pylint: disable=no-self-use """ Implement the Haystack 'formats' ops. Notes: Implement this method, only if you want to limit the format negotiation Returns: The grid format or None. If None, the API accept all formats ZINC, TRIO, JSON and CSV. """ return None # type: ignore
def get_customer_id(self) ‑> str
-
Override this for multi-tenant. May be, extract the customer id from the current
Principal
.Expand source code
def get_customer_id(self) -> str: # pylint: disable=no-self-use """ Override this for multi-tenant. May be, extract the customer id from the current `Principal`. """ return ''
def get_tz(self) ‑> pytz.tzinfo.BaseTzInfo
-
Return server time zone.
Expand source code
def get_tz(self) -> BaseTzInfo: # pylint: disable=no-self-use """ Return server time zone. """ return get_localzone()
def his_read(self, entity_id: shaystack.datatypes.Ref, dates_range: Tuple[datetime.datetime, datetime.datetime], date_version: Union[datetime.datetime, NoneType] = None) ‑> shaystack.grid.Grid
-
Implement the Haystack 'hisRead' ops.
Args
entity_id
- The entity to read
dates_range
- May be "today", "yesterday", {date}, ({date},{date}), ({datetime},{datetime}),
- {dateTime}
date_version
- The optional date version to read
Returns
A grid
Expand source code
@abstractmethod def his_read( self, entity_id: Ref, dates_range: Tuple[datetime, datetime], date_version: Optional[datetime] = None, ) -> Grid: # pylint: disable=no-self-use """ Implement the Haystack 'hisRead' ops. Args: entity_id: The entity to read dates_range: May be "today", "yesterday", {date}, ({date},{date}), ({datetime},{datetime}), {dateTime} date_version: The optional date version to read Returns: A grid """ raise NotImplementedError()
def his_write(self, entity_id: shaystack.datatypes.Ref, time_series: shaystack.grid.Grid, date_version: Union[datetime.datetime, NoneType] = None) ‑> shaystack.grid.Grid
-
Implement the Haystack 'hisWrite' ops.
Args
entity_id
- The entity to read
time_series
- A grid with a time series
date_version
- The optional date version to update
Returns
A grid
Expand source code
@abstractmethod def his_write( self, entity_id: Ref, time_series: Grid, date_version: Optional[datetime] = None ) -> Grid: # pylint: disable=no-self-use """ Implement the Haystack 'hisWrite' ops. Args: entity_id: The entity to read time_series: A grid with a time series date_version: The optional date version to update Returns: A grid """ raise NotImplementedError()
def invoke_action(self, entity_id: shaystack.datatypes.Ref, action: str, params: Dict[str, Any], date_version: Union[datetime.datetime, NoneType] = None) ‑> shaystack.grid.Grid
-
Implement the Haystack 'invokeAction' ops.
Args
entity_id
- The entity to read
action
- The action string
params
- A dictionary with parameters
date_version
- The optional date version to update
Returns
A grid
Expand source code
@abstractmethod def invoke_action( self, entity_id: Ref, action: str, params: Dict[str, Any], date_version: Optional[datetime] = None ) -> Grid: # pylint: disable=no-self-use """ Implement the Haystack 'invokeAction' ops. Args: entity_id: The entity to read action: The action string params: A dictionary with parameters date_version: The optional date version to update Returns: A grid """ raise NotImplementedError()
-
Implement the Haystack 'nav' ops. This operation allows servers to expose the database in a human-friendly tree (or graph) that can be explored
Args
nav_id
- The string for nav id column
Expand source code
@abstractmethod def nav(self, nav_id: str) -> Any: # pylint: disable=no-self-use """ Implement the Haystack 'nav' ops. This operation allows servers to expose the database in a human-friendly tree (or graph) that can be explored Args: nav_id: The string for nav id column """ raise NotImplementedError()
def ops(self) ‑> shaystack.grid.Grid
-
Implement the Haystack 'ops' ops.
Notes
Automatically calculate the implemented version.
Returns
A Grid containing 'ops' name operations and its related description
Expand source code
def ops(self) -> Grid: """ Implement the Haystack 'ops' ops. Notes: Automatically calculate the implemented version. Returns: A Grid containing 'ops' name operations and its related description """ grid = Grid( version=VER_3_0, columns={ "name": {}, "summary": {}, }, ) all_haystack_ops = { "about": "Summary information for server", "ops": "Operations supported by this server", "formats": "Grid data formats supported by this server", "read": "The read op is used to read a set of entity records either by their unique " "identifier or using a filter.", "nav": "The nav op is used navigate a project for learning and discovery", "watch_sub": "The watch_sub operation is used to create new watches " "or add entities to an existing watch.", "watch_unsub": "The watch_unsub operation is used to close a watch entirely " "or remove entities from a watch.", "watch_poll": "The watch_poll operation is used to poll a watch for " "changes to the subscribed entity records.", "point_write": "The point_write_read op is used to: read the current status of a " "writable point's priority array " "or write to a given level", "his_read": "The his_read op is used to read a time-series data " "from historized point.", "his_write": "The his_write op is used to post new time-series " "data to a historized point.", "invoke_action": "The invoke_action op is used to invoke a " "user action on a target record.", } # Remove abstract method # noinspection PyUnresolvedReferences for abstract_method in self.__class__.__base__.__abstractmethods__: all_haystack_ops.pop(abstract_method, None) if ( "point_write_read" in self.__class__.__base__.__abstractmethods__ or "point_write_write" in self.__class__.__base__.__abstractmethods__ ): all_haystack_ops.pop("point_write", None) all_haystack_ops = {_to_camel(k): v for k, v in all_haystack_ops.items()} grid.extend( [ {"name": name, "summary": summary} for name, summary in all_haystack_ops.items() ] ) return grid
def point_write_read(self, entity_id: shaystack.datatypes.Ref, date_version: Union[datetime.datetime, NoneType]) ‑> shaystack.grid.Grid
-
Implement the Haystack 'pointWrite' ops.
Args
entity_id
- The entity to update
date_version
- The optional date version to update
Returns
A Grid
Expand source code
@abstractmethod def point_write_read( self, entity_id: Ref, date_version: Optional[datetime] ) -> Grid: # pylint: disable=no-self-use """ Implement the Haystack 'pointWrite' ops. Args: entity_id: The entity to update date_version: The optional date version to update Returns: A Grid """ raise NotImplementedError()
def point_write_write(self, entity_id: shaystack.datatypes.Ref, level: int, val: Union[Any, NoneType], duration: shaystack.datatypes.Quantity, who: Union[str, NoneType], date_version: Union[datetime.datetime, NoneType] = None) ‑> NoneType
-
Implement the Haystack 'pointWrite' ops.
Args
entity_id
- The entity to update
level
- Number from 1-17 for level to write
val
- Value to write or null to auto the level
duration
- Number with duration unit if setting level 8
who
- Optional username performing the write, otherwise user dis is used
date_version
- The optional date version to update
Returns
None
Expand source code
@abstractmethod def point_write_write( self, entity_id: Ref, level: int, val: Optional[Any], duration: Quantity, who: Optional[str], date_version: Optional[datetime] = None, ) -> None: # pylint: disable=no-self-use """ Implement the Haystack 'pointWrite' ops. Args: entity_id: The entity to update level: Number from 1-17 for level to write val: Value to write or null to auto the level duration: Number with duration unit if setting level 8 who: Optional username performing the write, otherwise user dis is used date_version: The optional date version to update Returns: None """ raise NotImplementedError()
def read(self, limit: int, select: Union[str, NoneType], entity_ids: Union[List[shaystack.datatypes.Ref], NoneType], grid_filter: Union[str, NoneType], date_version: Union[datetime.datetime, NoneType]) ‑> shaystack.grid.Grid
-
Implement the Haystack 'read' ops.
Args
limit
- The number of record to return or zero
select
- The selected tag separated with comma, else '' or '*'
entity_ids
- A list en ids. If set, grid_filter and limit are ignored.
grid_filter
- A filter to apply. Ignored if entity_ids is set.
date_version
- The date of the ontology version.
Returns
The requested Grid
Expand source code
@abstractmethod def read( self, limit: int, select: Optional[str], entity_ids: Optional[List[Ref]], grid_filter: Optional[str], date_version: Optional[datetime], ) -> Grid: # pylint: disable=no-self-use """ Implement the Haystack 'read' ops. Args: limit: The number of record to return or zero select: The selected tag separated with comma, else '' or '*' entity_ids: A list en ids. If set, grid_filter and limit are ignored. grid_filter: A filter to apply. Ignored if entity_ids is set. date_version: The date of the ontology version. Returns: The requested Grid """ raise NotImplementedError()
def values_for_tag(self, tag: str, date_version: Union[datetime.datetime, NoneType] = None) ‑> List[Any]
-
Get all values for a given tag.
Args
tag
- The tag to analyse.
date_version
- version date of the ontology.
Returns
All unique values for a specific tag
Expand source code
def values_for_tag(self, tag: str, date_version: Optional[datetime] = None) -> List[Any]: """Get all values for a given tag. Args: tag: The tag to analyse. date_version: version date of the ontology. Returns: All unique values for a specific tag """ raise NotImplementedError()
def versions(self) ‑> List[datetime.datetime]
-
Return a list of versions fot the current ontology.
Returns
datetime for each version or empty array if unknown
Expand source code
def versions(self) -> List[datetime]: # pylint: disable=no-self-use """ Return a list of versions fot the current ontology. Returns: datetime for each version or empty array if unknown """ return []
def watch_poll(self, watch_id: str, refresh: bool) ‑> shaystack.grid.Grid
-
Implement the Haystack 'watchPoll' ops.
Args
watch_id
- The user watch_id to update or None
refresh
- Set to True for refreshing the data
Returns
A Grid where each row corresponds to a watched entity.
Expand source code
@abstractmethod def watch_poll( self, watch_id: str, refresh: bool ) -> Grid: # pylint: disable=no-self-use """ Implement the Haystack 'watchPoll' ops. Args: watch_id: The user watch_id to update or None refresh: Set to True for refreshing the data Returns: A Grid where each row corresponds to a watched entity. """ raise NotImplementedError()
def watch_sub(self, watch_dis: str, watch_id: Union[str, NoneType], ids: List[shaystack.datatypes.Ref], lease: Union[int, NoneType]) ‑> shaystack.grid.Grid
-
Implement the Haystack 'watchSub' ops.
Args
watch_dis
- Watch description
watch_id
- The user watch_id to update or None.
ids
- The list of ids to watch.
lease
- Lease to apply.
Returns
A Grid
Expand source code
@abstractmethod def watch_sub( self, watch_dis: str, watch_id: Optional[str], ids: List[Ref], lease: Optional[int], ) -> Grid: # pylint: disable=no-self-use """ Implement the Haystack 'watchSub' ops. Args: watch_dis: Watch description watch_id: The user watch_id to update or None. ids: The list of ids to watch. lease: Lease to apply. Returns: A Grid """ raise NotImplementedError()
def watch_unsub(self, watch_id: str, ids: List[shaystack.datatypes.Ref], close: bool) ‑> shaystack.grid.Grid
-
Implement the Haystack 'watchUnsub' ops.
Args
watch_id
- The user watch_id to update or None
ids
- The list of ids to watch
close
- Set to True to close
Returns
A Grid
Expand source code
@abstractmethod def watch_unsub( self, watch_id: str, ids: List[Ref], close: bool ) -> Grid: # pylint: disable=no-self-use """ Implement the Haystack 'watchUnsub' ops. Args: watch_id: The user watch_id to update or None ids: The list of ids to watch close: Set to True to close Returns: A Grid """ raise NotImplementedError()
class MetadataObject (initial: Union[NoneType, List[Tuple[str, Any]], Dict[str, Any]] = None, validate_fn: Union[Callable[[Any], bool], NoneType] = None)
-
An object that contains some metadata fields.
Used as a convenience base-class for grids and columns, both of which have metadata.
A dict-like object that permits value ordering/re-ordering.
Args
initial
- Initial values
validate_fn
- A validated function
Expand source code
class MetadataObject(SortableDict): # pylint: disable=too-many-ancestors """An object that contains some metadata fields. Used as a convenience base-class for grids and columns, both of which have metadata. """ def append(self, key: str, value: Any = MARKER, replace: bool = True) -> 'MetadataObject': """Append the item to the metadata. Args: key: The tag name value: The value replace: Flag to replace or not the value Returns `self` """ self.add_item(key, value, replace=replace) return self def extend(self, items: Iterable[Any], replace: bool = True) -> 'MetadataObject': """Append the items to the metadata. Args: items: A list of items replace: Flag to replace or not the value Returns `self` """ if isinstance(items, (dict, SortableDict)): items = list(items.items()) for (key, value) in items: self.append(key, value, replace=replace) return self def copy(self) -> 'MetadataObject': """ Deep copy of metadata Returns: A new metadata object """ return copy.deepcopy(self)
Ancestors
- shaystack.sortabledict.SortableDict
- collections.abc.MutableMapping
- collections.abc.Mapping
- collections.abc.Collection
- collections.abc.Sized
- collections.abc.Iterable
- collections.abc.Container
Methods
def append(self, key: str, value: Any = MARKER, replace: bool = True) ‑> shaystack.metadata.MetadataObject
-
Append the item to the metadata.
Args
key
- The tag name
value
- The value
replace
- Flag to replace or not the value
Returns
self
Expand source code
def append(self, key: str, value: Any = MARKER, replace: bool = True) -> 'MetadataObject': """Append the item to the metadata. Args: key: The tag name value: The value replace: Flag to replace or not the value Returns `self` """ self.add_item(key, value, replace=replace) return self
def copy(self) ‑> shaystack.metadata.MetadataObject
-
Deep copy of metadata
Returns
A new metadata object
Expand source code
def copy(self) -> 'MetadataObject': """ Deep copy of metadata Returns: A new metadata object """ return copy.deepcopy(self)
def extend(self, items: Iterable[Any], replace: bool = True) ‑> shaystack.metadata.MetadataObject
-
Append the items to the metadata.
Args
items
- A list of items
replace
- Flag to replace or not the value
Returns
self
Expand source code
def extend(self, items: Iterable[Any], replace: bool = True) -> 'MetadataObject': """Append the items to the metadata. Args: items: A list of items replace: Flag to replace or not the value Returns `self` """ if isinstance(items, (dict, SortableDict)): items = list(items.items()) for (key, value) in items: self.append(key, value, replace=replace) return self
class Quantity (value, units=None)
-
A quantity with unit. The quantity use the pint framework and can be converted. See here
Properties
value: The magnitude units: Pint unit symbol: The original symbol
Expand source code
class Quantity(unit_reg.Quantity): """ A quantity with unit. The quantity use the pint framework and can be converted. See [here](https://pint.readthedocs.io/en/stable/tutorial.html#defining-a-quantity) Properties: value: The magnitude units: Pint unit symbol: The original symbol """ def __new__(cls, value, units=None): new_quantity = unit_reg.Quantity.__new__(Quantity, value, _to_pint_unit(units) if units else None) new_quantity.symbol = units return new_quantity
Ancestors
- pint.quantity.build_quantity_class.
.Quantity - pint.quantity.Quantity
- pint.util.PrettyIPython
- pint.util.SharedRegistryObject
- pint.quantity.build_quantity_class.
class Ref (name: str, value: Union[str, NoneType] = None)
-
A reference to an object in Project Haystack.
Args
name
- the uniq id
value
- the comment to describe the reference
Expand source code
class Ref: """A reference to an object in Project Haystack. Args: name: the uniq id value: the comment to describe the reference """ __slots__ = "name", "value" def __init__(self, name: str, value: Optional[str] = None): if name.startswith("@"): name = name[1:] assert isinstance(name, str) and re.match("^[a-zA-Z0-9_:\\-.~]+$", name) self.name = name self.value = value @property def has_value(self): return self.value is not None def __repr__(self) -> str: return '%s(%r, %r)' % ( self.__class__.__name__, self.name, self.value ) def __str__(self) -> str: if self.has_value: return '@%s %r' % ( self.name, self.value ) return '@%s' % self.name def __eq__(self, other: 'Ref') -> bool: if not isinstance(other, Ref): return False return self.name == other.name def __ne__(self, other: 'Ref'): if not isinstance(other, Ref): return True return not self == other def __lt__(self, other: 'Ref') -> bool: return self.name.__lt__(other.name) def __le__(self, other: 'Ref') -> bool: return self.name.__le__(other.name) def __gt__(self, other: 'Ref') -> bool: return self.name.__gt__(other.name) def __ge__(self, other: 'Ref') -> bool: return self.name.__ge__(other.name) def __hash__(self) -> int: return hash(self.name)
Instance variables
var has_value
-
Expand source code
@property def has_value(self): return self.value is not None
var name
-
Return an attribute of instance, which is of type owner.
var value
-
Return an attribute of instance, which is of type owner.
class Uri (...)
-
A convenience class to allow identification of a URI from other string types.
Expand source code
class Uri(str): """A convenience class to allow identification of a URI from other string types. """ def __repr__(self) -> str: return '%s(%s)' % (self.__class__.__name__, super().__repr__()) def __eq__(self, other: 'Uri') -> bool: if not isinstance(other, Uri): return False return super().__eq__(other)
Ancestors
- builtins.str
class Version (ver_str: Union[str, ForwardRef('Version')])
-
A Project Haystack version number.
Args
ver_str
- The version str
Expand source code
class Version: """A Project Haystack version number. Args: ver_str: The version str """ __slots__ = "version_nums", "version_extra" def __init__(self, ver_str: Union[str, 'Version']): if isinstance(ver_str, Version): # Clone constructor self.version_nums = ver_str.version_nums self.version_extra = ver_str.version_extra else: match = _VERSION_RE.match(ver_str) if match is None: raise ValueError('Not a valid version string: %r' % ver_str) # We should have a nice friendly dotted decimal, followed # by anything else not recognised. Parse out the first bit. (version_nums, version_extra) = match.groups() if not version_nums: raise ValueError('Not a valid version string: %r' % ver_str) self.version_nums = tuple([int(p or 0) # pylint: disable=consider-using-generator for p in version_nums.split('.')]) self.version_extra = version_extra def __str__(self) -> str: base = '.'.join([str(p) for p in self.version_nums]) if self.version_extra: base += self.version_extra return base def _cmp(self, other: Union['Version', str]) -> int: """Compare two Project Haystack version strings Args: other: Other version Returns -1 if self < other, 0 if self == other or 1 if self > other. """ if isinstance(other, str): other = Version(other) num1 = self.version_nums num2 = other.version_nums # Pad both to be the same length ver_len = max(len(num1), len(num2)) num1 += tuple([0 for _ in range(len(num1), ver_len)]) # pylint: disable=consider-using-generator num2 += tuple([0 for _ in range(len(num2), ver_len)]) # pylint: disable=consider-using-generator # Compare the versions for (pair_1, pair_2) in zip(num1, num2): if pair_1 < pair_2: return -1 if pair_1 > pair_2: return 1 # All the same, compare the extra strings. # If a version misses the extra part; we consider that as coming *before*. if self.version_extra is None: if other.version_extra is None: return 0 return -1 if other.version_extra is None: return 1 if self.version_extra == other.version_extra: return 0 if self.version_extra < other.version_extra: return -1 return 1 def __hash__(self) -> int: return hash(str(self)) # Comparison operators def __lt__(self, other) -> bool: return self._cmp(other) < 0 def __le__(self, other) -> bool: return self._cmp(other) < 1 def __eq__(self, other) -> bool: return self._cmp(other) == 0 def __ne__(self, other) -> bool: return self._cmp(other) != 0 def __ge__(self, other) -> bool: return self._cmp(other) > -1 def __gt__(self, other) -> bool: return self._cmp(other) > 0 @classmethod def nearest(cls, ver: Union[str, 'Version']) -> 'Version': """Retrieve the official version nearest the one given. Args: ver: The version to analyse Returns: The version nearest the one given """ if isinstance(ver, str): ver = Version(ver) if ver in OFFICIAL_VERSIONS: return ver # We might not have an exact match for that. # See if we have one that's newer than the grid we're looking at. versions = list(OFFICIAL_VERSIONS) versions.sort(reverse=True) best = None for candidate in versions: # Due to ambiguities, we might have an exact match and not know it. # '2.0' will not hash to the same value as '2.0.0', but both are # equivalent. if candidate == ver: # We can't beat this, make a note of the match for later return candidate # If we have not seen a better candidate, and this is older # then we may have to settle for that. if (best is None) and (candidate < ver): warnings.warn('This version of shift-4-haystack does not yet ' 'support version %s, please seek a newer version ' 'or file a bug. Closest (older) version supported is %s.' % (ver, candidate)) return candidate # Probably the best so far, but see if we can go closer if candidate > ver: best = candidate # Unhappy path, no best option? This should not happen. assert best is not None warnings.warn('This version of shift-4-haystack does not yet ' 'support version %s, please seek a newer version ' 'or file a bug. Closest (newer) version supported is %s.' % (ver, best)) return best
Static methods
def nearest(ver: Union[str, ForwardRef('Version')]) ‑> shaystack.version.Version
-
Retrieve the official version nearest the one given.
Args
ver
- The version to analyse
Returns
The version nearest the one given
Expand source code
@classmethod def nearest(cls, ver: Union[str, 'Version']) -> 'Version': """Retrieve the official version nearest the one given. Args: ver: The version to analyse Returns: The version nearest the one given """ if isinstance(ver, str): ver = Version(ver) if ver in OFFICIAL_VERSIONS: return ver # We might not have an exact match for that. # See if we have one that's newer than the grid we're looking at. versions = list(OFFICIAL_VERSIONS) versions.sort(reverse=True) best = None for candidate in versions: # Due to ambiguities, we might have an exact match and not know it. # '2.0' will not hash to the same value as '2.0.0', but both are # equivalent. if candidate == ver: # We can't beat this, make a note of the match for later return candidate # If we have not seen a better candidate, and this is older # then we may have to settle for that. if (best is None) and (candidate < ver): warnings.warn('This version of shift-4-haystack does not yet ' 'support version %s, please seek a newer version ' 'or file a bug. Closest (older) version supported is %s.' % (ver, candidate)) return candidate # Probably the best so far, but see if we can go closer if candidate > ver: best = candidate # Unhappy path, no best option? This should not happen. assert best is not None warnings.warn('This version of shift-4-haystack does not yet ' 'support version %s, please seek a newer version ' 'or file a bug. Closest (newer) version supported is %s.' % (ver, best)) return best
Instance variables
var version_extra
-
Return an attribute of instance, which is of type owner.
var version_nums
-
Return an attribute of instance, which is of type owner.
class XStr (encoding: str, data: str)
-
A convenience class to allow identification of a XStr.
Args
encoding
- encoding format (accept
hex
andb64
) data
- The data
Expand source code
class XStr: """A convenience class to allow identification of a XStr. Args: encoding: encoding format (accept `hex` and `b64`) data: The data """ __slots__ = "encoding", "data" def __init__(self, encoding: str, data: str): self.encoding = encoding if encoding == "hex": self.data = bytearray.fromhex(data) elif encoding == "b64": self.data = base64.b64decode(data) else: self.data = data.encode('ascii') # Not decoded def data_to_string(self) -> str: """ Dump the binary data to string with the corresponding encoding. Returns: A string """ if self.encoding == "hex": return binascii.b2a_hex(self.data).decode("ascii") if self.encoding == "b64": return binascii.b2a_base64(self.data, newline=False).decode("ascii") raise ValueError("Ignore encoding") def __repr__(self) -> str: return 'XStr("%s","%s")' % (self.encoding, self.data_to_string()) def __eq__(self, other: 'XStr') -> bool: if not isinstance(other, XStr): return False return self.data == other.data # Check only binary data def __ne__(self, other: 'XStr') -> bool: if not isinstance(other, XStr): return True return not self.data == other.data # Check only binary data
Instance variables
var data
-
Return an attribute of instance, which is of type owner.
var encoding
-
Return an attribute of instance, which is of type owner.
Methods
def data_to_string(self) ‑> str
-
Dump the binary data to string with the corresponding encoding.
Returns
A string
Expand source code
def data_to_string(self) -> str: """ Dump the binary data to string with the corresponding encoding. Returns: A string """ if self.encoding == "hex": return binascii.b2a_hex(self.data).decode("ascii") if self.encoding == "b64": return binascii.b2a_base64(self.data, newline=False).decode("ascii") raise ValueError("Ignore encoding")