timeplus.stream

stream

This module defines stream class
:copyright: (c) 2022 by Timeplus
:license: Apache2, see LICENSE for more details.

View Source
  0"""
  1stream
  2
  3This module defines stream class  
  4:copyright: (c) 2022 by Timeplus  
  5:license: Apache2, see LICENSE for more details.  
  6"""
  7
  8import requests
  9import json
 10from datetime import datetime
 11
 12from timeplus.base import Base
 13from timeplus.resource import ResourceBase
 14from timeplus.error import TimeplusAPIError
 15
 16from timeplus.type import Type
 17
 18
 19class DateTimeEncoder(json.JSONEncoder):
 20    """
 21    DateTimeEncoder class defines how to encode datetime object for json
 22    """
 23
 24    def default(self, o):
 25        if isinstance(o, datetime):
 26            return o.isoformat()
 27        return json.JSONEncoder.default(self, o)
 28
 29
 30class StreamColumn(Base):
 31    """
 32    StreamColumn class defines columne of a stream
 33    """
 34
 35    def __init__(self):
 36        Base.__init__(self)
 37
 38    def name(self, *args):
 39        return self.prop("name", *args)
 40
 41    def type(self, *args):
 42        if len(args) == 0:
 43            return Type(self._get("type"))
 44        elif len(args) >= 1:
 45            if isinstance(args[0], Type):
 46                if args[0] == Type.Decimal:
 47                    # TODO : need validate the args
 48                    decimal_type = f"{args[0].value}({args[1]}, {args[2]})"
 49                    return self._set("type", decimal_type)
 50                elif args[0] == Type.Array and isinstance(args[1], Type):
 51                    array_type = f"{args[0].value}({args[1].value})"
 52                    return self._set("type", array_type)
 53                elif (
 54                    args[0] == Type.Map
 55                    and isinstance(args[1], Type)
 56                    and isinstance(args[2], Type)
 57                ):
 58                    map_type = f"{args[0].value}({args[1].value}, {args[2].value})"
 59                    return self._set("type", map_type)
 60                elif args[0] == Type.Tuple:
 61                    tuple_values = ",".join([a.value for a in args[1:]])
 62                    map_type = f"{args[0].value}({tuple_values})"
 63                    return self._set("type", map_type)
 64                else:
 65                    return self._set("type", args[0].value)
 66
 67            else:
 68                return self._set("type", args[0])
 69
 70    def default(self, *args):
 71        return self.prop("default", *args)
 72
 73    def compression_codec(self, *args):
 74        return self.prop("compression_codec", *args)
 75
 76    def nullable(self, *args):
 77        return self.prop("nullable", *args)
 78
 79    def skipping_index_expression(self, *args):
 80        return self.prop("skipping_index_expression", *args)
 81
 82    def ttl_expression(self, *args):
 83        return self.prop("ttl_expression", *args)
 84
 85
 86class Stream(ResourceBase):
 87    """
 88    Stream class defines stream object
 89    """
 90
 91    _resource_name = "streams"
 92
 93    def __init__(self, env=None):
 94        ResourceBase.__init__(self, env)
 95        self.prop("columns", [])
 96
 97    @classmethod
 98    def build(cls, val, env=None):
 99        obj = cls(env=env)
100        obj._data = val
101        return obj
102
103    # the list api is not implemented, has to manually implement it here
104    def get(self):
105        streams = Stream.list()
106        for s in streams:
107            if s.name() == self.name():
108                return s
109
110    def delete(self):
111        url = f"{self._base_url}/{self._resource_name}/{self.name()}"
112        self._logger.debug("delete {}", url)
113        try:
114            r = requests.delete(
115                url,
116                headers=self._headers,
117                timeout=self._env.http_timeout(),
118            )
119            if r.status_code < 200 or r.status_code > 299:
120                err_msg = f"failed to delete {self._resource_name} due to {r.text}"
121                raise TimeplusAPIError("delete", r.status_code, err_msg)
122            else:
123                self._logger.debug(f"delete {self._resource_name} success")
124                return self
125        except Exception as e:
126            self._logger.error(f"failed to delete {e}")
127            raise e
128
129    def insert(self, data, headers=None):
130        url = f"{self._base_url}/{self._resource_name}/{self.name()}/ingest"
131        self._logger.debug("post {}", url)
132
133        insert_headers = self.columnNames()
134        if headers is not None:
135            insert_headers = headers
136
137        insertRequest = {
138            "columns": insert_headers,
139            "data": data,
140        }
141        self._logger.debug(f"insert {insertRequest}")
142
143        try:
144            self._logger.debug(
145                f"insert {json.dumps(insertRequest, cls=DateTimeEncoder)}"
146            )
147            r = requests.post(
148                url,
149                data=json.dumps(insertRequest, cls=DateTimeEncoder),
150                headers=self._headers,
151                timeout=self._env.http_timeout(),
152            )
153            if r.status_code < 200 or r.status_code > 299:
154                err_msg = f"failed to insert into {self._resource_name} due to {r.text}"
155                raise TimeplusAPIError("post", r.status_code, err_msg)
156            else:
157                self._logger.debug("insert success")
158                return self
159        except Exception as e:
160            self._logger.error(f"failed to insert {e}")
161            raise e
162
163    def name(self, *args):
164        return self.prop("name", *args)
165
166    def event_time_column(self, *args):
167        return self.prop("event_time_column", *args)
168
169    def order_by_expression(self, *args):
170        return self.prop("order_by_expression", *args)
171
172    def order_by_granularity(self, *args):
173        return self.prop("order_by_granularity", *args)
174
175    def partition_by_granularity(self, *args):
176        return self.prop("partition_by_granularity", *args)
177
178    def replication_factor(self, *args):
179        return self.prop("replication_factor", *args)
180
181    def shards(self, *args):
182        return self.prop("shards", *args)
183
184    def ttl_expression(self, *args):
185        return self.prop("ttl_expression", *args)
186
187    # note, no way to remove/rename col for now
188    def column(self, col):
189        col_prop = self.prop("columns")
190        col_prop.append(col.data())
191        self.prop("columns", col_prop)
192        return self
193
194    def columnNames(self):
195        col_prop = self.prop("columns")
196        return [x["name"] for x in col_prop if not x["name"].startswith("_tp")]
#   class DateTimeEncoder(json.encoder.JSONEncoder):
View Source
20class DateTimeEncoder(json.JSONEncoder):
21    """
22    DateTimeEncoder class defines how to encode datetime object for json
23    """
24
25    def default(self, o):
26        if isinstance(o, datetime):
27            return o.isoformat()
28        return json.JSONEncoder.default(self, o)

DateTimeEncoder class defines how to encode datetime object for json

#   def default(self, o):
View Source
25    def default(self, o):
26        if isinstance(o, datetime):
27            return o.isoformat()
28        return json.JSONEncoder.default(self, o)

Implement this method in a subclass such that it returns a serializable object for o, or calls the base implementation (to raise a TypeError).

For example, to support arbitrary iterators, you could implement default like this::

def default(self, o):
    try:
        iterable = iter(o)
    except TypeError:
        pass
    else:
        return list(iterable)
    # Let the base class default method raise the TypeError
    return JSONEncoder.default(self, o)
Inherited Members
json.encoder.JSONEncoder
JSONEncoder
item_separator
key_separator
encode
iterencode
#   class StreamColumn(timeplus.base.Base):
View Source
31class StreamColumn(Base):
32    """
33    StreamColumn class defines columne of a stream
34    """
35
36    def __init__(self):
37        Base.__init__(self)
38
39    def name(self, *args):
40        return self.prop("name", *args)
41
42    def type(self, *args):
43        if len(args) == 0:
44            return Type(self._get("type"))
45        elif len(args) >= 1:
46            if isinstance(args[0], Type):
47                if args[0] == Type.Decimal:
48                    # TODO : need validate the args
49                    decimal_type = f"{args[0].value}({args[1]}, {args[2]})"
50                    return self._set("type", decimal_type)
51                elif args[0] == Type.Array and isinstance(args[1], Type):
52                    array_type = f"{args[0].value}({args[1].value})"
53                    return self._set("type", array_type)
54                elif (
55                    args[0] == Type.Map
56                    and isinstance(args[1], Type)
57                    and isinstance(args[2], Type)
58                ):
59                    map_type = f"{args[0].value}({args[1].value}, {args[2].value})"
60                    return self._set("type", map_type)
61                elif args[0] == Type.Tuple:
62                    tuple_values = ",".join([a.value for a in args[1:]])
63                    map_type = f"{args[0].value}({tuple_values})"
64                    return self._set("type", map_type)
65                else:
66                    return self._set("type", args[0].value)
67
68            else:
69                return self._set("type", args[0])
70
71    def default(self, *args):
72        return self.prop("default", *args)
73
74    def compression_codec(self, *args):
75        return self.prop("compression_codec", *args)
76
77    def nullable(self, *args):
78        return self.prop("nullable", *args)
79
80    def skipping_index_expression(self, *args):
81        return self.prop("skipping_index_expression", *args)
82
83    def ttl_expression(self, *args):
84        return self.prop("ttl_expression", *args)

StreamColumn class defines columne of a stream

#   StreamColumn()
View Source
36    def __init__(self):
37        Base.__init__(self)
#   def name(self, *args):
View Source
39    def name(self, *args):
40        return self.prop("name", *args)
#   def type(self, *args):
View Source
42    def type(self, *args):
43        if len(args) == 0:
44            return Type(self._get("type"))
45        elif len(args) >= 1:
46            if isinstance(args[0], Type):
47                if args[0] == Type.Decimal:
48                    # TODO : need validate the args
49                    decimal_type = f"{args[0].value}({args[1]}, {args[2]})"
50                    return self._set("type", decimal_type)
51                elif args[0] == Type.Array and isinstance(args[1], Type):
52                    array_type = f"{args[0].value}({args[1].value})"
53                    return self._set("type", array_type)
54                elif (
55                    args[0] == Type.Map
56                    and isinstance(args[1], Type)
57                    and isinstance(args[2], Type)
58                ):
59                    map_type = f"{args[0].value}({args[1].value}, {args[2].value})"
60                    return self._set("type", map_type)
61                elif args[0] == Type.Tuple:
62                    tuple_values = ",".join([a.value for a in args[1:]])
63                    map_type = f"{args[0].value}({tuple_values})"
64                    return self._set("type", map_type)
65                else:
66                    return self._set("type", args[0].value)
67
68            else:
69                return self._set("type", args[0])
#   def default(self, *args):
View Source
71    def default(self, *args):
72        return self.prop("default", *args)
#   def compression_codec(self, *args):
View Source
74    def compression_codec(self, *args):
75        return self.prop("compression_codec", *args)
#   def nullable(self, *args):
View Source
77    def nullable(self, *args):
78        return self.prop("nullable", *args)
#   def skipping_index_expression(self, *args):
View Source
80    def skipping_index_expression(self, *args):
81        return self.prop("skipping_index_expression", *args)
#   def ttl_expression(self, *args):
View Source
83    def ttl_expression(self, *args):
84        return self.prop("ttl_expression", *args)
Inherited Members
timeplus.base.Base
prop
data
id
View Source
 87class Stream(ResourceBase):
 88    """
 89    Stream class defines stream object
 90    """
 91
 92    _resource_name = "streams"
 93
 94    def __init__(self, env=None):
 95        ResourceBase.__init__(self, env)
 96        self.prop("columns", [])
 97
 98    @classmethod
 99    def build(cls, val, env=None):
100        obj = cls(env=env)
101        obj._data = val
102        return obj
103
104    # the list api is not implemented, has to manually implement it here
105    def get(self):
106        streams = Stream.list()
107        for s in streams:
108            if s.name() == self.name():
109                return s
110
111    def delete(self):
112        url = f"{self._base_url}/{self._resource_name}/{self.name()}"
113        self._logger.debug("delete {}", url)
114        try:
115            r = requests.delete(
116                url,
117                headers=self._headers,
118                timeout=self._env.http_timeout(),
119            )
120            if r.status_code < 200 or r.status_code > 299:
121                err_msg = f"failed to delete {self._resource_name} due to {r.text}"
122                raise TimeplusAPIError("delete", r.status_code, err_msg)
123            else:
124                self._logger.debug(f"delete {self._resource_name} success")
125                return self
126        except Exception as e:
127            self._logger.error(f"failed to delete {e}")
128            raise e
129
130    def insert(self, data, headers=None):
131        url = f"{self._base_url}/{self._resource_name}/{self.name()}/ingest"
132        self._logger.debug("post {}", url)
133
134        insert_headers = self.columnNames()
135        if headers is not None:
136            insert_headers = headers
137
138        insertRequest = {
139            "columns": insert_headers,
140            "data": data,
141        }
142        self._logger.debug(f"insert {insertRequest}")
143
144        try:
145            self._logger.debug(
146                f"insert {json.dumps(insertRequest, cls=DateTimeEncoder)}"
147            )
148            r = requests.post(
149                url,
150                data=json.dumps(insertRequest, cls=DateTimeEncoder),
151                headers=self._headers,
152                timeout=self._env.http_timeout(),
153            )
154            if r.status_code < 200 or r.status_code > 299:
155                err_msg = f"failed to insert into {self._resource_name} due to {r.text}"
156                raise TimeplusAPIError("post", r.status_code, err_msg)
157            else:
158                self._logger.debug("insert success")
159                return self
160        except Exception as e:
161            self._logger.error(f"failed to insert {e}")
162            raise e
163
164    def name(self, *args):
165        return self.prop("name", *args)
166
167    def event_time_column(self, *args):
168        return self.prop("event_time_column", *args)
169
170    def order_by_expression(self, *args):
171        return self.prop("order_by_expression", *args)
172
173    def order_by_granularity(self, *args):
174        return self.prop("order_by_granularity", *args)
175
176    def partition_by_granularity(self, *args):
177        return self.prop("partition_by_granularity", *args)
178
179    def replication_factor(self, *args):
180        return self.prop("replication_factor", *args)
181
182    def shards(self, *args):
183        return self.prop("shards", *args)
184
185    def ttl_expression(self, *args):
186        return self.prop("ttl_expression", *args)
187
188    # note, no way to remove/rename col for now
189    def column(self, col):
190        col_prop = self.prop("columns")
191        col_prop.append(col.data())
192        self.prop("columns", col_prop)
193        return self
194
195    def columnNames(self):
196        col_prop = self.prop("columns")
197        return [x["name"] for x in col_prop if not x["name"].startswith("_tp")]

Stream class defines stream object

#   Stream(env=None)
View Source
94    def __init__(self, env=None):
95        ResourceBase.__init__(self, env)
96        self.prop("columns", [])
#  
@classmethod
def build(cls, val, env=None):
View Source
 98    @classmethod
 99    def build(cls, val, env=None):
100        obj = cls(env=env)
101        obj._data = val
102        return obj
#   def get(self):
View Source
105    def get(self):
106        streams = Stream.list()
107        for s in streams:
108            if s.name() == self.name():
109                return s
#   def delete(self):
View Source
111    def delete(self):
112        url = f"{self._base_url}/{self._resource_name}/{self.name()}"
113        self._logger.debug("delete {}", url)
114        try:
115            r = requests.delete(
116                url,
117                headers=self._headers,
118                timeout=self._env.http_timeout(),
119            )
120            if r.status_code < 200 or r.status_code > 299:
121                err_msg = f"failed to delete {self._resource_name} due to {r.text}"
122                raise TimeplusAPIError("delete", r.status_code, err_msg)
123            else:
124                self._logger.debug(f"delete {self._resource_name} success")
125                return self
126        except Exception as e:
127            self._logger.error(f"failed to delete {e}")
128            raise e
#   def insert(self, data, headers=None):
View Source
130    def insert(self, data, headers=None):
131        url = f"{self._base_url}/{self._resource_name}/{self.name()}/ingest"
132        self._logger.debug("post {}", url)
133
134        insert_headers = self.columnNames()
135        if headers is not None:
136            insert_headers = headers
137
138        insertRequest = {
139            "columns": insert_headers,
140            "data": data,
141        }
142        self._logger.debug(f"insert {insertRequest}")
143
144        try:
145            self._logger.debug(
146                f"insert {json.dumps(insertRequest, cls=DateTimeEncoder)}"
147            )
148            r = requests.post(
149                url,
150                data=json.dumps(insertRequest, cls=DateTimeEncoder),
151                headers=self._headers,
152                timeout=self._env.http_timeout(),
153            )
154            if r.status_code < 200 or r.status_code > 299:
155                err_msg = f"failed to insert into {self._resource_name} due to {r.text}"
156                raise TimeplusAPIError("post", r.status_code, err_msg)
157            else:
158                self._logger.debug("insert success")
159                return self
160        except Exception as e:
161            self._logger.error(f"failed to insert {e}")
162            raise e
#   def name(self, *args):
View Source
164    def name(self, *args):
165        return self.prop("name", *args)
#   def event_time_column(self, *args):
View Source
167    def event_time_column(self, *args):
168        return self.prop("event_time_column", *args)
#   def order_by_expression(self, *args):
View Source
170    def order_by_expression(self, *args):
171        return self.prop("order_by_expression", *args)
#   def order_by_granularity(self, *args):
View Source
173    def order_by_granularity(self, *args):
174        return self.prop("order_by_granularity", *args)
#   def partition_by_granularity(self, *args):
View Source
176    def partition_by_granularity(self, *args):
177        return self.prop("partition_by_granularity", *args)
#   def replication_factor(self, *args):
View Source
179    def replication_factor(self, *args):
180        return self.prop("replication_factor", *args)
#   def shards(self, *args):
View Source
182    def shards(self, *args):
183        return self.prop("shards", *args)
#   def ttl_expression(self, *args):
View Source
185    def ttl_expression(self, *args):
186        return self.prop("ttl_expression", *args)
#   def column(self, col):
View Source
189    def column(self, col):
190        col_prop = self.prop("columns")
191        col_prop.append(col.data())
192        self.prop("columns", col_prop)
193        return self
#   def columnNames(self):
View Source
195    def columnNames(self):
196        col_prop = self.prop("columns")
197        return [x["name"] for x in col_prop if not x["name"].startswith("_tp")]