@@ -72,7 +72,7 @@ class TableType(JObjectWrapper):
72
72
j_object_type = jpy .get_type ("io.deephaven.kafka.KafkaTools$TableType" )
73
73
74
74
@staticmethod
75
- def blink ():
75
+ def blink () -> 'TableType' :
76
76
""" Consume all partitions into a single interleaved blink table, which will present only newly-available rows
77
77
to downstream operations and visualizations."""
78
78
return TableType (TableType .j_object_type .blink ())
@@ -85,12 +85,12 @@ def stream():
85
85
return TableType .blink ()
86
86
87
87
@staticmethod
88
- def append ():
88
+ def append () -> 'TableType' :
89
89
""" Consume all partitions into a single interleaved in-memory append-only table."""
90
90
return TableType (TableType .j_object_type .append ())
91
91
92
92
@staticmethod
93
- def ring (capacity : int ):
93
+ def ring (capacity : int ) -> 'TableType' :
94
94
""" Consume all partitions into a single in-memory ring table."""
95
95
return TableType (TableType .j_object_type .ring (capacity ))
96
96
@@ -101,14 +101,18 @@ def __init__(self, j_table_type: jpy.JType):
101
101
def j_object (self ) -> jpy .JType :
102
102
return self ._j_table_type
103
103
104
+
104
105
# TODO (https://github.com/deephaven/deephaven-core/issues/3853): Delete this attribute
105
106
TableType .Stream = TableType .blink ()
106
107
""" Deprecated, prefer TableType.blink(). Consume all partitions into a single interleaved blink table, which will
107
108
present only newly-available rows to downstream operations and visualizations."""
108
109
109
110
# TODO (https://github.com/deephaven/deephaven-core/issues/3853): Delete this attribute
110
111
TableType .Append = TableType .append ()
111
- """ Deprecated, prefer TableType.append(). Consume all partitions into a single interleaved in-memory append-only table."""
112
+ """ Deprecated, prefer TableType.append(). Consume all partitions into a single interleaved in-memory append-only
113
+ table."""
114
+
115
+ TableType .Blink = TableType .blink ()
112
116
113
117
114
118
def j_partitions (partitions ):
@@ -134,8 +138,8 @@ def consume(
134
138
offsets : Dict [int , int ] = None ,
135
139
key_spec : KeyValueSpec = None ,
136
140
value_spec : KeyValueSpec = None ,
137
- table_type : TableType = TableType . blink () ,
138
- ) -> Table :
141
+ table_type : TableType = None ,
142
+ ) -> Table :
139
143
"""Consume from Kafka to a Deephaven table.
140
144
141
145
Args:
@@ -158,17 +162,19 @@ def consume(
158
162
It can be the result of calling one of the functions: simple_spec(),avro_spec() or json_spec() in this
159
163
module, or the predefined KeyValueSpec.IGNORE or KeyValueSpec.FROM_PROPERTIES. The default is None which
160
164
works the same as KeyValueSpec.FROM_PROPERTIES, in which case, the kafka_config param should include values
161
- for dictionary keys 'deephaven.value.column.name' and 'deephaven.value.column.type', for the single resulting
165
+ for dictionary keys 'deephaven.value.column.name' and 'deephaven.value.column.type', for the single
166
+ resulting
162
167
column name and type
163
- table_type (TableType): a TableType, default is TableType.blink()
168
+ table_type (TableType): a TableType, default is None, meaning to use TableType.blink()
164
169
165
170
Returns:
166
171
a Deephaven live table that will update based on Kafka messages consumed for the given topic
167
172
168
173
Raises:
169
174
DHError
170
175
"""
171
-
176
+ if table_type is None :
177
+ table_type = TableType .blink ()
172
178
return _consume (kafka_config , topic , partitions , offsets , key_spec , value_spec , table_type , to_partitioned = False )
173
179
174
180
@@ -179,8 +185,8 @@ def consume_to_partitioned_table(
179
185
offsets : Dict [int , int ] = None ,
180
186
key_spec : KeyValueSpec = None ,
181
187
value_spec : KeyValueSpec = None ,
182
- table_type : TableType = TableType . blink () ,
183
- ) -> PartitionedTable :
188
+ table_type : TableType = None ,
189
+ ) -> PartitionedTable :
184
190
"""Consume from Kafka to a Deephaven partitioned table.
185
191
186
192
Args:
@@ -203,10 +209,11 @@ def consume_to_partitioned_table(
203
209
It can be the result of calling one of the functions: simple_spec(),avro_spec() or json_spec() in this
204
210
module, or the predefined KeyValueSpec.IGNORE or KeyValueSpec.FROM_PROPERTIES. The default is None which
205
211
works the same as KeyValueSpec.FROM_PROPERTIES, in which case, the kafka_config param should include values
206
- for dictionary keys 'deephaven.value.column.name' and 'deephaven.value.column.type', for the single resulting
212
+ for dictionary keys 'deephaven.value.column.name' and 'deephaven.value.column.type', for the single
213
+ resulting
207
214
column name and type
208
215
table_type (TableType): a TableType, specifying the type of the expected result's constituent tables,
209
- default is TableType.blink()
216
+ default is None, meaning to use TableType.blink()
210
217
211
218
Returns:
212
219
a Deephaven live partitioned table that will update based on Kafka messages consumed for the given topic,
@@ -216,7 +223,8 @@ def consume_to_partitioned_table(
216
223
Raises:
217
224
DHError
218
225
"""
219
-
226
+ if table_type is None :
227
+ table_type = TableType .blink ()
220
228
return _consume (kafka_config , topic , partitions , offsets , key_spec , value_spec , table_type , to_partitioned = True )
221
229
222
230
@@ -229,7 +237,7 @@ def _consume(
229
237
value_spec : KeyValueSpec = None ,
230
238
table_type : TableType = TableType .blink (),
231
239
to_partitioned : bool = False ,
232
- ) -> Union [Table , PartitionedTable ]:
240
+ ) -> Union [Table , PartitionedTable ]:
233
241
try :
234
242
partitions = j_partitions (partitions )
235
243
@@ -243,8 +251,8 @@ def _consume(
243
251
partitions_array = jpy .array ("int" , list (offsets .keys ()))
244
252
offsets_array = jpy .array ("long" , list (offsets .values ()))
245
253
offsets = _JKafkaTools .partitionToOffsetFromParallelArrays (
246
- partitions_array , offsets_array
247
- )
254
+ partitions_array , offsets_array
255
+ )
248
256
249
257
key_spec = KeyValueSpec .FROM_PROPERTIES if key_spec is None else key_spec
250
258
value_spec = KeyValueSpec .FROM_PROPERTIES if value_spec is None else value_spec
@@ -255,29 +263,32 @@ def _consume(
255
263
kafka_config = j_properties (kafka_config )
256
264
if not to_partitioned :
257
265
return Table (
258
- j_table = _JKafkaTools .consumeToTable (
259
- kafka_config ,
260
- topic ,
261
- partitions ,
262
- offsets ,
263
- key_spec .j_object ,
264
- value_spec .j_object ,
265
- table_type .j_object ,
266
- )
267
- )
266
+ j_table = _JKafkaTools .consumeToTable (
267
+ kafka_config ,
268
+ topic ,
269
+ partitions ,
270
+ offsets ,
271
+ key_spec .j_object ,
272
+ value_spec .j_object ,
273
+ table_type .j_object ,
274
+ )
275
+ )
268
276
else :
269
- return PartitionedTable (j_partitioned_table = _JKafkaTools .consumeToPartitionedTable (
270
- kafka_config ,
271
- topic ,
272
- partitions ,
273
- offsets ,
274
- key_spec .j_object ,
275
- value_spec .j_object ,
276
- table_type .j_object ,
277
- ))
277
+ return PartitionedTable (
278
+ j_partitioned_table = _JKafkaTools .consumeToPartitionedTable (
279
+ kafka_config ,
280
+ topic ,
281
+ partitions ,
282
+ offsets ,
283
+ key_spec .j_object ,
284
+ value_spec .j_object ,
285
+ table_type .j_object ,
286
+ )
287
+ )
278
288
except Exception as e :
279
289
raise DHError (e , "failed to consume a Kafka stream." ) from e
280
290
291
+
281
292
class ProtobufProtocol (JObjectWrapper ):
282
293
"""The protobuf serialization / deserialization protocol."""
283
294
@@ -310,7 +321,7 @@ def protobuf_spec(
310
321
message_class : Optional [str ] = None ,
311
322
include : Optional [List [str ]] = None ,
312
323
protocol : Optional [ProtobufProtocol ] = None ,
313
- ) -> KeyValueSpec :
324
+ ) -> KeyValueSpec :
314
325
"""Creates a spec for parsing a Kafka protobuf stream into a Deephaven table. Uses the schema, schema_version, and
315
326
schema_message_name to fetch the schema from the schema registry; or uses message_class to to get the schema from
316
327
the classpath.
@@ -344,13 +355,13 @@ def protobuf_spec(
344
355
parser_options_builder = _JProtobufDescriptorParserOptions .builder ()
345
356
if include is not None :
346
357
parser_options_builder .fieldOptions (
347
- _JFieldOptions .includeIf (
348
- _JFieldPath .anyMatches (j_array_list (include ))
349
- )
350
- )
358
+ _JFieldOptions .includeIf (
359
+ _JFieldPath .anyMatches (j_array_list (include ))
360
+ )
361
+ )
351
362
pb_consume_builder = (
352
- _JProtobufConsumeOptions .builder ()
353
- .parserOptions (parser_options_builder .build ())
363
+ _JProtobufConsumeOptions .builder ()
364
+ .parserOptions (parser_options_builder .build ())
354
365
)
355
366
if message_class :
356
367
if schema or schema_version or schema_message_name :
@@ -368,16 +379,16 @@ def protobuf_spec(
368
379
if protocol :
369
380
pb_consume_builder .protocol (protocol .j_object )
370
381
return KeyValueSpec (
371
- j_spec = _JKafkaTools_Consume .protobufSpec (pb_consume_builder .build ())
372
- )
382
+ j_spec = _JKafkaTools_Consume .protobufSpec (pb_consume_builder .build ())
383
+ )
373
384
374
385
375
386
def avro_spec (
376
387
schema : str ,
377
388
schema_version : str = "latest" ,
378
389
mapping : Dict [str , str ] = None ,
379
390
mapped_only : bool = False ,
380
- ) -> KeyValueSpec :
391
+ ) -> KeyValueSpec :
381
392
"""Creates a spec for how to use an Avro schema when consuming a Kafka stream to a Deephaven table.
382
393
383
394
Args:
@@ -388,7 +399,8 @@ def avro_spec(
388
399
the value of the Schema Server URL for fetching the schema definition
389
400
schema_version (str): the schema version to fetch from schema service, default is 'latest'
390
401
mapping (Dict[str, str]): a mapping from Avro field name to Deephaven table column name; the fields specified in
391
- the mapping will have their column names defined by it; if 'mapped_only' parameter is False, any other fields
402
+ the mapping will have their column names defined by it; if 'mapped_only' parameter is False,
403
+ any other fields
392
404
not mentioned in the mapping will use the same Avro field name for Deephaven table column; otherwise, these
393
405
unmapped fields will be ignored and will not be present in the resulting table. default is None
394
406
mapped_only (bool): whether to ignore Avro fields not present in the 'mapping' argument, default is False
@@ -407,22 +419,22 @@ def avro_spec(
407
419
jschema = _JKafkaTools .getAvroSchema (schema );
408
420
if mapping :
409
421
return KeyValueSpec (
410
- j_spec = _JKafkaTools_Consume .avroSpec (jschema , mapping )
411
- )
422
+ j_spec = _JKafkaTools_Consume .avroSpec (jschema , mapping )
423
+ )
412
424
else :
413
425
return KeyValueSpec (
414
- j_spec = _JKafkaTools_Consume .avroSpec (jschema )
415
- )
426
+ j_spec = _JKafkaTools_Consume .avroSpec (jschema )
427
+ )
416
428
417
429
else :
418
430
if mapping :
419
431
return KeyValueSpec (
420
- j_spec = _JKafkaTools_Consume .avroSpec (schema , schema_version , mapping )
421
- )
432
+ j_spec = _JKafkaTools_Consume .avroSpec (schema , schema_version , mapping )
433
+ )
422
434
else :
423
435
return KeyValueSpec (
424
- j_spec = _JKafkaTools_Consume .avroSpec (schema , schema_version )
425
- )
436
+ j_spec = _JKafkaTools_Consume .avroSpec (schema , schema_version )
437
+ )
426
438
except Exception as e :
427
439
raise DHError (e , "failed to create a Kafka key/value spec" ) from e
428
440
@@ -457,10 +469,11 @@ def json_spec(col_defs: Union[TableDefinitionLike, List[Tuple[str, DType]]], map
457
469
col_defs = [col .j_column_definition for col in table_def .values ()]
458
470
else :
459
471
warn (
460
- 'json_spec col_defs for List[Tuple[str, DType]] is deprecated for removal, prefer TableDefinitionLike' ,
461
- DeprecationWarning ,
462
- stacklevel = 2 ,
463
- )
472
+ 'json_spec col_defs for List[Tuple[str, DType]] is deprecated for removal, '
473
+ 'prefer TableDefinitionLike' ,
474
+ DeprecationWarning ,
475
+ stacklevel = 2 ,
476
+ )
464
477
col_defs = [col_def (* t ).j_column_definition for t in col_defs ]
465
478
466
479
if mapping is None :
@@ -489,8 +502,8 @@ def simple_spec(col_name: str, data_type: DType = None) -> KeyValueSpec:
489
502
if data_type is None :
490
503
return KeyValueSpec (j_spec = _JKafkaTools_Consume .simpleSpec (col_name ))
491
504
return KeyValueSpec (
492
- j_spec = _JKafkaTools_Consume .simpleSpec (col_name , data_type .qst_type .clazz ())
493
- )
505
+ j_spec = _JKafkaTools_Consume .simpleSpec (col_name , data_type .qst_type .clazz ())
506
+ )
494
507
except Exception as e :
495
508
raise DHError (e , "failed to create a Kafka key/value spec" ) from e
496
509
0 commit comments