Skip to content

Commit bcf3e11

Browse files
committed
Simplify the json_stream() utility
The previous implementation was generic over the kind of decoder and separator. However, the only use was with JSON decoder and newline-based splitting. Signed-off-by: Francesco Zardi <frazar0@hotmail.it>
1 parent c7d4ec1 commit bcf3e11

File tree

1 file changed

+9
-21
lines changed

1 file changed

+9
-21
lines changed

docker/utils/json_stream.py

Lines changed: 9 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import json
21
import json.decoder
32

43
from ..errors import StreamParseError
@@ -37,30 +36,12 @@ def json_stream(stream):
3736
This handles streams which are inconsistently buffered (some entries may
3837
be newline delimited, and others are not).
3938
"""
40-
return split_buffer(stream, json_splitter, json_decoder.decode)
41-
42-
43-
def line_splitter(buffer, separator='\n'):
44-
index = buffer.find(str(separator))
45-
if index == -1:
46-
return None
47-
return buffer[:index + 1], buffer[index + 1:]
48-
49-
50-
def split_buffer(stream, splitter=None, decoder=lambda a: a):
51-
"""Given a generator which yields strings and a splitter function,
52-
joins all input, splits on the separator and yields each chunk.
53-
Unlike string.split(), each chunk includes the trailing
54-
separator, except for the last one if none was found on the end
55-
of the input.
56-
"""
57-
splitter = splitter or line_splitter
5839
buffered = ''
5940

6041
for data in stream_as_text(stream):
6142
buffered += data
6243
while True:
63-
buffer_split = splitter(buffered)
44+
buffer_split = json_splitter(buffered)
6445
if buffer_split is None:
6546
break
6647

@@ -69,6 +50,13 @@ def split_buffer(stream, splitter=None, decoder=lambda a: a):
6950

7051
if buffered:
7152
try:
72-
yield decoder(buffered)
53+
yield json_decoder.decode(buffered)
7354
except Exception as e:
7455
raise StreamParseError(e) from e
56+
57+
58+
def line_splitter(buffer: str, separator='\n'):
59+
index = buffer.find(str(separator))
60+
if index == -1:
61+
return None
62+
return buffer[:index + 1], buffer[index + 1:]

0 commit comments

Comments
 (0)