Skip to content

API

lazylines.read_jsonl(path)

Read .jsonl file and turn it into a LazyLines object.

Source code in lazylines/__init__.py 
12
13
14
def read_jsonl(path: Union[str, Path]) -> LazyLines:
    """Read .jsonl file and turn it into a LazyLines object."""
    return LazyLines(srsly.read_jsonl(path))

lazylines.LazyLines

An object that can wrangle iterables of dictionaries (similar to JSONL).

from lazylines import LazyLines
Source code in lazylines/__init__.py 
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
class LazyLines:
    """
    An object that can wrangle iterables of dictionaries (similar to JSONL).

    ```python
    from lazylines import LazyLines
    ```
    """

    def __init__(self, g):
        self.g = g
        self.groups = set()

    def cache(self) -> LazyLines:
        """
        Cache the result internally by turning it into a list.

        It's recommended to store the result into another variable.

        ```python
        from lazylines import LazyLines

        items = ({"a": i} for i in range(100))

        # The intermediate representation is now a list.
        cached = (LazyLines(items).cache())
        ```
        """
        return LazyLines(g=list(self.g))

    def mutate(self, **kwargs: Dict[str, Callable]) -> LazyLines:
        """
        Adds/overwrites keys in the dictionary based on lambda.

        Arguments:
            kwargs: str/callable pairs that represent keys and a function to calculate it's value

        **Usage**:

        ```python
        from lazylines import LazyLines

        items = [{"a": 2}, {"a": 3}]
        results = (LazyLines(items).mutate(b=lambda d: d["a"] * 2))
        expected = [{"a": 2, "b": 4}, {"a": 3, "b": 6}]
        assert results.collect() == expected
        ```
        """

        def new_gen():
            for item in self.g:
                for k, v in kwargs.items():
                    item[k] = v(item)
                yield item

        return LazyLines(g=new_gen())

    def keep(self, *args: Callable) -> LazyLines:
        """
        Only keep a subset of the items in the generator based on lambda.

        Arguments:
            args: functions that can be used to filter the data, if it outputs `True` it will be kept around

        **Usage**:

        ```python
        from lazylines import LazyLines

        items = [{"a": 2}, {"a": 3}]
        results = (LazyLines(items).keep(lambda d: d["a"] % 2 == 0))
        expected = [{"a": 2}]
        assert results.collect() == expected
        ```
        """

        def new_gen():
            for item in self.g:
                allowed = True
                for func in args:
                    if not func(item):
                        allowed = False
                if allowed:
                    yield item

        return LazyLines(g=new_gen())

    def unnest(self, key: str="subset") -> LazyLines:
        """
        Explodes a key, effectively un-nesting it.

        Arguments:
            key: the key to un-nest

        **Usage**:

        ```python
        from lazylines import LazyLines

        data = [{'annotator': 'a',
                 'subset': [{'accept': True, 'text': 'foo'},
                            {'accept': True, 'text': 'foobar'}]},
                {'annotator': 'b',
                 'subset': [{'accept': True, 'text': 'foo'},
                            {'accept': True, 'text': 'foobar'}]}]

        expected = [
            {'accept': True, 'annotator': 'a', 'text': 'foo'},
            {'accept': True, 'annotator': 'a', 'text': 'foobar'},
            {'accept': True, 'annotator': 'b', 'text': 'foo'},
            {'accept': True, 'annotator': 'b', 'text': 'foobar'}
        ]

        result = LazyLines(data).unnest("subset")
        assert result.collect() == expected
        ```
        """

        def new_gen():
            for item in self.g:
                for value in item[key]:
                    orig = {k: v for k, v in item.items() if k != key}
                    d = {**{key: value}, **orig}
                    yield d

        return LazyLines(g=new_gen())

    def head(self, n=5) -> LazyLines:
        """
        Make a subset and only return the top `n` items.

        Arguments:
            n: the number of examples to take
        """
        if isinstance(self.g, list):
            return LazyLines(g=(i for i in self.g[:5]))

        def new_gen():
            for _ in range(n):
                yield next(self.g)

        return LazyLines(g=new_gen())

    def show(self, n: int=5) -> LazyLines:
        """
        Give a preview of the first `n` examples. 

        Arguments:
            n: the number of examples to preview
        """
        stream_orig, stream_copy = it.tee(self.g)
        for _ in range(n):
            pprint.pprint(next(stream_copy))
        return LazyLines(g=stream_orig)

    def map(self, func: Callable) -> LazyLines:
        """
        Apply a function to each item before yielding it back.

        Arguments:
            func: the function to call on each item
        """

        def new_gen():
            for item in self.g:
                yield func(item)

        return LazyLines(g=new_gen())

    def tee(self, n:int=2) -> Tuple[LazyLines]:
        """
        Copies the lazylines.

        Arguments:
            n: how often to `tee` the stream

        Usage:

        ```python
        from lazylines import LazyLines

        data = [
            {'accept': True, 'annotator': 'a', 'text': 'foo'},
            {'accept': True, 'annotator': 'a', 'text': 'foobar'},
            {'accept': True, 'annotator': 'b', 'text': 'foo'},
            {'accept': True, 'annotator': 'b', 'text': 'foobar'}
        ]

        lines1, lines2 = LazyLines(data).tee(n=2)
        lines1, lines2, lines3 = LazyLines(data).tee(n=3)
        ```
        """
        return tuple(LazyLines(g=gen) for gen in it.tee(self.g, n))

    def __iter__(self):
        return iter(self.g)

    def sort_by(self, *keys: str) -> LazyLines:
        """
        Sort the items based on a subset of the keys.

        Arguments:
            keys: the keys to use for sorting
        """
        return LazyLines(g=sorted(self.g, key=lambda d: tuple([d[c] for c in keys])))

    def rename(self, **kwargs: Dict[str, str]) -> LazyLines:
        """
        Rename a few keys in each item.

        Arguments:
            kwargs: str/str pairs that resemble the new name and old name of a key

        Usage:

        ```python
        from lazylines import LazyLines

        data = [
            {'labeller': 'a', 'text': 'foo'},
            {'labeller': 'a', 'text': 'foobar'},
            {'labeller': 'b', 'text': 'foo'},
            {'labeller': 'b', 'text': 'foobar'}
        ]

        expected = [
            {'annotator': 'a', 'text': 'foo'},
            {'annotator': 'a', 'text': 'foobar'},
            {'annotator': 'b', 'text': 'foo'},
            {'annotator': 'b', 'text': 'foobar'}
        ]

        result = (LazyLines(data).rename(annotator="labeller").collect())
        assert result == expected
        ```
        """

        def new_gen():
            for item in self.g:
                new_items = {k: item[v] for k, v in kwargs.items()}
                yield {**item, **new_items}

        return LazyLines(g=new_gen())

    def nest_by(self, *keys: str) -> LazyLines:
        """
        Group by keys and return nested collections.

        The opposite of `.unnest()`

        Arguments:
            keys: the keys to nest by

        **Usage**:

        ```python
        from lazylines import LazyLines

        data = [
            {'accept': True, 'annotator': 'a', 'text': 'foo'},
            {'accept': True, 'annotator': 'a', 'text': 'foobar'},
            {'accept': True, 'annotator': 'b', 'text': 'foo'},
            {'accept': True, 'annotator': 'b', 'text': 'foobar'}
        ]

        expected = [
                {'annotator': 'a',
                 'subset': [{'accept': True, 'text': 'foo'},
                            {'accept': True, 'text': 'foobar'}]},
                {'annotator': 'b',
                 'subset': [{'accept': True, 'text': 'foo'},
                            {'accept': True, 'text': 'foobar'}]}
        ]

        result = LazyLines(data).nest_by("annotator")
        assert result.collect() == expected
        ```
        """
        groups = {}
        for example in self.g:
            key = tuple(example.get(arg, None) for arg in keys)
            if key not in groups:
                groups[key] = []
            for arg in keys:
                del example[arg]
            groups[key].append(example)
        result = []
        for key, values in groups.items():
            result.append({**{k: v for k, v in zip(keys, key)}, "subset": values})
        return LazyLines(result)

    def progress(self) -> LazyLines:
        """
        Adds a progress bar. Meant to be used early.

        Will also run through the entire stream once to calculate the stream size.
        """
        stream_orig, stream_copy = it.tee(self.g)
        total = sum(1 for _ in stream_copy)
        return LazyLines(g=tqdm.tqdm(stream_orig, total=total))

    def collect(self) -> LazyLines:
        """
        Turns the (final) sequence into a list.

        Note that, as a consequence, this will also empty the lazyline object.
        """
        return [ex for ex in self.g]

    def write_jsonl(
        self, path, append: bool = False, append_new_line: bool = True
    ) -> LazyLines:
        """
        Write everything into a jsonl file.

        Note that, as a consequence, this will also empty the lazyline object.
        """
        srsly.write_jsonl(path, self.g, append=append, append_new_line=append_new_line)

    def select(self, *keys: str) -> LazyLines:
        """
        Only select specific keys from each dictionary.

        Arguments:
            keys: the names of the keys to be kept around

        Usage:

        ```python
        from lazylines import LazyLines

        data = [
            {'accept': True, 'annotator': 'a', 'text': 'foo'},
            {'accept': True, 'annotator': 'a', 'text': 'foobar'},
            {'accept': True, 'annotator': 'b', 'text': 'foo'},
            {'accept': True, 'annotator': 'b', 'text': 'foobar'}
        ]

        expected = [
            {'annotator': 'a', 'text': 'foo'},
            {'annotator': 'a', 'text': 'foobar'},
            {'annotator': 'b', 'text': 'foo'},
            {'annotator': 'b', 'text': 'foobar'}
        ]

        result = LazyLines(data).select("annotator", "text")
        assert result.collect() == expected
        ```
        """

        def new_gen():
            for ex in self.g:
                yield {k: v for k, v in ex.items() if k in keys}

        return LazyLines(g=new_gen())

    def drop(self, *args) -> LazyLines:
        """
        Drop specific keys from each dictionary.

        Arguments:
            keys: the names of the keys to be kept around

        Usage:

        ```python
        from lazylines import LazyLines

        data = [
            {'accept': True, 'annotator': 'a', 'text': 'foo'},
            {'accept': True, 'annotator': 'a', 'text': 'foobar'},
            {'accept': True, 'annotator': 'b', 'text': 'foo'},
            {'accept': True, 'annotator': 'b', 'text': 'foobar'}
        ]

        expected = [
            {'annotator': 'a', 'text': 'foo'},
            {'annotator': 'a', 'text': 'foobar'},
            {'annotator': 'b', 'text': 'foo'},
            {'annotator': 'b', 'text': 'foobar'}
        ]

        result = LazyLines(data).drop("accept")
        assert result.collect() == expected
        ```
        """

        def new_gen():
            for ex in self.g:
                yield {k: v for k, v in ex.items() if k not in args}

        return LazyLines(g=new_gen())

    def pipe(self, func, *args, **kwargs) -> LazyLines:
        """Call a function over the entire generator."""
        return LazyLines(g=func(self, *args, **kwargs))

    def foreach(self, func, *args, **kwargs) -> LazyLines:
        """Just call a function on each dictionary, but pass the original forward."""

        def new_gen():
            for ex in self.g:
                func(ex, *args, **kwargs)
                yield ex

        return LazyLines(g=new_gen())

    def agg(self, **kwargs: Dict[str, Callable]):
        data = [ex for ex in self.g]
        return {
            k: func(data) for k, func in kwargs.items()
        }

cache()

Cache the result internally by turning it into a list.

It's recommended to store the result into another variable.

from lazylines import LazyLines

items = ({"a": i} for i in range(100))

# The intermediate representation is now a list.
cached = (LazyLines(items).cache())
Source code in lazylines/__init__.py 
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
def cache(self) -> LazyLines:
    """
    Cache the result internally by turning it into a list.

    It's recommended to store the result into another variable.

    ```python
    from lazylines import LazyLines

    items = ({"a": i} for i in range(100))

    # The intermediate representation is now a list.
    cached = (LazyLines(items).cache())
    ```
    """
    return LazyLines(g=list(self.g))

collect()

Turns the (final) sequence into a list.

Note that, as a consequence, this will also empty the lazyline object.

Source code in lazylines/__init__.py 
318
319
320
321
322
323
324
def collect(self) -> LazyLines:
    """
    Turns the (final) sequence into a list.

    Note that, as a consequence, this will also empty the lazyline object.
    """
    return [ex for ex in self.g]

drop(*args)

Drop specific keys from each dictionary.

Parameters:

Name Type Description Default
keys

the names of the keys to be kept around

 required

Usage:

from lazylines import LazyLines

data = [
    {'accept': True, 'annotator': 'a', 'text': 'foo'},
    {'accept': True, 'annotator': 'a', 'text': 'foobar'},
    {'accept': True, 'annotator': 'b', 'text': 'foo'},
    {'accept': True, 'annotator': 'b', 'text': 'foobar'}
]

expected = [
    {'annotator': 'a', 'text': 'foo'},
    {'annotator': 'a', 'text': 'foobar'},
    {'annotator': 'b', 'text': 'foo'},
    {'annotator': 'b', 'text': 'foobar'}
]

result = LazyLines(data).drop("accept")
assert result.collect() == expected
Source code in lazylines/__init__.py 
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
def drop(self, *args) -> LazyLines:
    """
    Drop specific keys from each dictionary.

    Arguments:
        keys: the names of the keys to be kept around

    Usage:

    ```python
    from lazylines import LazyLines

    data = [
        {'accept': True, 'annotator': 'a', 'text': 'foo'},
        {'accept': True, 'annotator': 'a', 'text': 'foobar'},
        {'accept': True, 'annotator': 'b', 'text': 'foo'},
        {'accept': True, 'annotator': 'b', 'text': 'foobar'}
    ]

    expected = [
        {'annotator': 'a', 'text': 'foo'},
        {'annotator': 'a', 'text': 'foobar'},
        {'annotator': 'b', 'text': 'foo'},
        {'annotator': 'b', 'text': 'foobar'}
    ]

    result = LazyLines(data).drop("accept")
    assert result.collect() == expected
    ```
    """

    def new_gen():
        for ex in self.g:
            yield {k: v for k, v in ex.items() if k not in args}

    return LazyLines(g=new_gen())

foreach(func, *args, **kwargs)

Just call a function on each dictionary, but pass the original forward.

Source code in lazylines/__init__.py 
414
415
416
417
418
419
420
421
422
def foreach(self, func, *args, **kwargs) -> LazyLines:
    """Just call a function on each dictionary, but pass the original forward."""

    def new_gen():
        for ex in self.g:
            func(ex, *args, **kwargs)
            yield ex

    return LazyLines(g=new_gen())

head(n=5)

Make a subset and only return the top n items.

Parameters:

Name Type Description Default
n

the number of examples to take

 5
Source code in lazylines/__init__.py 
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
def head(self, n=5) -> LazyLines:
    """
    Make a subset and only return the top `n` items.

    Arguments:
        n: the number of examples to take
    """
    if isinstance(self.g, list):
        return LazyLines(g=(i for i in self.g[:5]))

    def new_gen():
        for _ in range(n):
            yield next(self.g)

    return LazyLines(g=new_gen())

keep(*args)

Only keep a subset of the items in the generator based on lambda.

Parameters:

Name Type Description Default
args Callable

functions that can be used to filter the data, if it outputs True it will be kept around

 ()

Usage:

from lazylines import LazyLines

items = [{"a": 2}, {"a": 3}]
results = (LazyLines(items).keep(lambda d: d["a"] % 2 == 0))
expected = [{"a": 2}]
assert results.collect() == expected
Source code in lazylines/__init__.py 
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
def keep(self, *args: Callable) -> LazyLines:
    """
    Only keep a subset of the items in the generator based on lambda.

    Arguments:
        args: functions that can be used to filter the data, if it outputs `True` it will be kept around

    **Usage**:

    ```python
    from lazylines import LazyLines

    items = [{"a": 2}, {"a": 3}]
    results = (LazyLines(items).keep(lambda d: d["a"] % 2 == 0))
    expected = [{"a": 2}]
    assert results.collect() == expected
    ```
    """

    def new_gen():
        for item in self.g:
            allowed = True
            for func in args:
                if not func(item):
                    allowed = False
            if allowed:
                yield item

    return LazyLines(g=new_gen())

map(func)

Apply a function to each item before yielding it back.

Parameters:

Name Type Description Default
func Callable

the function to call on each item

 required
Source code in lazylines/__init__.py 
172
173
174
175
176
177
178
179
180
181
182
183
184
def map(self, func: Callable) -> LazyLines:
    """
    Apply a function to each item before yielding it back.

    Arguments:
        func: the function to call on each item
    """

    def new_gen():
        for item in self.g:
            yield func(item)

    return LazyLines(g=new_gen())

mutate(**kwargs)

Adds/overwrites keys in the dictionary based on lambda.

Parameters:

Name Type Description Default
kwargs Dict[str, Callable]

str/callable pairs that represent keys and a function to calculate it's value

 {}

Usage:

from lazylines import LazyLines

items = [{"a": 2}, {"a": 3}]
results = (LazyLines(items).mutate(b=lambda d: d["a"] * 2))
expected = [{"a": 2, "b": 4}, {"a": 3, "b": 6}]
assert results.collect() == expected
Source code in lazylines/__init__.py 
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
def mutate(self, **kwargs: Dict[str, Callable]) -> LazyLines:
    """
    Adds/overwrites keys in the dictionary based on lambda.

    Arguments:
        kwargs: str/callable pairs that represent keys and a function to calculate it's value

    **Usage**:

    ```python
    from lazylines import LazyLines

    items = [{"a": 2}, {"a": 3}]
    results = (LazyLines(items).mutate(b=lambda d: d["a"] * 2))
    expected = [{"a": 2, "b": 4}, {"a": 3, "b": 6}]
    assert results.collect() == expected
    ```
    """

    def new_gen():
        for item in self.g:
            for k, v in kwargs.items():
                item[k] = v(item)
            yield item

    return LazyLines(g=new_gen())

nest_by(*keys)

Group by keys and return nested collections.

The opposite of .unnest()

Parameters:

Name Type Description Default
keys str

the keys to nest by

 ()

Usage:

from lazylines import LazyLines

data = [
    {'accept': True, 'annotator': 'a', 'text': 'foo'},
    {'accept': True, 'annotator': 'a', 'text': 'foobar'},
    {'accept': True, 'annotator': 'b', 'text': 'foo'},
    {'accept': True, 'annotator': 'b', 'text': 'foobar'}
]

expected = [
        {'annotator': 'a',
         'subset': [{'accept': True, 'text': 'foo'},
                    {'accept': True, 'text': 'foobar'}]},
        {'annotator': 'b',
         'subset': [{'accept': True, 'text': 'foo'},
                    {'accept': True, 'text': 'foobar'}]}
]

result = LazyLines(data).nest_by("annotator")
assert result.collect() == expected
Source code in lazylines/__init__.py 
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
def nest_by(self, *keys: str) -> LazyLines:
    """
    Group by keys and return nested collections.

    The opposite of `.unnest()`

    Arguments:
        keys: the keys to nest by

    **Usage**:

    ```python
    from lazylines import LazyLines

    data = [
        {'accept': True, 'annotator': 'a', 'text': 'foo'},
        {'accept': True, 'annotator': 'a', 'text': 'foobar'},
        {'accept': True, 'annotator': 'b', 'text': 'foo'},
        {'accept': True, 'annotator': 'b', 'text': 'foobar'}
    ]

    expected = [
            {'annotator': 'a',
             'subset': [{'accept': True, 'text': 'foo'},
                        {'accept': True, 'text': 'foobar'}]},
            {'annotator': 'b',
             'subset': [{'accept': True, 'text': 'foo'},
                        {'accept': True, 'text': 'foobar'}]}
    ]

    result = LazyLines(data).nest_by("annotator")
    assert result.collect() == expected
    ```
    """
    groups = {}
    for example in self.g:
        key = tuple(example.get(arg, None) for arg in keys)
        if key not in groups:
            groups[key] = []
        for arg in keys:
            del example[arg]
        groups[key].append(example)
    result = []
    for key, values in groups.items():
        result.append({**{k: v for k, v in zip(keys, key)}, "subset": values})
    return LazyLines(result)

pipe(func, *args, **kwargs)

Call a function over the entire generator.

Source code in lazylines/__init__.py 
410
411
412
def pipe(self, func, *args, **kwargs) -> LazyLines:
    """Call a function over the entire generator."""
    return LazyLines(g=func(self, *args, **kwargs))

progress()

Adds a progress bar. Meant to be used early.

Will also run through the entire stream once to calculate the stream size.

Source code in lazylines/__init__.py 
308
309
310
311
312
313
314
315
316
def progress(self) -> LazyLines:
    """
    Adds a progress bar. Meant to be used early.

    Will also run through the entire stream once to calculate the stream size.
    """
    stream_orig, stream_copy = it.tee(self.g)
    total = sum(1 for _ in stream_copy)
    return LazyLines(g=tqdm.tqdm(stream_orig, total=total))

rename(**kwargs)

Rename a few keys in each item.

Parameters:

Name Type Description Default
kwargs Dict[str, str]

str/str pairs that resemble the new name and old name of a key

 {}

Usage:

from lazylines import LazyLines

data = [
    {'labeller': 'a', 'text': 'foo'},
    {'labeller': 'a', 'text': 'foobar'},
    {'labeller': 'b', 'text': 'foo'},
    {'labeller': 'b', 'text': 'foobar'}
]

expected = [
    {'annotator': 'a', 'text': 'foo'},
    {'annotator': 'a', 'text': 'foobar'},
    {'annotator': 'b', 'text': 'foo'},
    {'annotator': 'b', 'text': 'foobar'}
]

result = (LazyLines(data).rename(annotator="labeller").collect())
assert result == expected
Source code in lazylines/__init__.py 
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
def rename(self, **kwargs: Dict[str, str]) -> LazyLines:
    """
    Rename a few keys in each item.

    Arguments:
        kwargs: str/str pairs that resemble the new name and old name of a key

    Usage:

    ```python
    from lazylines import LazyLines

    data = [
        {'labeller': 'a', 'text': 'foo'},
        {'labeller': 'a', 'text': 'foobar'},
        {'labeller': 'b', 'text': 'foo'},
        {'labeller': 'b', 'text': 'foobar'}
    ]

    expected = [
        {'annotator': 'a', 'text': 'foo'},
        {'annotator': 'a', 'text': 'foobar'},
        {'annotator': 'b', 'text': 'foo'},
        {'annotator': 'b', 'text': 'foobar'}
    ]

    result = (LazyLines(data).rename(annotator="labeller").collect())
    assert result == expected
    ```
    """

    def new_gen():
        for item in self.g:
            new_items = {k: item[v] for k, v in kwargs.items()}
            yield {**item, **new_items}

    return LazyLines(g=new_gen())

select(*keys)

Only select specific keys from each dictionary.

Parameters:

Name Type Description Default
keys str

the names of the keys to be kept around

 ()

Usage:

from lazylines import LazyLines

data = [
    {'accept': True, 'annotator': 'a', 'text': 'foo'},
    {'accept': True, 'annotator': 'a', 'text': 'foobar'},
    {'accept': True, 'annotator': 'b', 'text': 'foo'},
    {'accept': True, 'annotator': 'b', 'text': 'foobar'}
]

expected = [
    {'annotator': 'a', 'text': 'foo'},
    {'annotator': 'a', 'text': 'foobar'},
    {'annotator': 'b', 'text': 'foo'},
    {'annotator': 'b', 'text': 'foobar'}
]

result = LazyLines(data).select("annotator", "text")
assert result.collect() == expected
Source code in lazylines/__init__.py 
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
def select(self, *keys: str) -> LazyLines:
    """
    Only select specific keys from each dictionary.

    Arguments:
        keys: the names of the keys to be kept around

    Usage:

    ```python
    from lazylines import LazyLines

    data = [
        {'accept': True, 'annotator': 'a', 'text': 'foo'},
        {'accept': True, 'annotator': 'a', 'text': 'foobar'},
        {'accept': True, 'annotator': 'b', 'text': 'foo'},
        {'accept': True, 'annotator': 'b', 'text': 'foobar'}
    ]

    expected = [
        {'annotator': 'a', 'text': 'foo'},
        {'annotator': 'a', 'text': 'foobar'},
        {'annotator': 'b', 'text': 'foo'},
        {'annotator': 'b', 'text': 'foobar'}
    ]

    result = LazyLines(data).select("annotator", "text")
    assert result.collect() == expected
    ```
    """

    def new_gen():
        for ex in self.g:
            yield {k: v for k, v in ex.items() if k in keys}

    return LazyLines(g=new_gen())

show(n=5)

Give a preview of the first n examples.

Parameters:

Name Type Description Default
n int

the number of examples to preview

 5
Source code in lazylines/__init__.py 
160
161
162
163
164
165
166
167
168
169
170
def show(self, n: int=5) -> LazyLines:
    """
    Give a preview of the first `n` examples. 

    Arguments:
        n: the number of examples to preview
    """
    stream_orig, stream_copy = it.tee(self.g)
    for _ in range(n):
        pprint.pprint(next(stream_copy))
    return LazyLines(g=stream_orig)

sort_by(*keys)

Sort the items based on a subset of the keys.

Parameters:

Name Type Description Default
keys str

the keys to use for sorting

 ()
Source code in lazylines/__init__.py 
214
215
216
217
218
219
220
221
def sort_by(self, *keys: str) -> LazyLines:
    """
    Sort the items based on a subset of the keys.

    Arguments:
        keys: the keys to use for sorting
    """
    return LazyLines(g=sorted(self.g, key=lambda d: tuple([d[c] for c in keys])))

tee(n=2)

Copies the lazylines.

Parameters:

Name Type Description Default
n int

how often to tee the stream

 2

Usage:

from lazylines import LazyLines

data = [
    {'accept': True, 'annotator': 'a', 'text': 'foo'},
    {'accept': True, 'annotator': 'a', 'text': 'foobar'},
    {'accept': True, 'annotator': 'b', 'text': 'foo'},
    {'accept': True, 'annotator': 'b', 'text': 'foobar'}
]

lines1, lines2 = LazyLines(data).tee(n=2)
lines1, lines2, lines3 = LazyLines(data).tee(n=3)
Source code in lazylines/__init__.py 
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
def tee(self, n:int=2) -> Tuple[LazyLines]:
    """
    Copies the lazylines.

    Arguments:
        n: how often to `tee` the stream

    Usage:

    ```python
    from lazylines import LazyLines

    data = [
        {'accept': True, 'annotator': 'a', 'text': 'foo'},
        {'accept': True, 'annotator': 'a', 'text': 'foobar'},
        {'accept': True, 'annotator': 'b', 'text': 'foo'},
        {'accept': True, 'annotator': 'b', 'text': 'foobar'}
    ]

    lines1, lines2 = LazyLines(data).tee(n=2)
    lines1, lines2, lines3 = LazyLines(data).tee(n=3)
    ```
    """
    return tuple(LazyLines(g=gen) for gen in it.tee(self.g, n))

unnest(key='subset')

Explodes a key, effectively un-nesting it.

Parameters:

Name Type Description Default
key str

the key to un-nest

 'subset'

Usage:

from lazylines import LazyLines

data = [{'annotator': 'a',
         'subset': [{'accept': True, 'text': 'foo'},
                    {'accept': True, 'text': 'foobar'}]},
        {'annotator': 'b',
         'subset': [{'accept': True, 'text': 'foo'},
                    {'accept': True, 'text': 'foobar'}]}]

expected = [
    {'accept': True, 'annotator': 'a', 'text': 'foo'},
    {'accept': True, 'annotator': 'a', 'text': 'foobar'},
    {'accept': True, 'annotator': 'b', 'text': 'foo'},
    {'accept': True, 'annotator': 'b', 'text': 'foobar'}
]

result = LazyLines(data).unnest("subset")
assert result.collect() == expected
Source code in lazylines/__init__.py 
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
def unnest(self, key: str="subset") -> LazyLines:
    """
    Explodes a key, effectively un-nesting it.

    Arguments:
        key: the key to un-nest

    **Usage**:

    ```python
    from lazylines import LazyLines

    data = [{'annotator': 'a',
             'subset': [{'accept': True, 'text': 'foo'},
                        {'accept': True, 'text': 'foobar'}]},
            {'annotator': 'b',
             'subset': [{'accept': True, 'text': 'foo'},
                        {'accept': True, 'text': 'foobar'}]}]

    expected = [
        {'accept': True, 'annotator': 'a', 'text': 'foo'},
        {'accept': True, 'annotator': 'a', 'text': 'foobar'},
        {'accept': True, 'annotator': 'b', 'text': 'foo'},
        {'accept': True, 'annotator': 'b', 'text': 'foobar'}
    ]

    result = LazyLines(data).unnest("subset")
    assert result.collect() == expected
    ```
    """

    def new_gen():
        for item in self.g:
            for value in item[key]:
                orig = {k: v for k, v in item.items() if k != key}
                d = {**{key: value}, **orig}
                yield d

    return LazyLines(g=new_gen())

write_jsonl(path, append=False, append_new_line=True)

Write everything into a jsonl file.

Note that, as a consequence, this will also empty the lazyline object.

Source code in lazylines/__init__.py 
326
327
328
329
330
331
332
333
334
def write_jsonl(
    self, path, append: bool = False, append_new_line: bool = True
) -> LazyLines:
    """
    Write everything into a jsonl file.

    Note that, as a consequence, this will also empty the lazyline object.
    """
    srsly.write_jsonl(path, self.g, append=append, append_new_line=append_new_line)