Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

streaming-iterables

Package Overview
Dependencies
Maintainers
2
Versions
41
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

streaming-iterables - npm Package Compare versions

Comparing version 7.1.0 to 8.0.0

dist/index.d.ts.map

1013

dist/index.d.ts

@@ -1,625 +0,542 @@

/**
* Literally any `Iterable` (async or regular).
*/
export declare type AnyIterable<T> = Iterable<T> | AsyncIterable<T>;
declare module 'streaming-iterables' {
export type UnwrapAnyIterableArray<M extends AnyIterable<any>> = M extends Iterable<infer T> ? Generator<T[]> : M extends AsyncIterable<infer B> ? AsyncGenerator<B[]> : never;
export type CurriedBatchResult = <T, M extends AnyIterable<T>>(curriedIterable: M) => UnwrapAnyIterableArray<M>;
/**
* Batch objects from `iterable` into arrays of `size` length. The final array may be shorter than size if there is not enough items. Returns a sync iterator if the `iterable` is sync, otherwise an async iterator. Errors from the source `iterable` are immediately raised.
/**
* Batch objects from `iterable` into arrays of `size` length. The final array may be shorter than size if there is not enough items. Returns a sync iterator if the `iterable` is sync, otherwise an async iterator. Errors from the source `iterable` are immediately raised.
`size` can be between 1 and `Infinity`.
`size` can be between 1 and `Infinity`.
```ts
import { batch } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
```ts
import { batch } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
// batch 10 pokemon while we process them
for await (const pokemons of batch(10, getPokemon())) {
console.log(pokemons) // 10 pokemon at a time!
}
```
*/
export function batch(size: number): CurriedBatchResult;
export function batch<T, M extends AnyIterable<T>>(size: number, iterable: M): UnwrapAnyIterableArray<M>;
export type CurriedBatchWithTimeoutResult = <T, M extends AnyIterable<T>>(curriedIterable: M) => UnwrapAnyIterableArray<M>;
/**
* Like `batch` but flushes early if the `timeout` is reached. The batches may be shorter than size if there are not enough items. Returns a sync iterator if the `iterable` is sync, otherwise an async iterator. Errors from the source `iterable` are immediately raised.
// batch 10 pokemon while we process them
for await (const pokemons of batch(10, getPokemon())) {
console.log(pokemons) // 10 pokemon at a time!
}
```
*/
export declare function batch(size: number): CurriedBatchResult;
`size` can be between 1 and `Infinity`.
`timeout` can be between 0 and `Infinity`.
export declare function batch<T, M extends AnyIterable<T>>(size: number, iterable: M): UnwrapAnyIterableArray<M>;
```ts
import { batchWithTimeout } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
/**
* Like `batch` but flushes early if the `timeout` is reached. The batches may be shorter than size if there are not enough items. Returns a sync iterator if the `iterable` is sync, otherwise an async iterator. Errors from the source `iterable` are immediately raised.
// batch 10 pokemon while we process them
for await (const pokemons of batchWithTimeout(10, 100, getPokemon())) {
console.log(pokemons) // Up to 10 pokemon at a time!
}
```
*/
export function batchWithTimeout(size: number, timeout: number): CurriedBatchWithTimeoutResult;
export function batchWithTimeout<T, M extends AnyIterable<T>>(size: number, timeout: number, iterable: M): UnwrapAnyIterableArray<M>;
export type CurriedBufferResult = <T, M extends AnyIterable<T>>(curriedIterable: M) => UnwrapAnyIterable<M>;
/**
* Buffer keeps a number of objects in reserve available for immediate reading. This is helpful with async iterators as it will pre-fetch results so you don't have to wait for them to load. For sync iterables it will pre-compute up to `size` values and keep them in reserve. The internal buffer will start to be filled once `.next()` is called for the first time and will continue to fill until the source `iterable` is exhausted or the buffer is full. Errors from the source `iterable` will be raised after all buffered values are yielded.
`size` can be between 1 and `Infinity`.
`timeout` can be between 0 and `Infinity`.
`size` can be between 0 and `Infinity`.
```ts
import { batchWithTimeout } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
```ts
import { buffer } from 'streaming-iterables'
import { getPokemon, trainMonster } from 'iterable-pokedex'
// batch 10 pokemon while we process them
for await (const pokemons of batchWithTimeout(10, 100, getPokemon())) {
console.log(pokemons) // Up to 10 pokemon at a time!
}
```
*/
export declare function batchWithTimeout(size: number, timeout: number): CurriedBatchWithTimeoutResult;
// load 10 monsters in the background while we process them one by one
for await (const monster of buffer(10, getPokemon())) {
await trainMonster(monster) // got to do some pokéwork
}
```
*/
export function buffer(size: number): CurriedBufferResult;
export function buffer<T, M extends AnyIterable<T>>(size: number, iterable: M): UnwrapAnyIterable<M>;
export type UnwrapToPromiseOrAsyncIterable<M extends AnyIterable<any>> = M extends Iterable<infer T> ? T[] : M extends AsyncIterable<infer B> ? Promise<B[]> : never;
/**
* Collect all the values from an iterable into an array. Returns an array if you pass it an iterable and a promise for an array if you pass it an async iterable. Errors from the source `iterable` are raised immediately.
export declare function batchWithTimeout<T, M extends AnyIterable<T>>(size: number, timeout: number, iterable: M): UnwrapAnyIterableArray<M>;
```ts
import { collect } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
/**
* Buffer keeps a number of objects in reserve available for immediate reading. This is helpful with async iterators as it will pre-fetch results so you don't have to wait for them to load. For sync iterables it will pre-compute up to `size` values and keep them in reserve. The internal buffer will start to be filled once `.next()` is called for the first time and will continue to fill until the source `iterable` is exhausted or the buffer is full. Errors from the source `iterable` will be raised after all buffered values are yielded.
console.log(await collect(getPokemon()))
// [bulbasaur, ivysaur, venusaur, charmander, ...]
```
*/
export function collect<T, M extends AnyIterable<T>>(iterable: M): UnwrapToPromiseOrAsyncIterable<M>;
/**
* Combine multiple iterators into a single iterable. Reads each iterable completely one at a time. Returns a sync iterator if all `iterables` are sync, otherwise it returns an async iterable. Errors from the source `iterable` are raised immediately.
`size` can be between 0 and `Infinity`.
```ts
import { concat } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
import { getTransformers } from './util'
```ts
import { buffer } from 'streaming-iterables'
import { getPokemon, trainMonster } from 'iterable-pokedex'
for await (const hero of concat(getPokemon(2), getTransformers(2))) {
console.log(hero)
}
// charmander
// bulbasaur <- end of pokemon
// megatron
// bumblebee <- end of transformers
```
*/
export function concat<I extends Iterable<any>[]>(...iterables: I): Iterable<UnArrayAnyIterable<I>>;
export function concat<I extends AnyIterable<any>[]>(...iterables: I): AsyncIterable<UnArrayAnyIterable<I>>;
/**
* A promise that resolves after the function drains the iterable of all data. Useful for processing a pipeline of data. Errors from the source `iterable` are raised immediately.
// load 10 monsters in the background while we process them one by one
for await (const monster of buffer(10, getPokemon())) {
await trainMonster(monster) // got to do some pokéwork
}
```
*/
export declare function buffer(size: number): CurriedBufferResult;
```ts
import { consume, map } from 'streaming-iterables'
import { getPokemon, trainMonster } from 'iterable-pokedex'
export declare function buffer<T, M extends AnyIterable<T>>(size: number, iterable: M): UnwrapAnyIterable<M>;
const train = map(trainMonster)
await consume(train(getPokemon())) // load all the pokemon and train them!
```
*/
export function consume<T>(iterable: Iterable<T>): void;
export function consume<T>(iterable: AsyncIterable<T>): Promise<void>;
export type CurriedDropResult = <T, M extends AnyIterable<T>>(curriedIterable: M) => UnwrapAnyIterable<M>;
/**
* Returns a new iterator that skips a specific number of items from `iterable`. When used with generators it advances the generator `count` items, when used with arrays it gets a new iterator and skips `count` items.
/**
* Collect all the values from an iterable into an array. Returns an array if you pass it an iterable and a promise for an array if you pass it an async iterable. Errors from the source `iterable` are raised immediately.
```ts
import { pipeline, drop, collect } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
```ts
import { collect } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
const allButFirstFive = await collect(drop(5, getPokemon()))
// first five pokemon
```
*/
export function drop(count: number): CurriedDropResult;
export function drop<T, M extends AnyIterable<T>>(count: number, iterable: M): UnwrapAnyIterable<M>;
/**
* Takes a `filterFunc` and a `iterable`, and returns a new async iterator of the same type containing the members of the given iterable which cause the `filterFunc` to return true.
console.log(await collect(getPokemon()))
// [bulbasaur, ivysaur, venusaur, charmander, ...]
```
*/
export declare function collect<T, M extends AnyIterable<T>>(iterable: M): UnwrapToPromiseOrAsyncIterable<M>;
```ts
import { filter } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
/**
* Combine multiple iterators into a single iterable. Reads each iterable completely one at a time. Returns a sync iterator if all `iterables` are sync, otherwise it returns an async iterable. Errors from the source `iterable` are raised immediately.
const filterWater = filter(pokemon => pokemon.types.include('Water'))
```ts
import { concat } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
import { getTransformers } from './util'
for await (const pokemon of filterWater(getPokemon())) {
console.log(pokemon)
}
// squirtle
// vaporeon
// magikarp
```
*/
export function filter<T, S extends T>(filterFunc: (data: T) => data is S): <A extends T>(curriedIterable: AnyIterable<A>) => AsyncGenerator<S>;
export function filter<T>(filterFunc: (data: T) => boolean | Promise<boolean>): <A>(curriedIterable: AnyIterable<A>) => AsyncGenerator<A>;
export function filter<T, S extends T>(filterFunc: (data: T) => data is S, iterable: AnyIterable<T>): AsyncGenerator<S>;
export function filter<T>(filterFunc: (data: T) => boolean | Promise<boolean>, iterable: AnyIterable<T>): AsyncGenerator<T>;
/**
* Map `func` over the `iterable`, flatten the result and then ignore all null or undefined values. It's the transform function we've always needed. It's equivalent to;
for await (const hero of concat(getPokemon(2), getTransformers(2))) {
console.log(hero)
}
// charmander
// bulbasaur <- end of pokemon
// megatron
// bumblebee <- end of transformers
```
*/
export declare function concat<I extends Iterable<any>[]>(...iterables: I): Iterable<UnArrayAnyIterable<I>>;
```ts
(func, iterable) => filter(i => i !== undefined && i !== null, flatten(map(func, iterable)))
```
export declare function concat<I extends AnyIterable<any>[]>(...iterables: I): AsyncIterable<UnArrayAnyIterable<I>>;
*note*: The return value for `func` is `FlatMapValue<B>`. Typescript doesn't have recursive types but you can nest iterables as deep as you like.
/**
* A promise that resolves after the function drains the iterable of all data. Useful for processing a pipeline of data. Errors from the source `iterable` are raised immediately.
The ordering of the results is guaranteed. Errors from the source `iterable` are raised after all mapped values are yielded. Errors from `func` are raised after all previously mapped values are yielded.
```ts
import { consume, map } from 'streaming-iterables'
import { getPokemon, trainMonster } from 'iterable-pokedex'
```ts
import { flatMap } from 'streaming-iterables'
import { getPokemon, lookupStats } from 'iterable-pokedex'
const train = map(trainMonster)
await consume(train(getPokemon())) // load all the pokemon and train them!
```
*/
export declare function consume<T>(iterable: Iterable<T>): void;
async function getDefeatedGyms(pokemon) {
if (pokemon.gymBattlesWon > 0) {
const stats = await lookupStats(pokemon)
return stats.gyms
}
}
export declare function consume<T>(iterable: AsyncIterable<T>): Promise<void>;
for await (const gym of flatMap(getDefeatedGyms, getPokemon())) {
console.log(gym.name)
}
// "Pewter Gym"
// "Cerulean Gym"
// "Vermilion Gym"
```
*/
export function flatMap<T, B>(func: (data: T) => FlatMapValue<B>): (iterable: AnyIterable<T>) => AsyncGenerator<NonNullable<B>>;
export function flatMap<T, B>(func: (data: T) => FlatMapValue<B>, iterable: AnyIterable<T>): AsyncGenerator<NonNullable<B>>;
/**
* Returns a new iterator by pulling every item out of `iterable` (and all its sub iterables) and yielding them depth-first. Checks for the iterable interfaces and iterates it if it exists. If the value is a string it is not iterated as that ends up in an infinite loop. Errors from the source `iterable` are raised immediately.
export declare type CurriedBatchResult = <T, M extends AnyIterable<T>>(curriedIterable: M) => UnwrapAnyIterableArray<M>;
*note*: Typescript doesn't have recursive types but you can nest iterables as deep as you like.
export declare type CurriedBatchWithTimeoutResult = <T, M extends AnyIterable<T>>(curriedIterable: M) => UnwrapAnyIterableArray<M>;
```ts
import { flatten } from 'streaming-iterables'
export declare type CurriedBufferResult = <T, M extends AnyIterable<T>>(curriedIterable: M) => UnwrapAnyIterable<M>;
for await (const item of flatten([1, 2, [3, [4, 5], 6])) {
console.log(item)
}
// 1
// 2
// 3
// 4
// 5
// 6
```
*/
export function flatten<B>(iterable: AnyIterable<B | AnyIterable<B>>): AsyncIterableIterator<B>;
/**
* Map `func` over the `iterable`, flatten the result and then ignore all null or undefined values. Returned async iterables are flattened concurrently too. It's the transform function we've always wanted.
export declare type CurriedDropResult = <T, M extends AnyIterable<T>>(curriedIterable: M) => UnwrapAnyIterable<M>;
It's similar to;
export declare type CurriedTakeLastResult = <T, M extends AnyIterable<T>>(curriedIterable: M) => UnwrapAnyIterable<M>;
```ts
const filterEmpty = filter(i => i !== undefined && i !== null)
(concurrency, func, iterable) => filterEmpty(flatten(transform(concurrency, func, iterable)))
```
export declare type CurriedTakeResult = <T, M extends AnyIterable<T>>(curriedIterable: M) => UnwrapAnyIterable<M>;
*note*: The return value for `func` is `FlatMapValue<B>`. Typescript doesn't have recursive types but you can nest iterables as deep as you like. However only directly returned async iterables are processed concurrently. (Eg, if you use an async generator function as `func` it's output will be processed concurrently, but if it's nested inside other iterables it will be processed sequentially.)
export declare type CurriedTimeResult = <T, M extends AnyIterable<T>>(curriedIterable: M) => UnwrapAnyIterable<M>;
Order is determined by when async operations resolve. And it will run up to `concurrency` async operations at once. This includes promises and async iterables returned from `func`. Errors from the source `iterable` are raised after all transformed values are yielded. Errors from `func` are raised after all previously transformed values are yielded.
/**
* Returns a new iterator that skips a specific number of items from `iterable`. When used with generators it advances the generator `count` items, when used with arrays it gets a new iterator and skips `count` items.
`concurrency` can be between 1 and `Infinity`.
```ts
import { pipeline, drop, collect } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
Promise Example;
const allButFirstFive = await collect(drop(5, getPokemon()))
// first five pokemon
```
*/
export declare function drop(count: number): CurriedDropResult;
```ts
import { flatTransform } from 'streaming-iterables'
import { getPokemon, lookupStats } from 'iterable-pokedex'
export declare function drop<T, M extends AnyIterable<T>>(count: number, iterable: M): UnwrapAnyIterable<M>;
async function getDefeatedGyms(pokemon) {
if (pokemon.gymBattlesWon > 0) {
const stats = await lookupStats(pokemon)
return stats.gyms
}
}
/**
* Takes a `filterFunc` and a `iterable`, and returns a new async iterator of the same type containing the members of the given iterable which cause the `filterFunc` to return true.
// lookup 10 stats at a time
for await (const gym of flatTransform(10, getDefeatedGyms, getPokemon())) {
console.log(gym.name)
}
// "Pewter Gym"
// "Cerulean Gym"
// "Vermilion Gym"
```
```ts
import { filter } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
Async Generator Example
const filterWater = filter(pokemon => pokemon.types.include('Water'))
```ts
import { flatTransform } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
import { findFriendsFB, findFriendsMySpace } from './util'
for await (const pokemon of filterWater(getPokemon())) {
console.log(pokemon)
}
// squirtle
// vaporeon
// magikarp
```
*/
export declare function filter<T, S extends T>(filterFunc: (data: T) => data is S): <A extends T>(curriedIterable: AnyIterable<A>) => AsyncGenerator<S>;
export declare function filter<T>(filterFunc: (data: T) => boolean | Promise<boolean>): <A>(curriedIterable: AnyIterable<A>) => AsyncGenerator<A>;
async function* findFriends (pokemon) {
yield await findFriendsFB(pokemon.name)
yield await findFriendsMySpace(pokemon.name)
}
export declare function filter<T, S extends T>(filterFunc: (data: T) => data is S, iterable: AnyIterable<T>): AsyncGenerator<S>;
for await (const pokemon of flatTransform(10, findFriends, getPokemon())) {
console.log(pokemon.name)
}
// Pikachu
// Meowth
// Ash - FB
// Jessie - FB
// Misty - MySpace
// James - MySpace
```
*/
export function flatTransform(concurrency: number): {
<T, R>(func: (data: T) => FlatMapValue<R>, iterable: AnyIterable<T>): AsyncGenerator<R>;
<T, R>(func: (data: T) => FlatMapValue<R>): (iterable: AnyIterable<T>) => AsyncGenerator<R>;
};
export function flatTransform<T, R>(concurrency: number, func: (data: T) => FlatMapValue<R>): (iterable: AnyIterable<T>) => AsyncGenerator<R>;
export function flatTransform<T, R>(concurrency: number, func: (data: T) => FlatMapValue<R>, iterable: AnyIterable<T>): AsyncGenerator<R>;
export interface ReadableStreamish {
once: any;
read: any;
}
/**
* Wraps the stream in an async iterator or returns the stream if it already is an async iterator.
export declare function filter<T>(filterFunc: (data: T) => boolean | Promise<boolean>, iterable: AnyIterable<T>): AsyncGenerator<T>;
*note*: Since Node 10, streams already async iterators. This function may be used to ensure compatibility with older versions of Node.
/**
* Map `func` over the `iterable`, flatten the result and then ignore all null or undefined values. It's the transform function we've always needed. It's equivalent to;
```ts
import { fromStream } from 'streaming-iterables'
import { createReadStream } from 'fs'
```ts
(func, iterable) => filter(i => i !== undefined && i !== null, flatten(map(func, iterable)))
```
const pokeLog = fromStream(createReadStream('./pokedex-operating-system.log'))
*note*: The return value for `func` is `FlatMapValue<B>`. Typescript doesn't have recursive types but you can nest iterables as deep as you like.
for await (const pokeData of pokeLog) {
console.log(pokeData) // Buffer(...)
}
```
* @deprecated This method is deprecated since, node 10 is out of LTS. It may be removed in an upcoming major release.
*/
export function fromStream<T>(stream: ReadableStreamish): AsyncIterable<T>;
/**
* Get the iterator from any iterable or just return an iterator itself.
*/
export function getIterator<T>(iterable: Iterable<T> | Iterator<T>): Iterator<T>;
export function getIterator<T>(iterable: AsyncIterable<T> | AsyncIterator<T>): AsyncIterator<T>;
export function getIterator<T>(iterable: AsyncIterable<T> | Iterable<T>): AsyncIterator<T> | Iterator<T>;
/**
* Any iterable or iterator.
*/
export type Iterableish<T> = Iterable<T> | Iterator<T> | AsyncIterable<T> | AsyncIterator<T>;
/**
* Literally any `Iterable` (async or regular).
*/
export type AnyIterable<T> = Iterable<T> | AsyncIterable<T>;
/**
* A value, an array of that value, undefined, null or promises for any of them. Used in the `flatMap` and `flatTransform` functions as possible return values of the mapping function.
*/
export type FlatMapValue<B> = B | AnyIterable<B> | undefined | null | Promise<B | AnyIterable<B> | undefined | null>;
export type UnArrayAnyIterable<A extends AnyIterable<any>[]> = A extends AnyIterable<infer T>[] ? T : never;
export type UnwrapAnyIterable<M extends AnyIterable<any>> = M extends Iterable<infer T> ? Iterable<T> : M extends AsyncIterable<infer B> ? AsyncIterable<B> : never;
/**
* Map a function or async function over all the values of an iterable. Errors from the source `iterable` and `func` are raised immediately.
The ordering of the results is guaranteed. Errors from the source `iterable` are raised after all mapped values are yielded. Errors from `func` are raised after all previously mapped values are yielded.
```ts
import { consume, map } from 'streaming-iterables'
import got from 'got'
```ts
import { flatMap } from 'streaming-iterables'
import { getPokemon, lookupStats } from 'iterable-pokedex'
const urls = ['https://http.cat/200', 'https://http.cat/201', 'https://http.cat/202']
const download = map(got)
async function getDefeatedGyms(pokemon) {
if (pokemon.gymBattlesWon > 0) {
const stats = await lookupStats(pokemon)
return stats.gyms
}
}
// download one at a time
for await (page of download(urls)) {
console.log(page)
}
```
*/
export function map<T, B>(func: (data: T) => B | Promise<B>): (iterable: AnyIterable<T>) => AsyncGenerator<B>;
export function map<T, B>(func: (data: T) => B | Promise<B>, iterable: AnyIterable<T>): AsyncGenerator<B>;
/**
* Combine multiple iterators into a single iterable. Reads one item off each iterable in order repeatedly until they are all exhausted. If you care less about order and want them faster see `parallelMerge()`.
*/
export function merge<I extends AnyIterable<any>[]>(...iterables: I): AsyncIterable<UnArrayAnyIterable<I>>;
export function parallelFlatMap<T, R>(concurrency: number): {
(func: (data: T) => R | Promise<R>, iterable: AnyIterable<T>): AsyncGenerator<R>;
(func: (data: T) => R | Promise<R>): (iterable: AnyIterable<T>) => AsyncGenerator<R>;
};
export function parallelFlatMap<T, R>(concurrency: number, func: (data: T) => R | Promise<R>): (iterable: AnyIterable<T>) => AsyncGenerator<R>;
export function parallelFlatMap<T, R>(concurrency: number, func: (data: T) => R | Promise<R>, iterable: AnyIterable<T>): AsyncGenerator<R>;
export function parallelMap<T, R>(concurrency: number): {
(func: (data: T) => R | Promise<R>, iterable: AnyIterable<T>): AsyncIterableIterator<R>;
(func: (data: T) => R | Promise<R>): (iterable: AnyIterable<T>) => AsyncIterableIterator<R>;
};
export function parallelMap<T, R>(concurrency: number, func: (data: T) => R | Promise<R>): (iterable: AnyIterable<T>) => AsyncIterableIterator<R>;
export function parallelMap<T, R>(concurrency: number, func: (data: T) => R | Promise<R>, iterable: AnyIterable<T>): AsyncIterableIterator<R>;
/**
*Combine multiple iterators into a single iterable. Reads one item off of every iterable and yields them as they resolve. This is useful for pulling items out of a collection of iterables as soon as they're available. Errors `iterables` are raised immediately.
for await (const gym of flatMap(getDefeatedGyms, getPokemon())) {
console.log(gym.name)
}
// "Pewter Gym"
// "Cerulean Gym"
// "Vermilion Gym"
```
*/
export declare function flatMap<T, B>(func: (data: T) => FlatMapValue<B>): (iterable: AnyIterable<T>) => AsyncGenerator<NonNullable<B>>;
```ts
import { parallelMerge } from 'streaming-iterables'
import { getPokemon, getTransformer } from 'iterable-pokedex'
export declare function flatMap<T, B>(func: (data: T) => FlatMapValue<B>, iterable: AnyIterable<T>): AsyncGenerator<NonNullable<B>>;
// pokemon are much faster to load btw
const heros = parallelMerge(getPokemon(), getTransformer())
for await (const hero of heros) {
console.log(hero)
}
// charmander
// bulbasaur
// megatron
// pikachu
// eevee
// bumblebee
// jazz
```
*/
export function parallelMerge<I extends AnyIterable<any>[]>(...iterables: I): AsyncIterable<UnArrayAnyIterable<I>>;
/**
* Calls `firstFn` and then every function in `fns` with the result of the previous function. The final return is the result of the last function in `fns`.
/**
* A value, an array of that value, undefined, null or promises for any of them. Used in the `flatMap` and `flatTransform` functions as possible return values of the mapping function.
*/
export declare type FlatMapValue<B> = B | AnyIterable<B> | undefined | null | Promise<B | AnyIterable<B> | undefined | null>;
```ts
import { pipeline, map, collect } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
const getName = map(pokemon => pokemon.name)
/**
* Returns a new iterator by pulling every item out of `iterable` (and all its sub iterables) and yielding them depth-first. Checks for the iterable interfaces and iterates it if it exists. If the value is a string it is not iterated as that ends up in an infinite loop. Errors from the source `iterable` are raised immediately.
// equivalent to `await collect(getName(getPokemon()))`
await pipeline(getPokemon, getName, collect)
// charmander
// bulbasaur
// MissingNo.
```
*/
export function pipeline<T0>(firstFn: () => T0): T0;
export function pipeline<T0, T1>(a0: () => T0, a1: (a: T0) => T1): T1;
export function pipeline<T0, T1, T2>(a0: () => T0, a1: (a: T0) => T1, a2: (a: T1) => T2): T2;
export function pipeline<T0, T1, T2, T3>(a0: () => T0, a1: (a: T0) => T1, a2: (a: T1) => T2, a3: (a: T2) => T3): T3;
export function pipeline<T0, T1, T2, T3, T4>(a0: () => T0, a1: (a: T0) => T1, a2: (a: T1) => T2, a3: (a: T2) => T3, a4: (a: T3) => T4): T4;
export function pipeline<T0, T1, T2, T3, T4, T5>(a0: () => T0, a1: (a: T0) => T1, a2: (a: T1) => T2, a3: (a: T2) => T3, a4: (a: T3) => T4, a5: (a: T4) => T5): T5;
export function pipeline<T0, T1, T2, T3, T4, T5, T6>(a0: () => T0, a1: (a: T0) => T1, a2: (a: T1) => T2, a3: (a: T2) => T3, a4: (a: T3) => T4, a5: (a: T4) => T5, a6: (a: T5) => T6): T6;
export function pipeline<T0, T1, T2, T3, T4, T5, T6, T7>(a0: () => T0, a1: (a: T0) => T1, a2: (a: T1) => T2, a3: (a: T2) => T3, a4: (a: T3) => T4, a5: (a: T4) => T5, a6: (a: T5) => T6, a7: (a: T6) => T7): T7;
export function pipeline<T0, T1, T2, T3, T4, T5, T6, T7, T8>(a0: () => T0, a1: (a: T0) => T1, a2: (a: T1) => T2, a3: (a: T2) => T3, a4: (a: T3) => T4, a5: (a: T4) => T5, a6: (a: T5) => T6, a7: (a: T6) => T7, a8: (a: T7) => T8): T8;
export function pipeline<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9>(a0: () => T0, a1: (a: T0) => T1, a2: (a: T1) => T2, a3: (a: T2) => T3, a4: (a: T3) => T4, a5: (a: T4) => T5, a6: (a: T5) => T6, a7: (a: T6) => T7, a8: (a: T7) => T8, a9: (a: T8) => T9): T9;
/**
* An async function that takes a reducer function, an initial value and an iterable.
*note*: Typescript doesn't have recursive types but you can nest iterables as deep as you like.
Reduces an iterable to a value which is the accumulated result of running each value from the iterable thru `func`, where each successive invocation is supplied the return value of the previous. Errors are immediate raised.
*/
export function reduce<T, B>(func: (acc: B, value: T) => B): {
(start?: B): (iterable: AnyIterable<T>) => Promise<B>;
(start: B | undefined, iterable: AnyIterable<T>): Promise<B>;
};
export function reduce<T, B>(func: (acc: B, value: T) => B, start: B): (iterable: AnyIterable<T>) => Promise<B>;
export function reduce<T, B>(func: (acc: B, value: T) => B, start: B, iterable: AnyIterable<T>): Promise<B>;
export type CurriedTakeResult = <T, M extends AnyIterable<T>>(curriedIterable: M) => UnwrapAnyIterable<M>;
/**
* Returns a new iterator that reads a specific number of items from `iterable`. When used with generators it advances the generator, when used with arrays it gets a new iterator and starts from the beginning.
```ts
import { flatten } from 'streaming-iterables'
```ts
import { pipeline, take, collect } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
for await (const item of flatten([1, 2, [3, [4, 5], 6])) {
console.log(item)
}
// 1
// 2
// 3
// 4
// 5
// 6
```
*/
export declare function flatten<B>(iterable: AnyIterable<B | AnyIterable<B>>): AsyncIterableIterator<B>;
const topFive = await collect(take(5, getPokemon()))
// first five pokemon
```
*/
export function take(count: number): CurriedTakeResult;
export function take<T, M extends AnyIterable<T>>(count: number, iterable: M): UnwrapAnyIterable<M>;
export type CurriedTakeLastResult = <T, M extends AnyIterable<T>>(curriedIterable: M) => UnwrapAnyIterable<M>;
/**
* Returns a new iterator that reads a specific number of items from the end of `iterable` once it has completed. When used with generators it advances the generator, when used with arrays it gets a new iterator and starts from the beginning.
/**
* Map `func` over the `iterable`, flatten the result and then ignore all null or undefined values. Returned async iterables are flattened concurrently too. It's the transform function we've always wanted.
```ts
import { pipeline, takeLast, collect } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
It's similar to;
const bottomFive = await collect(takeLast(5, getPokemon()))
// last five pokemon
```
*/
export function takeLast(count: number): CurriedTakeLastResult;
export function takeLast<T, M extends AnyIterable<T>>(count: number, iterable: M): UnwrapAnyIterable<M>;
/**
* Takes a `predicate` and a `iterable`, and returns a new async iterator of the same type containing the members of the given iterable until the `predicate` returns false.
```ts
const filterEmpty = filter(i => i !== undefined && i !== null)
(concurrency, func, iterable) => filterEmpty(flatten(transform(concurrency, func, iterable)))
```
```ts
import { takeWhile } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
*note*: The return value for `func` is `FlatMapValue<B>`. Typescript doesn't have recursive types but you can nest iterables as deep as you like. However only directly returned async iterables are processed concurrently. (Eg, if you use an async generator function as `func` it's output will be processed concurrently, but if it's nested inside other iterables it will be processed sequentially.)
const firstSlowOnes = takeWhile(pokemon => pokemon.baseStats.speed < 100)
Order is determined by when async operations resolve. And it will run up to `concurrency` async operations at once. This includes promises and async iterables returned from `func`. Errors from the source `iterable` are raised after all transformed values are yielded. Errors from `func` are raised after all previously transformed values are yielded.
for await (const pokemon of firstSlowOnes(getPokemon())) {
console.log(pokemon)
}
// Abomasnow
// Abra
// Absol
```
*/
export function takeWhile<T, S extends T>(predicate: (data: T) => data is S): <A extends T>(curriedIterable: AnyIterable<A>) => AsyncGenerator<S>;
export function takeWhile<T>(predicate: (data: T) => boolean | Promise<boolean>): <A>(curriedIterable: AnyIterable<A>) => AsyncGenerator<A>;
export function takeWhile<T, S extends T>(predicate: (data: T) => data is S, iterable: AnyIterable<T>): AsyncGenerator<S>;
export function takeWhile<T>(predicate: (data: T) => boolean | Promise<boolean>, iterable: AnyIterable<T>): AsyncGenerator<T>;
/**
* Returns a new iterator that yields the data it consumes, passing the data through to a function. If you provide an async function, the iterator will wait for the promise to resolve before yielding the value. This is useful for logging, or processing information and passing it along.
*/
export function tap<T>(func: (data: T) => any): (iterable: AnyIterable<T>) => AsyncGenerator<T>;
export function tap<T>(func: (data: T) => any, iterable: AnyIterable<T>): AsyncGenerator<T>;
/**
* Throttles `iterable` at a rate of `limit` per `interval` without discarding data. Useful for throttling rate limited APIs.
`concurrency` can be between 1 and `Infinity`.
`limit` can be greater than 0 but less than `Infinity`.
`interval` can be greater than or equal to 0 but less than `Infinity`.
Promise Example;
```ts
import { throttle } from 'streaming-iterables'
import { getPokemon, trainMonster } from 'iterable-pokedex'
```ts
import { flatTransform } from 'streaming-iterables'
import { getPokemon, lookupStats } from 'iterable-pokedex'
// load monsters at a maximum rate of 1 per second
for await (const monster of throttle(1, 1000, getPokemon())) {
await trainMonster(monster)
}
```
*/
export function throttle<T>(limit: number, interval: number): (iterable: AnyIterable<T>) => AsyncGenerator<T>;
export function throttle<T>(limit: number, interval: number, iterable: AnyIterable<T>): AsyncGenerator<T>;
export interface TimeConfig {
progress?: (delta: [number, number], total: [number, number]) => any;
total?: (time: [number, number]) => any;
}
export type CurriedTimeResult = <T, M extends AnyIterable<T>>(curriedIterable: M) => UnwrapAnyIterable<M>;
/**
* Returns a new iterator that yields the data it consumes and calls the `progress` and `total` callbacks with the [`hrtime`](https://nodejs.org/api/process.html#process_process_hrtime_time) it took for `iterable` to provide a value when `.next()` was called on it. That is to say, the time returned is the time this iterator spent waiting for data, not the time it took to finish being read. The `hrtime` tuple looks like `[seconds, nanoseconds]`.
async function getDefeatedGyms(pokemon) {
if (pokemon.gymBattlesWon > 0) {
const stats = await lookupStats(pokemon)
return stats.gyms
}
}
```ts
import { consume, transform, time } from 'streaming-iterables'
import got from 'got'
// lookup 10 stats at a time
for await (const gym of flatTransform(10, getDefeatedGyms, getPokemon())) {
console.log(gym.name)
}
// "Pewter Gym"
// "Cerulean Gym"
// "Vermilion Gym"
```
const urls = ['https://http.cat/200', 'https://http.cat/201', 'https://http.cat/202']
const download = transform(1000, got)
const timer = time({
total: total => console.log(`Spent ${total[0]} seconds and ${total[1]}ns downloading cats`),
})
// download all of these at the same time
for await (page of timer(download(urls))) {
console.log(page)
}
```
*/
export function time(config?: TimeConfig): CurriedTimeResult;
export function time<T, M extends AnyIterable<T>>(config: TimeConfig, iterable: M): UnwrapAnyIterable<M>;
/**
* Map a function or async function over all the values of an iterable. Order is determined by when `func` resolves. And it will run up to `concurrency` async `func` operations at once. If you care about order see [`parallelMap()`](#parallelmap). Errors from the source `iterable` are raised after all transformed values are yielded. Errors from `func` are raised after all previously transformed values are yielded.
Async Generator Example
`concurrency` can be between 1 and `Infinity`.
```ts
import { flatTransform } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
import { findFriendsFB, findFriendsMySpace } from './util'
```ts
import { consume, transform } from 'streaming-iterables'
import got from 'got'
const urls = ['https://http.cat/200', 'https://http.cat/201', 'https://http.cat/202']
const download = transform(1000, got)
async function* findFriends (pokemon) {
yield await findFriendsFB(pokemon.name)
yield await findFriendsMySpace(pokemon.name)
}
// download all of these at the same time
for await (page of download(urls)) {
console.log(page)
}
```
*/
export function transform(concurrency: number): {
<T, R>(func: (data: T) => R | Promise<R>, iterable: AnyIterable<T>): AsyncIterableIterator<R>;
<T, R>(func: (data: T) => R | Promise<R>): (iterable: AnyIterable<T>) => AsyncIterableIterator<R>;
};
export function transform<T, R>(concurrency: number, func: (data: T) => R | Promise<R>): (iterable: AnyIterable<T>) => AsyncIterableIterator<R>;
export function transform<T, R>(concurrency: number, func: (data: T) => R | Promise<R>, iterable: AnyIterable<T>): AsyncIterableIterator<R>;
export interface WritableStreamish {
once: any;
write: any;
removeListener: any;
}
/**
* Writes the `iterable` to the stream respecting the stream back pressure. Resolves when the iterable is exhausted, rejects if the stream errors during calls to `write()` or if there are `error` events during the write.
for await (const pokemon of flatTransform(10, findFriends, getPokemon())) {
console.log(pokemon.name)
}
// Pikachu
// Meowth
// Ash - FB
// Jessie - FB
// Misty - MySpace
// James - MySpace
```
*/
export declare function flatTransform(concurrency: number): {
<T, R>(func: (data: T) => FlatMapValue<R>, iterable: AnyIterable<T>): AsyncGenerator<R>;
<T, R>(func: (data: T) => FlatMapValue<R>): (iterable: AnyIterable<T>) => AsyncGenerator<R>;
};
As it is when working with streams there are a few caveats;
export declare function flatTransform<T, R>(concurrency: number, func: (data: T) => FlatMapValue<R>): (iterable: AnyIterable<T>) => AsyncGenerator<R>;
- It is possible for the stream to error after `writeToStream()` has finished writing due to internal buffering and other concerns, so always handle errors on the stream as well.
- `writeToStream()` doesn't close the stream like `stream.pipe()` might. This is done so you can write to the stream multiple times. You can call `stream.write(null)` or any stream specific end function if you are done with the stream.
export declare function flatTransform<T, R>(concurrency: number, func: (data: T) => FlatMapValue<R>, iterable: AnyIterable<T>): AsyncGenerator<R>;
```ts
import { pipeline, map, writeToStream } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
import { createWriteStream } from 'fs'
/**
* Wraps the stream in an async iterator or returns the stream if it already is an async iterator.
*note*: Since Node 10, streams already async iterators. This function may be used to ensure compatibility with older versions of Node.
```ts
import { fromStream } from 'streaming-iterables'
import { createReadStream } from 'fs'
const pokeLog = fromStream(createReadStream('./pokedex-operating-system.log'))
for await (const pokeData of pokeLog) {
console.log(pokeData) // Buffer(...)
}
```
* @deprecated This method is deprecated since, node 10 is out of LTS. It may be removed in an upcoming major release.
*/
export declare function fromStream<T>(stream: ReadableStreamish): AsyncIterable<T>;
/**
* Get the iterator from any iterable or just return an iterator itself.
*/
export declare function getIterator<T>(iterable: Iterable<T> | Iterator<T>): Iterator<T>;
export declare function getIterator<T>(iterable: AsyncIterable<T> | AsyncIterator<T>): AsyncIterator<T>;
export declare function getIterator<T>(iterable: AsyncIterable<T> | Iterable<T>): AsyncIterator<T> | Iterator<T>;
/**
* Any iterable or iterator.
*/
export declare type Iterableish<T> = Iterable<T> | Iterator<T> | AsyncIterable<T> | AsyncIterator<T>;
/**
* Map a function or async function over all the values of an iterable. Errors from the source `iterable` and `func` are raised immediately.
```ts
import { consume, map } from 'streaming-iterables'
import got from 'got'
const urls = ['https://http.cat/200', 'https://http.cat/201', 'https://http.cat/202']
const download = map(got)
// download one at a time
for await (page of download(urls)) {
console.log(page)
}
```
*/
export declare function map<T, B>(func: (data: T) => B | Promise<B>): (iterable: AnyIterable<T>) => AsyncGenerator<B>;
export declare function map<T, B>(func: (data: T) => B | Promise<B>, iterable: AnyIterable<T>): AsyncGenerator<B>;
/**
* Combine multiple iterators into a single iterable. Reads one item off each iterable in order repeatedly until they are all exhausted. If you care less about order and want them faster see `parallelMerge()`.
*/
export declare function merge<I extends AnyIterable<any>[]>(...iterables: I): AsyncIterable<UnArrayAnyIterable<I>>;
export declare function parallelFlatMap<T, R>(concurrency: number): {
(func: (data: T) => R | Promise<R>, iterable: AnyIterable<T>): AsyncGenerator<R>;
(func: (data: T) => R | Promise<R>): (iterable: AnyIterable<T>) => AsyncGenerator<R>;
};
export declare function parallelFlatMap<T, R>(concurrency: number, func: (data: T) => R | Promise<R>): (iterable: AnyIterable<T>) => AsyncGenerator<R>;
export declare function parallelFlatMap<T, R>(concurrency: number, func: (data: T) => R | Promise<R>, iterable: AnyIterable<T>): AsyncGenerator<R>;
export declare function parallelMap<T, R>(concurrency: number): {
(func: (data: T) => R | Promise<R>, iterable: AnyIterable<T>): AsyncIterableIterator<R>;
(func: (data: T) => R | Promise<R>): (iterable: AnyIterable<T>) => AsyncIterableIterator<R>;
};
export declare function parallelMap<T, R>(concurrency: number, func: (data: T) => R | Promise<R>): (iterable: AnyIterable<T>) => AsyncIterableIterator<R>;
export declare function parallelMap<T, R>(concurrency: number, func: (data: T) => R | Promise<R>, iterable: AnyIterable<T>): AsyncIterableIterator<R>;
/**
*Combine multiple iterators into a single iterable. Reads one item off of every iterable and yields them as they resolve. This is useful for pulling items out of a collection of iterables as soon as they're available. Errors `iterables` are raised immediately.
```ts
import { parallelMerge } from 'streaming-iterables'
import { getPokemon, getTransformer } from 'iterable-pokedex'
// pokemon are much faster to load btw
const heros = parallelMerge(getPokemon(), getTransformer())
for await (const hero of heros) {
console.log(hero)
}
// charmander
// bulbasaur
// megatron
// pikachu
// eevee
// bumblebee
// jazz
```
*/
export declare function parallelMerge<I extends AnyIterable<any>[]>(...iterables: I): AsyncIterable<UnArrayAnyIterable<I>>;
/**
* Calls `firstFn` and then every function in `fns` with the result of the previous function. The final return is the result of the last function in `fns`.
```ts
import { pipeline, map, collect } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
const getName = map(pokemon => pokemon.name)
// equivalent to `await collect(getName(getPokemon()))`
await pipeline(getPokemon, getName, collect)
// charmander
// bulbasaur
// MissingNo.
```
*/
export declare function pipeline<T0>(firstFn: () => T0): T0;
export declare function pipeline<T0, T1>(a0: () => T0, a1: (a: T0) => T1): T1;
export declare function pipeline<T0, T1, T2>(a0: () => T0, a1: (a: T0) => T1, a2: (a: T1) => T2): T2;
export declare function pipeline<T0, T1, T2, T3>(a0: () => T0, a1: (a: T0) => T1, a2: (a: T1) => T2, a3: (a: T2) => T3): T3;
export declare function pipeline<T0, T1, T2, T3, T4>(a0: () => T0, a1: (a: T0) => T1, a2: (a: T1) => T2, a3: (a: T2) => T3, a4: (a: T3) => T4): T4;
export declare function pipeline<T0, T1, T2, T3, T4, T5>(a0: () => T0, a1: (a: T0) => T1, a2: (a: T1) => T2, a3: (a: T2) => T3, a4: (a: T3) => T4, a5: (a: T4) => T5): T5;
export declare function pipeline<T0, T1, T2, T3, T4, T5, T6>(a0: () => T0, a1: (a: T0) => T1, a2: (a: T1) => T2, a3: (a: T2) => T3, a4: (a: T3) => T4, a5: (a: T4) => T5, a6: (a: T5) => T6): T6;
export declare function pipeline<T0, T1, T2, T3, T4, T5, T6, T7>(a0: () => T0, a1: (a: T0) => T1, a2: (a: T1) => T2, a3: (a: T2) => T3, a4: (a: T3) => T4, a5: (a: T4) => T5, a6: (a: T5) => T6, a7: (a: T6) => T7): T7;
export declare function pipeline<T0, T1, T2, T3, T4, T5, T6, T7, T8>(a0: () => T0, a1: (a: T0) => T1, a2: (a: T1) => T2, a3: (a: T2) => T3, a4: (a: T3) => T4, a5: (a: T4) => T5, a6: (a: T5) => T6, a7: (a: T6) => T7, a8: (a: T7) => T8): T8;
export declare function pipeline<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9>(a0: () => T0, a1: (a: T0) => T1, a2: (a: T1) => T2, a3: (a: T2) => T3, a4: (a: T3) => T4, a5: (a: T4) => T5, a6: (a: T5) => T6, a7: (a: T6) => T7, a8: (a: T7) => T8, a9: (a: T8) => T9): T9;
export declare interface ReadableStreamish {
once: any;
read: any;
const file = createWriteStream('pokemon.ndjson')
const serialize = map(pokemon => `${JSON.stringify(pokemon)}\n`)
await pipeline(getPokemon, serialize, writeToStream(file))
file.end() // close the stream
// now all the pokemon are written to the file!
```
*/
export function writeToStream(stream: WritableStreamish): (iterable: AnyIterable<any>) => Promise<void>;
export function writeToStream(stream: WritableStreamish, iterable: AnyIterable<any>): Promise<void>;
}
/**
* An async function that takes a reducer function, an initial value and an iterable.
Reduces an iterable to a value which is the accumulated result of running each value from the iterable thru `func`, where each successive invocation is supplied the return value of the previous. Errors are immediate raised.
*/
export declare function reduce<T, B>(func: (acc: B, value: T) => B): {
(start?: B): (iterable: AnyIterable<T>) => Promise<B>;
(start: B | undefined, iterable: AnyIterable<T>): Promise<B>;
};
export declare function reduce<T, B>(func: (acc: B, value: T) => B, start: B): (iterable: AnyIterable<T>) => Promise<B>;
export declare function reduce<T, B>(func: (acc: B, value: T) => B, start: B, iterable: AnyIterable<T>): Promise<B>;
/**
* Returns a new iterator that reads a specific number of items from `iterable`. When used with generators it advances the generator, when used with arrays it gets a new iterator and starts from the beginning.
```ts
import { pipeline, take, collect } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
const topFive = await collect(take(5, getPokemon()))
// first five pokemon
```
*/
export declare function take(count: number): CurriedTakeResult;
export declare function take<T, M extends AnyIterable<T>>(count: number, iterable: M): UnwrapAnyIterable<M>;
/**
* Returns a new iterator that reads a specific number of items from the end of `iterable` once it has completed. When used with generators it advances the generator, when used with arrays it gets a new iterator and starts from the beginning.
```ts
import { pipeline, takeLast, collect } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
const bottomFive = await collect(takeLast(5, getPokemon()))
// last five pokemon
```
*/
export declare function takeLast(count: number): CurriedTakeLastResult;
export declare function takeLast<T, M extends AnyIterable<T>>(count: number, iterable: M): UnwrapAnyIterable<M>;
/**
* Takes a `predicate` and a `iterable`, and returns a new async iterator of the same type containing the members of the given iterable until the `predicate` returns false.
```ts
import { takeWhile } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
const firstSlowOnes = takeWhile(pokemon => pokemon.baseStats.speed < 100)
for await (const pokemon of firstSlowOnes(getPokemon())) {
console.log(pokemon)
}
// Abomasnow
// Abra
// Absol
```
*/
export declare function takeWhile<T, S extends T>(predicate: (data: T) => data is S): <A extends T>(curriedIterable: AnyIterable<A>) => AsyncGenerator<S>;
export declare function takeWhile<T>(predicate: (data: T) => boolean | Promise<boolean>): <A>(curriedIterable: AnyIterable<A>) => AsyncGenerator<A>;
export declare function takeWhile<T, S extends T>(predicate: (data: T) => data is S, iterable: AnyIterable<T>): AsyncGenerator<S>;
export declare function takeWhile<T>(predicate: (data: T) => boolean | Promise<boolean>, iterable: AnyIterable<T>): AsyncGenerator<T>;
/**
* Returns a new iterator that yields the data it consumes, passing the data through to a function. If you provide an async function, the iterator will wait for the promise to resolve before yielding the value. This is useful for logging, or processing information and passing it along.
*/
export declare function tap<T>(func: (data: T) => any): (iterable: AnyIterable<T>) => AsyncGenerator<T>;
export declare function tap<T>(func: (data: T) => any, iterable: AnyIterable<T>): AsyncGenerator<T>;
/**
* Throttles `iterable` at a rate of `limit` per `interval` without discarding data. Useful for throttling rate limited APIs.
`limit` can be greater than 0 but less than `Infinity`.
`interval` can be greater than or equal to 0 but less than `Infinity`.
```ts
import { throttle } from 'streaming-iterables'
import { getPokemon, trainMonster } from 'iterable-pokedex'
// load monsters at a maximum rate of 1 per second
for await (const monster of throttle(1, 1000, getPokemon())) {
await trainMonster(monster)
}
```
*/
export declare function throttle<T>(limit: number, interval: number): (iterable: AnyIterable<T>) => AsyncGenerator<T>;
export declare function throttle<T>(limit: number, interval: number, iterable: AnyIterable<T>): AsyncGenerator<T>;
/**
* Returns a new iterator that yields the data it consumes and calls the `progress` and `total` callbacks with the [`hrtime`](https://nodejs.org/api/process.html#process_process_hrtime_time) it took for `iterable` to provide a value when `.next()` was called on it. That is to say, the time returned is the time this iterator spent waiting for data, not the time it took to finish being read. The `hrtime` tuple looks like `[seconds, nanoseconds]`.
```ts
import { consume, transform, time } from 'streaming-iterables'
import got from 'got'
const urls = ['https://http.cat/200', 'https://http.cat/201', 'https://http.cat/202']
const download = transform(1000, got)
const timer = time({
total: total => console.log(`Spent ${total[0]} seconds and ${total[1]}ns downloading cats`),
})
// download all of these at the same time
for await (page of timer(download(urls))) {
console.log(page)
}
```
*/
export declare function time(config?: TimeConfig): CurriedTimeResult;
export declare function time<T, M extends AnyIterable<T>>(config: TimeConfig, iterable: M): UnwrapAnyIterable<M>;
export declare interface TimeConfig {
progress?: (delta: [number, number], total: [number, number]) => any;
total?: (time: [number, number]) => any;
}
/**
* Map a function or async function over all the values of an iterable. Order is determined by when `func` resolves. And it will run up to `concurrency` async `func` operations at once. If you care about order see [`parallelMap()`](#parallelmap). Errors from the source `iterable` are raised after all transformed values are yielded. Errors from `func` are raised after all previously transformed values are yielded.
`concurrency` can be between 1 and `Infinity`.
```ts
import { consume, transform } from 'streaming-iterables'
import got from 'got'
const urls = ['https://http.cat/200', 'https://http.cat/201', 'https://http.cat/202']
const download = transform(1000, got)
// download all of these at the same time
for await (page of download(urls)) {
console.log(page)
}
```
*/
export declare function transform(concurrency: number): {
<T, R>(func: (data: T) => R | Promise<R>, iterable: AnyIterable<T>): AsyncIterableIterator<R>;
<T, R>(func: (data: T) => R | Promise<R>): (iterable: AnyIterable<T>) => AsyncIterableIterator<R>;
};
export declare function transform<T, R>(concurrency: number, func: (data: T) => R | Promise<R>): (iterable: AnyIterable<T>) => AsyncIterableIterator<R>;
export declare function transform<T, R>(concurrency: number, func: (data: T) => R | Promise<R>, iterable: AnyIterable<T>): AsyncIterableIterator<R>;
export declare type UnArrayAnyIterable<A extends AnyIterable<any>[]> = A extends AnyIterable<infer T>[] ? T : never;
export declare type UnwrapAnyIterable<M extends AnyIterable<any>> = M extends Iterable<infer T> ? Iterable<T> : M extends AsyncIterable<infer B> ? AsyncIterable<B> : never;
export declare type UnwrapAnyIterableArray<M extends AnyIterable<any>> = M extends Iterable<infer T> ? Generator<T[]> : M extends AsyncIterable<infer B> ? AsyncGenerator<B[]> : never;
export declare type UnwrapToPromiseOrAsyncIterable<M extends AnyIterable<any>> = M extends Iterable<infer T> ? T[] : M extends AsyncIterable<infer B> ? Promise<B[]> : never;
export declare interface WritableStreamish {
once: any;
write: any;
removeListener: any;
}
/**
* Writes the `iterable` to the stream respecting the stream back pressure. Resolves when the iterable is exhausted, rejects if the stream errors during calls to `write()` or if there are `error` events during the write.
As it is when working with streams there are a few caveats;
- It is possible for the stream to error after `writeToStream()` has finished writing due to internal buffering and other concerns, so always handle errors on the stream as well.
- `writeToStream()` doesn't close the stream like `stream.pipe()` might. This is done so you can write to the stream multiple times. You can call `stream.write(null)` or any stream specific end function if you are done with the stream.
```ts
import { pipeline, map, writeToStream } from 'streaming-iterables'
import { getPokemon } from 'iterable-pokedex'
import { createWriteStream } from 'fs'
const file = createWriteStream('pokemon.ndjson')
const serialize = map(pokemon => `${JSON.stringify(pokemon)}\n`)
await pipeline(getPokemon, serialize, writeToStream(file))
file.end() // close the stream
// now all the pokemon are written to the file!
```
*/
export declare function writeToStream(stream: WritableStreamish): (iterable: AnyIterable<any>) => Promise<void>;
export declare function writeToStream(stream: WritableStreamish, iterable: AnyIterable<any>): Promise<void>;
export { }
//# sourceMappingURL=index.d.ts.map

2

dist/index.js

@@ -1098,4 +1098,2 @@ (function (global, factory) {

Object.defineProperty(exports, '__esModule', { value: true });
}));

50

package.json
{
"name": "streaming-iterables",
"version": "7.1.0",
"version": "8.0.0",
"description": "A collection of utilities for async iterables. Designed to replace your streams.",

@@ -17,7 +17,7 @@ "main": "./dist/index.js",

"test": "npm run unit-test && npm run lint",
"unit-test": "c8 -r html -r text mocha",
"unit-test": "c8 -r html -r text node -r ts-node/register --test lib/*-test.ts",
"check-coverage": "c8 check-coverage --lines 95 --functions 95 --branches 94",
"lint": "tsc && eslint lib/*.ts",
"format": "eslint lib/*.ts --fix",
"build": "tsc -p tsconfig-build.json && rollup -c && api-extractor run --local --verbose",
"build": "rollup -c && node bundle-types.mjs",
"prepare": "npm run build"

@@ -44,35 +44,27 @@ },

"devDependencies": {
"@microsoft/api-extractor": "7.29.5",
"@types/chai": "4.3.3",
"@types/mocha": "9.1.1",
"@types/node": "17.0.30",
"@types/sinon": "10.0.13",
"@typescript-eslint/eslint-plugin": "5.34.0",
"@typescript-eslint/parser": "5.34.0",
"@rollup/plugin-typescript": "^11.1.2",
"@types/chai": "4.3.5",
"@types/node": "20.4.9",
"@types/sinon": "10.0.16",
"@typescript-eslint/eslint-plugin": "6.3.0",
"@typescript-eslint/parser": "6.3.0",
"benchmark": "2.1.4",
"bluestream": "10.3.3",
"c8": "7.12.0",
"chai": "4.3.6",
"eslint": "8.22.0",
"c8": "8.0.1",
"chai": "4.3.7",
"dts-buddy": "^0.1.9",
"eslint": "8.46.0",
"eslint-config-airbnb-base": "15.0.0",
"eslint-plugin-import": "2.26.0",
"mocha": "10.0.0",
"prettier": "2.7.1",
"rollup": "2.78.1",
"sinon": "14.0.0",
"eslint-plugin-import": "2.28.0",
"prettier": "3.0.1",
"rollup": "3.28.0",
"sinon": "15.2.0",
"through2-concurrent": "2.0.0",
"ts-node": "10.9.1",
"tslib": "2.4.0",
"typescript": "4.7.4"
"ts-node": "^10.9.1",
"tslib": "^2.6.1",
"typescript": "5.1.6"
},
"engines": {
"node": ">=14"
},
"mocha": {
"bail": true,
"require": [
"ts-node/register"
],
"spec": "lib/*-test.ts"
"node": ">=18"
}
}

4

README.md

@@ -691,1 +691,5 @@ # streaming-iterables 🏄‍♂️

Writing docs and code is a lot of work! Thank you in advance for helping out.
### Publishing
Published with [`np`](https://www.npmjs.com/package/np)
SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc