steampipes - npm Package Compare versions

Comparing version 3.2.0 to 3.3.0

lib/pull-remit.js

@@ -0,1 +1,2 @@
+// Generated by CoffeeScript 2.4.1
 (function() {
@@ -378,13 +379,10 @@ 'use strict';
   //-----------------------------------------------------------------------------------------------------------
-  this.pull = async function(...transforms) {
+  this.pull = function(...transforms) {
     var d, drain, duct, first_bucket, on_end, ref;
     duct = this._pull(...transforms);
-    //.........................................................................................................
-    /* TAINT `await` makes this an async method; is that a problem? */
-    if (isa.function(duct.transforms[0].start)) {
-      duct.transforms[0].start();
-    } else if (isa.asyncfunction(duct.transforms[0].start)) {
-      await duct.transforms[0].start();
-    }
     if (duct.type !== 'circuit') {
+      // #.........................................................................................................
+      // ### TAINT `await` makes this an async method; is that a problem? ###
+      // if isa.function           duct.transforms[ 0 ].start  then        duct.transforms[ 0 ].start()
+      // else if isa.asyncfunction duct.transforms[ 0 ].start  then  await duct.transforms[ 0 ].start()
       //.........................................................................................................
@@ -489,1 +487,3 @@ return duct;
 }).call(this);
+
+//# sourceMappingURL=pull-remit.js.map

lib/tests/basic.test.js

@@ -1087,12 +1087,12 @@ // Generated by CoffeeScript 2.4.1
     // test @[ "$filter" ]
-    test(this["end push source (2)"]);
+    // test @[ "end push source (2)"             ]
+    // test @[ "remit 1"                         ]
+    // test @[ "drain with result"               ]
+    // test @[ "remit 2"                         ]
+    // test @[ "remit with end detection 1"      ]
+    // test @[ "duct_from_transforms"            ]
+    test(this["composability (through)"]);
   }
 
-  // test @[ "remit 1"                         ]
-// test @[ "drain with result"               ]
-// test @[ "remit 2"                         ]
-// test @[ "remit with end detection 1"      ]
-// test @[ "duct_from_transforms"            ]
-// test @[ "composability (through)"                 ]
-// test @[ "composability (source)"                 ]
+  // test @[ "composability (source)"                 ]
 // test @[ "composability (sink)" ]
@@ -1099,0 +1099,0 @@ // test @[ "remit with end detection 2"      ]

lib/tests/wye2.test.js

@@ -233,7 +233,9 @@ // Generated by CoffeeScript 2.4.1
   if (module.parent == null) {
-    // test @, 'timeout': 30000
-    test(this["leapfrogging compared to wye"]);
+    test(this, {
+      'timeout': 30000
+    });
   }
 
-  // test @[ "wye 3" ]
+  // test @[ "leapfrogging compared to wye" ]
+// test @[ "wye 3" ]
 
@@ -240,0 +242,0 @@ }).call(this);

package.json

 {
   "name": "steampipes",
-  "version": "3.2.0",
+  "version": "3.3.0",
   "description": "Fast, simple data pipelines",
@@ -29,6 +29,7 @@ "main": "lib/main.js",
   "dependencies": {
-    "cnd": "^4.6.0",
-    "intertype": "^2.5.0",
+    "@uppy/fs-tail-stream": "^1.2.0",
+    "cnd": "^4.9.0",
+    "intertype": "^2.8.0",
     "letsfreezethat": "^2.2.5",
-    "multimix": "^2.1.0",
+    "multimix": "^2.1.1",
     "pipedreams": "^12.1.0"
@@ -35,0 +36,0 @@ },

README.md

@@ -0,1 +1,6 @@
+
+
+# SteamPipes
+
+
 <!-- START doctoc generated TOC please keep comment here to allow auto update -->
@@ -5,9 +10,14 @@ <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
 
-- [SteamPipes](#steampipes)
-  - [Motivation](#motivation)
-  - [Notes](#notes)
-    - [Ducts](#ducts)
-      - [Duct Configurations](#duct-configurations)
-    - [Behavior for Ending Streams](#behavior-for-ending-streams)
-    - [Aborting Streams](#aborting-streams)
+- [Motivation](#motivation)
+- [How to Construct Sources, Transforms, and Sinks](#how-to-construct-sources-transforms-and-sinks)
+  - [Transforms](#transforms)
+  - [Sinks](#sinks)
+- [Asynchronous Sources and Transforms](#asynchronous-sources-and-transforms)
+- [Ducts](#ducts)
+  - [Duct Configurations](#duct-configurations)
+- [Behavior for Ending Streams](#behavior-for-ending-streams)
+- [Aborting Streams](#aborting-streams)
+- [Updates](#updates)
+- [To Do](#to-do)
+  - [To Do: Railway-Oriented Programming](#to-do-railway-oriented-programming)
 
@@ -18,5 +28,5 @@ <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
-# SteamPipes
 
-**Fast, simple data pipelines** built from first principles.
+**Fast, simple data pipelines** built from first principles. Basically, [datomic
+transducers](https://www.youtube.com/watch?v=6mTbuzafcII).
 
@@ -202,2 +212,3 @@ SteamPipes is the successor to [PipeStreams](https://github.com/loveencounterflow/pipestreams) and
 
+
 ## To Do
@@ -210,2 +221,4 @@
   SteamPipe buffers.
+* [ ] consider to adapt [Rich Hickey's terminology](https://youtu.be/6mTbuzafcII?t=878) and call transforms
+  'transducers' (it's the more pipestreamy word)
 * [ ] compare:
@@ -230,5 +243,64 @@ ```coffee
   downstream must rely on own `$drain()` to obtain data).
+* [ ] reflect once more about depth-first vs. breadth-first **doling mode**: (all sources and, so) async
+  sources (, too,) wait before doling out the next item until it has been **transduced** (dealt with)
+  completely; shouldn't asynchronous transforms behave likewise? Async transforms do have a `done()` method
+  to signal finishing, synchronous transforms don't have that, so it is not clear how to deal with a
+  situation where a transform happens to decide it doesn't want to `send()` anything (although, the
+  transform does return (stop running), so that might be a way)
+ * [ ] explain why using only `yield` instead of `send()` is not a good idea
 
 
 
+### To Do: Railway-Oriented Programming
 
+* https://zohaib.me/railway-programming-pattern-in-elixir/
+* https://fsharpforfunandprofit.com/rop/
+* https://github.com/zorbash/opus
+
+Transform categorization: functions may
+
+* acc. to result arity
+  * give back exactly one value for each input that we do care about (-> `$map()`)
+  * give back exactly one value for each input that we do not care about (-> `$watch()`)
+  * give back any number of values (-> `$`/`remit()`)
+  * never give back any value (-> `$watch()`)
+
+* acc. to iterability
+  * yield
+  * return
+
+* acc. to synchronicity
+  * be synchronous
+  * be asynchronous
+
+* acc. to happiness
+  * give back sad value on failure
+  * always give back happy failure, using `throw` for sad results
+  * return a sentinel value / error code (like JS `[].indexOf()`)
+
+* pipeline definition may take on this form:
+
+  ```coffee
+  ¶ = ( pipeline = [] ).push.bind pipeline
+  ¶ tee       other_pipeline, ( d ) -> 110 <= d <= 119    # optional filter, all `d`s stay in this pipeline, some also in other
+  ¶ switch    other_pipeline, ( d ) -> 110 <= d <= 119    # obligatory filter, each `d` in only one pipeline
+  ¶ watch     ( d ) -> ...                                # return value thrown away (does that respect async functions?)
+  ¶ guard     -1, $indexOf 'helo'                         # guard with filter value, saddens value when `true <- CND.equals(...)`
+  ¶ guard     ( ( d ) -> ... ), indexOf 'helo'            # guard with filter function, saddens value when `true <- filter()`
+  ¶ trycatch  map ( d       ) -> throw new Error "whaat" if d % 2 is 0; return d * 3 + 1
+  ¶ trycatch  $   ( d, send ) -> throw new Error "whaat" if d % 2 is 0; send d; send d * 3 + 1
+  ¶ if_sad $show_warning()
+  ¶ if_sad $ignore()
+  ¶ drain()
+  pull ¶
+  ```
+* pipe processing never calls any transform with sad value (except for those explicitly configured to accept
+  those)
+* but all sad values are still passed on, cause errors at pipeline end (near drain) when not being filtered
+  out
+* must **not** swallow exceptions implicitly as that would promote silent failures
+* benefit: simplify logic a great deal
+* benefit: may record errors and try to move on, then complain with summary of everything that went wrong
+
+
+

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Network access

Supply chain risk

This module accesses the network.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

956973

increased by8.23%

Number of package files

102

increased by6.25%

Lines of code

6756

increased by5.45%

Number of lines in readme file

302

increased by31.3%

Worsened metrics

Dependency count

6

increased by20%

Number of low supply chain risk alerts

17

increased by21.43%

Number of medium supply chain risk alerts

1

increased byInfinity%

Dependency changes

• + Added@uppy/fs-tail-stream@^1.2.0

• + Added@uppy/fs-tail-stream@1.2.0(transitive)

• + Addedonce@1.4.0(transitive)

• + Addedwrappy@1.0.2(transitive)

• Updatedcnd@^4.9.0

• Updatedintertype@^2.8.0

• Updatedmultimix@^2.1.1