@uwdata/flechette - npm Package Compare versions

Comparing version 0.0.3 to 0.0.4

dist/flechette.js

@@ -534,3 +534,4 @@ // src/constants.js
   /**
-   * Extract an array of values within the given index range.
+   * Extract an array of values within the given index range. Unlike
+   * Array.slice, all arguments are required and may not be negative indices.
    * @param {number} start The starting index, inclusive
@@ -561,3 +562,17 @@ * @param {number} end The ending index, exclusive
   /**
-   * Extract an array of values within the given index range.
+   * Create a new column batch with direct value array access.
+   * @param {object} options
+   * @param {number} options.length The length of the batch
+   * @param {number} options.nullCount The null value count
+   * @param {Uint8Array} [options.validity] Validity bitmap buffer
+   * @param {import('./types.js').TypedArray} options.values Values buffer
+   */
+  constructor(options) {
+    super(options);
+    const { length, values } = this;
+    this.values = values.subarray(0, length);
+  }
+  /**
+   * Extract an array of values within the given index range. Unlike
+   * Array.slice, all arguments are required and may not be negative indices.
    * When feasible, a zero-copy subarray of a typed array is returned.
@@ -564,0 +579,0 @@ * @param {number} start The starting index, inclusive

dist/flechette.min.js

@@ -1,1 +0,1 @@
-var w={V1:0,V2:1,V3:2,V4:3,V5:4},oe={Little:0,Big:1},S={NONE:0,Schema:1,DictionaryBatch:2,RecordBatch:3,Tensor:4,SparseTensor:5},c={Dictionary:-1,NONE:0,Null:1,Int:2,Float:3,Binary:4,Utf8:5,Bool:6,Decimal:7,Date:8,Time:9,Timestamp:10,Interval:11,List:12,Struct:13,Union:14,FixedSizeBinary:15,FixedSizeList:16,Map:17,Duration:18,LargeBinary:19,LargeUtf8:20,LargeList:21,RunEndEncoded:22,BinaryView:23,Utf8View:24,ListView:25,LargeListView:26},O={HALF:0,SINGLE:1,DOUBLE:2},k={DAY:0,MILLISECOND:1},E={SECOND:0,MILLISECOND:1,MICROSECOND:2,NANOSECOND:3},C={YEAR_MONTH:0,DAY_TIME:1,MONTH_DAY_NANO:2},H={Sparse:0,Dense:1};var ae=Uint8Array,wt=Uint16Array,Et=Uint32Array,ce=BigUint64Array,Mt=Int8Array,ie=Int16Array,D=Int32Array,T=BigInt64Array,zt=Float32Array,W=Float64Array;function Ht(e,t){let r=Math.log2(e)-3;return(t?[Mt,ie,D,T]:[ae,wt,Et,ce])[r]}function Tt(e,t){return(e[t>>3]&1<<t%8)!==0}var le=new TextDecoder("utf-8");function X(e){return le.decode(e)}function R(e,t){for(let[r,n]of Object.entries(e))if(n===t)return r;return"<Unknown>"}function N(e){if(e>Number.MAX_SAFE_INTEGER||e<Number.MIN_SAFE_INTEGER)throw Error(`BigInt exceeds integer number representation: ${e}`);return Number(e)}function Lt(e,t){return N(e/t)+N(e%t)/N(t)}function f(e,t){let r=t+h(e,t),n=r-h(e,r),s=x(e,n);return(o,a,i=null)=>{if(o<s){let l=x(e,n+o);if(l)return a(e,r+l)}return i}}function L(e,t){return t}function B(e,t){return!!ue(e,t)}function ue(e,t){return P(e,t)<<24>>24}function P(e,t){return e[t]}function x(e,t){return de(e,t)<<16>>16}function de(e,t){return e[t]|e[t+1]<<8}function h(e,t){return e[t]|e[t+1]<<8|e[t+2]<<16|e[t+3]<<24}function Pt(e,t){return h(e,t)>>>0}function fe(e,t){return BigInt.asIntN(64,BigInt(Pt(e,t))+(BigInt(Pt(e,t+4))<<BigInt(32)))}function v(e,t){return N(fe(e,t))}function V(e,t){let r=t+h(e,t),n=h(e,r);return r+=4,X(e.subarray(r,r+n))}function b(e,t){return{length:h(e,t+h(e,t)),base:t+h(e,t)+4}}var Y=class{static ArrayType=null;constructor({length:t,nullCount:r,validity:n,values:s,offsets:o,typeIds:a,children:i}){this.length=t,this.nullCount=r,this.validity=n,this.values=s,this.offsets=o,this.typeIds=a,this.children=i,r||(this.at=l=>this.value(l))}get[Symbol.toStringTag](){return"Batch"}at(t){return this.isValid(t)?this.value(t):null}isValid(t){return Tt(this.validity,t)}value(t){return this.values[t]}slice(t,r){let n=r-t,s=Array(n);for(let o=0;o<n;++o)s[o]=this.at(t+o);return s}*[Symbol.iterator](){let{length:t}=this;for(let r=0;r<t;++r)yield this.at(r)}},_=class extends Y{slice(t,r){return this.nullCount?super.slice(t,r):this.values.subarray(t,r)}[Symbol.iterator](){return this.nullCount?super[Symbol.iterator]():this.values[Symbol.iterator]()}},z=class extends Y{static ArrayType=W},g=class extends Y{static ArrayType=Array},q=class extends g{value(t){return null}},M=class extends z{value(t){return N(this.values[t])}},J=class extends z{value(t){let r=this.values[t],n=(r&31744)>>10,s=(r&1023)/1024,o=(-1)**((r&32768)>>15);switch(n){case 31:return o*(s?Number.NaN:1/0);case 0:return o*(s?6103515625e-14*s:0)}return o*2**(n-15)*(1+s)}},K=class extends g{value(t){return Tt(this.values,t)}},Yt=Array.from({length:8},(e,t)=>Math.pow(2,t*32)),Q=class extends z{constructor(t){super(t),this.stride=t.bitWidth>>5,this.scale=Math.pow(10,t.scale)}value(t){let r=this.values,n=this.stride,s=t<<2,o=0;if((r[n-1]|0)<0){for(let a=0;a<n;++a)o+=~r[a+s]*Yt[a];o=-(o+1)}else for(let a=0;a<n;++a)o+=r[a+s]*Yt[a];return o/this.scale}},tt=class extends g{constructor(t){super(t),this.source=t}value(t){return new Date(this.source.value(t))}},et=class extends z{value(t){return 864e5*this.values[t]}},$t=M,rt=class extends M{value(t){return super.value(t)*1e3}},Gt=M,nt=class extends M{value(t){return Lt(this.values[t],1000n)}},st=class extends M{value(t){return Lt(this.values[t],1000000n)}},ot=class extends g{value(t){return this.values.subarray(t<<1,t+1<<1)}},at=class extends g{value(t){let r=this.values[t];return Int32Array.of(Math.trunc(r/12),Math.trunc(r%12))}},ct=class extends g{value(t){let r=this.values,n=t<<2;return Float64Array.of(h(r,n),h(r,n+4),v(r,n+8))}},jt=({values:e,offsets:t},r)=>e.subarray(t[r],t[r+1]),Zt=({values:e,offsets:t},r)=>e.subarray(N(t[r]),N(t[r+1])),it=class extends g{value(t){return jt(this,t)}},lt=class extends g{value(t){return Zt(this,t)}},ut=class extends g{value(t){return X(jt(this,t))}},dt=class extends g{value(t){return X(Zt(this,t))}},ft=class extends g{value(t){let r=this.offsets;return this.children[0].slice(r[t],r[t+1])}},ht=class extends g{value(t){let r=this.offsets;return this.children[0].slice(N(r[t]),N(r[t+1]))}},pt=class extends g{constructor(t){super(t),this.stride=t.stride}},mt=class extends pt{value(t){let{stride:r,values:n}=this;return n.subarray(t*r,(t+1)*r)}},yt=class extends pt{value(t){let{children:r,stride:n}=this;return r[0].slice(t*n,(t+1)*n)}};function Wt({children:e,offsets:t},r){let[n,s]=e[0].children,o=t[r],a=t[r+1],i=[];for(let l=o;l<a;++l)i.push([n.at(l),s.at(l)]);return i}var gt=class extends g{value(t){return Wt(this,t)}},It=class extends g{value(t){return new Map(Wt(this,t))}},$=class extends g{constructor(t){super(t),this.map=t.map}value(t){let{typeIds:r,children:n,map:s}=this;return n[s[r[t]]].at(t)}},vt=class extends ${value(t){return super.value(this.offsets[t])}},xt=class extends g{constructor(t){super(t),this.names=t.names}value(t){let{children:r,names:n}=this,s=n.length,o={};for(let a=0;a<s;++a)o[n[a]]=r[a].at(t);return o}},bt=class extends g{constructor(t){super(t),this.cache=t.dictionary.cache()}value(t){return this.cache[this.key(t)]}key(t){return this.values[t]}};function Ut(){let e=[];return{add(t){return e.push(t),this},clear:()=>e=[],done:()=>new Bt(e)}}var Bt=class{constructor(t){this.length=t.reduce((s,o)=>s+o.length,0),this.nullCount=t.reduce((s,o)=>s+o.nullCount,0),this.data=t;let r=t.length,n=new Int32Array(r+1);if(r===1){let[s]=t;n[1]=s.length,this.at=o=>s.at(o)}else for(let s=0,o=0;s<r;++s)n[s+1]=o+=t[s].length;this.offsets=n}get[Symbol.toStringTag](){return"Column"}[Symbol.iterator](){let t=this.data;return t.length===1?t[0][Symbol.iterator]():he(t)}at(t){let{data:r,offsets:n}=this,s=0,o=n.length;do{let a=s+o>>>1;n[a]<=t?s=a+1:o=a}while(s<o);return r[--s]?.at(t-n[s])}toArray(){let{length:t,nullCount:r,data:n}=this,s=!r&&pe(n),o=n.length;if(s&&o===1)return n[0].values;let a=!o||r>0?Array:n[0].constructor.ArrayType??n[0].values.constructor,i=new a(t);return s?me(i,n):ye(i,n)}cache(){return this._cache??(this._cache=this.toArray())}};function*he(e){for(let t=0;t<e.length;++t){let r=e[t][Symbol.iterator]();for(let n=r.next();!n.done;n=r.next())yield n.value}}function pe(e){return e.length&&e[0]instanceof _}function me(e,t){for(let r=0,n=0;r<t.length;++r){let{values:s}=t[r];e.set(s,n),n+=s.length}return e}function ye(e,t){let r=-1;for(let n=0;n<t.length;++n){let s=t[n];for(let o=0;o<s.length;++o)e[++r]=s.at(o)}return e}function Ft(e,t,r,n){switch(r){case c.NONE:case c.Null:case c.Bool:return{typeId:r};case c.Binary:case c.Utf8:return{typeId:r,offsets:D};case c.LargeBinary:case c.LargeUtf8:return{typeId:r,offsets:T};case c.List:return{typeId:r,children:[n?.[0]??null],offsets:D};case c.LargeList:return{typeId:r,children:[n?.[0]??null],offsets:T};case c.Struct:return{typeId:r,children:n};case c.Int:return Ct(e,t);case c.Float:return ge(e,t);case c.Decimal:return Ie(e,t);case c.Date:return ve(e,t);case c.Time:return xe(e,t);case c.Timestamp:return be(e,t);case c.Interval:return Ae(e,t);case c.Duration:return Se(e,t);case c.FixedSizeBinary:return De(e,t);case c.FixedSizeList:return Ne(e,t,n);case c.Map:return we(e,t,n);case c.Union:return Ee(e,t,n)}throw new Error(`Unrecognized type: "${R(c,r)}" (id ${r})`)}function Ot(e,t){return{typeId:c.Int,bitWidth:e,signed:t,values:Ht(e,t)}}function Ct(e,t){let r=f(e,t);return Ot(r(4,h,0),r(6,B,!1))}function ge(e,t){let n=f(e,t)(4,x,O.HALF);return{typeId:c.Float,precision:n,values:n===O.HALF?wt:n===O.SINGLE?zt:W}}function Ie(e,t){let r=f(e,t),n=r(8,h,128);return{typeId:c.Decimal,precision:r(4,h,0),scale:r(6,h,0),bitWidth:n,values:Et}}function ve(e,t){let n=f(e,t)(4,x,k.MILLISECOND);return{typeId:c.Date,unit:n,values:n===k.DAY?D:T}}function xe(e,t){let r=f(e,t),n=r(6,h,32);return{typeId:c.Time,unit:r(4,x,E.MILLISECOND),bitWidth:n,values:n===32?D:T}}function be(e,t){let r=f(e,t);return{typeId:c.Timestamp,unit:r(4,x,E.SECOND),timezone:r(6,V),values:T}}function Ae(e,t){let n=f(e,t)(4,x,C.YEAR_MONTH);return{typeId:c.Interval,unit:n,values:n===C.MONTH_DAY_NANO?void 0:D}}function Se(e,t){let r=f(e,t);return{typeId:c.Duration,unit:r(4,x,E.MILLISECOND),values:T}}function De(e,t){let r=f(e,t);return{typeId:c.FixedSizeBinary,stride:r(4,h,0)}}function Ne(e,t,r){let n=f(e,t);return{typeId:c.FixedSizeList,stride:n(4,h,0),children:[r?.[0]??null]}}function we(e,t,r){let n=f(e,t);return{typeId:c.Map,keysSorted:n(4,B,!1),children:r,offsets:D}}function Ee(e,t,r){let n=f(e,t),{length:s,base:o}=b(e,n(6,L));return{typeId:c.Union,mode:n(4,x,H.Sparse),typeIds:new D(e.buffer,e.byteOffset+o,s),children:r??[],offsets:D}}function G(e,t){let{length:r,base:n}=b(e,t),s=r>0?new Map:null;for(let o=0;o<r;++o){let a=f(e,n+o*4),i=a(4,V),l=a(6,V);(i||l)&&s.set(i,l)}return s?.size?s:null}function At(e,t,r){let n=new Map,s=f(e,t);return{version:r,endianness:s(4,x,0),fields:s(6,(o,a)=>Me(o,a,n),[]),metadata:s(8,G),dictionaryTypes:n}}function Me(e,t,r){let{length:n,base:s}=b(e,t),o=[];for(let a=0;a<n;++a)o.push(Xt(e,s+a*4,r));return o}function Xt(e,t,r){let n=f(e,t),s=n(8,P,c.NONE),o=n(10,L,0),a=n(12,(m,u)=>Le(m,u)),i=n(14,Te),l;if(a){let{id:m}=a,u=r.get(m);u||(u=Ft(e,o,s,i),r.set(m,u)),a.type=u,l=a}else l=Ft(e,o,s,i);return{name:n(4,V),type:l,nullable:n(6,B,!1),metadata:n(16,G)}}function Te(e,t,r){let{length:n,base:s}=b(e,t),o=[];for(let a=0;a<n;++a){let i=s+a*4;o.push(Xt(e,i,r))}return o.length?o:null}function Le(e,t){if(!t)return null;let r=f(e,t);return{type:null,typeId:c.Dictionary,id:r(4,v,0),keys:r(6,Ct,Ot(32,!0)),ordered:r(8,B,!1)}}function St(e,t,r){let n=f(e,t);if(n(10,L,0))throw new Error("Record batch compression not implemented");let s=[],o=n(6,b);if(o){let{length:l,base:m}=o;for(let u=0;u<l;++u){let d=m+u*16;s.push({length:v(e,d),nullCount:v(e,d+8)})}}let a=[],i=n(8,b);if(i){let{length:l,base:m}=i,u=r<w.V4;for(let d=0;d<l;++d){let p=m+d*16+(u?8*(d+1):0);a.push({offset:v(e,p),length:v(e,p+8)})}}return{length:n(4,v,0),nodes:s,buffers:a}}function qt(e,t,r){let n=f(e,t);return{id:n(4,v,0),data:n(6,(s,o)=>St(s,o,r)),isDelta:n(8,B,!1)}}var Be=(e,t)=>`Expected to read ${e} metadata bytes, but only read ${t}.`,Ue=(e,t)=>`Expected to read ${e} bytes for message body, but only read ${t}.`,Fe=e=>`Unsupported message type: ${e} (${R(S,e)})`;function Dt(e,t){let r=h(e,t)||0;if(t+=4,r===-1&&(r=h(e,t)||0,t+=4),r===0)return null;let n=e.subarray(t,t+=r);if(n.byteLength<r)throw new Error(Be(r,n.byteLength));let s=f(n,0),o=s(4,x,w.V1),a=s(6,P,S.NONE),i=s(8,L,0),l=s(10,v,0),m;if(i){let u=a===S.Schema?At:a===S.DictionaryBatch?qt:a===S.RecordBatch?St:null;if(!u)throw new Error(Fe(a));if(m=u(n,i,o),l>0){let d=e.subarray(t,t+=l);if(d.byteLength<l)throw new Error(Ue(l,d.byteLength));m.body=d}}return{version:o,type:a,index:t,content:m}}function Oe(e,t){return{offset:v(e,t),metadataLength:h(e,t+8),bodyLength:v(e,t+16)}}function Vt(e,t){let{length:r,base:n}=b(e,t),s=[];for(let o=0;o<r;++o)s.push(Oe(e,n+o*24));return s}function Kt(e){let t=e instanceof ArrayBuffer?new Uint8Array(e):e;return!Array.isArray(t)&&Ce(t)?_e(t):Ve(t)}var Qt=Uint8Array.of(65,82,82,79,87,49);function Ce(e){if(!e||e.length<4)return!1;for(let t=0;t<6;++t)if(Qt[t]!==e[t])return!1;return!0}function Ve(e){let t=[e].flat(),r,n=[],s=[];for(let o of t){let a=0;for(;;){let i=Dt(o,a);if(i===null)break;if(a=i.index,!!i.content)switch(i.type){case S.Schema:r||(r=i.content);break;case S.RecordBatch:n.push(i.content);break;case S.DictionaryBatch:s.push(i.content);break}}}return{schema:r,dictionaries:s,records:n,metadata:null}}function _e(e){let t=e.byteLength-(Qt.length+4),r=h(e,t),n=f(e,t-r),s=n(4,x,w.V1),o=n(8,Vt,[]),a=n(10,Vt,[]);return{schema:n(6,(i,l)=>At(i,l,s)),dictionaries:o.map(({offset:i})=>Dt(e,i).content),records:a.map(({offset:i})=>Dt(e,i).content),metadata:n(12,G)}}var Nt=class e{constructor(t,r){this.schema=t,this.names=t.fields.map(n=>n.name),this.children=r}get[Symbol.toStringTag](){return"Table"}get numCols(){return this.names.length}get numRows(){return this.children[0]?.length??0}getChildAt(t){return this.children[t]}getChild(t){let r=this.names.findIndex(n=>n===t);return r>-1?this.children[r]:void 0}selectAt(t,r=[]){let{children:n,schema:s}=this,{fields:o}=s;return new e({...s,fields:t.map((a,i)=>ke(o[a],r[i]))},t.map(a=>n[a]))}select(t,r){let n=this.names,s=t.map(o=>n.indexOf(o));return this.selectAt(s,r)}toColumns(){let{children:t,names:r}=this,n={};return r.forEach((s,o)=>n[s]=t[o]?.toArray()??[]),n}*[Symbol.iterator](){let{children:t,names:r}=this,n=t[0]?.data.length??0;for(let s=0;s<n;++s){let o=t.map(i=>i.data[s]),a=o[0].length;for(let i=0;i<a;++i)yield te(r,o,i)}}toArray(){let{children:t,numRows:r,names:n}=this,s=t[0]?.data.length??0,o=Array(r);for(let a=0,i=-1;a<s;++a){let l=t.map(u=>u.data[a]),m=l?.[0].length;for(let u=0;u<m;++u)o[++i]=te(n,l,u)}return o}};function ke(e,t){return t!=null&&t!==e.name?{...e,name:t}:e}function te(e,t,r){let n={};for(let s=0;s<e.length;++s)n[e[s]]=t[s].at(r);return n}function Re(e,t){return ze(Kt(e),t)}function ze(e,t={}){let{schema:r={fields:[]},dictionaries:n,records:s}=e,{version:o,fields:a,dictionaryTypes:i}=r,l=new Map,m=He(t,o,l),u=new Map;for(let p of n){let{id:I,data:U,isDelta:F,body:j}=p,y=i.get(I),A=_t(y,m({...U,body:j}));if(u.has(I)){let Z=u.get(I);F||Z.clear(),Z.add(A)}else{if(F)throw new Error("Delta update can not be first dictionary batch.");u.set(I,Ut().add(A))}}u.forEach((p,I)=>l.set(I,p.done()));let d=a.map(()=>Ut());for(let p of s){let I=m(p);a.forEach((U,F)=>d[F].add(_t(U.type,I)))}return new Nt(r,d.map(p=>p.done()))}function He(e,t,r){let n={version:t,options:e,dictionary:s=>r.get(s)};return s=>{let{length:o,nodes:a,buffers:i,body:l}=s,m=-1,u=-1;return{...n,length:o,node:()=>a[++m],buffer:d=>{let{length:p,offset:I}=i[++u];return d?new d(l.buffer,l.byteOffset+I,p/d.BYTES_PER_ELEMENT):l.subarray(I,I+p)},visitAll(d){return d.map(p=>_t(p.type,this))}}}}function _t(e,t){let{typeId:r,bitWidth:n,precision:s,scale:o,stride:a,unit:i}=e,{useBigInt:l,useDate:m,useMap:u}=t.options;if(r===c.Null){let{length:y}=t;return new q({length:y,nullCount:y})}let d=t.node(),p=(y,A)=>new y({...d,...A,validity:t.buffer(),values:t.buffer(e.values)}),I=y=>new y({...d,validity:t.buffer(),offsets:t.buffer(e.offsets),values:t.buffer()}),U=y=>new y({...d,validity:t.buffer(),offsets:t.buffer(e.offsets),children:t.visitAll(e.children)}),F=(y,A)=>new y({...d,...A,validity:t.buffer(),children:t.visitAll(e.children)}),j=m?y=>new tt(p(y)):p;switch(r){case c.Bool:return p(K);case c.Int:case c.Time:case c.Duration:return p(n===64&&!l?M:_);case c.Float:return p(s===O.HALF?J:_);case c.Date:return j(i===k.DAY?et:$t);case c.Timestamp:return j(i===E.SECOND?rt:i===E.MILLISECOND?Gt:i===E.MICROSECOND?nt:st);case c.Decimal:return p(Q,{bitWidth:n,scale:o});case c.Interval:return p(i===C.DAY_TIME?ot:i===C.YEAR_MONTH?at:ct);case c.FixedSizeBinary:return p(mt,{stride:a});case c.Utf8:return I(ut);case c.LargeUtf8:return I(dt);case c.Binary:return I(it);case c.LargeBinary:return I(lt);case c.List:return U(ft);case c.LargeList:return U(ht);case c.Map:return U(u?It:gt);case c.FixedSizeList:return F(yt,{stride:a});case c.Struct:return F(xt,{names:e.children.map(y=>y.name)});case c.Dictionary:{let{id:y,keys:A}=e;return new bt({...d,validity:t.buffer(),values:t.buffer(A.values),dictionary:t.dictionary(y)})}case c.Union:{t.version<w.V5&&t.buffer();let y=e.mode===H.Sparse,A=t.buffer(Mt),Z=y?null:t.buffer(e.offsets),ee=t.visitAll(e.children),re=e.typeIds.reduce((Rt,ne,se)=>(Rt[ne]=se,Rt),{}),kt={...d,map:re,typeIds:A,offsets:Z,children:ee};return y?new $(kt):new vt(kt)}default:throw new Error(`Unsupported type: ${r}, (${R(c,r)})`)}}export{k as DateUnit,oe as Endianness,C as IntervalUnit,O as Precision,E as TimeUnit,c as Type,H as UnionMode,w as Version,Re as tableFromIPC};
+var w={V1:0,V2:1,V3:2,V4:3,V5:4},oe={Little:0,Big:1},S={NONE:0,Schema:1,DictionaryBatch:2,RecordBatch:3,Tensor:4,SparseTensor:5},c={Dictionary:-1,NONE:0,Null:1,Int:2,Float:3,Binary:4,Utf8:5,Bool:6,Decimal:7,Date:8,Time:9,Timestamp:10,Interval:11,List:12,Struct:13,Union:14,FixedSizeBinary:15,FixedSizeList:16,Map:17,Duration:18,LargeBinary:19,LargeUtf8:20,LargeList:21,RunEndEncoded:22,BinaryView:23,Utf8View:24,ListView:25,LargeListView:26},O={HALF:0,SINGLE:1,DOUBLE:2},k={DAY:0,MILLISECOND:1},E={SECOND:0,MILLISECOND:1,MICROSECOND:2,NANOSECOND:3},C={YEAR_MONTH:0,DAY_TIME:1,MONTH_DAY_NANO:2},H={Sparse:0,Dense:1};var ae=Uint8Array,wt=Uint16Array,Et=Uint32Array,ce=BigUint64Array,Mt=Int8Array,ie=Int16Array,D=Int32Array,T=BigInt64Array,zt=Float32Array,W=Float64Array;function Ht(e,t){let r=Math.log2(e)-3;return(t?[Mt,ie,D,T]:[ae,wt,Et,ce])[r]}function Tt(e,t){return(e[t>>3]&1<<t%8)!==0}var le=new TextDecoder("utf-8");function X(e){return le.decode(e)}function R(e,t){for(let[r,n]of Object.entries(e))if(n===t)return r;return"<Unknown>"}function N(e){if(e>Number.MAX_SAFE_INTEGER||e<Number.MIN_SAFE_INTEGER)throw Error(`BigInt exceeds integer number representation: ${e}`);return Number(e)}function Lt(e,t){return N(e/t)+N(e%t)/N(t)}function f(e,t){let r=t+h(e,t),n=r-h(e,r),s=x(e,n);return(o,a,i=null)=>{if(o<s){let l=x(e,n+o);if(l)return a(e,r+l)}return i}}function L(e,t){return t}function B(e,t){return!!ue(e,t)}function ue(e,t){return P(e,t)<<24>>24}function P(e,t){return e[t]}function x(e,t){return de(e,t)<<16>>16}function de(e,t){return e[t]|e[t+1]<<8}function h(e,t){return e[t]|e[t+1]<<8|e[t+2]<<16|e[t+3]<<24}function Pt(e,t){return h(e,t)>>>0}function fe(e,t){return BigInt.asIntN(64,BigInt(Pt(e,t))+(BigInt(Pt(e,t+4))<<BigInt(32)))}function v(e,t){return N(fe(e,t))}function V(e,t){let r=t+h(e,t),n=h(e,r);return r+=4,X(e.subarray(r,r+n))}function b(e,t){return{length:h(e,t+h(e,t)),base:t+h(e,t)+4}}var Y=class{static ArrayType=null;constructor({length:t,nullCount:r,validity:n,values:s,offsets:o,typeIds:a,children:i}){this.length=t,this.nullCount=r,this.validity=n,this.values=s,this.offsets=o,this.typeIds=a,this.children=i,r||(this.at=l=>this.value(l))}get[Symbol.toStringTag](){return"Batch"}at(t){return this.isValid(t)?this.value(t):null}isValid(t){return Tt(this.validity,t)}value(t){return this.values[t]}slice(t,r){let n=r-t,s=Array(n);for(let o=0;o<n;++o)s[o]=this.at(t+o);return s}*[Symbol.iterator](){let{length:t}=this;for(let r=0;r<t;++r)yield this.at(r)}},_=class extends Y{constructor(t){super(t);let{length:r,values:n}=this;this.values=n.subarray(0,r)}slice(t,r){return this.nullCount?super.slice(t,r):this.values.subarray(t,r)}[Symbol.iterator](){return this.nullCount?super[Symbol.iterator]():this.values[Symbol.iterator]()}},z=class extends Y{static ArrayType=W},g=class extends Y{static ArrayType=Array},q=class extends g{value(t){return null}},M=class extends z{value(t){return N(this.values[t])}},J=class extends z{value(t){let r=this.values[t],n=(r&31744)>>10,s=(r&1023)/1024,o=(-1)**((r&32768)>>15);switch(n){case 31:return o*(s?Number.NaN:1/0);case 0:return o*(s?6103515625e-14*s:0)}return o*2**(n-15)*(1+s)}},K=class extends g{value(t){return Tt(this.values,t)}},Yt=Array.from({length:8},(e,t)=>Math.pow(2,t*32)),Q=class extends z{constructor(t){super(t),this.stride=t.bitWidth>>5,this.scale=Math.pow(10,t.scale)}value(t){let r=this.values,n=this.stride,s=t<<2,o=0;if((r[n-1]|0)<0){for(let a=0;a<n;++a)o+=~r[a+s]*Yt[a];o=-(o+1)}else for(let a=0;a<n;++a)o+=r[a+s]*Yt[a];return o/this.scale}},tt=class extends g{constructor(t){super(t),this.source=t}value(t){return new Date(this.source.value(t))}},et=class extends z{value(t){return 864e5*this.values[t]}},$t=M,rt=class extends M{value(t){return super.value(t)*1e3}},Gt=M,nt=class extends M{value(t){return Lt(this.values[t],1000n)}},st=class extends M{value(t){return Lt(this.values[t],1000000n)}},ot=class extends g{value(t){return this.values.subarray(t<<1,t+1<<1)}},at=class extends g{value(t){let r=this.values[t];return Int32Array.of(Math.trunc(r/12),Math.trunc(r%12))}},ct=class extends g{value(t){let r=this.values,n=t<<2;return Float64Array.of(h(r,n),h(r,n+4),v(r,n+8))}},jt=({values:e,offsets:t},r)=>e.subarray(t[r],t[r+1]),Zt=({values:e,offsets:t},r)=>e.subarray(N(t[r]),N(t[r+1])),it=class extends g{value(t){return jt(this,t)}},lt=class extends g{value(t){return Zt(this,t)}},ut=class extends g{value(t){return X(jt(this,t))}},dt=class extends g{value(t){return X(Zt(this,t))}},ft=class extends g{value(t){let r=this.offsets;return this.children[0].slice(r[t],r[t+1])}},ht=class extends g{value(t){let r=this.offsets;return this.children[0].slice(N(r[t]),N(r[t+1]))}},pt=class extends g{constructor(t){super(t),this.stride=t.stride}},mt=class extends pt{value(t){let{stride:r,values:n}=this;return n.subarray(t*r,(t+1)*r)}},yt=class extends pt{value(t){let{children:r,stride:n}=this;return r[0].slice(t*n,(t+1)*n)}};function Wt({children:e,offsets:t},r){let[n,s]=e[0].children,o=t[r],a=t[r+1],i=[];for(let l=o;l<a;++l)i.push([n.at(l),s.at(l)]);return i}var gt=class extends g{value(t){return Wt(this,t)}},It=class extends g{value(t){return new Map(Wt(this,t))}},$=class extends g{constructor(t){super(t),this.map=t.map}value(t){let{typeIds:r,children:n,map:s}=this;return n[s[r[t]]].at(t)}},vt=class extends ${value(t){return super.value(this.offsets[t])}},xt=class extends g{constructor(t){super(t),this.names=t.names}value(t){let{children:r,names:n}=this,s=n.length,o={};for(let a=0;a<s;++a)o[n[a]]=r[a].at(t);return o}},bt=class extends g{constructor(t){super(t),this.cache=t.dictionary.cache()}value(t){return this.cache[this.key(t)]}key(t){return this.values[t]}};function Ut(){let e=[];return{add(t){return e.push(t),this},clear:()=>e=[],done:()=>new Bt(e)}}var Bt=class{constructor(t){this.length=t.reduce((s,o)=>s+o.length,0),this.nullCount=t.reduce((s,o)=>s+o.nullCount,0),this.data=t;let r=t.length,n=new Int32Array(r+1);if(r===1){let[s]=t;n[1]=s.length,this.at=o=>s.at(o)}else for(let s=0,o=0;s<r;++s)n[s+1]=o+=t[s].length;this.offsets=n}get[Symbol.toStringTag](){return"Column"}[Symbol.iterator](){let t=this.data;return t.length===1?t[0][Symbol.iterator]():he(t)}at(t){let{data:r,offsets:n}=this,s=0,o=n.length;do{let a=s+o>>>1;n[a]<=t?s=a+1:o=a}while(s<o);return r[--s]?.at(t-n[s])}toArray(){let{length:t,nullCount:r,data:n}=this,s=!r&&pe(n),o=n.length;if(s&&o===1)return n[0].values;let a=!o||r>0?Array:n[0].constructor.ArrayType??n[0].values.constructor,i=new a(t);return s?me(i,n):ye(i,n)}cache(){return this._cache??(this._cache=this.toArray())}};function*he(e){for(let t=0;t<e.length;++t){let r=e[t][Symbol.iterator]();for(let n=r.next();!n.done;n=r.next())yield n.value}}function pe(e){return e.length&&e[0]instanceof _}function me(e,t){for(let r=0,n=0;r<t.length;++r){let{values:s}=t[r];e.set(s,n),n+=s.length}return e}function ye(e,t){let r=-1;for(let n=0;n<t.length;++n){let s=t[n];for(let o=0;o<s.length;++o)e[++r]=s.at(o)}return e}function Ft(e,t,r,n){switch(r){case c.NONE:case c.Null:case c.Bool:return{typeId:r};case c.Binary:case c.Utf8:return{typeId:r,offsets:D};case c.LargeBinary:case c.LargeUtf8:return{typeId:r,offsets:T};case c.List:return{typeId:r,children:[n?.[0]??null],offsets:D};case c.LargeList:return{typeId:r,children:[n?.[0]??null],offsets:T};case c.Struct:return{typeId:r,children:n};case c.Int:return Ct(e,t);case c.Float:return ge(e,t);case c.Decimal:return Ie(e,t);case c.Date:return ve(e,t);case c.Time:return xe(e,t);case c.Timestamp:return be(e,t);case c.Interval:return Ae(e,t);case c.Duration:return Se(e,t);case c.FixedSizeBinary:return De(e,t);case c.FixedSizeList:return Ne(e,t,n);case c.Map:return we(e,t,n);case c.Union:return Ee(e,t,n)}throw new Error(`Unrecognized type: "${R(c,r)}" (id ${r})`)}function Ot(e,t){return{typeId:c.Int,bitWidth:e,signed:t,values:Ht(e,t)}}function Ct(e,t){let r=f(e,t);return Ot(r(4,h,0),r(6,B,!1))}function ge(e,t){let n=f(e,t)(4,x,O.HALF);return{typeId:c.Float,precision:n,values:n===O.HALF?wt:n===O.SINGLE?zt:W}}function Ie(e,t){let r=f(e,t),n=r(8,h,128);return{typeId:c.Decimal,precision:r(4,h,0),scale:r(6,h,0),bitWidth:n,values:Et}}function ve(e,t){let n=f(e,t)(4,x,k.MILLISECOND);return{typeId:c.Date,unit:n,values:n===k.DAY?D:T}}function xe(e,t){let r=f(e,t),n=r(6,h,32);return{typeId:c.Time,unit:r(4,x,E.MILLISECOND),bitWidth:n,values:n===32?D:T}}function be(e,t){let r=f(e,t);return{typeId:c.Timestamp,unit:r(4,x,E.SECOND),timezone:r(6,V),values:T}}function Ae(e,t){let n=f(e,t)(4,x,C.YEAR_MONTH);return{typeId:c.Interval,unit:n,values:n===C.MONTH_DAY_NANO?void 0:D}}function Se(e,t){let r=f(e,t);return{typeId:c.Duration,unit:r(4,x,E.MILLISECOND),values:T}}function De(e,t){let r=f(e,t);return{typeId:c.FixedSizeBinary,stride:r(4,h,0)}}function Ne(e,t,r){let n=f(e,t);return{typeId:c.FixedSizeList,stride:n(4,h,0),children:[r?.[0]??null]}}function we(e,t,r){let n=f(e,t);return{typeId:c.Map,keysSorted:n(4,B,!1),children:r,offsets:D}}function Ee(e,t,r){let n=f(e,t),{length:s,base:o}=b(e,n(6,L));return{typeId:c.Union,mode:n(4,x,H.Sparse),typeIds:new D(e.buffer,e.byteOffset+o,s),children:r??[],offsets:D}}function G(e,t){let{length:r,base:n}=b(e,t),s=r>0?new Map:null;for(let o=0;o<r;++o){let a=f(e,n+o*4),i=a(4,V),l=a(6,V);(i||l)&&s.set(i,l)}return s?.size?s:null}function At(e,t,r){let n=new Map,s=f(e,t);return{version:r,endianness:s(4,x,0),fields:s(6,(o,a)=>Me(o,a,n),[]),metadata:s(8,G),dictionaryTypes:n}}function Me(e,t,r){let{length:n,base:s}=b(e,t),o=[];for(let a=0;a<n;++a)o.push(Xt(e,s+a*4,r));return o}function Xt(e,t,r){let n=f(e,t),s=n(8,P,c.NONE),o=n(10,L,0),a=n(12,(m,u)=>Le(m,u)),i=n(14,Te),l;if(a){let{id:m}=a,u=r.get(m);u||(u=Ft(e,o,s,i),r.set(m,u)),a.type=u,l=a}else l=Ft(e,o,s,i);return{name:n(4,V),type:l,nullable:n(6,B,!1),metadata:n(16,G)}}function Te(e,t,r){let{length:n,base:s}=b(e,t),o=[];for(let a=0;a<n;++a){let i=s+a*4;o.push(Xt(e,i,r))}return o.length?o:null}function Le(e,t){if(!t)return null;let r=f(e,t);return{type:null,typeId:c.Dictionary,id:r(4,v,0),keys:r(6,Ct,Ot(32,!0)),ordered:r(8,B,!1)}}function St(e,t,r){let n=f(e,t);if(n(10,L,0))throw new Error("Record batch compression not implemented");let s=[],o=n(6,b);if(o){let{length:l,base:m}=o;for(let u=0;u<l;++u){let d=m+u*16;s.push({length:v(e,d),nullCount:v(e,d+8)})}}let a=[],i=n(8,b);if(i){let{length:l,base:m}=i,u=r<w.V4;for(let d=0;d<l;++d){let p=m+d*16+(u?8*(d+1):0);a.push({offset:v(e,p),length:v(e,p+8)})}}return{length:n(4,v,0),nodes:s,buffers:a}}function qt(e,t,r){let n=f(e,t);return{id:n(4,v,0),data:n(6,(s,o)=>St(s,o,r)),isDelta:n(8,B,!1)}}var Be=(e,t)=>`Expected to read ${e} metadata bytes, but only read ${t}.`,Ue=(e,t)=>`Expected to read ${e} bytes for message body, but only read ${t}.`,Fe=e=>`Unsupported message type: ${e} (${R(S,e)})`;function Dt(e,t){let r=h(e,t)||0;if(t+=4,r===-1&&(r=h(e,t)||0,t+=4),r===0)return null;let n=e.subarray(t,t+=r);if(n.byteLength<r)throw new Error(Be(r,n.byteLength));let s=f(n,0),o=s(4,x,w.V1),a=s(6,P,S.NONE),i=s(8,L,0),l=s(10,v,0),m;if(i){let u=a===S.Schema?At:a===S.DictionaryBatch?qt:a===S.RecordBatch?St:null;if(!u)throw new Error(Fe(a));if(m=u(n,i,o),l>0){let d=e.subarray(t,t+=l);if(d.byteLength<l)throw new Error(Ue(l,d.byteLength));m.body=d}}return{version:o,type:a,index:t,content:m}}function Oe(e,t){return{offset:v(e,t),metadataLength:h(e,t+8),bodyLength:v(e,t+16)}}function Vt(e,t){let{length:r,base:n}=b(e,t),s=[];for(let o=0;o<r;++o)s.push(Oe(e,n+o*24));return s}function Kt(e){let t=e instanceof ArrayBuffer?new Uint8Array(e):e;return!Array.isArray(t)&&Ce(t)?_e(t):Ve(t)}var Qt=Uint8Array.of(65,82,82,79,87,49);function Ce(e){if(!e||e.length<4)return!1;for(let t=0;t<6;++t)if(Qt[t]!==e[t])return!1;return!0}function Ve(e){let t=[e].flat(),r,n=[],s=[];for(let o of t){let a=0;for(;;){let i=Dt(o,a);if(i===null)break;if(a=i.index,!!i.content)switch(i.type){case S.Schema:r||(r=i.content);break;case S.RecordBatch:n.push(i.content);break;case S.DictionaryBatch:s.push(i.content);break}}}return{schema:r,dictionaries:s,records:n,metadata:null}}function _e(e){let t=e.byteLength-(Qt.length+4),r=h(e,t),n=f(e,t-r),s=n(4,x,w.V1),o=n(8,Vt,[]),a=n(10,Vt,[]);return{schema:n(6,(i,l)=>At(i,l,s)),dictionaries:o.map(({offset:i})=>Dt(e,i).content),records:a.map(({offset:i})=>Dt(e,i).content),metadata:n(12,G)}}var Nt=class e{constructor(t,r){this.schema=t,this.names=t.fields.map(n=>n.name),this.children=r}get[Symbol.toStringTag](){return"Table"}get numCols(){return this.names.length}get numRows(){return this.children[0]?.length??0}getChildAt(t){return this.children[t]}getChild(t){let r=this.names.findIndex(n=>n===t);return r>-1?this.children[r]:void 0}selectAt(t,r=[]){let{children:n,schema:s}=this,{fields:o}=s;return new e({...s,fields:t.map((a,i)=>ke(o[a],r[i]))},t.map(a=>n[a]))}select(t,r){let n=this.names,s=t.map(o=>n.indexOf(o));return this.selectAt(s,r)}toColumns(){let{children:t,names:r}=this,n={};return r.forEach((s,o)=>n[s]=t[o]?.toArray()??[]),n}*[Symbol.iterator](){let{children:t,names:r}=this,n=t[0]?.data.length??0;for(let s=0;s<n;++s){let o=t.map(i=>i.data[s]),a=o[0].length;for(let i=0;i<a;++i)yield te(r,o,i)}}toArray(){let{children:t,numRows:r,names:n}=this,s=t[0]?.data.length??0,o=Array(r);for(let a=0,i=-1;a<s;++a){let l=t.map(u=>u.data[a]),m=l?.[0].length;for(let u=0;u<m;++u)o[++i]=te(n,l,u)}return o}};function ke(e,t){return t!=null&&t!==e.name?{...e,name:t}:e}function te(e,t,r){let n={};for(let s=0;s<e.length;++s)n[e[s]]=t[s].at(r);return n}function Re(e,t){return ze(Kt(e),t)}function ze(e,t={}){let{schema:r={fields:[]},dictionaries:n,records:s}=e,{version:o,fields:a,dictionaryTypes:i}=r,l=new Map,m=He(t,o,l),u=new Map;for(let p of n){let{id:I,data:U,isDelta:F,body:j}=p,y=i.get(I),A=_t(y,m({...U,body:j}));if(u.has(I)){let Z=u.get(I);F||Z.clear(),Z.add(A)}else{if(F)throw new Error("Delta update can not be first dictionary batch.");u.set(I,Ut().add(A))}}u.forEach((p,I)=>l.set(I,p.done()));let d=a.map(()=>Ut());for(let p of s){let I=m(p);a.forEach((U,F)=>d[F].add(_t(U.type,I)))}return new Nt(r,d.map(p=>p.done()))}function He(e,t,r){let n={version:t,options:e,dictionary:s=>r.get(s)};return s=>{let{length:o,nodes:a,buffers:i,body:l}=s,m=-1,u=-1;return{...n,length:o,node:()=>a[++m],buffer:d=>{let{length:p,offset:I}=i[++u];return d?new d(l.buffer,l.byteOffset+I,p/d.BYTES_PER_ELEMENT):l.subarray(I,I+p)},visitAll(d){return d.map(p=>_t(p.type,this))}}}}function _t(e,t){let{typeId:r,bitWidth:n,precision:s,scale:o,stride:a,unit:i}=e,{useBigInt:l,useDate:m,useMap:u}=t.options;if(r===c.Null){let{length:y}=t;return new q({length:y,nullCount:y})}let d=t.node(),p=(y,A)=>new y({...d,...A,validity:t.buffer(),values:t.buffer(e.values)}),I=y=>new y({...d,validity:t.buffer(),offsets:t.buffer(e.offsets),values:t.buffer()}),U=y=>new y({...d,validity:t.buffer(),offsets:t.buffer(e.offsets),children:t.visitAll(e.children)}),F=(y,A)=>new y({...d,...A,validity:t.buffer(),children:t.visitAll(e.children)}),j=m?y=>new tt(p(y)):p;switch(r){case c.Bool:return p(K);case c.Int:case c.Time:case c.Duration:return p(n===64&&!l?M:_);case c.Float:return p(s===O.HALF?J:_);case c.Date:return j(i===k.DAY?et:$t);case c.Timestamp:return j(i===E.SECOND?rt:i===E.MILLISECOND?Gt:i===E.MICROSECOND?nt:st);case c.Decimal:return p(Q,{bitWidth:n,scale:o});case c.Interval:return p(i===C.DAY_TIME?ot:i===C.YEAR_MONTH?at:ct);case c.FixedSizeBinary:return p(mt,{stride:a});case c.Utf8:return I(ut);case c.LargeUtf8:return I(dt);case c.Binary:return I(it);case c.LargeBinary:return I(lt);case c.List:return U(ft);case c.LargeList:return U(ht);case c.Map:return U(u?It:gt);case c.FixedSizeList:return F(yt,{stride:a});case c.Struct:return F(xt,{names:e.children.map(y=>y.name)});case c.Dictionary:{let{id:y,keys:A}=e;return new bt({...d,validity:t.buffer(),values:t.buffer(A.values),dictionary:t.dictionary(y)})}case c.Union:{t.version<w.V5&&t.buffer();let y=e.mode===H.Sparse,A=t.buffer(Mt),Z=y?null:t.buffer(e.offsets),ee=t.visitAll(e.children),re=e.typeIds.reduce((Rt,ne,se)=>(Rt[ne]=se,Rt),{}),kt={...d,map:re,typeIds:A,offsets:Z,children:ee};return y?new $(kt):new vt(kt)}default:throw new Error(`Unsupported type: ${r}, (${R(c,r)})`)}}export{k as DateUnit,oe as Endianness,C as IntervalUnit,O as Precision,E as TimeUnit,c as Type,H as UnionMode,w as Version,Re as tableFromIPC};

dist/types/batch.d.ts

@@ -62,3 +62,4 @@ /**
     /**
-     * Extract an array of values within the given index range.
+     * Extract an array of values within the given index range. Unlike
+     * Array.slice, all arguments are required and may not be negative indices.
      * @param {number} start The starting index, inclusive
@@ -85,2 +86,17 @@ * @param {number} end The ending index, exclusive
 export class DirectBatch<T> extends Batch<T> {
+    /**
+     * Create a new column batch with direct value array access.
+     * @param {object} options
+     * @param {number} options.length The length of the batch
+     * @param {number} options.nullCount The null value count
+     * @param {Uint8Array} [options.validity] Validity bitmap buffer
+     * @param {import('./types.js').TypedArray} options.values Values buffer
+     */
+    constructor(options: {
+        length: number;
+        nullCount: number;
+        validity?: Uint8Array;
+        values: import("./types.js").TypedArray;
+    });
+    values: any;
 }
@@ -87,0 +103,0 @@ /**

package.json

 {
   "name": "@uwdata/flechette",
-  "version": "0.0.3",
+  "version": "0.0.4",
   "description": "Fast, lightweight access to Apache Arrow data.",
@@ -5,0 +5,0 @@ "keywords": [

src/batch.js

@@ -90,3 +90,4 @@ import { float64 } from './array-types.js';
   /**
-   * Extract an array of values within the given index range.
+   * Extract an array of values within the given index range. Unlike
+   * Array.slice, all arguments are required and may not be negative indices.
    * @param {number} start The starting index, inclusive
@@ -124,3 +125,20 @@ * @param {number} end The ending index, exclusive
   /**
-   * Extract an array of values within the given index range.
+   * Create a new column batch with direct value array access.
+   * @param {object} options
+   * @param {number} options.length The length of the batch
+   * @param {number} options.nullCount The null value count
+   * @param {Uint8Array} [options.validity] Validity bitmap buffer
+   * @param {import('./types.js').TypedArray} options.values Values buffer
+   */
+  constructor(options) {
+    super(options);
+    // underlying buffers may be padded, exceeding the logical batch length
+    // we trim the values array so we can safely access it directly
+    const { length, values } = this;
+    this.values = values.subarray(0, length);
+  }
+
+  /**
+   * Extract an array of values within the given index range. Unlike
+   * Array.slice, all arguments are required and may not be negative indices.
    * When feasible, a zero-copy subarray of a typed array is returned.
@@ -127,0 +145,0 @@ * @param {number} start The starting index, inclusive

No alert changes

Improved metrics

Total package byte prevSize

235121

increased by0.95%

Lines of code

6494

increased by0.74%

No dependency changes