[124] TiltLoader breaks in a module env

7 thoughts on “[124] TiltLoader breaks in a module env”

1. Hi, author of fflate here! The reason the async version exists in JSZip is beyond me, since it doesn’t actually offload the processing to other threads. On the other hand, if the assets in the ZIP are reasonably large (around 500kB), fflate will use worker threads to compress/decompress them faster.

   However, I looked at some of the demos using JSZip and they don’t seem to load that much data, so using the synchronous unzipping algorithm would be no different to the async version. You could even replace gunzip.js, which is used by the NRRD loader, with fflate because gunzip and unzip use the same core algorithm.

   Since it seems the ThreeJS team wants minified ES6 module scripts, I’ve hand-tuned a minified version of the library and pasted it below. It’s 4.4kB (2.2kB gzipped) and supports synchronous ZIP decompression (through the exported unzipSync method) and GZIP decompression (through gunzipSync).

   Usage:

   import { unzipSync, strFromU8 } from '../libs/fflate.module.min.js';
   let arrayBuffer = ... // From XHR, presumably
   const zip = unzipSync(new Uint8Array(arrayBuffer));
   
   // If we were using JSZip, the following would be:
   // zip.files['data.xml'].asUint8Array()
   const dataXmlU8 = zip['data.xml'];
   // zip.files['data.xml'].asArrayBuffer()
   const dataXmlBuffer = dataXmlU8.buffer;
   // zip.files['data.xml'].asText()
   const dataXmlText = strFromU8(dataXmlU8);

   import { gunzipSync } from '../libs/fflate.module.min.js';
   let arrayBuffer = ... // From XHR, presumably
   
   // If we were using gunzip.js, the following would be:
   // const gunzip = new Zlib.Gunzip(new Uint8Array(arrayBuffer));
   // const data = gunzip.decompress();
   const data = gunzipSync(new Uint8Array(arrayBuffer));

   Minified code (to put in fflate.module.min.js):

   /*!
   fflate - fast JavaScript compression/decompression
   <https://101arrowz.github.io/fflate>
   Licensed under MIT. https://github.com/101arrowz/fflate/blob/master/LICENSE
   Subset included: synchronous unzip and gunzip
   */
   "use strict";var r=Uint8Array,n=Uint16Array,e=Uint32Array,t=new r([0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0,0,0,0]),a=new r([0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,0,0]),i=new r([16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15]),f=function(r,e){for(var t=new n(31),a=0;a<31;++a)t[a]=e+=1<<r[a-1];return t},o=f(t,2);o[28]=258;for(var u=f(a,0),v=new n(32768),l=0;l<32768;++l){var c=(43690&l)>>>1|(21845&l)<<1;c=(61680&(c=(52428&c)>>>2|(13107&c)<<2))>>>4|(3855&c)<<4,v[l]=((65280&c)>>>8|(255&c)<<8)>>>1}var w=function(r,e){for(var t=r.length,a=0,i=new n(e);a<t;++a)++i[r[a]-1];var f=new n(1<<e),o=new n(e);for(a=0;a<e;++a)o[a]=o[a-1]+i[a-1]<<1;var u=15-e;for(a=0;a<t;++a)if(r[a])for(var l=a<<4|r[a],c=e-r[a],w=o[r[a]-1]++<<c,h=w|(1<<c)-1;w<=h;++w)f[v[w]>>>u]=l;return f},h=new r(288);for(l=0;l<144;++l)h[l]=8;for(l=144;l<256;++l)h[l]=9;for(l=256;l<280;++l)h[l]=7;for(l=280;l<288;++l)h[l]=8;var d=new r(32);for(l=0;l<32;++l)d[l]=5;var s=w(h,9),g=w(d,5),p=function(r){for(var n=r[0],e=1;e<r.length;++e)r[e]>n&&(n=r[e]);return n},y=function(r,n,e){var t=n/8>>0;return(r[t]|r[t+1]<<8)>>>(7&n)&e},b=function(r,n){var e=n/8>>0;return(r[e]|r[e+1]<<8|r[e+2]<<16)>>>(7&n)},x=function(r){return(r/8>>0)+(7&r&&1)},m=function(t,a,i){(null==a||a<0)&&(a=0),(null==i||i>t.length)&&(i=t.length);var f=new(t instanceof n?n:t instanceof e?e:r)(i-a);return f.set(t.subarray(a,i)),f},C=function(n,e,f){var v=n.length,l=!e||f,c=!f||f.i;f||(f={}),e||(e=new r(3*v));var h=function(n){var t=e.length;if(n>t){var a=new r(Math.max(2*t,n));a.set(e),e=a}},d=f.f||0,C=f.p||0,S=f.b||0,k=f.l,F=f.d,z=f.m,E=f.n;if(d&&!k)return e;var O=8*v;do{if(!k){f.f=d=y(n,C,1);var U=y(n,C+1,3);if(C+=3,!U){var A=n[(K=x(C)+4)-4]|n[K-3]<<8,D=K+A;if(D>v){if(c)throw"unexpected EOF";break}l&&h(S+A),e.set(n.subarray(K,D),S),f.b=S+=A,f.p=C=8*D;continue}if(1==U)k=s,F=g,z=9,E=5;else{if(2!=U)throw"invalid block type";var T=y(n,C,31)+257,M=y(n,C+10,15)+4,j=T+y(n,C+5,31)+1;C+=14;for(var q=new r(j),B=new r(19),G=0;G<M;++G)B[i[G]]=y(n,C+3*G,7);C+=3*M;var H=p(B),I=(1<<H)-1;if(!c&&C+j*(H+7)>O)break;var J=w(B,H);for(G=0;G<j;){var K,L=J[y(n,C,I)];if(C+=15&L,(K=L>>>4)<16)q[G++]=K;else{var N=0,P=0;for(16==K?(P=3+y(n,C,3),C+=2,N=q[G-1]):17==K?(P=3+y(n,C,7),C+=3):18==K&&(P=11+y(n,C,127),C+=7);P--;)q[G++]=N}}var Q=q.subarray(0,T),R=q.subarray(T);z=p(Q),E=p(R),k=w(Q,z),F=w(R,E)}if(C>O)throw"unexpected EOF"}l&&h(S+131072);for(var V=(1<<z)-1,W=(1<<E)-1,X=z+E+18;c||C+X<O;){var Y=(N=k[b(n,C)&V])>>>4;if((C+=15&N)>O)throw"unexpected EOF";if(!N)throw"invalid length/literal";if(Y<256)e[S++]=Y;else{if(256==Y){k=null;break}var Z=Y-254;if(Y>264){var $=t[G=Y-257];Z=y(n,C,(1<<$)-1)+o[G],C+=$}var _=F[b(n,C)&W],rr=_>>>4;if(!_)throw"invalid distance";if(C+=15&_,R=u[rr],rr>3&&($=a[rr],R+=b(n,C)&(1<<$)-1,C+=$),C>O)throw"unexpected EOF";l&&h(S+131072);for(var nr=S+Z;S<nr;S+=4)e[S]=e[S-R],e[S+1]=e[S+1-R],e[S+2]=e[S+2-R],e[S+3]=e[S+3-R];S=nr}}f.l=k,f.p=C,f.b=S,k&&(d=1,f.m=z,f.d=F,f.n=E)}while(!d);return S==e.length?e:m(e,0,S)},S=function(r,n){return r[n]|r[n+1]<<8},k=function(r,n){return(r[n]|r[n+1]
   

2. You have the spec, and you have reality. V8 and Gecko use a 64 bit unsigned integer to store the size of typed arrays, which means a theoretical max of a lot, but they both cap the actual size at 2³¹ on all devices I’ve tested. Node caps it at 2³². No Zip64 data is strictly necessary while still being possible in JavaScript.

   I understand that Zip64 is important, so I’ll add support for it. However, this might be a few days of work since it’s a much more messy specification.

3. My bad, updated the code in my most recent comment. I had forgotten to switch the CommonJS exports to ESM. UMD build should already be live on jsDelivr, I believe, but you could probably wrap the minified code to make the UMD build smaller too.

   On looking again at the source code here, it seems that a lot of compression libraries could be replaced by a single one, which is probably better to avoid fragmentation…Will create a more complete minified copy soon and create a PR.

   EDIT: Here are my (semi) final revisions for the versions:

   fflate.min.js (replaces gunzip.js, inflate.js, jszip.js):

   !function(f){typeof exports=="object"&&typeof module!="undefined"?module.exports=f():typeof define=='function'&&define.amd?define([],f):(typeof window!="undefined"?window:typeof global!="undefined"?global:typeof self!="undefined"&&self).fflate=f()}(function(_e){"use strict";var _e={},r=Uint8Array,n=Uint16Array,e=Uint32Array,t=new r([0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0,0,0,0]),a=new r([0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,0,0]),i=new r([16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15]),o=function(r,t){for(var a=new n(31),i=0;i<31;++i)a[i]=t+=1<<r[i-1];var o=new e(a[30]);for(i=1;i<30;++i)for(var f=a[i];f<a[i+1];++f)o[f]=f-a[i]<<5|i;return[a,o]},f=o(t,2),u=f[0],v=f[1];u[28]=258,v[258]=28;for(var c=o(a,0),l=c[0],w=(c[1],new n(32768)),h=0;h<32768;++h){var s=(43690&h)>>>1|(21845&h)<<1;s=(61680&(s=(52428&s)>>>2|(13107&s)<<2))>>>4|(3855&s)<<4,w[h]=((65280&s)>>>8|(255&s)<<8)>>>1}var d=function(r,e){for(var t=r.length,a=0,i=new n(e);a<t;++a)++i[r[a]-1];var o=new n(e);for(a=0;a<e;++a)o[a]=o[a-1]+i[a-1]<<1;var f=new n(1<<e),u=15-e;for(a=0;a<t;++a)if(r[a])for(var v=a<<4|r[a],c=e-r[a],l=o[r[a]-1]++<<c,h=l|(1<<c)-1;l<=h;++l)f[w[l]>>>u]=v;return f},p=new r(288);for(h=0;h<144;++h)p[h]=8;for(h=144;h<256;++h)p[h]=9;for(h=256;h<280;++h)p[h]=7;for(h=280;h<288;++h)p[h]=8;var g=new r(32);for(h=0;h<32;++h)g[h]=5;var y=d(p,9),b=d(g,5),x=function(r){for(var n=r[0],e=1;e<r.length;++e)r[e]>n&&(n=r[e]);return n},z=function(r,n,e){var t=n/8>>0;return(r[t]|r[t+1]<<8)>>>(7&n)&e},C=function(r,n){var e=n/8>>0;return(r[e]|r[e+1]<<8|r[e+2]<<16)>>>(7&n)},m=function(r){return(r/8>>0)+(7&r&&1)},S=function(t,a,i){(null==a||a<0)&&(a=0),(null==i||i>t.length)&&(i=t.length);var o=new(t instanceof n?n:t instanceof e?e:r)(i-a);return o.set(t.subarray(a,i)),o},F=function(n,e){var o=n.length,f=!e;e||(e=new r(3*o));var v,c,w,h,s=function(n){var t=e.length;if(n>t){var a=new r(Math.max(2*t,n));a.set(e),e=a}},p=0,g=0,F=0;if(p&&!v)return e;var E=8*o;do{p=z(n,g,1);var O=z(n,g+1,3);if(g+=3,O){if(1==O)v=y,c=b,w=9,h=5;else{if(2!=O)throw"invalid block type";var U=z(n,g,31)+257,k=z(n,g+10,15)+4,A=U+z(n,g+5,31)+1;g+=14;for(var D=new r(A),T=new r(19),M=0;M<k;++M)T[i[M]]=z(n,g+3*M,7);g+=3*k;var j=x(T),q=(1<<j)-1,B=d(T,j);for(M=0;M<A;){var G=B[z(n,g,q)];if(g+=15&G,(Y=G>>>4)<16)D[M++]=Y;else{var H=0,I=0;for(16==Y?(I=3+z(n,g,3),g+=2,H=D[M-1]):17==Y?(I=3+z(n,g,7),g+=3):18==Y&&(I=11+z(n,g,127),g+=7);I--;)D[M++]=H}}var J=D.subarray(0,U),K=D.subarray(U);w=x(J),h=x(K),v=d(J,w),c=d(K,h)}if(g>E)throw"unexpected EOF";f&&s(F+131072);for(var L=(1<<w)-1,N=(1<<h)-1;;){var P=(H=v[C(n,g)&L])>>>4;if((g+=15&H)>E)throw"unexpected EOF";if(!H)throw"invalid length/literal";if(P<256)e[F++]=P;else{if(256==P)break;var Q=P-254;if(P>264){var R=t[M=P-257];Q=z(n,g,(1<<R)-1)+u[M],g+=R}var V=c[C(n,g)&N],W=V>>>4;if(!V)throw"invalid distance";g+=15&V;K=l[W];if(W>3){R=a[W];K+=C(n,g)&(1<<R)-1,g+=R}if(g>E)throw"unexpected EOF";f&&s(F+131072);for(var X=F+Q;F<X;F+=4)e[F]=e[F-K],e[F+1]=e[F+1-K],e[F+2]=e[F+2-K],e[F+3]=e[F+3-K];F=X}}}else{var Y,Z=n[(Y=m(g)+4)-4]|n[Y-3]<<8,$=Y+Z;if($>o)throw"unexpected EOF";f&&s(F+Z),e.set(n.subarray(Y,$),F),F+=Z,g=8*$}}while(!p);return F==e.length?e:S(e,0,F)},E=function(r,n){return r[n]|r[n+1]<<8},O=function(r,n){return(r[n]|r[n+1]<<8|r[n+2]<<16)+2*(r[n+3]<<23)},U=function(r){if(31!=r[0]||139!=r[1]||8!=r[2])throw"invalid gzip data";var n=r[3],e=10;4&n&&(e+=r[10]|2+(r[11]<<8));for(var t=(n>>3&1)+(n>>4&1);<

4. Great! I’ve completed my first PR to migrate loaders from JSZip.js and gunzip.js to fflate.

   I change inflate to fflate in a separate PR when the first one was merged.

5. I’m not exactly sure what the resize option does, but it seems like an optimization more than anything. I checked its usage in the source code and it seems to change basically nothing; the only difference is allocating a new array of length LEN and copying bytes 0 to LEN into the new array vs. taking the subarray from 0 to LEN directly. If anything, the O(n) allocation + copy would be slower…

   I actually took a closer look and found this comment that indicates that it’s a memory optimization. This is applied by default in fflate because if it weren’t, memory would be about 2x worse.

6. I added the ZIP function into fflateDeflate.

   fflate-deflate.min.js:

   /*!
   fflate - fast JavaScript compression/decompression
   <https://101arrowz.github.io/fflate>
   Licensed under MIT. https://github.com/101arrowz/fflate/blob/master/LICENSE
   Subset included: synchronous deflate, synchronous zip
   */
   !function(f){typeof exports=="object"&&typeof module!="undefined"?module.exports=f():typeof define=='function'&&define.amd?define([],f):(typeof window!="undefined"?window:typeof global!="undefined"?global:typeof self!="undefined"&&self).fflateDeflate=f()}(function(){"use strict";var _e={},r=Uint8Array,n=Uint16Array,e=Uint32Array,t=new r([0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0,0,0,0]),f=new r([0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,0,0]),a=new r([16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15]),o=function(r,t){for(var f=new n(31),a=0;a<31;++a)f[a]=t+=1<<r[a-1];var o=new e(f[30]);for(a=1;a<30;++a)for(var v=f[a];v<f[a+1];++v)o[v]=v-f[a]<<5|a;return[f,o]},v=o(t,2),i=v[0],u=v[1];i[28]=258,u[258]=28;for(var l=o(f,0),c=(l[0],l[1]),s=new n(32768),h=0;h<32768;++h){var g=(43690&h)>>>1|(21845&h)<<1;g=(61680&(g=(52428&g)>>>2|(13107&g)<<2))>>>4|(3855&g)<<4,s[h]=((65280&g)>>>8|(255&g)<<8)>>>1}var w=function(r,e,t){for(var f=r.length,a=0,o=new n(e);a<f;++a)++o[r[a]-1];var v,i=new n(e);for(a=0;a<e;++a)i[a]=i[a-1]+o[a-1]<<1;if(t){v=new n(1<<e);var u=15-e;for(a=0;a<f;++a)if(r[a])for(var l=a<<4|r[a],c=e-r[a],h=i[r[a]-1]++<<c,g=h|(1<<c)-1;h<=g;++h)v[s[h]>>>u]=l}else for(v=new n(f),a=0;a<f;++a)v[a]=s[i[r[a]-1]++]>>>15-r[a];return v},d=new r(288);for(h=0;h<144;++h)d[h]=8;for(h=144;h<256;++h)d[h]=9;for(h=256;h<280;++h)d[h]=7;for(h=280;h<288;++h)d[h]=8;var m=new r(32);for(h=0;h<32;++h)m[h]=5;var p=w(d,9,0),y=w(m,5,0),M=function(r){return(r/8>>0)+(7&r&&1)},b=function(r,n,e){e<<=7&n;var t=n/8>>0;r[t]|=e,r[t+1]|=e>>>8},x=function(r,n,e){e<<=7&n;var t=n/8>>0;r[t]|=e,r[t+1]|=e>>>8,r[t+2]|=e>>>16},A=function(e,t){for(var f=[],a=0;a<e.length;++a)e[a]&&f.push({s:a,f:e[a]});var o=f.length,v=f.slice();if(!o)return[new r(0),0];if(1==o){var i=new r(f[0].s+1);return i[f[0].s]=1,[i,1]}f.sort((function(r,n){return r.f-n.f})),f.push({s:-1,f:25001});var u=f[0],l=f[1],c=0,s=1,h=2;for(f[0]={s:-1,f:u.f+l.f,l:u,r:l};s!=o-1;)u=f[f[c].f<f[h].f?c++:h++],l=f[c!=s&&f[c].f<f[h].f?c++:h++],f[s++]={s:-1,f:u.f+l.f,l:u,r:l};var g=v[0].s;for(a=1;a<o;++a)v[a].s>g&&(g=v[a].s);var w=new n(g+1),d=U(f[s-1],w,0);if(d>t){a=0;var m=0,p=d-t,y=1<<p;for(v.sort((function(r,n){return w[n.s]-w[r.s]||r.f-n.f}));a<o;++a){var M=v[a].s;if(!(w[M]>t))break;m+=y-(1<<d-w[M]),w[M]=t}for(m>>>=p;m>0;){var b=v[a].s;w[b]<t?m-=1<<t-w[b]++-1:++a}for(;a>=0&&m;--a){var x=v[a].s;w[x]==t&&(--w[x],++m)}d=t}return[new r(w),d]},U=function(r,n,e){return-1==r.s?Math.max(U(r.l,n,e+1),U(r.r,n,e+1)):n[r.s]=e},D=function(r){for(var e=r.length;e&&!r[--e];);for(var t=new n(++e),f=0,a=r[0],o=1,v=function(r){t[f++]=r},i=1;i<=e;++i)if(r[i]==a&&i!=e)++o;else{if(!a&&o>2){for(;o>138;o-=138)v(32754);o>2&&(v(o>10?o-11<<5|28690:o-3<<5|12305),o=0)}else if(o>3){for(v(a),--o;o>6;o-=6)v(8304);o>2&&(v(o-3<<5|8208),o=0)}for(;o--;)v(a);o=1,a=r[i]}return[t.subarray(0,f),e]},S=function(r,n){for(var e=0,t=0;t<n.length;++t)e+=r[t]*n[t];return e},T=function(r,n,e){var t=e.length,f=M(n+2);r[f]=255&t,r[f+1]=t>>>8,r[f+2]=255^r[f],r[f+3]=255^r[f+1];for(var a=0;a<t;++a)r[f+a+4]=e[a];return 8*(f+4+t)},k=function(r,e,o,v,i,u,l,c,s,h,g){b(e,g++,o),++i[256];for(var M=A(i,15),U=M[0],k=M[1],z=A(u,15),C=z[0],E=z[1],F=D(U),H=F[0],Y=F[1],j=D(C),q=j[0],B=j[1],G=new n(19),I=0;I<H.length;++I)G[31<

7. Side note: in case @sciecode is still working on this, fflate.unzlibSync(u8Data, new Uint8Array(uncompressedSize)) will work well for you. See the docs.

   Also, if you’re still looking into the EXR exporter, try fflateDeflate.zlibSync(u8Data, options). Docs

   yep, still plan on finalizing the exporter. I had already taken a note to update current deflate to fflate’s deflate, but thanks for the heads up 👍

Comments are closed.