feat: native character encoding with Effect streams

.changeset/quiet-encodings-stream.md

@@ -0,0 +1,10 @@
+---
+"effect": minor
+---
+
+Add CharacterEncoding with Unicode and table-driven legacy character codecs,
+typed conversion errors, and incremental encode/decode/transcode Effect streams.
+Mapping data is derived from iconv-lite; no runtime dependency is added.
+Import explicit codecs from effect/encoding/* for selective bundles, or opt into
+the complete effect/encoding/All registry. Custom registries support runtime
+labels without importing unrelated encodings.

packages/effect/benchmark/CharacterEncoding.alpha-results-2026-09-09.jsonl

@@ -0,0 +1,14 @@
+{"node":"v24.20.0","platform":"darwin","arch":"arm64","iconvLite":"1.0.0-alpha.2","nodeIconv":"3.0.1","rounds":5,"duration":150,"size":65536,"chunkSize":4093,"measuredAt":"2026-09-09","effectRevision":"53900c09f767ef35023a8754fc8ab6a92dfccc74","iconvLiteRevision":"2472166ea5a4825ca091b9550713c403852a566b"}
+{"workload":"utf8/encode","inputBytes":65538,"mibPerSecond":{"effect":491.52028864780533,"iconv-lite":487.3065537676007,"node-iconv":198.23883528429502},"pairedEffectVsIconvLitePercent":0.8811665241137856,"samples":{"effect":[487.72373111684834,495.53173759687303,489.9137047809593,491.52028864780533,495.4576050672845],"iconv-lite":[491.4682919924715,487.3065537676007,489.61414894276766,487.22700736248566,486.72315178998866],"node-iconv":[197.14760236603948,199.49964929111994,198.23883528429502,196.90770048565744,200.5828833419985]}}
+{"workload":"utf8/decode","inputBytes":65538,"mibPerSecond":{"effect":1262.835592037425,"iconv-lite":1261.0110678604856,"node-iconv":261.53770921148094},"pairedEffectVsIconvLitePercent":0.1446874038968593,"samples":{"effect":[1254.4439072913583,1287.4659830687588,1280.5942420461,1249.9832893727016,1262.835592037425],"iconv-lite":[1257.0326566589852,1280.7141164617603,1252.796215128341,1278.331506540169,1261.0110678604856],"node-iconv":[261.53770921148094,265.4101689812336,265.2946236099697,260.9424498050623,254.08753883082852]}}
+{"workload":"utf8->utf16le/stream","inputBytes":65538,"mibPerSecond":{"effect":657.6314940714428,"iconv-lite":319.68288987010806,"node-iconv":348.9781140475917},"pairedEffectVsIconvLitePercent":109.73263194972773,"samples":{"effect":[670.1177033877198,633.5526992814594,657.2186560690284,673.2601864599877,657.6314940714428],"iconv-lite":[333.2089272164591,329.5160452155511,300.413818103037,319.68288987010806,313.55706928289305],"node-iconv":[349.07676966060524,348.9781140475917,335.6774218295154,348.9714292444394,350.33247882165466]}}
+{"workload":"utf16le/encode","inputBytes":65538,"mibPerSecond":{"effect":14250.897341624406,"iconv-lite":510.6277936613516,"node-iconv":228.11491181466957},"pairedEffectVsIconvLitePercent":2716.330120697302,"samples":{"effect":[14250.897341624406,14571.22126925651,14370.8588631969,14228.573944144648,14219.89079671459],"iconv-lite":[499.9608197219404,517.3832840891815,510.6277936613516,503.71877924290476,513.1482998813789],"node-iconv":[230.04093484740497,231.396575924199,228.11491181466957,221.00308908856783,218.17818085375754]}}
+{"workload":"utf16le/decode","inputBytes":89370,"mibPerSecond":{"effect":1998.4427989590167,"iconv-lite":1991.503740819421,"node-iconv":389.42872846981356},"pairedEffectVsIconvLitePercent":-0.8340274995004449,"samples":{"effect":[2007.0774388437233,1998.9181470312617,1974.894051967407,1998.4427989590167,1983.765722108115],"iconv-lite":[2037.858093710382,1939.970096891375,1991.503740819421,2042.420669401198,1946.458912537985],"node-iconv":[392.9553142205567,374.6649838878996,382.824906105301,389.42872846981356,394.27337437596395]}}
+{"workload":"cp1251/encode","inputBytes":65541,"mibPerSecond":{"effect":745.7015336421241,"iconv-lite":616.481847556986,"node-iconv":253.0278021113857},"pairedEffectVsIconvLitePercent":20.86273663655902,"samples":{"effect":[727.808644290426,762.3032294412511,745.7015336421241,743.1701906512748,747.3737188808433],"iconv-lite":[616.481847556986,601.1924986884432,616.9821686931433,624.5517005034703,590.837051745732],"node-iconv":[254.81836079965692,253.0278021113857,257.43326310137167,252.36644288048333,250.7366434413001]}}
+{"workload":"cp1251/decode","inputBytes":37452,"mibPerSecond":{"effect":789.5793631593011,"iconv-lite":220.8062096964244,"node-iconv":204.39009558652057},"pairedEffectVsIconvLitePercent":258.01636102577146,"samples":{"effect":[789.5793631593011,787.2317523394315,815.3914130440828,791.0704277116477,788.3203380325908],"iconv-lite":[216.94013350983488,213.5966953427877,230.91715900368112,220.95929511296922,220.8062096964244],"node-iconv":[200.4181111890772,200.61519216723337,204.80611268252125,204.39009558652057,207.32852865131292]}}
+{"workload":"cp1251->utf8/stream","inputBytes":37452,"mibPerSecond":{"effect":216.89630661857703,"iconv-lite":263.9984646852462,"node-iconv":215.68419157019088},"pairedEffectVsIconvLitePercent":-17.77758201288143,"samples":{"effect":[216.89630661857703,207.6634907872032,214.28319463871557,217.84568663368773,219.67993684930033],"iconv-lite":[259.72299792114376,263.9984646852462,241.9106529341241,271.853389324671,267.17766544364656],"node-iconv":[215.68419157019088,214.68543834283858,206.36355136046433,217.62476503609187,216.66297514380543]}}
+{"workload":"cp932/encode","inputBytes":65550,"mibPerSecond":{"effect":591.0594297853604,"iconv-lite":478.20292164435034,"node-iconv":208.50020403295073},"pairedEffectVsIconvLitePercent":23.60012936619904,"samples":{"effect":[594.3390545424661,587.4176203602999,591.0594297853604,588.9869048667623,594.7254994776059],"iconv-lite":[479.74938914049557,477.16748915466076,478.20292164435034,475.8313589186596,485.90270042233016],"node-iconv":[209.03268367836185,203.73557514880383,206.11451202789243,208.50020403295073,210.39960614152767]}}
+{"workload":"cp932/decode","inputBytes":44574,"mibPerSecond":{"effect":199.96136455861554,"iconv-lite":264.1314083255627,"node-iconv":222.36843852024364},"pairedEffectVsIconvLitePercent":-24.85246955771433,"samples":{"effect":[195.76822124118036,198.48823047909008,201.2206031296855,202.94118627449336,199.96136455861554],"iconv-lite":[263.7065855448581,264.1314083255627,265.6874227613002,270.8836717421465,259.5333358537847],"node-iconv":[222.95479149197527,218.27148254177067,222.36843852024364,221.2830287020514,226.46038336824694]}}
+{"workload":"cp932->utf8/stream","inputBytes":44574,"mibPerSecond":{"effect":130.75206511867324,"iconv-lite":169.6616813990804,"node-iconv":226.51757664472834},"pairedEffectVsIconvLitePercent":-22.79364653225342,"samples":{"effect":[130.75206511867324,129.87776991158245,131.67616954368455,131.0096023291126,127.19776950254402],"iconv-lite":[169.24462127762078,169.6616813990804,167.11663757137285,169.68759233505654,169.7071011407984],"node-iconv":[222.35071886784814,227.347458479794,225.8304671647944,226.51757664472834,227.92357094112998]}}
+{"workload":"gb18030/encode","inputBytes":65538,"mibPerSecond":{"effect":278.82448629485737,"iconv-lite":282.38233230661336,"node-iconv":180.03809128497912},"pairedEffectVsIconvLitePercent":-2.12593113304409,"samples":{"effect":[276.37907838989105,278.70648156500636,278.82448629485737,280.095490291118,279.3516231436458],"iconv-lite":[282.38233230661336,275.17993225998293,284.96239112120674,278.02981677159806,285.43079806620227],"node-iconv":[179.94076487212217,176.99276846887153,181.2073096298429,180.03809128497912,183.49387438913644]}}
+{"workload":"gb18030/decode","inputBytes":50974,"mibPerSecond":{"effect":176.78754278882036,"iconv-lite":216.76496486198823,"node-iconv":204.63484755565074},"pairedEffectVsIconvLitePercent":-17.869298892817053,"samples":{"effect":[176.78754278882036,174.62203064008222,177.525868142612,180.42215126037033,172.8164455352665],"iconv-lite":[215.05178924305517,216.76496486198823,216.15043552463482,217.01219726063349,217.6127505799458],"node-iconv":[203.5139759005652,205.42558025918112,203.60816074532596,204.63484755565074,204.8371533560087]}}

packages/effect/benchmark/CharacterEncoding.alpha-results.jsonl

@@ -0,0 +1,14 @@
+{"node":"v24.20.0","platform":"darwin","arch":"arm64","iconvLite":"1.0.0-alpha.2","nodeIconv":"3.0.1","rounds":5,"duration":150,"size":65536,"chunkSize":4093}
+{"workload":"utf8/encode","inputBytes":65538,"mibPerSecond":{"effect":487.51527066214373,"iconv-lite":488.8025024385244,"node-iconv":197.61163295845526},"pairedEffectVsIconvLitePercent":-0.33178019967873285,"samples":{"effect":[492.48493579441663,487.51527066214373,487.1807525198992,488.6067957924329,485.39916556189405],"iconv-lite":[484.92145364556393,484.17754301759567,488.8025024385244,493.38255842430425,488.90685527100277],"node-iconv":[194.31469451860485,197.61163295845526,198.51360350018507,197.3877038205831,199.83602041417055]}}
+{"workload":"utf8/decode","inputBytes":65538,"mibPerSecond":{"effect":1275.4818266831865,"iconv-lite":1274.4830994580968,"node-iconv":263.4414064908368},"pairedEffectVsIconvLitePercent":-0.7883433462487655,"samples":{"effect":[1275.4818266831865,1263.1504906100392,1283.5246254019605,1278.45627627648,1265.3582733177013],"iconv-lite":[1274.4830994580968,1273.187580183683,1245.3422442692722,1295.151476406623,1283.3525285932428],"node-iconv":[256.5579076257914,263.6808485323972,269.2115987028487,258.4357132341747,263.4414064908368]}}
+{"workload":"utf8->utf16le/stream","inputBytes":65538,"mibPerSecond":{"effect":299.1140633542997,"iconv-lite":331.5303624200096,"node-iconv":355.0465268964054},"pairedEffectVsIconvLitePercent":-8.529761004697056,"samples":{"effect":[292.6870658152856,299.1140633542997,298.75090699384486,304.7913820231815,303.25161484757683],"iconv-lite":[339.1905302074942,339.33382344111914,301.12377003106025,328.8245037258651,331.5303624200096],"node-iconv":[348.6801757151757,355.0465268964054,360.97314838925877,371.06708912522697,350.9058178339048]}}
+{"workload":"utf16le/encode","inputBytes":65538,"mibPerSecond":{"effect":486.0524601700659,"iconv-lite":509.39696302235706,"node-iconv":224.11231307630376},"pairedEffectVsIconvLitePercent":-4.180062927883698,"samples":{"effect":[475.5868113546589,492.11259335138857,486.0524601700659,489.2442505679895,483.403286791313],"iconv-lite":[500.32228420410206,511.0830296247498,507.2560836731207,509.50179392067105,509.39696302235706],"node-iconv":[224.11231307630376,226.63461491778804,219.83664677166564,226.50634679832248,223.1387037667954]}}
+{"workload":"utf16le/decode","inputBytes":89370,"mibPerSecond":{"effect":1974.2650325803488,"iconv-lite":1976.6177170896224,"node-iconv":396.56798191322537},"pairedEffectVsIconvLitePercent":-0.11902577260805014,"samples":{"effect":[1968.8062374246695,2002.111931048082,1976.4541183970798,1974.2650325803488,1934.0457006916865],"iconv-lite":[1951.456218997974,1967.450705419892,1982.4632748133108,1976.6177170896224,1988.282269810045],"node-iconv":[396.56798191322537,397.71742723246246,393.6334293761263,397.4286423982321,386.80394634139816]}}
+{"workload":"cp1251/encode","inputBytes":65541,"mibPerSecond":{"effect":754.9472940630842,"iconv-lite":603.1280606550656,"node-iconv":253.78275494969438},"pairedEffectVsIconvLitePercent":24.507066638204723,"samples":{"effect":[734.2333025237033,756.2993958274681,740.5119909091761,756.075622404098,754.9472940630842],"iconv-lite":[589.712152369033,609.546702744223,610.296423406723,599.1968253108902,603.1280606550656],"node-iconv":[255.20221804532557,253.78275494969438,253.98308323041806,252.88327866061434,253.22520444657604]}}
+{"workload":"cp1251/decode","inputBytes":37452,"mibPerSecond":{"effect":797.5737759564122,"iconv-lite":222.28177647296602,"node-iconv":207.46442109597788},"pairedEffectVsIconvLitePercent":259.4118409651573,"samples":{"effect":[796.5701249101,807.8954400595376,788.1568106664396,797.5737759564122,797.9187471024125],"iconv-lite":[222.97661480379188,222.28177647296602,222.63401097934545,221.73001811947174,222.00680560765542],"node-iconv":[209.12299846374927,209.2797071493059,203.53206181427447,207.46442109597788,206.1784295428369]}}
+{"workload":"cp1251->utf8/stream","inputBytes":37452,"mibPerSecond":{"effect":215.88418127871518,"iconv-lite":144.50353105530715,"node-iconv":216.65978620938384},"pairedEffectVsIconvLitePercent":46.527630112019104,"samples":{"effect":[209.28197413265508,215.88418127871518,220.3245850799632,218.65172191585555,211.7375994835271],"iconv-lite":[171.87459293371214,140.57148543733797,142.9014890248983,167.55401714977157,144.50353105530715],"node-iconv":[213.75464448478004,216.65978620938384,220.72852205576365,220.47782202252074,216.5382968112042]}}
+{"workload":"cp932/encode","inputBytes":65550,"mibPerSecond":{"effect":413.4371237570646,"iconv-lite":484.5307260109217,"node-iconv":209.82972801516834},"pairedEffectVsIconvLitePercent":-14.67267160519652,"samples":{"effect":[413.4371237570646,408.87881329941234,410.4207739836271,416.6310341518153,418.5048823177263],"iconv-lite":[484.5307260109217,485.30763634006973,486.4736117975467,482.68021178151827,470.6264382449228],"node-iconv":[205.13819560788795,209.82972801516834,210.24623186634244,210.289822348648,206.885054927634]}}
+{"workload":"cp932/decode","inputBytes":44574,"mibPerSecond":{"effect":199.33447297102433,"iconv-lite":265.7361178918456,"node-iconv":224.89265749480842},"pairedEffectVsIconvLitePercent":-24.5151442535934,"samples":{"effect":[201.47870654037212,199.07491833748023,199.83997544874617,199.33447297102433,198.40700907742067],"iconv-lite":[266.91275295967347,262.5901307854984,263.9994376559414,265.7361178918456,269.1925367470487],"node-iconv":[224.89265749480842,226.44524064552374,224.5916468454687,224.34000404456975,226.95882599135274]}}
+{"workload":"cp932->utf8/stream","inputBytes":44574,"mibPerSecond":{"effect":127.51363548402524,"iconv-lite":171.96905545056143,"node-iconv":230.26009259623632},"pairedEffectVsIconvLitePercent":-24.23683992573241,"samples":{"effect":[122.76006588288672,128.14355652740363,127.07218972898934,130.28919075921485,127.51363548402524],"iconv-lite":[160.46056484270315,175.5622802720143,162.47955486680135,171.96905545056143,173.031709470168],"node-iconv":[228.28254038914008,232.84161029785344,230.47606925003998,230.26009259623632,226.05844194022464]}}
+{"workload":"gb18030/encode","inputBytes":65538,"mibPerSecond":{"effect":268.5319634199401,"iconv-lite":285.33597162097755,"node-iconv":182.3467758074069},"pairedEffectVsIconvLitePercent":-6.174815636956543,"samples":{"effect":[264.91352715772666,263.9845855762561,268.5319634199401,270.3925477326028,270.8145910742895],"iconv-lite":[282.34799532360176,280.89272030065734,286.56636782558934,285.33597162097755,289.90866925791124],"node-iconv":[181.23071236908052,182.3467758074069,185.30794812714578,182.17163912766387,185.5062792174846]}}
+{"workload":"gb18030/decode","inputBytes":50974,"mibPerSecond":{"effect":174.542398824946,"iconv-lite":212.23791281656963,"node-iconv":212.1568957676428},"pairedEffectVsIconvLitePercent":-17.332953133519723,"samples":{"effect":[175.45081485651318,174.02902800676316,174.542398824946,176.52259353175856,174.12061122489786],"iconv-lite":[212.23791281656963,213.66620085725546,210.82546074868486,210.90865156568847,213.65469863263712],"node-iconv":[208.3579188924209,201.5647723274151,212.1568957676428,214.50598700845978,213.27275044510344]}}

packages/effect/benchmark/CharacterEncoding.bundle-results.jsonl

@@ -0,0 +1,6 @@
+{"node":"v24.20.0","esbuild":"0.28.2","rolldown":"1.2.7"}
+{"entry":"operators-only","esbuild":{"bytes":14199,"gzip":5202,"mappingModules":0},"rolldown":{"bytes":13290,"gzip":4796,"mappingModules":0}}
+{"entry":"utf8-only","esbuild":{"bytes":16987,"gzip":6326,"mappingModules":0},"rolldown":{"bytes":16028,"gzip":5831,"mappingModules":0}}
+{"entry":"cp1251-only","esbuild":{"bytes":16646,"gzip":6333,"mappingModules":1},"rolldown":{"bytes":15711,"gzip":5904,"mappingModules":1}}
+{"entry":"restricted-registry","esbuild":{"bytes":97801,"gzip":33229,"mappingModules":1},"rolldown":{"bytes":94242,"gzip":30731,"mappingModules":1}}
+{"entry":"all-encodings","esbuild":{"bytes":484458,"gzip":292691,"mappingModules":89},"rolldown":{"bytes":481040,"gzip":289117,"mappingModules":89}}