Use new text encoder for stream frames

This pattern was introduced with Exec and fixes the multibyte encoding
issue that the native window.atob has.
This commit is contained in:
Michael Lange
2020-03-30 12:50:43 -07:00
parent b3a23c9b07
commit c30a153d1f
2 changed files with 21 additions and 6 deletions

View File

@@ -1,3 +1,8 @@
import { TextDecoderLite } from 'text-encoder-lite';
import base64js from 'base64-js';
const decoder = new TextDecoderLite('utf-8');
/**
*
* @param {string} chunk
@@ -15,7 +20,7 @@ export function decode(chunk) {
const frames = lines.map(line => JSON.parse(line)).filter(frame => frame.Data);
if (frames.length) {
frames.forEach(frame => (frame.Data = b64DecodeUnicode(frame.Data)));
frames.forEach(frame => (frame.Data = b64decode(frame.Data)));
return {
offset: frames[frames.length - 1].Offset,
message: frames.mapBy('Data').join(''),
@@ -25,9 +30,6 @@ export function decode(chunk) {
return {};
}
function b64DecodeUnicode(str) {
// from bytestream, to percent-encoding, to original string.
return decodeURIComponent(window.atob(str).split('').map(function(c) {
return '%' + ('00' + c.charCodeAt(0).toString(16)).slice(-2);
}).join(''));
function b64decode(str) {
return decoder.decode(base64js.toByteArray(str));
}

View File

@@ -1,5 +1,10 @@
import { module, test } from 'qunit';
import { decode } from 'nomad-ui/utils/stream-frames';
import { TextEncoderLite } from 'text-encoder-lite';
import base64js from 'base64-js';
const Encoder = new TextEncoderLite('utf-8');
const encode = str => base64js.fromByteArray(Encoder.encode(str));
module('Unit | Util | stream-frames', function() {
const { btoa } = window;
@@ -31,6 +36,14 @@ module('Unit | Util | stream-frames', function() {
in: '',
out: {},
},
{
name: 'Multi-byte unicode',
in: `{"Offset":1,"Data":"${encode('ワンワン 🐶')}"}`,
out: {
offset: 1,
message: 'ワンワン 🐶',
},
},
];
decodeTestCases.forEach(testCase => {