互转的方法相信网上一搜有一大堆，都是比较简单的互转没有考虑到中文或者是偏僻的中文。
理论上来说，互转的话，转过去再转回来应该是同一个东西，打印的内容应该一致，我们来尝试一下网上给出的方法：
function Uint8ArrayToString(fileData){
var dataString = "";
for (var i = 0; i < fileData.length; i++) {
dataString += String.fromCharCode(fileData[i]);
}
return dataString
}
function stringToUint8Array(str){
var arr = [];
for (var i = 0, j = str.length; i < j; ++i) {
arr.push(str.charCodeAt(i));
}
var tmpUint8Array = new Uint8Array(arr);
return tmpUint8Array
}
我们的实验代码也很简单：
var before= "𠮷中a";
var after = Uint8ArrayToString(stringToUint8Array(origin))
console.log(before,"===",after,before===after);
打印结果：
结果
什么鬼！！完全不一致。
不过要是我们的字符串中只有简单字符，这种转换也够我们使用了。
再解开谜题之前，我们先复习一下功课。
总结成图表就是下面这个：
二进制表示
UTF-8：
1字节 0xxxxxxx 提供7个有效位
2字节 110xxxxx 10xxxxxx 提供11个有效位
3字节 1110xxxx 10xxxxxx 10xxxxxx 提供16个有效位
4字节 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx 提供21个有效位，下面两个不大用到
5字节 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 提供26个有效位
6字节 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 提供31个有效位
UTF-16：
小于等于0xFFFF的字符
直接用0xFFFF 填充码点
大于0xFFFF的字符需要把码点减去0x10000,然后再填充到下面两个UTF-16中，即32位
1101 10xx xxxxxxxx 1101 11xx xxxxxxxx
即第一个开头必须是1101 10;第二个开头必须是1101 11 提供20个有效位
上面的x代表了码点填充的有效位置，可1可0。
我们的JavaScript/Java语言都是使用UTF-16作为字符编码格式。
1 对中文字单独分析
let ch1= '中';
let ch2 = '𠮷';
console.log("中.length="+ch1.length+",unicode="+ch1.charCodeAt(0));
console.log("𠮷.length="+ch2 .length+",unicode="+ch2 .charCodeAt(0)+",unicode2="+ch2 .charCodeAt(1));
分析汉字
同样是中文，由于'中'的unicode码点小于0x10000，所以用一个UTF-16就可以表示，而'𠮷'的unicode大于0xFFFF，所以用了两个UTF-16,我们看到他的length是2.
而我们的Uin8Array单个元素只能存最大0xFF，而中文字都大于0xFF，用上面网上搜的方法不足以存下完整数据，所以这就导致一次来回转换数据丢失了。
一般情况下，我们想用Uint8Array保存的是字符串的UTF-8编码时候的序列。所以对任意字符串，我们首先要取到每个字的unicode码点，然后再对照UTF-8的编码方式填入Uin8Array。
取出来的时候当然也是先把UTF-8的编码转成unicode码点，然后按照UTF-16的形式转成字符串。
用图表示就是：
string 转 Uint8Array
char --> UTF-16 -> Unicode->UTF8 -> Uin8Array
Uint8Array 转string
就是上面的序列反过来
由于我是在cocos creator环境下使用TypeScript，我编写了如下的类：
interface UnicodeOk {
unicode?: number
ok: boolean
}
interface UnicodeLen {
unicode?: number
len: number
}
export class BufferBigEndian {
buffer: number[];//uint8array
private readOffset: number;
constructor() {
this.buffer = [];
this.readOffset = 0;
}
initWithUint8Array(array: ArrayLike<number>, len?: number) {
len = len || array.length;
this.buffer = [];
for (let i = 0; i < len && i < array.length; i++)
this.buffer[i] = array[i];
this.readOffset = 0;
}
getUint8(): number {
// console.log("getUint8 readOffset=" + this.readOffset + ",total=" + this.buffer.length);
if (this.readOffset + 1 > this.buffer.length)
return null;
return this.buffer[this.readOffset++];
}
pushUint8(value: number): void {
if (value > 255)
throw Error("BufferBigEndian pushUint8 value need <= 255");
this.buffer.push(value);
}
getUint16(): number {
if (this.readOffset + 2 > this.buffer.length)
return null;
let uint1 = this.getUint8();
let uint2 = this.getUint8();
return (uint1 << 8) | uint2;
}
pushUint16(value: number): void {
this.pushUint8((value >> 8) & 0xFF);
this.pushUint8(value & 0xFF);
}
getUint32(): number {
/*
实验：
let a = 34302;
let b = 32200;
console.log("a << 16 | b=" + (a << 16 | b) + ",a * 65536 + b=" + (a * 65536 + b)
+ ",(a * 65536) | b=" + ((a * 65536) | b) + ",(a << 16) + b=" + ((a << 16) + b));
实际打印结果 a << 16 | b=-2046919224,a * 65536 + b=2248048072,
(a * 65536) | b=-2046919224,(a << 16) + b=-2046919224
结果表明 js的位运算仅限在4字节32位。如果想要扩展到8字节64位，那么只能用乘除加减的方法。
*/
if (this.readOffset + 4 > this.buffer.length)
return null;
let uint1 = this.getUint16();
let uint2 = this.getUint16();
return uint1 * 65536 + uint2;
}
pushUint32(value: number): void {
/*
这里可以直接这样使用。
buf.initWithUint8Array([]);
buf.pushUint32(2248048072);
console.log("2248048072 uint16 =" + buf.getUint16() + ",uint16=" + buf.getUint16());
buf.changeReadOffset(-4);
console.log("2248048072 uint32 =" + buf.getUint32());
*/
this.pushUint16((value >> 16) & 0xFFFF);
this.pushUint16(value & 0xFFFF);
}
getInt64(): number {
let hi = this.getUint32();
// console.log("hi=" + hi);
let lo = this.getUint32();
// console.log("lo=" + lo);
if (hi >> 31 == 1)
return -(hi * 4294967296 + lo);
return hi * 4294967296 + lo;
}
pushUnicodeWithUtf8(value: number): void {
// console.log("encodeUnicode value=" + value);
if (value <= 0x7F) {
this.pushUint8(value);
} else if (value <= 0xFF) {
this.pushUint8((value >> 6) | 0xC0);
this.pushUint8((value & 0x3F) | 0x80);
} else if (value <= 0xFFFF) {
this.pushUint8((value >> 12) | 0xE0);
this.pushUint8(((value >> 6) & 0x3F) | 0x80);
this.pushUint8((value & 0x3F) | 0x80);
} else if (value <= 0x1FFFFF) {
this.pushUint8((value >> 18) | 0xF0);
this.pushUint8(((value >> 12) & 0x3F) | 0x80);
this.pushUint8(((value >> 6) & 0x3F) | 0x80);
this.pushUint8((value & 0x3F) | 0x80);
} else if (value <= 0x3FFFFFF) {//后面两种情况一般不大接触到，看了下protobuf.js中的utf8，他没去实现
this.pushUint8((value >> 24) | 0xF8);
this.pushUint8(((value >> 18) & 0x3F) | 0x80);
this.pushUint8(((value >> 12) & 0x3F) | 0x80);
this.pushUint8(((value >> 6) & 0x3F) | 0x80);
this.pushUint8((value & 0x3F) | 0x80);
} else {//Math.pow(2, 32) - 1
this.pushUint8((value >> 30) & 0x1 | 0xFC);
this.pushUint8(((value >> 24) & 0x3F) | 0x80);
this.pushUint8(((value >> 18) & 0x3F) | 0x80);
this.pushUint8(((value >> 12) & 0x3F) | 0x80);
this.pushUint8(((value >> 6) & 0x3F) | 0x80);
this.pushUint8((value & 0x3F) | 0x80);
}
}
getUnicodeWithUtf8(): UnicodeLen {
let result;
let start = this.getUint8();
if (start == null)
return null;
let n = 7;
while (((start >> n) & 1) == 1)
n--;
n = 7 - n;
if (n == 0)
result = start;
else
result = start & (Math.pow(2, 7 - n) - 1);
// console.log("start=" + start.toString(16).toUpperCase() + ",n=" + n + ",result=" + result);
for (let i = 1; i < n; i++) {
let follow = this.getUint8();
if ((follow & 0x80) == 0x80) {
result = result << 6 | (follow & 0x3F);
} else {
//不是标准的UTF8字符串。。我们直接取第一个。
result = start;
this.changeReadOffset(1 - n);
n = 0;
break;
}
}
return {unicode: result, len: n == 0 ? 1 : n};
}
parseUnicodeFromUtf16(ch1: number, ch2: number): UnicodeOk {
if ((ch1 & 0xFC00) === 0xD800 && (ch2 & 0xFC00) === 0xDC00) {
return {unicode: (((ch1 & 0x3FF) << 10) | (ch2 & 0x3FF)) + 0x10000, ok: true}
}
return {ok: false}
}
pushStringWithUtf8(value: string): number {
let oldlen = this.buffer.length;
for (let i = 0; i < value.length; i++) {
let ch1 = value.charCodeAt(i);
// console.log("pushStringWithUtf8 i=" + i + ",ch1=" + ch1 + "," + ch1.toString(16).toUpperCase());
if (ch1 < 128)
this.pushUnicodeWithUtf8(ch1);
else if (ch1 < 2048) {
this.pushUnicodeWithUtf8(ch1);
} else {
let ch2 = value.charCodeAt(i + 1);
// console.log("pushStringWithUtf8 i=" + i + ",ch2=" + ch2 + "," + ch2.toString(16).toUpperCase());
let unicodeOk = this.parseUnicodeFromUtf16(ch1, ch2);
// console.log("unicodeOk=" + JSON.stringify(unicodeOk));
if (unicodeOk.ok) {
this.pushUnicodeWithUtf8(unicodeOk.unicode);
i++;
} else {
this.pushUnicodeWithUtf8(ch1);
}
}
}
return this.buffer.length - oldlen;
}
getStringWithUtf8(len: number): string {
if (len < 1)
return "";
// console.log("this.readOffset=" + this.readOffset + ",len=" + len + ",total=" + this.buffer.length);
if (this.readOffset + len > this.buffer.length)
return "";
let str = "";
let read = 0;
while (read < len) {
let unicodeLen = this.getUnicodeWithUtf8();
if (!unicodeLen) {
break;
}
read += unicodeLen.len;
// console.log("read unicode=" + JSON.stringify(unicodeLen));
if (unicodeLen.unicode < 0x10000) {
str += String.fromCharCode(unicodeLen.unicode);
} else {
let minus = unicodeLen.unicode - 0x10000;
let ch1 = (minus >> 10) | 0xD800;
let ch2 = (minus & 0x3FF) | 0xDC00;
str += String.fromCharCode(ch1, ch2)
}
}
// console.log("getStringWithUtf8 len=" + len + ",str.len=" + str.length);
return str;
}
pushStringWithUtf16(value: string): number {
let oldlen = this.buffer.length;
for (let i = 0; i < value.length; i++) {
let ch = value[i].charCodeAt(0);
this.pushUint16(ch);
}
return this.buffer.length - oldlen;
}
getStringWithUtf16(len: number): string {
if (len < 1)
return "";
if (this.readOffset + len > this.buffer.length || len % 2 != 0)
return "";
let str = "";
for (let i = 0; i < len; i += 2) {
let ch1 = this.getUint16();
let ch2 = this.getUint16();
str += String.fromCharCode(ch1, ch2);
}
return str;
}
pushUint8List(val: ArrayLike<number>) {
for (let i = 0; i < val.length; i++)
this.pushUint8(val[i]);
}
getUint8List(len?: number): Uint8Array {
len = len || this.buffer.length;
return new Uint8Array(this.buffer.slice(this.readOffset, this.readOffset + len));
}
tostring(): string {
let result = "";
for (let i = 0; i < this.buffer.length; i++) {
let ch = this.buffer[i].toString(16);
result += ch.length == 1 ? "0" + ch.toUpperCase() : ch.toUpperCase();
}
return result;
}
toUint8Array(): Uint8Array {
let array = new Uint8Array(this.buffer.length);
for (let i = 0; i < this.buffer.length; i++)
array[i] = this.buffer[i];
return array;
}
changeReadOffset(len: number) {
this.readOffset = Math.max(0, Math.min(this.buffer.length, this.readOffset + len))
}
}
测试代码
let str0 = '𠮷';
let str1 = '中';//𝌆𠮷
let str2 = 'a';
let strAll = str0 + str1 + str2;
let buf = new BufferBigEndian();
let len = buf.pushStringWithUtf8(strAll);
console.log("buffer HEX=" + buf.tostring(), "encodeURI=" + encodeURI(strAll));
console.log("转换前:" + strAll + ",转换后:" + buf.getStringWithUtf8(len));
打印结果
这就是我们要的结果了。
