Delete utils.js
Browse files
utils.js
DELETED
@@ -1,115 +0,0 @@
|
|
1 |
-
const { v4: uuidv4 } = require('uuid');
|
2 |
-
const zlib = require('zlib');
|
3 |
-
const $root = require('./message.js');
|
4 |
-
|
5 |
-
const regex = /<\|BEGIN_SYSTEM\|>.*?<\|END_SYSTEM\|>.*?<\|BEGIN_USER\|>.*?<\|END_USER\|>/s;
|
6 |
-
|
7 |
-
async function stringToHex(messages, modelName) {
|
8 |
-
const formattedMessages = messages.map((msg) => ({
|
9 |
-
...msg,
|
10 |
-
role: msg.role === 'user' ? 1 : 2,
|
11 |
-
message_id: uuidv4(),
|
12 |
-
}));
|
13 |
-
|
14 |
-
const message = {
|
15 |
-
messages: formattedMessages,
|
16 |
-
instructions: {
|
17 |
-
instruction: 'Always respond in 中文',
|
18 |
-
},
|
19 |
-
projectPath: '/path/to/project',
|
20 |
-
model: {
|
21 |
-
name: modelName,
|
22 |
-
empty: '',
|
23 |
-
},
|
24 |
-
requestId: uuidv4(),
|
25 |
-
summary: '',
|
26 |
-
conversationId: uuidv4(),
|
27 |
-
};
|
28 |
-
const errMsg = $root.ChatMessage.verify(message);
|
29 |
-
if (errMsg) throw Error(errMsg);
|
30 |
-
|
31 |
-
const messageInstance = $root.ChatMessage.create(message);
|
32 |
-
|
33 |
-
const buffer = $root.ChatMessage.encode(messageInstance).finish();
|
34 |
-
const hexString = (buffer.length.toString(16).padStart(10, '0') + buffer.toString('hex')).toUpperCase();
|
35 |
-
|
36 |
-
return Buffer.from(hexString, 'hex');
|
37 |
-
}
|
38 |
-
|
39 |
-
async function chunkToUtf8String(chunk) {
|
40 |
-
try {
|
41 |
-
let hex = Buffer.from(chunk).toString('hex');
|
42 |
-
console.log("debug [42] :", hex)
|
43 |
-
|
44 |
-
let offset = 0;
|
45 |
-
let results = [];
|
46 |
-
|
47 |
-
while (offset < hex.length) {
|
48 |
-
if (offset + 10 > hex.length) break;
|
49 |
-
|
50 |
-
const dataLength = parseInt(hex.slice(offset, offset + 10), 16);
|
51 |
-
offset += 10;
|
52 |
-
|
53 |
-
if (offset + dataLength * 2 > hex.length) break;
|
54 |
-
|
55 |
-
const messageHex = hex.slice(offset, offset + dataLength * 2);
|
56 |
-
offset += dataLength * 2;
|
57 |
-
|
58 |
-
console.log("debug [57] :", messageHex)
|
59 |
-
const messageBuffer = Buffer.from(messageHex, 'hex');
|
60 |
-
const message = $root.ResMessage.decode(messageBuffer);
|
61 |
-
results.push(message.msg);
|
62 |
-
}
|
63 |
-
|
64 |
-
if (results.length == 0) {
|
65 |
-
console.log("debug [63] :", chunk)
|
66 |
-
return gunzip(chunk);
|
67 |
-
}
|
68 |
-
return results.join('');
|
69 |
-
} catch (err) {
|
70 |
-
console.log("debug [68] :", chunk)
|
71 |
-
return gunzip(chunk);
|
72 |
-
}
|
73 |
-
}
|
74 |
-
|
75 |
-
function gunzip(chunk) {
|
76 |
-
return new Promise((resolve, reject) => {
|
77 |
-
zlib.gunzip(chunk.slice(5), (err, decompressed) => {
|
78 |
-
if (err) {
|
79 |
-
resolve('');
|
80 |
-
} else {
|
81 |
-
const text = decompressed.toString('utf-8');
|
82 |
-
// 这里只是尝试解析错误数据,如果是包含了全量的返回结果直接忽略
|
83 |
-
if (regex.test(text)) {
|
84 |
-
resolve('');
|
85 |
-
} else {
|
86 |
-
resolve(text);
|
87 |
-
}
|
88 |
-
}
|
89 |
-
});
|
90 |
-
});
|
91 |
-
}
|
92 |
-
|
93 |
-
function getRandomIDPro({ size, dictType, customDict }) {
|
94 |
-
let random = '';
|
95 |
-
if (!customDict) {
|
96 |
-
switch (dictType) {
|
97 |
-
case 'alphabet':
|
98 |
-
customDict = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
|
99 |
-
break;
|
100 |
-
case 'max':
|
101 |
-
customDict = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-';
|
102 |
-
break;
|
103 |
-
default:
|
104 |
-
customDict = '0123456789';
|
105 |
-
}
|
106 |
-
}
|
107 |
-
for (; size--; ) random += customDict[(Math.random() * customDict.length) | 0];
|
108 |
-
return random;
|
109 |
-
}
|
110 |
-
|
111 |
-
module.exports = {
|
112 |
-
stringToHex,
|
113 |
-
chunkToUtf8String,
|
114 |
-
getRandomIDPro,
|
115 |
-
};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|