ltxlong commited on
Commit
f6c8a02
·
verified ·
1 Parent(s): 00fedd5

Delete utils.js

Browse files
Files changed (1) hide show
  1. utils.js +0 -115
utils.js DELETED
@@ -1,115 +0,0 @@
1
- const { v4: uuidv4 } = require('uuid');
2
- const zlib = require('zlib');
3
- const $root = require('./message.js');
4
-
5
- const regex = /<\|BEGIN_SYSTEM\|>.*?<\|END_SYSTEM\|>.*?<\|BEGIN_USER\|>.*?<\|END_USER\|>/s;
6
-
7
- async function stringToHex(messages, modelName) {
8
- const formattedMessages = messages.map((msg) => ({
9
- ...msg,
10
- role: msg.role === 'user' ? 1 : 2,
11
- message_id: uuidv4(),
12
- }));
13
-
14
- const message = {
15
- messages: formattedMessages,
16
- instructions: {
17
- instruction: 'Always respond in 中文',
18
- },
19
- projectPath: '/path/to/project',
20
- model: {
21
- name: modelName,
22
- empty: '',
23
- },
24
- requestId: uuidv4(),
25
- summary: '',
26
- conversationId: uuidv4(),
27
- };
28
- const errMsg = $root.ChatMessage.verify(message);
29
- if (errMsg) throw Error(errMsg);
30
-
31
- const messageInstance = $root.ChatMessage.create(message);
32
-
33
- const buffer = $root.ChatMessage.encode(messageInstance).finish();
34
- const hexString = (buffer.length.toString(16).padStart(10, '0') + buffer.toString('hex')).toUpperCase();
35
-
36
- return Buffer.from(hexString, 'hex');
37
- }
38
-
39
- async function chunkToUtf8String(chunk) {
40
- try {
41
- let hex = Buffer.from(chunk).toString('hex');
42
- console.log("debug [42] :", hex)
43
-
44
- let offset = 0;
45
- let results = [];
46
-
47
- while (offset < hex.length) {
48
- if (offset + 10 > hex.length) break;
49
-
50
- const dataLength = parseInt(hex.slice(offset, offset + 10), 16);
51
- offset += 10;
52
-
53
- if (offset + dataLength * 2 > hex.length) break;
54
-
55
- const messageHex = hex.slice(offset, offset + dataLength * 2);
56
- offset += dataLength * 2;
57
-
58
- console.log("debug [57] :", messageHex)
59
- const messageBuffer = Buffer.from(messageHex, 'hex');
60
- const message = $root.ResMessage.decode(messageBuffer);
61
- results.push(message.msg);
62
- }
63
-
64
- if (results.length == 0) {
65
- console.log("debug [63] :", chunk)
66
- return gunzip(chunk);
67
- }
68
- return results.join('');
69
- } catch (err) {
70
- console.log("debug [68] :", chunk)
71
- return gunzip(chunk);
72
- }
73
- }
74
-
75
- function gunzip(chunk) {
76
- return new Promise((resolve, reject) => {
77
- zlib.gunzip(chunk.slice(5), (err, decompressed) => {
78
- if (err) {
79
- resolve('');
80
- } else {
81
- const text = decompressed.toString('utf-8');
82
- // 这里只是尝试解析错误数据,如果是包含了全量的返回结果直接忽略
83
- if (regex.test(text)) {
84
- resolve('');
85
- } else {
86
- resolve(text);
87
- }
88
- }
89
- });
90
- });
91
- }
92
-
93
- function getRandomIDPro({ size, dictType, customDict }) {
94
- let random = '';
95
- if (!customDict) {
96
- switch (dictType) {
97
- case 'alphabet':
98
- customDict = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
99
- break;
100
- case 'max':
101
- customDict = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-';
102
- break;
103
- default:
104
- customDict = '0123456789';
105
- }
106
- }
107
- for (; size--; ) random += customDict[(Math.random() * customDict.length) | 0];
108
- return random;
109
- }
110
-
111
- module.exports = {
112
- stringToHex,
113
- chunkToUtf8String,
114
- getRandomIDPro,
115
- };