1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
|
const std = @import("std");
const options = @import("build_options");
const assert = std.debug.assert;
const indexOf = @import("index_of.zig").indexOf;
// vt.cpp
extern "c" fn ghostty_simd_decode_utf8_until_control_seq(
input: [*]const u8,
count: usize,
output: [*]u32,
output_count: *usize,
) usize;
const DecodeResult = struct {
consumed: usize,
decoded: usize,
};
pub fn utf8DecodeUntilControlSeq(
input: []const u8,
output: []u32,
) DecodeResult {
assert(output.len >= input.len);
if (comptime options.simd) {
var decoded: usize = 0;
const consumed = ghostty_simd_decode_utf8_until_control_seq(
input.ptr,
input.len,
output.ptr,
&decoded,
);
return .{ .consumed = consumed, .decoded = decoded };
}
return utf8DecodeUntilControlSeqScalar(input, output);
}
fn utf8DecodeUntilControlSeqScalar(
input: []const u8,
output: []u32,
) DecodeResult {
// Find our escape
const idx = indexOf(input, 0x1B) orelse input.len;
const decode = input[0..idx];
// Go through and decode one item at a time.
var decode_offset: usize = 0;
var decode_count: usize = 0;
while (decode_offset < decode.len) {
const decode_rem = decode[decode_offset..];
const cp_len = std.unicode.utf8ByteSequenceLength(decode_rem[0]) catch {
// Note, this is matching our SIMD behavior, but it is admittedly
// a bit weird. See our "decode invalid leading byte" test too.
// SIMD should be our source of truth then we copy behavior here.
break;
};
// If we don't have that number of bytes available. we finish. We
// assume this is a partial input and we defer to the future.
if (decode_rem.len < cp_len) break;
// We have the bytes available, so move forward
const cp_bytes = decode_rem[0..cp_len];
decode_offset += cp_len;
if (std.unicode.utf8Decode(cp_bytes)) |cp| {
output[decode_count] = @intCast(cp);
decode_count += 1;
} else |_| {
// If decoding failed, we replace the leading byte with the
// replacement char and then continue decoding after that
// byte. This matches the SIMD behavior and is tested by the
// "invalid UTF-8" tests.
output[decode_count] = 0xFFFD;
decode_count += 1;
decode_offset -= cp_len - 1;
}
}
return .{
.consumed = decode_offset,
.decoded = decode_count,
};
}
test "decode no escape" {
const testing = std.testing;
var output: [1024]u32 = undefined;
// TODO: many more test cases
{
const str = "hello" ** 128;
try testing.expectEqual(DecodeResult{
.consumed = str.len,
.decoded = str.len,
}, utf8DecodeUntilControlSeq(str, &output));
}
}
test "decode ASCII to escape" {
const testing = std.testing;
var output: [1024]u32 = undefined;
// TODO: many more test cases
{
const prefix = "hello" ** 64;
const str = prefix ++ "\x1b" ++ ("world" ** 64);
try testing.expectEqual(DecodeResult{
.consumed = prefix.len,
.decoded = prefix.len,
}, utf8DecodeUntilControlSeq(str, &output));
}
}
test "decode immediate esc sequence" {
const testing = std.testing;
var output: [64]u32 = undefined;
const str = "\x1b[?5s";
try testing.expectEqual(DecodeResult{
.consumed = 0,
.decoded = 0,
}, utf8DecodeUntilControlSeq(str, &output));
}
test "decode incomplete UTF-8" {
const testing = std.testing;
var output: [64]u32 = undefined;
// 2-byte
{
const str = "hello\xc2";
try testing.expectEqual(DecodeResult{
.consumed = 5,
.decoded = 5,
}, utf8DecodeUntilControlSeq(str, &output));
}
// 3-byte
{
const str = "hello\xe0\x00";
try testing.expectEqual(DecodeResult{
.consumed = 5,
.decoded = 5,
}, utf8DecodeUntilControlSeq(str, &output));
}
// 4-byte
{
const str = "hello\xf0\x90";
try testing.expectEqual(DecodeResult{
.consumed = 5,
.decoded = 5,
}, utf8DecodeUntilControlSeq(str, &output));
}
}
test "decode invalid UTF-8" {
const testing = std.testing;
var output: [64]u32 = undefined;
// Invalid leading 2-byte sequence
{
const str = "hello\xc2\x01";
try testing.expectEqual(DecodeResult{
.consumed = 7,
.decoded = 7,
}, utf8DecodeUntilControlSeq(str, &output));
}
// Replacement will only replace the invalid leading byte.
try testing.expectEqual(@as(u32, 0xFFFD), output[5]);
try testing.expectEqual(@as(u32, 0x01), output[6]);
}
// This is testing our current behavior so that we know we have to handle
// this case in terminal/stream.zig. If we change this behavior, we can
// remove the special handling in terminal/stream.zig.
test "decode invalid leading byte isn't consumed or replaced" {
const testing = std.testing;
var output: [64]u32 = undefined;
{
const str = "hello\xFF";
try testing.expectEqual(DecodeResult{
.consumed = 5,
.decoded = 5,
}, utf8DecodeUntilControlSeq(str, &output));
}
}
|
