-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtokenizer_fix_v2.diff
More file actions
89 lines (82 loc) · 3.02 KB
/
tokenizer_fix_v2.diff
File metadata and controls
89 lines (82 loc) · 3.02 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
diff --git a/src/parser/tokenizer.ts b/src/parser/tokenizer.ts
index 9247b38..338a6c9 100644
--- a/src/parser/tokenizer.ts
+++ b/src/parser/tokenizer.ts
@@ -6,6 +6,30 @@ export type TokenResult = Token[] | TokenError;
export class Tokenizer {
private input: string = '';
+ private equalsAssignmentMode = false;
private position: number = 0;
tokenize(input: string): TokenResult {
@@ -26,6 +49,7 @@ export class Tokenizer {
while (this.position < this.input.length) {
// Skip whitespace
+ this.equalsAssignmentMode = false;
while (this.position < this.input.length && /\s/.test(this.input[this.position] || '')) {
this.position++;
}
@@ -42,6 +66,11 @@ export class Tokenizer {
if ('error' in token) {
return token; // Return error immediately
}
+ // If this token follows an equals sign assignment, mark it specially
+ if (token.type === 'EQUALS') {
+ this.equalsAssignmentMode = true;
+ }
+
tokens.push(token);
}
@@ -57,11 +85,21 @@ export class Tokenizer {
const currentChar = this.input[this.position];
if (currentChar === '-') {
if (this.position + 1 < this.input.length && this.input[this.position + 1] === '-') {
+ // If we're processing a value after flag assignment (either --flag=value or --flag value),
+ // and we encounter a pattern that looks like a flag, treat it as a plain word
+ if (this.equalsAssignmentMode) {
+ // We're in a situation like --message="Build Status: success, Test Status: success"
+ // which has been split by the shell, so we're processing "Build Status: success, Test Status: success"
+ // In this case, just grab everything until the end
+ return this.readRemainingAsValue();
+ }
+
// Long flag: --flagname
this.position += 2; // Skip --
let flagName = '';
@@ -69,13 +107,23 @@ export class Tokenizer {
}
return {
- type: 'FLAG',
+ type: this.equalsAssignmentMode ? 'WORD' : 'FLAG',
value: flagName,
position: startPos,
raw: `--${flagName}`
};
} else {
+ // Short flag: -f
+ if (this.equalsAssignmentMode) {
+ // If we're in an equals assignment context, treat this as part of the value
+ return this.readRemainingAsValue();
+ }
+
// Short flag: -f
this.position++; // Skip -
@@ -147,6 +195,22 @@ export class Tokenizer {
}
}
+ private readRemainingAsValue(): Token {
+ // If we're in equals assignment mode, read everything until whitespace
+ const startPos = this.position;
+ let tokenValue = '';
+
+ while (this.position < this.input.length &&
+ !/\s/.test(this.input[this.position] || '')) {
+ tokenValue += this.input[this.position];
+ this.position++;
+ }
+
+ return { type: 'STRING', value: tokenValue, position: startPos, raw: tokenValue };
+ }
+
// Check for equals sign
if (currentChar === '=') {
this.position++;