this repo has no description
www.jonmsterling.com/01HC/
jonmsterling.com
1// SPDX-FileCopyrightText: 2025 The Project Pterodactyl Developers
2//
3// SPDX-License-Identifier: MPL-2.0
4
5import Foundation
6
7/// This tokenises a string, without handling block layout at all. The tokens produced here should be fed into the ``BlockLayoutProcessor``.
8public struct Lexer {
9 private let input: String
10 private var index: String.Index
11 private var tokens: [PterodactylSyntax.Token]
12
13 public init(input: String) {
14 self.input = input
15 self.index = input.startIndex
16 self.tokens = []
17 }
18
19 private var isAtEnd: Bool { index >= input.endIndex }
20
21 private var peek: Character? {
22 guard index < input.endIndex else { return nil }
23 return input[index]
24 }
25
26 private func lookahead() -> Character? {
27 guard index < input.endIndex else { return nil }
28 let next = input.index(after: index)
29 guard next < input.endIndex else { return nil }
30 return input[next]
31 }
32
33 private mutating func advance() -> Character {
34 let c = input[index]
35 index = input.index(after: index)
36
37 return c
38 }
39
40 private mutating func consume(while predicate: (Character) -> Bool) {
41 while let c = peek, predicate(c) {
42 _ = advance()
43 }
44 }
45
46 func text(from start: String.Index) -> String {
47 let range = start..<index
48 return String(input[range])
49 }
50
51 public mutating func nextToken() -> (kind: TokenKind, text: String)? {
52 guard let c = peek else {
53 return nil
54 }
55
56 let start = index
57
58 if c.isNewline {
59 _ = advance()
60 return (kind: .newline, text: text(from: start))
61 }
62
63 if c.isWhitespace && !c.isNewline {
64 consume { $0.isWhitespace && !$0.isNewline }
65 return (kind: .whitespace, text: text(from: start))
66 }
67
68 if c == "/" && lookahead() == "/" {
69 _ = advance()
70 _ = advance()
71 consume { $0 != "\n" }
72 return (kind: .lineComment, text: text(from: start))
73 }
74
75 if c == "/" && lookahead() == "*" {
76 _ = advance() // consume '/'
77 _ = advance() // consume '*'
78 var terminated = false
79
80 while let ch = peek {
81 if ch == "*" && lookahead() == "/" {
82 _ = advance() // consume '*'
83 _ = advance() // consume '/'
84 terminated = true
85 break
86 }
87 _ = advance()
88 }
89
90 return (kind: .blockComment(terminated: terminated), text: text(from: start))
91 }
92
93 if c == "<" && lookahead() == "=" {
94 _ = advance()
95 _ = advance()
96 return (kind: .punctuation(.doubleLeftArrow), text: text(from: start))
97 }
98
99 if c == "=" && lookahead() == ">" {
100 _ = advance()
101 _ = advance()
102 return (kind: .punctuation(.doubleRightArrow), text: text(from: start))
103 }
104
105 if let punct = Punctuation(rawValue: String(c)) {
106 _ = advance()
107 return (.punctuation(punct), String(c))
108 } else if c.isLetter || c == "_" {
109 _ = advance()
110 consume { $0.isLetter || $0.isNumber || $0 == "_" }
111 let text = text(from: start)
112 if let keyword = Keyword(rawValue: text) {
113 return (kind: .keyword(keyword), text: text)
114 }
115 return (kind: .identifier, text: text)
116 }
117
118 // Invalid single char (don’t drop input)
119 let ch = advance()
120 return (kind: .error, text: String(ch))
121 }
122
123 public mutating func tokenize() -> [Token] {
124 var tokens: [Token] = []
125
126 while !isAtEnd {
127 guard let token = nextToken() else { break }
128 tokens.append(
129 Token(kind: token.kind, text: token.text)
130 )
131 }
132
133 let eofToken = Token(kind: .eof, text: "")
134 tokens.append(eofToken)
135
136 return tokens
137 }
138}