Adding token infrastructure

Changed files
+148 -4
Sources
+5 -1
Package.swift
···
import PackageDescription
let package = Package(
-
name: "PterodactylKernel",
+
name: "Pterodactyl",
+
platforms: [.macOS(.v15)],
products: [
// Products define the executables and libraries a package produces, making them visible to other packages.
.library(
···
// Targets can depend on other targets in this package and products from dependencies.
.target(
name: "PterodactylKernel",
+
),
+
.target(
+
name: "PterodactylSyntax"
),
.testTarget(
name: "PterodactylKernelTests",
-3
Sources/PterodactylKernel/PterodactylKernel.swift
···
-
// SPDX-FileCopyrightText: 2025 The Project Pterodactyl Developers
-
//
-
// SPDX-License-Identifier: MPL-2.0
+21
Sources/PterodactylSyntax/Cursor.swift
···
+
import Foundation
+
+
public struct Cursor<L: Language>: Sendable {
+
let node: Child<L>
+
let utf16Offset: Int
+
let children: [Cursor<L>]
+
+
init(node: Child<L>, utf16Offset: Int) {
+
self.node = node
+
self.utf16Offset = utf16Offset
+
+
var children: [Cursor<L>] = []
+
var utf16Offset = utf16Offset
+
for childNode in node.children {
+
children.append(Self(node: childNode, utf16Offset: utf16Offset))
+
utf16Offset += childNode.utf16Length
+
}
+
+
self.children = children
+
}
+
}
+16
Sources/PterodactylSyntax/Protocols.swift
···
+
import Foundation
+
+
public protocol TreeKindType: Equatable, Sendable {
+
static var error: Self { get }
+
}
+
+
public protocol TokenKindType: Equatable, Sendable {
+
static var eof: Self { get }
+
}
+
+
public protocol Language {
+
associatedtype TokenKind: TokenKindType
+
associatedtype TreeKind: TreeKindType
+
associatedtype TokenMetadata: Sendable, Equatable
+
associatedtype TreeMetadata: Sendable, Equatable
+
}
+15
Sources/PterodactylSyntax/Token.swift
···
+
import Foundation
+
+
public struct Token<Kind> {
+
public let kind: Kind
+
public let text: String
+
public let utf16Length: Int
+
+
init(kind: Kind, text: String) {
+
self.kind = kind
+
self.text = text
+
self.utf16Length = text.utf16.count
+
}
+
}
+
+
extension Token: Sendable where Kind: Sendable {}
+71
Sources/PterodactylSyntax/Tree.swift
···
+
import Foundation
+
+
public struct Tree<L: Language>: Sendable {
+
public let kind: L.TreeKind
+
public let metadata: L.TreeMetadata?
+
public let children: [Child<L>]
+
public let utf16Length: Int
+
+
public init(kind: L.TreeKind, metadata: L.TreeMetadata?, children: [Child<L>]) {
+
self.kind = kind
+
self.metadata = metadata
+
self.children = children
+
self.utf16Length = children.reduce(0) { length, child in
+
length + child.utf16Length
+
}
+
}
+
}
+
+
public enum Child<L: Language>: Sendable {
+
case token(Token<L.TokenKind>, metadata: L.TokenMetadata?)
+
case tree(Tree<L>)
+
}
+
+
extension Tree {
+
public var text: String {
+
children.map(\.text).joined()
+
}
+
}
+
+
extension Child {
+
var text: String {
+
switch self {
+
case let .token(tok, _): tok.text
+
case let .tree(tree): tree.text
+
}
+
}
+
+
var utf16Length: Int {
+
switch self {
+
case let .token(token, metadata): token.utf16Length
+
case let .tree(tree): tree.utf16Length
+
}
+
}
+
+
var children: [Self] {
+
switch self {
+
case .token: []
+
case let .tree(tree): tree.children
+
}
+
}
+
}
+
+
extension Tree: CustomStringConvertible {
+
public var description: String {
+
prettyPrint()
+
}
+
+
/// Pretty-print the tree with indentation.
+
func prettyPrint(indent: String = "") -> String {
+
var result = "\(indent)\(kind)"
+
for child in children {
+
switch child {
+
case .token(let token, _):
+
result += "\n\(indent) \(token.kind): \(token.text.replacingOccurrences(of: "\n", with: "\\n"))"
+
case .tree(let subtree):
+
result += "\n" + subtree.prettyPrint(indent: indent + " ")
+
}
+
}
+
return result
+
}
+
}
+20
Sources/PterodactylSyntax/Utf16Position.swift
···
+
import Foundation
+
+
public struct Utf16Position: Equatable, Comparable, Sendable {
+
public var absoluteOffset: Int
+
public var line: Int
+
public var column : Int
+
public init(absoluteOffset: Int, line: Int, column: Int) {
+
self.absoluteOffset = absoluteOffset
+
self.line = line
+
self.column = column
+
}
+
+
public static func < (lhs: Utf16Position, rhs: Utf16Position) -> Bool {
+
lhs.absoluteOffset < rhs.absoluteOffset
+
}
+
}
+
+
public extension Utf16Position {
+
static var zero : Self { Self(absoluteOffset: 0, line: 0, column: 0) }
+
}