Bringing in the grammar, some LSP compatibility

Much of this code is adapted from an earlier experiment. Relative to my old
experiment, I am now distinguishing "green" and "red" trees in the style of
Roslyn, where the latter instrument the former with global positioning
information via a line map. For me a "green tree" is a syntax tree, and a "red
tree" is a cursor.

I am also starting to hook up the tokenisation and parsing into the llbuild2fx
action graph.

+9 -2
Package.swift
···
)
],
dependencies: [
-
.package(url: "https://github.com/apple/swift-llbuild2.git", branch: "main")
+
// .package(url: "https://github.com/ChimeHQ/LanguageServer", branch: "main"),
+
.package(url: "https://github.com/ChimeHQ/LanguageServerProtocol", branch: "main"),
+
.package(url: "https://github.com/apple/swift-llbuild2.git", branch: "main"),
+
.package(url: "https://github.com/apple/swift-algorithms", from: "1.2.0")
],
targets: [
// Targets are the basic building blocks of a package, defining a module or a test suite.
···
name: "PterodactylKernel",
),
.target(
-
name: "PterodactylSyntax"
+
name: "PterodactylSyntax",
+
dependencies: [
+
.product(name: "Algorithms", package: "swift-algorithms"),
+
"LanguageServerProtocol"
+
]
),
.target(
name: "PterodactylBuild",
+15
Sources/PterodactylBuild/FXValue+Conformances.swift
···
+
// SPDX-FileCopyrightText: 2025 The Project Pterodactyl Developers
+
//
+
// SPDX-License-Identifier: MPL-2.0
+
+
import Foundation
+
import llbuild2fx
+
import PterodactylSyntax
+
+
extension SyntaxTree: FXValue {}
+
extension Token: FXValue {}
+
extension Graph: FXValue where Vertex: Codable {}
+
+
extension String: @retroactive FXValue {}
+
extension Set: @retroactive FXValue where Element: Codable {}
+
extension LLBDataID: @retroactive FXValue {}
+11 -8
Sources/PterodactylBuild/Keys/AnalyseImports.swift Sources/PterodactylBuild/Keys/Blob-local operations/BlobImports.swift
···
public mutating func parseHeader() {
while true {
-
let token = nextSignificantToken()
+
guard let token = nextSignificantToken() else { return }
switch token.kind {
case .keyword(.import): parseImportStatement()
default: return
···
}
/// Returns the next non-whitespace token.
-
private mutating func nextSignificantToken() -> Token {
+
private mutating func nextSignificantToken() -> Token? {
var token = lexer.nextToken()
-
while case .whitespace = token.kind {
+
while token?.kind.isTrivia == true {
token = lexer.nextToken()
}
-
return token
+
+
guard let token else { return nil}
+
return Token(kind: token.kind, text: token.text)
}
/// Parses a single `import xyz` line.
private mutating func parseImportStatement() {
-
let next = nextSignificantToken()
+
guard let next = nextSignificantToken() else { return }
guard next.kind == .identifier else { return }
imports.append(next.text)
}
···
extension Keys {
-
struct AnalyseImports: BuildKey {
+
struct BlobImports: BuildKey {
typealias ValueType = [UnitName]
let blobId: LLBDataID
+
+
static let versionDependencies: [any FXVersioning.Type] = [BlobContents.self]
func computeValue(_ ctx: BuildContext<Self>) async throws -> [UnitName] {
-
let contents = try await ctx.load(blobId)
-
let code = try await String(decoding: Data(ctx.read(blob: contents.blob!)), as: UTF8.self)
+
let code = try await ctx.request(BlobContents(blobId: blobId))
var importParser = ImportParser(input: code)
importParser.parseHeader()
+22
Sources/PterodactylBuild/Keys/Blob-local operations/BlobContents.swift
···
+
// SPDX-FileCopyrightText: 2025 The Project Pterodactyl Developers
+
//
+
// SPDX-License-Identifier: MPL-2.0
+
+
import Foundation
+
import TSCBasic
+
import llbuild2fx
+
+
extension Keys {
+
struct BlobContents: BuildKey {
+
typealias ValueType = String
+
+
let blobId: LLBDataID
+
+
static let versionDependencies: [any FXVersioning.Type] = []
+
+
func computeValue(_ ctx: BuildContext<Self>) async throws -> String {
+
let contents = try await ctx.load(blobId)
+
return try await String(decoding: Data(ctx.read(blob: contents.blob!)), as: UTF8.self)
+
}
+
}
+
}
+26
Sources/PterodactylBuild/Keys/Blob-local operations/BlobSyntaxTree.swift
···
+
// SPDX-FileCopyrightText: 2025 The Project Pterodactyl Developers
+
//
+
// SPDX-License-Identifier: MPL-2.0
+
+
import Foundation
+
import PterodactylSyntax
+
import TSCBasic
+
import llbuild2fx
+
+
extension Keys {
+
struct BlobSyntaxTree: BuildKey {
+
typealias ValueType = SyntaxTree
+
+
let blobId: LLBDataID
+
+
static let versionDependencies: [any FXVersioning.Type] = [BlobContents.self, BlobTokens.self]
+
+
func computeValue(_ ctx: BuildContext<Self>) async throws -> ValueType {
+
let code = try await ctx.request(BlobContents(blobId: blobId))
+
let tokens = try await ctx.request(BlobTokens(blobId: blobId))
+
var parser = Parser(source: code, tokens: tokens)
+
PterodactylSyntax.Document.parse(&parser)
+
return parser.tree
+
}
+
}
+
}
+25
Sources/PterodactylBuild/Keys/Blob-local operations/BlobTokens.swift
···
+
// SPDX-FileCopyrightText: 2025 The Project Pterodactyl Developers
+
//
+
// SPDX-License-Identifier: MPL-2.0
+
+
import Foundation
+
import TSCBasic
+
import llbuild2fx
+
import PterodactylSyntax
+
+
extension Keys {
+
struct BlobTokens: BuildKey {
+
typealias ValueType = [Token]
+
+
let blobId: LLBDataID
+
+
static let versionDependencies: [any FXVersioning.Type] = [BlobContents.self]
+
+
func computeValue(_ ctx: BuildContext<Self>) async throws -> ValueType {
+
let code = try await ctx.request(BlobContents(blobId: blobId))
+
var lexer = PterodactylSyntax.Lexer(input: code)
+
let flatTokens = lexer.tokenize()
+
return BlockLayoutProcessor(tokens: flatTokens).layout()
+
}
+
}
+
}
+1
Sources/PterodactylBuild/Keys/Blob-local operations/README.md
···
+
Certain operations do not require knowledge of the entire source tree, only a specific blob inside the source tree. These include import analysis, tokenisation, line maps, parsing, etc.
+2 -4
Sources/PterodactylBuild/Keys/DependencyGraphOfSourceTree.swift
···
import TSCBasic
import llbuild2fx
-
extension Graph: FXValue where Vertex: Codable {}
-
extension Keys {
struct DependencyGraphOfSourceTree: BuildKey {
typealias ValueType = Graph<UnitName>
let sourceTreeId: LLBDataID
-
static let versionDependencies: [any FXVersioning.Type] = [Keys.UnitMapOfSourceTree.self, Keys.AnalyseImports.self]
+
static let versionDependencies: [any FXVersioning.Type] = [Keys.UnitMapOfSourceTree.self, Keys.BlobImports.self]
func computeValue(_ ctx: BuildContext<Self>) async throws -> Graph<UnitName> {
let unitMap = try await ctx.request(Keys.UnitMapOfSourceTree(sourceTreeId: sourceTreeId))
···
for (unitName, unitInfo) in unitMap.units {
if edges[unitName] == nil { edges[unitName] = [] }
-
let imports = try await ctx.request(Keys.AnalyseImports(blobId: unitInfo.blobId))
+
let imports = try await ctx.request(Keys.BlobImports(blobId: unitInfo.blobId))
for importedUnitName in imports {
edges[unitName]!.insert(importedUnitName)
}
+4 -6
Sources/PterodactylBuild/Keys/NarrowSourceTree.swift
···
extension Keys {
/// Narrows a source tree to just the transitive dependencies of a given unit
struct NarrowSourceTree: BuildKey {
-
struct ValueType: Codable, FXValue {
-
let sourceTreeId: LLBDataID
-
}
-
+
typealias ValueType = LLBDataID
+
let sourceTreeId: LLBDataID
let unitName: UnitName
static let versionDependencies: [any FXVersioning.Type] = [TransitiveDependencies.self, UnitMapOfSourceTree.self]
func computeValue(_ ctx: BuildContext<Self>) async throws -> ValueType {
-
let dependencies = try await ctx.request(TransitiveDependencies(sourceTreeId: sourceTreeId, unitName: unitName)).dependencies
+
let dependencies = try await ctx.request(TransitiveDependencies(sourceTreeId: sourceTreeId, unitName: unitName))
let unitMap = try await ctx.request(UnitMapOfSourceTree(sourceTreeId: sourceTreeId))
var sourceTree = try await LLBCASFileTree.load(id: sourceTreeId, in: ctx)
···
}
}
-
return ValueType(sourceTreeId: sourceTree.id)
+
return sourceTree.id
}
}
}
-32
Sources/PterodactylBuild/Keys/SourceCode.swift
···
-
// SPDX-FileCopyrightText: 2025 The Project Pterodactyl Developers
-
//
-
// SPDX-License-Identifier: MPL-2.0
-
-
import Foundation
-
import TSCBasic
-
import llbuild2fx
-
-
extension Keys {
-
struct SourceCode: BuildKey {
-
struct ValueType: Codable, FXValue {
-
let code: String
-
}
-
-
enum SourceCodeError: Error {
-
case unitNotFound
-
}
-
-
let sourceTreeId: LLBDataID
-
let unitName: UnitName
-
-
static let versionDependencies: [any FXVersioning.Type] = [UnitMapOfSourceTree.self]
-
-
func computeValue(_ ctx: BuildContext<Self>) async throws -> ValueType {
-
let unitMap = try await ctx.request(UnitMapOfSourceTree(sourceTreeId: sourceTreeId))
-
guard let unitInfo = unitMap.units[unitName] else { throw SourceCodeError.unitNotFound }
-
let contents = try await ctx.load(unitInfo.blobId)
-
let code = try await String(decoding: Data(ctx.read(blob: contents.blob!)), as: UTF8.self)
-
return ValueType(code: code)
-
}
-
}
-
}
+2 -4
Sources/PterodactylBuild/Keys/TransitiveDependencies.swift
···
extension Keys {
struct TransitiveDependencies: BuildKey {
-
struct ValueType: Codable, FXValue {
-
var dependencies: Set<UnitName>
-
}
+
typealias ValueType = Set<UnitName>
let sourceTreeId: LLBDataID
let unitName: UnitName
···
func computeValue(_ ctx: BuildContext<Self>) async throws -> ValueType {
let graph = try await ctx.request(Keys.DependencyGraphOfSourceTree(sourceTreeId: sourceTreeId))
-
return ValueType(dependencies: graph.verticesReachableFrom(unitName))
+
return graph.verticesReachableFrom(unitName)
}
}
}
+44 -16
Sources/PterodactylSyntax/Cursor.swift
···
// SPDX-License-Identifier: MPL-2.0
import Foundation
+
import LanguageServerProtocol
-
public struct Cursor: Sendable {
-
let node: SyntaxTree.Child
-
let utf16Offset: Int
-
let children: [Cursor]
-
-
init(node: SyntaxTree.Child, utf16Offset: Int) {
-
self.node = node
-
self.utf16Offset = utf16Offset
-
-
var children: [Cursor] = []
-
var utf16Offset = utf16Offset
-
for childNode in node.children {
-
children.append(Self(node: childNode, utf16Offset: utf16Offset))
-
utf16Offset += childNode.utf16Length
+
extension SyntaxTree {
+
public final class Cursor {
+
public let lineMap: LineMap
+
public let node: SyntaxTree.Child
+
public let utf16Offset: Int
+
+
public private(set) lazy var children: [Cursor] = {
+
var children: [Cursor] = []
+
var utf16Offset = utf16Offset
+
for childNode in node.children {
+
children.append(Self(lineMap: lineMap, node: childNode, utf16Offset: utf16Offset))
+
utf16Offset += childNode.utf16Length
+
}
+
+
return children
+
}()
+
+
public var utf16Range: Range<Int> {
+
utf16Offset..<utf16Offset + node.utf16Length
}
-
-
self.children = children
+
+
init(lineMap: LineMap, node: SyntaxTree.Child, utf16Offset: Int) {
+
self.lineMap = lineMap
+
self.node = node
+
self.utf16Offset = utf16Offset
+
}
}
}
+
+
extension SyntaxTree.Cursor {
+
public func firstChild<T>(mapping: (SyntaxTree.Cursor) -> T?) -> T? {
+
for child in children {
+
if let result = mapping(child) {
+
return result
+
} else {
+
continue
+
}
+
}
+
return nil
+
}
+
+
public func children<T>(mapping: (SyntaxTree.Cursor) -> T?) -> [T] {
+
children.compactMap(mapping)
+
}
+
}
+
+63
Sources/PterodactylSyntax/FoldingRanges.swift
···
+
// SPDX-FileCopyrightText: 2025 The Project Pterodactyl Developers
+
//
+
// SPDX-License-Identifier: MPL-2.0
+
+
import Foundation
+
import LanguageServerProtocol
+
+
private enum ArraySymmetry {
+
case identity
+
case reverse
+
}
+
+
extension Array {
+
fileprivate func apply(symmetry: ArraySymmetry) -> any Collection<Element> {
+
switch symmetry {
+
case .identity: self
+
case .reverse: reversed()
+
}
+
}
+
}
+
+
extension SyntaxTree.Cursor {
+
private func firstVisibleNode(under symmetry: ArraySymmetry) -> SyntaxTree.Cursor? {
+
switch node {
+
case .token(let token, _):
+
return token.kind.isVisible ? self : nil
+
case .tree:
+
for child in children.apply(symmetry: symmetry) {
+
if let visibleChild = child.firstVisibleNode(under: symmetry) { return visibleChild }
+
continue
+
}
+
+
return nil
+
}
+
}
+
+
private var visibleUtf16Range: Range<Int>? {
+
guard
+
let firstNode = firstVisibleNode(under: .identity),
+
let lastNode = firstVisibleNode(under: .reverse)
+
else { return nil }
+
return firstNode.utf16Range.lowerBound..<lastNode.utf16Range.upperBound
+
}
+
+
private func collectFoldingRanges(_ sink: inout [FoldingRange]) {
+
if let foldingRangeKind = node.tree?.metadata?.delimitedFoldingRangeKind, let visibleUtf16Range {
+
let startLocation = lineMap.location(at: visibleUtf16Range.lowerBound)
+
let endLocation = lineMap.location(at: visibleUtf16Range.upperBound)
+
let foldingRange = FoldingRange(startLine: startLocation.line, endLine: endLocation.line, kind: foldingRangeKind)
+
sink.append(foldingRange)
+
}
+
+
for child in children {
+
child.collectFoldingRanges(&sink)
+
}
+
}
+
+
public var foldingRanges: [FoldingRange] {
+
var sink: [FoldingRange] = []
+
collectFoldingRanges(&sink)
+
return sink
+
}
+
}
+16 -5
Sources/PterodactylSyntax/Grammar.swift
···
import Foundation
-
public protocol Grammar {
-
// static var kind: L.TreeKind { get }
+
public protocol Grammar: Sendable {
+
static var kinds: [SyntaxTreeKind] { get }
static func before(_ parser: inout Parser) -> Bool
-
static func inside(_ parser: inout Parser) -> SyntaxTreeKind
+
static func inside(_ parser: inout Parser) -> ParseResult
+
}
+
+
public struct ParseResult {
+
public var kind: SyntaxTreeKind
+
public var metadata: SyntaxTreeMetadata? = nil
+
+
public init(kind: SyntaxTreeKind, metadata: SyntaxTreeMetadata? = nil) {
+
self.kind = kind
+
self.metadata = metadata
+
}
}
+
public extension Grammar {
static func tryParse(_ parser: inout Parser) -> Bool {
···
static func parse(_ parser: inout Parser) {
let mark = parser.open()
-
let kind = inside(&parser)
-
parser.close(mark: mark, kind: kind)
+
let result = inside(&parser)
+
parser.close(mark: mark, kind: result.kind, metadata: result.metadata)
}
}
+51
Sources/PterodactylSyntax/Grammar/Document.swift
···
+
// SPDX-FileCopyrightText: 2025 The Project Pterodactyl Developers
+
//
+
// SPDX-License-Identifier: MPL-2.0
+
+
import Foundation
+
+
public enum Document: Grammar {
+
public static let kind = SyntaxTreeKind(name: "document")
+
public static let kinds = [kind]
+
+
public static func before(_ parser: inout Parser) -> Bool {
+
true
+
}
+
+
public static func inside(_ parser: inout Parser) -> ParseResult {
+
parser.eatTrivia()
+
+
// Parse imports
+
while !parser.isAt(kind: .eof) {
+
parser.eatTrivia()
+
if Theory.before(&parser) { break }
+
+
if !Import.tryParse(&parser) {
+
parser.advance(error: "Expected to see either an import or a theory declaration, but instead got \(parser.currentToken.kind): \(parser.currentToken.text)")
+
}
+
}
+
+
// Theories section
+
while !parser.isAt(kind: .eof) {
+
if !Theory.tryParse(&parser) {
+
if parser.isAt(kindSatisfying: \.isVisible) {
+
let token = parser.currentToken
+
parser.advance(error: "Unexpected token: \(token.kind)")
+
} else {
+
parser.advance(metadata: nil)
+
}
+
}
+
parser.eatTrivia()
+
}
+
+
parser.eatTrivia()
+
_ = parser.eat(kind: .eof, metadata: nil)
+
+
return ParseResult(kind: Self.kind)
+
}
+
}
+
+
extension SyntaxView<Document> {
+
var imports: [SyntaxView<Import>] { matchingSubviews() }
+
var theories: [SyntaxView<Theory>] { matchingSubviews() }
+
}
+46
Sources/PterodactylSyntax/Grammar/Document/Import.swift
···
+
// SPDX-FileCopyrightText: 2025 The Project Pterodactyl Developers
+
//
+
// SPDX-License-Identifier: MPL-2.0
+
+
import Foundation
+
+
enum ImportName: Grammar {
+
static let kind = SyntaxTreeKind(name: "import.name")
+
static let kinds = [kind]
+
+
static func before(_ parser: inout Parser) -> Bool {
+
parser.isAt(kind: .identifier)
+
}
+
+
static func inside(_ parser: inout Parser) -> ParseResult {
+
parser.expect(kind: .identifier, metadata: TokenMetadata(semanticTokenType: .namespace))
+
return ParseResult(kind: Self.kind)
+
}
+
}
+
+
enum Import: Grammar {
+
static let kind = SyntaxTreeKind(name: "import")
+
static let kinds = [kind]
+
+
static func before(_ parser: inout Parser) -> Bool {
+
parser.isAt(kind: .keyword(.import))
+
}
+
+
static func inside(_ parser: inout Parser) -> ParseResult {
+
parser.expect(kind: .keyword(.import), metadata: TokenMetadata(semanticTokenType: .keyword))
+
parser.eatTrivia()
+
ImportName.parse(&parser)
+
return ParseResult(kind: Self.kind)
+
}
+
}
+
+
+
extension SyntaxView<ImportName> {
+
var text: String { cursor.node.text }
+
}
+
+
extension SyntaxView<Import> {
+
var name: SyntaxView<ImportName>? {
+
matchingSubview()
+
}
+
}
+33
Sources/PterodactylSyntax/Grammar/Document/Theory.swift
···
+
// SPDX-FileCopyrightText: 2025 The Project Pterodactyl Developers
+
//
+
// SPDX-License-Identifier: MPL-2.0
+
+
import Foundation
+
+
enum Theory: Grammar {
+
static let kind = SyntaxTreeKind(name: "theory")
+
static let kinds = [kind]
+
+
static func before(_ parser: inout Parser) -> Bool {
+
parser.isAt(kind: .keyword(.theory))
+
}
+
+
static func inside(_ parser: inout Parser) -> ParseResult {
+
parser.expect(kind: .keyword(.theory), metadata: TokenMetadata(semanticTokenType: .keyword))
+
parser.eatTrivia()
+
if !TheoryName.tryParse(&parser) {
+
parser.advance(error: "Expected theory name")
+
}
+
+
parser.eatTrivia()
+
+
TheoryBlock.parse(&parser)
+
+
return ParseResult(kind: Self.kind)
+
}
+
}
+
+
extension SyntaxView<Theory> {
+
var name: SyntaxView<TheoryName>? { matchingSubview() }
+
var block: SyntaxView<TheoryBlock>? { matchingSubview() }
+
}
+52
Sources/PterodactylSyntax/Grammar/Document/Theory/Declaration.swift
···
+
// SPDX-FileCopyrightText: 2025 The Project Pterodactyl Developers
+
//
+
// SPDX-License-Identifier: MPL-2.0
+
+
import Foundation
+
+
enum Declaration: Grammar {
+
enum Kinds {
+
static let claim = SyntaxTreeKind(name: "decl.claim")
+
static let refine = SyntaxTreeKind(name: "decl.refine")
+
static let define = SyntaxTreeKind(name: "decl.define")
+
}
+
+
static let kinds = [Kinds.claim, Kinds.refine, Kinds.define]
+
+
static func before(_ parser: inout Parser) -> Bool {
+
Lhs.before(&parser)
+
}
+
+
static let punctuationMap: [Punctuation: SyntaxTreeKind] = [
+
.colon: Kinds.claim,
+
.doubleLeftArrow: Kinds.refine,
+
.doubleRightArrow: Kinds.define
+
]
+
+
static func inside(_ parser: inout Parser) -> ParseResult {
+
Lhs.parse(&parser)
+
parser.eatTrivia()
+
+
var kind: SyntaxTreeKind = .error
+
for cell in punctuationMap {
+
if parser.eat(kind: .punctuation(cell.key), metadata: TokenMetadata(semanticTokenType: .operator)) {
+
kind = cell.value
+
break
+
}
+
}
+
+
if kind == .error {
+
parser.advance(error: "Expected one of \(punctuationMap.keys.map(\.rawValue)) in declaration")
+
}
+
+
parser.eatTrivia()
+
Rhs.parse(&parser)
+
+
return ParseResult(kind: kind)
+
}
+
}
+
+
extension SyntaxView<Declaration> {
+
var lhs: SyntaxView<Lhs>? { matchingSubview() }
+
var rhs: SyntaxView<Rhs>? { matchingSubview() }
+
}
+19
Sources/PterodactylSyntax/Grammar/Document/Theory/Declaration/Lhs.swift
···
+
// SPDX-FileCopyrightText: 2025 The Project Pterodactyl Developers
+
//
+
// SPDX-License-Identifier: MPL-2.0
+
+
import Foundation
+
+
enum Lhs: Grammar {
+
static let kind = SyntaxTreeKind(name: "declaration.lhs")
+
static let kinds = [kind]
+
+
static func before(_ parser: inout Parser) -> Bool {
+
parser.isAt(kind: .identifier)
+
}
+
+
static func inside(_ parser: inout Parser) -> ParseResult {
+
parser.expect(kind: .identifier, metadata: TokenMetadata(semanticTokenType: .method))
+
return ParseResult(kind: Self.kind)
+
}
+
}
+19
Sources/PterodactylSyntax/Grammar/Document/Theory/Declaration/Rhs.swift
···
+
// SPDX-FileCopyrightText: 2025 The Project Pterodactyl Developers
+
//
+
// SPDX-License-Identifier: MPL-2.0
+
+
import Foundation
+
+
enum Rhs: Grammar {
+
static let kind = SyntaxTreeKind(name: "declaration.lhs")
+
static let kinds = [kind]
+
+
static func before(_ parser: inout Parser) -> Bool {
+
parser.isAt(kind: .identifier)
+
}
+
+
static func inside(_ parser: inout Parser) -> ParseResult {
+
parser.expect(kind: .identifier, metadata: TokenMetadata(semanticTokenType: .method))
+
return ParseResult(kind: Self.kind)
+
}
+
}
+46
Sources/PterodactylSyntax/Grammar/Document/Theory/TheoryBlock.swift
···
+
// SPDX-FileCopyrightText: 2025 The Project Pterodactyl Developers
+
//
+
// SPDX-License-Identifier: MPL-2.0
+
+
import Foundation
+
+
enum TheoryBlock: Grammar {
+
static let kind = SyntaxTreeKind(name: "theory.block")
+
static let kinds = [kind]
+
+
static func before(_ parser: inout Parser) -> Bool {
+
parser.isAt(kind: .keyword(.where))
+
}
+
+
static func inside(_ parser: inout Parser) -> ParseResult {
+
parser.expect(kind: .keyword(.where), metadata: TokenMetadata(semanticTokenType: .keyword))
+
+
parser.eatTrivia()
+
if parser.eat(kind: .blockBegin, metadata: nil) {
+
parser.eatTrivia()
+
+
while Declaration.tryParse(&parser) {
+
parser.eatTrivia()
+
if parser.eat(kind: .blockSep, metadata: nil) {
+
parser.eatTrivia()
+
continue
+
} else {
+
break
+
}
+
}
+
+
_ = parser.eat(kind: .blockEnd, metadata: nil)
+
}
+
+
var metadata = SyntaxTreeMetadata()
+
metadata.delimitedFoldingRangeKind = .region
+
+
return ParseResult(kind: kind, metadata: metadata)
+
}
+
}
+
+
extension SyntaxView<TheoryBlock> {
+
var declarations: [SyntaxView<Declaration>] {
+
matchingSubviews()
+
}
+
}
+23
Sources/PterodactylSyntax/Grammar/Document/Theory/TheoryName.swift
···
+
// SPDX-FileCopyrightText: 2025 The Project Pterodactyl Developers
+
//
+
// SPDX-License-Identifier: MPL-2.0
+
+
import Foundation
+
+
enum TheoryName: Grammar {
+
static let kind = SyntaxTreeKind(name: "theory.name")
+
static let kinds = [kind]
+
+
static func before(_ parser: inout Parser) -> Bool {
+
parser.isAt(kind: .identifier)
+
}
+
+
static func inside(_ parser: inout Parser) -> ParseResult {
+
parser.advance(metadata: TokenMetadata(semanticTokenType: .interface))
+
return ParseResult(kind: Self.kind)
+
}
+
}
+
+
extension SyntaxView<TheoryName> {
+
var text: String { cursor.node.text }
+
}
+130 -69
Sources/PterodactylSyntax/Lexer.swift
···
//
// SPDX-License-Identifier: MPL-2.0
-
//
-
// SPDX-License-Identifier: MPL-2.0
-
-
import Foundation
-
+
import Foundation
+
public struct Lexer {
-
private let input: String
-
private var index: String.Index
+
private let input: String
+
private var index: String.Index
private var tokens: [PterodactylSyntax.Token]
-
+
public init(input: String) {
-
self.input = input
-
self.index = input.startIndex
-
self.tokens = []
-
}
-
-
static let keywords: [String: Keyword] = [
-
"import": .import
-
]
-
-
public mutating func nextToken() -> Token {
-
guard !isAtEnd else {
-
return Token(kind: .eof, text: "")
-
}
-
-
let char = currentChar
-
-
if char.isNewline {
-
advance()
-
return Token(kind: .whitespace(.newline), text: String(char))
-
}
-
-
if char.isWhitespace {
-
let text = readWhile { $0.isWhitespace && !$0.isNewline }
-
return Token(kind: .whitespace(.other), text: text)
-
}
-
-
if char.isLetter {
-
let word = readWhile { $0.isLetter || $0.isNumber || $0 == "_" }
-
let kind: TokenKind =
-
if let keyword = Self.keywords[word] {
-
.keyword(keyword)
-
} else {
-
.identifier
-
}
-
return Token(kind: kind, text: word)
-
}
-
-
advance()
-
return Token(kind: .error, text: String(char))
-
}
-
-
private mutating func readWhile(_ condition: (Character) -> Bool) -> String {
-
var result = ""
-
while !isAtEnd && condition(currentChar) {
-
result.append(currentChar)
-
advance()
-
}
-
return result
-
}
-
-
private var currentChar: Character { input[index] }
-
-
private var isAtEnd: Bool {
-
index == input.endIndex
-
}
-
-
private mutating func advance() {
-
guard index < input.endIndex else { return }
-
index = input.index(after: index)
-
}
-
}
+
self.input = input
+
self.index = input.startIndex
+
self.tokens = []
+
}
+
+
private var isAtEnd: Bool { index >= input.endIndex }
+
+
private var peek: Character? {
+
guard index < input.endIndex else { return nil }
+
return input[index]
+
}
+
+
private func lookahead() -> Character? {
+
guard index < input.endIndex else { return nil }
+
let next = input.index(after: index)
+
guard next < input.endIndex else { return nil }
+
return input[next]
+
}
+
+
private mutating func advance() -> Character {
+
let c = input[index]
+
index = input.index(after: index)
+
+
return c
+
}
+
+
private mutating func consume(while predicate: (Character) -> Bool) {
+
while let c = peek, predicate(c) {
+
_ = advance()
+
}
+
}
+
+
func text(from start: String.Index) -> String {
+
let range = start..<index
+
return String(input[range])
+
}
+
+
public mutating func nextToken() -> (kind: TokenKind, text: String)? {
+
guard let c = peek else {
+
return nil
+
}
+
+
let start = index
+
+
if c.isNewline {
+
_ = advance()
+
return (kind: .newline, text: text(from: start))
+
}
+
+
if c.isWhitespace && !c.isNewline {
+
consume { $0.isWhitespace && !$0.isNewline }
+
return (kind: .whitespace, text: text(from: start))
+
}
+
+
if c == "/" && lookahead() == "/" {
+
_ = advance()
+
_ = advance()
+
consume { $0 != "\n" }
+
return (kind: .lineComment, text: text(from: start))
+
}
+
+
if c == "/" && lookahead() == "*" {
+
_ = advance() // consume '/'
+
_ = advance() // consume '*'
+
var terminated = false
+
+
while let ch = peek {
+
if ch == "*" && lookahead() == "/" {
+
_ = advance() // consume '*'
+
_ = advance() // consume '/'
+
terminated = true
+
break
+
}
+
_ = advance()
+
}
+
+
return (kind: .blockComment(terminated: terminated), text: text(from: start))
+
}
+
+
if c == "<" && lookahead() == "=" {
+
_ = advance()
+
_ = advance()
+
return (kind: .punctuation(.doubleLeftArrow), text: text(from: start))
+
}
+
+
if c == "=" && lookahead() == ">" {
+
_ = advance()
+
_ = advance()
+
return (kind: .punctuation(.doubleRightArrow), text: text(from: start))
+
}
+
+
if let punct = Punctuation(rawValue: String(c)) {
+
_ = advance()
+
return (.punctuation(punct), String(c))
+
} else if c.isLetter || c == "_" {
+
_ = advance()
+
consume { $0.isLetter || $0.isNumber || $0 == "_" }
+
let text = text(from: start)
+
if let keyword = Keyword(rawValue: text) {
+
return (kind: .keyword(keyword), text: text)
+
}
+
return (kind: .identifier, text: text)
+
}
+
+
// Invalid single char (don’t drop input)
+
let ch = advance()
+
return (kind: .error, text: String(ch))
+
}
+
+
public mutating func tokenize() -> [Token] {
+
var tokens: [Token] = []
+
+
while !isAtEnd {
+
guard let token = nextToken() else { break }
+
tokens.append(
+
Token(kind: token.kind, text: token.text)
+
)
+
}
+
+
let eofToken = Token(kind: .eof, text: "")
+
tokens.append(eofToken)
+
+
return tokens
+
}
+
}
+30
Sources/PterodactylSyntax/LineMap.swift
···
+
// SPDX-FileCopyrightText: 2025 The Project Pterodactyl Developers
+
//
+
// SPDX-License-Identifier: MPL-2.0
+
+
import Algorithms
+
import Foundation
+
+
public struct LineMap: Codable {
+
private var utf16LineOffsets: [Int] = [0]
+
+
public init(source: String) {
+
for idx in source.indices {
+
let c = source[idx]
+
if c == "\n" || c == "\r\n" || c == "\r" {
+
let next = source.index(after: idx)
+
let utf16Offset = next.utf16Offset(in: source)
+
utf16LineOffsets.append(utf16Offset)
+
}
+
}
+
}
+
+
public func location(at utf16Offset: Int) -> (line: Int, column: Int) {
+
let partitioningIndex = utf16LineOffsets.partitioningIndex { $0 > utf16Offset }
+
let lineIndex = partitioningIndex == 0 ? 0 : partitioningIndex - 1
+
let lineStart = utf16LineOffsets[lineIndex]
+
let lineNumber = lineIndex + 1
+
let columnNumber = utf16Offset - lineStart + 1
+
return (lineNumber, columnNumber)
+
}
+
}
+52 -23
Sources/PterodactylSyntax/Parser.swift
···
public struct Parser {
enum Event: Equatable {
-
case open(kind: SyntaxTreeKind)
+
case open(kind: SyntaxTreeKind, metadata: SyntaxTreeMetadata?)
case close
-
case advance
+
case advance(metadata: TokenMetadata?)
}
public struct MarkOpened {
···
public mutating func open() -> MarkOpened {
let mark = MarkOpened(index: events.count)
-
events.append(.open(kind: .error))
+
events.append(.open(kind: .error, metadata: nil))
return mark
}
-
public mutating func close(mark: MarkOpened, kind: SyntaxTreeKind) {
-
events[mark.index] = .open(kind: kind)
+
public mutating func close(mark: MarkOpened, kind: SyntaxTreeKind, metadata: SyntaxTreeMetadata?) {
+
events[mark.index] = .open(kind: kind, metadata: metadata)
events.append(.close)
}
-
public mutating func advance() {
+
public mutating func advance(metadata: TokenMetadata?) {
precondition(!isEndOfFile)
-
events.append(.advance)
+
events.append(.advance(metadata: metadata))
absoluteUtf16Offset += currentToken.utf16Length
position += 1
fuel = 256
}
-
public mutating func advance(error: String?) {
+
public mutating func advance(error: String?, metadata: TokenMetadata? = nil) {
let mark = open()
if let error {
let diagnostic = Diagnostic(
···
diagnostics.append(diagnostic)
}
-
advance()
-
close(mark: mark, kind: .error)
+
advance(metadata: metadata)
+
close(mark: mark, kind: .error, metadata: nil)
}
public mutating func lookahead(_ k: Int) -> TokenKind? {
···
return tokens[index].kind
}
-
public mutating func eat(kindSatisfying predicate: (TokenKind) -> Bool) -> Bool {
+
public mutating func eat(kindSatisfying predicate: (TokenKind) -> Bool, metadata: TokenMetadata?) -> Bool {
guard !isEndOfFile && isAt(kindSatisfying: predicate) else { return false }
-
advance()
+
advance(metadata: metadata)
return true
}
-
public mutating func eat(kind: TokenKind) -> Bool {
-
eat { $0 == kind }
+
public mutating func eat(kind: TokenKind, metadata: TokenMetadata?) -> Bool {
+
eat(kindSatisfying: { $0 == kind }, metadata: metadata)
}
-
public mutating func expect(kind: TokenKind, error: String? = nil) {
-
if eat(kind: kind) { return }
+
public mutating func expect(kind: TokenKind, metadata: TokenMetadata?, error: String? = nil) {
+
if eat(kind: kind, metadata: metadata) { return }
let diagnostic = Diagnostic(
message: error ?? "Expected \(kind) but got \(currentToken.kind): `\(currentToken.text)`",
absoluteRange: absoluteRangeAtCursor
···
public var tree: SyntaxTree {
var events = events
-
var stack: [SyntaxTree] = []
+
var stack: [SyntaxTree.Builder] = []
var cursor: Int = 0
precondition(events.popLast() == .close)
for event in events {
switch event {
-
case .open(let kind):
-
stack.append(SyntaxTree(kind: kind, children: []))
+
case .open(let kind, let metadata):
+
stack.append(SyntaxTree.Builder(kind: kind, metadata: metadata, children: []))
case .close:
let tree = stack.popLast()!
stack.modifyLast { last in
-
last.children.append(.tree(tree))
+
last.children.append(.tree(tree.tree))
}
-
case .advance:
+
case .advance(let metadata):
let token = tokens[cursor]
cursor += 1
stack.modifyLast { last in
-
last.children.append(.token(token))
+
last.children.append(.token(token, metadata: metadata))
}
}
}
assert(stack.count == 1)
-
return stack.popLast()!
+
return stack.popLast()!.tree
+
}
+
+
mutating func eatTrivium() -> Bool {
+
switch currentToken.kind {
+
case .whitespace:
+
advance(metadata: nil)
+
return true
+
case .blockComment(let terminated):
+
let metadata = TokenMetadata(
+
semanticTokenType: .comment,
+
delimitedFoldingRangeKind: .comment
+
)
+
if terminated {
+
advance(metadata: metadata)
+
} else {
+
advance(error: "Block comment was not terminated")
+
}
+
return true
+
case .lineComment:
+
advance(metadata: TokenMetadata(semanticTokenType: .comment))
+
return true
+
default:
+
return false
+
}
}
+
+
mutating func eatTrivia() {
+
while !isEndOfFile && eatTrivium() {}
+
}
+
}
extension Array {
+69
Sources/PterodactylSyntax/SemanticToken.swift
···
+
// SPDX-FileCopyrightText: 2025 The Project Pterodactyl Developers
+
//
+
// SPDX-License-Identifier: MPL-2.0
+
+
import Foundation
+
import LanguageServerProtocol
+
+
extension SemanticTokenTypes {
+
public var index: Int {
+
Self.allCases.firstIndex(of: self)!
+
}
+
}
+
+
extension SemanticTokenModifiers {
+
private static let modifierBitPositions: [SemanticTokenModifiers: Int] = {
+
var dict: [SemanticTokenModifiers: Int] = [:]
+
for (i, modifier) in SemanticTokenModifiers.allCases.enumerated() {
+
dict[modifier] = i
+
}
+
return dict
+
}()
+
+
static func encodeBitset(_ modifiers: Set<SemanticTokenModifiers>) -> UInt32 {
+
var bitset: UInt32 = 0
+
for modifier in modifiers {
+
if let bit = modifierBitPositions[modifier] {
+
bitset |= (1 << bit)
+
}
+
}
+
return bitset
+
}
+
}
+
+
+
struct SingleLineRange {
+
let line: Int
+
let char: Int
+
let length: Int
+
}
+
+
+
extension TokenMetadata {
+
func semanticToken(range: SingleLineRange) -> SemanticToken? {
+
guard range.length > 0 else { return nil }
+
return SemanticToken(
+
line: UInt32(range.line),
+
char: UInt32(range.char),
+
length: UInt32(range.length),
+
type: UInt32(semanticTokenType.index),
+
modifiers: SemanticTokenModifiers.encodeBitset(semanticTokenModifiers)
+
)
+
}
+
}
+
+
extension SyntaxTree.Cursor {
+
var singleLineRanges: [SingleLineRange] {
+
var result: [SingleLineRange] = []
+
var location = lineMap.location(at: utf16Offset)
+
+
for line in node.text.split(omittingEmptySubsequences: false, whereSeparator: \.isNewline) {
+
let length = line.utf16.count
+
result.append(SingleLineRange(line: location.line, char: location.column, length: length))
+
location.line += 1
+
location.column = 0
+
}
+
+
return result
+
}
+
}
+40 -15
Sources/PterodactylSyntax/SyntaxTree.swift
···
import Foundation
public struct SyntaxTree: Codable, Sendable {
-
public var kind: SyntaxTreeKind
-
public var metadata: SyntaxTreeMetadata?
-
public var children: [Child]
+
public let kind: SyntaxTreeKind
+
public let metadata: SyntaxTreeMetadata?
+
public let children: [Child]
+
public let utf16Length: Int
-
public var utf16Length: Int {
-
children.reduce(0) { length, child in
-
length + child.utf16Length
-
}
+
public enum Child: Codable, Sendable {
+
case token(Token, metadata: TokenMetadata?)
+
case tree(SyntaxTree)
}
-
public enum Child: Codable, Sendable {
-
case token(Token, metadata: TokenMetadata? = nil)
-
case tree(SyntaxTree)
-
}
-
-
public init(kind: SyntaxTreeKind, metadata: SyntaxTreeMetadata? = nil, children: [SyntaxTree.Child]) {
+
public init(kind: SyntaxTreeKind, metadata: SyntaxTreeMetadata? = nil, children: [SyntaxTree.Child]) {
self.kind = kind
self.metadata = metadata
self.children = children
+
self.utf16Length = children.reduce(0) { length, child in
+
length + child.utf16Length
+
}
+
}
+
}
+
+
extension SyntaxTree {
+
/// A mutable version of ``SyntaxTree`` that does not keep track of textual length, for use when constructing trees.
+
public struct Builder {
+
public var kind: SyntaxTreeKind
+
public var metadata: SyntaxTreeMetadata?
+
public var children: [Child]
+
+
var tree: SyntaxTree {
+
SyntaxTree(kind: kind, metadata: metadata, children: children)
+
}
}
}
···
}
extension SyntaxTree.Child {
-
var text: String {
+
public var text: String {
switch self {
case let .token(tok, _): tok.text
case let .tree(tree): tree.text
}
}
+
+
public var tree: SyntaxTree? {
+
switch self {
+
case let .tree(tree): tree
+
default: nil
+
}
+
}
+
var token: (Token, TokenMetadata?)? {
+
switch self {
+
case let .token(token, metadata): (token, metadata)
+
default: nil
+
}
+
}
+
var utf16Length: Int {
switch self {
case let .token(token, _): token.utf16Length
case let .tree(tree): tree.utf16Length
}
}
-
+
var children: [Self] {
switch self {
case .token: []
+21
Sources/PterodactylSyntax/SyntaxView.swift
···
+
// SPDX-FileCopyrightText: 2025 The Project Pterodactyl Developers
+
//
+
// SPDX-License-Identifier: MPL-2.0
+
+
import Foundation
+
+
struct SyntaxView<G: Grammar> {
+
let cursor: SyntaxTree.Cursor
+
init?(_ cursor: SyntaxTree.Cursor) {
+
guard let kind = cursor.node.tree?.kind, G.kinds.contains(kind) else { return nil }
+
self.cursor = cursor
+
}
+
+
func matchingSubview<X: Grammar>() -> SyntaxView<X>? {
+
return cursor.firstChild(mapping: SyntaxView<X>.init)
+
}
+
+
func matchingSubviews<X: Grammar>() -> [SyntaxView<X>] {
+
return cursor.children(mapping: SyntaxView<X>.init)
+
}
+
}
+1 -1
Sources/PterodactylSyntax/Token.swift
···
public let text: String
public let utf16Length: Int
-
init(kind: TokenKind, text: String) {
+
public init(kind: TokenKind, text: String) {
self.kind = kind
self.text = text
self.utf16Length = text.utf16.count
+60 -30
Sources/PterodactylSyntax/Types.swift
···
// SPDX-License-Identifier: MPL-2.0
import Foundation
+
import LanguageServerProtocol
-
public enum Keyword: Codable, Equatable, Sendable {
-
case `import`
-
case theory
-
case `where`
+
public enum Keyword: String, Codable, Sendable, CaseIterable {
+
case theory = "theory"
+
case `where` = "where"
+
case `import` = "import"
}
-
public enum Whitespace: Codable, Equatable, Sendable {
-
case newline
-
case other
+
public enum Punctuation: String, CaseIterable, Codable, Equatable, Sendable {
+
case lparen = "("
+
case rparen = ")"
+
case lbrace = "{"
+
case rbrace = "}"
+
case comma = ","
+
case dot = "."
+
case colon = ":"
+
case doubleLeftArrow = "<="
+
case doubleRightArrow = "=>"
+
case equal = "="
}
public enum TokenKind: Codable, Equatable, Sendable {
-
case eof
+
case eof
case keyword(Keyword)
+
case punctuation(Punctuation)
case error
case identifier
-
case whitespace(Whitespace)
+
case newline
+
case whitespace
case blockBegin
case blockEnd
case blockSep
+
case lineComment
+
case blockComment(terminated: Bool)
+
}
+
+
extension TokenKind {
+
public var isTrivia: Bool {
+
switch self {
+
case .whitespace, .newline, .lineComment, .blockComment: true
+
default: false
+
}
+
}
+
+
public var isVisible: Bool {
+
switch self {
+
case .whitespace, .blockBegin, .blockSep, .blockEnd, .newline: false
+
default: true
+
}
+
}
+
public var canDetermineLayoutColumn: Bool {
+
switch self {
+
case .whitespace, .eof: false
+
default: true
+
}
+
}
+
+
public var isBlockHerald: Bool {
+
switch self {
+
case .keyword(.where): true
+
default: false
+
}
+
}
+
}
public final class SyntaxTreeKind: Codable, Equatable, Sendable {
static let error: SyntaxTreeKind = .init(name: "error")
-
+
public static func == (lhs: SyntaxTreeKind, rhs: SyntaxTreeKind) -> Bool {
lhs === rhs
}
-
+
let name: String
var description: String { name }
-
+
required init(name: String) {
self.name = name
}
}
-
public struct TokenMetadata: Codable, Equatable, Sendable {
+
public struct TokenMetadata: Equatable, Codable, Sendable {
+
public var semanticTokenType: SemanticTokenTypes
+
public var semanticTokenModifiers: Set<SemanticTokenModifiers> = []
+
public var delimitedFoldingRangeKind: FoldingRangeKind? = nil
}
public struct SyntaxTreeMetadata: Codable, Equatable, Sendable {
-
-
}
-
-
extension TokenKind {
-
var canDetermineLayoutColumn: Bool {
-
switch self {
-
case .whitespace, .eof: false
-
default: true
-
}
-
}
-
-
var isBlockHerald: Bool {
-
switch self {
-
case .keyword(.where): true
-
default: false
-
}
-
}
+
public var delimitedFoldingRangeKind: FoldingRangeKind? = nil
}
+1 -1
Tests/PterodactylBuildTests/Test.swift
···
]
)
-
let dependenciesOfBaz = try await engine.build(key: Keys.TransitiveDependencies(sourceTreeId: treeID, unitName: baz), ctx).get().dependencies
+
let dependenciesOfBaz = try await engine.build(key: Keys.TransitiveDependencies(sourceTreeId: treeID, unitName: baz), ctx).get()
#expect(dependenciesOfBaz == [foo, bar])
return
}