[New feature] Externals #2

closed
opened by danabra.mov targeting main from external

A step towards https://tangled.org/@danabra.mov/typelex/issues/5.

This adds support for Externals, which are namespaces with @external decorator.

@external
namespace com.atproto.label.defs {
  model Label { }
}

@external
namespace app.bsky.graph.defs {
  model StarterPackViewBasic { }
  model ListViewBasic { }
}

They do NOT get emitted as JSON. They only exist to represent external Lexicons for which you have only JSON.

Models inside @external namespaces must be empty. However, they're allowed to be @tokens which we need to know to disambiguate unions and knownValues. (In the future, it's possible we won't rely on knowing that, but for now we do.)

I've converted the playground to use externals properly. I will convert tests in a follow-up.

Changed files
+310 -95
packages
emitter
lib
src
test
spec
basic
output
com
example
external
input
test
output
playground
samples
+15
packages/emitter/lib/decorators.tsp
···
import "../dist/tsp-index.js";
/**
* Specifies the maximum number of graphemes (user-perceived characters) allowed.
* Used alongside maxLength for proper Unicode text handling.
···
* ```
*/
extern dec errors(target: unknown, ...errors: unknown[]);
···
import "../dist/tsp-index.js";
+
using TypeSpec.Reflection;
+
/**
* Specifies the maximum number of graphemes (user-perceived characters) allowed.
* Used alongside maxLength for proper Unicode text handling.
···
* ```
*/
extern dec errors(target: unknown, ...errors: unknown[]);
+
+
/**
+
* Marks a namespace as external, preventing it from emitting JSON output.
+
* This decorator can only be applied to namespaces.
+
* Useful for importing definitions from other lexicons without re-emitting them.
+
*
+
* @example
+
* ```typespec
+
* @external
+
* namespace com.atproto.repo.defs;
+
* ```
+
*/
+
extern dec external(target: Namespace);
+2
packages/emitter/src/tsp-index.ts
···
$inline,
$maxBytes,
$minBytes,
} from "./decorators.js";
/** @internal */
···
inline: $inline,
maxBytes: $maxBytes,
minBytes: $minBytes,
},
};
···
$inline,
$maxBytes,
$minBytes,
+
$external,
} from "./decorators.js";
/** @internal */
···
inline: $inline,
maxBytes: $maxBytes,
minBytes: $minBytes,
+
external: $external,
},
};
+23 -1
packages/emitter/test/spec.test.ts
···
assert.deepStrictEqual(actual, expected);
});
} else {
-
it.skip(`TODO: ${expectedPath} (add ${inputPath})`, function () {});
}
}
});
}
});
···
assert.deepStrictEqual(actual, expected);
});
} else {
+
it(`should emit ${expectedPath}`, function () {
+
assert.fail(
+
`Expected output file ${expectedPath} has no corresponding input file ${inputPath}. ` +
+
`Either add the input file or remove the expected output.`
+
);
+
});
}
}
+
+
// Check for unexpected emitted files
+
it("should not emit unexpected files", function () {
+
const emittedFiles = Object.keys(emitResult.files).filter(f => f.endsWith(".json"));
+
const expectedPaths = Object.keys(expectedFiles)
+
.filter(f => f.endsWith(".json"))
+
.map(normalizePathToPosix);
+
+
const unexpected = emittedFiles.filter(f => !expectedPaths.includes(f));
+
+
if (unexpected.length > 0) {
+
assert.fail(
+
`Unexpected files were emitted: ${unexpected.join(", ")}. ` +
+
`Either add expected output files or ensure these should not be emitted.`
+
);
+
}
+
});
});
}
});
+21
packages/emitter/test/spec/basic/output/com/example/other.json
···
···
+
{
+
"lexicon": 1,
+
"id": "com.example.other",
+
"defs": {
+
"main": {
+
"type": "object",
+
"properties": {}
+
},
+
"someDef": {
+
"type": "object",
+
"required": [
+
"value"
+
],
+
"properties": {
+
"value": {
+
"type": "string"
+
}
+
}
+
}
+
}
+
}
+7
packages/emitter/test/spec/external/input/test/normal.tsp
···
···
+
import "@typelex/emitter";
+
+
namespace test.normal {
+
model Main {
+
name?: string;
+
}
+
}
+14
packages/emitter/test/spec/external/output/test/normal.json
···
···
+
{
+
"lexicon": 1,
+
"id": "test.normal",
+
"defs": {
+
"main": {
+
"type": "object",
+
"properties": {
+
"name": {
+
"type": "string"
+
}
+
}
+
}
+
}
+
}
+66 -3
packages/playground/samples/build.js
···
// @ts-check
-
import { writeFileSync, mkdirSync } from "fs";
import { dirname, resolve, join } from "path";
import { fileURLToPath } from "url";
import { lexicons, bundleLexicon } from "./index.js";
const __dirname = dirname(fileURLToPath(import.meta.url));
const outputDir = resolve(__dirname, "dist");
// Create output directory
mkdirSync(outputDir, { recursive: true });
-
// Write each bundled lexicon to disk
const samplesList = {};
for (const [namespace, lexicon] of lexicons) {
···
writeFileSync(filepath, bundled);
samplesList[namespace] = {
filename: `samples/dist/${filename}`,
preferredEmitter: "@typelex/emitter",
···
const samplesIndex = `export default ${JSON.stringify(samplesList, null, 2)};`;
writeFileSync(join(outputDir, "samples.js"), samplesIndex);
-
console.log(`Wrote ${Object.keys(samplesList).length} bundled samples to disk`);
···
// @ts-check
+
import { writeFileSync, mkdirSync, readFileSync } from "fs";
import { dirname, resolve, join } from "path";
import { fileURLToPath } from "url";
+
import { deepStrictEqual } from "assert";
import { lexicons, bundleLexicon } from "./index.js";
+
import { createTestHost, findTestPackageRoot, resolveVirtualPath } from "@typespec/compiler/testing";
const __dirname = dirname(fileURLToPath(import.meta.url));
const outputDir = resolve(__dirname, "dist");
+
const pkgRoot = await findTestPackageRoot(import.meta.url);
+
+
// TypeSpec library setup for testing
+
const TypelexTestLibrary = {
+
name: "@typelex/emitter",
+
packageRoot: pkgRoot.replace("/playground", "/emitter"),
+
files: [
+
{ realDir: "", pattern: "package.json", virtualPath: "./node_modules/@typelex/emitter" },
+
{ realDir: "dist", pattern: "**/*.js", virtualPath: "./node_modules/@typelex/emitter/dist" },
+
{ realDir: "lib/", pattern: "*.tsp", virtualPath: "./node_modules/@typelex/emitter/lib" },
+
],
+
};
// Create output directory
mkdirSync(outputDir, { recursive: true });
+
// Write each bundled lexicon to disk and verify it compiles correctly
const samplesList = {};
for (const [namespace, lexicon] of lexicons) {
···
writeFileSync(filepath, bundled);
+
const host = await createTestHost({ libraries: [TypelexTestLibrary] });
+
host.addTypeSpecFile("main.tsp", bundled);
+
+
const baseOutputPath = resolveVirtualPath("test-output/");
+
const [, diagnostics] = await host.compileAndDiagnose("main.tsp", {
+
outputDir: baseOutputPath,
+
noEmit: false,
+
emit: ["@typelex/emitter"],
+
});
+
+
if (diagnostics.length > 0) {
+
console.error(`❌ ${namespace}: Compilation errors`);
+
diagnostics.forEach(d => console.error(` ${d.message}`));
+
process.exit(1);
+
}
+
+
// Get emitted JSON
+
const outputFiles = [...host.fs.entries()]
+
.filter(([name]) => name.startsWith(baseOutputPath))
+
.map(([name, value]) => {
+
let relativePath = name.replace(baseOutputPath, "");
+
if (relativePath.startsWith("@typelex/emitter/")) {
+
relativePath = relativePath.replace("@typelex/emitter/", "");
+
}
+
return [relativePath, value];
+
});
+
+
const expectedJsonPath = namespace.replace(/\./g, "/") + ".json";
+
const emittedJson = outputFiles.find(([path]) => path === expectedJsonPath);
+
+
if (!emittedJson) {
+
console.error(`❌ ${namespace}: No JSON output found (expected ${expectedJsonPath})`);
+
process.exit(1);
+
}
+
+
// Compare with expected JSON
+
const expectedJsonFile = join(
+
pkgRoot.replace("/playground", "/emitter"),
+
"test/integration",
+
lexicon.suite,
+
"output",
+
lexicon.file.replace(".tsp", ".json")
+
);
+
+
const expectedJson = JSON.parse(readFileSync(expectedJsonFile, "utf-8"));
+
const actualJson = JSON.parse(emittedJson[1]);
+
+
deepStrictEqual(actualJson, expectedJson);
+
samplesList[namespace] = {
filename: `samples/dist/${filename}`,
preferredEmitter: "@typelex/emitter",
···
const samplesIndex = `export default ${JSON.stringify(samplesList, null, 2)};`;
writeFileSync(join(outputDir, "samples.js"), samplesIndex);
+
console.log(`\n✅ ${lexicons.size} samples verified successfully`);
+151 -91
packages/playground/samples/index.js
···
const __dirname = dirname(fileURLToPath(import.meta.url));
-
// Get all tsp files
-
function getAllTspFiles(dir, baseDir = dir) {
const files = [];
const entries = readdirSync(dir);
···
const stat = statSync(fullPath);
if (stat.isDirectory()) {
-
files.push(...getAllTspFiles(fullPath, baseDir));
-
} else if (entry.endsWith(".tsp")) {
files.push(relative(baseDir, fullPath));
}
}
···
return files.sort();
}
-
// Extract dependencies from a file
-
function extractDependencies(content) {
-
const deps = new Set();
-
// Match namespace references like "com.atproto.label.defs.Label" or "com.atproto.repo.strongRef.Main"
-
// Pattern: word.word.word... followed by dot and identifier starting with capital letter
-
const pattern =
-
/\b([a-z]+(?:\.[a-z]+)+(?:\.[a-z][a-zA-Z]*)*)\.[A-Z][a-zA-Z]*/g;
-
const withoutDeclaration = content.replace(/namespace\s+[a-z.]+\s*\{/, "");
-
-
const matches = withoutDeclaration.matchAll(pattern);
-
for (const match of matches) {
-
deps.add(match[1]);
}
-
-
return Array.from(deps);
}
-
const atprotoInputDir = join(
-
__dirname,
-
"../../emitter/test/integration/atproto/input",
-
);
-
const lexiconExamplesDir = join(
-
__dirname,
-
"../../emitter/test/integration/lexicon-examples/input",
-
);
-
-
const atprotoFiles = getAllTspFiles(atprotoInputDir);
-
const lexiconExampleFiles = getAllTspFiles(lexiconExamplesDir);
-
-
// Build dependency graph
-
const lexicons = new Map(); // namespace -> { file, content, deps }
-
-
// Process atproto files
-
for (const file of atprotoFiles) {
-
const fullPath = join(atprotoInputDir, file);
-
const content = readFileSync(fullPath, "utf-8");
-
const namespace = file.replace(/\.tsp$/, "").replace(/\//g, ".");
-
const deps = extractDependencies(content);
-
-
lexicons.set(namespace, { file: `atproto/${file}`, content, deps });
-
}
-
// Process lexicon-examples files
-
for (const file of lexiconExampleFiles) {
-
const fullPath = join(lexiconExamplesDir, file);
-
const content = readFileSync(fullPath, "utf-8");
-
const namespace = file.replace(/\.tsp$/, "").replace(/\//g, ".");
-
const deps = extractDependencies(content);
-
lexicons.set(namespace, { file: `examples/${file}`, content, deps });
-
}
-
// Recursively collect all dependencies (topological sort)
-
function collectDependencies(
-
namespace,
-
collected = new Set(),
-
visiting = new Set(),
-
) {
-
if (collected.has(namespace)) return;
-
if (visiting.has(namespace)) return; // circular dependency
-
const lexicon = lexicons.get(namespace);
-
if (!lexicon) return;
-
visiting.add(namespace);
-
// First collect all dependencies
-
for (const dep of lexicon.deps) {
-
collectDependencies(dep, collected, visiting);
}
-
visiting.delete(namespace);
-
collected.add(namespace);
}
-
// Bundle a lexicon with all its dependencies
-
function bundleLexicon(namespace) {
-
const collected = new Set();
-
collectDependencies(namespace, collected);
-
// Put the main lexicon FIRST, then its dependencies
-
const mainLexicon = lexicons.get(namespace);
-
const deps = Array.from(collected).filter((ns) => ns !== namespace);
-
let bundled = 'import "@typelex/emitter";\n\n';
-
// Main lexicon first (so it shows in the playground)
-
if (mainLexicon) {
-
const contentWithoutImport = mainLexicon.content.replace(
-
/^import "@typelex\/emitter";\s*\n/,
-
"",
-
);
-
bundled += `// ${mainLexicon.file}\n${contentWithoutImport}\n`;
-
}
-
// Then dependencies
-
for (const ns of deps) {
-
const lexicon = lexicons.get(ns);
-
if (!lexicon) continue;
-
const contentWithoutImport = lexicon.content.replace(
-
/^import "@typelex\/emitter";\s*\n/,
-
"",
-
);
-
bundled += `// ${lexicon.file}\n${contentWithoutImport}\n`;
}
return bundled;
···
const __dirname = dirname(fileURLToPath(import.meta.url));
+
// Get all tsp and json files
+
function getAllFiles(dir, baseDir = dir) {
const files = [];
const entries = readdirSync(dir);
···
const stat = statSync(fullPath);
if (stat.isDirectory()) {
+
files.push(...getAllFiles(fullPath, baseDir));
+
} else if (entry.endsWith(".tsp") || entry.endsWith(".json")) {
files.push(relative(baseDir, fullPath));
}
}
···
return files.sort();
}
+
// Extract all refs from JSON (recursively search for strings with #)
+
function extractRefsFromJson(obj, refs = new Map()) {
+
if (typeof obj === "string") {
+
// Match pattern like "foo.bar#baz" or "foo.barCamel#baz" (must have # to be a ref)
+
const match = obj.match(/^([a-z][a-zA-Z.]+)#([a-z][a-zA-Z]*)$/);
+
if (match) {
+
const ns = match[1];
+
const def = match[2];
+
const modelName = def.charAt(0).toUpperCase() + def.slice(1);
+
if (!refs.has(ns)) {
+
refs.set(ns, new Set());
+
}
+
refs.get(ns).add(modelName);
+
} else {
+
// Also match plain namespace refs like "foo.bar.baz" or "foo.bar.bazCamel" (must have at least 2 dots)
+
const nsMatch = obj.match(/^([a-z][a-zA-Z]*(?:\.[a-z][a-zA-Z]*){2,})$/);
+
if (nsMatch) {
+
const ns = nsMatch[1];
+
if (!refs.has(ns)) {
+
refs.set(ns, new Set());
+
}
+
refs.get(ns).add("Main");
+
}
+
}
+
} else if (Array.isArray(obj)) {
+
for (const item of obj) {
+
extractRefsFromJson(item, refs);
+
}
+
} else if (obj && typeof obj === "object") {
+
for (const value of Object.values(obj)) {
+
extractRefsFromJson(value, refs);
+
}
}
+
return refs;
}
+
const integrationDir = join(__dirname, "../../emitter/test/integration");
+
// Get all test suite directories
+
const testSuites = readdirSync(integrationDir).filter((name) => {
+
const fullPath = join(integrationDir, name);
+
return statSync(fullPath).isDirectory() && !name.startsWith(".");
+
});
+
// Build lexicons with refs extracted from JSON
+
const lexicons = new Map(); // namespace -> { file, content, refs, suite }
+
// Process all test suites
+
for (const suite of testSuites) {
+
const inputDir = join(integrationDir, suite, "input");
+
const outputDir = join(integrationDir, suite, "output");
+
const inputFiles = getAllFiles(inputDir).filter((f) => f.endsWith(".tsp"));
+
+
for (const file of inputFiles) {
+
const fullPath = join(inputDir, file);
+
const content = readFileSync(fullPath, "utf-8");
+
const namespace = file.replace(/\.tsp$/, "").replace(/\//g, ".");
+
// Find corresponding JSON output
+
const jsonFile = file.replace(/\.tsp$/, ".json");
+
const jsonPath = join(outputDir, jsonFile);
+
const jsonContent = readFileSync(jsonPath, "utf-8");
+
const jsonData = JSON.parse(jsonContent);
+
const refs = extractRefsFromJson(jsonData);
+
lexicons.set(namespace, { file, content, refs, suite });
}
+
}
+
// TypeSpec reserved keywords that need escaping
+
const TYPESPEC_KEYWORDS = new Set([
+
"record",
+
"pub",
+
"interface",
+
"model",
+
"namespace",
+
"op",
+
"import",
+
"export",
+
"using",
+
"alias",
+
"enum",
+
"union",
+
"scalar",
+
"extends",
+
]);
+
+
// Escape a namespace part if it's a reserved keyword
+
function escapeNamespacePart(part) {
+
return TYPESPEC_KEYWORDS.has(part) ? `\`${part}\`` : part;
}
+
// Escape a full namespace path
+
function escapeNamespace(namespace) {
+
return namespace.split(".").map(escapeNamespacePart).join(".");
+
}
+
// Get the JSON for a lexicon to check its definitions
+
function getLexiconJson(namespace) {
+
const lexicon = lexicons.get(namespace);
+
if (!lexicon) return null;
+
+
const jsonPath = join(
+
integrationDir,
+
lexicon.suite,
+
"output",
+
lexicon.file.replace(".tsp", ".json"),
+
);
+
+
try {
+
return JSON.parse(readFileSync(jsonPath, "utf-8"));
+
} catch {
+
return null;
+
}
+
}
+
// Check if a definition in JSON is a token
+
function isToken(lexiconJson, defName) {
+
if (!lexiconJson || !lexiconJson.defs) return false;
+
const def = lexiconJson.defs[defName];
+
return def && def.type === "token";
+
}
+
// Bundle a lexicon with stubs for referenced types (from JSON)
+
function bundleLexicon(namespace) {
+
const mainLexicon = lexicons.get(namespace);
+
if (!mainLexicon) return "";
+
+
let bundled = mainLexicon.content;
+
+
// Add stubs from refs extracted from JSON output (excluding self-references)
+
if (mainLexicon.refs.size > 0) {
+
let hasExternalRefs = false;
+
for (const [ns] of mainLexicon.refs) {
+
if (ns !== namespace) {
+
hasExternalRefs = true;
+
break;
+
}
+
}
+
if (hasExternalRefs) {
+
bundled += "\n// --- Externals ---\n";
+
}
+
for (const [ns, models] of mainLexicon.refs) {
+
// Skip if this is the current namespace
+
if (ns === namespace) continue;
+
+
// Get the JSON for this referenced namespace to check for tokens
+
const refJson = getLexiconJson(ns);
+
+
const escapedNs = escapeNamespace(ns);
+
bundled += `\n@external\nnamespace ${escapedNs} {\n`;
+
for (const model of models) {
+
// Check if this definition exists in the JSON and is a token
+
const defName = model.charAt(0).toLowerCase() + model.slice(1);
+
if (refJson && isToken(refJson, defName)) {
+
bundled += ` @token model ${model} { }\n`;
+
} else {
+
bundled += ` model ${model} { }\n`;
+
}
+
}
+
bundled += `}\n`;
+
}
}
return bundled;
+11
packages/emitter/src/emitter.ts
···
// Skip external namespaces - they don't emit JSON files
if (isExternal(this.program, ns)) {
return;
}
···
// Skip external namespaces - they don't emit JSON files
if (isExternal(this.program, ns)) {
+
// Validate that all models in external namespaces are empty (stub-only)
+
for (const [_, model] of ns.models) {
+
if (model.properties && model.properties.size > 0) {
+
this.program.reportDiagnostic({
+
code: "external-model-not-empty",
+
severity: "error",
+
message: `Models in @external namespaces must be empty stubs. Model '${model.name}' in namespace '${fullName}' has properties.`,
+
target: model,
+
});
+
}
+
}
return;
}