···
from scipy.stats import ttest_rel
from tabulate import tabulate
11
+
from typing import Final
def flatten_data(json_data: dict) -> dict:
···
90
+
def metric_table_name(name: str, explain: bool) -> str:
92
+
Returns the name of the metric, plus a footnote to explain it if needed.
94
+
return f"{name}[^{name}]" if explain else name
97
+
METRIC_EXPLANATION_FOOTNOTE: Final[str] = """
99
+
[^time.cpu]: Number of seconds of CPU time accounted by the OS to the Nix evaluator process. On UNIX systems, this comes from [`getrusage(RUSAGE_SELF)`](https://man7.org/linux/man-pages/man2/getrusage.2.html).
100
+
[^time.gc]: Number of seconds of CPU time accounted by the Boehm garbage collector to performing GC.
101
+
[^time.gcFraction]: What fraction of the total CPU time is accounted towards performing GC.
102
+
[^gc.cycles]: Number of times garbage collection has been performed.
103
+
[^gc.heapSize]: Size in bytes of the garbage collector heap.
104
+
[^gc.totalBytes]: Size in bytes of all allocations in the garbage collector.
105
+
[^envs.bytes]: Size in bytes of all `Env` objects allocated by the Nix evaluator. These are almost exclusively created by [`nix-env`](https://nix.dev/manual/nix/stable/command-ref/nix-env.html).
106
+
[^list.bytes]: Size in bytes of all [lists](https://nix.dev/manual/nix/stable/language/syntax.html#list-literal) allocated by the Nix evaluator.
107
+
[^sets.bytes]: Size in bytes of all [attrsets](https://nix.dev/manual/nix/stable/language/syntax.html#list-literal) allocated by the Nix evaluator.
108
+
[^symbols.bytes]: Size in bytes of all items in the Nix evaluator symbol table.
109
+
[^values.bytes]: Size in bytes of all values allocated by the Nix evaluator.
110
+
[^envs.number]: The count of all `Env` objects allocated.
111
+
[^nrAvoided]: The number of thunks avoided being created.
112
+
[^nrExprs]: The number of expression objects ever created.
113
+
[^nrFunctionCalls]: The number of function calls ever made.
114
+
[^nrLookups]: The number of lookups into an attrset ever made.
115
+
[^nrOpUpdateValuesCopied]: The number of attrset values copied in the process of merging attrsets.
116
+
[^nrOpUpdates]: The number of attrsets merge operations (`//`) performed.
117
+
[^nrPrimOpCalls]: The number of function calls to primops (Nix builtins) ever made.
118
+
[^nrThunks]: The number of [thunks](https://nix.dev/manual/nix/latest/language/evaluation.html#laziness) ever made. A thunk is a delayed computation, represented by an expression reference and a closure.
119
+
[^sets.number]: The number of attrsets ever made.
120
+
[^symbols.number]: The number of symbols ever added to the symbol table.
121
+
[^values.number]: The number of values ever made.
122
+
[^envs.elements]: The number of values contained within an `Env` object.
123
+
[^list.concats]: The number of list concatenation operations (`++`) performed.
124
+
[^list.elements]: The number of values contained within a list.
125
+
[^sets.elements]: The number of values contained within an attrset.
126
+
[^sizes.Attr]: Size in bytes of the `Attr` type.
127
+
[^sizes.Bindings]: Size in bytes of the `Bindings` type.
128
+
[^sizes.Env]: Size in bytes of the `Env` type.
129
+
[^sizes.Value]: Size in bytes of the `Value` type.
def metric_sort_key(name: str) -> str:
if name in ("time.cpu", "time.gc", "time.gcFraction"):
···
102
-
def dataframe_to_markdown(df: pd.DataFrame) -> str:
146
+
def dataframe_to_markdown(df: pd.DataFrame, explain: bool) -> str:
by=df.columns[0], ascending=True, key=lambda s: s.map(metric_sort_key)
···
headers = [str(column) for column in df.columns]
155
+
# The metric acts as its own footnote name
156
+
metric_table_name(row["metric"], explain),
# Check for no change and NaN in p_value/t_stat
*[None if np.isnan(val) or np.allclose(val, 0) else val for val in row[1:]],
for _, row in df.iterrows()
118
-
return tabulate(table, headers, tablefmt="github", floatfmt=".4f", missingval="-")
163
+
result = tabulate(table, headers, tablefmt="github", floatfmt=".4f", missingval="-")
165
+
result += METRIC_EXPLANATION_FOOTNOTE
def perform_pairwise_tests(before_metrics: dict, after_metrics: dict) -> pd.DataFrame:
···
description="Performance comparison of Nix evaluation statistics"
225
+
"--explain", action="store_true", help="Explain the evaluation statistics"
227
+
parser.add_argument(
"before", help="File or directory containing baseline (data before)"
···
before_metrics = load_all_metrics(before_stats)
after_metrics = load_all_metrics(after_stats)
df1 = perform_pairwise_tests(before_metrics, after_metrics)
194
-
markdown_table = dataframe_to_markdown(df1)
245
+
markdown_table = dataframe_to_markdown(df1, explain=options.explain)
pyrox.dev