···
3
-
from scipy.stats import ttest_rel
6
-
from pathlib import Path
8
-
# Define metrics of interest (can be expanded as needed)
9
-
METRIC_PREFIXES = ("nr", "gc")
11
-
def flatten_data(json_data: dict) -> dict:
13
-
Extracts and flattens metrics from JSON data.
14
-
This is needed because the JSON data can be nested.
15
-
For example, the JSON data entry might look like this:
17
-
"gc":{"cycles":13,"heapSize":5404549120,"totalBytes":9545876464}
22
-
"gc.heapSize": 5404549120
26
-
json_data (dict): JSON data containing metrics.
28
-
dict: Flattened metrics with keys as metric names.
31
-
for k, v in json_data.items():
32
-
if isinstance(v, (int, float)):
34
-
elif isinstance(v, dict):
35
-
for sub_k, sub_v in v.items():
36
-
flat_metrics[f"{k}.{sub_k}"] = sub_v
42
-
def load_all_metrics(directory: Path) -> dict:
44
-
Loads all stats JSON files in the specified directory and extracts metrics.
47
-
directory (Path): Directory containing JSON files.
49
-
dict: Dictionary with filenames as keys and extracted metrics as values.
52
-
for system_dir in directory.iterdir():
53
-
assert system_dir.is_dir()
55
-
for chunk_output in system_dir.iterdir():
56
-
with chunk_output.open() as f:
58
-
metrics[f"{system_dir.name}/${chunk_output.name}"] = flatten_data(data)
62
-
def dataframe_to_markdown(df: pd.DataFrame) -> str:
65
-
# Header (get column names and format them)
66
-
header = '\n| ' + ' | '.join(df.columns) + ' |'
67
-
markdown_lines.append(header)
68
-
markdown_lines.append("| - " * (len(df.columns)) + "|") # Separator line
70
-
# Iterate over rows to build Markdown rows
71
-
for _, row in df.iterrows():
72
-
# TODO: define threshold for highlighting
75
-
fmt = lambda x: f"**{x}**" if highlight else f"{x}"
77
-
# Check for no change and NaN in p_value/t_stat
80
-
if isinstance(val, float) and np.isnan(val): # For NaN values in p-value or t-stat
81
-
row_values.append("-") # Custom symbol for NaN
82
-
elif isinstance(val, float) and val == 0: # For no change (mean_diff == 0)
83
-
row_values.append("-") # Custom symbol for no change
85
-
row_values.append(fmt(f"{val:.4f}" if isinstance(val, float) else str(val)))
87
-
markdown_lines.append('| ' + ' | '.join(row_values) + ' |')
89
-
return '\n'.join(markdown_lines)
92
-
def perform_pairwise_tests(before_metrics: dict, after_metrics: dict) -> pd.DataFrame:
93
-
common_files = sorted(set(before_metrics) & set(after_metrics))
94
-
all_keys = sorted({ metric_keys for file_metrics in before_metrics.values() for metric_keys in file_metrics.keys() })
98
-
for key in all_keys:
99
-
before_vals, after_vals = [], []
101
-
for fname in common_files:
102
-
if key in before_metrics[fname] and key in after_metrics[fname]:
103
-
before_vals.append(before_metrics[fname][key])
104
-
after_vals.append(after_metrics[fname][key])
106
-
if len(before_vals) >= 2:
107
-
before_arr = np.array(before_vals)
108
-
after_arr = np.array(after_vals)
110
-
diff = after_arr - before_arr
111
-
pct_change = 100 * diff / before_arr
112
-
t_stat, p_val = ttest_rel(after_arr, before_arr)
116
-
"mean_before": np.mean(before_arr),
117
-
"mean_after": np.mean(after_arr),
118
-
"mean_diff": np.mean(diff),
119
-
"mean_%_change": np.mean(pct_change),
124
-
df = pd.DataFrame(results).sort_values("p_value")
128
-
if __name__ == "__main__":
129
-
before_dir = os.environ.get("BEFORE_DIR")
130
-
after_dir = os.environ.get("AFTER_DIR")
132
-
if not before_dir or not after_dir:
133
-
print("Error: Environment variables 'BEFORE_DIR' and 'AFTER_DIR' must be set.")
136
-
before_metrics = load_all_metrics(Path(before_dir) / "stats")
137
-
after_metrics = load_all_metrics(Path(after_dir) / "stats")
139
-
df1 = perform_pairwise_tests(before_metrics, after_metrics)
140
-
markdown_table = dataframe_to_markdown(df1)
141
-
print(markdown_table)
pyrox.dev