···
-
from scipy.stats import ttest_rel
-
from pathlib import Path
-
# Define metrics of interest (can be expanded as needed)
-
METRIC_PREFIXES = ("nr", "gc")
-
def flatten_data(json_data: dict) -> dict:
-
Extracts and flattens metrics from JSON data.
-
This is needed because the JSON data can be nested.
-
For example, the JSON data entry might look like this:
-
"gc":{"cycles":13,"heapSize":5404549120,"totalBytes":9545876464}
-
"gc.heapSize": 5404549120
-
json_data (dict): JSON data containing metrics.
-
dict: Flattened metrics with keys as metric names.
-
for k, v in json_data.items():
-
if isinstance(v, (int, float)):
-
elif isinstance(v, dict):
-
for sub_k, sub_v in v.items():
-
flat_metrics[f"{k}.{sub_k}"] = sub_v
-
def load_all_metrics(directory: Path) -> dict:
-
Loads all stats JSON files in the specified directory and extracts metrics.
-
directory (Path): Directory containing JSON files.
-
dict: Dictionary with filenames as keys and extracted metrics as values.
-
for system_dir in directory.iterdir():
-
assert system_dir.is_dir()
-
for chunk_output in system_dir.iterdir():
-
with chunk_output.open() as f:
-
metrics[f"{system_dir.name}/${chunk_output.name}"] = flatten_data(data)
-
def dataframe_to_markdown(df: pd.DataFrame) -> str:
-
# Header (get column names and format them)
-
header = '\n| ' + ' | '.join(df.columns) + ' |'
-
markdown_lines.append(header)
-
markdown_lines.append("| - " * (len(df.columns)) + "|") # Separator line
-
# Iterate over rows to build Markdown rows
-
for _, row in df.iterrows():
-
# TODO: define threshold for highlighting
-
fmt = lambda x: f"**{x}**" if highlight else f"{x}"
-
# Check for no change and NaN in p_value/t_stat
-
if isinstance(val, float) and np.isnan(val): # For NaN values in p-value or t-stat
-
row_values.append("-") # Custom symbol for NaN
-
elif isinstance(val, float) and val == 0: # For no change (mean_diff == 0)
-
row_values.append("-") # Custom symbol for no change
-
row_values.append(fmt(f"{val:.4f}" if isinstance(val, float) else str(val)))
-
markdown_lines.append('| ' + ' | '.join(row_values) + ' |')
-
return '\n'.join(markdown_lines)
-
def perform_pairwise_tests(before_metrics: dict, after_metrics: dict) -> pd.DataFrame:
-
common_files = sorted(set(before_metrics) & set(after_metrics))
-
all_keys = sorted({ metric_keys for file_metrics in before_metrics.values() for metric_keys in file_metrics.keys() })
-
before_vals, after_vals = [], []
-
for fname in common_files:
-
if key in before_metrics[fname] and key in after_metrics[fname]:
-
before_vals.append(before_metrics[fname][key])
-
after_vals.append(after_metrics[fname][key])
-
if len(before_vals) >= 2:
-
before_arr = np.array(before_vals)
-
after_arr = np.array(after_vals)
-
diff = after_arr - before_arr
-
pct_change = 100 * diff / before_arr
-
t_stat, p_val = ttest_rel(after_arr, before_arr)
-
"mean_before": np.mean(before_arr),
-
"mean_after": np.mean(after_arr),
-
"mean_diff": np.mean(diff),
-
"mean_%_change": np.mean(pct_change),
-
df = pd.DataFrame(results).sort_values("p_value")
-
if __name__ == "__main__":
-
before_dir = os.environ.get("BEFORE_DIR")
-
after_dir = os.environ.get("AFTER_DIR")
-
if not before_dir or not after_dir:
-
print("Error: Environment variables 'BEFORE_DIR' and 'AFTER_DIR' must be set.")
-
before_metrics = load_all_metrics(Path(before_dir) / "stats")
-
after_metrics = load_all_metrics(Path(after_dir) / "stats")
-
df1 = perform_pairwise_tests(before_metrics, after_metrics)
-
markdown_table = dataframe_to_markdown(df1)
pyrox.dev