Spaces:
Sleeping
Sleeping
File size: 2,939 Bytes
3cfadc8 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 |
import os
import pandas as pd
import numpy as np
compare_to_gt = True
ground_truth = pd.read_csv("data/zipfiles.csv", sep="\t")
results = pd.read_csv("data/results.csv", sep="\t")
verbose = 0
eval_readme = []
eval_training = []
eval_evaluating = []
eval_licensing = []
eval_weights = []
eval_dependencies = []
full_results = []
for (index1, row1), (index2, row2) in zip(ground_truth.iterrows(), results.iterrows()):
if (pd.isna(row1["training"])):
continue
print(f"\nEvaluating {index1+1} out of {len(ground_truth.index)} papers...")
print(f'Paper title - "{row1["title"]}" ({row1["year"]})')
print(f'Repository link - {row1["url"]}')
if ((not(pd.isna(row1["dependencies"]))) & (row2["pred_dependencies"] is not None)):
eval_dependencies.append(row2["pred_dependencies"] == row1["dependencies"])
if (row2["pred_dependencies"] != row1["dependencies"]):
print(f"Dependencies acc. - {row2['pred_dependencies']} (GT:{row1['dependencies']})")
if ((not(pd.isna(row1["training"]))) & (row2["pred_dependencies"] is not None)):
eval_training.append(row1["training"] == row2["pred_training"])
if (row1["training"] != row2["pred_training"]):
print(f"Training acc. -{row2['pred_training']} (GT:{row1['training']})")
if ((not(pd.isna(row1["evaluation"]))) & (row2["pred_dependencies"] is not None)):
eval_evaluating.append(row1["evaluation"] == row2["pred_evaluation"])
if (row1["evaluation"] != row2["pred_evaluation"]):
print(f"Evaluating acc. - {row2['pred_evaluation']} (GT:{row1['evaluation']})")
if ((not(pd.isna(row1["weights"]))) & (row2["pred_dependencies"] is not None)):
eval_weights.append(row1["weights"] == row2["pred_weights"])
if (row1["weights"] != row2["pred_weights"]):
print(f"Weights acc. - {row2['pred_weights']} (GT:{row1['weights']})")
if ((not(pd.isna(row1["readme"]))) & (row2["pred_dependencies"] is not None)):
eval_readme.append(row1["readme"] == row2["pred_readme"])
if (row1["readme"] != row2["pred_readme"]):
print(f"README acc. - {row2['pred_readme']} (GT:{row1['readme']})")
if ((not(pd.isna(row1["license"]))) & (row2["pred_dependencies"] is not None)):
eval_licensing.append(("No" if row1["license"] == "No" else "Yes") == row2["pred_license"])
if (("No" if row1["license"] == "No" else "Yes") != row2["pred_license"]):
print(f"LICENSE acc. - {row2['pred_license']} (GT:{row1['license']})")
print("\nSummary:")
print(f"Dependencies acc. - {int(100 * np.mean(eval_dependencies))}%")
print(f"Training acc. - {int(100 * np.mean(eval_training))}%")
print(f"Evaluating acc. - {int(100 * np.mean(eval_evaluating))}%")
print(f"Weights acc. - {int(100 * np.mean(eval_weights))}%")
print(f"README acc. - {int(100 * np.mean(eval_readme))}%")
print(f"LICENSE acc. - {int(100 * np.mean(eval_licensing))}%") |