Spaces:
Sleeping
Sleeping
| import pandas as pd | |
| import os | |
| from evaluations import documentation, requirements, training, validating, license, weights, pitfalls | |
| from evaluations.utils import * | |
| import zipfile | |
| import os | |
| import numpy as np | |
| from huggingface_hub import InferenceClient | |
| def evaluate(llm, verbose, repo_url, title=None, year=None, zip=None): | |
| try: | |
| if (not(llm)): | |
| log(verbose, "LOG", "No LLM will be used for the evaluation.") | |
| results = { "pred_live": "Yes", "pred_dependencies": None, "pred_training": None, "pred_evaluation": None, "pred_weights": None, "pred_readme": None, "pred_license": None, "pred_stars": None, "pred_citations": None, "pred_valid": False} | |
| if ((title != None) & (year != None) & (title != "") & (year != "")): | |
| res = fetch_openalex(verbose, title, year) | |
| if ((res != None)): | |
| res = res["results"] | |
| if (len(res) > 0): | |
| res = res[0] | |
| results["pred_citations"] = res["cited_by_count"] | |
| if (get_api_link(repo_url) != ""): | |
| results["pred_valid"] = True | |
| else: | |
| return results | |
| username, repo_name = decompose_url(repo_url) | |
| # If you don't provide a zip file, it will be fetched from github. For this, you need to provide a github token. | |
| if (zip is None): | |
| token = os.getenv("githubToken") | |
| repository_zip_name = "data/repo.zip" | |
| log(verbose, "LOG", f"Fetching github repository: https://github.com/{username}/{repo_name}") | |
| fetch_repo(verbose, repo_url, repository_zip_name, token) | |
| if (not(os.path.exists(repository_zip_name))): | |
| results["pred_live"] = "No" | |
| return results | |
| results["pred_stars"] = fetch_repo_stars(verbose, repo_url, token) | |
| zip = zipfile.ZipFile(repository_zip_name) | |
| readme = fetch_readme(zip) | |
| results["NA"] = documentation.is_applicable(verbose, llm, readme) | |
| results["pred_license"] = license.evaluate(verbose, llm, zip, readme) | |
| if (len(zip.namelist()) <= 2): | |
| log(verbose, "LOG", "The repository is empty.") | |
| results["pred_dependencies"] = requirements.evaluate(verbose, llm, zip, readme) | |
| results["pred_training"] = training.evaluate(verbose, llm, zip, readme) | |
| results["pred_evaluation"] = validating.evaluate(verbose, llm, zip, readme) | |
| results["pred_weights"] = weights.evaluate(verbose, llm, zip, readme) | |
| results["pred_readme"] = documentation.evaluate(verbose, llm, zip, readme) | |
| results["pred_codetocomment"] = documentation.get_code_to_comment_ratio(zip) | |
| pitfalls.evaluate(verbose, llm, zip, readme) | |
| return results | |
| except Exception as e: | |
| log(verbose, "ERROR", "Evaluating repository failed: " + str(e)) | |
| results["pred_live"] = "No" | |
| return results | |
| def full_evaluation(): | |
| paper_dump = pd.read_csv("data/zipfiles.csv", sep="\t") | |
| full_results = [] | |
| for idx, row in paper_dump.iterrows(): | |
| if (pd.isna(row["url"]) | (row["url"] == "")): | |
| continue | |
| print(str(int(100 * idx / paper_dump["title"].count())) + "% done") | |
| result = evaluate(None, False, row["url"], row["title"], row["year"], zip=zipfile.ZipFile(row["zip_idx"])) | |
| for column in result.keys(): | |
| row[column] = result[column] | |
| full_results.append(row) | |
| return pd.DataFrame(full_results) | |