Create app.py

app.py

@@ -0,0 +1,224 @@
+import os
+import io
+import requests
+import dotenv
+from bs4 import BeautifulSoup
+import gradio as gr
+import pandas as pd
+import time
+
+# 加载环境变量（其中需要包含 S2_API_KEY）
+dotenv.load_dotenv()
+
+def get_papers_since(url, since_year=2023):
+    """
+    给定一个 Google Scholar citations page URL，返回指定年份（默认2023）及之后的论文列表。
+    
+    每篇论文用一个字典表示：{"title": ..., "year": ...}
+    """
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"
+    }
+    
+    papers = []
+    start = 0
+    pagesize = 20  # Google Scholar 通常每页显示 20 条记录
+    
+    while True:
+        params = {"cstart": start, "pagesize": pagesize}
+        response = requests.get(url, params=params, headers=headers)
+        if response.status_code != 200:
+            break
+        
+        soup = BeautifulSoup(response.text, "html.parser")
+        rows = soup.find_all("tr", class_="gsc_a_tr")
+        if not rows:
+            break
+        
+        for row in rows:
+            title_tag = row.find("a", class_="gsc_a_at")
+            title = title_tag.text.strip() if title_tag else "Unknown Title"
+            year_tag = row.find("span", class_="gsc_a_h")
+            try:
+                year = int(year_tag.text.strip()) if year_tag else None
+            except ValueError:
+                year = None
+            
+            if year and year >= since_year:
+                papers.append({"title": title, "year": year})
+        
+        next_button = soup.find("button", id="gsc_bpf_more")
+        if next_button and "disabled" not in next_button.attrs:
+            start += pagesize
+        else:
+            break
+    
+    return papers
+
+def search_paper_by_title(paper_title: str):
+    """
+    使用 Semantic Scholar "match" endpoint 根据标题搜索论文，返回 top-1 匹配结果。
+    
+    返回:
+    dict: 包含 'paperId', 'title', 'abstract', 'venue' 四个键的字典。
+          若找不到对应论文（HTTP 404 或无结果），则返回 None。
+          若出现其它错误则抛出异常。
+    """
+    base_url = "https://api.semanticscholar.org/graph/v1/paper/search/match"
+    params = {
+        "query": paper_title,
+        "fields": "paperId,abstract,title,venue"
+    }
+    headers = {
+        "x-api-key": os.getenv("S2_API_KEY")
+    }
+    
+    response = requests.get(base_url, params=params, headers=headers)
+
+    if response.status_code == 404:
+        return None
+    elif response.status_code != 200:
+        raise Exception(f"错误 {response.status_code}: {response.text}")
+    
+    data_list = response.json().get("data", [])
+    if not data_list:
+        return None
+    
+    paper_data = data_list[0]
+    return {
+        "paperId": paper_data.get("paperId"),
+        "title": paper_data.get("title"),
+        "abstract": paper_data.get("abstract"),
+        "venue": paper_data.get("venue")
+    }
+
+def process_profiles(profiles_text, wechat, progress=None):
+    """
+    1. 将用户输入的多个 Google Scholar profile links (以换行分隔) 转为列表
+    2. 爬取所有链接在 2023 年及之后的论文
+    3. 对每篇论文调用 Semantic Scholar 获取匹配信息，若无匹配则丢弃
+    4. 根据 paperId 去重
+    5. 返回一个 CSV 文件内容（bytes），用于用户下载
+    """
+    log_messages = ["开始处理..."]
+
+    def update_log(message):
+        log_messages.append(message)
+        return "\n".join(log_messages)
+    
+    if not profiles_text.strip():
+        return update_log("错误: 未提供任何链接"), None
+
+    if not wechat.strip():
+        return update_log("错误: 未提供微信号"), None
+
+    # 将用户输入按换行拆分
+    profile_urls = [line.strip() for line in profiles_text.splitlines() if line.strip()]
+    
+    message = f"已识别 {len(profile_urls)} 个档案链接，开始处理..."
+    yield update_log(message), None
+
+    all_papers = []
+    for i, url in enumerate(profile_urls):
+        message = f"处理第 {i+1}/{len(profile_urls)} 个档案: {url}"
+        yield update_log(message), None
+        
+        papers = get_papers_since(url, 2023)
+        all_papers.extend(papers)
+        
+        message = f"已从档案 {i+1}/{len(profile_urls)} 中收集 {len(papers)} 篇论文"
+        yield update_log(message), None
+
+    message = f"共收集到 {len(all_papers)} 篇论文，开始获取详细信息..."
+    yield update_log(message), None
+
+    paperid_map = {}
+    total_papers = len(all_papers)
+    
+    for i, paper in enumerate(all_papers):
+        title = paper["title"]
+        year = paper["year"]
+        
+        if i % 10 == 0 or i == total_papers - 1:
+            percent = round((i+1)/total_papers*100)
+            message = f"正在处理论文 {i+1}/{total_papers} ({percent}% 完成)"
+            yield update_log(message), None
+        
+        if i > 0 and i % 10 == 0:
+            time.sleep(1)
+        
+        try:
+            paper_info = search_paper_by_title(title)
+        except Exception as e:
+            continue
+        
+        if not paper_info or not paper_info.get("paperId"):
+            continue
+
+        pid = paper_info["paperId"]
+        if pid not in paperid_map:
+            paperid_map[pid] = {
+                "paperId": pid,
+                "title": paper_info["title"],
+                "abstract": paper_info["abstract"],
+                "venue": paper_info["venue"],
+                "year": year
+            }
+    
+    if not paperid_map:
+        message = "错误: 未找到任何匹配的论文"
+        return update_log(message), None
+
+    message = f"去重后共有 {len(paperid_map)} 篇论文，正在生成CSV..."
+    yield update_log(message), None
+
+    df = pd.DataFrame(list(paperid_map.values()))
+
+    # 使用微信号作为 CSV 文件名
+    temp_csv_path = f"{wechat.strip()}.csv"
+    df.to_csv(temp_csv_path, index=False, encoding='utf-8')
+
+    message = f"任务完成! 已生成包含 {len(paperid_map)} 篇论文的CSV文件: {temp_csv_path}"
+    yield update_log(message), temp_csv_path
+
+def build_app():
+    """
+    使用 gradio 搭建一个小型 Demo app。
+    """
+    with gr.Blocks() as demo:
+        gr.Markdown("## Google Scholar & Semantic Scholar 信息整合工具")
+        gr.Markdown("在下方输入任意多个 Google Scholar Profile 链接，每行一个，然后输入微信号，点击 **开始爬取**。")
+
+        profile_input = gr.Textbox(
+            lines=5,
+            placeholder="粘贴或输入多个 Google Scholar 个人页面链接（每行一个，需要下面格式 'https://scholar.google.com/citations?user=NVii64oAAAAJ'）"
+        )
+        # 新增微信号输入框
+        wechat_input = gr.Textbox(
+            label="微信号",
+            placeholder="请输入您的微信号"
+        )
+        
+        progress_output = gr.Textbox(
+            label="进度", 
+            value="等待开始...", 
+            lines=10,
+            interactive=False
+        )
+
+        download_output = gr.File(label="下载结果 CSV")
+
+        run_button = gr.Button("开始爬取")
+
+        # 传入两个输入值
+        run_button.click(
+            fn=process_profiles,
+            inputs=[profile_input, wechat_input],
+            outputs=[progress_output, download_output]
+        )
+
+    return demo
+
+if __name__ == "__main__":
+    app = build_app()
+    app.launch()