adding app files

app.py

@@ -0,0 +1,95 @@
+import streamlit as st
+from transformers import pipeline
+import torch
+from transformers import AutoModelForSequenceClassification
+import pandas as pd
+from typing import Dict
+from transformers import DistilBertTokenizer
+from typing import List
+
+
+USED_MODEL = "distilbert-base-cased"
+
+@st.cache_resource  # кэширование
+def load_model():
+    # csv локально прочитать очень быстро, так что его не кешируем, хотя это не сложно было бы добавить наверное
+    arxiv_topics_df = pd.read_csv('arxiv_topics.csv')
+    tag_to_index = {}
+    for i, row in arxiv_topics_df.iterrows():
+        tag_to_index[row['tag']] = i
+    index_to_tag = {value: key for key, value in tag_to_index.items()}
+
+    return AutoModelForSequenceClassification.from_pretrained(
+        "bumchik2/train_distilbert-base-cased-tags-classification-simple", 
+        problem_type="multi_label_classification", 
+        num_labels=len(tag_to_index),
+        id2label=index_to_tag,
+        label2id=tag_to_index
+    ).to(torch.device('cuda'))
+
+model = load_model()
+
+
+@st.cache_resource()
+def get_tokenizer():
+    return DistilBertTokenizer.from_pretrained(USED_MODEL)
+
+
+def tokenize_function(text):
+    tokenizer = get_tokenizer()
+    return tokenizer(text, padding="max_length", truncation=True)
+
+
+@torch.no_grad
+def get_category_probs_dict(model, title: str, summary: str) -> Dict[str, float]:
+    # csv локально прочитать очень быстро, так что его не кешируем, хотя это не сложно было бы добавить наверное
+    arxiv_topics_df = pd.read_csv('arxiv_topics.csv')
+    tag_to_index = {}
+    tag_to_category = {}
+    for i, row in arxiv_topics_df.iterrows():
+        tag_to_category[row['tag']] = row['category']
+        tag_to_index[row['tag']] = i
+    index_to_tag = {value: key for key, value in tag_to_index.items()}
+
+    text = f'{title} $ {summary}'
+    tags_logits = model(**{key: torch.tensor(value).to(model.device).unsqueeze(0) for key, value in tokenize_function(text).items()}).logits
+    sigmoid = torch.nn.Sigmoid()
+    tags_probs = sigmoid(tags_logits.squeeze().cpu()).numpy()
+    tags_probs /= tags_probs.sum()
+    category_probs_dict = {category: 0.0 for category in set(arxiv_topics_df['category'])}
+    for index in range(len(index_to_tag)):
+        category_probs_dict[tag_to_category[index_to_tag[index]]] += float(tags_probs[index])
+    return category_probs_dict
+
+
+def get_most_probable_keys(probs_dict: Dict[str, float], target_probability: float, print_probabilities: bool) -> List[str]:
+    current_p = 0
+    probs_list = sorted([(value, key) for key, value in probs_dict.items()])[::-1]
+    current_index = 0
+    answer = []
+    while current_p <= target_probability:
+        current_p += probs_list[current_index][0]
+        if not print_probabilities:
+            answer.append(probs_list[current_index][1])
+        else:
+            answer.append(f'{probs_list[current_index][1]} ({probs_list[current_index][0]})')
+        current_index += 1
+        if current_index >= len(probs_list):
+            break
+    return answer
+
+
+title = st.text_input("Article title", value="Enter title here...")
+summary = st.text_input("Article summary", value="Enter summary here...")
+
+need_to_print_probabilities = st.radio("Need to print probabilities: ", ('Yes', 'No'), index=0)
+st.session_state['need_to_print_probabilities'] = need_to_print_probabilities
+
+target_probability = st.slider("Select minimum probability sum", 0.0, 1.0, step=0.01, value=0.95)
+st.session_state['target_probability'] = 'target_probability'
+
+
+if title and summary:
+    category_probs_dict = get_category_probs_dict(model=model, title=title, summary=summary)
+    result = get_most_probable_keys(probs_dict=category_probs_dict, target_probability=target_probability, print_probabilities=need_to_print_probabilities=='Yes')
+    st.write(result)

arxiv_topics.csv

@@ -0,0 +1,156 @@
+tag,topic,category
+cs.AI,Artificial Intelligence,Computer Science
+cs.AR,Hardware Architecture,Computer Science
+cs.CC,Computational Complexity,Computer Science
+cs.CE,"Computational Engineering, Finance, and Science",Computer Science
+cs.CG,Computational Geometry,Computer Science
+cs.CL,Computation and Language,Computer Science
+cs.CR,Cryptography and Security,Computer Science
+cs.CV,Computer Vision and Pattern Recognition,Computer Science
+cs.CY,Computers and Society,Computer Science
+cs.DB,Databases,Computer Science
+cs.DC,"Distributed, Parallel, and Cluster Computing",Computer Science
+cs.DL,Digital Libraries,Computer Science
+cs.DM,Discrete Mathematics,Computer Science
+cs.DS,Data Structures and Algorithms,Computer Science
+cs.ET,Emerging Technologies,Computer Science
+cs.FL,Formal Languages and Automata Theory,Computer Science
+cs.GL,General Literature,Computer Science
+cs.GR,Graphics,Computer Science
+cs.GT,Computer Science and Game Theory,Computer Science
+cs.HC,Human-Computer Interaction,Computer Science
+cs.IR,Information Retrieval,Computer Science
+cs.IT,Information Theory,Computer Science
+cs.LG,Machine Learning,Computer Science
+cs.LO,Logic in Computer Science,Computer Science
+cs.MA,Multiagent Systems,Computer Science
+cs.MM,Multimedia,Computer Science
+cs.MS,Mathematical Software,Computer Science
+cs.NA,Numerical Analysis,Computer Science
+cs.NE,Neural and Evolutionary Computing,Computer Science
+cs.NI,Networking and Internet Architecture,Computer Science
+cs.OH,Other Computer Science,Computer Science
+cs.OS,Operating Systems,Computer Science
+cs.PF,Performance,Computer Science
+cs.PL,Programming Languages,Computer Science
+cs.RO,Robotics,Computer Science
+cs.SC,Symbolic Computation,Computer Science
+cs.SD,Sound,Computer Science
+cs.SE,Software Engineering,Computer Science
+cs.SI,Social and Information Networks,Computer Science
+cs.SY,Systems and Control,Computer Science
+econ.EM,Econometrics,Economics
+econ.GN,General Economics,Economics
+econ.TH,Theoretical Economics,Economics
+eess.AS,Audio and Speech Processing,Electrical Engineering and Systems Science
+eess.IV,Image and Video Processing,Electrical Engineering and Systems Science
+eess.SP,Signal Processing,Electrical Engineering and Systems Science
+eess.SY,Systems and Control,Electrical Engineering and Systems Science
+math.AC,Commutative Algebra,Mathematics
+math.AG,Algebraic Geometry,Mathematics
+math.AP,Analysis of PDEs,Mathematics
+math.AT,Algebraic Topology,Mathematics
+math.CA,Classical Analysis and ODEs,Mathematics
+math.CO,Combinatorics,Mathematics
+math.CT,Category Theory,Mathematics
+math.CV,Complex Variables,Mathematics
+math.DG,Differential Geometry,Mathematics
+math.DS,Dynamical Systems,Mathematics
+math.FA,Functional Analysis,Mathematics
+math.GM,General Mathematics,Mathematics
+math.GN,General Topology,Mathematics
+math.GR,Group Theory,Mathematics
+math.GT,Geometric Topology,Mathematics
+math.HO,History and Overview,Mathematics
+math.IT,Information Theory,Mathematics
+math.KT,K-Theory and Homology,Mathematics
+math.LO,Logic,Mathematics
+math.MG,Metric Geometry,Mathematics
+math.MP,Mathematical Physics,Mathematics
+math.NA,Numerical Analysis,Mathematics
+math.NT,Number Theory,Mathematics
+math.OA,Operator Algebras,Mathematics
+math.OC,Optimization and Control,Mathematics
+math.PR,Probability,Mathematics
+math.QA,Quantum Algebra,Mathematics
+math.RA,Rings and Algebras,Mathematics
+math.RT,Representation Theory,Mathematics
+math.SG,Symplectic Geometry,Mathematics
+math.SP,Spectral Theory,Mathematics
+math.ST,Statistics Theory,Mathematics
+astro-ph.CO,Cosmology and Nongalactic Astrophysics,Physics
+astro-ph.EP,Earth and Planetary Astrophysics,Physics
+astro-ph.GA,Astrophysics of Galaxies,Physics
+astro-ph.HE,High Energy Astrophysical Phenomena,Physics
+astro-ph.IM,Instrumentation and Methods for Astrophysics,Physics
+astro-ph.SR,Solar and Stellar Astrophysics,Physics
+cond-mat.dis-nn,Disordered Systems and Neural Networks,Physics
+cond-mat.mes-hall,Mesoscale and Nanoscale Physics,Physics
+cond-mat.mtrl-sci,Materials Science,Physics
+cond-mat.other,Other Condensed Matter,Physics
+cond-mat.quant-gas,Quantum Gases,Physics
+cond-mat.soft,Soft Condensed Matter,Physics
+cond-mat.stat-mech,Statistical Mechanics,Physics
+cond-mat.str-el,Strongly Correlated Electrons,Physics
+cond-mat.supr-con,Superconductivity,Physics
+gr-qc,General Relativity and Quantum Cosmology,Physics
+hep-ex,High Energy Physics - Experiment,Physics
+hep-lat,High Energy Physics - Lattice,Physics
+hep-ph,High Energy Physics - Phenomenology,Physics
+hep-th,High Energy Physics - Theory,Physics
+math-ph,Mathematical Physics,Physics
+nlin.AO,Adaptation and Self-Organizing Systems,Physics
+nlin.CD,Chaotic Dynamics,Physics
+nlin.CG,Cellular Automata and Lattice Gases,Physics
+nlin.PS,Pattern Formation and Solitons,Physics
+nlin.SI,Exactly Solvable and Integrable Systems,Physics
+nucl-ex,Nuclear Experiment,Physics
+nucl-th,Nuclear Theory,Physics
+physics.acc-ph,Accelerator Physics,Physics
+physics.ao-ph,Atmospheric and Oceanic Physics,Physics
+physics.app-ph,Applied Physics,Physics
+physics.atm-clus,Atomic and Molecular Clusters,Physics
+physics.atom-ph,Atomic Physics,Physics
+physics.bio-ph,Biological Physics,Physics
+physics.chem-ph,Chemical Physics,Physics
+physics.class-ph,Classical Physics,Physics
+physics.comp-ph,Computational Physics,Physics
+physics.data-an,"Data Analysis, Statistics and Probability",Physics
+physics.ed-ph,Physics Education,Physics
+physics.flu-dyn,Fluid Dynamics,Physics
+physics.gen-ph,General Physics,Physics
+physics.geo-ph,Geophysics,Physics
+physics.hist-ph,History and Philosophy of Physics,Physics
+physics.ins-det,Instrumentation and Detectors,Physics
+physics.med-ph,Medical Physics,Physics
+physics.optics,Optics,Physics
+physics.plasm-ph,Plasma Physics,Physics
+physics.pop-ph,Popular Physics,Physics
+physics.soc-ph,Physics and Society,Physics
+physics.space-ph,Space Physics,Physics
+quant-ph,Quantum Physics,Physics
+q-bio.BM,Biomolecules,Quantitative Biology
+q-bio.CB,Cell Behavior,Quantitative Biology
+q-bio.GN,Genomics,Quantitative Biology
+q-bio.MN,Molecular Networks,Quantitative Biology
+q-bio.NC,Neurons and Cognition,Quantitative Biology
+q-bio.OT,Other Quantitative Biology,Quantitative Biology
+q-bio.PE,Populations and Evolution,Quantitative Biology
+q-bio.QM,Quantitative Methods,Quantitative Biology
+q-bio.SC,Subcellular Processes,Quantitative Biology
+q-bio.TO,Tissues and Organs,Quantitative Biology
+q-fin.CP,Computational Finance,Quantitative Finance
+q-fin.EC,Economics,Quantitative Finance
+q-fin.GN,General Finance,Quantitative Finance
+q-fin.MF,Mathematical Finance,Quantitative Finance
+q-fin.PM,Portfolio Management,Quantitative Finance
+q-fin.PR,Pricing of Securities,Quantitative Finance
+q-fin.RM,Risk Management,Quantitative Finance
+q-fin.ST,Statistical Finance,Quantitative Finance
+q-fin.TR,Trading and Market Microstructure,Quantitative Finance
+stat.AP,Applications,Statistics
+stat.CO,Computation,Statistics
+stat.ME,Methodology,Statistics
+stat.ML,Machine Learning,Statistics
+stat.OT,Other Statistics,Statistics
+stat.TH,Statistics Theory,Statistics

requirements.txt

@@ -0,0 +1,3 @@
+transformers
+torch
+pandas