Upload updated files

pages/0 FileChecker.py

@@ -1,5 +1,7 @@
 import streamlit as st
 import pandas as pd
+import json
+from tools import sourceformat as sf
 
 #===config===
 st.set_page_config(
@@ -34,10 +36,23 @@ def get_ext(extype):
 @st.cache_data(ttl=3600)
 def upload(extype):
     keywords = pd.read_csv(uploaded_file)
+    if "dimensions" in uploaded_file.name.lower():
+        keywords = sf.dim(keywords)
+        col_dict = {'MeSH terms': 'Keywords',
+        'PubYear': 'Year',
+        'Times cited': 'Cited by',
+        'Publication Type': 'Document Type'
+        }
+        keywords.rename(columns=col_dict, inplace=True)
     return keywords
 
 @st.cache_data(ttl=3600)
 def conv_txt(extype):
+    if("PMID" in (uploaded_file.read()).decode()):
+        uploaded_file.seek(0)
+        papers = sf.medline(uploaded_file)
+        print(papers)
+        return papers
     col_dict = {'TI': 'Title',
             'SO': 'Source title',
             'DE': 'Author Keywords',
@@ -45,16 +60,48 @@ def conv_txt(extype):
             'AB': 'Abstract',
             'TC': 'Cited by',
             'PY': 'Year',
-            'ID': 'Keywords Plus'}
-    keywords = pd.read_csv(uploaded_file, sep='\t', lineterminator='\r')
-    keywords.rename(columns=col_dict, inplace=True)
+            'ID': 'Keywords Plus',
+            'rights_date_used': 'Year'}
+    uploaded_file.seek(0)
+    papers = pd.read_csv(uploaded_file, sep='\t')
+    if("htid" in papers.columns):
+        papers = sf.htrc(papers)
+    papers.rename(columns=col_dict, inplace=True)
+    print(papers)
+    return papers
+
+
+@st.cache_data(ttl=3600)
+def conv_json(extype):
+    col_dict={'title': 'title',
+    'rights_date_used': 'Year',
+    'content_provider_code':'Source title'
+    }
+
+    data = json.load(uploaded_file)
+    hathifile = data['gathers']
+    keywords = pd.DataFrame.from_records(hathifile)
+
+    keywords = sf.htrc(keywords)
+    keywords['Cited by'] = keywords.groupby(['Keywords'])['Keywords'].transform('size')
+    keywords.rename(columns=col_dict,inplace=True)
+    return keywords
+
+@st.cache_data(ttl=3600)
+def conv_pub(extype):
+    if (get_ext(extype)).endswith('.tar.gz'):
+        bytedata = extype.read()
+        keywords = sf.readPub(bytedata)
+    elif (get_ext(extype)).endswith('.xml'):
+        bytedata = extype.read()
+        keywords = sf.readxml(bytedata)
     return keywords
 
 st.header('File Checker', anchor=False)
 st.subheader('Put your file here...', anchor=False)
 
 #===read data===
-uploaded_file = st.file_uploader('', type=['csv','txt'], on_change=reset_data)
+uploaded_file = st.file_uploader('', type=['csv','txt','json', 'tar.gz', 'xml'], on_change=reset_data)
 
 if uploaded_file is not None:
     extype = get_ext(uploaded_file)
@@ -64,8 +111,15 @@ if uploaded_file is not None:
     elif extype.endswith('.txt'):
         data = conv_txt(extype)
 
-    col1, col2, col3 = st.columns(3)
+    elif extype.endswith('.json'):
+        data = conv_json(extype)
     
+    elif extype.endswith('.tar.gz') or extype.endswith('.xml'):
+        data = conv_pub(uploaded_file)
+
+
+    col1, col2, col3 = st.columns(3)
+  
     with col1:
         #===check keywords===  
         keycheck = list(data.columns)
@@ -134,4 +188,4 @@ if uploaded_file is not None:
             container6.write("Unfortunately, you don't have a column containing object in your data. Please check again.")
         else:
             container6.subheader('✔️ Scattertext', divider='blue', anchor=False)
-            container6.write('Congratulations! You can use Scattertext')
+            container6.write('Congratulations! You can use Scattertext')

pages/1 Scattertext.py

@@ -10,6 +10,8 @@ nltk.download('stopwords')
 from nltk.corpus import stopwords
 import time
 import sys
+import json
+from tools import sourceformat as sf
 
 #===config===
 st.set_page_config(
@@ -37,6 +39,9 @@ with st.popover("🔗 Menu"):
     st.page_link("pages/4 Sunburst.py", label="Sunburst", icon="4️⃣")
     st.page_link("pages/5 Burst Detection.py", label="Burst Detection", icon="5️⃣")
     st.page_link("pages/6 Keywords Stem.py", label="Keywords Stem", icon="6️⃣")
+    st.page_link("pages/7 Sentiment Analysis.py", label="Sentiment Analysis", icon="7️⃣")
+    st.page_link("pages/8 Shifterator.py", label="Shifterator", icon="8️⃣")
+    st.page_link("pages/9 Summarization.py", label = "Summarization",icon ="9️⃣")
     
 st.header("Scattertext", anchor=False)
 st.subheader('Put your file here...', anchor=False)
@@ -57,19 +62,66 @@ def upload(extype):
     if 'Publication Year' in papers.columns:
                papers.rename(columns={'Publication Year': 'Year', 'Citing Works Count': 'Cited by',
                                      'Publication Type': 'Document Type', 'Source Title': 'Source title'}, inplace=True)
+    
+    elif "About the data" in papers.columns[0]:
+        papers = sf.dim(papers)
+        col_dict = {'MeSH terms': 'Keywords',
+        'PubYear': 'Year',
+        'Times cited': 'Cited by',
+        'Publication Type': 'Document Type'
+        }
+        papers.rename(columns=col_dict, inplace=True)
+    
     return papers
 
 @st.cache_data(ttl=3600)
 def conv_txt(extype):
+    if("PMID" in (uploaded_file.read()).decode()):
+        uploaded_file.seek(0)
+        papers = sf.medline(uploaded_file)
+        print(papers)
+        return papers
     col_dict = {'TI': 'Title',
             'SO': 'Source title',
+            'DE': 'Author Keywords',
             'DT': 'Document Type',
             'AB': 'Abstract',
-            'PY': 'Year'}
-    papers = pd.read_csv(uploaded_file, sep='\t', lineterminator='\r')
+            'TC': 'Cited by',
+            'PY': 'Year',
+            'ID': 'Keywords Plus',
+            'rights_date_used': 'Year'}
+    uploaded_file.seek(0)
+    papers = pd.read_csv(uploaded_file, sep='\t')
+    if("htid" in papers.columns):
+        papers = sf.htrc(papers)
     papers.rename(columns=col_dict, inplace=True)
+    print(papers)
     return papers
 
+@st.cache_data(ttl=3600)
+def conv_json(extype):
+    col_dict={'title': 'title',
+    'rights_date_used': 'Year',
+    }
+
+    data = json.load(uploaded_file)
+    hathifile = data['gathers']
+    keywords = pd.DataFrame.from_records(hathifile)
+
+    keywords = sf.htrc(keywords)
+    keywords.rename(columns=col_dict,inplace=True)
+    return keywords
+
+@st.cache_data(ttl=3600)
+def conv_pub(extype):
+    if (get_ext(extype)).endswith('.tar.gz'):
+        bytedata = extype.read()
+        keywords = sf.readPub(bytedata)
+    elif (get_ext(extype)).endswith('.xml'):
+        bytedata = extype.read()
+        keywords = sf.readxml(bytedata)
+    return keywords
+
 @st.cache_data(ttl=3600)
 def get_data(extype): 
     df_col = sorted(papers.select_dtypes(include=['object']).columns.tolist())
@@ -122,18 +174,24 @@ def clean_csv(extype):
           
     #===lemmatize===
     lemmatizer = WordNetLemmatizer()
+    
+    @st.cache_data(ttl=3600)
     def lemmatize_words(text):
         words = text.split()
         words = [lemmatizer.lemmatize(word) for word in words]
         return ' '.join(words)
+        
     paper[ColCho] = paper[ColCho].apply(lemmatize_words)
     
     words_rmv = [word.strip() for word in words_to_remove.split(";")]
     remove_set = set(words_rmv)
+    
+    @st.cache_data(ttl=3600)
     def remove_words(text):
         words = text.split()  
         cleaned_words = [word for word in words if word not in remove_set]
         return ' '.join(cleaned_words) 
+        
     paper[ColCho] = paper[ColCho].apply(remove_words)
          
     return paper
@@ -155,7 +213,11 @@ def running_scattertext(cat_col, catname, noncatname):
                                 nlp = stx.whitespace_nlp_with_sentences,
                                 ).build().get_unigram_corpus().remove_infrequent_words(minimum_term_count = min_term)        
                                 
-        st.toast('Building corpus completed', icon='🎉')
+        #table results
+        disp = stx.Dispersion(corpus)
+        disp_df = disp.get_df()
+
+        disp_csv = disp_df.to_csv(index=False).encode('utf-8')
             
         try:
             html = stx.produce_scattertext_explorer(corpus,
@@ -175,11 +237,8 @@ def running_scattertext(cat_col, catname, noncatname):
                                                 width_in_pixels = 900,
                                                 minimum_term_frequency = 0,
                                                 save_svg_button=True)
-    
-        st.toast('Process completed', icon='🎉')
-        time.sleep(1)
-        st.toast('Visualizing', icon='⏳')
-        components.html(html, height = 1200, scrolling = True) 
+
+        return disp_csv, html 
 
     except ValueError:
         st.warning('Please decrease the Minimum term count in the advanced settings.', icon="⚠️")
@@ -226,7 +285,7 @@ def df_years(first_range, second_range):
     return filtered_df 
 
 #===Read data===
-uploaded_file = st.file_uploader('', type=['csv', 'txt'], on_change=reset_all)
+uploaded_file = st.file_uploader('', type=['csv', 'txt', 'json', 'tar.gz','xml'], on_change=reset_all)
 
 if uploaded_file is not None:
     try:
@@ -236,6 +295,10 @@ if uploaded_file is not None:
              papers = upload(extype) 
         elif extype.endswith('.txt'):
              papers = conv_txt(extype)
+        elif extype.endswith('.json'):
+            papers = conv_json(extype)
+        elif extype.endswith('.tar.gz') or extype.endswith('.xml'):
+            papers = conv_pub(uploaded_file)
     
         df_col, selected_cols = get_data(extype)
         comparison = check_comparison(extype)
@@ -264,7 +327,7 @@ if uploaded_file is not None:
         
         paper = clean_csv(extype)
     
-        tab1, tab2, tab3 = st.tabs(["📈 Generate visualization", "📃 Reference", "📓 Recommended Reading"])
+        tab1, tab2, tab3, tab4 = st.tabs(["📈 Generate visualization", "📃 Reference", "📓 Recommended Reading", "⬇️ Download Help"])
     
         with tab1:
              #===visualization===
@@ -286,7 +349,7 @@ if uploaded_file is not None:
                     st.warning(f'We cannot find {text2} in your document.', icon="⚠️")
                 else:
                     with st.spinner('Processing. Please wait until the visualization comes up'):
-                        running_scattertext('Topic', 'First Term', 'Second Term')
+                        disp_df, html = running_scattertext('Topic', 'First Term', 'Second Term')
         
             elif compare == 'Manual label':
                 col1, col2, col3 = st.columns(3)
@@ -313,7 +376,7 @@ if uploaded_file is not None:
                 filtered_df = paper[paper[column_selected].isin([label1, label2])].reset_index(drop=True)
                 
                 with st.spinner('Processing. Please wait until the visualization comes up'):
-                    running_scattertext(column_selected, label1, label2)
+                    disp_df, html = running_scattertext(column_selected, label1, label2)
         
             elif compare == 'Sources':
                 col1, col2, col3 = st.columns([4,0.1,4])
@@ -334,7 +397,7 @@ if uploaded_file is not None:
                 filtered_df = df_sources(stitle1, stitle2)
         
                 with st.spinner('Processing. Please wait until the visualization comes up'):
-                    running_scattertext('Source title', stitle1, stitle2)
+                    disp_df, html = running_scattertext('Source title', stitle1, stitle2)
         
             elif compare == 'Years':
                 col1, col2, col3 = st.columns([4,0.1,4])
@@ -348,19 +411,44 @@ if uploaded_file is not None:
                     filtered_df = df_years(first_range, second_range)
         
                     with st.spinner('Processing. Please wait until the visualization comes up'):
-                        running_scattertext('Topic Range', 'First range', 'Second range')
-    
+                        disp_df, html = running_scattertext('Topic Range', 'First range', 'Second range')
+                        
                 else:
                     st.write('You only have data in ', (MAX))
+
+            if html:
+                st.toast('Process completed', icon='🎉')
+                time.sleep(1)
+                st.toast('Visualizing', icon='⏳')
+                components.html(html, height = 1200, scrolling = True)
+    
+                st.download_button(
+                    "📥 Click to download result",
+                    disp_df,
+                    "scattertext_dataframe.csv",
+                    "text/csv",
+                    on_click="ignore")
     
         with tab2:
             st.markdown('**Jason Kessler. 2017. Scattertext: a Browser-Based Tool for Visualizing how Corpora Differ. In Proceedings of ACL 2017, System Demonstrations, pages 85–90, Vancouver, Canada. Association for Computational Linguistics.** https://doi.org/10.48550/arXiv.1703.00565')
     
         with tab3:
+            st.markdown('**Sánchez-Franco, M. J., & Rey-Tienda, S. (2023). The role of user-generated content in tourism decision-making: an exemplary study of Andalusia, Spain. Management Decision, 62(7).** https://doi.org/10.1108/md-06-2023-0966')
             st.markdown('**Marrone, M., & Linnenluecke, M.K. (2020). Interdisciplinary Research Maps: A new technique for visualizing research topics. PLoS ONE, 15.** https://doi.org/10.1371/journal.pone.0242283')
             st.markdown('**Moreno, A., & Iglesias, C.A. (2021). Understanding Customers’ Transport Services with Topic Clustering and Sentiment Analysis. Applied Sciences.** https://doi.org/10.3390/app112110169')
-            st.markdown('**Sánchez-Franco, M.J., & Rey-Tienda, S. (2023). The role of user-generated content in tourism decision-making: an exemplary study of Andalusia, Spain. Management Decision.** https://doi.org/10.1108/MD-06-2023-0966')
+            st.markdown('**Santosa, F. A. (2025). Artificial Intelligence in Library Studies: A Textual Analysis. JLIS.It, 16(1).** https://doi.org/10.36253/jlis.it-626')
 
-    except:
+        with tab4:
+            st.subheader(':blue[Image]', anchor=False)
+            st.write("Click the :blue[Download SVG] on the right side.")  
+            st.divider()
+            st.subheader(':blue[Scattertext Dataframe]', anchor=False)
+            st.button('📥 Click to download result')
+            st.text("Click the Download button to get the CSV result.")
+
+    except NameError:
+        pass
+    
+    except Exception as e:
         st.error("Please ensure that your file is correct. Please contact us if you find that this is an error.", icon="🚨")
-        st.stop()
+        st.stop()

pages/2 Topic Modeling.py

@@ -1,445 +1,659 @@
-#import module
-import streamlit as st
-import streamlit.components.v1 as components
-import pandas as pd
-import numpy as np
-import re
-import nltk
-nltk.download('wordnet')
-from nltk.stem import WordNetLemmatizer
-nltk.download('stopwords')
-from nltk.corpus import stopwords
-#from scipy import triu
-import gensim
-import gensim.corpora as corpora
-from gensim.corpora import Dictionary
-from gensim.models.coherencemodel import CoherenceModel
-from gensim.models.ldamodel import LdaModel
-from pprint import pprint
-import pickle
-import pyLDAvis
-import pyLDAvis.gensim_models as gensimvis
-from io import StringIO
-from ipywidgets.embed import embed_minimal_html
-from nltk.stem.snowball import SnowballStemmer
-from bertopic import BERTopic
-import plotly.express as px
-from sklearn.cluster import KMeans
-import bitermplus as btm
-import tmplot as tmp
-import tomotopy
-import sys
-import spacy
-import en_core_web_md
-import pipeline
-from html2image import Html2Image
-from umap import UMAP
-import os
-import time
-
-
-#===config===
-st.set_page_config(
-    page_title="Coconut",
-    page_icon="🥥",
-    layout="wide",
-    initial_sidebar_state="collapsed"
-)
-
-hide_streamlit_style = """
-            <style>
-            #MainMenu 
-            {visibility: hidden;}
-            footer {visibility: hidden;}
-            [data-testid="collapsedControl"] {display: none}
-            </style>
-            """
-st.markdown(hide_streamlit_style, unsafe_allow_html=True)
-
-with st.popover("🔗 Menu"):
-    st.page_link("https://www.coconut-libtool.com/", label="Home", icon="🏠")
-    st.page_link("pages/1 Scattertext.py", label="Scattertext", icon="1️⃣")
-    st.page_link("pages/2 Topic Modeling.py", label="Topic Modeling", icon="2️⃣")
-    st.page_link("pages/3 Bidirected Network.py", label="Bidirected Network", icon="3️⃣")
-    st.page_link("pages/4 Sunburst.py", label="Sunburst", icon="4️⃣")
-    st.page_link("pages/5 Burst Detection.py", label="Burst Detection", icon="5️⃣")
-    st.page_link("pages/6 Keywords Stem.py", label="Keywords Stem", icon="6️⃣")
-
-st.header("Topic Modeling", anchor=False)
-st.subheader('Put your file here...', anchor=False)
-
-#========unique id========
-@st.cache_resource(ttl=3600)
-def create_list():
-    l = [1, 2, 3]
-    return l
-
-l = create_list()
-first_list_value = l[0]
-l[0] = first_list_value + 1
-uID = str(l[0])
-
-@st.cache_data(ttl=3600)
-def get_ext(uploaded_file):
-    extype = uID+uploaded_file.name
-    return extype
-
-#===clear cache===
-
-def reset_biterm():
-    try:
-        biterm_map.clear()
-        biterm_bar.clear()
-    except NameError:
-        biterm_topic.clear()
-
-def reset_all():
-    st.cache_data.clear()
-
-#===avoiding deadlock===
-os.environ["TOKENIZERS_PARALLELISM"] = "false"
-        
-#===upload file===
-@st.cache_data(ttl=3600)
-def upload(file):
-    papers = pd.read_csv(uploaded_file)
-    return papers
-
-@st.cache_data(ttl=3600)
-def conv_txt(extype):
-    col_dict = {'TI': 'Title',
-            'SO': 'Source title',
-            'DT': 'Document Type',
-            'AB': 'Abstract',
-            'PY': 'Year'}
-    papers = pd.read_csv(uploaded_file, sep='\t', lineterminator='\r')
-    papers.rename(columns=col_dict, inplace=True)
-    return papers
-
-
-#===Read data===
-uploaded_file = st.file_uploader('', type=['csv', 'txt'], on_change=reset_all)
-
-if uploaded_file is not None:
-    try:
-        extype = get_ext(uploaded_file)
-    
-        if extype.endswith('.csv'):
-             papers = upload(extype) 
-        elif extype.endswith('.txt'):
-             papers = conv_txt(extype)
-    
-        coldf = sorted(papers.select_dtypes(include=['object']).columns.tolist())
-            
-        c1, c2 = st.columns([3,4])
-        method = c1.selectbox(
-                'Choose method',
-                ('Choose...', 'pyLDA', 'Biterm', 'BERTopic'), on_change=reset_all)
-        num_cho = c1.number_input('Choose number of topics', min_value=2, max_value=30, value=5)
-        ColCho = c2.selectbox(
-                'Choose column',
-                (coldf), on_change=reset_all)
-        words_to_remove = c2.text_input("Remove specific words. Separate words by semicolons (;)")
-        rem_copyright = c1.toggle('Remove copyright statement', value=True, on_change=reset_all)
-        rem_punc = c2.toggle('Remove punctuation', value=True, on_change=reset_all)
-         
-        #===clean csv===
-        @st.cache_data(ttl=3600, show_spinner=False)
-        def clean_csv(extype):
-            paper = papers.dropna(subset=[ColCho])
-                     
-            #===mapping===
-            paper['Abstract_pre'] = paper[ColCho].map(lambda x: x.lower())
-            if rem_punc:
-                 paper['Abstract_pre'] = paper['Abstract_pre'].map(lambda x: re.sub('[,:;\.!-?•=]', ' ', x))
-                 paper['Abstract_pre'] = paper['Abstract_pre'].str.replace('\u201c|\u201d', '', regex=True) 
-            if rem_copyright:  
-                 paper['Abstract_pre'] = paper['Abstract_pre'].map(lambda x: re.sub('©.*', '', x))
-            
-            #===stopword removal===
-            stop = stopwords.words('english')
-            paper['Abstract_stop'] = paper['Abstract_pre'].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)]))
-              
-            #===lemmatize===
-            lemmatizer = WordNetLemmatizer()
-            def lemmatize_words(text):
-                words = text.split()
-                words = [lemmatizer.lemmatize(word) for word in words]
-                return ' '.join(words)
-            paper['Abstract_lem'] = paper['Abstract_stop'].apply(lemmatize_words)
-        
-            words_rmv = [word.strip() for word in words_to_remove.split(";")]
-            remove_dict = {word: None for word in words_rmv}
-            def remove_words(text):
-                 words = text.split()
-                 cleaned_words = [word for word in words if word not in remove_dict]
-                 return ' '.join(cleaned_words) 
-            paper['Abstract_lem'] = paper['Abstract_lem'].map(remove_words)
-             
-            topic_abs = paper.Abstract_lem.values.tolist()
-            return topic_abs, paper
-    
-        d1, d2 = st.columns([7,3]) 
-        d2.info("Don't do anything during the computing", icon="⚠️")
-        topic_abs, paper=clean_csv(extype) 
-    
-        #===advance settings===
-        with d1.expander("🧮 Show advance settings"): 
-             t1, t2 = st.columns([5,5])
-             if method == 'pyLDA':
-                  py_random_state = t1.number_input('Random state', min_value=0, max_value=None, step=1)
-                  py_chunksize = t2.number_input('Chunk size', value=100 , min_value=10, max_value=None, step=1)
-             elif method == 'Biterm':
-                  btm_seed = t1.number_input('Random state seed', value=100 , min_value=1, max_value=None, step=1)
-                  btm_iterations = t2.number_input('Iterations number', value=20 , min_value=2, max_value=None, step=1)
-             elif method == 'BERTopic':
-                  bert_top_n_words = t1.number_input('top_n_words', value=5 , min_value=5, max_value=25, step=1)
-                  bert_random_state = t1.number_input('random_state', value=42 , min_value=1, max_value=None, step=1)
-                  bert_n_components = t2.number_input('n_components', value=5 , min_value=1, max_value=None, step=1)
-                  bert_n_neighbors = t2.number_input('n_neighbors', value=15 , min_value=1, max_value=None, step=1)
-                  bert_embedding_model = st.radio(
-                       "embedding_model", 
-                       ["all-MiniLM-L6-v2", "paraphrase-multilingual-MiniLM-L12-v2", "en_core_web_md"], index=0, horizontal=True)
-             else:
-                  st.write('Please choose your preferred method')
-        if st.button("Submit", on_click=reset_all):
-             num_topic = num_cho  
-    
-        if method == 'BERTopic':
-            st.info('BERTopic is an expensive process when dealing with a large volume of text with our existing resources. Please kindly wait until the visualization appears.', icon="ℹ️")
-               
-        #===topic===
-        if method == 'Choose...':
-            st.write('')
-    
-        elif method == 'pyLDA':       
-            tab1, tab2, tab3 = st.tabs(["📈 Generate visualization", "📃 Reference", "📓 Recommended Reading"])
-    
-            with tab1:
-            #===visualization===
-                @st.cache_data(ttl=3600, show_spinner=False)
-                def pylda(extype):
-                    topic_abs_LDA = [t.split(' ') for t in topic_abs]
-                    id2word = Dictionary(topic_abs_LDA)
-                    corpus = [id2word.doc2bow(text) for text in topic_abs_LDA]
-                    #===LDA===
-                    lda_model = LdaModel(corpus=corpus,
-                                id2word=id2word,
-                                num_topics=num_topic, 
-                                random_state=py_random_state,
-                                chunksize=py_chunksize,
-                                alpha='auto',
-                                per_word_topics=True)
-         
-                    pprint(lda_model.print_topics())
-                    doc_lda = lda_model[corpus]
-         
-                    #===visualization===
-                    coherence_model_lda = CoherenceModel(model=lda_model, texts=topic_abs_LDA, dictionary=id2word, coherence='c_v')
-                    coherence_lda = coherence_model_lda.get_coherence()
-                    vis = pyLDAvis.gensim_models.prepare(lda_model, corpus, id2word)
-                    py_lda_vis_html = pyLDAvis.prepared_data_to_html(vis)
-                    return py_lda_vis_html, coherence_lda, vis
-                       
-                with st.spinner('Performing computations. Please wait ...'):
-                    try:
-                        py_lda_vis_html, coherence_lda, vis = pylda(extype)
-                        st.write('Coherence score: ', coherence_lda)
-                        components.html(py_lda_vis_html, width=1500, height=800)
-                        st.markdown('Copyright (c) 2015, Ben Mabey. https://github.com/bmabey/pyLDAvis')
-                           
-                        @st.cache_data(ttl=3600, show_spinner=False)
-                        def img_lda(vis):
-                            pyLDAvis.save_html(vis, 'output.html')
-                            hti = Html2Image()
-                            hti.browser.flags = ['--default-background-color=ffffff', '--hide-scrollbars']
-                            css = "body {background: white;}"
-                            hti.screenshot(
-                                other_file='output.html', css_str=css, size=(1500, 800),
-                                save_as='ldavis_img.png'
-                            )
-                                 
-                        img_lda(vis)   
-                        with open("ldavis_img.png", "rb") as file:
-                            btn = st.download_button(
-                                label="Download image",
-                                data=file,
-                                file_name="ldavis_img.png",
-                                mime="image/png"
-                                )
-                           
-                    except NameError:
-                        st.warning('🖱️ Please click Submit')
-    
-            with tab2:
-                st.markdown('**Sievert, C., & Shirley, K. (2014). LDAvis: A method for visualizing and interpreting topics. Proceedings of the Workshop on Interactive Language Learning, Visualization, and Interfaces.** https://doi.org/10.3115/v1/w14-3110')
-    
-            with tab3:
-                st.markdown('**Chen, X., & Wang, H. (2019, January). Automated chat transcript analysis using topic modeling for library reference services. Proceedings of the Association for Information Science and Technology, 56(1), 368–371.** https://doi.org/10.1002/pra2.31')
-                st.markdown('**Joo, S., Ingram, E., & Cahill, M. (2021, December 15). Exploring Topics and Genres in Storytime Books: A Text Mining Approach. Evidence Based Library and Information Practice, 16(4), 41–62.** https://doi.org/10.18438/eblip29963')
-                st.markdown('**Lamba, M., & Madhusudhan, M. (2021, July 31). Topic Modeling. Text Mining for Information Professionals, 105–137.** https://doi.org/10.1007/978-3-030-85085-2_4')
-                st.markdown('**Lamba, M., & Madhusudhan, M. (2019, June 7). Mapping of topics in DESIDOC Journal of Library and Information Technology, India: a study. Scientometrics, 120(2), 477–505.** https://doi.org/10.1007/s11192-019-03137-5')
-         
-         #===Biterm===
-        elif method == 'Biterm':            
-                 
-            #===optimize Biterm===
-            @st.cache_data(ttl=3600, show_spinner=False)
-            def biterm_topic(extype):
-                X, vocabulary, vocab_dict = btm.get_words_freqs(topic_abs)
-                tf = np.array(X.sum(axis=0)).ravel()
-                docs_vec = btm.get_vectorized_docs(topic_abs, vocabulary)
-                docs_lens = list(map(len, docs_vec))
-                biterms = btm.get_biterms(docs_vec)
-                model = btm.BTM(X, vocabulary, seed=btm_seed, T=num_topic, M=20, alpha=50/8, beta=0.01)
-                model.fit(biterms, iterations=btm_iterations)
-                p_zd = model.transform(docs_vec)
-                coherence = model.coherence_
-                phi = tmp.get_phi(model)
-                topics_coords = tmp.prepare_coords(model)
-                totaltop = topics_coords.label.values.tolist()
-                perplexity = model.perplexity_
-                return topics_coords, phi, totaltop, perplexity
-    
-            tab1, tab2, tab3 = st.tabs(["📈 Generate visualization", "📃 Reference", "📓 Recommended Reading"])
-            with tab1:
-                try:
-                    with st.spinner('Performing computations. Please wait ...'): 
-                        topics_coords, phi, totaltop, perplexity = biterm_topic(extype)            
-                        col1, col2 = st.columns([4,6])
-                      
-                        @st.cache_data(ttl=3600)
-                        def biterm_map(extype):
-                            btmvis_coords = tmp.plot_scatter_topics(topics_coords, size_col='size', label_col='label', topic=numvis)
-                            return btmvis_coords
-                                
-                        @st.cache_data(ttl=3600)
-                        def biterm_bar(extype):
-                            terms_probs = tmp.calc_terms_probs_ratio(phi, topic=numvis, lambda_=1)
-                            btmvis_probs = tmp.plot_terms(terms_probs, font_size=12)
-                            return btmvis_probs
-                                
-                        with col1:
-                            st.write('Perplexity score: ', perplexity)
-                            st.write('')
-                            numvis = st.selectbox(
-                                'Choose topic',
-                                (totaltop), on_change=reset_biterm)
-                            btmvis_coords = biterm_map(extype)
-                            st.altair_chart(btmvis_coords)
-                        with col2:
-                            btmvis_probs = biterm_bar(extype)
-                            st.altair_chart(btmvis_probs, use_container_width=True)
-    
-                except ValueError:
-                    st.error('🙇‍♂️ Please raise the number of topics and click submit')
-                except NameError:
-                    st.warning('🖱️ Please click Submit')
-    
-            with tab2: 
-                st.markdown('**Yan, X., Guo, J., Lan, Y., & Cheng, X. (2013, May 13). A biterm topic model for short texts. Proceedings of the 22nd International Conference on World Wide Web.** https://doi.org/10.1145/2488388.2488514')
-            with tab3:
-                st.markdown('**Cai, M., Shah, N., Li, J., Chen, W. H., Cuomo, R. E., Obradovich, N., & Mackey, T. K. (2020, August 26). Identification and characterization of tweets related to the 2015 Indiana HIV outbreak: A retrospective infoveillance study. PLOS ONE, 15(8), e0235150.** https://doi.org/10.1371/journal.pone.0235150')
-                st.markdown('**Chen, Y., Dong, T., Ban, Q., & Li, Y. (2021). What Concerns Consumers about Hypertension? A Comparison between the Online Health Community and the Q&A Forum. International Journal of Computational Intelligence Systems, 14(1), 734.** https://doi.org/10.2991/ijcis.d.210203.002')
-                st.markdown('**George, Crissandra J., "AMBIGUOUS APPALACHIANNESS: A LINGUISTIC AND PERCEPTUAL INVESTIGATION INTO ARC-LABELED PENNSYLVANIA COUNTIES" (2022). Theses and Dissertations-- Linguistics. 48.** https://doi.org/10.13023/etd.2022.217')
-                st.markdown('**Li, J., Chen, W. H., Xu, Q., Shah, N., Kohler, J. C., & Mackey, T. K. (2020). Detection of self-reported experiences with corruption on twitter using unsupervised machine learning. Social Sciences & Humanities Open, 2(1), 100060.** https://doi.org/10.1016/j.ssaho.2020.100060')
-              
-         #===BERTopic===
-        elif method == 'BERTopic':
-            @st.cache_data(ttl=3600, show_spinner=False)
-            def bertopic_vis(extype):
-                umap_model = UMAP(n_neighbors=bert_n_neighbors, n_components=bert_n_components, 
-                    min_dist=0.0, metric='cosine', random_state=bert_random_state)   
-                cluster_model = KMeans(n_clusters=num_topic)
-                if bert_embedding_model == 'all-MiniLM-L6-v2':
-                    emb_mod = 'all-MiniLM-L6-v2'
-                    lang = 'en'
-                elif bert_embedding_model == 'en_core_web_md':
-                    emb_mod = en_core_web_md.load(exclude=['tagger', 'parser', 'ner', 'attribute_ruler', 'lemmatizer'])
-                    lang = 'en'
-                elif bert_embedding_model == 'paraphrase-multilingual-MiniLM-L12-v2':
-                    emb_mod = 'paraphrase-multilingual-MiniLM-L12-v2'
-                    lang = 'multilingual'
-                topic_model = BERTopic(embedding_model=emb_mod, hdbscan_model=cluster_model, language=lang, umap_model=umap_model, top_n_words=bert_top_n_words)
-                topics, probs = topic_model.fit_transform(topic_abs)
-                return topic_model, topics, probs
-            
-            @st.cache_data(ttl=3600, show_spinner=False)
-            def Vis_Topics(extype):
-                fig1 = topic_model.visualize_topics()
-                return fig1
-            
-            @st.cache_data(ttl=3600, show_spinner=False)
-            def Vis_Documents(extype):
-                fig2 = topic_model.visualize_documents(topic_abs)
-                return fig2
-    
-            @st.cache_data(ttl=3600, show_spinner=False)
-            def Vis_Hierarchy(extype):
-                fig3 = topic_model.visualize_hierarchy(top_n_topics=num_topic)
-                return fig3
-        
-            @st.cache_data(ttl=3600, show_spinner=False)
-            def Vis_Heatmap(extype):
-                global topic_model
-                fig4 = topic_model.visualize_heatmap(n_clusters=num_topic-1, width=1000, height=1000)
-                return fig4
-    
-            @st.cache_data(ttl=3600, show_spinner=False)
-            def Vis_Barchart(extype):
-                fig5 = topic_model.visualize_barchart(top_n_topics=num_topic)
-                return fig5
-           
-            tab1, tab2, tab3 = st.tabs(["📈 Generate visualization", "📃 Reference", "📓 Recommended Reading"])
-            with tab1:
-                try:
-                    with st.spinner('Performing computations. Please wait ...'):
-                   
-                        topic_model, topics, probs = bertopic_vis(extype)
-                        time.sleep(.5)
-                        st.toast('Visualize Topics', icon='🏃')
-                        fig1 = Vis_Topics(extype)
-                       
-                        time.sleep(.5)
-                        st.toast('Visualize Document', icon='🏃')
-                        fig2 = Vis_Documents(extype)
-                       
-                        time.sleep(.5)
-                        st.toast('Visualize Document Hierarchy', icon='🏃')
-                        fig3 = Vis_Hierarchy(extype)
-                       
-                        time.sleep(.5)
-                        st.toast('Visualize Topic Similarity', icon='🏃')
-                        fig4 = Vis_Heatmap(extype)
-                       
-                        time.sleep(.5)
-                        st.toast('Visualize Terms', icon='🏃')
-                        fig5 = Vis_Barchart(extype)
-                       
-                        with st.expander("Visualize Topics"):
-                            st.write(fig1)
-                        with st.expander("Visualize Terms"):
-                            st.write(fig5)
-                        with st.expander("Visualize Documents"):
-                            st.write(fig2)
-                        with st.expander("Visualize Document Hierarchy"):  
-                            st.write(fig3)
-                        with st.expander("Visualize Topic Similarity"):
-                            st.write(fig4)
-                                            
-                except ValueError:
-                    st.error('🙇‍♂️ Please raise the number of topics and click submit')
-              
-                except NameError:
-                    st.warning('🖱️ Please click Submit')
-    
-            with tab2:
-                st.markdown('**Grootendorst, M. (2022). BERTopic: Neural topic modeling with a class-based TF-IDF procedure. arXiv preprint arXiv:2203.05794.** https://doi.org/10.48550/arXiv.2203.05794')
-              
-            with tab3:
-                st.markdown('**Jeet Rawat, A., Ghildiyal, S., & Dixit, A. K. (2022, December 1). Topic modelling of legal documents using NLP and bidirectional encoder representations from transformers. Indonesian Journal of Electrical Engineering and Computer Science, 28(3), 1749.** https://doi.org/10.11591/ijeecs.v28.i3.pp1749-1755')
-                st.markdown('**Yao, L. F., Ferawati, K., Liew, K., Wakamiya, S., & Aramaki, E. (2023, April 20). Disruptions in the Cystic Fibrosis Community’s Experiences and Concerns During the COVID-19 Pandemic: Topic Modeling and Time Series Analysis of Reddit Comments. Journal of Medical Internet Research, 25, e45249.** https://doi.org/10.2196/45249')
-
-    except:
-        st.error("Please ensure that your file is correct. Please contact us if you find that this is an error.", icon="🚨")
-        st.stop()
+#import module
+import streamlit as st
+import streamlit.components.v1 as components
+import pandas as pd
+import numpy as np
+import re
+import string
+import nltk
+nltk.download('wordnet')
+from nltk.stem import WordNetLemmatizer
+nltk.download('stopwords')
+from nltk.corpus import stopwords
+import gensim
+import gensim.corpora as corpora
+from gensim.corpora import Dictionary
+from gensim.models.coherencemodel import CoherenceModel
+from gensim.models.ldamodel import LdaModel
+from gensim.models import Phrases
+from gensim.models.phrases import Phraser
+from pprint import pprint
+import pickle
+import pyLDAvis
+import pyLDAvis.gensim_models as gensimvis
+from io import StringIO
+from ipywidgets.embed import embed_minimal_html
+from nltk.stem.snowball import SnowballStemmer
+from bertopic import BERTopic
+from bertopic.representation import KeyBERTInspired, MaximalMarginalRelevance, OpenAI, TextGeneration
+import plotly.express as px
+from sklearn.cluster import KMeans
+from sklearn.feature_extraction.text import CountVectorizer
+import bitermplus as btm
+import tmplot as tmp
+import tomotopy
+import sys
+import spacy
+import en_core_web_sm
+import pipeline
+from html2image import Html2Image
+from umap import UMAP
+import os
+import time
+import json
+from tools import sourceformat as sf
+import datamapplot
+from sentence_transformers import SentenceTransformer
+import openai
+from transformers import pipeline
+
+#===config===
+st.set_page_config(
+    page_title="Coconut",
+    page_icon="🥥",
+    layout="wide",
+    initial_sidebar_state="collapsed"
+)
+
+hide_streamlit_style = """
+            <style>
+            #MainMenu 
+            {visibility: hidden;}
+            footer {visibility: hidden;}
+            [data-testid="collapsedControl"] {display: none}
+            </style>
+            """
+st.markdown(hide_streamlit_style, unsafe_allow_html=True)
+
+with st.popover("🔗 Menu"):
+    st.page_link("https://www.coconut-libtool.com/", label="Home", icon="🏠")
+    st.page_link("pages/1 Scattertext.py", label="Scattertext", icon="1️⃣")
+    st.page_link("pages/2 Topic Modeling.py", label="Topic Modeling", icon="2️⃣")
+    st.page_link("pages/3 Bidirected Network.py", label="Bidirected Network", icon="3️⃣")
+    st.page_link("pages/4 Sunburst.py", label="Sunburst", icon="4️⃣")
+    st.page_link("pages/5 Burst Detection.py", label="Burst Detection", icon="5️⃣")
+    st.page_link("pages/6 Keywords Stem.py", label="Keywords Stem", icon="6️⃣")
+    st.page_link("pages/7 Sentiment Analysis.py", label="Sentiment Analysis", icon="7️⃣")
+
+st.header("Topic Modeling", anchor=False)
+st.subheader('Put your file here...', anchor=False)
+
+#========unique id========
+@st.cache_resource(ttl=3600)
+def create_list():
+    l = [1, 2, 3]
+    return l
+
+l = create_list()
+first_list_value = l[0]
+l[0] = first_list_value + 1
+uID = str(l[0])
+
+@st.cache_data(ttl=3600)
+def get_ext(uploaded_file):
+    extype = uID+uploaded_file.name
+    return extype
+
+#===clear cache===
+
+def reset_biterm():
+    try:
+        biterm_map.clear()
+        biterm_bar.clear()
+    except NameError:
+        biterm_topic.clear()
+
+def reset_all():
+    st.cache_data.clear()
+
+#===avoiding deadlock===
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+        
+#===upload file===
+@st.cache_data(ttl=3600)
+def upload(file):
+    papers = pd.read_csv(uploaded_file)
+    if "About the data" in papers.columns[0]:
+        papers = sf.dim(papers)
+        col_dict = {'MeSH terms': 'Keywords',
+        'PubYear': 'Year',
+        'Times cited': 'Cited by',
+        'Publication Type': 'Document Type'
+        }
+        papers.rename(columns=col_dict, inplace=True)
+    
+    return papers
+
+@st.cache_data(ttl=3600)
+def conv_txt(extype):
+    if("PMID" in (uploaded_file.read()).decode()):
+        uploaded_file.seek(0)
+        papers = sf.medline(uploaded_file)
+        print(papers)
+        return papers
+    col_dict = {'TI': 'Title',
+            'SO': 'Source title',
+            'DE': 'Author Keywords',
+            'DT': 'Document Type',
+            'AB': 'Abstract',
+            'TC': 'Cited by',
+            'PY': 'Year',
+            'ID': 'Keywords Plus',
+            'rights_date_used': 'Year'}
+    uploaded_file.seek(0)
+    papers = pd.read_csv(uploaded_file, sep='\t')
+    if("htid" in papers.columns):
+        papers = sf.htrc(papers)
+    papers.rename(columns=col_dict, inplace=True)
+    print(papers)
+    return papers
+
+
+@st.cache_data(ttl=3600)
+def conv_json(extype):
+    col_dict={'title': 'title',
+    'rights_date_used': 'Year',
+    }
+
+    data = json.load(uploaded_file)
+    hathifile = data['gathers']
+    keywords = pd.DataFrame.from_records(hathifile)
+    
+    keywords = sf.htrc(keywords)
+    keywords.rename(columns=col_dict,inplace=True)
+    return keywords
+
+@st.cache_resource(ttl=3600)
+def conv_pub(extype):
+    if (get_ext(extype)).endswith('.tar.gz'):
+        bytedata = extype.read()
+        keywords = sf.readPub(bytedata)
+    elif (get_ext(extype)).endswith('.xml'):
+        bytedata = extype.read()
+        keywords = sf.readxml(bytedata)
+    return keywords
+
+#===Read data===
+uploaded_file = st.file_uploader('', type=['csv', 'txt','json','tar.gz','xml'], on_change=reset_all)
+
+if uploaded_file is not None:
+    try:
+        extype = get_ext(uploaded_file)
+    
+        if extype.endswith('.csv'):
+             papers = upload(extype) 
+        elif extype.endswith('.txt'):
+             papers = conv_txt(extype)
+
+        elif extype.endswith('.json'):
+            papers = conv_json(extype)
+        elif extype.endswith('.tar.gz') or extype.endswith('.xml'):
+            papers = conv_pub(uploaded_file)
+
+        coldf = sorted(papers.select_dtypes(include=['object']).columns.tolist())
+            
+        c1, c2, c3 = st.columns([3,3,4])
+        method = c1.selectbox(
+                'Choose method',
+                ('Choose...', 'pyLDA', 'Biterm', 'BERTopic'))
+        ColCho = c2.selectbox('Choose column', (coldf))
+        num_cho = c3.number_input('Choose number of topics', min_value=2, max_value=30, value=5)
+
+        d1, d2 = st.columns([3,7])
+        xgram = d1.selectbox("N-grams", ("1", "2", "3"))
+        xgram = int(xgram)
+        words_to_remove = d2.text_input("Remove specific words. Separate words by semicolons (;)")
+    
+        rem_copyright = d1.toggle('Remove copyright statement', value=True)
+        rem_punc = d2.toggle('Remove punctuation', value=True)
+
+        #===advance settings===
+        with st.expander("🧮 Show advance settings"): 
+            t1, t2, t3 = st.columns([3,3,4])
+            if method == 'pyLDA':
+                py_random_state = t1.number_input('Random state', min_value=0, max_value=None, step=1)
+                py_chunksize = t2.number_input('Chunk size', value=100 , min_value=10, max_value=None, step=1)
+                opt_threshold = t3.number_input('Threshold', value=100 , min_value=1, max_value=None, step=1)
+                
+            elif method == 'Biterm':
+                btm_seed = t1.number_input('Random state seed', value=100 , min_value=1, max_value=None, step=1)
+                btm_iterations = t2.number_input('Iterations number', value=20 , min_value=2, max_value=None, step=1)
+                opt_threshold = t3.number_input('Threshold', value=100 , min_value=1, max_value=None, step=1)
+                
+            elif method == 'BERTopic':
+                u1, u2 = st.columns([5,5])
+                
+                bert_top_n_words = u1.number_input('top_n_words', value=5 , min_value=5, max_value=25, step=1)
+                bert_random_state = u2.number_input('random_state', value=42 , min_value=1, max_value=None, step=1)
+                bert_n_components = u1.number_input('n_components', value=5 , min_value=1, max_value=None, step=1)
+                bert_n_neighbors = u2.number_input('n_neighbors', value=15 , min_value=1, max_value=None, step=1)
+                bert_embedding_model = st.radio(
+                    "embedding_model", 
+                    ["all-MiniLM-L6-v2", "paraphrase-multilingual-MiniLM-L12-v2", "en_core_web_sm"], index=0, horizontal=True)
+                
+                fine_tuning = st.toggle("Use Fine-tuning")
+                if fine_tuning:
+                    topic_labelling = st.toggle("Automatic topic labelling")
+                    if topic_labelling:
+                        llm_provider = st.selectbox("Provider",["OpenAI","HuggingFace"])
+                        if llm_provider == "OpenAI":
+                            api_key = st.text_input("API Key")
+                
+            else:
+                st.write('Please choose your preferred method')
+
+        #===clean csv===
+        @st.cache_data(ttl=3600, show_spinner=False)
+        def clean_csv(extype):
+            paper = papers.dropna(subset=[ColCho])
+                     
+            #===mapping===
+            paper['Abstract_pre'] = paper[ColCho].map(lambda x: x.lower())
+            if rem_punc:
+                paper['Abstract_pre'] = paper['Abstract_pre'].map(
+                    lambda x: re.sub(f"[{re.escape(string.punctuation)}]", " ", x)
+                ).map(lambda x: re.sub(r"\s+", " ", x).strip())
+                paper['Abstract_pre'] = paper['Abstract_pre'].str.replace('[\u2018\u2019\u201c\u201d]', '', regex=True)
+            if rem_copyright:  
+                paper['Abstract_pre'] = paper['Abstract_pre'].map(lambda x: re.sub('©.*', '', x))
+            
+            #===stopword removal===
+            stop = stopwords.words('english')
+            paper['Abstract_stop'] = paper['Abstract_pre'].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)]))
+              
+            #===lemmatize===
+            lemmatizer = WordNetLemmatizer()
+            
+            @st.cache_resource(ttl=3600)
+            def lemmatize_words(text):
+                words = text.split()
+                words = [lemmatizer.lemmatize(word) for word in words]
+                return ' '.join(words)
+            paper['Abstract_lem'] = paper['Abstract_stop'].apply(lemmatize_words)
+        
+            words_rmv = [word.strip() for word in words_to_remove.split(";")]
+            remove_dict = {word: None for word in words_rmv}
+            
+            @st.cache_resource(ttl=3600)
+            def remove_words(text):
+                 words = text.split()
+                 cleaned_words = [word for word in words if word not in remove_dict]
+                 return ' '.join(cleaned_words) 
+            paper['Abstract_lem'] = paper['Abstract_lem'].map(remove_words)
+             
+            topic_abs = paper.Abstract_lem.values.tolist()
+            return topic_abs, paper
+    
+        topic_abs, paper=clean_csv(extype) 
+                 
+        if st.button("Submit", on_click=reset_all):
+            num_topic = num_cho  
+    
+        if method == 'BERTopic':
+            st.info('BERTopic is an expensive process when dealing with a large volume of text with our existing resources. Please kindly wait until the visualization appears.', icon="ℹ️")
+               
+        #===topic===
+        if method == 'Choose...':
+            st.write('')
+    
+        elif method == 'pyLDA':       
+            tab1, tab2, tab3, tab4 = st.tabs(["📈 Generate visualization", "📃 Reference", "📓 Recommended Reading", "⬇️ Download Help"])
+    
+            with tab1:
+            #===visualization===
+                @st.cache_data(ttl=3600, show_spinner=False)
+                def pylda(extype):
+                    topic_abs_LDA = [t.split(' ') for t in topic_abs]
+
+                    bigram = Phrases(topic_abs_LDA, min_count=xgram, threshold=opt_threshold)
+                    trigram = Phrases(bigram[topic_abs_LDA], threshold=opt_threshold)
+                    bigram_mod = Phraser(bigram)
+                    trigram_mod = Phraser(trigram)
+                    
+                    topic_abs_LDA = [trigram_mod[bigram_mod[doc]] for doc in topic_abs_LDA]
+
+                    id2word = Dictionary(topic_abs_LDA)
+                    corpus = [id2word.doc2bow(text) for text in topic_abs_LDA]
+                    #===LDA===
+                    lda_model = LdaModel(corpus=corpus,
+                                id2word=id2word,
+                                num_topics=num_topic, 
+                                random_state=py_random_state,
+                                chunksize=py_chunksize,
+                                alpha='auto',
+                                per_word_topics=False)
+                    pprint(lda_model.print_topics())
+                    doc_lda = lda_model[corpus]
+                    topics = lda_model.show_topics(num_words = 30,formatted=False)
+
+                    #===visualization===
+                    coherence_model_lda = CoherenceModel(model=lda_model, texts=topic_abs_LDA, dictionary=id2word, coherence='c_v')
+                    coherence_lda = coherence_model_lda.get_coherence()
+                    vis = pyLDAvis.gensim_models.prepare(lda_model, corpus, id2word)
+                    py_lda_vis_html = pyLDAvis.prepared_data_to_html(vis)
+                    return py_lda_vis_html, coherence_lda, vis, topics
+                       
+                with st.spinner('Performing computations. Please wait ...'):
+                    try:
+                        py_lda_vis_html, coherence_lda, vis, topics = pylda(extype)
+                        st.write('Coherence score: ', coherence_lda)
+                        components.html(py_lda_vis_html, width=1500, height=800)
+                        st.markdown('Copyright (c) 2015, Ben Mabey. https://github.com/bmabey/pyLDAvis')
+                           
+                        @st.cache_data(ttl=3600, show_spinner=False)
+                        def img_lda(vis):
+                            pyLDAvis.save_html(vis, 'output.html')
+                            hti = Html2Image()
+                            hti.browser.flags = ['--default-background-color=ffffff', '--hide-scrollbars']
+                            hti.browser.use_new_headless = None
+                            css = "body {background: white;}"
+                            hti.screenshot( 
+                                other_file='output.html', css_str=css, size=(1500, 800),
+                                save_as='ldavis_img.png'
+                            )
+
+                        img_lda(vis)
+                        
+                        d1, d2 = st.columns(2)
+                        with open("ldavis_img.png", "rb") as file:
+                            btn = d1.download_button(
+                                label="Download image",
+                                data=file,
+                                file_name="ldavis_img.png",
+                                mime="image/png"
+                                )
+                        
+                        #===download results===#
+                        resultf = pd.DataFrame(topics)
+                        #formatting
+                        resultf = resultf.transpose()
+                        resultf = resultf.drop([0])
+                        resultf = resultf.explode(list(range(len(resultf.columns))), ignore_index=False)
+                        
+                        resultcsv = resultf.to_csv().encode("utf-8")
+                        d2.download_button(
+                            label = "Download Results",
+                            data=resultcsv,
+                            file_name="results.csv",
+                            mime="text\csv",
+                            on_click="ignore")
+
+                    except NameError as f:
+                        st.warning('🖱️ Please click Submit')
+    
+            with tab2:
+                st.markdown('**Sievert, C., & Shirley, K. (2014). LDAvis: A method for visualizing and interpreting topics. Proceedings of the Workshop on Interactive Language Learning, Visualization, and Interfaces.** https://doi.org/10.3115/v1/w14-3110')
+    
+            with tab3:
+                st.markdown('**Chen, X., & Wang, H. (2019, January). Automated chat transcript analysis using topic modeling for library reference services. Proceedings of the Association for Information Science and Technology, 56(1), 368–371.** https://doi.org/10.1002/pra2.31')
+                st.markdown('**Joo, S., Ingram, E., & Cahill, M. (2021, December 15). Exploring Topics and Genres in Storytime Books: A Text Mining Approach. Evidence Based Library and Information Practice, 16(4), 41–62.** https://doi.org/10.18438/eblip29963')
+                st.markdown('**Lamba, M., & Madhusudhan, M. (2021, July 31). Topic Modeling. Text Mining for Information Professionals, 105–137.** https://doi.org/10.1007/978-3-030-85085-2_4')
+                st.markdown('**Lamba, M., & Madhusudhan, M. (2019, June 7). Mapping of topics in DESIDOC Journal of Library and Information Technology, India: a study. Scientometrics, 120(2), 477–505.** https://doi.org/10.1007/s11192-019-03137-5')
+         
+            with tab4:
+                st.subheader(':blue[pyLDA]', anchor=False)
+                st.button('Download image')
+                st.text("Click Download Image button.")
+                st.divider()
+                st.subheader(':blue[Downloading CSV Results]', anchor=False)
+                st.button("Download Results")
+                st.text("Click Download results button at bottom of page")
+
+         #===Biterm===
+        elif method == 'Biterm':            
+                 
+            #===optimize Biterm===
+            @st.cache_data(ttl=3600, show_spinner=False)
+            def biterm_topic(extype):
+                tokenized_abs = [t.split(' ') for t in topic_abs]
+
+                bigram = Phrases(tokenized_abs, min_count=xgram, threshold=opt_threshold)
+                trigram = Phrases(bigram[tokenized_abs], threshold=opt_threshold)
+                bigram_mod = Phraser(bigram)
+                trigram_mod = Phraser(trigram)
+            
+                topic_abs_ngram = [trigram_mod[bigram_mod[doc]] for doc in tokenized_abs]
+            
+                topic_abs_str = [' '.join(doc) for doc in topic_abs_ngram]
+
+                
+                X, vocabulary, vocab_dict = btm.get_words_freqs(topic_abs_str)
+                tf = np.array(X.sum(axis=0)).ravel()
+                docs_vec = btm.get_vectorized_docs(topic_abs, vocabulary)
+                docs_lens = list(map(len, docs_vec))
+                biterms = btm.get_biterms(docs_vec)
+                
+                model = btm.BTM(X, vocabulary, seed=btm_seed, T=num_topic, M=20, alpha=50/8, beta=0.01)
+                model.fit(biterms, iterations=btm_iterations)
+                
+                p_zd = model.transform(docs_vec)
+                coherence = model.coherence_
+                phi = tmp.get_phi(model)
+                topics_coords = tmp.prepare_coords(model)
+                totaltop = topics_coords.label.values.tolist()
+                perplexity = model.perplexity_
+                top_topics = model.df_words_topics_
+                
+                return topics_coords, phi, totaltop, perplexity, top_topics
+    
+            tab1, tab2, tab3, tab4 = st.tabs(["📈 Generate visualization", "📃 Reference", "📓 Recommended Reading", "⬇️ Download Help"])
+            with tab1:
+                try:
+                    with st.spinner('Performing computations. Please wait ...'): 
+                        topics_coords, phi, totaltop, perplexity, top_topics = biterm_topic(extype)            
+                        col1, col2 = st.columns([4,6])
+                      
+                        @st.cache_data(ttl=3600)
+                        def biterm_map(extype):
+                            btmvis_coords = tmp.plot_scatter_topics(topics_coords, size_col='size', label_col='label', topic=numvis)
+                            return btmvis_coords
+                                
+                        @st.cache_data(ttl=3600)
+                        def biterm_bar(extype):
+                            terms_probs = tmp.calc_terms_probs_ratio(phi, topic=numvis, lambda_=1)
+                            btmvis_probs = tmp.plot_terms(terms_probs, font_size=12)
+                            return btmvis_probs
+                                
+                        with col1:
+                            st.write('Perplexity score: ', perplexity)
+                            st.write('')
+                            numvis = st.selectbox(
+                                'Choose topic',
+                                (totaltop), on_change=reset_biterm)
+                            btmvis_coords = biterm_map(extype)
+                            st.altair_chart(btmvis_coords)
+                        with col2:
+                            btmvis_probs = biterm_bar(extype)
+                            st.altair_chart(btmvis_probs, use_container_width=True)
+    
+                        #===download results===#
+                        resultcsv = top_topics.to_csv().encode("utf-8")
+                        st.download_button(label = "Download Results", data=resultcsv, file_name="results.csv", mime="text\csv", on_click="ignore")
+
+                except ValueError as g:
+                    st.error('🙇‍♂️ Please raise the number of topics and click submit')
+                    
+                except NameError as f:
+                    st.warning('🖱️ Please click Submit')
+    
+            with tab2: 
+                st.markdown('**Yan, X., Guo, J., Lan, Y., & Cheng, X. (2013, May 13). A biterm topic model for short texts. Proceedings of the 22nd International Conference on World Wide Web.** https://doi.org/10.1145/2488388.2488514')
+            with tab3:
+                st.markdown('**Cai, M., Shah, N., Li, J., Chen, W. H., Cuomo, R. E., Obradovich, N., & Mackey, T. K. (2020, August 26). Identification and characterization of tweets related to the 2015 Indiana HIV outbreak: A retrospective infoveillance study. PLOS ONE, 15(8), e0235150.** https://doi.org/10.1371/journal.pone.0235150')
+                st.markdown('**Chen, Y., Dong, T., Ban, Q., & Li, Y. (2021). What Concerns Consumers about Hypertension? A Comparison between the Online Health Community and the Q&A Forum. International Journal of Computational Intelligence Systems, 14(1), 734.** https://doi.org/10.2991/ijcis.d.210203.002')
+                st.markdown('**George, Crissandra J., "AMBIGUOUS APPALACHIANNESS: A LINGUISTIC AND PERCEPTUAL INVESTIGATION INTO ARC-LABELED PENNSYLVANIA COUNTIES" (2022). Theses and Dissertations-- Linguistics. 48.** https://doi.org/10.13023/etd.2022.217')
+                st.markdown('**Li, J., Chen, W. H., Xu, Q., Shah, N., Kohler, J. C., & Mackey, T. K. (2020). Detection of self-reported experiences with corruption on twitter using unsupervised machine learning. Social Sciences & Humanities Open, 2(1), 100060.** https://doi.org/10.1016/j.ssaho.2020.100060')
+            with tab4:
+                st.subheader(':blue[Biterm]', anchor=False)
+                st.text("Click the three dots at the top right then select the desired format.")
+                st.markdown("![Downloading visualization](https://raw.githubusercontent.com/faizhalas/library-tools/main/images/download_biterm.jpg)")  
+                st.divider()
+                st.subheader(':blue[Downloading CSV Results]', anchor=False)
+                st.button("Download Results")
+                st.text("Click Download results button at bottom of page")
+
+
+         #===BERTopic===
+        elif method == 'BERTopic':
+            @st.cache_resource(ttl = 3600, show_spinner=False)
+            #@st.cache_data(ttl=3600, show_spinner=False)
+            def bertopic_vis(extype):
+                umap_model = UMAP(n_neighbors=bert_n_neighbors, n_components=bert_n_components, 
+                    min_dist=0.0, metric='cosine', random_state=bert_random_state)   
+                cluster_model = KMeans(n_clusters=num_topic)
+                if bert_embedding_model == 'all-MiniLM-L6-v2':
+                    model = SentenceTransformer('all-MiniLM-L6-v2')
+                    lang = 'en'
+                    embeddings = model.encode(topic_abs, show_progress_bar=True)
+                    
+                elif bert_embedding_model == 'en_core_web_sm':
+                    nlp = en_core_web_sm.load(exclude=['tagger', 'parser', 'ner', 'attribute_ruler', 'lemmatizer'])
+                    model = nlp
+                    lang = 'en'
+                    embeddings = np.array([nlp(text).vector for text in topic_abs])
+                    
+                elif bert_embedding_model == 'paraphrase-multilingual-MiniLM-L12-v2':
+                    model = SentenceTransformer('paraphrase-multilingual-MiniLM-L12-v2')
+                    lang = 'multilingual'
+                    embeddings = model.encode(topic_abs, show_progress_bar=True)
+
+                representation_model = ""
+
+                if fine_tuning:
+                    keybert = KeyBERTInspired()
+                    mmr = MaximalMarginalRelevance(diversity=0.3)
+                    representation_model = {
+                        "KeyBERT": keybert,
+                        "MMR": mmr,
+                    }
+                    if topic_labelling:
+                        if llm_provider == "OpenAI":
+                            client = openai.OpenAI(api_key=api_key)
+                            representation_model = {
+                                "KeyBERT": keybert,
+                                "MMR": mmr,
+                                "test": OpenAI(client, model = "gpt-4o-mini", delay_in_seconds=10)
+                            }
+                        elif llm_provider == "HuggingFace":
+                            gennie = pipeline("text2text-generation", model = "google/flan-t5-base")
+                            clientmod = TextGeneration(gennie)
+                            representation_model = {
+                                "KeyBERT": keybert,
+                                "MMR": mmr,
+                                "test": clientmod
+                            }
+
+                vectorizer_model = CountVectorizer(ngram_range=(1, xgram), stop_words='english')
+                topic_model = BERTopic(representation_model = representation_model, embedding_model=model, hdbscan_model=cluster_model, language=lang, umap_model=umap_model, vectorizer_model=vectorizer_model, top_n_words=bert_top_n_words)
+                topics, probs = topic_model.fit_transform(topic_abs, embeddings=embeddings)
+
+                if(fine_tuning and topic_labelling):
+                    generated_labels = [label[0][0].split("\n")[0] for label in topic_model.get_topics(full=True)["test"].values()]
+                    topic_model.set_topic_labels(generated_labels)
+
+                return topic_model, topics, probs, embeddings
+            
+            @st.cache_resource(ttl = 3600, show_spinner=False)
+            def Vis_Topics(extype):
+                fig1 = topic_model.visualize_topics()
+                return fig1
+            @st.cache_resource(ttl = 3600, show_spinner=False)
+            def Vis_Documents(extype):
+                fig2 = topic_model.visualize_document_datamap(topic_abs, embeddings=embeddings, custom_labels = True)
+                return fig2
+            @st.cache_resource(ttl = 3600, show_spinner=False)
+            def Vis_Hierarchy(extype):
+                fig3 = topic_model.visualize_hierarchy(top_n_topics=num_topic, custom_labels = True)
+                return fig3
+            @st.cache_resource(ttl = 3600, show_spinner=False)
+            def Vis_Heatmap(extype):
+                global topic_model
+                fig4 = topic_model.visualize_heatmap(n_clusters=num_topic-1, width=1000, height=1000, custom_labels = True)
+                return fig4
+            @st.cache_resource(ttl = 3600, show_spinner=False)
+            def Vis_Barchart(extype):
+                fig5 = topic_model.visualize_barchart(top_n_topics=num_topic, custom_labels = True)
+                return fig5
+           
+            tab1, tab2, tab3, tab4 = st.tabs(["📈 Generate visualization", "📃 Reference", "📓 Recommended Reading", "⬇️ Download Help"])
+            with tab1:
+                try:
+                    with st.spinner('Performing computations. Please wait ...'):
+                   
+                        topic_model, topics, probs, embeddings = bertopic_vis(extype)
+                        time.sleep(.5)
+                        st.toast('Visualize Topics', icon='🏃')
+                        fig1 = Vis_Topics(extype)
+                       
+                        time.sleep(.5)
+                        st.toast('Visualize Document', icon='🏃')
+                        fig2 = Vis_Documents(extype)
+                       
+                        time.sleep(.5)
+                        st.toast('Visualize Document Hierarchy', icon='🏃')
+                        fig3 = Vis_Hierarchy(extype)
+                       
+                        time.sleep(.5)
+                        st.toast('Visualize Topic Similarity', icon='🏃')
+                        fig4 = Vis_Heatmap(extype)
+                       
+                        time.sleep(.5)
+                        st.toast('Visualize Terms', icon='🏃')
+                        fig5 = Vis_Barchart(extype)
+                       
+                        bertab1, bertab2, bertab3, bertab4, bertab5 = st.tabs(["Visualize Topics", "Visualize Terms", "Visualize Documents",
+                                                                              "Visualize Document Hierarchy", "Visualize Topic Similarity"])
+                        
+                        with bertab1:
+                            st.plotly_chart(fig1, use_container_width=True)
+                        with bertab2:
+                            st.plotly_chart(fig5, use_container_width=True)
+                        with bertab3:
+                            st.plotly_chart(fig2, use_container_width=True)
+                        with bertab4:  
+                            st.plotly_chart(fig3, use_container_width=True)
+                        with bertab5:
+                            st.plotly_chart(fig4, use_container_width=True)
+                      
+                        #===download results===#
+                        results = topic_model.get_topic_info()
+                        resultf = pd.DataFrame(results)
+                        resultcsv = resultf.to_csv().encode("utf-8")
+                        st.download_button(
+                            label = "Download Results",
+                            data=resultcsv,
+                            file_name="results.csv",
+                            mime="text\csv",
+                            on_click="ignore",
+                        )
+
+                except ValueError as e:
+                    st.write(e)
+                    st.error('🙇‍♂️ Please raise the number of topics and click submit')
+
+
+                except NameError as e:
+                    st.warning('🖱️ Please click Submit')
+                    st.write(e)
+    
+            with tab2:
+                st.markdown('**Grootendorst, M. (2022). BERTopic: Neural topic modeling with a class-based TF-IDF procedure. arXiv preprint arXiv:2203.05794.** https://doi.org/10.48550/arXiv.2203.05794')
+              
+            with tab3:
+                st.markdown('**Jeet Rawat, A., Ghildiyal, S., & Dixit, A. K. (2022, December 1). Topic modelling of legal documents using NLP and bidirectional encoder representations from transformers. Indonesian Journal of Electrical Engineering and Computer Science, 28(3), 1749.** https://doi.org/10.11591/ijeecs.v28.i3.pp1749-1755')
+                st.markdown('**Yao, L. F., Ferawati, K., Liew, K., Wakamiya, S., & Aramaki, E. (2023, April 20). Disruptions in the Cystic Fibrosis Community’s Experiences and Concerns During the COVID-19 Pandemic: Topic Modeling and Time Series Analysis of Reddit Comments. Journal of Medical Internet Research, 25, e45249.** https://doi.org/10.2196/45249')
+
+            with tab4:
+                st.divider()
+                st.subheader(':blue[BERTopic]', anchor=False)
+                st.text("Click the camera icon on the top right menu")
+                st.markdown("![Downloading visualization](https://raw.githubusercontent.com/faizhalas/library-tools/main/images/download_bertopic.jpg)")
+                st.divider()
+                st.subheader(':blue[Downloading CSV Results]', anchor=False)
+                st.button("Download Results")
+                st.text("Click Download results button at bottom of page")
+
+    except Exception as e:
+        st.error("Please ensure that your file is correct. Please contact us if you find that this is an error.", icon="🚨")
+        st.write(e)
+        st.stop()

pages/3 Bidirected Network.py

@@ -1,276 +1,370 @@
-#import module
-import streamlit as st
-import pandas as pd
-import re
-import nltk
-nltk.download('punkt')
-from nltk.tokenize import word_tokenize
-from mlxtend.preprocessing import TransactionEncoder
-te = TransactionEncoder()
-from mlxtend.frequent_patterns import fpgrowth
-from mlxtend.frequent_patterns import association_rules
-from streamlit_agraph import agraph, Node, Edge, Config
-import nltk
-nltk.download('wordnet')
-from nltk.stem import WordNetLemmatizer
-nltk.download('stopwords')
-from nltk.corpus import stopwords
-from nltk.stem.snowball import SnowballStemmer
-import sys
-import time
-
-#===config===
-st.set_page_config(
-    page_title="Coconut",
-    page_icon="🥥",
-    layout="wide",
-    initial_sidebar_state="collapsed"
-)
-
-hide_streamlit_style = """
-            <style>
-            #MainMenu 
-            {visibility: hidden;}
-            footer {visibility: hidden;}
-            [data-testid="collapsedControl"] {display: none}
-            </style>
-            """
-st.markdown(hide_streamlit_style, unsafe_allow_html=True)
-
-with st.popover("🔗 Menu"):
-    st.page_link("https://www.coconut-libtool.com/", label="Home", icon="🏠")
-    st.page_link("pages/1 Scattertext.py", label="Scattertext", icon="1️⃣")
-    st.page_link("pages/2 Topic Modeling.py", label="Topic Modeling", icon="2️⃣")
-    st.page_link("pages/3 Bidirected Network.py", label="Bidirected Network", icon="3️⃣")
-    st.page_link("pages/4 Sunburst.py", label="Sunburst", icon="4️⃣")
-    st.page_link("pages/5 Burst Detection.py", label="Burst Detection", icon="5️⃣")
-    st.page_link("pages/6 Keywords Stem.py", label="Keywords Stem", icon="6️⃣")
-    
-st.header("Bidirected Network", anchor=False)
-st.subheader('Put your file here...', anchor=False)
-
-#===clear cache===
-def reset_all():
-    st.cache_data.clear()
-
-#===check type===
-@st.cache_data(ttl=3600)
-def get_ext(extype):
-    extype = uploaded_file.name
-    return extype
-
-@st.cache_data(ttl=3600)
-def upload(extype):
-    papers = pd.read_csv(uploaded_file)
-    return papers
-
-@st.cache_data(ttl=3600)
-def conv_txt(extype):
-    col_dict = {'TI': 'Title',
-            'SO': 'Source title',
-            'DT': 'Document Type',
-            'DE': 'Author Keywords',
-            'ID': 'Keywords Plus'}
-    papers = pd.read_csv(uploaded_file, sep='\t', lineterminator='\r')
-    papers.rename(columns=col_dict, inplace=True)
-    return papers
-
-#===Read data===
-uploaded_file = st.file_uploader('', type=['csv', 'txt'], on_change=reset_all)
-
-if uploaded_file is not None:
-    try:
-        extype = get_ext(uploaded_file)
-        if extype.endswith('.csv'):
-             papers = upload(extype) 
-        elif extype.endswith('.txt'):
-             papers = conv_txt(extype)
-        
-        @st.cache_data(ttl=3600)
-        def get_data_arul(extype):
-            list_of_column_key = list(papers.columns)
-            list_of_column_key = [k for k in list_of_column_key if 'Keyword' in k]
-            return papers, list_of_column_key
-         
-        papers, list_of_column_key = get_data_arul(extype)
-    
-        col1, col2 = st.columns(2)
-        with col1:
-            method = st.selectbox(
-                 'Choose method',
-               ('Lemmatization', 'Stemming'), on_change=reset_all)
-        with col2:
-            keyword = st.selectbox(
-                'Choose column',
-               (list_of_column_key), on_change=reset_all)
-    
-    
-        #===body=== 
-        @st.cache_data(ttl=3600)
-        def clean_arul(extype):
-            global keyword, papers
-            try:
-                arul = papers.dropna(subset=[keyword])
-            except KeyError:
-                st.error('Error: Please check your Author/Index Keywords column.')
-                sys.exit(1)
-            arul[keyword] = arul[keyword].map(lambda x: re.sub('-—–', ' ', x))
-            arul[keyword] = arul[keyword].map(lambda x: re.sub('; ', ' ; ', x))
-            arul[keyword] = arul[keyword].map(lambda x: x.lower())
-            arul[keyword] = arul[keyword].dropna()
-            return arul
-    
-        arul = clean_arul(extype)   
-    
-        #===stem/lem===
-        @st.cache_data(ttl=3600)
-        def lemma_arul(extype):
-            lemmatizer = WordNetLemmatizer()
-            def lemmatize_words(text):
-                 words = text.split()
-                 words = [lemmatizer.lemmatize(word) for word in words]
-                 return ' '.join(words)
-            arul[keyword] = arul[keyword].apply(lemmatize_words)
-            return arul
-        
-        @st.cache_data(ttl=3600)
-        def stem_arul(extype):
-            stemmer = SnowballStemmer("english")
-            def stem_words(text):
-                words = text.split()
-                words = [stemmer.stem(word) for word in words]
-                return ' '.join(words)
-            arul[keyword] = arul[keyword].apply(stem_words)
-            return arul
-    
-        if method is 'Lemmatization':
-            arul = lemma_arul(extype)
-        else:
-            arul = stem_arul(extype)
-        
-        @st.cache_data(ttl=3600)
-        def arm(extype):
-            arule = arul[keyword].str.split(' ; ')
-            arule_list = arule.values.tolist()  
-            te_ary = te.fit(arule_list).transform(arule_list)
-            df = pd.DataFrame(te_ary, columns=te.columns_)
-            return df
-        df = arm(extype)
-    
-        col1, col2, col3 = st.columns(3)
-        with col1:
-            supp = st.slider(
-                'Select value of Support',
-                0.001, 1.000, (0.010), on_change=reset_all)
-        with col2:
-            conf = st.slider(
-                'Select value of Confidence',
-                0.001, 1.000, (0.050), on_change=reset_all)
-        with col3:
-            maxlen = st.slider(
-                'Maximum length of the itemsets generated',
-                2, 8, (2), on_change=reset_all)
-    
-        tab1, tab2, tab3 = st.tabs(["📈 Result & Generate visualization", "📃 Reference", "📓 Recommended Reading"])
-        
-        with tab1:
-            #===Association rules===
-            @st.cache_data(ttl=3600)
-            def freqitem(extype):
-                freq_item = fpgrowth(df, min_support=supp, use_colnames=True, max_len=maxlen)
-                return freq_item
-    
-            freq_item = freqitem(extype)
-            col1, col2 = st.columns(2)
-            with col1:
-                 st.write('🚨 The more data you have, the longer you will have to wait.')
-            with col2:
-                 showall = st.checkbox('Show all nodes', value=True, on_change=reset_all)
-    
-            @st.cache_data(ttl=3600)
-            def arm_table(extype):
-                restab = association_rules(freq_item, metric='confidence', min_threshold=conf) 
-                restab = restab[['antecedents', 'consequents', 'antecedent support', 'consequent support', 'support', 'confidence', 'lift', 'conviction']]
-                restab['antecedents'] = restab['antecedents'].apply(lambda x: ', '.join(list(x))).astype('unicode')
-                restab['consequents'] = restab['consequents'].apply(lambda x: ', '.join(list(x))).astype('unicode')
-                if showall:
-                     restab['Show'] = True
-                else:
-                     restab['Show'] = False
-                return restab 
-             
-            if freq_item.empty:
-                st.error('Please lower your value.', icon="🚨")
-            else:
-                restab = arm_table(extype)
-                restab = st.data_editor(restab, use_container_width=True)
-                res = restab[restab['Show'] == True] 
-                       
-                 #===visualize===
-                    
-                if st.button('📈 Generate network visualization', on_click=reset_all):
-                    with st.spinner('Visualizing, please wait ....'): 
-                         @st.cache_data(ttl=3600)
-                         def map_node(extype):
-                            res['to'] = res['antecedents'] + ' → ' + res['consequents'] + '\n Support = ' +  res['support'].astype(str) + '\n Confidence = ' +  res['confidence'].astype(str) + '\n Conviction = ' +  res['conviction'].astype(str)
-                            res_ant = res[['antecedents','antecedent support']].rename(columns={'antecedents': 'node', 'antecedent support': 'size'}) 
-                            res_con = res[['consequents','consequent support']].rename(columns={'consequents': 'node', 'consequent support': 'size'}) 
-                            res_node = pd.concat([res_ant, res_con]).drop_duplicates(keep='first')
-                            return res_node, res
-                         
-                         res_node, res = map_node(extype)
-    
-                         @st.cache_data(ttl=3600)
-                         def arul_network(extype):
-                            nodes = []
-                            edges = []
-    
-                            for w,x in zip(res_node['size'], res_node['node']):
-                                nodes.append( Node(id=x, 
-                                                label=x,
-                                                size=50*w+10,
-                                                shape="dot",
-                                                labelHighlightBold=True,
-                                                group=x,
-                                                opacity=10,
-                                                mass=1)
-                                        )   
-    
-                            for y,z,a,b in zip(res['antecedents'],res['consequents'],res['confidence'],res['to']):
-                                edges.append( Edge(source=y, 
-                                                target=z,
-                                                title=b,
-                                                width=a*2,
-                                                physics=True,
-                                                smooth=True
-                                                ) 
-                                        )  
-                            return nodes, edges
-    
-                         nodes, edges = arul_network(extype)
-                         config = Config(width=1200,
-                                         height=800,
-                                         directed=True, 
-                                         physics=True, 
-                                         hierarchical=False,
-                                         maxVelocity=5
-                                         )
-    
-                         return_value = agraph(nodes=nodes, 
-                                               edges=edges, 
-                                               config=config)
-                         time.sleep(1)
-                         st.toast('Process completed', icon='📈')
-                        
-        with tab2:
-             st.markdown('**Santosa, F. A. (2023). Adding Perspective to the Bibliometric Mapping Using Bidirected Graph. Open Information Science, 7(1), 20220152.** https://doi.org/10.1515/opis-2022-0152')
-             
-        with tab3:
-            st.markdown('**Agrawal, R., Imieliński, T., & Swami, A. (1993). Mining association rules between sets of items in large databases. In ACM SIGMOD Record (Vol. 22, Issue 2, pp. 207–216). Association for Computing Machinery (ACM).** https://doi.org/10.1145/170036.170072')
-            st.markdown('**Brin, S., Motwani, R., Ullman, J. D., & Tsur, S. (1997). Dynamic itemset counting and implication rules for market basket data. ACM SIGMOD Record, 26(2), 255–264.** https://doi.org/10.1145/253262.253325')
-            st.markdown('**Edmonds, J., & Johnson, E. L. (2003). Matching: A Well-Solved Class of Integer Linear Programs. Combinatorial Optimization — Eureka, You Shrink!, 27–30.** https://doi.org/10.1007/3-540-36478-1_3') 
-            st.markdown('**Li, M. (2016, August 23). An exploration to visualise the emerging trends of technology foresight based on an improved technique of co-word analysis and relevant literature data of WOS. Technology Analysis & Strategic Management, 29(6), 655–671.** https://doi.org/10.1080/09537325.2016.1220518')
-
-    except:
-        st.error("Please ensure that your file is correct. Please contact us if you find that this is an error.", icon="🚨")
-        st.stop()
+#import module
+import streamlit as st
+import pandas as pd
+import re
+import nltk
+nltk.download('punkt')
+from nltk.tokenize import word_tokenize
+from mlxtend.preprocessing import TransactionEncoder
+te = TransactionEncoder()
+from mlxtend.frequent_patterns import fpgrowth
+from mlxtend.frequent_patterns import association_rules
+from streamlit_agraph import agraph, Node, Edge, Config
+import nltk
+nltk.download('wordnet')
+from nltk.stem import WordNetLemmatizer
+nltk.download('stopwords')
+from nltk.corpus import stopwords
+from nltk.stem.snowball import SnowballStemmer
+import sys
+import time
+import json
+from tools import sourceformat as sf
+
+import networkx as nx
+import matplotlib.pyplot as plt
+import plotly.graph_objects as go
+
+import altair as alt
+import altair_nx as anx
+
+#===config===
+st.set_page_config(
+    page_title="Coconut",
+    page_icon="🥥",
+    layout="wide",
+    initial_sidebar_state="collapsed"
+)
+
+hide_streamlit_style = """
+            <style>
+            #MainMenu 
+            {visibility: hidden;}
+            footer {visibility: hidden;}
+            [data-testid="collapsedControl"] {display: none}
+            </style>
+            """
+st.markdown(hide_streamlit_style, unsafe_allow_html=True)
+
+with st.popover("🔗 Menu"):
+    st.page_link("https://www.coconut-libtool.com/", label="Home", icon="🏠")
+    st.page_link("pages/1 Scattertext.py", label="Scattertext", icon="1️⃣")
+    st.page_link("pages/2 Topic Modeling.py", label="Topic Modeling", icon="2️⃣")
+    st.page_link("pages/3 Bidirected Network.py", label="Bidirected Network", icon="3️⃣")
+    st.page_link("pages/4 Sunburst.py", label="Sunburst", icon="4️⃣")
+    st.page_link("pages/5 Burst Detection.py", label="Burst Detection", icon="5️⃣")
+    st.page_link("pages/6 Keywords Stem.py", label="Keywords Stem", icon="6️⃣")
+    st.page_link("pages/7 Sentiment Analysis.py", label="Sentiment Analysis", icon="7️⃣")
+    st.page_link("pages/8 Shifterator.py", label="Shifterator", icon="8️⃣")
+    
+st.header("Bidirected Network", anchor=False)
+st.subheader('Put your file here...', anchor=False)
+
+#===clear cache===
+def reset_all():
+    st.cache_data.clear()
+
+#===check type===
+@st.cache_data(ttl=3600)
+def get_ext(extype):
+    extype = uploaded_file.name
+    return extype
+
+@st.cache_data(ttl=3600)
+def upload(extype):
+    papers = pd.read_csv(uploaded_file)
+
+    if "About the data" in papers.columns[0]:
+        papers = sf.dim(papers)
+        col_dict = {'MeSH terms': 'Keywords',
+        'PubYear': 'Year',
+        'Times cited': 'Cited by',
+        'Publication Type': 'Document Type'
+        }
+        papers.rename(columns=col_dict, inplace=True)
+    
+    return papers
+
+    return papers
+
+@st.cache_data(ttl=3600)
+def conv_txt(extype):
+    if("PMID" in (uploaded_file.read()).decode()):
+        uploaded_file.seek(0)
+        papers = sf.medline(uploaded_file)
+        print(papers)
+        return papers
+    col_dict = {'TI': 'Title',
+            'SO': 'Source title',
+            'DE': 'Author Keywords',
+            'DT': 'Document Type',
+            'AB': 'Abstract',
+            'TC': 'Cited by',
+            'PY': 'Year',
+            'ID': 'Keywords Plus',
+            'rights_date_used': 'Year'}
+    uploaded_file.seek(0)
+    papers = pd.read_csv(uploaded_file, sep='\t')
+    if("htid" in papers.columns):
+        papers = sf.htrc(papers)
+    papers.rename(columns=col_dict, inplace=True)
+    print(papers)
+    return papers
+
+
+@st.cache_data(ttl=3600)
+def conv_json(extype):
+    col_dict={'title': 'title',
+    'rights_date_used': 'Year',
+    }
+
+    data = json.load(uploaded_file)
+    hathifile = data['gathers']
+    keywords = pd.DataFrame.from_records(hathifile)
+    
+    keywords = sf.htrc(keywords)
+    keywords.rename(columns=col_dict,inplace=True)
+    return keywords
+
+@st.cache_data(ttl=3600)
+def conv_pub(extype):
+    if (get_ext(extype)).endswith('.tar.gz'):
+        bytedata = extype.read()
+        keywords = sf.readPub(bytedata)
+    elif (get_ext(extype)).endswith('.xml'):
+        bytedata = extype.read()
+        keywords = sf.readxml(bytedata)
+    return keywords
+
+#===Read data===
+uploaded_file = st.file_uploader('', type=['csv', 'txt','json','tar.gz', 'xml'], on_change=reset_all)
+
+if uploaded_file is not None:
+    try:
+        extype = get_ext(uploaded_file)
+        if extype.endswith('.csv'):
+             papers = upload(extype) 
+        elif extype.endswith('.txt'):
+             papers = conv_txt(extype)
+        elif extype.endswith('.json'):
+            papers = conv_json(extype)
+        elif extype.endswith('.tar.gz') or extype.endswith('.xml'):
+            papers = conv_pub(uploaded_file)
+        
+        @st.cache_data(ttl=3600)
+        def get_data_arul(extype):
+            list_of_column_key = list(papers.columns)
+            list_of_column_key = [k for k in list_of_column_key if 'Keyword' in k]
+            return papers, list_of_column_key
+         
+        papers, list_of_column_key = get_data_arul(extype)
+    
+        col1, col2 = st.columns(2)
+        with col1:
+            method = st.selectbox(
+                 'Choose method',
+               ('Lemmatization', 'Stemming'), on_change=reset_all)
+            layout = st.selectbox(
+            'Choose graph layout',
+            ['Circular','Kamada Kawai','Random','Spring','Shell']
+        ) 
+        with col2:
+            keyword = st.selectbox(
+                'Choose column',
+               (list_of_column_key), on_change=reset_all)
+
+    
+        #===body=== 
+        @st.cache_data(ttl=3600)
+        def clean_arul(extype):
+            global keyword, papers
+            try:
+                arul = papers.dropna(subset=[keyword])
+            except KeyError:
+                st.error('Error: Please check your Author/Index Keywords column.')
+                sys.exit(1)
+            arul[keyword] = arul[keyword].map(lambda x: re.sub('-—–', ' ', x))
+            arul[keyword] = arul[keyword].map(lambda x: re.sub('; ', ' ; ', x))
+            arul[keyword] = arul[keyword].map(lambda x: x.lower())
+            arul[keyword] = arul[keyword].dropna()
+            return arul
+    
+        arul = clean_arul(extype)   
+    
+        #===stem/lem===
+        @st.cache_data(ttl=3600)
+        def lemma_arul(extype):
+            lemmatizer = WordNetLemmatizer()
+            def lemmatize_words(text):
+                 words = text.split()
+                 words = [lemmatizer.lemmatize(word) for word in words]
+                 return ' '.join(words)
+            arul[keyword] = arul[keyword].apply(lemmatize_words)
+            return arul
+        
+        @st.cache_data(ttl=3600)
+        def stem_arul(extype):
+            stemmer = SnowballStemmer("english")
+            def stem_words(text):
+                words = text.split()
+                words = [stemmer.stem(word) for word in words]
+                return ' '.join(words)
+            arul[keyword] = arul[keyword].apply(stem_words)
+            return arul
+    
+        if method is 'Lemmatization':
+            arul = lemma_arul(extype)
+        else:
+            arul = stem_arul(extype)
+        
+        @st.cache_data(ttl=3600)
+        def arm(extype):
+            arule = arul[keyword].str.split(' ; ')
+            arule_list = arule.values.tolist()  
+            te_ary = te.fit(arule_list).transform(arule_list)
+            df = pd.DataFrame(te_ary, columns=te.columns_)
+            return df
+        df = arm(extype)
+    
+        col1, col2, col3 = st.columns(3)
+        with col1:
+            supp = st.slider(
+                'Select value of Support',
+                0.001, 1.000, (0.010), on_change=reset_all)
+        with col2:
+            conf = st.slider(
+                'Select value of Confidence',
+                0.001, 1.000, (0.050), on_change=reset_all)
+        with col3:
+            maxlen = st.slider(
+                'Maximum length of the itemsets generated',
+                2, 8, (2), on_change=reset_all)
+    
+        tab1, tab2, tab3, tab4 = st.tabs(["📈 Result & Generate visualization", "📃 Reference", "📓 Recommended Reading", "⬇️ Download Help"])
+        
+        with tab1:
+            #===Association rules===
+            @st.cache_data(ttl=3600)
+            def freqitem(extype):
+                freq_item = fpgrowth(df, min_support=supp, use_colnames=True, max_len=maxlen)
+                return freq_item
+    
+            freq_item = freqitem(extype)
+            col1, col2 = st.columns(2)
+            with col1:
+                 st.write('🚨 The more data you have, the longer you will have to wait.')
+            with col2:
+                 showall = st.checkbox('Show all nodes', value=True, on_change=reset_all)
+    
+            @st.cache_data(ttl=3600)
+            def arm_table(extype):
+                restab = association_rules(freq_item, metric='confidence', min_threshold=conf) 
+                restab = restab[['antecedents', 'consequents', 'antecedent support', 'consequent support', 'support', 'confidence', 'lift', 'conviction']]
+                restab['antecedents'] = restab['antecedents'].apply(lambda x: ', '.join(list(x))).astype('unicode')
+                restab['consequents'] = restab['consequents'].apply(lambda x: ', '.join(list(x))).astype('unicode')
+                if showall:
+                     restab['Show'] = True
+                else:
+                     restab['Show'] = False
+                return restab 
+             
+            if freq_item.empty:
+                st.error('Please lower your value.', icon="🚨")
+            else:
+                restab = arm_table(extype)
+                restab = st.data_editor(restab, use_container_width=True)
+                res = restab[restab['Show'] == True] 
+                       
+                 #===visualize===
+                    
+                if st.button('📈 Generate network visualization', on_click=reset_all):
+                    with st.spinner('Visualizing, please wait ....'): 
+                         @st.cache_data(ttl=3600)
+                         def map_node(extype):
+                            res['to'] = res['antecedents'] + ' → ' + res['consequents'] + '\n Support = ' +  res['support'].astype(str) + '\n Confidence = ' +  res['confidence'].astype(str) + '\n Conviction = ' +  res['conviction'].astype(str)
+                            res_ant = res[['antecedents','antecedent support']].rename(columns={'antecedents': 'node', 'antecedent support': 'size'}) 
+                            res_con = res[['consequents','consequent support']].rename(columns={'consequents': 'node', 'consequent support': 'size'}) 
+                            res_node = pd.concat([res_ant, res_con]).drop_duplicates(keep='first')
+                            return res_node, res
+                         
+                         res_node, res = map_node(extype)
+                    ___='''
+                    @st.cache_data(ttl=3600)
+                    def arul_net(extype):
+                        nodes = []
+                        edges = []
+                
+                        for w,x in zip(res_node['size'], res_node['node']):
+                            nodes.append(x)
+                        for y,z,a,b in zip(res['antecedents'],res['consequents'],res['confidence'],res['to']):
+                            edge = (y,z)
+
+                            edges.append(edge)
+
+                        return nodes, edges
+
+                    #nodes, edges =  arul_net(res)
+                    '''
+
+                    @st.cache_data(ttl=3600)
+                    def graphmaker(__netgraph):
+
+                        #add nodes, w is weight, x is node label
+                        for w,x in zip(res_node['size'], res_node['node']):
+                            __netgraph.add_node(x, size = w)
+                        #add edges, y is startpoint, z is endpoint, a is edge weight, b is title
+                        for y,z,a,b in zip(res['antecedents'],res['consequents'],res['confidence'],res['to']):
+                            __netgraph.add_edge(y,z, weight = int(a*100))
+
+
+                    #Make graph with NetworkX
+
+                    G=nx.DiGraph()
+
+                    graphmaker(G)
+
+                    #G.add_edges_from(edges) ##### remove this later
+
+                    #Graph layout    
+                    if(layout=="Spring"):
+                        pos=nx.spring_layout(G)
+                    elif(layout == "Kamada Kawai"):
+                        pos=nx.kamada_kawai_layout(G)                    
+                    elif(layout == "Circular"):
+                        pos = nx.circular_layout(G)
+                    elif(layout=="Random"):
+                        pos = nx.random_layout(G)
+                    elif(layout=="Shell"):
+                        pos=nx.shell_layout(G)
+                  
+                    graph = anx.draw_networkx(G,pos, node_label = 'node',
+                     edge_width = 'weight',
+                     node_size = 'size',
+                     curved_edges = True,
+                      node_font_size=12,
+                      chart_width=1920,
+                      chart_height=1080).interactive()
+                
+                    st.altair_chart(graph)
+                
+
+        with tab2:
+             st.markdown('**Santosa, F. A. (2023). Adding Perspective to the Bibliometric Mapping Using Bidirected Graph. Open Information Science, 7(1), 20220152.** https://doi.org/10.1515/opis-2022-0152')
+             
+        with tab3:
+            st.markdown('**Agrawal, R., Imieliński, T., & Swami, A. (1993). Mining association rules between sets of items in large databases. In ACM SIGMOD Record (Vol. 22, Issue 2, pp. 207–216). Association for Computing Machinery (ACM).** https://doi.org/10.1145/170036.170072')
+            st.markdown('**Brin, S., Motwani, R., Ullman, J. D., & Tsur, S. (1997). Dynamic itemset counting and implication rules for market basket data. ACM SIGMOD Record, 26(2), 255–264.** https://doi.org/10.1145/253262.253325')
+            st.markdown('**Edmonds, J., & Johnson, E. L. (2003). Matching: A Well-Solved Class of Integer Linear Programs. Combinatorial Optimization — Eureka, You Shrink!, 27–30.** https://doi.org/10.1007/3-540-36478-1_3') 
+            st.markdown('**Li, M. (2016, August 23). An exploration to visualise the emerging trends of technology foresight based on an improved technique of co-word analysis and relevant literature data of WOS. Technology Analysis & Strategic Management, 29(6), 655–671.** https://doi.org/10.1080/09537325.2016.1220518')
+        with tab4:
+            st.subheader("Download visualization")
+            st.text("Zoom in, zoom out, or shift the nodes as desired, then right-click and select Save image as ...")
+            st.markdown("![Downloading graph](https://raw.githubusercontent.com/faizhalas/library-tools/main/images/download_bidirected.jpg)")     
+            st.subheader("Download table as CSV")
+            st.text("Hover cursor over table, and click download arrow")
+            st.image("images/tablenetwork.png")
+            
+    except Exception as e:
+        st.write(e)
+        st.error("Please ensure that your file is correct. Please contact us if you find that this is an error.", icon="🚨")
+        st.stop()

pages/4 Sunburst.py

@@ -4,6 +4,9 @@ import pandas as pd
 import plotly.express as px
 import numpy as np
 import sys
+import json
+from tools import sourceformat as sf
+
 
 #===config===
 st.set_page_config(
@@ -31,6 +34,7 @@ with st.popover("🔗 Menu"):
     st.page_link("pages/4 Sunburst.py", label="Sunburst", icon="4️⃣")
     st.page_link("pages/5 Burst Detection.py", label="Burst Detection", icon="5️⃣")
     st.page_link("pages/6 Keywords Stem.py", label="Keywords Stem", icon="6️⃣")
+    st.page_link("pages/7 Sentiment Analysis.py", label="Sentiment Analysis", icon="7️⃣")
     
 st.header("Sunburst Visualization", anchor=False)
 st.subheader('Put your file here...', anchor=False)
@@ -52,24 +56,71 @@ def upload(extype):
     if 'Publication Year' in papers.columns:
             papers.rename(columns={'Publication Year': 'Year', 'Citing Works Count': 'Cited by',
                                     'Publication Type': 'Document Type', 'Source Title': 'Source title'}, inplace=True)
+    if "About the data" in papers.columns[0]:
+        papers = sf.dim(papers)
+        col_dict = {'MeSH terms': 'Keywords',
+        'PubYear': 'Year',
+        'Times cited': 'Cited by',
+        'Publication Type': 'Document Type'
+        }
+        papers.rename(columns=col_dict, inplace=True)
+    
     return papers
 
 @st.cache_data(ttl=3600)
 def conv_txt(extype):
+    if("PMID" in (uploaded_file.read()).decode()):
+        uploaded_file.seek(0)
+        papers = sf.medline(uploaded_file)
+        print(papers)
+        return papers
     col_dict = {'TI': 'Title',
             'SO': 'Source title',
-            'DT': 'Document Type',
             'DE': 'Author Keywords',
-            'ID': 'Keywords Plus',
+            'DT': 'Document Type',
             'AB': 'Abstract',
             'TC': 'Cited by',
-            'PY': 'Year',}
-    papers = pd.read_csv(uploaded_file, sep='\t', lineterminator='\r')
+            'PY': 'Year',
+            'ID': 'Keywords Plus',
+            'rights_date_used': 'Year'}
+    uploaded_file.seek(0)
+    papers = pd.read_csv(uploaded_file, sep='\t')
+    if("htid" in papers.columns):
+        papers = sf.htrc(papers)
     papers.rename(columns=col_dict, inplace=True)
+    print(papers)
     return papers
 
+@st.cache_data(ttl=3600)
+def conv_json(extype):
+    col_dict={'title': 'title',
+    'rights_date_used': 'Year',
+    'content_provider_code': 'Document Type',
+    'Keywords':'Source title'
+    }
+
+    data = json.load(uploaded_file)
+    hathifile = data['gathers']
+    keywords = pd.DataFrame.from_records(hathifile)
+
+    keywords = sf.htrc(keywords)
+    keywords['Cited by'] = keywords.groupby(['Keywords'])['Keywords'].transform('size')
+    keywords.rename(columns=col_dict,inplace=True)
+    return keywords
+
+def conv_pub(extype):
+    if (get_ext(extype)).endswith('.tar.gz'):
+        bytedata = extype.read()
+        keywords = sf.readPub(bytedata)
+    elif (get_ext(extype)).endswith('.xml'):
+        bytedata = extype.read()
+        keywords = sf.readxml(bytedata)
+    keywords['Cited by'] = keywords.groupby(['Keywords'])['Keywords'].transform('size')
+    st.write(keywords)
+    return keywords
+
 #===Read data===
-uploaded_file = st.file_uploader('', type=['csv', 'txt'], on_change=reset_all)
+uploaded_file = st.file_uploader('', type=['csv', 'txt','json','tar.gz', 'xml'], on_change=reset_all)
 
 if uploaded_file is not None:
     try:
@@ -79,36 +130,45 @@ if uploaded_file is not None:
        
         elif extype.endswith('.txt'):
              papers = conv_txt(extype)
-        
+        elif extype.endswith('.json'):
+            papers = conv_json(extype)
+        elif extype.endswith('.tar.gz') or extype.endswith('.xml'):
+            papers = conv_pub(uploaded_file)
+
         @st.cache_data(ttl=3600)
         def get_minmax(extype):
             extype = extype
             MIN = int(papers['Year'].min())
             MAX = int(papers['Year'].max())
+            MIN1 = int(papers['Cited by'].min())
+            MAX1 = int(papers['Cited by'].max())
             GAP = MAX - MIN
-            return papers, MIN, MAX, GAP
-        
-        tab1, tab2 = st.tabs(["📈 Generate visualization", "📓 Recommended Reading"])
+            return papers, MIN, MAX, GAP, MIN1, MAX1
+
+        tab1, tab2, tab3 = st.tabs(["📈 Generate visualization", "📓 Recommended Reading", "⬇️ Download Help"])
         
         with tab1:    
             #===sunburst===
             try:
-                papers, MIN, MAX, GAP = get_minmax(extype)
+                papers, MIN, MAX, GAP, MIN1, MAX1 = get_minmax(extype)
             except KeyError:
                 st.error('Error: Please check again your columns.')
                 sys.exit(1)
             
             if (GAP != 0):
                 YEAR = st.slider('Year', min_value=MIN, max_value=MAX, value=(MIN, MAX), on_change=reset_all)
+                KEYLIM = st.slider('Cited By Count',min_value = MIN1, max_value = MAX1, value = (MIN1,MAX1), on_change=reset_all)
             else:
                 st.write('You only have data in ', (MAX))
                 YEAR = (MIN, MAX)
-            
+                KEYLIM = (MIN1,MAX1)
             @st.cache_data(ttl=3600)
             def listyear(extype):
                 global papers
                 years = list(range(YEAR[0],YEAR[1]+1))
+                cited = list(range(KEYLIM[0],KEYLIM[1]+1))
                 papers = papers.loc[papers['Year'].isin(years)]
+                papers = papers.loc[papers['Cited by'].isin(cited)]
                 return years, papers
             
             @st.cache_data(ttl=3600)
@@ -118,19 +178,23 @@ if uploaded_file is not None:
                 vis[['doctype','source','citby','year']] = papers[['Document Type','Source title','Cited by','Year']]
                 viz=vis.groupby(['doctype', 'source', 'year'])['citby'].agg(['sum','count']).reset_index()  
                 viz.rename(columns={'sum': 'cited by', 'count': 'total docs'}, inplace=True)
-                                
+
+
+
                 fig = px.sunburst(viz, path=['doctype', 'source', 'year'], values='total docs',
                               color='cited by', 
                               color_continuous_scale='RdBu',
                               color_continuous_midpoint=np.average(viz['cited by'], weights=viz['total docs']))
                 fig.update_layout(height=800, width=1200)
-                return fig
+                return fig, viz
             
             years, papers = listyear(extype)
     
             if {'Document Type','Source title','Cited by','Year'}.issubset(papers.columns):
-                fig = vis_sunbrust(extype)
+                fig, viz = vis_sunbrust(extype)
                 st.plotly_chart(fig, height=800, width=1200) #use_container_width=True)
+                
+                st.dataframe(viz)
                
             else:
                 st.error('We require these columns: Document Type, Source title, Cited by, Year', icon="🚨")
@@ -138,7 +202,10 @@ if uploaded_file is not None:
         with tab2:
             st.markdown('**numpy.average — NumPy v1.24 Manual. (n.d.). Numpy.Average — NumPy v1.24 Manual.** https://numpy.org/doc/stable/reference/generated/numpy.average.html')
             st.markdown('**Sunburst. (n.d.). Sunburst Charts in Python.** https://plotly.com/python/sunburst-charts/')
-    
+
+        with tab3:
+            st.text("Click the camera icon on the top right menu (you may need to hover your cursor within the visualization)")
+            st.markdown("![Downloading visualization](https://raw.githubusercontent.com/faizhalas/library-tools/main/images/download_bertopic.jpg)")
     except:
         st.error("Please ensure that your file is correct. Please contact us if you find that this is an error.", icon="🚨")
-        st.stop()
+        st.stop()

pages/5 Burst Detection.py

@@ -15,6 +15,9 @@ import plotly.graph_objects as go
 from plotly.subplots import make_subplots
 import plotly.io as pio
 import sys
+import json
+from tools import sourceformat as sf
+
 
 #===config===
 st.set_page_config(
@@ -42,6 +45,8 @@ with st.popover("🔗 Menu"):
     st.page_link("pages/4 Sunburst.py", label="Sunburst", icon="4️⃣")
     st.page_link("pages/5 Burst Detection.py", label="Burst Detection", icon="5️⃣")
     st.page_link("pages/6 Keywords Stem.py", label="Keywords Stem", icon="6️⃣")
+    st.page_link("pages/7 Sentiment Analysis.py", label="Sentiment Analysis", icon="7️⃣")
+    st.page_link("pages/8 Shifterator.py", label="Shifterator", icon="8️⃣")
 
 st.header("Burst Detection", anchor=False)
 st.subheader('Put your file here...', anchor=False)
@@ -51,7 +56,7 @@ def reset_all():
     st.cache_data.clear()
 
 # Initialize NLP model
-nlp = spacy.load("en_core_web_md")
+nlp = spacy.load("en_core_web_sm")
 
 @st.cache_data(ttl=3600)
 def upload(extype):
@@ -60,6 +65,15 @@ def upload(extype):
     if 'Publication Year' in df.columns:
                df.rename(columns={'Publication Year': 'Year', 'Citing Works Count': 'Cited by',
                                      'Publication Type': 'Document Type', 'Source Title': 'Source title'}, inplace=True)
+    if "About the data" in df.columns[0]:
+        df = sf.dim(df)
+        col_dict = {'MeSH terms': 'Keywords',
+        'PubYear': 'Year',
+        'Times cited': 'Cited by',
+        'Publication Type': 'Document Type'
+        }
+        df.rename(columns=col_dict, inplace=True)
+    
     return df
 
 @st.cache_data(ttl=3600)
@@ -76,14 +90,49 @@ def get_minmax(df):
 
 @st.cache_data(ttl=3600)
 def conv_txt(extype):
+    if("PMID" in (uploaded_file.read()).decode()):
+        uploaded_file.seek(0)
+        papers = sf.medline(uploaded_file)
+        print(papers)
+        return papers
     col_dict = {'TI': 'Title',
             'SO': 'Source title',
+            'DE': 'Author Keywords',
             'DT': 'Document Type',
             'AB': 'Abstract',
-            'PY': 'Year'}
-    df = pd.read_csv(uploaded_file, sep='\t', lineterminator='\r')
-    df.rename(columns=col_dict, inplace=True)
-    return df
+            'TC': 'Cited by',
+            'PY': 'Year',
+            'ID': 'Keywords Plus',
+            'rights_date_used': 'Year'}
+    uploaded_file.seek(0)
+    papers = pd.read_csv(uploaded_file, sep='\t')
+    if("htid" in papers.columns):
+        papers = sf.htrc(papers)
+    papers.rename(columns=col_dict, inplace=True)
+    print(papers)
+    return papers
+
+def conv_json(extype):
+    col_dict={'title': 'title',
+    'rights_date_used': 'Year',
+    }
+
+    data = json.load(uploaded_file)
+    hathifile = data['gathers']
+    keywords = pd.DataFrame.from_records(hathifile)
+    
+    keywords = sf.htrc(keywords)
+    keywords.rename(columns=col_dict,inplace=True)
+    return keywords
+
+def conv_pub(extype):
+    if (get_ext(extype)).endswith('.tar.gz'):
+        bytedata = extype.read()
+        keywords = sf.readPub(bytedata)
+    elif (get_ext(extype)).endswith('.xml'):
+        bytedata = extype.read()
+        keywords = sf.readxml(bytedata)
+    return keywords
 
 # Helper Functions
 @st.cache_data(ttl=3600)
@@ -107,6 +156,10 @@ def load_data(uploaded_file):
          df = upload(extype) 
     elif extype.endswith('.txt'):
          df = conv_txt(extype)
+    elif extype.endswith('.json'):
+        df = conv_json(extype)
+    elif extype.endswith('.tar.gz') or extype.endswith('.xml'):
+        df = conv_pub(uploaded_file)
 
     df['Year'] = pd.to_numeric(df['Year'], errors='coerce')
     df = df.dropna(subset=['Year'])
@@ -133,23 +186,29 @@ def clean_data(df):
     
     # Preprocess text
     df['processed'] = df.apply(lambda row: preprocess_text(f"{row.get(col_name, '')}"), axis=1)
+
+    ngram_range = (1, xgram)
     
     # Vectorize processed text
     if count_method == "Document Frequency":
-        vectorizer = CountVectorizer(lowercase=False, tokenizer=lambda x: x.split(), binary=True)
+        vectorizer = CountVectorizer(lowercase=False, tokenizer=lambda x: x.split(), binary=True, ngram_range=ngram_range)
     else:
-        vectorizer = CountVectorizer(lowercase=False, tokenizer=lambda x: x.split())
+        vectorizer = CountVectorizer(lowercase=False, tokenizer=lambda x: x.split(), ngram_range=ngram_range)
     X = vectorizer.fit_transform(df['processed'].tolist())
     
     # Create DataFrame from the Document-Term Matrix (DTM)
     dtm = pd.DataFrame(X.toarray(), columns=vectorizer.get_feature_names_out(), index=df['Year'].values)
     yearly_term_frequency = dtm.groupby(dtm.index).sum()
 
-    # User inputs for top words analysis and exclusions
-    excluded_words = [word.strip() for word in excluded_words_input.split(',')]
+    # excluded & included words
+    if exc_inc == "Words to exclude":
+        excluded_words = [word.strip() for word in words_input.split(',')]
+        filtered_words = [word for word in yearly_term_frequency.columns if word not in excluded_words]
     
-    # Identify top words, excluding specified words
-    filtered_words = [word for word in yearly_term_frequency.columns if word not in excluded_words]
+    elif exc_inc == "Focus on these words":
+        included_words = [word.strip() for word in words_input.split(',')]   
+        filtered_words = [word for word in yearly_term_frequency.columns if word in included_words]
+
     top_words = yearly_term_frequency[filtered_words].sum().nlargest(top_n).index.tolist()
     
     return yearly_term_frequency, top_words
@@ -205,27 +264,38 @@ def convert_df(df):
     return df.to_csv().encode("utf-8")
 
 @st.cache_data(ttl=3600)
-def scattervis(bursts, freq_data):
-    freq_data.reset_index(inplace=True)
+def scattervis(bursts, freq_data, top_n):
+    freq_data = freq_data.reset_index()
     freq_data.rename(columns={"index": "Year"}, inplace=True)
-                        
+    
     freq_data_melted = freq_data.melt(id_vars=["Year"], var_name="Category", value_name="Value")
     freq_data_melted = freq_data_melted[freq_data_melted["Value"] > 0]
-    wordlist = freq_data_melted["Category"].unique()
     
+    wordlist = freq_data_melted["Category"].unique()
     years = freq_data["Year"].tolist()
+    
     bursts["begin"] = bursts["begin"].apply(lambda x: years[min(x, len(years) - 1)] if x < len(years) else None)
     bursts["end"] = bursts["end"].apply(lambda x: years[min(x, len(years) - 1)] if x < len(years) else None)
+
     burst_points = []
-    
     for _, row in bursts.iterrows():
         for year in range(row["begin"], row["end"] + 1):
             burst_points.append((year, row["label"], row["weight"]))
-                    
     burst_points_df = pd.DataFrame(burst_points, columns=["Year", "Category", "Weight"])
-                    
+
+    min_year = min(years)
+    max_year = max(years)
+    n_years = max_year - min_year + 1
+    n_labels = len(wordlist)
+
+    label_spacing = 50   
+    year_spacing = 60    
+
+    plot_height = n_labels * label_spacing + 100
+    plot_width = n_years * year_spacing + 150
+
     fig = go.Figure()
-    
+
     # scatter trace for burst points
     fig.add_trace(go.Scatter(
         x=burst_points_df["Year"],
@@ -233,14 +303,15 @@ def scattervis(bursts, freq_data):
         mode='markers',
         marker=dict(
             symbol='square',
-            size=40,  
+            size=40,
             color='red',
-            opacity=0.5),
+            opacity=0.5
+        ),
         hoverinfo='text',
         text=burst_points_df["Weight"],
         showlegend=False
     ))
-                    
+
     # scatter trace for freq_data
     fig.add_trace(go.Scatter(
         x=freq_data_melted["Year"],
@@ -251,26 +322,43 @@ def scattervis(bursts, freq_data):
             size=30,
             color=freq_data_melted["Value"],
             colorscale='Blues',
-            showscale=False),
+            showscale=False
+        ),
         text=freq_data_melted["Value"],
         textposition="middle center",
         textfont=dict(
             size=16,
-            color=['white' if value > freq_data_melted["Value"].max()/2 else 'black' for value in freq_data_melted["Value"]])
+            color=['white' if value > freq_data_melted["Value"].max()/2 else 'black'
+                   for value in freq_data_melted["Value"]]
+        )
     ))
-    
-    min_year = min(years)
-    max_year = max(years)
-                    
+
+    # Layout
     fig.update_layout(
-        xaxis=dict(tickmode='linear', dtick=1, range=[(min_year-1), (max_year+1)], tickfont = dict(size=16), automargin=True, showgrid=False, zeroline=False),
-        yaxis=dict(tickvals=wordlist, ticktext=wordlist, tickmode='array', tickfont = dict(size=16), automargin=True, showgrid=False, zeroline=False),
-        plot_bgcolor='white',  
-        paper_bgcolor='white',  
+        xaxis=dict(
+            tickmode='linear',
+            dtick=1,
+            range=[min_year - 1, max_year + 1],
+            tickfont=dict(size=16),
+            automargin=True,
+            showgrid=False,
+            zeroline=False
+        ),
+        yaxis=dict(
+            tickvals=wordlist,
+            ticktext=wordlist,
+            tickmode='array',
+            tickfont=dict(size=16),
+            automargin=True,
+            showgrid=False,
+            zeroline=False
+        ),
+        plot_bgcolor='white',
+        paper_bgcolor='white',
         showlegend=False,
-        margin=dict(l=1, r=1, t=1, b=1),
-        height=top_n*50+2,
-        width=(max_year-min_year)*52+100,
+        margin=dict(l=20, r=20, t=20, b=20),
+        height=plot_height,
+        width=plot_width,
         autosize=False
     )
                     
@@ -289,15 +377,13 @@ def linegraph(bursts, freq_data):
             line_shape='linear',
             hoverinfo='text',
             hovertext=[f"Year: {index}<br>Frequency: {freq}" for index, freq in zip(freq_data.index, freq_data[column])],
-            text=freq_data[column],
+            #text=freq_data[column],
             textposition='top center'
         ), row=row, col=col)
-                
         # Add area charts
         for _, row_data in bursts[bursts['label'] == column].iterrows():
             x_values = freq_data.index[row_data['begin']:row_data['end']+1]
             y_values = freq_data[column][row_data['begin']:row_data['end']+1]
-                        
             #middle_y = sum(y_values) / len(y_values)
             y_post = min(freq_data[column]) + 1 if running_total == "Running total" else sum(y_values) / len(y_values)
             x_offset = 0.1
@@ -326,7 +412,19 @@ def linegraph(bursts, freq_data):
                 textangle=270,
                 row=row, col=col
                 )
-                
+            
+            # Add labels for values only in bursts
+            fig.add_trace(go.Scatter(
+            x=x_values, y=y_values, mode='lines+markers+text', name=column,
+            line_shape='linear',
+            hoverinfo='text',
+            hovertext=[f"Year: {index}<br>Frequency: {freq}" for index, freq in zip(freq_data.index, freq_data[column])],
+            text=y_values,
+            textposition='top center'
+        ), row=row, col=col)
+            print(freq_data[column])
+
+
         col += 1
         if col > 2:
             col = 1
@@ -349,36 +447,41 @@ def download_result(freq_data, bursts):
     csv2 = convert_df(bursts)
     return csv1, csv2
       
-uploaded_file = st.file_uploader('', type=['csv', 'txt'], on_change=reset_all)
+uploaded_file = st.file_uploader('', type=['csv', 'txt','json','tar.gz','xml'], on_change=reset_all)
 
 if uploaded_file is not None:
     try:
-        c1, c2, c3, c4 = st.columns([2,2,3,3])
+        c1, c2, c3 = st.columns([3,3,4])
         top_n = c1.number_input("Number of top words to analyze", min_value=5, value=10, step=1, on_change=reset_all)
         viz_selected = c2.selectbox("Option for visualization",
-            ("Line graph", "Scatter plot"), on_change=reset_all)
+            ("Line graph", "Heatmap"), on_change=reset_all)
         running_total = c3.selectbox("Calculation method",
             ("Running total", "By occurrences each year"), on_change=reset_all)
-        count_method = c4.selectbox("Count by",
+        count_method = c1.selectbox("Count by",
             ("Term Frequency", "Document Frequency"), on_change=reset_all)
 
-        d1, d2 = st.columns([2,8])
         df, coldf, MIN, MAX, GAP = load_data(uploaded_file)
-        col_name = d1.selectbox("Select column to analyze",
+        col_name = c2.selectbox("Select column to analyze",
             (coldf), on_change=reset_all)
-        excluded_words_input = d2.text_input("Words to exclude (comma-separated)", on_change=reset_all)
+        xgram = c3.selectbox("N-grams", ("1", "2", "3"), on_change=reset_all)
+        xgram = int(xgram)
+
+        st.divider()
+        d1, d2 = st.columns([3,7])
+        exc_inc = d1.radio("Select to exclude or focus on specific words", ["Words to exclude","Focus on these words"], horizontal=True, on_change=reset_all)
+        words_input = d2.text_input("Words to exclude or focus on (comma-separated)", on_change=reset_all)
 
         if (GAP != 0):
             YEAR = st.slider('Year', min_value=MIN, max_value=MAX, value=(MIN, MAX), on_change=reset_all)
         else:
-            e1.write('You only have data in ', (MAX))
+            c1.write('You only have data in ', (MAX))
             sys.exit(1)
       
         yearly_term_frequency, top_words = clean_data(df) 
         
         bursts, freq_data, num_unique_labels, num_rows = apply_burst_detection(top_words, yearly_term_frequency)
 
-        tab1, tab2, tab3 = st.tabs(["📈 Generate visualization", "📃 Reference", "📓 Recommended Reading"])
+        tab1, tab2, tab3, tab4 = st.tabs(["📈 Generate visualization", "📃 Reference", "📓 Recommended Reading", "⬇️ Download Help"])
 
         with tab1:        
             if bursts.empty:
@@ -394,7 +497,7 @@ if uploaded_file is not None:
                     linegraph(bursts, freq_data)
                     
                 elif viz_selected =="Scatter plot":
-                    scattervis(bursts, freq_data)
+                    scattervis(bursts, freq_data, top_n)
                 
                 csv1, csv2 = download_result(freq_data, bursts)
                 e1, e2, e3 = st.columns(3)
@@ -424,7 +527,23 @@ if uploaded_file is not None:
             st.markdown('**Li, M., Zheng, Z., & Yi, Q. (2024). The landscape of hot topics and research frontiers in Kawasaki disease: scientometric analysis. Heliyon, 10(8), e29680–e29680.** https://doi.org/10.1016/j.heliyon.2024.e29680')
             st.markdown('**Domicián Máté, Ni Made Estiyanti and Novotny, A. (2024) ‘How to support innovative small firms? Bibliometric analysis and visualization of start-up incubation’, Journal of Innovation and Entrepreneurship, 13(1).** https://doi.org/10.1186/s13731-024-00361-z')
             st.markdown('**Lamba, M., Madhusudhan, M. (2022). Burst Detection. In: Text Mining for Information Professionals. Springer, Cham.** https://doi.org/10.1007/978-3-030-85085-2_6')
+            st.markdown('**Santosa, F. A. (2025). Artificial Intelligence in Library Studies: A Textual Analysis. JLIS.It, 16(1).** https://doi.org/10.36253/jlis.it-626')
+        
+        with tab4:
+            st.subheader(':blue[Burst Detection]', anchor=False)
+            st.button('📊 Download high resolution image', on_click=None)
+            st.text("Click download button.") 
+
+            st.divider()
+            st.subheader(':blue[Top words]', anchor=False)
+            st.button('👉 Press to download list of top words', on_click=None)
+            st.text("Click download button.")  
+
+            st.divider()
+            st.subheader(':blue[Burst]', anchor=False)
+            st.button('👉 Press to download the list of detected bursts', on_click=None)
+            st.text("Click download button.")
             
-    except:
-        st.error("Please ensure that your file is correct. Please contact us if you find that this is an error.", icon="🚨")
-        st.stop()
+    except Exception as e:
+        st.error("Please ensure that your file or settings are correct. If you think there is a mistake, feel free to reach out to us!", icon="🚨")
+        st.stop()

pages/6 Keywords Stem.py

@@ -1,238 +1,298 @@
-import streamlit as st
-import pandas as pd
-import numpy as np
-import re
-import nltk
-nltk.download('wordnet')
-from nltk.stem import WordNetLemmatizer
-nltk.download('stopwords')
-from nltk.corpus import stopwords
-from pprint import pprint
-import pickle
-import streamlit.components.v1 as components
-from io import StringIO
-from nltk.stem.snowball import SnowballStemmer
-import csv
-import sys
-
-#===config===
-st.set_page_config(
-    page_title="Coconut",
-    page_icon="🥥",
-    layout="wide",
-    initial_sidebar_state="collapsed"
-)
-
-hide_streamlit_style = """
-            <style>
-            #MainMenu 
-            {visibility: hidden;}
-            footer {visibility: hidden;}
-            [data-testid="collapsedControl"] {display: none}
-            </style>
-            """
-st.markdown(hide_streamlit_style, unsafe_allow_html=True)
-
-with st.popover("🔗 Menu"):
-    st.page_link("https://www.coconut-libtool.com/", label="Home", icon="🏠")
-    st.page_link("pages/1 Scattertext.py", label="Scattertext", icon="1️⃣")
-    st.page_link("pages/2 Topic Modeling.py", label="Topic Modeling", icon="2️⃣")
-    st.page_link("pages/3 Bidirected Network.py", label="Bidirected Network", icon="3️⃣")
-    st.page_link("pages/4 Sunburst.py", label="Sunburst", icon="4️⃣")
-    st.page_link("pages/5 Burst Detection.py", label="Burst Detection", icon="5️⃣")
-    st.page_link("pages/6 Keywords Stem.py", label="Keywords Stem", icon="6️⃣")
-    
-
-st.header("Keywords Stem", anchor=False)
-st.subheader('Put your file here...', anchor=False)
-
-def reset_data():
-     st.cache_data.clear()
-
-#===check filetype===
-@st.cache_data(ttl=3600)
-def get_ext(extype):
-    extype = uploaded_file.name
-    return extype
-     
-#===upload===
-@st.cache_data(ttl=3600)
-def upload(extype):
-    keywords = pd.read_csv(uploaded_file)
-    return keywords
-
-@st.cache_data(ttl=3600)
-def conv_txt(extype):
-    col_dict = {'TI': 'Title',
-            'SO': 'Source title',
-            'DE': 'Author Keywords',
-            'ID': 'Keywords Plus'}
-    keywords = pd.read_csv(uploaded_file, sep='\t', lineterminator='\r')
-    keywords.rename(columns=col_dict, inplace=True)
-    return keywords
-
-@st.cache_data(ttl=3600)
-def rev_conv_txt(extype):
-    col_dict_rev = {'Title': 'TI',
-            'Source title': 'SO',
-            'Author Keywords': 'DE',
-            'Keywords Plus': 'ID'}
-    keywords.rename(columns=col_dict_rev, inplace=True)
-    return keywords
-
-@st.cache_data(ttl=3600)
-def get_data(extype):
-    list_of_column_key = list(keywords.columns)
-    list_of_column_key = [k for k in list_of_column_key if 'Keyword' in k]
-    return list_of_column_key
-
-uploaded_file = st.file_uploader('', type=['csv','txt'], on_change=reset_data)
-
-if uploaded_file is not None:
-    try:
-        extype = get_ext(uploaded_file)
-        if extype.endswith('.csv'):
-            keywords = upload(extype) 
-                      
-        elif extype.endswith('.txt'):
-            keywords = conv_txt(extype)
-             
-        list_of_column_key = get_data(extype)
-    
-        col1, col2 = st.columns(2)
-        with col1:
-            method = st.selectbox(
-                'Choose method',
-                ('Lemmatization', 'Stemming'), on_change=reset_data)
-        with col2:
-            keyword = st.selectbox(
-                'Choose column',
-                (list_of_column_key), on_change=reset_data)
-    
-        @st.cache_data(ttl=3600)
-        def clean_keyword(extype):      
-            global keyword, keywords
-            try:
-                key = keywords[keyword]
-            except KeyError:
-                st.error('Error: Please check your Author/Index Keywords column.')
-                sys.exit(1)
-            keywords = keywords.replace(np.nan, '', regex=True)
-            keywords[keyword] = keywords[keyword].astype(str)
-            keywords[keyword] = keywords[keyword].map(lambda x: re.sub('-', ' ', x))
-            keywords[keyword] = keywords[keyword].map(lambda x: re.sub('; ', ' ; ', x))
-            keywords[keyword] = keywords[keyword].map(lambda x: x.lower())
-            
-            #===Keywords list===
-            key = key.dropna()
-            key = pd.concat([key.str.split('; ', expand=True)], axis=1)
-            key = pd.Series(np.ravel(key)).dropna().drop_duplicates().sort_values().reset_index()
-            key[0] = key[0].map(lambda x: re.sub('-', ' ', x))
-            key['new']=key[0].map(lambda x: x.lower())
-    
-            return keywords, key
-         
-        #===stem/lem===
-        @st.cache_data(ttl=3600)
-        def Lemmatization(extype):
-            lemmatizer = WordNetLemmatizer()
-            def lemmatize_words(text):
-                words = text.split()
-                words = [lemmatizer.lemmatize(word) for word in words]
-                return ' '.join(words)
-            keywords[keyword] = keywords[keyword].apply(lemmatize_words)
-            key['new'] = key['new'].apply(lemmatize_words)
-            keywords[keyword] = keywords[keyword].map(lambda x: re.sub(' ; ', '; ', x))
-            return keywords, key
-                    
-        @st.cache_data(ttl=3600)
-        def Stemming(extype):
-            stemmer = SnowballStemmer("english")
-            def stem_words(text):
-                words = text.split()
-                words = [stemmer.stem(word) for word in words]
-                return ' '.join(words)
-            keywords[keyword] = keywords[keyword].apply(stem_words)
-            key['new'] = key['new'].apply(stem_words)
-            keywords[keyword] = keywords[keyword].map(lambda x: re.sub(' ; ', '; ', x))
-            return keywords, key
-         
-        keywords, key = clean_keyword(extype) 
-         
-        if method is 'Lemmatization':
-            keywords, key = Lemmatization(extype)
-        else:
-            keywords, key = Stemming(extype)
-                
-        st.write('Congratulations! 🤩 You choose',keyword ,'with',method,'method. Now, you can easily download the result by clicking the button below')
-        st.divider()
-              
-        #===show & download csv===
-        tab1, tab2, tab3, tab4 = st.tabs(["📥 Result", "📥 List of Keywords", "📃 Reference", "📃 Recommended Reading"])
-         
-        with tab1:
-            st.dataframe(keywords, use_container_width=True, hide_index=True)
-            @st.cache_data(ttl=3600)
-            def convert_df(extype):
-                return keywords.to_csv(index=False).encode('utf-8')
-             
-            @st.cache_data(ttl=3600)
-            def convert_txt(extype):
-                return keywords.to_csv(index=False, sep='\t', lineterminator='\r').encode('utf-8')
-             
-            if extype.endswith('.csv'):
-                csv = convert_df(extype)
-                st.download_button(
-                    "Press to download result 👈",
-                    csv,
-                    "result.csv",
-                    "text/csv")
-      
-            elif extype.endswith('.txt'):
-                keywords = rev_conv_txt(extype)
-                txt = convert_txt(extype)
-                st.download_button(
-                    "Press to download result 👈",
-                    txt,
-                    "result.txt",
-                    "text/csv")    
-             
-        with tab2:
-            @st.cache_data(ttl=3600)
-            def table_keyword(extype):
-                keytab = key.drop(['index'], axis=1).rename(columns={0: 'label'})
-                return keytab
-                
-            #===coloring the same keywords===
-            @st.cache_data(ttl=3600)
-            def highlight_cells(value):
-                if keytab['new'].duplicated(keep=False).any() and keytab['new'].duplicated(keep=False)[keytab['new'] == value].any():
-                    return 'background-color: yellow'
-                return '' 
-            keytab = table_keyword(extype) 
-            st.dataframe(keytab.style.applymap(highlight_cells, subset=['new']), use_container_width=True, hide_index=True)
-                      
-            @st.cache_data(ttl=3600)
-            def convert_dfs(extype):
-                return key.to_csv(index=False).encode('utf-8')
-                    
-            csv = convert_dfs(extype)
-    
-            st.download_button(
-                "Press to download keywords 👈",
-                csv,
-                "keywords.csv",
-                "text/csv")
-                 
-        with tab3:
-            st.markdown('**Santosa, F. A. (2023). Prior steps into knowledge mapping: Text mining application and comparison. Issues in Science and Technology Librarianship, 102.** https://doi.org/10.29173/istl2736')
-         
-        with tab4:
-            st.markdown('**Beri, A. (2021, January 27). Stemming vs Lemmatization. Medium.** https://towardsdatascience.com/stemming-vs-lemmatization-2daddabcb221')
-            st.markdown('**Khyani, D., Siddhartha B S, Niveditha N M, &amp; Divya B M. (2020). An Interpretation of Lemmatization and Stemming in Natural Language Processing. Journal of University of Shanghai for Science and Technology , 22(10), 350–357.**  https://jusst.org/an-interpretation-of-lemmatization-and-stemming-in-natural-language-processing/')
-            st.markdown('**Lamba, M., & Madhusudhan, M. (2021, July 31). Text Pre-Processing. Text Mining for Information Professionals, 79–103.** https://doi.org/10.1007/978-3-030-85085-2_3')
-
-
-    except:
-        st.error("Please ensure that your file is correct. Please contact us if you find that this is an error.", icon="🚨")
-        st.stop()     
+import streamlit as st
+import pandas as pd
+import numpy as np
+import re
+import nltk
+nltk.download('wordnet')
+from nltk.stem import WordNetLemmatizer
+nltk.download('stopwords')
+from nltk.corpus import stopwords
+from pprint import pprint
+import pickle
+import streamlit.components.v1 as components
+from io import StringIO
+from nltk.stem.snowball import SnowballStemmer
+import csv
+import sys
+import json
+from tools import sourceformat as sf
+
+
+#===config===
+st.set_page_config(
+    page_title="Coconut",
+    page_icon="🥥",
+    layout="wide",
+    initial_sidebar_state="collapsed"
+)
+
+hide_streamlit_style = """
+            <style>
+            #MainMenu 
+            {visibility: hidden;}
+            footer {visibility: hidden;}
+            [data-testid="collapsedControl"] {display: none}
+            </style>
+            """
+st.markdown(hide_streamlit_style, unsafe_allow_html=True)
+
+with st.popover("🔗 Menu"):
+    st.page_link("https://www.coconut-libtool.com/", label="Home", icon="🏠")
+    st.page_link("pages/1 Scattertext.py", label="Scattertext", icon="1️⃣")
+    st.page_link("pages/2 Topic Modeling.py", label="Topic Modeling", icon="2️⃣")
+    st.page_link("pages/3 Bidirected Network.py", label="Bidirected Network", icon="3️⃣")
+    st.page_link("pages/4 Sunburst.py", label="Sunburst", icon="4️⃣")
+    st.page_link("pages/5 Burst Detection.py", label="Burst Detection", icon="5️⃣")
+    st.page_link("pages/6 Keywords Stem.py", label="Keywords Stem", icon="6️⃣")
+    st.page_link("pages/7 Sentiment Analysis.py", label="Sentiment Analysis", icon="7️⃣")
+    
+
+st.header("Keywords Stem", anchor=False)
+st.subheader('Put your file here...', anchor=False)
+
+def reset_data():
+     st.cache_data.clear()
+
+#===check filetype===
+@st.cache_data(ttl=3600)
+def get_ext(extype):
+    extype = uploaded_file.name
+    return extype
+     
+#===upload===
+@st.cache_data(ttl=3600)
+def upload(extype):
+    keywords = pd.read_csv(uploaded_file)
+
+    if "About the data" in keywords.columns[0]:
+        keywords = sf.dim(keywords)
+        col_dict = {'MeSH terms': 'Keywords',
+        'PubYear': 'Year',
+        'Times cited': 'Cited by',
+        'Publication Type': 'Document Type'
+        }
+        keywords.rename(columns=col_dict, inplace=True)
+
+    return keywords
+@st.cache_data(ttl=3600)
+def conv_txt(extype):
+    if("PMID" in (uploaded_file.read()).decode()):
+        uploaded_file.seek(0)
+        papers = sf.medline(uploaded_file)
+        print(papers)
+        return papers
+    col_dict = {'TI': 'Title',
+            'SO': 'Source title',
+            'DE': 'Author Keywords',
+            'DT': 'Document Type',
+            'AB': 'Abstract',
+            'TC': 'Cited by',
+            'PY': 'Year',
+            'ID': 'Keywords Plus',
+            'rights_date_used': 'Year'}
+    uploaded_file.seek(0)
+    papers = pd.read_csv(uploaded_file, sep='\t')
+    if("htid" in papers.columns):
+        papers = sf.htrc(papers)
+    papers.rename(columns=col_dict, inplace=True)
+    print(papers)
+    return papers
+
+@st.cache_data(ttl=3600)
+def rev_conv_txt(extype):
+    col_dict_rev = {'Title': 'TI',
+            'Source title': 'SO',
+            'Author Keywords': 'DE',
+            'Keywords Plus': 'ID'}
+    keywords.rename(columns=col_dict_rev, inplace=True)
+    return keywords
+
+@st.cache_data(ttl=3600)
+def conv_json(extype):
+    col_dict={'title': 'title',
+    'rights_date_used': 'Year',
+    }
+
+    data = json.load(uploaded_file)
+    hathifile = data['gathers']
+    keywords = pd.DataFrame.from_records(hathifile)
+
+    keywords = sf.htrc(keywords)
+    keywords.rename(columns=col_dict,inplace=True)
+    return keywords
+
+def conv_pub(extype):
+    if (get_ext(extype)).endswith('.tar.gz'):
+        bytedata = extype.read()
+        keywords = sf.readPub(bytedata)
+    elif (get_ext(extype)).endswith('.xml'):
+        bytedata = extype.read()
+        keywords = sf.readxml(bytedata)
+    return keywords
+
+@st.cache_data(ttl=3600)
+def get_data(extype):
+    list_of_column_key = list(keywords.columns)
+    list_of_column_key = [k for k in list_of_column_key if 'Keyword' in k]
+    return list_of_column_key
+
+uploaded_file = st.file_uploader('', type=['csv','txt','json','tar.gz','xml'], on_change=reset_data)
+
+if uploaded_file is not None:
+    try:
+        extype = get_ext(uploaded_file)
+        if extype.endswith('.csv'):
+            keywords = upload(extype) 
+                      
+        elif extype.endswith('.txt'):
+            keywords = conv_txt(extype)
+
+        elif extype.endswith('.json'):
+            keywords = conv_json(extype)
+        elif extype.endswith('.tar.gz') or extype.endswith('.xml'):
+            keywords = conv_pub(uploaded_file)
+
+        list_of_column_key = get_data(extype)
+    
+        col1, col2 = st.columns(2)
+        with col1:
+            method = st.selectbox(
+                'Choose method',
+                ('Lemmatization', 'Stemming'), on_change=reset_data)
+        with col2:
+            keyword = st.selectbox(
+                'Choose column',
+                (list_of_column_key), on_change=reset_data)
+    
+        @st.cache_data(ttl=3600)
+        def clean_keyword(extype):      
+            global keyword, keywords
+            try:
+                key = keywords[keyword]
+            except KeyError:
+                st.error('Error: Please check your Author/Index Keywords column.')
+                sys.exit(1)
+            keywords = keywords.replace(np.nan, '', regex=True)
+            keywords[keyword] = keywords[keyword].astype(str)
+            keywords[keyword] = keywords[keyword].map(lambda x: re.sub('-', ' ', x))
+            keywords[keyword] = keywords[keyword].map(lambda x: re.sub('; ', ' ; ', x))
+            keywords[keyword] = keywords[keyword].map(lambda x: x.lower())
+            
+            #===Keywords list===
+            key = key.dropna()
+            key = pd.concat([key.str.split('; ', expand=True)], axis=1)
+            key = pd.Series(np.ravel(key)).dropna().drop_duplicates().sort_values().reset_index()
+            key[0] = key[0].map(lambda x: re.sub('-', ' ', x))
+            key['new']=key[0].map(lambda x: x.lower())
+    
+            return keywords, key
+         
+        #===stem/lem===
+        @st.cache_data(ttl=3600)
+        def Lemmatization(extype):
+            lemmatizer = WordNetLemmatizer()
+            def lemmatize_words(text):
+                words = text.split()
+                words = [lemmatizer.lemmatize(word) for word in words]
+                return ' '.join(words)
+            keywords[keyword] = keywords[keyword].apply(lemmatize_words)
+            key['new'] = key['new'].apply(lemmatize_words)
+            keywords[keyword] = keywords[keyword].map(lambda x: re.sub(' ; ', '; ', x))
+            return keywords, key
+                    
+        @st.cache_data(ttl=3600)
+        def Stemming(extype):
+            stemmer = SnowballStemmer("english")
+            def stem_words(text):
+                words = text.split()
+                words = [stemmer.stem(word) for word in words]
+                return ' '.join(words)
+            keywords[keyword] = keywords[keyword].apply(stem_words)
+            key['new'] = key['new'].apply(stem_words)
+            keywords[keyword] = keywords[keyword].map(lambda x: re.sub(' ; ', '; ', x))
+            return keywords, key
+         
+        keywords, key = clean_keyword(extype) 
+         
+        if method is 'Lemmatization':
+            keywords, key = Lemmatization(extype)
+        else:
+            keywords, key = Stemming(extype)
+                
+        st.write('Congratulations! 🤩 You choose',keyword ,'with',method,'method. Now, you can easily download the result by clicking the button below')
+        st.divider()
+              
+        #===show & download csv===
+        tab1, tab2, tab3, tab4, tab5 = st.tabs(["📥 Result", "📥 List of Keywords", "📃 Reference", "📃 Recommended Reading", "⬇️ Download Help"])
+         
+        with tab1:
+            st.dataframe(keywords, use_container_width=True, hide_index=True)
+            @st.cache_data(ttl=3600)
+            def convert_df(extype):
+                return keywords.to_csv(index=False).encode('utf-8')
+             
+            @st.cache_data(ttl=3600)
+            def convert_txt(extype):
+                return keywords.to_csv(index=False, sep='\t', lineterminator='\r').encode('utf-8')
+             
+            if extype.endswith('.csv'):
+                csv = convert_df(extype)
+                st.download_button(
+                    "Press to download result 👈",
+                    csv,
+                    "result.csv",
+                    "text/csv")
+      
+            elif extype.endswith('.txt'):
+                keywords = rev_conv_txt(extype)
+                txt = convert_txt(extype)
+                st.download_button(
+                    "Press to download result 👈",
+                    txt,
+                    "result.txt",
+                    "text/csv")    
+             
+        with tab2:
+            @st.cache_data(ttl=3600)
+            def table_keyword(extype):
+                keytab = key.drop(['index'], axis=1).rename(columns={0: 'label'})
+                return keytab
+                
+            #===coloring the same keywords===
+            @st.cache_data(ttl=3600)
+            def highlight_cells(value):
+                if keytab['new'].duplicated(keep=False).any() and keytab['new'].duplicated(keep=False)[keytab['new'] == value].any():
+                    return 'background-color: yellow'
+                return '' 
+            keytab = table_keyword(extype) 
+            st.dataframe(keytab.style.applymap(highlight_cells, subset=['new']), use_container_width=True, hide_index=True)
+                      
+            @st.cache_data(ttl=3600)
+            def convert_dfs(extype):
+                return key.to_csv(index=False).encode('utf-8')
+                    
+            csv = convert_dfs(extype)
+    
+            st.download_button(
+                "Press to download keywords 👈",
+                csv,
+                "keywords.csv",
+                "text/csv")
+                 
+        with tab3:
+            st.markdown('**Santosa, F. A. (2023). Prior steps into knowledge mapping: Text mining application and comparison. Issues in Science and Technology Librarianship, 102.** https://doi.org/10.29173/istl2736')
+         
+        with tab4:
+            st.markdown('**Beri, A. (2021, January 27). Stemming vs Lemmatization. Medium.** https://towardsdatascience.com/stemming-vs-lemmatization-2daddabcb221')
+            st.markdown('**Khyani, D., Siddhartha B S, Niveditha N M, &amp; Divya B M. (2020). An Interpretation of Lemmatization and Stemming in Natural Language Processing. Journal of University of Shanghai for Science and Technology , 22(10), 350–357.**  https://jusst.org/an-interpretation-of-lemmatization-and-stemming-in-natural-language-processing/')
+            st.markdown('**Lamba, M., & Madhusudhan, M. (2021, July 31). Text Pre-Processing. Text Mining for Information Professionals, 79–103.** https://doi.org/10.1007/978-3-030-85085-2_3')
+
+        with tab5:
+            st.text("Download keywords at bottom of table")
+            st.divider()
+            st.text("Download table")
+            st.markdown("![Downloading visualization](https://raw.githubusercontent.com/faizhalas/library-tools/mainimages/downloadtable.png")
+    except Exception as e:
+        st.write(e)
+        st.error("Please ensure that your file is correct. Please contact us if you find that this is an error.", icon="🚨")
+        st.stop()     

pages/7 Sentiment Analysis.py

@@ -0,0 +1,357 @@
+#import module
+import streamlit as st
+import streamlit.components.v1 as components
+import pandas as pd
+import re
+import nltk
+import pandas as pd
+from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
+nltk.download('stopwords')
+from nltk.corpus import stopwords
+from nltk.tokenize import word_tokenize
+from nltk.stem import WordNetLemmatizer
+nltk.download('punkt_tab')
+nltk.download('vader_lexicon')
+from textblob import TextBlob
+import os
+import numpy as np
+import plotly.express as px
+import json
+from tools import sourceformat as sf
+
+#===config===
+st.set_page_config(
+    page_title="Coconut",
+    page_icon="🥥",
+    layout="wide",
+    initial_sidebar_state="collapsed"
+)
+
+hide_streamlit_style = """
+            <style>
+            #MainMenu 
+            {visibility: hidden;}
+            footer {visibility: hidden;}
+            [data-testid="collapsedControl"] {display: none}
+            </style>
+            """
+st.markdown(hide_streamlit_style, unsafe_allow_html=True)
+
+with st.popover("🔗 Menu"):
+    st.page_link("https://www.coconut-libtool.com/", label="Home", icon="🏠")
+    st.page_link("pages/1 Scattertext.py", label="Scattertext", icon="1️⃣")
+    st.page_link("pages/2 Topic Modeling.py", label="Topic Modeling", icon="2️⃣")
+    st.page_link("pages/3 Bidirected Network.py", label="Bidirected Network", icon="3️⃣")
+    st.page_link("pages/4 Sunburst.py", label="Sunburst", icon="4️⃣")
+    st.page_link("pages/5 Burst Detection.py", label="Burst Detection", icon="5️⃣")
+    st.page_link("pages/6 Keywords Stem.py", label="Keywords Stem", icon="6️⃣")
+    st.page_link("pages/7 Sentiment Analysis.py", label="Sentiment Analysis", icon="7️⃣")
+
+st.header("Sentiment Analysis", anchor=False)
+st.subheader('Put your file here...', anchor=False)
+
+#========unique id========
+@st.cache_resource(ttl=3600)
+def create_list():
+    l = [1, 2, 3]
+    return l
+
+l = create_list()
+first_list_value = l[0]
+l[0] = first_list_value + 1
+uID = str(l[0])
+
+@st.cache_data(ttl=3600)
+def get_ext(uploaded_file):
+    extype = uID+uploaded_file.name
+    return extype
+
+#===clear cache===
+
+
+def reset_all():
+    st.cache_data.clear()
+
+#===avoiding deadlock===
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+        
+#===upload file===
+@st.cache_data(ttl=3600)
+def upload(file):
+    papers = pd.read_csv(uploaded_file)
+    if "About the data" in papers.columns[0]:
+        papers = sf.dim(papers)
+        col_dict = {'MeSH terms': 'Keywords',
+        'PubYear': 'Year',
+        'Times cited': 'Cited by',
+        'Publication Type': 'Document Type'
+        }
+        papers.rename(columns=col_dict, inplace=True)
+    return papers
+
+@st.cache_data(ttl=3600)
+def conv_txt(extype):
+    if("PMID" in (uploaded_file.read()).decode()):
+        uploaded_file.seek(0)
+        papers = sf.medline(uploaded_file)
+        print(papers)
+        return papers
+    col_dict = {'TI': 'Title',
+            'SO': 'Source title',
+            'DE': 'Author Keywords',
+            'DT': 'Document Type',
+            'AB': 'Abstract',
+            'TC': 'Cited by',
+            'PY': 'Year',
+            'ID': 'Keywords Plus',
+            'rights_date_used': 'Year'}
+    uploaded_file.seek(0)
+    papers = pd.read_csv(uploaded_file, sep='\t')
+    if("htid" in papers.columns):
+        papers = sf.htrc(papers)
+    papers.rename(columns=col_dict, inplace=True)
+    print(papers)
+    return papers
+
+
+@st.cache_data(ttl=3600)
+def conv_json(extype):
+    col_dict={'title': 'title',
+    'rights_date_used': 'Year',
+    }
+
+    data = json.load(uploaded_file)
+    hathifile = data['gathers']
+    keywords = pd.DataFrame.from_records(hathifile)
+
+    keywords = sf.htrc(keywords)
+    keywords.rename(columns=col_dict,inplace=True)
+    return keywords
+
+@st.cache_resource(ttl=3600)
+def conv_pub(extype):
+    if (get_ext(extype)).endswith('.tar.gz'):
+        bytedata = extype.read()
+        keywords = sf.readPub(bytedata)
+    elif (get_ext(extype)).endswith('.xml'):
+        bytedata = extype.read()
+        keywords = sf.readxml(bytedata)
+    return keywords
+
+#===Read data===
+uploaded_file = st.file_uploader('', type=['csv', 'txt','json','tar.gz', 'xml'], on_change=reset_all)
+
+if uploaded_file is not None:
+    try:
+        extype = get_ext(uploaded_file)
+    
+        if extype.endswith('.csv'):
+             papers = upload(extype) 
+        elif extype.endswith('.txt'):
+             papers = conv_txt(extype)
+
+        elif extype.endswith('.json'):
+            papers = conv_json(extype)
+        elif extype.endswith('.tar.gz') or extype.endswith('.xml'):
+            papers = conv_pub(uploaded_file)
+
+        coldf = sorted(papers.select_dtypes(include=['object']).columns.tolist())
+            
+        c1, c2 = st.columns(2)
+        ColCho = c1.selectbox(
+                'Choose column',
+                (coldf), on_change=reset_all)
+        method = c2.selectbox(
+            'Choose method',[
+            'TextBlob','NLTKvader']
+        )
+        words_to_remove = c1.text_input("Remove specific words. Separate words by semicolons (;)")        
+        wordcount = c2.number_input(label = "Words displayed", min_value = 1, step = 1, value=5)-1
+        rem_copyright = c1.toggle('Remove copyright statement', value=True, on_change=reset_all)
+        rem_punc = c2.toggle('Remove punctuation', value=True, on_change=reset_all)
+
+        #===clean csv===
+        @st.cache_data(ttl=3600, show_spinner=False)
+        def clean_csv(extype):
+            paper = papers.dropna(subset=[ColCho])
+                     
+            #===mapping===
+            paper['Abstract_pre'] = paper[ColCho].map(lambda x: x.lower())
+            if rem_punc:
+                 paper['Abstract_pre'] = paper['Abstract_pre'].map(lambda x: re.sub('[,:;\.!-?•=]', ' ', x))
+                 paper['Abstract_pre'] = paper['Abstract_pre'].str.replace('\u201c|\u201d', '', regex=True) 
+            if rem_copyright:  
+                 paper['Abstract_pre'] = paper['Abstract_pre'].map(lambda x: re.sub('©.*', '', x))
+            
+            #===stopword removal===
+            stop = stopwords.words('english')
+            paper[ColCho] = paper['Abstract_pre'].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)]))
+              
+            words_rmv = [word.strip() for word in words_to_remove.split(";")]
+            remove_dict = {word: None for word in words_rmv}
+            
+            @st.cache_resource(ttl=3600)
+            def remove_words(text):
+                 words = text.split()
+                 cleaned_words = [word for word in words if word not in remove_dict]
+                 return ' '.join(cleaned_words) 
+            
+            paper['Sentences__'] = paper['Abstract_pre'].map(remove_words)
+
+            return paper
+        paper=clean_csv(extype) 
+    
+        if method == 'NLTKvader':
+            analyzer = SentimentIntensityAnalyzer()
+
+            @st.cache_resource(ttl=3600)
+            def get_sentiment(text):
+                score = analyzer.polarity_scores(text)
+                return score
+
+            tab1, tab2, tab3, tab4 = st.tabs(["📈 Result", "📃 Reference", "📓 Recommended Reading", "⬇️ Download Help"])
+            with tab1:
+                
+                paper['Scores'] = paper['Sentences__'].apply(get_sentiment)
+
+                scoreframe = pd.DataFrame()
+
+                scoreframe['Phrase'] = pd.Series(paper['Sentences__'])
+
+                scoreframe[['Negativity','Neutrality','Positivity','Compound']] = pd.DataFrame.from_records(paper['Scores'])
+
+                scoreframe = scoreframe.groupby(scoreframe.columns.tolist(),as_index=False).size()
+
+                scoreframe = scoreframe.truncate(after = wordcount)
+
+                with st.expander("Sentence and Results"):
+                    finalframe = pd.DataFrame()
+                    finalframe['Sentence'] = scoreframe['Phrase']
+                    finalframe[['Negativity','Neutrality','Positivity','Compound']] = scoreframe[['Negativity','Neutrality','Positivity','Compound']]
+                    finalframe[['Count']] = scoreframe[['size']]
+
+                    st.dataframe(finalframe, use_container_width=True)
+
+            with tab2:
+                st.markdown('**Hutto, C. and Gilbert, E. (2014) ‘VADER: A Parsimonious Rule-Based Model for Sentiment Analysis of Social Media Text’, Proceedings of the International AAAI Conference on Web and Social Media, 8(1), pp. 216–225.** https://doi.org/10.1609/icwsm.v8i1.14550')
+
+            with tab3:
+                st.markdown('**Lamba, M., & Madhusudhan, M. (2021, July 31). Sentiment Analysis. Text Mining for Information Professionals, 191–211.** https://doi.org/10.1007/978-3-030-85085-2_7')
+
+            with tab4:
+                st.subheader(':blue[CSV Results]', anchor=False)
+                st.text("Click Download button")
+                st.markdown("![Downloading results](https://raw.githubusercontent.com/faizhalas/library-tools/main/images/sentitable.png)")
+        
+        elif(method == 'TextBlob'):
+            
+            @st.cache_resource(ttl=3600)
+            def get_sentimentb(text):
+                line = TextBlob(text)
+                return line.sentiment
+
+            @st.cache_resource(ttl=3600)
+            def get_assessments(frame):
+                text = TextBlob(str(frame))
+
+                polar, subject, assessment = text.sentiment_assessments
+
+                try:
+                    phrase, phrasepolar, phrasesubject, unknown = assessment[0]
+                except: #this only happens if assessment is empty
+                    phrase, phrasepolar, phrasesubject = "empty", 0, 0
+
+                return phrase, phrasepolar, phrasesubject
+
+            @st.cache_resource(ttl=3600)
+            def mergelist(data):
+                return ' '.join(data)
+
+            @st.cache_resource(ttl=3600)
+            def assignscore(data):
+                if data>0:
+                    return "Positive"
+                elif data<0:
+                    return "Negative"
+                else:
+                    return "Neutral"
+
+            phrases = paper['Sentences__'].apply(get_assessments)
+
+            phraselist = phrases.to_list()
+
+            phraseframe = pd.DataFrame(phraselist, columns =["Phrase","Polarity","Subjectivity"])
+
+            phraseframe["Phrase"] = phraseframe["Phrase"].apply(mergelist)
+
+            phraseframe = phraseframe.groupby(phraseframe.columns.tolist(),as_index=False).size()
+
+            phraseframe["Score"] = phraseframe["Polarity"].apply(assignscore)
+
+            neut = phraseframe.loc[phraseframe['Score']=="Neutral"]
+            neut.reset_index(inplace = True)
+
+            pos = phraseframe.loc[phraseframe['Score']=="Positive"]
+            pos.reset_index(inplace = True)
+
+            neg = phraseframe.loc[phraseframe['Score']=="Negative"]
+            neg.reset_index(inplace = True)
+
+            paper['Sentiment'] = paper['Sentences__'].apply(get_sentimentb)
+
+            pos.sort_values(by=["size"], inplace = True, ascending = False, ignore_index = True)
+            pos = pos.truncate(after = wordcount)
+
+            neg.sort_values(by=["size"], inplace = True, ascending = False, ignore_index = True)
+            neg = neg.truncate(after = wordcount)
+        
+            neut.sort_values(by=["size"], inplace = True, ascending = False, ignore_index = True)
+            neut = neut.truncate(after = wordcount)
+
+            tab1, tab2, tab3, tab4 = st.tabs(["📈 Generate visualization", "📃 Reference", "📓 Recommended Reading", "⬇️ Download Help"])
+            with tab1:
+                #display tables and graphs
+    
+                with st.expander("Positive Sentiment"):
+                    st.dataframe(pos, use_container_width=True)
+                    figpos = px.bar(pos, x="Phrase", y="size", labels={"size": "Count", "Phrase": "Word"})      
+                    st.plotly_chart(figpos, use_container_width=True)
+    
+                with st.expander("Negative Sentiment"):
+                    st.dataframe(neg, use_container_width=True)
+                    figneg = px.bar(neg, x="Phrase", y="size", labels={"size": "Count", "Phrase": "Word"}, color_discrete_sequence=["#e57d7d"])
+                    st.plotly_chart(figneg, use_container_width=True)
+    
+                with st.expander("Neutral Sentiment"):
+                    st.dataframe(neut, use_container_width=True)
+                    figneut = px.bar(neut, x="Phrase", y="size", labels={"size": "Count", "Phrase": "Word"}, color_discrete_sequence=["#737a72"])
+                    st.plotly_chart(figneut, use_container_width=True)
+
+
+                with st.expander("Sentence and Results"):
+                    finalframe = pd.DataFrame()
+                    finalframe['Sentence'] = paper['Sentences__']
+                    finalframe[['Polarity','Subjectivity']] = pd.DataFrame(paper['Sentiment'].tolist(), index = paper.index)
+            
+                    st.dataframe(finalframe, use_container_width=True)
+
+            with tab2:
+                st.markdown('**Steven, L. et al. (2018) TextBlob: Simplified Text Processing — TextBlob 0.15.2 documentation, Readthedocs.io.** https://textblob.readthedocs.io/en/dev/')
+
+            with tab3:
+                st.markdown('**Lamba, M., & Madhusudhan, M. (2021, July 31). Sentiment Analysis. Text Mining for Information Professionals, 191–211.** https://doi.org/10.1007/978-3-030-85085-2_7')
+
+            with tab4:
+                st.subheader(':blue[Sentiment Analysis]', anchor=False)
+                st.write("Click the three dots at the top right then select the desired format")
+                st.markdown("![Downloading visualization](https://raw.githubusercontent.com/faizhalas/library-tools/main/images/download_sentiment.png)")
+                st.divider()
+                st.subheader(':blue[CSV Results]', anchor=False)
+                st.text("Click Download button")
+                st.markdown("![Downloading results](https://raw.githubusercontent.com/faizhalas/library-tools/main/images/sentitable.png)")
+
+    
+    except Exception as e:
+        st.write(e)
+        st.error("Please ensure that your file is correct. Please contact us if you find that this is an error.", icon="🚨")
+        st.stop()

pages/8 Shifterator.py

@@ -0,0 +1,524 @@
+import streamlit as st
+import streamlit.components.v1 as components
+import shifterator as sh
+from shifterator import ProportionShift
+import pandas as pd
+import re
+import nltk
+nltk.download('wordnet')
+from nltk.stem import WordNetLemmatizer
+nltk.download('stopwords')
+from nltk.corpus import stopwords
+import time
+import sys
+import json
+from tools import sourceformat as sf
+from collections import Counter
+import io
+
+#===config===
+st.set_page_config(
+    page_title="Coconut",
+    page_icon="🥥",
+    layout="wide",
+    initial_sidebar_state="collapsed"
+)
+
+hide_streamlit_style = """
+            <style>
+            #MainMenu 
+            {visibility: hidden;}
+            footer {visibility: hidden;}
+            [data-testid="collapsedControl"] {display: none}
+            </style>
+            """
+st.markdown(hide_streamlit_style, unsafe_allow_html=True)
+
+with st.popover("🔗 Menu"):
+    st.page_link("https://www.coconut-libtool.com/", label="Home", icon="🏠")
+    st.page_link("pages/1 Scattertext.py", label="Scattertext", icon="1️⃣")
+    st.page_link("pages/2 Topic Modeling.py", label="Topic Modeling", icon="2️⃣")
+    st.page_link("pages/3 Bidirected Network.py", label="Bidirected Network", icon="3️⃣")
+    st.page_link("pages/4 Sunburst.py", label="Sunburst", icon="4️⃣")
+    st.page_link("pages/5 Burst Detection.py", label="Burst Detection", icon="5️⃣")
+    st.page_link("pages/6 Keywords Stem.py", label="Keywords Stem", icon="6️⃣")
+    st.page_link("pages/7 Sentiment Analysis.py", label="Sentiment Analysis", icon="7️⃣")
+    st.page_link("pages/8 Shifterator.py", label="Shifterator", icon="8️⃣")
+    
+st.header("Shifterator", anchor=False)
+st.subheader('Put your file here...', anchor=False)
+
+def reset_all():
+    st.cache_data.clear()
+
+@st.cache_data(ttl=3600)
+def get_ext(extype):
+    extype = uploaded_file.name
+    return extype
+
+#===upload file===
+@st.cache_data(ttl=3600)
+def upload(extype):
+    papers = pd.read_csv(uploaded_file)
+    #lens.org
+    if 'Publication Year' in papers.columns:
+               papers.rename(columns={'Publication Year': 'Year', 'Citing Works Count': 'Cited by',
+                                     'Publication Type': 'Document Type', 'Source Title': 'Source title'}, inplace=True)
+    
+    if "dimensions" in uploaded_file.name.lower():
+        papers = sf.dim(papers)
+        col_dict = {'MeSH terms': 'Keywords',
+        'PubYear': 'Year',
+        'Times cited': 'Cited by',
+        'Publication Type': 'Document Type'
+        }
+        papers.rename(columns=col_dict, inplace=True)
+    
+    return papers
+
+@st.cache_data(ttl=3600)
+def conv_txt(extype):
+    if("pmc" in uploaded_file.name.lower() or "pubmed" in uploaded_file.name.lower()):
+        file = uploaded_file
+        papers = sf.medline(file)
+
+    elif("hathi" in uploaded_file.name.lower()):
+        papers = pd.read_csv(uploaded_file,sep = '\t')
+        papers = sf.htrc(papers)
+        col_dict={'title': 'title',
+        'rights_date_used': 'Year',
+        }
+        papers.rename(columns=col_dict, inplace=True)
+        
+    else:
+        col_dict = {'TI': 'Title',
+                'SO': 'Source title',
+                'DE': 'Author Keywords',
+                'DT': 'Document Type',
+                'AB': 'Abstract',
+                'TC': 'Cited by',
+                'PY': 'Year',
+                'ID': 'Keywords Plus'}
+        papers = pd.read_csv(uploaded_file, sep='\t', lineterminator='\r')
+        papers.rename(columns=col_dict, inplace=True)
+    print(papers)
+    return papers
+
+@st.cache_data(ttl=3600)
+def conv_json(extype):
+    col_dict={'title': 'title',
+    'rights_date_used': 'Year',
+    }
+
+    data = json.load(uploaded_file)
+    hathifile = data['gathers']
+    keywords = pd.DataFrame.from_records(hathifile)
+
+    keywords = sf.htrc(keywords)
+    keywords.rename(columns=col_dict,inplace=True)
+    return keywords
+
+@st.cache_data(ttl=3600)
+def conv_pub(extype):
+    if (get_ext(extype)).endswith('.tar.gz'):
+        bytedata = extype.read()
+        keywords = sf.readPub(bytedata)
+    elif (get_ext(extype)).endswith('.xml'):
+        bytedata = extype.read()
+        keywords = sf.readxml(bytedata)
+    return keywords
+
+@st.cache_data(ttl=3600)
+def get_data(extype): 
+    df_col = sorted(papers.select_dtypes(include=['object']).columns.tolist())
+    list_title = [col for col in df_col if col.lower() == "title"]
+    abstract_pattern = re.compile(r'abstract', re.IGNORECASE)
+    list_abstract = [col for col in df_col if abstract_pattern.search(col)]
+
+    if all(col in df_col for col in list_title) and all(col in df_col for col in list_abstract):
+        selected_cols = list_abstract + list_title
+    elif all(col in df_col for col in list_title):
+        selected_cols = list_title
+    elif all(col in df_col for col in list_abstract):
+        selected_cols = list_abstract
+    else:
+        selected_cols = df_col
+
+    if not selected_cols:
+        selected_cols = df_col
+    
+    return df_col, selected_cols
+
+@st.cache_data(ttl=3600)
+def check_comparison(extype):
+    comparison = ['Word-to-word', 'Manual label']
+    
+    if any('year' in col.lower() for col in papers.columns):
+        comparison.append('Years')
+    if any('source title' in col.lower() for col in papers.columns):
+        comparison.append('Sources')
+
+    comparison.sort(reverse=False)
+    return comparison
+
+#===clean csv===
+@st.cache_data(ttl=3600, show_spinner=False)
+def clean_csv(extype):
+    paper = papers.dropna(subset=[ColCho])
+                 
+    #===mapping===
+    paper[ColCho] = paper[ColCho].map(lambda x: x.lower())
+    if rem_punc:
+        paper[ColCho] = paper[ColCho].map(lambda x: re.sub('[,:;\.!-?•=]', ' ', x))
+        paper[ColCho] = paper[ColCho].str.replace('\u201c|\u201d', '', regex=True) 
+    if rem_copyright:
+        paper[ColCho] = paper[ColCho].map(lambda x: re.sub('©.*', '', x))
+        
+    #===stopword removal===
+    stop = stopwords.words('english')
+    paper[ColCho] = paper[ColCho].apply(lambda x: ' '.join([word for word in x.split() if word not in stop]))
+          
+    #===lemmatize===
+    lemmatizer = WordNetLemmatizer()
+    
+    @st.cache_data(ttl=3600)
+    def lemmatize_words(text):
+        words = text.split()
+        words = [lemmatizer.lemmatize(word) for word in words]
+        return ' '.join(words)
+        
+    paper[ColCho] = paper[ColCho].apply(lemmatize_words)
+    
+    words_rmv = [word.strip() for word in words_to_remove.split(";")]
+    remove_set = set(words_rmv)
+    
+    @st.cache_data(ttl=3600)
+    def remove_words(text):
+        words = text.split()  
+        cleaned_words = [word for word in words if word not in remove_set]
+        return ' '.join(cleaned_words) 
+        
+    paper[ColCho] = paper[ColCho].apply(remove_words)
+         
+    return paper
+
+@st.cache_data(ttl=3600)
+def get_minmax(extype):
+    MIN = int(papers['Year'].min())
+    MAX = int(papers['Year'].max())
+    GAP = MAX - MIN
+    MID = round((MIN + MAX) / 2)
+    return MIN, MAX, GAP, MID
+
+@st.cache_data(ttl=3600)
+def running_shifterator(dict1, dict2):
+    try:
+        if method_shifts == 'Proportion Shifts':
+            proportion_shift = sh.ProportionShift(type2freq_1=dict1, type2freq_2=dict2)
+            ax = proportion_shift.get_shift_graph(system_names = ['Topic 1', 'Topic 2'], title='Proportion Shifts')     
+            
+        elif method_shifts == 'Shannon Entropy Shifts':
+            entropy_shift = sh.EntropyShift(type2freq_1=dict1,
+                                type2freq_2=dict2,
+                                base=2)
+            ax = entropy_shift.get_shift_graph(system_names = ['Topic 1', 'Topic 2'], title='Shannon Entropy Shifts')     
+            
+        elif method_shifts == 'Tsallis Entropy Shifts':
+            entropy_shift = sh.EntropyShift(type2freq_1=dict1,
+                                type2freq_2=dict2,
+                                base=2,
+                                alpha=0.8)
+            ax = entropy_shift.get_shift_graph(system_names = ['Topic 1', 'Topic 2'], title='Tsallis Entropy Shifts')     
+            
+        elif method_shifts == 'Kullback-Leibler Divergence Shifts':
+            kld_shift = sh.KLDivergenceShift(type2freq_1=dict1,
+                                 type2freq_2=dict2,
+                                 base=2)
+            ax = kld_shift.get_shift_graph(system_names = ['Topic 1', 'Topic 2'], title='Kullback-Leibler Divergence Shifts')     
+            
+        elif method_shifts == 'Jensen-Shannon Divergence Shifts':
+            jsd_shift = sh.JSDivergenceShift(type2freq_1=dict1,
+                                 type2freq_2=dict2,
+                                 weight_1=0.5,
+                                 weight_2=0.5,
+                                 base=2,
+                                 alpha=1)
+            ax = jsd_shift.get_shift_graph(system_names = ['Topic 1', 'Topic 2'], title='Jensen-Shannon Divergence Shifts')     
+
+        fig = ax.get_figure()
+        
+        buf = io.BytesIO()
+        fig.savefig(buf, format="png", bbox_inches='tight')
+        buf.seek(0)
+        
+        return fig, buf
+
+    except ValueError:
+        st.warning('Please check your data.', icon="⚠️")
+        sys.exit()
+
+@st.cache_data(ttl=3600)
+def df2dict(df_1, df_2):
+    text1 = ' '.join(df_1.dropna().astype(str))
+    text2 = ' '.join(df_2.dropna().astype(str))
+                
+    text1_clean = re.sub(r'\d+', '', text1)
+    text2_clean = re.sub(r'\d+', '', text2)
+                
+    tokens1 = re.findall(r'\b\w+\b', text1_clean.lower())
+    tokens2 = re.findall(r'\b\w+\b', text2_clean.lower())
+                
+    type2freq_1 = {k: int(v) for k, v in Counter(tokens1).items()}
+    type2freq_2 = {k: int(v) for k, v in Counter(tokens2).items()}
+
+    return type2freq_1, type2freq_2
+
+@st.cache_data(ttl=3600)
+def dict_w2w(search_terms1, search_terms2):
+    selected_col = [ColCho]
+    dfs1 = pd.DataFrame()
+    for term in search_terms1:
+        dfs1 = pd.concat([dfs1, paper[paper[selected_col[0]].str.contains(r'\b' + term + r'\b', case=False, na=False)]], ignore_index=True)
+    dfs1['Topic'] = 'First Term'
+    dfs1 = dfs1.drop_duplicates()
+        
+    dfs2 = pd.DataFrame()
+    for term in search_terms2:
+        dfs2 = pd.concat([dfs2, paper[paper[selected_col[0]].str.contains(r'\b' + term + r'\b', case=False, na=False)]], ignore_index=True)
+    dfs2['Topic'] = 'Second Term'
+    dfs2 = dfs2.drop_duplicates()
+    
+    type2freq_1, type2freq_2 = df2dict(dfs1[selected_col[0]], dfs2[selected_col[0]])
+    
+    return type2freq_1, type2freq_2
+
+@st.cache_data(ttl=3600)
+def dict_sources(stitle1, stitle2):
+    selected_col = [ColCho]
+    dfs1 = paper[paper['Source title'].str.contains(stitle1, case=False, na=False)]
+    dfs1['Topic'] = stitle1
+    dfs2 = paper[paper['Source title'].str.contains(stitle2, case=False, na=False)]
+    dfs2['Topic'] = stitle2
+
+    type2freq_1, type2freq_2 = df2dict(dfs1[selected_col[0]], dfs2[selected_col[0]])
+    
+    return type2freq_1, type2freq_2
+
+@st.cache_data(ttl=3600)
+def dict_years(first_range, second_range):
+    selected_col = [ColCho]
+    first_filter_df = paper[(paper['Year'] >= first_range[0]) & (paper['Year'] <= first_range[1])].copy()
+    first_filter_df['Topic Range'] = 'First range'
+        
+    second_filter_df = paper[(paper['Year'] >= second_range[0]) & (paper['Year'] <= second_range[1])].copy()
+    second_filter_df['Topic Range'] = 'Second range'
+
+    type2freq_1, type2freq_2 = df2dict(first_filter_df[selected_col[0]], second_filter_df[selected_col[0]])
+    
+    return type2freq_1, type2freq_2
+    
+
+#===Read data===
+uploaded_file = st.file_uploader('', type=['csv', 'txt', 'json', 'tar.gz','xml'], on_change=reset_all)
+
+if uploaded_file is not None:
+    try:
+        extype = get_ext(uploaded_file)
+    
+        if extype.endswith('.csv'):
+             papers = upload(extype) 
+        elif extype.endswith('.txt'):
+             papers = conv_txt(extype)
+        elif extype.endswith('.json'):
+            papers = conv_json(extype)
+        elif extype.endswith('.tar.gz') or extype.endswith('.xml'):
+            papers = conv_pub(uploaded_file)
+    
+        df_col, selected_cols = get_data(extype)
+        comparison = check_comparison(extype)
+    
+        #Menu
+        c1, c2, c3 = st.columns([4,0.1,4])
+        ColCho = c1.selectbox(
+                'Choose column to analyze',
+                (selected_cols), on_change=reset_all)
+    
+        c2.write('')
+    
+        compare = c3.selectbox(
+                'Type of comparison',
+                (comparison), on_change=reset_all)
+        
+        with st.expander("🧮 Show advance settings"):
+            y1, y2, y3 = st.columns([4,0.1,4])
+            t1, t2 = st.columns([3,3])
+            words_to_remove = y1.text_input('Input your text', on_change=reset_all, placeholder='Remove specific words. Separate words by semicolons (;)')
+            method_shifts = y3.selectbox("Choose preferred method",('Proportion Shifts','Shannon Entropy Shifts', 'Tsallis Entropy Shifts','Kullback-Leibler Divergence Shifts', 
+                                                                   'Jensen-Shannon Divergence Shifts'), on_change=reset_all)
+            rem_copyright = t1.toggle('Remove copyright statement', value=True, on_change=reset_all)
+            rem_punc = t2.toggle('Remove punctuation', value=False, on_change=reset_all)
+    
+        if method_shifts == 'Kullback-Leibler Divergence Shifts':
+            st.info('The Kullback-Leibler Divergence is only well-defined if every single word in the comparison text is also in the reference text.', icon="ℹ️")
+        
+        paper = clean_csv(extype)
+    
+        tab1, tab2, tab3, tab4 = st.tabs(["📈 Generate visualization", "📃 Reference", "📓 Recommended Reading", "⬇️ Download Help"])
+    
+        with tab1:
+             #===visualization===
+            if compare == 'Word-to-word':
+                col1, col2, col3 = st.columns([4,0.1,4])
+                text1 = col1.text_input('First Term', on_change=reset_all, placeholder='put comma if you have more than one')
+                search_terms1 = [term.strip() for term in text1.split(",") if term.strip()]
+                col2.write('')
+                text2 = col3.text_input('Second Term', on_change=reset_all, placeholder='put comma if you have more than one')
+                search_terms2 = [term.strip() for term in text2.split(",") if term.strip()]
+                
+                type2freq_1, type2freq_2 = dict_w2w(search_terms1, search_terms2)
+        
+                if not type2freq_1 and not type2freq_2:
+                    st.warning('We cannot find anything in your document.', icon="⚠️")
+                elif not type2freq_1:
+                    st.warning(f'We cannot find {text1} in your document.', icon="⚠️")
+                elif not type2freq_2:
+                    st.warning(f'We cannot find {text2} in your document.', icon="⚠️")
+                else:
+                    with st.spinner('Processing. Please wait until the visualization comes up'):
+                        fig, buf = running_shifterator(type2freq_1, type2freq_2)
+                        st.pyplot(fig)
+        
+            elif compare == 'Manual label':
+                col1, col2, col3 = st.columns(3)
+        
+                df_col_sel = sorted([col for col in paper.columns.tolist()])
+                     
+                column_selected = col1.selectbox(
+                    'Choose column',
+                    (df_col_sel), on_change=reset_all)
+        
+                list_words = paper[column_selected].values.tolist()
+                list_unique = sorted(list(set(list_words)))
+                
+                if column_selected is not None:
+                    label1 = col2.selectbox(
+                        'Choose first label',
+                        (list_unique), on_change=reset_all)
+        
+                    default_index = 0 if len(list_unique) == 1 else 1
+                    label2 = col3.selectbox(
+                        'Choose second label',
+                        (list_unique), on_change=reset_all, index=default_index)
+        
+                filtered_df = paper[paper[column_selected].isin([label1, label2])].reset_index(drop=True)
+                
+                dfs1 = filtered_df[filtered_df[column_selected] == label1].reset_index(drop=True)
+                dfs2 = filtered_df[filtered_df[column_selected] == label2].reset_index(drop=True)
+
+                type2freq_1, type2freq_2 = df2dict(dfs1[ColCho], dfs2[ColCho])
+                
+                with st.spinner('Processing. Please wait until the visualization comes up'):
+                    fig, buf = running_shifterator(type2freq_1, type2freq_2)
+                    st.pyplot(fig)
+        
+            elif compare == 'Sources':
+                col1, col2, col3 = st.columns([4,0.1,4])
+        
+                unique_stitle = set()
+                unique_stitle.update(paper['Source title'].dropna())
+                list_stitle = sorted(list(unique_stitle))
+                     
+                stitle1 = col1.selectbox(
+                    'Choose first label',
+                    (list_stitle), on_change=reset_all)
+                col2.write('')
+                default_index = 0 if len(list_stitle) == 1 else 1
+                stitle2 = col3.selectbox(
+                    'Choose second label',
+                    (list_stitle), on_change=reset_all, index=default_index)
+        
+                type2freq_1, type2freq_2 = dict_sources(stitle1, stitle2)
+        
+                with st.spinner('Processing. Please wait until the visualization comes up'):
+                    fig, buf = running_shifterator(type2freq_1, type2freq_2)
+                    st.pyplot(fig)
+        
+            elif compare == 'Years':
+                col1, col2, col3 = st.columns([4,0.1,4])
+                
+                MIN, MAX, GAP, MID = get_minmax(extype)
+                if (GAP != 0):
+                    first_range = col1.slider('First Range', min_value=MIN, max_value=MAX, value=(MIN, MID), on_change=reset_all)
+                    col2.write('')
+                    second_range = col3.slider('Second Range', min_value=MIN, max_value=MAX, value=(MID, MAX), on_change=reset_all)
+                
+                    type2freq_1, type2freq_2 = dict_years(first_range, second_range)
+        
+                    with st.spinner('Processing. Please wait until the visualization comes up'):
+                        fig, buf = running_shifterator(type2freq_1, type2freq_2)
+                        st.pyplot(fig)
+
+                else:
+                    st.write('You only have data in ', (MAX))
+
+            d1, d2 = st.columns(2)
+                
+            d1.download_button(
+                label="📥 Download Graph",
+                data=buf,
+                file_name="shifterator.png",
+                mime="image/png"
+            )
+
+            @st.cache_data(ttl=3600)
+            def shifts_dfs(type2freq_1, type2freq_2):
+                proportion_shift = ProportionShift(type2freq_1=type2freq_1, type2freq_2=type2freq_2)
+                
+                words = list(proportion_shift.types)
+                shift_scores = proportion_shift.get_shift_scores()
+                freq1 = proportion_shift.type2freq_1
+                freq2 = proportion_shift.type2freq_2
+
+                data = []
+                for word, score in shift_scores.items():
+                    data.append({
+                        'word': word,
+                        'freq_text1': proportion_shift.type2freq_1.get(word, 0),
+                        'freq_text2': proportion_shift.type2freq_2.get(word, 0),
+                        'shift_score': score
+                    })
+                
+                df_shift = pd.DataFrame(data)
+                df_shift = df_shift.sort_values('shift_score')
+                
+                return df_shift.to_csv(index=False).encode('utf-8')
+
+            csv = shifts_dfs(type2freq_1, type2freq_2)
+
+            d2.download_button(
+                "📥 Click to download result",
+                csv,
+                "shiftertor_dataframe.csv",
+                "text/csv")
+    
+        with tab2:
+            st.markdown('**Gallagher, R.J., Frank, M.R., Mitchell, L. et al. (2021). Generalized Word Shift Graphs: A Method for Visualizing and Explaining Pairwise Comparisons Between Texts. EPJ Data Science, 10(4).** https://doi.org/10.1140/epjds/s13688-021-00260-3')
+    
+        with tab3:
+            st.markdown('**Sánchez-Franco, M. J., & Rey-Tienda, S. (2023). The role of user-generated content in tourism decision-making: an exemplary study of Andalusia, Spain. Management Decision, 62(7).** https://doi.org/10.1108/md-06-2023-0966')
+            st.markdown('**Ipek Baris Schlicht, Fernandez, E., Chulvi, B., & Rosso, P. (2023). Automatic detection of health misinformation: a systematic review. Journal of Ambient Intelligence and Humanized Computing, 15.** https://doi.org/10.1007/s12652-023-04619-4')
+            st.markdown('**Torricelli, M., Falkenberg, M., Galeazzi, A., Zollo, F., Quattrociocchi, W., & Baronchelli, A. (2023). Hurricanes Increase Climate Change Conversations on Twitter. PLOS Climate, 2(11)** https://doi.org/10.1371/journal.pclm.0000277')
+
+        with tab4:
+            st.subheader(':blue[Result]', anchor=False)
+            st.button('📥 Download Graph')
+            st.text("Click Download Graph button.")  
+
+            st.divider()
+            st.subheader(':blue[Shifterator Dataframe]', anchor=False)
+            st.button('📥 Click to download result')
+            st.text("Click the Download button to get the CSV result.") 
+
+    except Exception as e:
+        st.error("Please ensure that your file is correct. Please contact us if you find that this is an error.", icon="🚨")
+        st.stop()

pages/9 Summarization.py

@@ -0,0 +1,304 @@
+import streamlit as st
+import streamlit.components.v1 as components
+import nltk
+import spacy
+import pytextrank
+import pandas as pd
+from rouge_score import rouge_scorer
+from nltk.translate.bleu_score import sentence_bleu
+from transformers import pipeline, PegasusForConditionalGeneration, PegasusTokenizer, T5ForConditionalGeneration, T5Tokenizer
+nltk.download('punkt')
+
+#===config===
+st.set_page_config(
+    page_title="Coconut",
+    page_icon="🥥",
+    layout="wide",
+    initial_sidebar_state="collapsed"
+)
+
+hide_streamlit_style = """
+            <style>
+            #MainMenu 
+            {visibility: hidden;}
+            footer {visibility: hidden;}
+            [data-testid="collapsedControl"] {display: none}
+            </style>
+            """
+st.markdown(hide_streamlit_style, unsafe_allow_html=True)
+
+with st.popover("🔗 Menu"):
+    st.page_link("https://www.coconut-libtool.com/", label="Home", icon="🏠")
+    st.page_link("pages/1 Scattertext.py", label="Scattertext", icon="1️⃣")
+    st.page_link("pages/2 Topic Modeling.py", label="Topic Modeling", icon="2️⃣")
+    st.page_link("pages/3 Bidirected Network.py", label="Bidirected Network", icon="3️⃣")
+    st.page_link("pages/4 Sunburst.py", label="Sunburst", icon="4️⃣")
+    st.page_link("pages/5 Burst Detection.py", label="Burst Detection", icon="5️⃣")
+    st.page_link("pages/6 Keywords Stem.py", label="Keywords Stem", icon="6️⃣")
+    st.page_link("pages/7 Sentiment Analysis.py", label="Sentiment Analysis", icon="7️⃣")
+    st.page_link("pages/8 Shifterator.py", label="Shifterator", icon="8️⃣")
+    
+st.header("Summarization test", anchor=False)
+st.subheader('Put your file here...', anchor=False)
+
+#========unique id========
+@st.cache_resource(ttl=3600)
+def create_list():
+    l = [1, 2, 3]
+    return l
+
+l = create_list()
+first_list_value = l[0]
+l[0] = first_list_value + 1
+uID = str(l[0])
+
+@st.cache_data(ttl=3600)
+def get_ext(uploaded_file):
+    extype = uID+uploaded_file.name
+    return extype
+
+#===clear cache===
+def reset_all():
+    st.cache_data.clear()
+
+#===text reading===
+def read_txt(intext):
+    return (intext.read()).decode()
+
+#===csv reading===
+def read_csv(uploaded_file):
+    fulltexts = pd.read_csv(uploaded_file)
+    fulltexts.rename(columns={fulltexts.columns[0]: "texts"}, inplace = True)
+    return fulltexts
+    
+
+#===Read data===
+uploaded_file = st.file_uploader('', type=['txt','csv'], on_change=reset_all)
+
+
+if uploaded_file is not None:
+    try:
+        extype = get_ext(uploaded_file)
+
+        if extype.endswith(".txt"):
+            fulltext = read_txt(uploaded_file)
+        elif extype.endswith(".csv"):
+            texts = read_csv(uploaded_file)
+
+        #Menu
+        
+        method = st.selectbox("Method",("Extractive","Abstractive"))
+        if method == "Abstractive":
+            ab_method = st.selectbox("Abstractive method", ("Pegasus x-sum","FalconsAI t5"))
+            min_length = st.number_input("Minimum length", min_value = 0)
+            max_length = st.number_input("Maximum length", min_value = 1)        
+
+        if method == "Extractive":
+            ex_method = st.selectbox("Extractive method", ("t5","Spacy PyTextRank"))
+            if ex_method == "Spacy PyTextRank":
+                phrase_limit = st.number_input("Phrase length limit", min_value = 0)
+                sentence_limit = st.number_input("Sentence limit", min_value = 0)
+            elif ex_method == "t5" or ex_method == "FalconsAI t5":
+                min_length = st.number_input("Minimum length", min_value = 0)
+                max_length = st.number_input("Maximum length", min_value = 1)                  
+
+        
+
+        if st.button("Submit", on_click=reset_all):
+        
+            tab1, tab2, tab3 = st.tabs(["📈 Generate visualization", "📃 Reference", "⬇️ Download Help"])
+                
+            with tab1:
+                
+                def SpacyRank(text):
+                    nlp = spacy.load("en_core_web_lg")
+                    nlp.add_pipe("textrank")
+                    doc = nlp(text)
+                    summary = ""
+                    for sent in doc._.textrank.summary(limit_phrases = phrase_limit, limit_sentences = sentence_limit):
+                        summary+=str(sent) + '\n'
+                    return summary
+
+                def t5summ(text):
+                    model = T5ForConditionalGeneration.from_pretrained('t5-small')
+                    tokenizer = T5Tokenizer.from_pretrained('t5-small')
+                    
+                    input_text = "summarize: " + text
+                    input_ids = tokenizer.encode(input_text,return_tensors='pt')
+                    
+                    summed = model.generate(input_ids, max_length = max_length, min_length = min_length)
+
+                    summary = tokenizer.decode(summed[0],skip_special_tokens=True)     
+                    return summary       
+
+                def xsum(text):
+                    model_name = "google/pegasus-xsum"
+                    
+                    pegasus_tokenizer = PegasusTokenizer.from_pretrained(model_name)
+
+                    summarizer = pipeline("summarization", 
+                    model=model_name, 
+                    tokenizer=pegasus_tokenizer, 
+                    framework="pt")
+                    
+                    summed = summarizer(text, min_length = min_length, max_length = max_length)
+                    summary = summed[0]["summary_text"]   
+
+                    return summary                     
+
+                def falcsum(text):
+                    summarizer = pipeline("summarization",model = "Falconsai/text_summarization")
+                    summed = summarizer(text, max_length = max_length, min_length = min_length, do_sample = False)                    
+                    summary = summed[0]["summary_text"]
+                    return summary
+
+                def bulkScore(combined):
+                    
+                    scorelist = []
+
+                    for column in range(len(combined)):
+                        ref = combined[column][0]
+                        cand = combined[column][1]
+                    
+                        BLEuscore = nltk.translate.bleu_score.sentence_bleu([ref], cand)
+                        scorer = rouge_scorer.RougeScorer(['rouge1', 'rougeL'], use_stemmer=True)
+                        rougescores = scorer.score(ref, cand)
+
+                        Bscore = f"{BLEuscore:.2f}"
+                        Rscore = f"{rougescores['rouge1'].fmeasure:.2f}"
+
+                        scoreTuplet = Bscore, Rscore
+
+                        scorelist.append(scoreTuplet)
+
+                    return scorelist
+
+
+                with st.spinner('Performing computations. Please wait ...'):
+                
+                    c1, c2 = st.columns([0.5,0.5], border=True)
+                
+                    if(extype.endswith(".txt")):
+
+                        with c1:
+                            if(extype.endswith(".txt")):
+                                st.header("Original text")
+                                with st.container(border=True):
+                                    st.write(fulltext)
+
+                            if method == "Extractive":
+                                if(ex_method == "Spacy PyTextRank"):
+                                    summary = SpacyRank(fulltext)
+                                elif(ex_method == "t5"):
+                                    summary = t5summ(fulltext)
+
+                            elif method == "Abstractive":
+                                if ab_method == "Pegasus x-sum":
+                                    summary = xsum(fulltext)
+
+                                elif ab_method == "FalconsAI t5":
+                                    summary = t5summ(fulltext)
+                        with c2:
+                            
+                            st.header("Summarized")
+                            with st.container(border = True):
+                                st.write(summary)
+                            st.header("Performance scores")
+                            with st.container(border = True):
+                                
+                                #performance metrics
+                                reference = fulltext
+                                candidate = summary      
+
+                                BLEuscore = nltk.translate.bleu_score.sentence_bleu([reference], candidate)
+
+                                scorer = rouge_scorer.RougeScorer(['rouge1', 'rougeL'], use_stemmer=True)
+                                rougescores = scorer.score(reference, candidate)
+
+                                st.write(f"BLEU Score (NLTK): {BLEuscore:.2f}")
+                                st.write(f"ROUGE-1 F1 Score: {rougescores['rouge1'].fmeasure:.2f}")
+
+                                text_file = summary
+                                st.download_button(
+                                    label = "Download Results",
+                                    data=text_file,
+                                    file_name="Summary.txt",
+                                    mime="text\csv",
+                                    on_click="ignore",)
+
+                    elif(extype.endswith(".csv")):
+                        if method == "Extractive":
+                            if(ex_method == "Spacy PyTextRank"):
+                                summaries = texts['texts'].apply(SpacyRank)
+                                fullnsums = summaries.to_frame()
+                                fullnsums['full'] = texts['texts']
+                                fullnsums['combined'] = fullnsums.values.tolist()
+
+
+                            elif(ex_method == "t5"):
+                                summaries = texts['texts'].apply(t5summ)
+                                fullnsums = summaries.to_frame()
+                                fullnsums['full'] = texts['texts']
+                                fullnsums['combined'] = fullnsums.values.tolist()
+                                
+
+                        elif method == "Abstractive":
+                            if ab_method == "Pegasus x-sum":
+                                summaries = texts['texts'].apply(xsum)
+                                fullnsums = summaries.to_frame()
+                                fullnsums['full'] = texts['texts']
+                                fullnsums['combined'] = fullnsums.values.tolist()
+
+                            elif ab_method == "FalconsAI t5":
+                                summaries = texts['texts'].apply(falcsum)
+                                fullnsums = summaries.to_frame()
+                                fullnsums['full'] = texts['texts']
+                                fullnsums['combined'] = fullnsums.values.tolist()
+
+                        with c1:
+                            st.header("Download bulk summarization results")
+
+                            result = summaries.to_csv()
+                            st.download_button(
+                                label = "Download Results",
+                                data = result,
+                                file_name = "Summaries.csv",
+                                mime="text\csv",
+                                on_click = "ignore"
+                            )
+
+                        with c2:
+                            st.header("Scores and summaries results")
+                            scores  = pd.DataFrame.from_records(bulkScore(fullnsums.combined.to_list()),columns = ["BLEU","Rouge"])
+                        
+                            summariesscores = fullnsums.join(scores)
+
+                            summariesscores.drop("combined", axis = 1, inplace = True)
+                            summariesscores.rename(columns = {"texts":"summarized"}, inplace = True)
+
+                            result2 = summariesscores.to_csv()
+
+                            st.download_button(
+                                label = "Download scores and results",
+                                data = result2,
+                                file_name = "ScoredSummaries.csv",
+                                mime = "test\csv",
+                                on_click = "ignore"
+                            )
+
+
+                            
+
+
+            #do this
+            with tab2:
+                st.write("")
+
+            with tab3:
+                st.header("Summarization result (.txt)")
+                st.write("Click the download button (example) to get the text file result")
+                st.button(label = "Download Results")
+
+
+    except Exception as e:
+        st.write(e)

tools/sourceformat.py

@@ -0,0 +1,328 @@
+from io import StringIO, BytesIO
+import pymarc
+import requests
+import string
+import pandas as pd
+import tarfile
+try:
+    from lxml import etree as ET
+except ImportError:
+    import xml.etree.ElementTree as ET
+
+#metadata for htrc worksets
+def htrc(self):
+    
+    #variables/arrays and stuff
+    
+    #string of keywords per volume/htid
+    keywords = ""
+    
+    #array of all the keywords per each volume/htid, to add to the file
+    keylist = []
+
+    #get htids of the volumes
+    htids = self['htid'].values.tolist()
+    #iterate through list of htids
+    for id in range(len(htids)):
+        htid = htids[id]
+        
+        #api call for the extra metadata using htid
+        extradata = requests.get("https://catalog.hathitrust.org/api/volumes/full/htid/"+htid+".json")
+        
+        #turn the request into a json file
+        extradata = extradata.json()
+
+        #get record id and use it to get the xml/marc file with the actual metadata
+        recid = extradata['items'][0]['fromRecord']
+        xmlmarc = extradata['records'][recid]['marc-xml']
+
+        #turn the formatted xml into an actual pymarc
+        xml = StringIO(xmlmarc)
+        marc = pymarc.parse_xml_to_array(xml)[0]
+        xml.close()
+
+        for term in marc.get_fields('650'):
+            if "http" in (term.value()).lower():
+                keywords+= ""
+            elif "ocolc" in (term.value()).lower():
+                keywords+=""
+            else:
+                keywords+=term.value().translate(str.maketrans('','', string.punctuation))+"; "
+        keylist.append(keywords)
+    self['Keywords'] = keylist
+    return self
+
+def htrcxtra(self):
+        
+    #variables/arrays and stuff
+    
+    #string of keywords per volume/htid
+    pages = ""
+    
+    #array of all the keywords per each volume/htid, to add to the file
+    pagecount = []
+
+    #get htids of the volumes
+    htids = self['htid'].values.tolist()
+    #iterate through list of htids
+    for id in range(len(htids)):
+        htid = htids[id]
+        
+        #api call for the extra metadata using htid
+        extradata = requests.get("https://catalog.hathitrust.org/api/volumes/full/htid/"+htid+".json")
+        
+        #turn the request into a json file
+        extradata = extradata.json()
+
+        #get record id and use it to get the xml/marc file with the actual metadata
+        recid = extradata['items'][0]['fromRecord']
+        xmlmarc = extradata['records'][recid]['marc-xml']
+
+        #turn the formatted xml into an actual pymarc
+        xml = StringIO(xmlmarc)
+        marc = pymarc.parse_xml_to_array(xml)[0]
+        xml.close()
+
+    for term in marc.get_fields('350'):
+        pages+=term.value()
+    pagecount.append(pages)
+    self['pages'] = pagecount
+    return self
+
+
+#format files from dimensions
+def dim(file):
+    formatted = file.drop(file.columns[[0]],axis=1)
+
+    done = pd.read_csv(StringIO((formatted.to_csv(header=False,index=False))))
+  
+    return done
+
+
+
+def readPub(tar):
+
+    #list to put xmls from tarfile in
+    xmllist = []
+
+    readfile = BytesIO(tar)
+
+    #get the files from the tarfile into the list
+    files = tarfile.open(fileobj=readfile, mode = 'r:gz', )
+    for member in files.getmembers():
+        singlefile = files.extractfile(member)
+        if singlefile is not None:
+            article = singlefile.read()
+            article = article.decode("utf-8")
+            article = StringIO(article)
+            xmllist.append(article)
+
+    #lists for each data point
+    titles = []
+    years = []
+    keys = []
+    authors = []
+    publishers = []
+    journaltitles = []
+    
+    #go through each xml file in the list
+    for art in range(len(xmllist)):
+
+        #make a parseable element tree out of the xml file
+        tree = ET.parse(xmllist[art])
+        root = tree.getroot()
+
+        #remove parts of the main branch that do not have metadata that we care about
+        for child in list(root):
+            if(child.tag!="front"):
+                root.remove(child)
+
+        #names to concatnate for each article
+        firstname = []
+        lastname = []
+
+        #individual strings for multiple keywords/titles
+        key = ""
+        title = ""
+        
+
+        for target in root.iter('article-title'):
+            if target.text is not None:
+                title += target.text + ", "
+            else:
+                title += " "
+        for target in root.iter('kwd'):
+            if target.text is not None:
+                key+=target.text+ "; "
+            else:
+                key += " "
+        for target in root.iter('year'):
+            year=int(target.text)
+            years.append(year)
+        for names in root.iter('given-names'):
+            firstname.append(names.text)
+        for names in root.iter('surname'):
+            lastname.append(names.text)
+        for target in root.iter('journal-title'):
+            jtitle = target.text
+            journaltitles.append(jtitle)
+        for target in root.iter('publisher-name'):
+            publisher = target.text
+            publishers.append(publisher)
+
+        titles.append(title)
+        keys.append(key)
+
+        fullnames = [first + ' ' + last for first, last in zip(firstname,lastname)]
+
+        #join the names into a single string with authors
+        author = str.join(', ', fullnames)
+
+        authors.append(author)
+
+    data = pd.DataFrame()
+
+    data["Title"] = pd.Series(titles)
+    data["Keywords"] = pd.Series(keys)
+    data["Authors"] = pd.Series(authors)
+    data["Year"] = pd.Series(years)
+    data["Document Type"] = pd.Series(publisher)
+    data["Source title"] = pd.Series(journaltitles)
+
+    data.fillna(value = "empty", inplace = True)
+
+    return data
+
+
+def readxml(file):
+    root = ET.fromstring(file)
+
+
+
+    #remove stuff from the xml that we do not need
+    for child in list(root):
+        for lchild in list(child):
+            if(lchild.tag!="front"):
+                child.remove(lchild)
+
+    #get stuff
+
+    keys = []
+    titles = []
+    authors = []
+    jtitle = []
+    publishers = []
+    years = []
+
+    for child in list(root):
+        for article in list(child):
+            key = ""
+            firstname = []
+            lastname = []
+            for target in article.iter('article-title'):
+                
+                if target.text is not None:
+                    titles.append(target.text)
+                else:
+                    titles.append("empty")
+            for target in article.iter('kwd'):
+                if target.text is not None:
+                    key+= target.text + "; "
+                else:
+                    key += ""
+            keys.append(key)
+            for target in article.iter('given-names'):
+                firstname.append(target.text)
+            for target in article.iter('surname'):
+                lastname.append(target.text)
+            
+            fullnames = [first + ' ' + last for first, last in zip(firstname,lastname)]
+            author = str.join(', ', fullnames)
+            authors.append(author)
+
+            for target in article.iter('journal-title'):
+                jtitle.append(target.text)
+            for target in article.iter('publisher-name'):
+                publishers.append(target.text)
+
+            for target in article.iter('year'):
+                years.append(int(target.text))
+
+    frame = pd.DataFrame()
+
+    frame["Title"] = pd.Series(titles)
+    frame["Keywords"] = pd.Series(keys)
+    frame["Authors"] = pd.Series(authors)
+    frame["Year"] = pd.Series(years)
+    frame["Document Type"] = pd.Series(jtitle)
+    frame["Source title"] = pd.Series(publishers)
+
+    frame.fillna(value = "empty", inplace = True)
+
+    return frame
+
+def medline(file):
+
+    textfile = file.read()
+
+
+    text = textfile.decode()
+
+
+
+
+
+    authors = []
+    titles = []
+    year = []
+    meshkeys = []
+    otherkeys = []
+
+    #articles are separated by newlines so seperate them
+    articles = text.split('\n\n')
+
+    for paper in articles:
+        names = ""
+        meshk = ""
+        otherk = ""         
+        largetext = paper.splitlines()
+        for line in largetext:
+            #title
+            if "TI  - " in line:
+                #checking if the title goes over another line, and to add it if it does
+                startpos = line.index("-") + 2
+                if "- " not in(largetext[largetext.index(line)+1]):
+                    titles.append(line[startpos:] +  " " + largetext[largetext.index(line)+1].strip())
+                else:
+                    titles.append(line[startpos:])
+            #author
+            if "FAU - " in line:
+                startpos = line.index("-") + 2
+                names+= line[startpos:] + "; "
+            #year
+            if "DP  - " in line:
+                startpos = line.index("-") + 2
+                year.append(int(line[startpos:startpos+4]))
+            #key terms
+            if "MH  - " in line:
+                startpos = line.index("-") + 2
+                meshk += line[startpos:] + "; "
+            if"OT  - " in line:
+                startpos = line.index("-") + 2
+                otherk += line[startpos:] + "; "
+    
+        authors.append(names)
+        meshkeys.append(meshk)
+        otherkeys.append(otherk)
+
+    frame = pd.DataFrame()
+    
+    frame['Title'] = pd.Series(titles)
+    frame['Authors'] = pd.Series(authors)
+    frame['Year'] = pd.Series(year)
+    frame['MeSH Keywords'] = pd.Series(meshkeys)
+    frame['Other Keywords'] = pd.Series(otherkeys)
+
+    frame.fillna(value = "empty", inplace = True)
+
+    return frame