Update app.py

app.py

@@ -9,29 +9,14 @@ from hdbscan import HDBSCAN
 from sklearn.feature_extraction.text import CountVectorizer
 
 st.set_page_config(page_title="HF-BERTopic")
-#st.title("HF-BERTopic A front end for BERTopic")
-#st.caption("By Allen Roush")
-#st.caption("github: https://github.com/Hellisotherpeople")
-#st.caption("Linkedin: https://www.linkedin.com/in/allen-roush-27721011b/")
-#st.image("https://raw.githubusercontent.com/MaartenGr/BERTopic/master/images/logo.png", width = 380)
-#st.caption("By Maarten Grootendorst")
-#st.caption("github: https://github.com/MaartenGr/BERTopic")
-#st.caption("Linkedin: https://www.linkedin.com/in/mgrootendorst/")
-#st.image("https://maartengr.github.io/BERTopic/img/algorithm.png")
-
-
 form = st.sidebar.form("Main Settings")
-
 form.header("Main Settings")
-#form.image("https://maartengr.github.io/BERTopic/img/algorithm.png", width = 270)
-
-
 dataset_name = form.text_area("Enter the name of the huggingface dataset to do analysis of:", value = "Hellisotherpeople/DebateSum")
 dataset_name_2 = form.text_area("Enter the name of the config for the dataset if it has one", value = "")
 
 split_name = form.text_area("Enter the name of the split of the dataset that you want to use", value = "train")
 
-number_of_records = form.number_input("Enter the number of documents that you want to analyze from the dataset", value = 200)
+number_of_records = form.number_input("Enter the number of documents that you want to analyze from the dataset", value = 10)
 
 column_name = form.text_area("Enter the name of the column that we are doing analysis on (the X value)", value = "Full-Document")
 
@@ -50,7 +35,6 @@ form.caption("Leave this if you want HDBScan to choose the number of topics (clu
 number_of_topics = form.number_input("Enter the number of topics to use if doing Manual topic reduction", value = 4)
 use_random_seed = form.checkbox("Do you want to make the results reproducible? This significantly slows down BERTopic", value = False)
 
-
 form.header("CounterVectorizer Settings")
 cv_lowercase = form.checkbox("Shall we automatically lowercase the text?", value = True)
 cv_ngram_min = form.number_input("What's the lower boundary of the range of n-values for different word n-grams or char n-grams to be extracted", value = 1)
@@ -64,8 +48,6 @@ cv_min_df = form.number_input("Ignore terms that have a document frequency stric
 form.caption("This parameter represents a proportion of documents if a float is given")
 cv_max_features = form.number_input("Enter the maximum number of n-grams to be featurized", value = 100000)
 
-
-
 form.header("HDBScan Settings")
 hdbscan_min_cluster_size = form.number_input("Enter the number of points necessary to form a new cluster", value = 3)
 form.caption("Set it to the smallest size grouping that you wish to consider a cluster. This is the most impactful setting for HDBscan")
@@ -73,8 +55,6 @@ hdbscan_min_samples = form.number_input("Enter the minimum number of points to b
 form.caption("The larger the value of min_samples you provide, the more conservative the clustering – more points will be declared as noise, and clusters will be restricted to progressively more dense areas.")
 hdbscan_metric = form.text_area("Enter the name of the metric used for computing distances for HDBscan. Common metrics for NLP are euclidean and cosine. Cosine is not supported by HDBscan", value = "euclidean")
 
-
-
 form.header("UMAP Settings")
 umap_n_neighbors = form.number_input("Enter the number of neighbors used by UMAP for generating its manifold", value = 15)
 form.caption("This parameter controls how UMAP balances local versus global structure in the data. It does this by constraining the size of the local neighborhood UMAP will look at when attempting to learn the manifold structure of the data. This means that low values of n_neighbors will force UMAP to concentrate on very local structure (potentially to the detriment of the big picture), while large values will push UMAP to look at larger neighborhoods of each point when estimating the manifold structure of the data, losing fine detail structure for the sake of getting the broader of the data.")
@@ -86,18 +66,8 @@ form.caption("UMAP is used in BERTopic primarily to allow the highly effective c
 umap_metric = form.text_area("Enter the name of the metric used for computing distances. Common metrics for NLP are euclidean and cosine", value = "cosine")
 form.caption("A complete list of all available metrics supported by UMAP can be found here: https://umap-learn.readthedocs.io/en/latest/parameters.html#metric")
 
-
-
-
 form.form_submit_button("Submit")
 
-
-
-
-
-
-
-
 @st.cache
 def load_and_process_data(path, name, streaming, split_name, number_of_records):
     dataset = load_dataset(path = path, name = name, streaming=streaming)
@@ -106,8 +76,6 @@ def load_and_process_data(path, name, streaming, split_name, number_of_records):
     df = pd.DataFrame.from_dict(dataset_head)
     return df
 
-
-
 hdbscan_model = HDBSCAN(min_cluster_size=hdbscan_min_cluster_size, min_samples = hdbscan_min_samples, metric=hdbscan_metric, prediction_data=True)
 if use_random_seed:
     umap_model = UMAP(n_neighbors=umap_n_neighbors, n_components=umap_n_components, min_dist=umap_min_dist, metric=umap_metric, random_state = 42)
@@ -115,8 +83,6 @@ else:
     umap_model = UMAP(n_neighbors=umap_n_neighbors, n_components=umap_n_components, min_dist=umap_min_dist, metric=umap_metric)
 vectorizer_model = CountVectorizer(lowercase = cv_lowercase, ngram_range=(cv_ngram_min, cv_ngram_max), analyzer=cv_analyzer, max_df=cv_max_df, min_df=cv_min_df, stop_words="english")
 
-
-
 @st.cache(allow_output_mutation=True)
 def load_model(model_name, hdbscan_model=hdbscan_model, umap_model=umap_model, vectorizer_model=vectorizer_model, use_topic_reduction = use_topic_reduction, number_of_topics = number_of_topics):
     sentence_model = SentenceTransformer(model_name)
@@ -133,17 +99,12 @@ def fit_transform(model, docs):
     topics, probs = model.fit_transform(docs)
     return topics, probs
 
-
 model = load_model(model_name=model_name)
-
 df = load_and_process_data(dataset_name, dataset_name_2, True, split_name, number_of_records)
-
 X = df[column_name]
-
 st.header("Original Dataset")
 st.write(df)
 
-
 topics, probs = fit_transform(model, X)
 
 st.header("Topic assignment for each example")
@@ -170,7 +131,6 @@ if labels:
     st.header("Topics per class")
     topics_per_class = model.topics_per_class(X, classes=y)
     st.plotly_chart(model.visualize_topics_per_class(topics_per_class))
-#TODO:Each of these need there own options!
 
 st.header("Visualizations")