Updated

app.py

@@ -2,6 +2,7 @@ import gradio as gr
 import numpy as np
 from datetime import date
 from predictors import predict_bc_scores, predict_mc_scores
+from predictors import update, correct_text, split_text
 from analysis import depth_analysis
 from predictors import predict_quillbot
 from plagiarism import plagiarism_check, build_date
@@ -112,6 +113,12 @@ with gr.Blocks() as demo:
     char_count = gr.Textbox(label="Minumum Character Limit Check")
     input_text.change(fn=len_validator, inputs=input_text, outputs=char_count)
 
+    with gr.Row():
+        btn = gr.Button("Bias Buster")
+        out = gr.Textbox(label="Bias Corrected Full Input", interactive=False)
+        corrections_output = gr.Textbox(label="Bias Corrections", interactive=False) 
+        btn.click(fn=update, inputs=input_text, outputs=[out, corrections_output]) 
+
     with gr.Row():
         models = gr.Dropdown(
             model_list,
@@ -310,6 +317,5 @@ with gr.Blocks() as demo:
     date_from = ""
     date_to = ""
 
-
 if __name__ == "__main__":  
-    demo.launch(share=True, server_name="0.0.0.0", auth=("polygraf-admin", "test@aisd"))
+    demo.launch(share=True, server_name="0.0.0.0", server_port = 80, auth=("polygraf-admin", "test@aisd"))

predictors.py

@@ -21,6 +21,15 @@ import os
 from utils import *
 import joblib
 from optimum.bettertransformer import BetterTransformer
+import gc
+from cleantext import clean
+import gradio as gr
+from tqdm.auto import tqdm
+from transformers import pipeline
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+import nltk
+from nltk.tokenize import sent_tokenize
+from optimum.pipelines import pipeline
 
 with open("config.yaml", "r") as file:
     params = yaml.safe_load(file)
@@ -37,6 +46,8 @@ mc_label_map = params["MC_OUTPUT_LABELS"]
 text_1on1_label_map = params["1ON1_OUTPUT_LABELS"]
 mc_token_size = int(params["MC_TOKEN_SIZE"])
 bc_token_size = int(params["BC_TOKEN_SIZE"])
+bias_checker_model_name = params['BIAS_CHECKER_MODEL_PATH']
+bias_corrector_model_name = params['BIAS_CORRECTOR_MODEL_PATH']
 text_bc_tokenizer = AutoTokenizer.from_pretrained(text_bc_model_path)
 text_bc_model = AutoModelForSequenceClassification.from_pretrained(
     text_bc_model_path
@@ -57,6 +68,21 @@ for model_name, model in zip(mc_label_map, text_1on1_models):
         model
     ).to(device)
 
+
+bias_model_checker = AutoModelForSequenceClassification.from_pretrained(bias_checker_model_name)
+tokenizer = AutoTokenizer.from_pretrained(bias_checker_model_name)
+bias_model_checker = BetterTransformer.transform(bias_model_checker, keep_original_model=False)
+bias_checker = pipeline(
+    "text-classification",
+    model=model,
+    tokenizer=tokenizer,
+)
+gc.collect()
+bias_corrector = pipeline(
+    "text2text-generation", model=bias_corrector_model_name, accelerator="ort"
+
+)
+
 # proxy models for explainability
 mini_bc_model_name = "polygraf-ai/bc-model-bert-mini"
 bc_tokenizer_mini = AutoTokenizer.from_pretrained(mini_bc_model_name)
@@ -79,6 +105,34 @@ quillbot_model = BetterTransformer.transform(quillbot_model)
 iso_reg = joblib.load("isotonic_regression_model.joblib")
 
 
+def split_text(text: str) -> list:
+    sentences = sent_tokenize(text)
+    return [[sentence] for sentence in sentences]
+
+def correct_text(text: str, bias_checker, bias_corrector, separator: str = " ") -> tuple:
+    sentence_batches = split_text(text)
+    corrected_text = []
+    corrections = []
+    for batch in tqdm(sentence_batches, total=len(sentence_batches), desc="correcting text.."):
+        raw_text = " ".join(batch)
+        results = bias_checker(raw_text)
+        if results[0]["label"] != "LABEL_1" or (results[0]["label"] == "LABEL_1" and results[0]["score"] < 0.9):
+            corrected_batch = bias_corrector(raw_text)
+            corrected_version = corrected_batch[0]["generated_text"]
+            corrected_text.append(corrected_version)
+            corrections.append((raw_text, corrected_version)) 
+        else:
+            corrected_text.append(raw_text)
+    corrected_text = separator.join(corrected_text)
+    return corrected_text, corrections
+
+def update(text: str):
+    text = clean(text, lower=False)
+    corrected_text, corrections = correct_text(text, bias_checker, bias_corrector)
+    corrections_display = "\n\n".join([f"Original: {orig}\nCorrected: {corr}" for orig, corr in corrections])
+    return corrected_text, corrections_display
+
+
 def split_text_allow_complete_sentences_nltk(
     text,
     max_length=256,
@@ -252,6 +306,7 @@ def predict_bc_scores(input):
     human_score = 1 - ai_score
     bc_score = {"AI": ai_score, "HUMAN": human_score}
     print(f"Calibration BC scores: AI: {ai_score}, HUMAN: {human_score}")
+    print(f"Input Text: {cleaned_text_bc}")
     return bc_score
 
 

requirements.txt

@@ -26,4 +26,6 @@ Unidecode
 python-dotenv
 lime
 joblib
-optimum
+optimum
+clean-text
+optimum[onnxruntime]

utils.py

@@ -31,13 +31,28 @@ def remove_accents(input_str):
 
 
 def remove_special_characters(text):
-    text = text.replace("<s>", "").replace("</s>", "")
-    text = remove_accents(text)
-    pattern = r'[^\w\s\d.,!?\'"()-;]+'
-    text = re.sub(pattern, "", text)
+    text = re.sub(r'https?://\S+|www\.\S+', '', text)
+    emoji_pattern = re.compile("["  
+        u"\U0001F600-\U0001F64F"  # emoticons
+        u"\U0001F300-\U0001F5FF"  # symbols & pictographs
+        u"\U0001F680-\U0001F6FF"  # transport & map symbols
+        u"\U0001F700-\U0001F77F"  # alchemical symbols
+        u"\U0001F780-\U0001F7FF"  # Geometric Shapes Extended
+        u"\U0001F800-\U0001F8FF"  # Supplemental Arrows-C
+        u"\U0001F900-\U0001F9FF"  # Supplemental Symbols and Pictographs
+        u"\U0001FA00-\U0001FA6F"  # Chess Symbols
+        u"\U0001FA70-\U0001FAFF"  # Symbols and Pictographs Extended-A
+        u"\U00002702-\U000027B0"  # Dingbats
+        u"\U000024C2-\U0001F251" 
+        "]+", flags=re.UNICODE)
+    text = emoji_pattern.sub('', text) 
+    text = re.sub(r'#\w+', '', text)  
+    text = re.sub(r'[^\w\s\d.,!?\'"()-;]', '', text) 
+    text = re.sub(r'\s+([.,!?;])', r'\1', text)
+    text = re.sub(r'([.,!?;])(\S)', r'\1 \2', text)
+    text = re.sub(r'\s+', ' ', text).strip()
     return text
 
-
 def remove_special_characters_2(text):
     pattern = r"[^a-zA-Z0-9 ]+"
     text = re.sub(pattern, "", text)