from sentence_transformers import SentenceTransformer, util import torch import difflib from utils.GetDB import GetDB postgreSQL_pool = GetDB().get_db_connection() embedder = SentenceTransformer('all-MiniLM-L6-v2') def get_question(): # Connect to the PostgreSQL database conn = postgreSQL_pool.getconn() # Create a cursor object cur = conn.cursor() # Execute a SELECT query to fetch data from the "users" table cur.execute("SELECT question FROM chat_history ORDER BY created_at DESC") # Fetch all the results as a list of tuples results = cur.fetchall() results = [x[0] for x in results] # Close the cursor and connection # # cur.close() postgreSQL_pool.putconn(conn) return results def count_top_questions(questions_array): corpus_embeddings = embedder.encode(questions_array, convert_to_tensor=True) top_questions_array = {} for question in questions_array: query_embedding = embedder.encode([question], convert_to_tensor=True) # We use cosine-similarity and torch.topk to find the highest 5 scores cos_scores = util.cos_sim(query_embedding, corpus_embeddings)[0] top_results = torch.topk(cos_scores, k=100) counter = 0 for score, idx in zip(top_results[0][1:], top_results[1][1:]): if score.item() >= 0.8: counter += 1 top_questions_array[question] = counter # removing duplicate tuples return sorted(top_questions_array.items(), key=lambda x: x[1], reverse=True)[:50] def remove_redundancy(redundant_raw_top_asked_questions): for raw_top_asked_question in redundant_raw_top_asked_questions: for raw_top_asked_question_inner in redundant_raw_top_asked_questions: matching_ratio = difflib.SequenceMatcher(None, raw_top_asked_question_inner[0], raw_top_asked_question[0]).ratio() if 0.7 <= matching_ratio < 1.0: redundant_raw_top_asked_questions.remove(raw_top_asked_question_inner) return redundant_raw_top_asked_questions def remove_greetings(sanitised_questions_array): greeting_array = ['hey', 'hi', 'hello', "Hello!", "Hi there!", "Hey!", "Good morning!", "Good afternoon!", "Good evening!", "Howdy!", "Greetings!", "Nice to see you!", "What's up?", "Hi!", "hiiii", "Hello!", "Hey!", "How are you?", "What is your name?", "Where are you from?", "What do you do?", "How can I help you?", "What's the weather like?", "Do you have any plans for the weekend?", "Have you seen any good movies lately?", "What's your favorite food?", "What are your hobbies?", "hi, hello"] greetings_embeddings = embedder.encode(greeting_array, convert_to_tensor=True) for raw_top_asked_question in sanitised_questions_array[:10]: query_embedding = embedder.encode([raw_top_asked_question[0]], convert_to_tensor=True) cos_scores = util.cos_sim(query_embedding, greetings_embeddings)[0] top_results = torch.topk(cos_scores, k=1) for score, idx in zip(top_results[0], top_results[1]): if score.item() >= 0.87: sanitised_questions_array.remove(raw_top_asked_question) return sanitised_questions_array def final_phase_filtering(raw_first_phase_filtered_questions, limit=20): raw_first_phase_filtered_questions = raw_first_phase_filtered_questions[:limit] for raw_first_phase_filtered_question in raw_first_phase_filtered_questions: for raw_first_phase_filtered_question_inner in raw_first_phase_filtered_questions: emb1 = embedder.encode(raw_first_phase_filtered_question[0]) emb2 = embedder.encode(raw_first_phase_filtered_question_inner[0]) cos_sim = util.cos_sim(emb1, emb2) if 0.85 <= cos_sim.item() < 1.0000001192092896: raw_first_phase_filtered_questions.remove(raw_first_phase_filtered_question_inner) return raw_first_phase_filtered_questions def return_top_question(limit=5): questions = get_question() print('questions') count_top_questions_ = count_top_questions(questions) print('count_top_questions_') remove_redundancy_ = remove_redundancy(count_top_questions_) print('remove_redundancy_') remove_greetings_ = remove_greetings(remove_redundancy_) print('remove_greetings_') final_phase_filtering_ = final_phase_filtering(remove_greetings_)[:limit] print('final_phase_filtering_') message = 'These are the top questions asked on the ask twimbit/platform by the users:' for key, final_phase_filtering__ in enumerate(final_phase_filtering_): message = message + '\n {}: '.format(key + 1) + final_phase_filtering__[0] return message def return_recent_posts(limit=5, strategy='recent'): import os import requests import json HASURA_URL = os.environ['HASURA_URL'] HASURA_ADMIN_SECRET = os.environ['HASURA_ADMIN_SECRET'] url = HASURA_URL body = """query homeFeedQuery($strategy: Strategy, $limit: Int){ feed(strategy: $strategy, limit: $limit) { hits { link title date author } } }""" variables = {'strategy': strategy, 'limit': limit} response = requests.post(url=url, json={'query': body, 'variables': variables}, headers={ 'x-hasura-admin-secret': HASURA_ADMIN_SECRET}) message = 'These are the recent Articles/Posts on the platform/twimbit website: \n' if response.status_code == 200: data = json.loads(response.content) posts = data.get('data').get('feed').get('hits') for key, post in enumerate(posts): title = post.get('title') link = post.get('link') date = post.get('date') authors = ','.join(post.get('author')) message += 'Post/Article {}:- \n\tPost/Article Title:- {}\n\tPost/Article Link/URL:- {}\n\tPost/Article Publish Date:- {}\n\tPost/Article Author:- {}\n'.format( key + 1, title, link, date, authors) return message