Merge pull request #6 from gperdrizet/dev

functions/helper_functions.py

@@ -1,8 +1,33 @@
 '''Helper functions for MCP tools.'''
 
 import logging
+from types import GeneratorType
+
 import feedparser
-from findfeed import search
+from findfeed import search as feed_search
+from googlesearch import search as google_search
+
+def get_url(company_name: str) -> str:
+    '''Finds the website associated with the name of a company or
+    publication.
+
+    Args:
+        company_name: the name of the company, publication or site to find
+        the URL for
+
+    Returns:
+        The URL for the company, publication or website.
+    '''
+
+    logger = logging.getLogger(__name__ + '.get_url')
+
+    query = f'{company_name} official website'
+
+    for url in google_search(query, num_results=5):
+        if 'facebook' not in url and 'linkedin' not in url:
+            return url
+
+    return None
 
 
 def get_feed(website_url: str) -> str:
@@ -16,8 +41,13 @@ def get_feed(website_url: str) -> str:
     '''
 
     logger = logging.getLogger(__name__ + '.get_content')
+    logger.info('Getting feed URI for: %s', website_url)
+
+    feeds = feed_search(website_url)
 
-    feeds = search(website_url)
+    logger.info('Feeds search result is: %s', type(feeds))
+    logger.info('Feeds search results: %s', len(feeds))
+    logger.info('Feeds results: %s', list(feeds))
 
     if len(feeds) > 0:
         return str(feeds[0].url)

functions/tools.py

@@ -1,24 +1,56 @@
 '''Tool functions for MCP server'''
 
 import logging
+from urllib.parse import urlparse
+import validators
 import functions.helper_functions as helper_funcs
 
+FEED_URIS = {}
+RSS_EXTENSIONS = ['xml', 'rss', 'atom']
 
-def get_content(website_url: str) -> list:
+
+def get_content(website: str) -> list:
     '''Gets RSS feed content from a given website.
     
     Args:
-        website_url: URL of website to extract RSS feed content from
+        website_url: URL or nam of website to extract RSS feed content from
 
     Returns:
-        List of titles for the 10 most recent entries in the RSS feed.
+        List of titles for the 10 most recent entries in the RSS feed from the
+        requested website.
     '''
 
     logger = logging.getLogger(__name__ + '.get_content')
-    logger.info('Getting feed content for: %s', website_url)
+    logger.info('Getting feed content for: %s', website)
+
+    # Find the feed URI
+    feed_uri = None
+
+    # If the website contains xml, rss or atom, assume it's an RSS URI
+    if any(extension in website.lower() for extension in RSS_EXTENSIONS):
+        feed_uri = website
+        logger.info('%s looks like a feed URI already - using it directly', website)
+
+    # Next, check the cache to see if we alreay have this feed's URI
+    elif website in FEED_URIS.keys():
+        feed_uri = FEED_URIS[website]
+        logger.info('%s feed URI in cache: %s', website, feed_uri)
+
+    # If neither of those get it - try feedparse if it looks like a url
+    # or else just google it
+    else:
+        if validators.url(website):
+            website_url = website
+            logger.info('%s looks like a website URL', website)
+
+        else:
+            website_url = helper_funcs.get_url(website)
+            logger.info('Google result for %s: %s', website, website_url)
+
+        feed_uri = helper_funcs.get_feed(website_url)
+        logger.info('get_feed() returned %s', feed_uri)
 
-    feed_uri = helper_funcs.get_feed(website_url)
-    logger.info('get_feed() returned %s', feed_uri)
+        FEED_URIS[website] = feed_uri
 
     content = helper_funcs.parse_feed(feed_uri)
     logger.info('parse_feed() returned %s', content)

requirements.txt

@@ -1,4 +1,5 @@
 feedparser
 findfeed
 gradio
-mcp
+mcp
+validators

rss_server.py

@@ -21,7 +21,7 @@ logging.basicConfig(
         backupCount=10,
         mode='w'
     )],
-    level=logging.DEBUG,
+    level=logging.INFO,
     format='%(levelname)s - %(name)s - %(message)s'
 )
 
@@ -34,7 +34,7 @@ with gr.Blocks() as demo:
         gr.HTML(html.TITLE)
 
     gr.Markdown(html.DESCRIPTION)
-    website_url = gr.Textbox('hackernews.com', label='Website URL')
+    website_url = gr.Textbox('hackernews.com', label='Website')
     output = gr.Textbox(label='RSS entry titles', lines=10)
     submit_button = gr.Button('Submit')