Added some fancy fallback options to handle requests for feeds that are not urls, also added caching so we don't have to look up the same feed URI more than once in a session.

functions/helper_functions.py

@@ -1,8 +1,33 @@
 '''Helper functions for MCP tools.'''
 
 import logging
+from types import GeneratorType
+
 import feedparser
-from findfeed import search
+from findfeed import search as feed_search
+from googlesearch import search as google_search
+
+def get_url(company_name: str) -> str:
+    '''Finds the website associated with the name of a company or
+    publication.
+
+    Args:
+        company_name: the name of the company, publication or site to find
+        the URL for
+
+    Returns:
+        The URL for the company, publication or website.
+    '''
+
+    logger = logging.getLogger(__name__ + '.get_url')
+
+    query = f'{company_name} official website'
+
+    for url in google_search(query, num_results=5):
+        if 'facebook' not in url and 'linkedin' not in url:
+            return url
+
+    return None
 
 
 def get_feed(website_url: str) -> str:
@@ -16,8 +41,13 @@ def get_feed(website_url: str) -> str:
     '''
 
     logger = logging.getLogger(__name__ + '.get_content')
+    logger.info('Getting feed URI for: %s', website_url)
+
+    feeds = feed_search(website_url)
 
-    feeds = search(website_url)
+    logger.info('Feeds search result is: %s', type(feeds))
+    logger.info('Feeds search results: %s', len(feeds))
+    logger.info('Feeds results: %s', list(feeds))
 
     if len(feeds) > 0:
         return str(feeds[0].url)

functions/tools.py

@@ -1,24 +1,56 @@
 '''Tool functions for MCP server'''
 
 import logging
+from urllib.parse import urlparse
+import validators
 import functions.helper_functions as helper_funcs
 
+FEED_URIS = {}
+RSS_EXTENSIONS = ['xml', 'rss', 'atom']
 
-def get_content(website_url: str) -> list:
+
+def get_content(website: str) -> list:
     '''Gets RSS feed content from a given website.
     
     Args:
-        website_url: URL of website to extract RSS feed content from
+        website_url: URL or nam of website to extract RSS feed content from
 
     Returns:
-        List of titles for the 10 most recent entries in the RSS feed.
+        List of titles for the 10 most recent entries in the RSS feed from the
+        requested website.
     '''
 
     logger = logging.getLogger(__name__ + '.get_content')
-    logger.info('Getting feed content for: %s', website_url)
+    logger.info('Getting feed content for: %s', website)
+
+    # Find the feed URI
+    feed_uri = None
+
+    # If the website contains xml, rss or atom, assume it's an RSS URI
+    if any(extension in website.lower() for extension in RSS_EXTENSIONS):
+        feed_uri = website
+        logger.info('%s looks like a feed URI already - using it directly', website)
+
+    # Next, check the cache to see if we alreay have this feed's URI
+    elif website in FEED_URIS.keys():
+        feed_uri = FEED_URIS[website]
+        logger.info('%s feed URI in cache: %s', website, feed_uri)
+
+    # If neither of those get it - try feedparse if it looks like a url
+    # or else just google it
+    else:
+        if validators.url(website):
+            website_url = website
+            logger.info('%s looks like a website URL', website)
+
+        else:
+            website_url = helper_funcs.get_url(website)
+            logger.info('Google result for %s: %s', website, website_url)
+
+        feed_uri = helper_funcs.get_feed(website_url)
+        logger.info('get_feed() returned %s', feed_uri)
 
-    feed_uri = helper_funcs.get_feed(website_url)
-    logger.info('get_feed() returned %s', feed_uri)
+        FEED_URIS[website] = feed_uri
 
     content = helper_funcs.parse_feed(feed_uri)
     logger.info('parse_feed() returned %s', content)

rss_server.py

@@ -21,7 +21,7 @@ logging.basicConfig(
         backupCount=10,
         mode='w'
     )],
-    level=logging.DEBUG,
+    level=logging.INFO,
     format='%(levelname)s - %(name)s - %(message)s'
 )
 
@@ -34,7 +34,7 @@ with gr.Blocks() as demo:
         gr.HTML(html.TITLE)
 
     gr.Markdown(html.DESCRIPTION)
-    website_url = gr.Textbox('hackernews.com', label='Website URL')
+    website_url = gr.Textbox('hackernews.com', label='Website')
     output = gr.Textbox(label='RSS entry titles', lines=10)
     submit_button = gr.Button('Submit')