Spaces:
Running
Running
File size: 4,868 Bytes
7c3be27 97420da 7c3be27 3542083 97420da 869d1ed 32dec0d 97420da 16990e2 97420da 16990e2 97420da 7c3be27 97420da 32dec0d 7c3be27 16990e2 7c3be27 16990e2 7c3be27 97420da 7c3be27 97420da 7c3be27 869d1ed 97420da |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 |
# gather_news.py
# News Source Integration
# This script integrates with various news sources to fetch the latest articles from the specified news sources,
# extracts relevant information such as title, URL, Source, Author and Publish date, and extracts full content.
import requests
import os
from extract_news import extract_news_articles, create_dataframe, save_to_csv
def fetch_newsapi_top_headlines(min_length=100, max_articles=25):
#import config
url = 'https://newsapi.org/v2/top-headlines'
api_key = os.environ.get("api_key")
params = {
'apiKey': api_key,
'language': 'en',
'pageSize': max_articles
}
response = requests.get(url, params=params)
if response.status_code != 200:
print(f"Error: Failed to fetch news from NewsAPI Top Headlines. Status code: {response.status_code}")
return []
articles = response.json().get("articles", [])
if not articles:
print("No articles found in NewsAPI Top Headlines.")
return []
meta_by_url = {}
urls = []
for article in articles:
url = article.get("url", "#")
meta = {
"url": url,
"title": article.get("title", ""),
"source": article.get("source", {}).get("name", ""),
"author": article.get("author", "Unknown"),
"publishedAt": article.get("publishedAt", "Unknown"),
}
meta_by_url[url] = meta
urls.append(url)
print(f"Fetched {len(urls)} article URLs from NewsAPI Top Headlines.")
extracted_articles = extract_news_articles(urls, min_length=min_length)
merged_articles = []
for art in extracted_articles:
meta = meta_by_url.get(art.get("original_url"))
if not meta:
meta = {
"title": art.get("title", "Untitled"),
"source": "",
"author": "Unknown",
"publishedAt": "Unknown"
}
merged = {
"url": art.get("url"),
"title": art.get("title") if art.get("title") and art.get("title") != "Untitled" else meta["title"],
"source": meta["source"],
"author": meta["author"],
"publishedAt": meta["publishedAt"],
"text": art.get("text", ""),
}
merged_articles.append(merged)
print(f"Usable articles after extraction (NewsAPI Top Headlines): {len(merged_articles)}")
return merged_articles
def fetch_newsapi_everything(topic, min_length=100, max_articles=50):
#import config
url = 'https://newsapi.org/v2/everything'
api_key = os.environ.get("api_key")
params = {
'apiKey': api_key,
'language': 'en',
'q': topic,
'pageSize': max_articles,
'sortBy': 'publishedAt'
}
response = requests.get(url, params=params)
if response.status_code != 200:
print(f"Error: Failed to fetch news from NewsAPI Everything. Status code: {response.status_code}")
return []
articles = response.json().get("articles", [])
if not articles:
print("No articles found in NewsAPI Everything.")
return []
meta_by_url = {}
urls = []
for article in articles:
url = article.get("url", "#")
meta = {
"url": url,
"title": article.get("title", ""),
"source": article.get("source", {}).get("name", ""),
"author": article.get("author", "Unknown"),
"publishedAt": article.get("publishedAt", "Unknown"),
}
meta_by_url[url] = meta
urls.append(url)
print(f"Fetched {len(urls)} article URLs from NewsAPI Everything.")
extracted_articles = extract_news_articles(urls, min_length=min_length)
merged_articles = []
for art in extracted_articles:
meta = meta_by_url.get(art.get("original_url"))
if not meta:
meta = {
"title": art.get("title", "Untitled"),
"source": "",
"author": "Unknown",
"publishedAt": "Unknown"
}
merged = {
"url": art.get("url"),
"title": art.get("title") if art.get("title") and art.get("title") != "Untitled" else meta["title"],
"source": meta["source"],
"author": meta["author"],
"publishedAt": meta["publishedAt"],
"text": art.get("text", ""),
}
merged_articles.append(merged)
print(f"Usable articles after extraction (NewsAPI Everything): {len(merged_articles)}")
return merged_articles
def fetch_articles(topic=None, min_length=100, max_articles=25):
if topic and topic.strip():
return fetch_newsapi_everything(topic, min_length=min_length, max_articles=max_articles)
else:
return fetch_newsapi_top_headlines(min_length=min_length, max_articles=max_articles) |