Upload 5 files

GenerateAIPodcast.py

@@ -0,0 +1,25 @@
+# Import the required module
+import pyttsx3
+from numpy.ma.core import concatenate
+
+
+
+def generateMp3(text, number):
+    # Initialize the Pyttsx3 engine
+    pyttsx3.init(driverName='sapi5')
+    engine = pyttsx3.init()
+
+    voices = engine.getProperty('voices')
+
+    engine.setProperty('voice', voices[2].id)
+
+    newVoiceRate = 145
+    engine.setProperty('rate', newVoiceRate)
+
+    # We can use file extension as mp3 and wav, both will work
+    fileName = "Ttldr - " + str(number) + ".mp3"
+    engine.save_to_file(text, fileName)
+
+    # Wait until above command is not finished.
+    engine.runAndWait()
+

app.py

@@ -1,6 +1,9 @@
 from fastapi import FastAPI
 from transformers import pipeline
 
+import crawl_archive
+import GenerateAIPodcast
+
 
 from llama_cpp import Llama
 
@@ -12,19 +15,6 @@ llm = Llama.from_pretrained(
 	filename="llama-3.2-1b-instruct-q8_0.gguf",
 )
 
-check  = llm.create_chat_completion(
-	messages = [
-		{
-			"role": "user",
-			"content": "What is the capital of France?"
-		}
-	]
-)
-
-print(check['choices'][0]['message']['content'])
-
-## create a new FASTAPI app instance
-app=FastAPI()
 @app.get("/")
 def home():
     return {"message":"Hello World"}
@@ -32,18 +22,20 @@ def home():
 # Define a function to handle the GET request at `/generate`
 
 
+
 @app.get("/generate")
-def generate(text:str):
+def generate(link:str):
     ## use the pipeline to generate text from given input text
     output= llm.create_chat_completion(
 	messages = [
-		{
-			"role": "user",
-			"content": f"{text}"
-		}
+		{"role": "system",
+		 "content": "Always answer short and most detailled and dont use * in your answers. It should be good to hear as a Podcast"},
+		{"role": "user", "content": f"Please summarize this website: {link}."}
 	]
 )
 
     ## return the generate text in Json reposnfe
-    return {"output":output[0]['generated_text']}
+    return output['choices'][0]['message']['content']
+
 
+crawl_archive.run_tldr_crawler()

crawl_archive.py

@@ -0,0 +1,136 @@
+import asyncio
+
+from llama_cpp import Llama
+from openai import OpenAI
+from selenium import webdriver
+from selenium.common import WebDriverException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.chrome.service import Service
+import concurrent.futures
+from webdriver_manager.chrome import ChromeDriverManager
+import time
+from datetime import datetime
+import os
+from GenerateAIPodcast import generateMp3
+from btts import generateAudioFile
+
+# client = OpenAI(base_url="http://localhost:8080/v1", api_key="lm-studio")
+
+
+'''def make_request(link):
+    print("-----------------------------------------------------------------------------------------")
+    print("Make Request is called")
+    try:
+
+        completion = client.chat.completions.create(
+            model="model-identifier",
+
+            messages=[
+                {"role": "system",
+                 "content": "Always answer short and most detailled and dont use * in your answers. It should be good to hear as a Podcast"},
+                {"role": "user", "content": f"Please summarize this website: {link}."}
+            ],
+            temperature=0.7,
+        )
+        # print(f"Thread: {completion.choices[0].message}")
+        # print("TEST:", completion.choices[0].message)
+        message = completion.choices[0].message.content
+        return message
+    except Exception as e:
+        print(f"Thread  encountered an error: {e}^")
+        '''
+
+
+
+llm = Llama.from_pretrained(
+	repo_id="hugging-quants/Llama-3.2-1B-Instruct-Q8_0-GGUF",
+	filename="llama-3.2-1b-instruct-q8_0.gguf",
+)
+
+def generate(link:str):
+    ## use the pipeline to generate text from given input text
+    output= llm.create_chat_completion(
+	messages = [
+		{"role": "system",
+		 "content": "Always answer short and most detailled and dont use * in your answers. It should be good to hear as a Podcast"},
+		{"role": "user", "content": f"Please summarize this website: {link}."}
+	]
+)
+
+    ## return the generate text in Json reposnfe
+    return output['choices'][0]['message']['content']
+
+
+
+def run_tldr_crawler():
+    # Setup Selenium WebDriver
+    options = webdriver.ChromeOptions()
+    # options.add_argument()  # Run in headless mode (no browser UI)
+    options.add_argument('--disable-gpu')
+    options.add_argument('--no-sandbox')
+
+    # Initialize the WebDriver
+    service = Service(ChromeDriverManager().install())
+    driver = webdriver.Chrome(service=service, options=options)
+    try:
+        date = datetime.today().strftime('%Y-%m-%d')
+        print(date)
+        # Comment this if you want run this at a weekend
+        date = '2025-03-07'
+        # Step 1: Navigate to the TLDR archives page
+        url = f"https://tldr.tech/tech/{date}"
+        driver.get(url)
+
+        # Wait for the page to load
+        time.sleep(2)
+
+        # Step 3: Extract all links on the new page
+        links = driver.find_elements(By.TAG_NAME, 'a')
+
+        # Collect the href attributes
+        # extracted_links = [link.get_attribute('href') for link in links if link.get_attribute('href') is not None]
+
+        extracted_links = [
+            link.get_attribute('href')
+            for link in links
+            if link.get_attribute('href') is not None and
+               not link.get_attribute('href').startswith("https://tldr.tech") and
+               not link.get_attribute('href').startswith("https://jobs") and
+               not "advertise" in link.get_attribute('href')
+        ]
+
+        # Output the extracted links
+        print("Extracted Links:")
+        print(len(extracted_links))
+        for idx, link in enumerate(extracted_links, start=1):
+            print(f"{idx}. {link}")
+
+        # Die maximale Anzahl von Threads, die gleichzeitig laufen sollen
+        max_threads = 4
+
+        # ThreadPoolExecutor verwenden, um maximal 4 Threads gleichzeitig auszuführen
+        with concurrent.futures.ThreadPoolExecutor(max_workers=max_threads) as executor:
+            # Für jeden Link in extracted_links wird makerequest aufgerufen
+            # enumerate gibt auch den Index zurück, falls du ihn brauchst
+            futures = []
+            for idx, link in enumerate(extracted_links, start=1):
+                future = executor.submit(generate, link)
+                futures.append((idx, link, future))
+                # print(f"{idx}. {link}")
+
+                # print(future.result())
+
+            for idx, link, future in futures:
+                result = future.result()
+                # print(f"{idx}. {link} - Result {result}")
+                asyncio.run(generateAudioFile(result, idx))
+
+
+    except WebDriverException as e:
+        print(f"Fehler beim Laden der Seite: {e}")
+
+
+    finally:
+        # Close the WebDriver
+        driver.quit()