feat: Update blog data processing to create vector database and enhance stats tracking

py-src/app.py

@@ -10,7 +10,7 @@ load_dotenv()
 import pipeline
 #build vector store
 print("=== Blog Data Update ===")
-pipeline.main()
+pipeline.create_vector_database(force_recreate=True)
 print("========================")
 
 import chainlit as cl

py-src/pipeline.py

@@ -18,7 +18,7 @@ import argparse
 from datetime import datetime
 import json
 from pathlib import Path
-from lets_talk.config import VECTOR_STORAGE_PATH
+from lets_talk.config import VECTOR_STORAGE_PATH, DATA_DIR
 
 # Import the blog utilities module
 import lets_talk.utils.blog as blog
@@ -28,8 +28,8 @@ def parse_args():
     parser = argparse.ArgumentParser(description="Update blog data vector store")
     parser.add_argument("--force-recreate", action="store_true", 
                         help="Force recreation of the vector store")
-    parser.add_argument("--data-dir", default=blog.DATA_DIR,
-                        help=f"Directory containing blog posts (default: {blog.DATA_DIR})")
+    parser.add_argument("--data-dir", default=DATA_DIR,
+                        help=f"Directory containing blog posts (default: {DATA_DIR})")
     return parser.parse_args()
 
 def save_stats(stats, output_dir="./stats"):
@@ -57,7 +57,7 @@ def save_stats(stats, output_dir="./stats"):
     print(f"Saved stats to {filename}")
     return filename
 
-def create_vector_database(documents, data_dir, storage_path=VECTOR_STORAGE_PATH, force_recreate=False):
+def create_vector_database(data_dir=DATA_DIR, storage_path=VECTOR_STORAGE_PATH, force_recreate=False):
     """
     Create or update the vector database with blog documents.
     
@@ -70,7 +70,21 @@ def create_vector_database(documents, data_dir, storage_path=VECTOR_STORAGE_PATH
     Returns:
         Tuple of (success status, message)
     """
+
+    
+    
     try:
+        # Load and process documents
+        documents = blog.load_blog_posts(data_dir)
+        documents = blog.update_document_metadata(documents)
+        
+        # Get stats
+        stats = blog.get_document_stats(documents)
+        blog.display_document_stats(stats)
+        
+        # Save stats for tracking
+        stats_file = save_stats(stats)
+
         create_vector_store = (not Path.exists(Path(storage_path))) or force_recreate
         
         if create_vector_store:
@@ -82,9 +96,9 @@ def create_vector_database(documents, data_dir, storage_path=VECTOR_STORAGE_PATH
             )
             vector_store.client.close()
             print("Vector store reference file saved.")
-            return True, f"Vector store successfully created at {storage_path}"
+            return True, f"Vector store successfully created at {storage_path}",stats, stats_file
         else:
-            return True, f"Vector store already exists at {storage_path} (use --force-recreate to rebuild)"
+            return True, f"Vector store already exists at {storage_path} (use --force-recreate to rebuild)",stats, stats_file
     except Exception as e:
         return False, f"Error creating vector store: {str(e)}"
 
@@ -98,20 +112,10 @@ def main():
     print("========================")
     
     try:
-        # Load and process documents
-        documents = blog.load_blog_posts(args.data_dir)
-        documents = blog.update_document_metadata(documents)
-        
-        # Get stats
-        stats = blog.get_document_stats(documents)
-        blog.display_document_stats(stats)
-        
-        # Save stats for tracking
-        stats_file = save_stats(stats)
+       
 
         # Create or update vector database
-        success, message = create_vector_database(
-            documents, 
+        success, message,stats,stats_file = create_vector_database(
             args.data_dir, 
             storage_path=VECTOR_STORAGE_PATH, 
             force_recreate=args.force_recreate

pyproject.toml

@@ -18,6 +18,7 @@ dependencies = [
     "langchain-qdrant>=0.2.0",
     "langchain-text-splitters>=0.3.8",
     "langgraph>=0.4.3",
+    "libmagic>=1.0",
     "pandas>=2.2.3",
     "python-dotenv>=1.1.0",
     "qdrant-client>=1.14.2",

stats/blog_stats_20250511_100823.json

@@ -0,0 +1,8 @@
+{
+  "timestamp": "20250511_100823",
+  "total_documents": 14,
+  "total_characters": 106275,
+  "min_length": 1900,
+  "max_length": 13468,
+  "avg_length": 7591.071428571428
+}

uv.lock

@@ -1143,6 +1143,7 @@ dependencies = [
     { name = "langchain-qdrant" },
     { name = "langchain-text-splitters" },
     { name = "langgraph" },
+    { name = "libmagic" },
     { name = "pandas" },
     { name = "python-dotenv" },
     { name = "qdrant-client" },
@@ -1165,6 +1166,7 @@ requires-dist = [
     { name = "langchain-qdrant", specifier = ">=0.2.0" },
     { name = "langchain-text-splitters", specifier = ">=0.3.8" },
     { name = "langgraph", specifier = ">=0.4.3" },
+    { name = "libmagic", specifier = ">=1.0" },
     { name = "pandas", specifier = ">=2.2.3" },
     { name = "python-dotenv", specifier = ">=1.1.0" },
     { name = "qdrant-client", specifier = ">=1.14.2" },
@@ -1172,6 +1174,12 @@ requires-dist = [
     { name = "websockets", specifier = ">=15.0.1" },
 ]
 
+[[package]]
+name = "libmagic"
+version = "1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/83/86/419ddfc3879b4565a60e0c75b6d19baec48428cbc2f15aca5320b3d136f6/libmagic-1.0.tar.gz", hash = "sha256:649f1ce7fb7c92796badbb812555e4a926351da4f5cdf82e810b5cd371aedf8d", size = 3665 }
+
 [[package]]
 name = "literalai"
 version = "0.1.201"