Update apis/Bubble_API_Calls.py

apis/Bubble_API_Calls.py

@@ -91,7 +91,28 @@ def fetch_linkedin_token_from_bubble(url_user_token_str: str):
     print(status_message) # Log the final status message
     return parsed_token_dict
 
-def fetch_linkedin_posts_data_from_bubble(constraint_value: str, data_type:str, constraint_key:str, constraint_type:str, additional_constraints: list = None):
+def fetch_linkedin_posts_data_from_bubble(
+    constraint_value: str, 
+    data_type: str, 
+    constraint_key: str, 
+    constraint_type: str, 
+    additional_constraints: list = None
+):
+    """
+    Fetches data from the Bubble.io Data API, handling pagination to retrieve all results.
+
+    Args:
+        constraint_value: The value to match in the constraint.
+        data_type: The Bubble data type (table name) to query.
+        constraint_key: The field key for the constraint.
+        constraint_type: The type of constraint (e.g., 'equals', 'contains').
+        additional_constraints: A list of additional constraint dictionaries.
+
+    Returns:
+        A tuple containing a pandas DataFrame with all results and an error message string.
+        If successful, the error message is None.
+        If an error occurs, the DataFrame is None.
+    """
     bubble_api_key = os.environ.get("Bubble_API")
     if not bubble_api_key:
         error_msg = "❌ Bubble API Error: The 'Bubble_API' environment variable is not set."
@@ -99,34 +120,64 @@ def fetch_linkedin_posts_data_from_bubble(constraint_value: str, data_type:str,
         return None, error_msg
 
     base_url = f"https://app.ingaze.ai/version-test/api/1.1/obj/{data_type}"
+    headers = {"Authorization": f"Bearer {bubble_api_key}"}
+    
+    # --- Main Constraint Setup ---
     constraints = [{"key": constraint_key, "constraint_type": constraint_type, "value": constraint_value}]
     if additional_constraints:
         constraints.extend(additional_constraints)
-    params = {'constraints': json.dumps(constraints)}
-    headers = {"Authorization": f"Bearer {bubble_api_key}"}
 
-    status_message = f"Attempting to fetch posts from Bubble for urn: {constraint_key}..."
-    print(status_message)
+    # --- Pagination Logic ---
+    all_results = []
+    cursor = 0  # Start at the beginning
+    
+    print(f"Attempting to fetch data from Bubble for {constraint_key} = {constraint_value}...")
 
-    try:
-        response = requests.get(base_url, params=params, headers=headers, timeout=15)
-        response.raise_for_status()
+    while True: # Loop until all pages are fetched
+        # Parameters for the API call, including constraints and pagination info
+        params = {
+            'constraints': json.dumps(constraints),
+            'cursor': cursor,
+            'limit': 100  # Fetch up to 100 items per request (the max)
+        }
+        
+        try:
+            # --- Make the API Request ---
+            response = requests.get(base_url, params=params, headers=headers, timeout=30)
+            response.raise_for_status()  # Raises an HTTPError for bad responses (4xx or 5xx)
+            
+            data = response.json().get("response", {})
+            results_on_page = data.get("results", [])
 
-        data = response.json()
-        results = data.get("response", {}).get("results", [])
+            if results_on_page:
+                all_results.extend(results_on_page)
+                print(f"Retrieved {len(results_on_page)} results on this page. Total so far: {len(all_results)}.")
+            
+            # --- Check if there are more pages ---
+            # The API returns the number of results remaining. If 0, we're done.
+            remaining = data.get("remaining", 0)
+            if remaining == 0:
+                print("No more pages to fetch.")
+                break # Exit the while loop
+            
+            # --- Update cursor for the next page ---
+            # The new cursor is the current cursor + number of items just received.
+            cursor += len(results_on_page)
 
-        if results:
-            df = pd.DataFrame(results)
-            print(f"Successfully retrieved {len(df)} posts.")
-            return df, None
-        else:
-            print("No posts found for the given org_urn.")
-            return pd.DataFrame(), None
+        except requests.exceptions.RequestException as e:
+            error_msg = f"❌ Bubble API Error: {str(e)}"
+            print(error_msg)
+            return None, error_msg
 
-    except requests.exceptions.RequestException as e:
-        error_msg = f"❌ Bubble API Error: {str(e)}"
-        print(error_msg)
-        return None, error_msg
+    # --- Final Processing ---
+    if all_results:
+        df = pd.DataFrame(all_results)
+        print(f"✅ Successfully retrieved a total of {len(df)} posts.")
+        return df, None
+    else:
+        print("No posts found for the given constraints.")
+        # Return an empty DataFrame if nothing was found
+        return pd.DataFrame(), None
 
 
 # def bulk_upload_to_bubble(data, data_type):