Initial commit of FlexChunk Hugging Face app

.gitattributes

@@ -0,0 +1 @@
+*.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,51 @@
+---
+title: FlexChunk SpMV Demo
+emoji: ⚡
+colorFrom: blue
+colorTo: green
+sdk: gradio
+sdk_version: 4.39.0 # Or check your gradio version
+app_file: app.py
+pinned: false
+---
+
+# FlexChunk: Enabling 100M×100M Out-of-Core SpMV
+
+This Hugging Face Space demonstrates **FlexChunk**, an algorithm for performing Sparse Matrix-Vector Multiplication (SpMV) on matrices potentially too large to fit in RAM.
+
+**Key Idea:** Divide the matrix into manageable horizontal chunks, process them sequentially, and use minimal memory (~1.7 GB for 100M×100M SpMV).
+
+## Interactive Demo
+
+The app above allows you to:
+1.  Generate a sparse matrix (adjust **Size**, **Density**, **Chunks**, **Challenging** flag).
+2.  Run SpMV using **FlexChunk** (chunking to disk) and **SciPy** (emulating disk load).
+3.  Compare performance (timings) and correctness.
+
+**Note:** Larger matrices/densities will take longer. Limits are set for public infrastructure.
+
+## Performance Highlights
+
+FlexChunk demonstrates near-linear scaling in time and memory for increasing matrix dimensions.
+
+**Time Performance and Memory Usage:** See the original article for detailed performance graphs and analysis.
+
+### Benchmark Results
+
+Selected results showing performance on large matrices:
+
+| Matrix Size        | Non-zero Elements | Total Time      | Peak RAM Usage |
+|--------------------|-------------------|-----------------|----------------|
+| ...                | ...               | ...             | ...            |
+| 1.0M × 1.0M        | 1.2M              | 1.07 s          | 17.00 MB       |
+| 10.0M × 10.0M      | 12.0M             | 10.21 s         | 170.00 MB      |
+| 30.0M × 30.0M      | 36.0M             | 31.13 s         | 510.00 MB      |
+| 50.0M × 50.0M      | 62.5M             | 55.27 s         | 850.00 MB      |
+| 70.0M × 70.0M      | 88.2M             | 1 min 17.1 s    | 1.19 GB        |
+| **100.0M × 100.0M**| **120.0M**        | **1 min 47.1 s**| **1.70 GB**    |
+
+*(Full table in the original [README](https://github.com/your-repo/FlexChunk/blob/main/README.md) - **Update this link!**)*
+
+## More Information
+
+See the original post for technical details: [FlexChunk: Enabling 100M×100M Out-of-Core SpMV](https://www.lesswrong.com/posts/zpRhsdDkWygTDScxb/flexchunk-enabling-100m-100m-out-of-core-spmv-1-8-min-1-7-gb) 

app.py

@@ -0,0 +1,221 @@
+import gradio as gr
+import numpy as np
+import scipy.sparse as sparse
+import time
+import os
+import shutil
+import math
+import sys
+from pathlib import Path
+
+# Assuming flex_chunk.py and matrix_multiply.py are in the same directory
+from flex_chunk import FlexChunk, save_chunk, load_chunk
+from matrix_multiply import prepare_chunks, load_chunks, matrix_vector_multiply
+
+# --- Matrix Generation (copied from test_vs_scipy.py) ---
+
+def generate_sparse_matrix(size, density, challenging=False):
+    """
+    Generate a sparse test matrix with optional challenging patterns.
+    
+    Args:
+        size: Matrix size (n x n)
+        density: Target density
+        challenging: Whether to include challenging patterns and extreme values
+        
+    Returns:
+        A scipy.sparse.csr_matrix
+    """
+    # Calculate number of non-zeros
+    nnz = int(size * size * density)
+    if nnz == 0: # Ensure at least one non-zero element if density is very low
+        nnz = 1 
+        
+    if not challenging:
+        # Simple random matrix
+        rows = np.random.randint(0, size, nnz)
+        cols = np.random.randint(0, size, nnz)
+        data = np.random.rand(nnz)
+        # Ensure the matrix actually has the specified size if nnz is small
+        if nnz < size:
+             # Add diagonal elements to ensure size
+             diag_indices = np.arange(min(nnz, size))
+             rows = np.concatenate([rows, diag_indices])
+             cols = np.concatenate([cols, diag_indices])
+             data = np.concatenate([data, np.ones(len(diag_indices))]) # Use 1 for diagonal
+        
+        matrix = sparse.csr_matrix((data, (rows, cols)), shape=(size, size))
+        matrix.sum_duplicates() # Consolidate duplicate entries
+        return matrix
+    
+    # --- Challenging matrix with specific patterns ---
+    # Base random matrix (80% of non-zeros)
+    base_nnz = int(nnz * 0.8)
+    rows = np.random.randint(0, size, base_nnz)
+    cols = np.random.randint(0, size, base_nnz)
+    data = np.random.rand(base_nnz)
+    
+    # Add diagonal elements (10% of non-zeros)
+    diag_nnz = int(nnz * 0.1)
+    diag_indices = np.random.choice(size, diag_nnz, replace=False)
+    
+    # Add extreme values (10% of non-zeros)
+    extreme_nnz = max(0, nnz - base_nnz - diag_nnz) # Ensure non-negative
+    extreme_rows = np.random.randint(0, size, extreme_nnz)
+    extreme_cols = np.random.randint(0, size, extreme_nnz)
+    
+    # Mix of very large and very small values
+    extreme_data = np.concatenate([
+        np.random.uniform(1e6, 1e9, extreme_nnz // 2),
+        np.random.uniform(1e-9, 1e-6, extreme_nnz - extreme_nnz // 2)
+    ]) if extreme_nnz > 0 else np.array([])
+    if extreme_nnz > 0:
+        np.random.shuffle(extreme_data)
+    
+    # Combine all components
+    all_rows = np.concatenate([rows, diag_indices, extreme_rows])
+    all_cols = np.concatenate([cols, diag_indices, extreme_cols])
+    all_data = np.concatenate([data, np.random.rand(diag_nnz), extreme_data])
+    
+    matrix = sparse.csr_matrix((all_data, (all_rows, all_cols)), shape=(size, size))
+    matrix.sum_duplicates() # Consolidate duplicate entries
+    return matrix
+
+# --- Benchmark Function (Placeholder) ---
+
+def run_benchmark(size, density, num_chunks, challenging, progress=gr.Progress()):
+    # This function will contain the main logic from test_vs_scipy.py
+    # Adapted for Gradio inputs and outputs
+    progress(0, desc="Starting Benchmark...")
+    time.sleep(1) # Placeholder
+    
+    # 1. Setup storage
+    storage_dir = Path("./flex_chunk_temp_space")
+    if storage_dir.exists():
+        shutil.rmtree(storage_dir)
+    storage_dir.mkdir(exist_ok=True)
+    
+    progress(0.1, desc="Generating Matrix...")
+    # 2. Generate matrix and vector
+    matrix = generate_sparse_matrix(size, density, challenging)
+    vector = np.random.rand(size)
+    actual_nnz = matrix.nnz
+    actual_density = actual_nnz / (size * size) if size > 0 else 0
+    
+    matrix_info = f"Matrix: {size}x{size}, Target Density: {density:.6f}, Actual Density: {actual_density:.6f}, NNZ: {actual_nnz}"
+    print(matrix_info) # For debugging in Hugging Face console
+    
+    # --- FlexChunk Run ---
+    progress(0.2, desc="Preparing FlexChunks...")
+    prepare_start = time.time()
+    prepare_chunks(matrix, num_chunks, str(storage_dir), verbose=False)
+    prepare_time = time.time() - prepare_start
+
+    progress(0.4, desc="Loading FlexChunks...")
+    load_start = time.time()
+    chunks = load_chunks(str(storage_dir), verbose=False)
+    load_time = time.time() - load_start
+    
+    progress(0.6, desc="Running FlexChunk SpMV...")
+    flex_compute_start = time.time()
+    flex_result = matrix_vector_multiply(chunks, vector, verbose=False)
+    flex_compute_time = time.time() - flex_compute_start
+    flex_total_time = load_time + flex_compute_time
+    
+    # --- SciPy Run ---
+    progress(0.7, desc="Saving SciPy data...")
+    scipy_temp_dir = storage_dir / "scipy_temp"
+    scipy_temp_dir.mkdir(exist_ok=True)
+    matrix_file = scipy_temp_dir / "matrix.npz"
+    vector_file = scipy_temp_dir / "vector.npy"
+    
+    scipy_save_start = time.time()
+    sparse.save_npz(matrix_file, matrix)
+    np.save(vector_file, vector)
+    scipy_save_time = time.time() - scipy_save_start
+    
+    progress(0.8, desc="Loading SciPy data...")
+    scipy_load_start = time.time()
+    loaded_matrix = sparse.load_npz(matrix_file)
+    loaded_vector = np.load(vector_file)
+    scipy_load_time = time.time() - scipy_load_start
+    
+    progress(0.9, desc="Running SciPy SpMV...")
+    scipy_compute_start = time.time()
+    scipy_result = loaded_matrix @ loaded_vector
+    scipy_compute_time = time.time() - scipy_compute_start
+    scipy_total_time = scipy_load_time + scipy_compute_time
+    
+    # --- Comparison ---
+    progress(0.95, desc="Comparing results...")
+    diff = np.abs(scipy_result - flex_result)
+    max_diff = np.max(diff) if len(diff) > 0 else 0
+    mean_diff = np.mean(diff) if len(diff) > 0 else 0
+    is_close = np.allclose(scipy_result, flex_result, atol=1e-9) # Increased tolerance slightly
+    comparison_result = f"✅ Results Match! (Max Diff: {max_diff:.2e}, Mean Diff: {mean_diff:.2e})" if is_close else f"❌ Results Differ! (Max Diff: {max_diff:.2e}, Mean Diff: {mean_diff:.2e})"
+
+    # --- Cleanup ---
+    shutil.rmtree(storage_dir)
+    
+    progress(1.0, desc="Benchmark Complete")
+    
+    # --- Format Output ---
+    results_summary = f"""
+{matrix_info}
+
+**FlexChunk Performance:**
+- Prepare Chunks Time: {prepare_time:.4f}s
+- Load Chunks Time:    {load_time:.4f}s
+- Compute Time:        {flex_compute_time:.4f}s
+- **Total (Load+Compute): {flex_total_time:.4f}s**
+
+**SciPy Performance (Out-of-Core Emulation):**
+- Save Data Time:   {scipy_save_time:.4f}s (For reference)
+- Load Data Time:   {scipy_load_time:.4f}s
+- Compute Time:     {scipy_compute_time:.4f}s
+- **Total (Load+Compute): {scipy_total_time:.4f}s**
+
+**Comparison:**
+{comparison_result}
+"""
+    
+    return results_summary
+
+# --- Gradio Interface ---
+
+with gr.Blocks() as demo:
+    gr.Markdown("""
+    # FlexChunk: Out-of-Core Sparse Matrix-Vector Multiplication (SpMV) Demo
+    
+    This demo compares the performance of FlexChunk against standard SciPy for SpMV, 
+    simulating an out-of-core scenario where the matrix doesn't fit entirely in memory.
+    
+    FlexChunk splits the matrix into smaller chunks, processing them sequentially to reduce peak memory usage. 
+    SciPy performance includes the time to save and load the matrix from disk to mimic this out-of-core access.
+    """)
+    
+    with gr.Row():
+        with gr.Column(scale=1):
+            gr.Markdown("**Benchmark Parameters**")
+            size_input = gr.Slider(label="Matrix Size (N x N)", minimum=100, maximum=50000, value=10000, step=100)
+            # Max density adjusted to prevent excessive nnz for large matrices in demo
+            density_input = gr.Slider(label="Matrix Density", minimum=0.00001, maximum=0.01, value=0.0001, step=0.00001, format="%.5f") 
+            chunks_input = gr.Slider(label="Number of Chunks", minimum=1, maximum=32, value=4, step=1)
+            challenging_input = gr.Checkbox(label="Use Challenging Matrix (Extreme Values)", value=False)
+            run_button = gr.Button("Run Benchmark", variant="primary")
+            
+        with gr.Column(scale=2):
+            gr.Markdown("**Results**")
+            output_textbox = gr.Markdown(label="Benchmark Summary")
+
+    run_button.click(
+        fn=run_benchmark, 
+        inputs=[size_input, density_input, chunks_input, challenging_input], 
+        outputs=[output_textbox]
+    )
+    
+    gr.Markdown("--- Developed based on the [FlexChunk concept](https://www.lesswrong.com/posts/zpRhsdDkWygTDScxb/flexchunk-enabling-100m-100m-out-of-core-spmv-1-8-min-1-7-gb).")
+
+# Launch the app
+if __name__ == "__main__":
+    demo.launch() 

flex_chunk.py

@@ -0,0 +1,214 @@
+"""
+FlexChunk - Minimal implementation of optimized data structure for sparse matrix chunks.
+
+Ref: T4, T5, T13
+"""
+
+import numpy as np
+import os
+import struct
+from typing import Tuple, Optional
+import scipy.sparse as sparse
+
+# Magic number for binary format identification
+FLEX_CHUNK_MAGIC = b'FLXCHK01'
+
+class FlexChunk:
+    """
+    A flexible sparse matrix chunk representation optimized for efficient operations.
+    
+    Ref: T4, T10
+    """
+    def __init__(self, 
+                start_row: int,
+                num_rows: int,
+                row_offsets: np.ndarray,
+                col_indices: np.ndarray,
+                data: np.ndarray,
+                shape: Optional[Tuple[int, int]] = None):
+        """
+        Initialize a FlexChunk from raw CSR data
+        
+        Ref: T4
+        
+        Args:
+            start_row: Global starting row index
+            num_rows: Number of rows in this chunk
+            row_offsets: CSR row pointer array (length num_rows+1)
+            col_indices: CSR column indices array
+            data: CSR data values array
+            shape: Optional matrix shape (rows, cols). If not provided, will be inferred.
+        """
+        self.start_row = start_row
+        self.num_rows = num_rows
+        self.end_row = start_row + num_rows
+        
+        # Validate row_offsets
+        if len(row_offsets) != num_rows + 1:
+            raise ValueError(f"row_offsets must have length {num_rows + 1}, got {len(row_offsets)}")
+        if not np.all(np.diff(row_offsets) >= 0):
+            raise ValueError("row_offsets must be monotonically increasing")
+            
+        # [T4] Preserve structural representation
+        self.row_offsets = row_offsets
+        self.col_indices = col_indices
+        self.data = data
+        
+        # Determine number of columns
+        if shape is not None:
+            self.n_cols = shape[1]
+        elif len(col_indices) > 0:
+            # If shape not provided, determine by max column index
+            self.n_cols = col_indices.max() + 1
+        else:
+            self.n_cols = 0
+        
+        # Save full matrix shape
+        self.shape = (num_rows, self.n_cols)
+        
+        # Stats
+        self.nnz = len(data)
+    
+    def process_with_vector(self, vector: np.ndarray) -> np.ndarray:
+        """
+        Multiply chunk with a vector
+        
+        Ref: T5, T13
+        
+        Args:
+            vector: Vector to multiply with
+            
+        Returns:
+            Result of multiplication
+        """
+        if len(vector) != self.n_cols:
+            raise ValueError(f"Vector length {len(vector)} does not match matrix columns {self.n_cols}")
+        
+        # [T5] Skip processing for empty data
+        if self.nnz == 0:
+            return np.zeros(self.num_rows, dtype=vector.dtype)
+        
+        # Create result buffer
+        result = np.zeros(self.num_rows, dtype=vector.dtype)
+        
+        # [T13] Optimize computation flow
+        for i in range(self.num_rows):
+            start_idx = self.row_offsets[i]
+            end_idx = self.row_offsets[i+1]
+            
+            # [T5] Process only non-zero elements
+            for j in range(start_idx, end_idx):
+                col = self.col_indices[j]
+                if col < len(vector):
+                    result[i] += self.data[j] * vector[col]
+        
+        return result
+    
+    @classmethod
+    def from_csr_matrix(cls, 
+                       matrix: sparse.csr_matrix,
+                       start_row: int = 0,
+                       end_row: Optional[int] = None) -> 'FlexChunk':
+        """
+        Create a FlexChunk from a CSR matrix (full or slice)
+        
+        Ref: T4, T9
+        
+        Args:
+            matrix: A scipy.sparse.csr_matrix
+            start_row: Global start row index
+            end_row: Global end row index (optional)
+            
+        Returns:
+            A new FlexChunk
+        """
+        if not sparse.isspmatrix_csr(matrix):
+            matrix = matrix.tocsr()
+            
+        if end_row is None:
+            end_row = start_row + matrix.shape[0]
+            
+        num_rows = end_row - start_row
+        
+        if num_rows != matrix.shape[0]:
+            raise ValueError(f"Matrix shape {matrix.shape} doesn't match row range {start_row}:{end_row}")
+        
+        # [T4] Maintain data structure integrity
+        row_offsets = matrix.indptr.copy()
+        col_indices = matrix.indices.copy()
+        data = matrix.data.copy()
+        
+        return cls(
+            start_row=start_row,
+            num_rows=num_rows,
+            row_offsets=row_offsets,
+            col_indices=col_indices,
+            data=data,
+            shape=matrix.shape
+        )
+
+def save_chunk(chunk: FlexChunk, filepath: str) -> None:
+    """
+    Save a FlexChunk to a binary file.
+    
+    Ref: T4
+    
+    Args:
+        chunk: The FlexChunk to save
+        filepath: Path to save the file
+    """
+    with open(filepath, 'wb') as f:
+        # Write the magic number
+        f.write(FLEX_CHUNK_MAGIC)
+        
+        # [T4] Store structural representation
+        f.write(struct.pack('q', chunk.start_row))
+        f.write(struct.pack('q', chunk.num_rows))
+        f.write(struct.pack('q', chunk.nnz))
+        f.write(struct.pack('q', chunk.n_cols))
+        
+        # Write arrays
+        f.write(chunk.row_offsets.astype(np.int32).tobytes())
+        f.write(chunk.col_indices.astype(np.int32).tobytes())
+        f.write(chunk.data.astype(np.float64).tobytes())
+
+def load_chunk(filepath: str) -> FlexChunk:
+    """
+    Load a FlexChunk from a binary file.
+    
+    Ref: T4
+    
+    Args:
+        filepath: Path to the file
+        
+    Returns:
+        Loaded FlexChunk
+    """
+    with open(filepath, 'rb') as f:
+        # Verify the magic number
+        magic = f.read(len(FLEX_CHUNK_MAGIC))
+        if magic != FLEX_CHUNK_MAGIC:
+            raise ValueError(f"Invalid file format for {filepath}")
+            
+        # [T4] Restore structural representation
+        start_row = struct.unpack('q', f.read(8))[0]
+        num_rows = struct.unpack('q', f.read(8))[0]
+        nnz = struct.unpack('q', f.read(8))[0]
+        n_cols = struct.unpack('q', f.read(8))[0]
+        
+        # Read arrays
+        row_offsets = np.frombuffer(f.read((num_rows + 1) * 4), dtype=np.int32)
+        col_indices = np.frombuffer(f.read(nnz * 4), dtype=np.int32)
+        data = np.frombuffer(f.read(nnz * 8), dtype=np.float64)
+        
+        # Create the FlexChunk with explicit shape
+        chunk = FlexChunk(
+            start_row=start_row,
+            num_rows=num_rows,
+            row_offsets=row_offsets,
+            col_indices=col_indices,
+            data=data,
+            shape=(num_rows, n_cols)
+        )
+        
+        return chunk 

matrix_multiply.py

@@ -0,0 +1,198 @@
+"""
+Minimal implementation of matrix-vector multiplication using FlexChunk format.
+Includes only direct (single-process) multiplication and chunking functions.
+
+Ref: T9, T10, T13
+"""
+
+import os
+import math
+import time
+import numpy as np
+import scipy.sparse as sparse
+from typing import List, Optional
+
+from flex_chunk import FlexChunk, save_chunk, load_chunk
+
+def prepare_chunks(matrix: sparse.csr_matrix, 
+                  num_chunks: int, 
+                  storage_dir: str,
+                  verbose: bool = False) -> List[str]:
+    """
+    Prepare chunks from a sparse matrix for processing.
+    
+    Ref: T4, T9
+    
+    Args:
+        matrix: Sparse matrix to split into chunks
+        num_chunks: Number of chunks to create
+        storage_dir: Directory to store chunks
+        verbose: Whether to print debug information
+        
+    Returns:
+        List of paths to the created chunks
+    """
+    if not sparse.isspmatrix_csr(matrix):
+        matrix = matrix.tocsr()
+    
+    # Ensure the storage directory exists
+    os.makedirs(storage_dir, exist_ok=True)
+    os.makedirs(os.path.join(storage_dir, "chunks"), exist_ok=True)
+    
+    # [T9] Divide data into independent processing units
+    rows_per_chunk = max(1, math.ceil(matrix.shape[0] / num_chunks))
+    
+    # Create and save chunks
+    chunk_paths = []
+    for i in range(num_chunks):
+        start_row = i * rows_per_chunk
+        end_row = min((i + 1) * rows_per_chunk, matrix.shape[0])
+        
+        if start_row >= matrix.shape[0]:
+            break
+        
+        # Extract the submatrix for this chunk
+        chunk_matrix = matrix[start_row:end_row, :]
+        
+        # [T4] Preserve data structure in chunks
+        chunk = FlexChunk.from_csr_matrix(
+            matrix=chunk_matrix,
+            start_row=start_row,
+            end_row=end_row
+        )
+        
+        # Save chunk to file
+        chunk_path = os.path.join(storage_dir, "chunks", f"chunk_{i}.bin")
+        save_chunk(chunk, chunk_path)
+        chunk_paths.append(chunk_path)
+        
+        if verbose:
+            print(f"Created chunk {i}: rows {start_row}-{end_row}, nnz: {chunk.nnz}, saved to {chunk_path}")
+    
+    # Also save matrix dimensions for later use
+    info_path = os.path.join(storage_dir, "matrix_info.npy")
+    np.save(info_path, np.array([matrix.shape[0], matrix.shape[1]], dtype=np.int64))
+    
+    if verbose:
+        print(f"Matrix chunks prepared and saved to {storage_dir}")
+        print(f"Total chunks: {len(chunk_paths)}")
+        print(f"Matrix shape: {matrix.shape}")
+    
+    return chunk_paths
+
+def load_chunks(storage_dir: str, verbose: bool = False) -> List[FlexChunk]:
+    """
+    Load precomputed chunks from storage directory.
+    
+    Ref: T4, T13
+    
+    Args:
+        storage_dir: Directory containing saved chunks
+        verbose: Whether to print debug information
+        
+    Returns:
+        List of loaded FlexChunk objects
+    """
+    chunks_dir = os.path.join(storage_dir, "chunks")
+    if not os.path.exists(chunks_dir):
+        raise ValueError(f"Chunks directory {chunks_dir} does not exist")
+    
+    # Find all chunk files
+    chunk_files = sorted([f for f in os.listdir(chunks_dir) if f.startswith("chunk_") and f.endswith(".bin")],
+                        key=lambda x: int(x.split('_')[1].split('.')[0]))
+    
+    if not chunk_files:
+        raise ValueError(f"No chunk files found in {chunks_dir}")
+    
+    # [T4] Restore structural representation from storage
+    chunks = []
+    for chunk_file in chunk_files:
+        chunk_path = os.path.join(chunks_dir, chunk_file)
+        chunks.append(load_chunk(chunk_path))
+    
+    if verbose:
+        print(f"Loaded {len(chunks)} chunks from {storage_dir}")
+        print(f"Matrix shape: ({chunks[-1].end_row}, {chunks[0].n_cols})")
+    
+    return chunks
+
+def matrix_vector_multiply(chunks: List[FlexChunk], 
+                          vector: np.ndarray,
+                          verbose: bool = False) -> np.ndarray:
+    """
+    Multiply a sparse matrix with a vector using direct mode and precomputed chunks.
+    
+    Ref: T5, T10, T13
+    
+    Args:
+        chunks: List of FlexChunk objects representing the matrix
+        vector: Vector to multiply with
+        verbose: Whether to print debug information
+        
+    Returns:
+        Result vector from the multiplication
+    """
+    start_time = time.time()
+    
+    if verbose:
+        print("Starting matrix-vector multiplication (direct mode)")
+    
+    # Convert vector to numpy array if needed
+    vector = np.asarray(vector)
+    
+    # Validate chunks
+    if not chunks:
+        raise ValueError("No chunks provided for multiplication")
+    
+    # Check vector dimensions
+    if vector.shape[0] != chunks[0].n_cols:
+        raise ValueError(f"Vector length {vector.shape[0]} does not match matrix columns {chunks[0].n_cols}")
+    
+    # Calculate result size based on the end row of the last chunk
+    result_size = max(chunk.end_row for chunk in chunks)
+    
+    # Initialize result vector
+    result = np.zeros(result_size, dtype=vector.dtype)
+    
+    # [T13] Direct computation through optimized pathways
+    for i, chunk in enumerate(chunks):
+        if verbose:
+            print(f"Processing chunk {i} with {chunk.nnz} non-zeros")
+        
+        # [T5] Skip processing for empty chunks
+        if chunk.nnz == 0:
+            continue
+            
+        # Multiply chunk with vector
+        chunk_result = chunk.process_with_vector(vector)
+        
+        # [T10] Map results to output coordinates
+        result[chunk.start_row:chunk.end_row] = chunk_result
+    
+    if verbose:
+        elapsed = time.time() - start_time
+        print(f"Direct multiplication completed in {elapsed:.4f}s")
+    
+    return result
+
+def process_matrix_file(storage_dir: str, 
+                       vector: np.ndarray, 
+                       verbose: bool = False) -> np.ndarray:
+    """
+    Convenience function to load chunks from storage and multiply with vector.
+    
+    Ref: T13
+    
+    Args:
+        storage_dir: Directory containing saved chunks
+        vector: Vector to multiply with
+        verbose: Whether to print debug information
+        
+    Returns:
+        Result vector from the multiplication
+    """
+    # [T13] Optimize data processing flow
+    chunks = load_chunks(storage_dir, verbose=verbose)
+    
+    # Perform multiplication
+    return matrix_vector_multiply(chunks, vector, verbose=verbose) 

requirements.txt

@@ -0,0 +1,3 @@
+numpy
+scipy
+gradio