Merge branch 'final_project' of github.com:lorenzotomada/devtools_scicomp_project_2025 into final_project

Author: Gaspare99

scripts/profiling_memory.py renamed to scripts/profiling_memory_and_time.py

@@ -16,6 +16,7 @@
 import os
 import csv
 import sys
+from time import time
 from mpi4py import MPI
 
 
@@ -50,7 +51,6 @@
 plot = kwargs["plot"]
 
 # Now we build the matrix on rank 0
-# It is a scipy sparse matrix with the structure of a 2D Poisson problem matrix obtained using finite differences
 if rank == 0:
     eig = np.arange(1, dim + 1)
     A = np.diag(eig)
@@ -69,17 +69,24 @@
 # We actually call it twice: the first time to ensure that the function is JIT-compiled by Numba, the second one for memory profiling
 if rank == 0:
     print("Precompiling Lanczos...")
+
     Q, diag, off_diag = Lanczos_PRO(A_np, np.ones_like(np.diag(A_np)) * 1.0)
+
     print("Done. Now reducing using Lanczos...")
+
     gc.collect()
     proc = psutil.Process()
     mem_before_lanczos = proc.memory_info().rss / 1024 / 1024  # MB
+    begin_lanczos = time()
 
     Q, diag, off_diag = Lanczos_PRO(A_np, np.ones_like(np.diag(A_np)) * 1.0)
 
+    end_lanczos = time()
     gc.collect()
     mem_after_lanczos = proc.memory_info().rss / 1024 / 1024  # MB
     delta_mem_lanczos = mem_after_lanczos - mem_before_lanczos
+    delta_t_lanzos = end_lanczos - begin_lanczos
+
     print("Done. Now computing eigenvalues...")
 else:
     diag = off_diag = None
@@ -91,43 +98,55 @@
 gc.collect()
 proc = psutil.Process()
 mem_before = proc.memory_info().rss / 1024 / 1024  # MB
+time_before_parallel = time()
 
 eigvals, eigvecs = parallel_tridiag_eigen(
     diag, off_diag, comm=comm, min_size=1, tol_factor=1e-10
 )
 
+time_after_parallel = time()
 gc.collect()
 mem_after = proc.memory_info().rss / 1024 / 1024
 delta_mem = mem_after - mem_before
+delta_t_parallel = time_after_parallel - time_before_parallel
 
 total_mem_children = comm.reduce(delta_mem, op=MPI.SUM, root=0)
+total_time_children = comm.reduce(delta_t_parallel, op=MPI.SUM, root=0)
 
 # Collect the information across all ranks
 if rank == 0:
     total_mem_all = delta_mem_lanczos
+    total_time_all = delta_t_lanzos + total_time_children
     print("Eigenvalues computed.")
     process = psutil.Process()
 
-    print(f"Total memory across all processes: {total_mem_all:.2f} MB")
-
+    print(f"[D&I] Total memory across all processes: {total_mem_all:.4f} MB")
+    print(f"[D&I] Total time: across all processes: {total_time_all:.4f} s")
     # We also profile numpy and scipy memory consumption
-    mem_np = profile_numpy_eigvals(A_np)
-    print(f"NumPy eig memory usage: {mem_np:.2f} MB")
+    mem_np, time_np = profile_numpy_eigvals(A_np)
+    print(f"[NumPy] eig memory usage: {mem_np:.4f} MB")
+    print(f"[NumPy] eig total time: {time_np:.4f} s")
 
-    mem_sp = profile_scipy_eigvals(A_np)
-    print(f"SciPy eig memory usage: {mem_sp:.2f} MB")
+    mem_sp, time_sp = profile_scipy_eigvals(A_np)
+    print(f"[SciPy] eig memory usage: {mem_sp:.4f} MB")
+    print(f"[SciPy] eig total time: {time_sp:.4f} s")
 
     # Save to the logs folder
     os.makedirs("logs", exist_ok=True)
-    log_file = "logs/memory_profile.csv"
+    log_file = "logs/profile.csv"
     fieldnames = [
         "matrix_size",
         "n_processes",
-        "mem_lanzos_mb",
+        "mem_lanczos_mb",
         "mem_tridiag_mb",
         "mem_total_mb",
         "mem_numpy_mb",
         "mem_scipy_mb",
+        "time_lanczos",
+        "time_tridiag",
+        "time_total",
+        "time_numpy",
+        "time_scipy",
     ]
 
     write_header = not os.path.exists(log_file)
@@ -139,11 +158,16 @@
             {
                 "matrix_size": dim,
                 "n_processes": size,
-                "mem_lanzos_mb": round(delta_mem_lanczos, 2),
+                "mem_lanczos_mb": round(delta_mem_lanczos, 2),
                 "mem_tridiag_mb": round(total_mem_children, 2),
                 "mem_total_mb": round(total_mem_all, 2),
                 "mem_numpy_mb": round(mem_np, 2),
                 "mem_scipy_mb": round(mem_sp, 2),
+                "time_lanczos": round(delta_t_lanzos, 2),
+                "time_tridiag": round(total_time_children, 2),
+                "time_total": round(total_time_all, 2),
+                "time_numpy": round(time_np, 2),
+                "time_scipy": round(time_sp, 2),
             }
         )
 
@@ -153,9 +177,10 @@
         import matplotlib.pyplot as plt
         import pandas as pd
 
-        df = pd.read_csv("logs/memory_profile.csv")
+        df = pd.read_csv("logs/profile.csv")
         nproc_values = sorted(df["n_processes"].unique())
 
+        # First we plot the memoy usage, then the execution time
         plt.figure(figsize=(10, 6))
 
         numpy_avg = df.groupby("matrix_size")["mem_numpy_mb"].mean()
@@ -206,3 +231,54 @@
 
         plt.savefig("logs/memory_profiling.png", bbox_inches="tight")
         plt.show()
+
+        plt.figure(figsize=(10, 6))
+
+        numpy_avg = df.groupby("matrix_size")["time_numpy"].mean()
+        plt.plot(
+            numpy_avg.index,
+            numpy_avg.values,
+            color="green",
+            marker="x",
+            linestyle="--",
+            label="NumPy",
+        )
+
+        scipy_avg = df.groupby("matrix_size")["time_scipy"].mean()
+        plt.plot(
+            scipy_avg.index,
+            scipy_avg.values,
+            color="red",
+            marker="^",
+            linestyle=":",
+            label="SciPy",
+        )
+
+        for nproc in nproc_values:
+            subset = df[df["n_processes"] == nproc].sort_values("matrix_size")
+            label = f"Divide et impera ({nproc} proc{'s' if nproc > 1 else ''})"
+            plt.plot(
+                subset["matrix_size"],
+                subset["time_total"],
+                marker="o",
+                linestyle="-",
+                label=label,
+            )
+
+        plt.xlabel("Matrix size")
+        plt.ylabel("Total time (s)")
+        plt.xscale("log")
+        plt.title("Execution time vs. Matrix size")
+        plt.grid(True)
+        plt.tight_layout()
+
+        plt.legend(
+            bbox_to_anchor=(1.05, 1),
+            loc="upper left",
+            borderaxespad=0.0,
+            title="Method",
+        )
+        plt.subplots_adjust(right=0.75)
+
+        plt.savefig("logs/time_profiling.png", bbox_inches="tight")
+        plt.show()

shell/submit.sbatch

@@ -61,7 +61,7 @@ for dim in "${matrix_sizes[@]}"; do
       sed -i "s/^plot: .*/plot: true/" $CONFIG_FILE
     fi
 
-    srun --mpi=openmpi -n ${n_p} python scripts/profiling_memory.py
+    srun --mpi=openmpi -n ${n_p} python scripts/profiling_memory_and_time.py
   done
 done
 

shell/submit.sh

@@ -2,7 +2,7 @@
 
 # Ranges over which we iterate
 n_processes=(1 2 4 8)
-matrix_sizes=(10 50 100 500 1000)
+matrix_sizes=(10 50 100 500 1000 1500)
 
 last_dim="${matrix_sizes[-1]}"
 last_nproc="${n_processes[-1]}"
@@ -30,7 +30,7 @@ for dim in "${matrix_sizes[@]}"; do
       sed -i "s/^plot: .*/plot: true/" $CONFIG_FILE
     fi
 
-    mpirun -np ${n_p} python scripts/profiling_memory.py
+    mpirun -np ${n_p} python scripts/profiling_memory_and_time.py
   done
 done
 

src/pyclassify/utils.py

@@ -5,6 +5,7 @@
 import psutil
 import gc
 import yaml
+from time import time
 
 # import cProfile
 # from memory_profiler import memory_usage
@@ -162,14 +163,17 @@ def profile_numpy_eigvals(A):
     gc.collect()
     process = psutil.Process()
     mem_before = process.memory_info().rss / 1024 / 1024
+    time_begin = time()
 
     # NumPy symmetric eig solver
     eigvals, eigvecs = np.linalg.eigh(A)
 
+    time_end = time()
+    elapsed_time = time_end - time_begin
     gc.collect()
     mem_after = process.memory_info().rss / 1024 / 1024
     delta_mem = mem_after - mem_before
-    return delta_mem
+    return delta_mem, elapsed_time
 
 
 def profile_scipy_eigvals(A):
@@ -191,14 +195,16 @@ def profile_scipy_eigvals(A):
     gc.collect()
     process = psutil.Process()
     mem_before = process.memory_info().rss / 1024 / 1024
+    time_begin = time()
 
-    # SciPy symmetric eig solver
     eigvals, eigvecs = scipy.linalg.eigh(A)
 
+    time_end = time()
+    elapsed_time = time_end - time_begin
     gc.collect()
     mem_after = process.memory_info().rss / 1024 / 1024
     delta_mem = mem_after - mem_before
-    return delta_mem
+    return delta_mem, elapsed_time
 
 
 ######### OUTDATED FUNCTIONS: working but no longer used #########