diff --git a/monai/csrc/filtering/permutohedral/permutohedral_cuda.cu b/monai/csrc/filtering/permutohedral/permutohedral_cuda.cu
index 5ec3967aa4..d1d78eb940 100644
--- a/monai/csrc/filtering/permutohedral/permutohedral_cuda.cu
+++ b/monai/csrc/filtering/permutohedral/permutohedral_cuda.cu
@@ -530,6 +530,8 @@ void PermutohedralCuda(scalar_t* values, scalar_t* positions, int elementCount,
 
   destroyHashTable<scalar_t>();
   cudaFree(table_values);
+  cudaFree(scaleFactor);
+  cudaFree(matrix);
 }
 
 #define DECLARATION(dc, fc)                                                                                         \