diff --git a/deepspeed/module_inject/replace_module.py b/deepspeed/module_inject/replace_module.py
index 8a1d661bc4d6..ec653b613e64 100644
--- a/deepspeed/module_inject/replace_module.py
+++ b/deepspeed/module_inject/replace_module.py
@@ -142,6 +142,8 @@ def __init__(self, q_int8=True, group_size=1, num_bits=8):
         self.group_size = group_size
         self.num_bits = num_bits
         self.q_int8 = q_int8
+        # TODO(jeff): need to check w. Reza on why this is needed when changing tp size w. bloom
+        self.num_groups = 32
 
     def quantize(self, inputs, qkv=True, count=1, parallel_dim=0):
         if not self.q_int8 or not qkv: