From 9f4cc3839cfbba6310c7e5c0ee9a3d1e9c52c216 Mon Sep 17 00:00:00 2001 From: Jeff Rasley Date: Thu, 22 Dec 2022 16:41:05 -0800 Subject: [PATCH] Fix issue w. bloom when changing tp size --- deepspeed/module_inject/replace_module.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deepspeed/module_inject/replace_module.py b/deepspeed/module_inject/replace_module.py index 8a1d661bc4d6..ec653b613e64 100644 --- a/deepspeed/module_inject/replace_module.py +++ b/deepspeed/module_inject/replace_module.py @@ -142,6 +142,8 @@ def __init__(self, q_int8=True, group_size=1, num_bits=8): self.group_size = group_size self.num_bits = num_bits self.q_int8 = q_int8 + # TODO(jeff): need to check w. Reza on why this is needed when changing tp size w. bloom + self.num_groups = 32 def quantize(self, inputs, qkv=True, count=1, parallel_dim=0): if not self.q_int8 or not qkv: