From 48bd1822d83bb45bb3f393cbceb59a725728b7da Mon Sep 17 00:00:00 2001 From: GuanhuaWang Date: Fri, 9 Sep 2022 23:07:53 +0000 Subject: [PATCH 01/15] add quant unit test --- .../ops/transformer/inference/test_quant.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 tests/unit/ops/transformer/inference/test_quant.py diff --git a/tests/unit/ops/transformer/inference/test_quant.py b/tests/unit/ops/transformer/inference/test_quant.py new file mode 100644 index 000000000000..a16410a7ada6 --- /dev/null +++ b/tests/unit/ops/transformer/inference/test_quant.py @@ -0,0 +1,33 @@ +import torch +import pytest +from deepspeed.ops import op_builder + +quantizer_cuda_module = op_builder.QuantizerBuilder().load() + +def allclose(x,y): + assert x.dtype == y.dtype + rtol, atol = {torch.float32: (1e-2, 1e-3), torch.float16: (1e-2, 1e-3)}[x.dtype] + return torch.allclose(x,y, rtol=rtol, atol=atol) + + +def quantize_ref(inputs, bit, num_groups=1): + q_range = 2**bit + input_flat = inputs.float().reshape(num_groups, -1).contiguous() + input_flat = torch.nan_to_num(input_flat, nan=0.0) + input_min = input_flat.amin(-1, keepdim=True) + input_max = input_flat.amax(-1, keepdim=True) + + scale = q_range / (2 * torch.max(input_min.abs(), input_max.abs())) + input_flat = (input_flat * scale).round().clamp(-q_range // 2, q_range // 2 - 1) + + return input_flat.reshape(inputs.shape).to(torch.int8)/scale.view(-1).to(torch.float16) + +@pytest.mark.inference +@pytest.mark.parametrize("input_tensor", torch.rand(8, 8, dtype=torch.float16).cuda()) +def test_quant_dequant(input_tensor): + ref_input = input_tensor.clone().detach() + ref_out = quantize_ref(ref_input,8) + + ds_out = quantizer_cuda_module.ds_quantize_fp16(input_tensor,1,8) + + assert (allclose(ds_out, ref_out)) \ No newline at end of file From 7f56db63412793cf8c5e0150e630fb63fda34a7b Mon Sep 17 00:00:00 2001 From: GuanhuaWang Date: Fri, 9 Sep 2022 23:16:29 +0000 Subject: [PATCH 02/15] add codeowner --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 6ad02de7a02f..e3fc6d679849 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1 @@ -* @jeffra @samyam @tjruwase @ShadenSmith @conglongli @awan-10 @cli99 @eltonzheng @minjiaz @RezaYazdaniAminabadi @duli2012 @mrwyattii @yaozhewei @arashb @xiaoxiawu-microsoft @samadejacobs @cmikeh2 +* @jeffra @samyam @tjruwase @ShadenSmith @conglongli @awan-10 @cli99 @eltonzheng @minjiaz @RezaYazdaniAminabadi @duli2012 @mrwyattii @yaozhewei @arashb @xiaoxiawu-microsoft @samadejacobs @cmikeh2 @GuanhuaWang From e696ed3bc21f9099f188763773cab7774dda2d03 Mon Sep 17 00:00:00 2001 From: GuanhuaWang Date: Fri, 9 Sep 2022 23:20:26 +0000 Subject: [PATCH 03/15] format fix --- .../ops/transformer/inference/test_quant.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/unit/ops/transformer/inference/test_quant.py b/tests/unit/ops/transformer/inference/test_quant.py index a16410a7ada6..d7960372786a 100644 --- a/tests/unit/ops/transformer/inference/test_quant.py +++ b/tests/unit/ops/transformer/inference/test_quant.py @@ -4,10 +4,11 @@ quantizer_cuda_module = op_builder.QuantizerBuilder().load() -def allclose(x,y): + +def allclose(x, y): assert x.dtype == y.dtype rtol, atol = {torch.float32: (1e-2, 1e-3), torch.float16: (1e-2, 1e-3)}[x.dtype] - return torch.allclose(x,y, rtol=rtol, atol=atol) + return torch.allclose(x, y, rtol=rtol, atol=atol) def quantize_ref(inputs, bit, num_groups=1): @@ -17,17 +18,19 @@ def quantize_ref(inputs, bit, num_groups=1): input_min = input_flat.amin(-1, keepdim=True) input_max = input_flat.amax(-1, keepdim=True) - scale = q_range / (2 * torch.max(input_min.abs(), input_max.abs())) + scale = q_range / (2 * torch.max(input_min.abs(), input_max.abs())) input_flat = (input_flat * scale).round().clamp(-q_range // 2, q_range // 2 - 1) - return input_flat.reshape(inputs.shape).to(torch.int8)/scale.view(-1).to(torch.float16) + return input_flat.reshape(inputs.shape).to(torch.int8) / scale.view(-1).to( + torch.float16) + @pytest.mark.inference @pytest.mark.parametrize("input_tensor", torch.rand(8, 8, dtype=torch.float16).cuda()) def test_quant_dequant(input_tensor): ref_input = input_tensor.clone().detach() - ref_out = quantize_ref(ref_input,8) + ref_out = quantize_ref(ref_input, 8) - ds_out = quantizer_cuda_module.ds_quantize_fp16(input_tensor,1,8) + ds_out = quantizer_cuda_module.ds_quantize_fp16(input_tensor, 1, 8) - assert (allclose(ds_out, ref_out)) \ No newline at end of file + assert (allclose(ds_out, ref_out)) From b5a5508356275ae65d950199060301d79a23870e Mon Sep 17 00:00:00 2001 From: GuanhuaWang Date: Fri, 9 Sep 2022 23:27:44 +0000 Subject: [PATCH 04/15] fix undefined symbol: curandSetPseudoRandomGeneratorSeed --- op_builder/quantizer.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/op_builder/quantizer.py b/op_builder/quantizer.py index 43bc5778ea20..314f5a8ad44b 100644 --- a/op_builder/quantizer.py +++ b/op_builder/quantizer.py @@ -20,3 +20,9 @@ def sources(self): def include_paths(self): return ['csrc/includes'] + + def extra_ldflags(self): + if not self.is_rocm_pytorch(): + return ['-lcurand'] + else: + return [] From 0f5dc1b0f71dd72a44ec97ac245af9775ed8c1e4 Mon Sep 17 00:00:00 2001 From: GuanhuaWang Date: Fri, 9 Sep 2022 23:56:07 +0000 Subject: [PATCH 05/15] modify ref fn name and add comment --- tests/unit/ops/transformer/inference/test_quant.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/ops/transformer/inference/test_quant.py b/tests/unit/ops/transformer/inference/test_quant.py index d7960372786a..70e7de4a73b5 100644 --- a/tests/unit/ops/transformer/inference/test_quant.py +++ b/tests/unit/ops/transformer/inference/test_quant.py @@ -11,7 +11,7 @@ def allclose(x, y): return torch.allclose(x, y, rtol=rtol, atol=atol) -def quantize_ref(inputs, bit, num_groups=1): +def quantize_dequantize_ref(inputs, bit, num_groups=1): q_range = 2**bit input_flat = inputs.float().reshape(num_groups, -1).contiguous() input_flat = torch.nan_to_num(input_flat, nan=0.0) @@ -29,8 +29,9 @@ def quantize_ref(inputs, bit, num_groups=1): @pytest.mark.parametrize("input_tensor", torch.rand(8, 8, dtype=torch.float16).cuda()) def test_quant_dequant(input_tensor): ref_input = input_tensor.clone().detach() - ref_out = quantize_ref(ref_input, 8) + ref_out = quantize_dequantize_ref(ref_input, 8) + # ds_quantize will do quantize then dequantize and return the dequantized value. ds_out = quantizer_cuda_module.ds_quantize_fp16(input_tensor, 1, 8) assert (allclose(ds_out, ref_out)) From dfd3ee70e34038a448643b7dae2fcacf12a1a998 Mon Sep 17 00:00:00 2001 From: GuanhuaWang Date: Sat, 10 Sep 2022 00:01:26 +0000 Subject: [PATCH 06/15] add comments --- tests/unit/ops/transformer/inference/test_quant.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/ops/transformer/inference/test_quant.py b/tests/unit/ops/transformer/inference/test_quant.py index 70e7de4a73b5..c8b513a223c1 100644 --- a/tests/unit/ops/transformer/inference/test_quant.py +++ b/tests/unit/ops/transformer/inference/test_quant.py @@ -12,6 +12,7 @@ def allclose(x, y): def quantize_dequantize_ref(inputs, bit, num_groups=1): + # quantize q_range = 2**bit input_flat = inputs.float().reshape(num_groups, -1).contiguous() input_flat = torch.nan_to_num(input_flat, nan=0.0) @@ -20,7 +21,7 @@ def quantize_dequantize_ref(inputs, bit, num_groups=1): scale = q_range / (2 * torch.max(input_min.abs(), input_max.abs())) input_flat = (input_flat * scale).round().clamp(-q_range // 2, q_range // 2 - 1) - + # dequantize return input_flat.reshape(inputs.shape).to(torch.int8) / scale.view(-1).to( torch.float16) From 5cd45c8592eeddb93de725fc2003438c57ca7ead Mon Sep 17 00:00:00 2001 From: GuanhuaWang Date: Mon, 12 Sep 2022 21:03:52 +0000 Subject: [PATCH 07/15] add 4bit quant 16groups --- tests/unit/ops/quantizer/test_quant.py | 51 +++++++++++++++++++ .../ops/transformer/inference/test_quant.py | 38 -------------- 2 files changed, 51 insertions(+), 38 deletions(-) create mode 100644 tests/unit/ops/quantizer/test_quant.py delete mode 100644 tests/unit/ops/transformer/inference/test_quant.py diff --git a/tests/unit/ops/quantizer/test_quant.py b/tests/unit/ops/quantizer/test_quant.py new file mode 100644 index 000000000000..a9938475a4c8 --- /dev/null +++ b/tests/unit/ops/quantizer/test_quant.py @@ -0,0 +1,51 @@ +import torch +import pytest +from deepspeed.ops import op_builder + +quantizer_cuda_module = None + +def allclose(x, y): + assert x.dtype == y.dtype + rtol, atol = {torch.float32: (2e-1, 5e-2), torch.float16: (2e-1, 5e-2)}[x.dtype] + return torch.allclose(x, y, rtol=rtol, atol=atol) + +def quantize_dequantize_ref(inputs, bit, num_groups=1): + # quantize + q_range = 2**bit + input_flat = inputs.float().reshape(num_groups, -1).contiguous() + input_flat = torch.nan_to_num(input_flat, nan=0.0) + input_min = input_flat.amin(-1, keepdim=True) + input_max = input_flat.amax(-1, keepdim=True) + + scale = q_range / (2 * torch.max(input_min.abs(), input_max.abs())) + input_flat = (input_flat * scale).round().clamp(-q_range // 2, q_range // 2 - 1) + # dequantize + return input_flat.reshape(inputs.shape).to(torch.int8) / scale.view(-1).to( + torch.float16) + +def run_quant_dequant(inputs, groups, bits): + global quantizer_cuda_module + if quantizer_cuda_module is None: + quantizer_cuda_module = op_builder.QuantizerBuilder().load() + return quantizer_cuda_module.ds_quantize_fp16(inputs, groups, bits) + + +@pytest.mark.inference +def test_quant_dequant(): + + input_small_tensor = torch.rand(8, 8, dtype=torch.float16).cuda() + input_big_tensor = torch.rand(128, 256, dtype=torch.float16).cuda() + + # test 8bit quant/dequant on 8x8 small tensor partitioned in 1 group. + ref_input_small_8bit_1group = input_small_tensor.clone().detach() + ref_out_small_8bit_1group = quantize_dequantize_ref(ref_input_small_8bit_1group, 8) + # run_quant_dequant will do quantize then dequantize and return the dequantized value. + ds_out_small_8bit_1group = run_quant_dequant(input_small_tensor, 1, 8) + assert (allclose(ds_out_small_8bit_1group, ref_out_small_8bit_1group)) + + # test 4bit quant/dequant on 128x256 big tensor partitioned into 16 groups. + # Note that we have an explict boundary for groups as ((size / groups) - 1) / 4096 + 1) <= MAX_REG. + ref_input_big_4bit_16group = input_big_tensor.clone().detach() + ref_out_big_4bit_16group = quantize_dequantize_ref(ref_input_big_4bit_16group, 4) + ds_out_big_4bit_16group = run_quant_dequant(input_big_tensor, 16, 4) + assert (allclose(ds_out_big_4bit_16group,ref_out_big_4bit_16group)) diff --git a/tests/unit/ops/transformer/inference/test_quant.py b/tests/unit/ops/transformer/inference/test_quant.py deleted file mode 100644 index c8b513a223c1..000000000000 --- a/tests/unit/ops/transformer/inference/test_quant.py +++ /dev/null @@ -1,38 +0,0 @@ -import torch -import pytest -from deepspeed.ops import op_builder - -quantizer_cuda_module = op_builder.QuantizerBuilder().load() - - -def allclose(x, y): - assert x.dtype == y.dtype - rtol, atol = {torch.float32: (1e-2, 1e-3), torch.float16: (1e-2, 1e-3)}[x.dtype] - return torch.allclose(x, y, rtol=rtol, atol=atol) - - -def quantize_dequantize_ref(inputs, bit, num_groups=1): - # quantize - q_range = 2**bit - input_flat = inputs.float().reshape(num_groups, -1).contiguous() - input_flat = torch.nan_to_num(input_flat, nan=0.0) - input_min = input_flat.amin(-1, keepdim=True) - input_max = input_flat.amax(-1, keepdim=True) - - scale = q_range / (2 * torch.max(input_min.abs(), input_max.abs())) - input_flat = (input_flat * scale).round().clamp(-q_range // 2, q_range // 2 - 1) - # dequantize - return input_flat.reshape(inputs.shape).to(torch.int8) / scale.view(-1).to( - torch.float16) - - -@pytest.mark.inference -@pytest.mark.parametrize("input_tensor", torch.rand(8, 8, dtype=torch.float16).cuda()) -def test_quant_dequant(input_tensor): - ref_input = input_tensor.clone().detach() - ref_out = quantize_dequantize_ref(ref_input, 8) - - # ds_quantize will do quantize then dequantize and return the dequantized value. - ds_out = quantizer_cuda_module.ds_quantize_fp16(input_tensor, 1, 8) - - assert (allclose(ds_out, ref_out)) From 325b86a629b7475e30434c3056ecc7e04b8b7a53 Mon Sep 17 00:00:00 2001 From: GuanhuaWang Date: Mon, 12 Sep 2022 21:07:00 +0000 Subject: [PATCH 08/15] fix --- tests/unit/ops/quantizer/test_quant.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/unit/ops/quantizer/test_quant.py b/tests/unit/ops/quantizer/test_quant.py index a9938475a4c8..38472ec966a8 100644 --- a/tests/unit/ops/quantizer/test_quant.py +++ b/tests/unit/ops/quantizer/test_quant.py @@ -4,11 +4,13 @@ quantizer_cuda_module = None + def allclose(x, y): assert x.dtype == y.dtype rtol, atol = {torch.float32: (2e-1, 5e-2), torch.float16: (2e-1, 5e-2)}[x.dtype] return torch.allclose(x, y, rtol=rtol, atol=atol) + def quantize_dequantize_ref(inputs, bit, num_groups=1): # quantize q_range = 2**bit @@ -23,6 +25,7 @@ def quantize_dequantize_ref(inputs, bit, num_groups=1): return input_flat.reshape(inputs.shape).to(torch.int8) / scale.view(-1).to( torch.float16) + def run_quant_dequant(inputs, groups, bits): global quantizer_cuda_module if quantizer_cuda_module is None: @@ -44,8 +47,8 @@ def test_quant_dequant(): assert (allclose(ds_out_small_8bit_1group, ref_out_small_8bit_1group)) # test 4bit quant/dequant on 128x256 big tensor partitioned into 16 groups. - # Note that we have an explict boundary for groups as ((size / groups) - 1) / 4096 + 1) <= MAX_REG. + # Note that we have an explicit boundary for groups as ((size / groups) - 1) / 4096 + 1) <= MAX_REG. ref_input_big_4bit_16group = input_big_tensor.clone().detach() ref_out_big_4bit_16group = quantize_dequantize_ref(ref_input_big_4bit_16group, 4) ds_out_big_4bit_16group = run_quant_dequant(input_big_tensor, 16, 4) - assert (allclose(ds_out_big_4bit_16group,ref_out_big_4bit_16group)) + assert (allclose(ds_out_big_4bit_16group, ref_out_big_4bit_16group)) From caa716fb011f7c94280bf4bbb4f17769ae1d7a0a Mon Sep 17 00:00:00 2001 From: GuanhuaWang Date: Mon, 12 Sep 2022 21:50:20 +0000 Subject: [PATCH 09/15] modify groups in ref code --- tests/unit/ops/quantizer/test_quant.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/ops/quantizer/test_quant.py b/tests/unit/ops/quantizer/test_quant.py index 38472ec966a8..915bd8380f51 100644 --- a/tests/unit/ops/quantizer/test_quant.py +++ b/tests/unit/ops/quantizer/test_quant.py @@ -22,8 +22,8 @@ def quantize_dequantize_ref(inputs, bit, num_groups=1): scale = q_range / (2 * torch.max(input_min.abs(), input_max.abs())) input_flat = (input_flat * scale).round().clamp(-q_range // 2, q_range // 2 - 1) # dequantize - return input_flat.reshape(inputs.shape).to(torch.int8) / scale.view(-1).to( - torch.float16) + dequant_flat = torch.t(input_flat.to(torch.int8)) / scale.view(-1).to(torch.float16) + return torch.t(dequant_flat).reshape(inputs.shape) def run_quant_dequant(inputs, groups, bits): @@ -49,6 +49,6 @@ def test_quant_dequant(): # test 4bit quant/dequant on 128x256 big tensor partitioned into 16 groups. # Note that we have an explicit boundary for groups as ((size / groups) - 1) / 4096 + 1) <= MAX_REG. ref_input_big_4bit_16group = input_big_tensor.clone().detach() - ref_out_big_4bit_16group = quantize_dequantize_ref(ref_input_big_4bit_16group, 4) + ref_out_big_4bit_16group = quantize_dequantize_ref(ref_input_big_4bit_16group, 4, 16) ds_out_big_4bit_16group = run_quant_dequant(input_big_tensor, 16, 4) assert (allclose(ds_out_big_4bit_16group, ref_out_big_4bit_16group)) From 9410fcdc570c3c104b682435e2cbef8883434350 Mon Sep 17 00:00:00 2001 From: GuanhuaWang Date: Mon, 12 Sep 2022 22:12:03 +0000 Subject: [PATCH 10/15] parameterize tensor shape --- tests/unit/ops/quantizer/test_quant.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/unit/ops/quantizer/test_quant.py b/tests/unit/ops/quantizer/test_quant.py index 915bd8380f51..90540a142bdd 100644 --- a/tests/unit/ops/quantizer/test_quant.py +++ b/tests/unit/ops/quantizer/test_quant.py @@ -34,10 +34,12 @@ def run_quant_dequant(inputs, groups, bits): @pytest.mark.inference -def test_quant_dequant(): +@pytest.mark.parametrize("small_tensor_shape", [8, 8]) +@pytest.mark.parametrize("big_tensor_shape", [128, 256]) +def test_quant_dequant(small_tensor_shape, big_tensor_shape): - input_small_tensor = torch.rand(8, 8, dtype=torch.float16).cuda() - input_big_tensor = torch.rand(128, 256, dtype=torch.float16).cuda() + input_small_tensor = torch.rand((small_tensor_shape), dtype=torch.float16).cuda() + input_big_tensor = torch.rand((big_tensor_shape), dtype=torch.float16).cuda() # test 8bit quant/dequant on 8x8 small tensor partitioned in 1 group. ref_input_small_8bit_1group = input_small_tensor.clone().detach() From 1ebd819cdd0c18ca671b0bb74e8d84a96be9f3a5 Mon Sep 17 00:00:00 2001 From: GuanhuaWang Date: Mon, 12 Sep 2022 22:23:48 +0000 Subject: [PATCH 11/15] single param --- tests/unit/ops/quantizer/test_quant.py | 29 ++++++++++++-------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/tests/unit/ops/quantizer/test_quant.py b/tests/unit/ops/quantizer/test_quant.py index 90540a142bdd..837606776f44 100644 --- a/tests/unit/ops/quantizer/test_quant.py +++ b/tests/unit/ops/quantizer/test_quant.py @@ -34,23 +34,20 @@ def run_quant_dequant(inputs, groups, bits): @pytest.mark.inference -@pytest.mark.parametrize("small_tensor_shape", [8, 8]) -@pytest.mark.parametrize("big_tensor_shape", [128, 256]) -def test_quant_dequant(small_tensor_shape, big_tensor_shape): +@pytest.mark.parametrize("tensor_shape", [(8, 8), (128, 256)]) +def test_quant_dequant(tensor_shape): + input_tensor = torch.rand((tensor_shape), dtype=torch.float16).cuda() - input_small_tensor = torch.rand((small_tensor_shape), dtype=torch.float16).cuda() - input_big_tensor = torch.rand((big_tensor_shape), dtype=torch.float16).cuda() - - # test 8bit quant/dequant on 8x8 small tensor partitioned in 1 group. - ref_input_small_8bit_1group = input_small_tensor.clone().detach() - ref_out_small_8bit_1group = quantize_dequantize_ref(ref_input_small_8bit_1group, 8) + # test 8bit quant/dequant on tensor partitioned in 1 group. + ref_input_8bit_1group = input_tensor.clone().detach() + ref_out_8bit_1group = quantize_dequantize_ref(ref_input_8bit_1group, 8) # run_quant_dequant will do quantize then dequantize and return the dequantized value. - ds_out_small_8bit_1group = run_quant_dequant(input_small_tensor, 1, 8) - assert (allclose(ds_out_small_8bit_1group, ref_out_small_8bit_1group)) + ds_out_8bit_1group = run_quant_dequant(input_tensor, 1, 8) + assert (allclose(ds_out_8bit_1group, ref_out_8bit_1group)) - # test 4bit quant/dequant on 128x256 big tensor partitioned into 16 groups. + # test 4bit quant/dequant on tensor partitioned into 16 groups. # Note that we have an explicit boundary for groups as ((size / groups) - 1) / 4096 + 1) <= MAX_REG. - ref_input_big_4bit_16group = input_big_tensor.clone().detach() - ref_out_big_4bit_16group = quantize_dequantize_ref(ref_input_big_4bit_16group, 4, 16) - ds_out_big_4bit_16group = run_quant_dequant(input_big_tensor, 16, 4) - assert (allclose(ds_out_big_4bit_16group, ref_out_big_4bit_16group)) + ref_input_4bit_16group = input_tensor.clone().detach() + ref_out_4bit_16group = quantize_dequantize_ref(ref_input_4bit_16group, 4, 16) + ds_out_4bit_16group = run_quant_dequant(input_tensor, 16, 4) + assert (allclose(ds_out_4bit_16group, ref_out_4bit_16group)) From 9f6fa41b586c93953c2be64fe3935cb5f75151c6 Mon Sep 17 00:00:00 2001 From: GuanhuaWang Date: Mon, 12 Sep 2022 23:15:46 +0000 Subject: [PATCH 12/15] detach tensor --- tests/unit/ops/quantizer/test_quant.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unit/ops/quantizer/test_quant.py b/tests/unit/ops/quantizer/test_quant.py index 837606776f44..1f16d3bdba3f 100644 --- a/tests/unit/ops/quantizer/test_quant.py +++ b/tests/unit/ops/quantizer/test_quant.py @@ -40,14 +40,16 @@ def test_quant_dequant(tensor_shape): # test 8bit quant/dequant on tensor partitioned in 1 group. ref_input_8bit_1group = input_tensor.clone().detach() + ds_input_8bit_1group = input_tensor.clone().detach() ref_out_8bit_1group = quantize_dequantize_ref(ref_input_8bit_1group, 8) # run_quant_dequant will do quantize then dequantize and return the dequantized value. - ds_out_8bit_1group = run_quant_dequant(input_tensor, 1, 8) + ds_out_8bit_1group = run_quant_dequant(ds_input_8bit_1group, 1, 8) assert (allclose(ds_out_8bit_1group, ref_out_8bit_1group)) # test 4bit quant/dequant on tensor partitioned into 16 groups. # Note that we have an explicit boundary for groups as ((size / groups) - 1) / 4096 + 1) <= MAX_REG. ref_input_4bit_16group = input_tensor.clone().detach() + ds_input_4bit_16group = input_tensor.clone().detach() ref_out_4bit_16group = quantize_dequantize_ref(ref_input_4bit_16group, 4, 16) - ds_out_4bit_16group = run_quant_dequant(input_tensor, 16, 4) + ds_out_4bit_16group = run_quant_dequant(ds_input_4bit_16group, 16, 4) assert (allclose(ds_out_4bit_16group, ref_out_4bit_16group)) From 6bc80bb38ce77b36812a32b37443f46cc419df05 Mon Sep 17 00:00:00 2001 From: GuanhuaWang Date: Tue, 13 Sep 2022 00:11:57 +0000 Subject: [PATCH 13/15] remove -lcurand flag --- op_builder/quantizer.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/op_builder/quantizer.py b/op_builder/quantizer.py index 314f5a8ad44b..43bc5778ea20 100644 --- a/op_builder/quantizer.py +++ b/op_builder/quantizer.py @@ -20,9 +20,3 @@ def sources(self): def include_paths(self): return ['csrc/includes'] - - def extra_ldflags(self): - if not self.is_rocm_pytorch(): - return ['-lcurand'] - else: - return [] From 2cf46f198415d7894a90d14e495821720c404983 Mon Sep 17 00:00:00 2001 From: GuanhuaWang Date: Tue, 13 Sep 2022 01:04:48 +0000 Subject: [PATCH 14/15] add back -lcurand flag --- op_builder/quantizer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/op_builder/quantizer.py b/op_builder/quantizer.py index 43bc5778ea20..3555a95e3179 100644 --- a/op_builder/quantizer.py +++ b/op_builder/quantizer.py @@ -20,3 +20,6 @@ def sources(self): def include_paths(self): return ['csrc/includes'] + + def extra_ldflags(self): + return ['-lcurand'] From 2a5b3ad9adc6399383f236e820be4ecd50411acc Mon Sep 17 00:00:00 2001 From: GuanhuaWang Date: Wed, 14 Sep 2022 16:46:02 +0000 Subject: [PATCH 15/15] format --- tests/unit/ops/quantizer/test_quant.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/ops/quantizer/test_quant.py b/tests/unit/ops/quantizer/test_quant.py index 1f16d3bdba3f..ea6b35860873 100644 --- a/tests/unit/ops/quantizer/test_quant.py +++ b/tests/unit/ops/quantizer/test_quant.py @@ -28,6 +28,7 @@ def quantize_dequantize_ref(inputs, bit, num_groups=1): def run_quant_dequant(inputs, groups, bits): global quantizer_cuda_module + if quantizer_cuda_module is None: quantizer_cuda_module = op_builder.QuantizerBuilder().load() return quantizer_cuda_module.ds_quantize_fp16(inputs, groups, bits)