From 1de131d88f64eb3ab1e03765c1569cf18babe41f Mon Sep 17 00:00:00 2001 From: Ammar Ahmad Awan Date: Wed, 31 Aug 2022 10:58:31 -0700 Subject: [PATCH 1/6] add a new unit test. --- .../transformer/inference/test_bias_gelu.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tests/unit/ops/transformer/inference/test_bias_gelu.py diff --git a/tests/unit/ops/transformer/inference/test_bias_gelu.py b/tests/unit/ops/transformer/inference/test_bias_gelu.py new file mode 100644 index 000000000000..bad14294b876 --- /dev/null +++ b/tests/unit/ops/transformer/inference/test_bias_gelu.py @@ -0,0 +1,46 @@ +import pytest +import torch +import deepspeed +from deepspeed.ops.op_builder import InferenceBuilder + +if not deepspeed.ops.__compatible_ops__[InferenceBuilder.NAME]: + pytest.skip("Inference ops are not available on this system", + allow_module_level=True) + +inference_module = InferenceBuilder().load() + + +def allclose(x, y): + assert x.dtype == y.dtype + rtol, atol = {torch.float32: (5e-4, 5e-5), torch.float16: (3e-2, 2e-3)}[x.dtype] + return torch.allclose(x, y, rtol=rtol, atol=atol) + + +def run_bias_gelu_reference(activations, bias): + # Expected behavior is that of casting to float32 internally and using the tanh approximation + return torch.nn.functional.gelu(activations.to(torch.float32) + + bias.to(torch.float32), + approximate='tanh').to(activations.dtype) + + +def run_bias_gelu_ds(activations, bias): + if activations.dtype == torch.float16: + return inference_module.bias_gelu_fp16(activations, bias) + else: + return inference_module.bias_gelu_fp32(activations, bias) + + +@pytest.mark.parametrize("batch", [1, 2]) +@pytest.mark.parametrize("sequence", [1, 128, 255]) +@pytest.mark.parametrize("channels", [512, 1232, 4096]) +@pytest.mark.parametrize("dtype", [torch.float16, torch.float32]) +def test_bias_gelu(batch, sequence, channels, dtype): + activations_ds = torch.randn((batch, sequence, channels), dtype=dtype, device='cuda') + bias_ds = torch.randn((channels), dtype=dtype, device='cuda') + + activations_ref = activations_ds.clone().detach() + bias_ref = bias_ds.clone().detach() + + ds_out = run_bias_gelu_ds(activations_ds, bias_ds) + ref_out = run_bias_gelu_reference(activations_ref, bias_ref) + assert (allclose(ds_out, ref_out)) From ff7e8abb7976a2eb8e5dce5ace978e65199ad616 Mon Sep 17 00:00:00 2001 From: Ammar Ahmad Awan Date: Wed, 31 Aug 2022 13:04:44 -0700 Subject: [PATCH 2/6] load module only when used. --- tests/unit/ops/transformer/inference/test_bias_gelu.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/ops/transformer/inference/test_bias_gelu.py b/tests/unit/ops/transformer/inference/test_bias_gelu.py index bad14294b876..df82ae0c17ce 100644 --- a/tests/unit/ops/transformer/inference/test_bias_gelu.py +++ b/tests/unit/ops/transformer/inference/test_bias_gelu.py @@ -7,7 +7,7 @@ pytest.skip("Inference ops are not available on this system", allow_module_level=True) -inference_module = InferenceBuilder().load() +inference_module = None def allclose(x, y): @@ -24,6 +24,9 @@ def run_bias_gelu_reference(activations, bias): def run_bias_gelu_ds(activations, bias): + global inference_module + if inference_module is None: + inference_module = InferenceBuilder().load() if activations.dtype == torch.float16: return inference_module.bias_gelu_fp16(activations, bias) else: From fa79b20634808cd7b6119239784bbc5c712cf8f1 Mon Sep 17 00:00:00 2001 From: cmikeh2 Date: Wed, 31 Aug 2022 21:33:52 +0000 Subject: [PATCH 3/6] Test PyTorch version for appropriate gelu --- .../ops/transformer/inference/test_bias_gelu.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/unit/ops/transformer/inference/test_bias_gelu.py b/tests/unit/ops/transformer/inference/test_bias_gelu.py index df82ae0c17ce..6289e4492ba7 100644 --- a/tests/unit/ops/transformer/inference/test_bias_gelu.py +++ b/tests/unit/ops/transformer/inference/test_bias_gelu.py @@ -8,6 +8,7 @@ allow_module_level=True) inference_module = None +torch_minor_version = None def allclose(x, y): @@ -16,11 +17,21 @@ def allclose(x, y): return torch.allclose(x, y, rtol=rtol, atol=atol) +def version_appropriate_gelu(activations): + global torch_minor_version + if torch_minor_version is None: + torch_minor_version = int(torch.__version__.split('.')[1]) + # If torch version = 1.12 + if torch_minor_version < 12: + return torch.nn.functional.gelu(activations) + else: + return torch.nn.functional.gelu(activations, approximate='tanh') + + def run_bias_gelu_reference(activations, bias): # Expected behavior is that of casting to float32 internally and using the tanh approximation - return torch.nn.functional.gelu(activations.to(torch.float32) + - bias.to(torch.float32), - approximate='tanh').to(activations.dtype) + return version_appropriate_gelu( + activations.to(torch.float32) + bias.to(torch.float32)).to(activations.dtype) def run_bias_gelu_ds(activations, bias): From 2a790f8168aa57e317b522195e0c289fbff86dc7 Mon Sep 17 00:00:00 2001 From: Ammar Ahmad Awan Date: Wed, 31 Aug 2022 15:04:31 -0700 Subject: [PATCH 4/6] mark test as an inference test. --- tests/unit/ops/transformer/inference/test_bias_gelu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/ops/transformer/inference/test_bias_gelu.py b/tests/unit/ops/transformer/inference/test_bias_gelu.py index 6289e4492ba7..39d889dc028d 100644 --- a/tests/unit/ops/transformer/inference/test_bias_gelu.py +++ b/tests/unit/ops/transformer/inference/test_bias_gelu.py @@ -10,7 +10,6 @@ inference_module = None torch_minor_version = None - def allclose(x, y): assert x.dtype == y.dtype rtol, atol = {torch.float32: (5e-4, 5e-5), torch.float16: (3e-2, 2e-3)}[x.dtype] @@ -44,6 +43,7 @@ def run_bias_gelu_ds(activations, bias): return inference_module.bias_gelu_fp32(activations, bias) +@pytest.mark.inference @pytest.mark.parametrize("batch", [1, 2]) @pytest.mark.parametrize("sequence", [1, 128, 255]) @pytest.mark.parametrize("channels", [512, 1232, 4096]) From 4cb88250bc5aa7b0bfb559b696b8c5da18f08c0d Mon Sep 17 00:00:00 2001 From: Ammar Ahmad Awan Date: Wed, 31 Aug 2022 15:05:16 -0700 Subject: [PATCH 5/6] remove forked --- .github/workflows/nv-inference.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nv-inference.yml b/.github/workflows/nv-inference.yml index 5d588f08f3e9..681fc90be78f 100644 --- a/.github/workflows/nv-inference.yml +++ b/.github/workflows/nv-inference.yml @@ -61,4 +61,4 @@ jobs: if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi cd tests EXPECTED_TORCH=$(pip index versions torch | grep -oP -m1 "^\s*LATEST.*\s\K\d+\.\d+") - TRANSFORMERS_CACHE=/blob/transformers_cache/ TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --forked -n 4 --verbose -m 'inference' unit/ --torch_ver=$EXPECTED_TORCH --cuda_ver="11.3" + TRANSFORMERS_CACHE=/blob/transformers_cache/ TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 -n 4 --verbose -m 'inference' unit/ --torch_ver=$EXPECTED_TORCH --cuda_ver="11.3" From bba3da0dce96d2c7d86c6a870415a440f65986e2 Mon Sep 17 00:00:00 2001 From: Ammar Ahmad Awan Date: Wed, 31 Aug 2022 15:06:41 -0700 Subject: [PATCH 6/6] fix format --- tests/unit/ops/transformer/inference/test_bias_gelu.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/ops/transformer/inference/test_bias_gelu.py b/tests/unit/ops/transformer/inference/test_bias_gelu.py index 39d889dc028d..773ea6556462 100644 --- a/tests/unit/ops/transformer/inference/test_bias_gelu.py +++ b/tests/unit/ops/transformer/inference/test_bias_gelu.py @@ -10,6 +10,7 @@ inference_module = None torch_minor_version = None + def allclose(x, y): assert x.dtype == y.dtype rtol, atol = {torch.float32: (5e-4, 5e-5), torch.float16: (3e-2, 2e-3)}[x.dtype]