diff --git a/olive/data/component/text_generation.py b/olive/data/component/text_generation.py
index 0934f977d2..508f423608 100644
--- a/olive/data/component/text_generation.py
+++ b/olive/data/component/text_generation.py
@@ -168,7 +168,7 @@ def text_gen_corpus_pre_process(_dataset, tokenizer, all_kwargs):
             # no randomization, just use contiguous blocks of tokens
             if args.corpus_strategy == TextGenCorpusStrategy.JOIN_SLIDING_WINDOW:
                 # we use the stride as both the step between sequences and the context size
-                step, context = args.stride, args.stride
+                step, context = args.stride, seqlen - args.stride
             else:
                 # JOIN corpus_strategy
                 # text is split into non-overlapping sequences and there is no context