From a5b7bccc893e9b898cabcafee77926f7ede1de38 Mon Sep 17 00:00:00 2001 From: Jambay Kinley Date: Tue, 19 Sep 2023 01:05:07 +0000 Subject: [PATCH] use correct context --- olive/data/component/text_generation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/olive/data/component/text_generation.py b/olive/data/component/text_generation.py index 0934f977d2..508f423608 100644 --- a/olive/data/component/text_generation.py +++ b/olive/data/component/text_generation.py @@ -168,7 +168,7 @@ def text_gen_corpus_pre_process(_dataset, tokenizer, all_kwargs): # no randomization, just use contiguous blocks of tokens if args.corpus_strategy == TextGenCorpusStrategy.JOIN_SLIDING_WINDOW: # we use the stride as both the step between sequences and the context size - step, context = args.stride, args.stride + step, context = args.stride, seqlen - args.stride else: # JOIN corpus_strategy # text is split into non-overlapping sequences and there is no context