fastchat finetuning chatglm3 error

Nobody has claimed this yet.

Traceback (most recent call last):
File "/mnt/project/Code/FastChat/fastchat/train/train_mem.py", line 13, in
train()
File "/mnt/project/Code/FastChat/fastchat/train/train.py", line 308, in train
trainer.train()
File "/home/anaconda3/envs/fschat/lib/python3.10/site-packages/transformers/trainer.py", line 1537, in train
return inner_training_loop(
File "/home/anaconda3/envs/fschat/lib/python3.10/site-packages/transformers/trainer.py", line 1821, in _inner_training_loop
for step, inputs in enumerate(epoch_iterator):
File "/home/anaconda3/envs/fschat/lib/python3.10/site-packages/accelerate/data_loader.py", line 448, in __iter__
current_batch = next(dataloader_iter)
File "/home/anaconda3/envs/fschat/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 630, in __next__
data = self._next_data()
File "/home/anaconda3/envs/fschat/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 674, in _next_data
data = self._dataset_fetcher.fetch(index) # may raise StopIteration
File "/home/anaconda3/envs/fschat/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 51, in fetch
data = [self.dataset[idx] for idx in possibly_batched_index]
File "/home/anaconda3/envs/fschat/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 51, in
data = [self.dataset[idx] for idx in possibly_batched_index]
File "/mnt/project/Code/FastChat/fastchat/train/train.py", line 224, in __getitem__
ret = preprocess([self.raw_data[i]["conversations"]], self.tokenizer)
File "/mnt/project/Code/FastChat/fastchat/train/train.py", line 114, in preprocess
input_ids = tokenizer(
File "/home/anaconda3/envs/fschat/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2802, in __call__
encodings = self._call_one(text=text, text_pair=text_pair, **all_kwargs)
File "/home/anaconda3/envs/fschat/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2888, in _call_one
return self.batch_encode_plus(
File "/home/anaconda3/envs/fschat/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 3079, in batch_encode_plus
return self._batch_encode_plus(
File "/home/anaconda3/envs/fschat/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 807, in _batch_encode_plus
batch_outputs = self._batch_prepare_for_model(
File "/home/anaconda3/envs/fschat/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 879, in _batch_prepare_for_model
batch_outputs = self.pad(
File "/home/anaconda3/envs/fschat/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 3286, in pad
outputs = self._pad(
File "/home/.cache/huggingface/modules/transformers_modules/chatglm3-6b/tokenization_chatglm.py", line 299, in _pad
assert self.padding_side == "left"
AssertionError

Start in fastchat/train/train.py, especially preprocess at line 114 and __getitem__ at line 224, then inspect the ChatGLM3 tokenizer's _pad implementation named in the traceback. Reproduce the finetuning failure and determine what change prevents the padding assertion while keeping preprocessing functional.

Written by the indexing model from the issue text.