Vicuna v1.5 giving wrong repsones in a different language when trying to do a vanila inference

Nobody has claimed this yet.


I am not able to understand what is wrong in the code I wrote since its based on the same prompt template given in the repo

```
import torch
import transformers
from transformers import (
AutoModelForCausalLM,
AutoTokenizer,
set_seed,
Trainer,
BitsAndBytesConfig,
DataCollatorForLanguageModeling,
TrainingArguments,
AutoConfig,
pipeline,
)
from peft import LoraConfig, PeftModel, get_peft_model, prepare_model_for_kbit_training
import torch.nn as nn
from trl import SFTTrainer
from datasets import load_dataset, DatasetDict

import os

DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
DEFAULT_SYSTEM_PROMPT = """A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. \n"""
MODEL_NAME = "lmsys/vicuna-13b-v1.5"

def remove_substring(string, substring):
return string.replace(substring, "")

def remove_trailing_newlines(text):
return text.rstrip("\n")

def cut_off_text(text, prompt):
cutoff_phrase = prompt
index = text.find(cutoff_phrase)
if index != -1:
return text[:index]
else:
return text

def generate_findings_prompt(finding, prompt=DEFAULT_SYSTEM_PROMPT):
return prompt + " USER: " + finding + " ASSISTANT: "

def generate_code_from_prompt(model, tokenizer, input, prompt=DEFAULT_SYSTEM_PROMPT):
prompt = generate_findings_prompt(input, prompt)

inputs = tokenizer([prompt])
inputs = {k: torch.tensor(v).to(DEVICE) for k, v in inputs.items()}
output_ids = model.generate(
**inputs,
do_sample=True,
temperature=0.7,
repetition_penalty=1.0,
max_new_tokens=512,
)

generated_text = tokenizer.decode(
output_ids[0],
skip_special_tokens=True,
spaces_between_special_tokens=False,
clean_up_tokenization_spaces=True,
)
# prompt = remove_substring(prompt, "")
generated_text = remove_substring(generated_text, prompt)
generated_text = remove_trailing_newlines(generated_text)
return generated_text

def chat_with_llms():
print(
"vicuna Chatbot Initialized. Ask a question based on a medical report or type 'exit' to end the chat."
)
tokenizer = AutoTokenizer.from_pretrained(
MODEL_NAME,
)
tokenizer.pad_token = tokenizer.eos_token
tokenizer.padding_side = "right"

model = AutoModelForCausalLM.from_pretrained(
MODEL_NAME,
torch_dtype=torch.bfloat16,
low_cpu_mem_usage=True,
device_map="auto",
)
while True:
user_input = input("USER: ")
if user_input.lower() == "exit":
print("\n Vicuna Chatbot: Goodbye!")
break
response = generate_code_from_prompt(model, tokenizer, user_input)
print(f"Vicuna Chatbot: {response} \n \n")

if __name__ == "__main__":
chat_with_llms()
```

Start with the prompt construction in generate_findings_prompt and generation flow in generate_code_from_prompt, then compare them with the repository's Vicuna prompt template. Reproduce the behavior with chat_with_llms and a minimal vanilla inference; done means Vicuna produces responses in the requested language.

Written by the indexing model from the issue text.