File size: 2,309 Bytes
3ddf4a0 5e289b7 3ddf4a0 5e289b7 3ddf4a0 5e289b7 3ddf4a0 5e289b7 3ddf4a0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 |
#run these two commands to get setup (adjust the string after serve to point to the folder with the model):
#pip install vllm
#vllm serve "/root/llama_3.1_8b_function_calling/checkpoint-669" --chat-template /root/template_chatml.jinja
from openai import OpenAI
openai_api_key = "EMPTY"
openai_api_base = "http://localhost:8000/v1"
client = OpenAI(
api_key=openai_api_key,
base_url=openai_api_base,
)
chat_response = client.chat.completions.create(
model="/root/llama_3.1_8b_function_calling/checkpoint-669",
messages=[
{"role": "system", "content": """You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. For each function call return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows:
<tool_call>
{"name": <function-name>,"arguments": <args-dict>}
</tool_call>
Here are the available tools:
<tools> {
"name": "create_new_user",
"description": "Creates a new user in the database with the provided username, email, and encrypted password.",
"parameters": {
"properties": {
"username": {
"type": "string",
"description": "The username for the new user."
},
"email": {
"type": "string",
"description": "The email address for the new user.",
"format": "email"
},
"password": {
"type": "string",
"description": "The password for the new user which will be encrypted before storage."
}
},
"required": [
"username",
"email",
"password"
]
},
"required": [
"username",
"email",
"password"
]
} </tools>"""},
{"role": "user", "content": "Hey, I need to create a new account for our project management system. Can you help me with that?"}]
# {"role": "assistant", "content": """Sure, I can help with that. I'll need some details from you. Can you provide the username, email, and password you'd like to use?"""},
# {"role": "user", "content": """Sure, here they are: username: johndoe, email: [email protected], password: password123"""}]
)
print("Chat response:", chat_response) |