File size: 2,257 Bytes
5e289b7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 |
#vllm serve "/root/llama_3.1_8b_function_calling/checkpoint-669" --chat-template /root/template_chatml.jinja
from openai import OpenAI
# Set OpenAI's API key and API base to use vLLM's API server.
openai_api_key = "EMPTY"
openai_api_base = "http://localhost:8000/v1"
client = OpenAI(
api_key=openai_api_key,
base_url=openai_api_base,
)
chat_response = client.chat.completions.create(
model="/root/llama_3.1_8b_function_calling/checkpoint-669",
messages=[
# {"role": "system", "content": """You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. For each function call return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows:
# <tool_call>
# {"name": <function-name>,"arguments": <args-dict>}
# </tool_call>
# Here are the available tools:
# <tools> {
# "name": "create_new_user",
# "description": "Creates a new user in the database with the provided username, email, and encrypted password.",
# "parameters": {
# "properties": {
# "username": {
# "type": "string",
# "description": "The username for the new user."
# },
# "email": {
# "type": "string",
# "description": "The email address for the new user.",
# "format": "email"
# },
# "password": {
# "type": "string",
# "description": "The password for the new user which will be encrypted before storage."
# }
# },
# "required": [
# "username",
# "email",
# "password"
# ]
# },
# "required": [
# "username",
# "email",
# "password"
# ]
# } </tools>"""},
{"role": "user", "content": """Who's the president of the France?"""}]
# {"role": "assistant", "content": """Sure, I can help with that. I'll need some details from you. Can you provide the username, email, and password you'd like to use?"""},
# {"role": "user", "content": """Sure, here they are: username: johndoe, email: [email protected], password: password123"""}]
)
print("Chat response:", chat_response)
|