forked from abetlen/llama-cpp-python
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathserver.py
More file actions
59 lines (44 loc) · 1.28 KB
/
server.py
File metadata and controls
59 lines (44 loc) · 1.28 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
import gradio as gr
from openai import OpenAI
client = OpenAI(base_url="http://localhost:8000/v1", api_key="llama.cpp")
model = "gpt-3.5-turbo"
def predict(message, history):
messages = []
for user_message, assistant_message in history:
messages.append({"role": "user", "content": user_message})
messages.append({"role": "assistant", "content": assistant_message})
messages.append({"role": "user", "content": message})
response = client.chat.completions.create(
model=model, messages=messages, stream=True
)
text = ""
for chunk in response:
content = chunk.choices[0].delta.content
if content:
text += content
yield text
js = """function () {
gradioURL = window.location.href
if (!gradioURL.endsWith('?__theme=dark')) {
window.location.replace(gradioURL + '?__theme=dark');
}
}"""
css = """
footer {
visibility: hidden;
}
full-height {
height: 100%;
}
"""
with gr.Blocks(theme=gr.themes.Soft(), js=js, css=css, fill_height=True) as demo:
gr.ChatInterface(
predict,
fill_height=True,
examples=[
"What is the capital of France?",
"Who was the first person on the moon?",
],
)
if __name__ == "__main__":
demo.launch()