Infino - LangChain LLM 监控示例 (Infino - LangChain LLM Monitoring Example)

该示例展示了在通过LangChain和Infino调用OpenAI模型时，如何跟踪以下内容：

• 提示输入 (prompt input)
• chatgpt或任何其他LangChain模型的响应 (response)
• 延迟 (latency)
• 错误 (errors)
• 消耗的令牌数量 (number of tokens consumed)

# 安装必要的依赖项。
pip install infinopy
pip install matplotlib

# 在Infino/LangChain集成的PR合并后，删除(1) import sys 和 sys.path.append(..) 和 (2) 取消注释 `!pip install langchain`。
import sys

sys.path.append("../../../../../langchain")
#!pip install langchain


import datetime as dt
from infinopy import InfinoClient
import json
from langchain.llms import OpenAI
from langchain.callbacks import InfinoCallbackHandler
import matplotlib.pyplot as plt
import matplotlib.dates as md
import os
import time
import sys

    Requirement already satisfied: matplotlib in /Users/vinaykakade/.pyenv/versions/3.10.11/lib/python3.10/site-packages (3.7.1)
    Requirement already satisfied: contourpy>=1.0.1 in /Users/vinaykakade/.pyenv/versions/3.10.11/lib/python3.10/site-packages (from matplotlib) (1.0.7)
    Requirement already satisfied: cycler>=0.10 in /Users/vinaykakade/.pyenv/versions/3.10.11/lib/python3.10/site-packages (from matplotlib) (0.11.0)
    Requirement already satisfied: fonttools>=4.22.0 in /Users/vinaykakade/.pyenv/versions/3.10.11/lib/python3.10/site-packages (from matplotlib) (4.39.4)
    Requirement already satisfied: kiwisolver>=1.0.1 in /Users/vinaykakade/.pyenv/versions/3.10.11/lib/python3.10/site-packages (from matplotlib) (1.4.4)
    Requirement already satisfied: numpy>=1.20 in /Users/vinaykakade/.pyenv/versions/3.10.11/lib/python3.10/site-packages (from matplotlib) (1.24.3)
    Requirement already satisfied: packaging>=20.0 in /Users/vinaykakade/.pyenv/versions/3.10.11/lib/python3.10/site-packages (from matplotlib) (23.1)
    Requirement already satisfied: pillow>=6.2.0 in /Users/vinaykakade/.pyenv/versions/3.10.11/lib/python3.10/site-packages (from matplotlib) (9.5.0)
    Requirement already satisfied: pyparsing>=2.3.1 in /Users/vinaykakade/.pyenv/versions/3.10.11/lib/python3.10/site-packages (from matplotlib) (3.0.9)
    Requirement already satisfied: python-dateutil>=2.7 in /Users/vinaykakade/.pyenv/versions/3.10.11/lib/python3.10/site-packages (from matplotlib) (2.8.2)
    Requirement already satisfied: six>=1.5 in /Users/vinaykakade/.pyenv/versions/3.10.11/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)
    Requirement already satisfied: infinopy in /Users/vinaykakade/.pyenv/versions/3.10.11/lib/python3.10/site-packages (0.0.1)
    Requirement already satisfied: docker in /Users/vinaykakade/.pyenv/versions/3.10.11/lib/python3.10/site-packages (from infinopy) (6.1.3)
    Requirement already satisfied: requests in /Users/vinaykakade/.pyenv/versions/3.10.11/lib/python3.10/site-packages (from infinopy) (2.31.0)
    Requirement already satisfied: packaging>=14.0 in /Users/vinaykakade/.pyenv/versions/3.10.11/lib/python3.10/site-packages (from docker->infinopy) (23.1)
    Requirement already satisfied: urllib3>=1.26.0 in /Users/vinaykakade/.pyenv/versions/3.10.11/lib/python3.10/site-packages (from docker->infinopy) (2.0.2)
    Requirement already satisfied: websocket-client>=0.32.0 in /Users/vinaykakade/.pyenv/versions/3.10.11/lib/python3.10/site-packages (from docker->infinopy) (1.5.2)
    Requirement already satisfied: charset-normalizer<4,>=2 in /Users/vinaykakade/.pyenv/versions/3.10.11/lib/python3.10/site-packages (from requests->infinopy) (3.1.0)
    Requirement already satisfied: idna<4,>=2.5 in /Users/vinaykakade/.pyenv/versions/3.10.11/lib/python3.10/site-packages (from requests->infinopy) (3.4)
    Requirement already satisfied: certifi>=2017.4.17 in /Users/vinaykakade/.pyenv/versions/3.10.11/lib/python3.10/site-packages (from requests->infinopy) (2023.5.7)

启动Infino服务器，初始化Infino客户端 (Start Infino server, initialize the Infino client)

# 使用Infino Docker镜像启动服务器。
docker run --rm --detach --name infino-example -p 3000:3000 infinohq/infino:latest

# 创建Infino客户端。
client = InfinoClient()

    497a621125800abdd19f57ce7e033349b3cf83ca8cea6a74e8e28433a42ecadd

读取问题数据集 (Read the questions dataset)

# 这些是从斯坦福大学的QA数据集中提取的一部分问题 -
# https://rajpurkar.github.io/SQuAD-explorer/
data = """诺曼底位于哪个国家？
诺曼人何时在诺曼底？
诺斯人来自哪些国家？
谁是诺斯人的领袖？
诺曼人何时首次获得独立身份？
在1000年和1100年期间，谁给了诺曼底他们的名字？
法国是哪个地区的一部分？
查理三世向谁宣誓效忠？
法兰克身份是在什么时候出现的？
在哈斯丁斯战役中，谁是公爵？
谁统治了诺曼底公国？
诺曼人信仰什么宗教？
诺曼王朝对现代欧洲产生了什么重大影响？
谁因其基督教精神而闻名？
谁吸收了罗马语言？
谁统治了诺曼底国家？
威廉征服者建立了哪个公国？
诺曼人这个词的原始含义是什么？
诺曼人这个词的拉丁语版本首次记录于何时？
哪个名称来自英文单词Normans/Normanz？
"""

questions = data.split("\n")

LangChain OpenAI Q&A; 将指标和日志发布到Infino

# 在这里设置你的密钥。
# os.environ["OPENAI_API_KEY"] = "YOUR_API_KEY"

# 创建回调处理程序。这将记录延迟、错误、令牌使用情况、提示以及Infino的提示响应。
handler = InfinoCallbackHandler(
    model_id="test_openai", model_version="0.1", verbose=False
)

# 创建LLM。
llm = OpenAI(temperature=0.1)

# 要向OpenAI模型提问的问题数量。在这里我们限制为一个较小的数字，以节省运行此演示时的费用。
num_questions = 10

questions = questions[0:num_questions]
for question in questions:
    print(question)

    # 我们将问题发送到OpenAI API，并使用Infino回调。
    llm_result = llm.generate([question], callbacks=[handler])
    print(llm_result)

    Normandy位于哪个国家？
    generations=[[Generation(text='\n\nNormandy is located in France.', generation_info={'finish_reason': 'stop', 'logprobs': None})]] llm_output={'token_usage': {'total_tokens': 16, 'completion_tokens': 9, 'prompt_tokens': 7}, 'model_name': 'text-davinci-003'} run=RunInfo(run_id=UUID('8de21639-acec-4bd1-a12d-8124de1e20da'))
    Normandy的诺曼人是在什么时候在诺曼底？
    generations=[[Generation(text='\n\nThe Normans first settled in Normandy in the late 9th century.', generation_info={'finish_reason': 'stop', 'logprobs': None})]] llm_output={'token_usage': {'total_tokens': 24, 'completion_tokens': 16, 'prompt_tokens': 8}, 'model_name': 'text-davinci-003'} run=RunInfo(run_id=UUID('cf81fc86-250b-4e6e-9d92-2df3bebb019a'))
    诺尔斯人来自哪些国家？
    generations=[[Generation(text='\n\nThe Norse originated from Scandinavia, which includes modern-day Norway, Sweden, and Denmark.', generation_info={'finish_reason': 'stop', 'logprobs': None})]] llm_output={'token_usage': {'total_tokens': 29, 'completion_tokens': 21, 'prompt_tokens': 8}, 'model_name': 'text-davinci-003'} run=RunInfo(run_id=UUID('50f42f5e-b4a4-411a-a049-f92cb573a74f'))
    诺尔斯领袖是谁？
    generations=[[Generation(text='\n\nThe most famous Norse leader was the legendary Viking king Ragnar Lodbrok. He is believed to have lived in the 9th century and is renowned for his exploits in England and France.', generation_info={'finish_reason': 'stop', 'logprobs': None})]] llm_output={'token_usage': {'total_tokens': 45, 'completion_tokens': 39, 'prompt_tokens': 6}, 'model_name': 'text-davinci-003'} run=RunInfo(run_id=UUID('e32f31cb-ddc9-4863-8e6e-cb7a281a0ada'))
    诺曼人何时首次获得独立身份？
    generations=[[Generation(text='\n\nThe Normans first gained their separate identity in the 11th century.', generation_info={'finish_reason': 'stop', 'logprobs': None})]] llm_output={'token_usage': {'total_tokens': 28, 'completion_tokens': 16, 'prompt_tokens': 12}, 'model_name': 'text-davinci-003'} run=RunInfo(run_id=UUID('da9d8f73-b3b3-4bc5-8495-da8b11462a51'))
    谁在1000年和1100年间将他们的名字赋予了诺曼底？
    generations=[[Generation(text='\n\nThe Normans, a people from northern France, gave their name to Normandy in the 1000s and 1100s. The Normans were descended from Viking settlers who had come to the region in the late 800s.', generation_info={'finish_reason': 'stop', 'logprobs': None})]] llm_output={'token_usage': {'total_tokens': 58, 'completion_tokens': 45, 'prompt_tokens': 13}, 'model_name': 'text-davinci-003'} run=RunInfo(run_id=UUID('bb5829bf-b6a6-4429-adfa-414ac5be46e5'))
    法国是哪个地区的一部分？
    generations=[[Generation(text='\n\nFrance is a region of Europe.', generation_info={'finish_reason': 'stop', 'logprobs': None})]] llm_output={'token_usage': {'total_tokens': 16, 'completion_tokens': 9, 'prompt_tokens': 7}, 'model_name': 'text-davinci-003'} run=RunInfo(run_id=UUID('6943880b-b4e4-4c74-9ca1-8c03c10f7e9c'))
    查尔斯三世向谁宣誓效忠？
    generations=[[Generation(text='\n\nKing Charles III swore fealty to Pope Innocent III.', generation_info={'finish_reason': 'stop', 'logprobs': None})]] llm_output={'token_usage': {'total_tokens': 23, 'completion_tokens': 13, 'prompt_tokens': 10}, 'model_name': 'text-davinci-003'} run=RunInfo(run_id=UUID('c91fd663-09e6-4d00-b746-4c7fd96f9ceb'))
    弗兰克身份是在什么时候出现的？
    generations=[[Generation(text='\n\nThe Frankish identity began to emerge in the late 5th century, when the Franks began to expand their power and influence in the region. The Franks were a Germanic tribe that had migrated to the area from the east and had established a kingdom in what is now modern-day France. The Franks were eventually able to establish a powerful kingdom that lasted until the 10th century.', generation_info={'finish_reason': 'stop', 'logprobs': None})]] llm_output={'token_usage': {'total_tokens': 86, 'completion_tokens': 78, 'prompt_tokens': 8}, 'model_name': 'text-davinci-003'} run=RunInfo(run_id=UUID('23f86775-e592-4cb8-baa3-46ebe74305b2'))
    哈斯廷斯战役中的公爵是谁？
    generations=[[Generation(text='\n\nThe Duke of Normandy, William the Conqueror, was the leader of the Norman forces at the Battle of Hastings in 1066.', generation_info={'finish_reason': 'stop', 'logprobs': None})]] llm_output={'token_usage': {'total_tokens': 39, 'completion_tokens': 28, 'prompt_tokens': 11}, 'model_name': 'text-davinci-003'} run=RunInfo(run_id=UUID('ad5b7984-8758-4d95-a5eb-ee56e0218f6b'))

创建度量图表 (Create Metric Charts)

我们现在使用matplotlib来创建延迟、错误和消耗的令牌的图表。

# 使用matplotlib创建图表的辅助函数
def plot(data, title):
    data = json.loads(data)

    # 从数据中提取x和y的值
    timestamps = [item["time"] for item in data]
    dates = [dt.datetime.fromtimestamp(ts) for ts in timestamps]
    y = [item["value"] for item in data]

    plt.rcParams["figure.figsize"] = [6, 4]
    plt.subplots_adjust(bottom=0.2)
    plt.xticks(rotation=25)
    ax = plt.gca()
    xfmt = md.DateFormatter("%Y-%m-%d %H:%M:%S")
    ax.xaxis.set_major_formatter(xfmt)

    # 创建图表
    plt.plot(dates, y)

    # 设置标签和标题
    plt.xlabel("时间")
    plt.ylabel("值")
    plt.title(title)

    plt.show()


response = client.search_ts("__name__", "latency", 0, int(time.time()))
plot(response.text, "延迟 (Latency)")

response = client.search_ts("__name__", "error", 0, int(time.time()))
plot(response.text, "错误 (Errors)")

response = client.search_ts("__name__", "prompt_tokens", 0, int(time.time()))
plot(response.text, "提示令牌 (Prompt Tokens)")

response = client.search_ts("__name__", "completion_tokens", 0, int(time.time()))
plot(response.text, "完成令牌 (Completion Tokens)")

response = client.search_ts("__name__", "total_tokens", 0, int(time.time()))
plot(response.text, "总令牌 (Total Tokens)")

在提示或提示输出上进行全文查询

# 搜索特定的提示文本。
query = "诺曼底"
response = client.search_log(query, 0, int(time.time()))
print("查询", query, "的结果:", response.text)

print("===")

query = "查理三世"
response = client.search_log("查理三世", 0, int(time.time()))
print("查询", query, "的结果:", response.text)

    查询 诺曼底 的结果: [{"time":1686821979,"fields":{"prompt":"诺曼底位于哪个国家？"},"text":"诺曼底位于哪个国家？"},{"time":1686821982,"fields":{"prompt_response":"\n\n诺曼底位于法国。"},"text":"\n\n诺曼底位于法国。"},{"time":1686821984,"fields":{"prompt_response":"\n\n诺曼底人在9世纪末首次定居在诺曼底。"},"text":"\n\n诺曼底人在9世纪末首次定居在诺曼底。"},{"time":1686821993,"fields":{"prompt":"在1000年和1100年，谁给了诺曼底他们的名字？"},"text":"在1000年和1100年，谁给了诺曼底他们的名字？"},{"time":1686821997,"fields":{"prompt_response":"\n\n诺曼底人是来自法国北部的人民，在1000年和1100年给了诺曼底他们的名字。诺曼底人是维京定居者的后裔，他们在9世纪末来到该地区。"},"text":"\n\n诺曼底人是来自法国北部的人民，在1000年和1100年给了诺曼底他们的名字。诺曼底人是维京定居者的后裔，他们在9世纪末来到该地区。"}]
    ===
    查询 查理三世 的结果: [{"time":1686821998,"fields":{"prompt":"查理三世向谁宣誓效忠？"},"text":"查理三世向谁宣誓效忠？"},{"time":1686822000,"fields":{"prompt_response":"\n\n查理三世向教皇无辜三世宣誓效忠。"},"text":"\n\n查理三世向教皇无辜三世宣誓效忠。"}]

步骤5: 停止infino服务器 (Step 5: Stop infino server)

docker rm -f infino-example

    infino-example