새로운 버전의 에이전트로 백테스트 실행하기

import getpass
import os

# 테스트할 프로젝트 이름을 설정합니다
project_name = "Tweet Writing Task"
os.environ["LANGSMITH_PROJECT"] = project_name
os.environ["LANGSMITH_TRACING"] = "true"

if not os.environ.get("LANGSMITH_API_KEY"):
    os.environ["LANGSMITH_API_KEY"] = getpass.getpass("YOUR API KEY")

# 선택 사항. OpenAI를 다른 도구 호출 채팅 모델로 변경할 수 있습니다.
os.environ["OPENAI_API_KEY"] = "YOUR OPENAI API KEY"

# 선택 사항. 원한다면 Tavily를 무료 DuckDuckGo 검색 도구로 변경할 수 있습니다.
# Tavily API 키 받기: https://tavily.com
os.environ["TAVILY_API_KEY"] = "YOUR TAVILY API KEY"

from langchain.chat_models import init_chat_model
from langchain.agents import create_agent
from langchain_community.tools import DuckDuckGoSearchRun, TavilySearchResults
from langchain_core.rate_limiters import InMemoryRateLimiter


# GPT-3.5 Turbo를 기준선으로 사용하고 GPT-4o와 비교할 것입니다
gpt_3_5_turbo = init_chat_model(
    "gpt-3.5-turbo",
    temperature=1,
    configurable_fields=("model", "model_provider"),
)

# 지시사항은 시스템 메시지로 에이전트에 전달됩니다
instructions = """You are a tweet writing assistant. Given a topic, do some research and write a relevant and engaging tweet about it.
- Use at least 3 emojis in each tweet
- The tweet should be no longer than 280 characters
- Always use the search tool to gather recent information on the tweet topic
- Write the tweet only based on the search content. Do not rely on your internal knowledge
- When relevant, link to your sources
- Make your tweet as engaging as possible"""

# 에이전트가 사용할 도구를 정의합니다
# 더 높은 등급의 Tavily API 플랜이 있다면 이 값을 늘릴 수 있습니다
rate_limiter = InMemoryRateLimiter(requests_per_second=0.08)

# Tavily API 키가 없다면 DuckDuckGo를 사용하세요:
# tools = [DuckDuckGoSearchRun(rate_limiter=rate_limiter)]
tools = [TavilySearchResults(max_results=5, rate_limiter=rate_limiter)]

agent = create_agent(gpt_3_5_turbo, tools=tools, system_prompt=instructions)

fake_production_inputs = [
    "Alan turing's early childhood",
    "Economic impacts of the European Union",
    "Underrated philosophers",
    "History of the Roxie theater in San Francisco",
    "ELI5: gravitational waves",
    "The arguments for and against a parliamentary system",
    "Pivotal moments in music history",
    "Big ideas in programming languages",
    "Big questions in biology",
    "The relationship between math and reality",
    "What makes someone funny",
]

agent.batch(
    [{"messages": [{"role": "user", "content": content}]} for content in fake_production_inputs],
)

from datetime import datetime, timedelta, timezone
from uuid import uuid4
from langsmith import Client
from langsmith.beta import convert_runs_to_test

# 데이터셋/실험으로 변환할 실행을 가져옵니다
client = Client()

# 데이터셋에 포함할 실행을 샘플링하는 방법
end_time = datetime.now(tz=timezone.utc)
start_time = end_time - timedelta(days=1)
run_filter = f'and(gt(start_time, "{start_time.isoformat()}"), lt(end_time, "{end_time.isoformat()}"))'
prod_runs = list(
    client.list_runs(
        project_name=project_name,
        is_root=True,
        filter=run_filter,
    )
)

# 생성할 데이터셋 이름
dataset_name = f'{project_name}-backtesting {start_time.strftime("%Y-%m-%d")}-{end_time.strftime("%Y-%m-%d")}'
# 과거 실행에서 생성할 실험 이름
baseline_experiment_name = f"prod-baseline-gpt-3.5-turbo-{str(uuid4())[:4]}"

# 실행을 데이터셋 + 실험으로 변환합니다
convert_runs_to_test(
    prod_runs,
    # 결과 데이터셋의 이름
    dataset_name=dataset_name,
    # 실행 출력을 참조/정답으로 포함할지 여부
    include_outputs=False,
    # 결과 실험에 전체 트레이스를 포함할지 여부
    # (기본값은 루트 실행만 포함)
    load_child_runs=True,
    # 나중에 평가자를 적용할 수 있도록 실험 이름 지정
    test_project_name=baseline_experiment_name
)

import emoji
from pydantic import BaseModel, Field
from langchain_core.messages import convert_to_openai_messages

class Grade(BaseModel):
    """응답이 컨텍스트에 의해 뒷받침되는지 평가합니다."""
    grounded: bool = Field(..., description="Is the majority of the response supported by the retrieved context?")

grounded_instructions = f"""You have given somebody some contextual information and asked them to write a statement grounded in that context.

Grade whether their response is fully supported by the context you have provided. \
If any meaningful part of their statement is not backed up directly by the context you provided, then their response is not grounded. \
Otherwise it is grounded."""
grounded_model = init_chat_model(model="gpt-4o").with_structured_output(Grade)

def lt_280_chars(outputs: dict) -> bool:
    messages = convert_to_openai_messages(outputs["messages"])
    return len(messages[-1]['content']) <= 280

def gte_3_emojis(outputs: dict) -> bool:
    messages = convert_to_openai_messages(outputs["messages"])
    return len(emoji.emoji_list(messages[-1]['content'])) >= 3

async def is_grounded(outputs: dict) -> bool:
    context = ""
    messages = convert_to_openai_messages(outputs["messages"])
    for message in messages:
        if message["role"] == "tool":
            # 도구 메시지 출력은 Tavily/DuckDuckGo 도구에서 반환된 결과입니다
            context += "\n\n" + message["content"]
    tweet = messages[-1]["content"]
    user = f"""CONTEXT PROVIDED:
    {context}

    RESPONSE GIVEN:
    {tweet}"""
    grade = await grounded_model.ainvoke([
        {"role": "system", "content": grounded_instructions},
        {"role": "user", "content": user}
    ])
    return grade.grounded

baseline_results = await client.aevaluate(
    baseline_experiment_name,
    evaluators=[lt_280_chars, gte_3_emojis, is_grounded],
)
# pandas가 설치되어 있다면 결과를 DataFrame으로 쉽게 탐색할 수 있습니다:
# baseline_results.to_pandas()

candidate_results = await client.aevaluate(
    agent.with_config(model="gpt-4o"),
    data=dataset_name,
    evaluators=[lt_280_chars, gte_3_emojis, is_grounded],
    experiment_prefix="candidate-gpt-4o",
)
# pandas가 설치되어 있다면 결과를 DataFrame으로 쉽게 탐색할 수 있습니다:
# candidate_results.to_pandas()