SageMaker AI 엔드포인트에 호스팅된 LLM을 위한 Strands 에이전트용 커스텀 모델 공급자 구축

# Install Yeoman globally
npm install -g yo
# Clone and install ml-container-creator
git clone https://github.com/awslabs/ml-container-creator
cd ml-container-creator
npm install && npm link

# Run the generator
yo ml-container-creator
# Configuration: Framework=transformers, Model Server=sglang, Model=meta-llama/Llama-3.1-8B-Instruct

def stream(self, messages: List[Dict[str, Any]], tool_specs: list, system_prompt: Optional[str], **kwargs):
    # ... (중략)
    payload = {
        "messages": payload_messages,
        "stream": True
    }
    response = self.runtime_client.invoke_endpoint_with_response_stream(
        EndpointName=self.endpoint_name,
        Body=json.dumps(payload)
    )
    for event in response['Body']:
        chunk = event['PayloadPart']['Bytes'].decode('utf-8')
        # SSE 포맷 파싱 및 Strands 포맷으로 yield
        if 'content' in delta:
            yield {
                "type": "contentBlockDelta",
                "delta": {"text": delta['content']},
                "contentBlockIndex": 0
            }

from strands.agent import Agent

provider = LlamaModelProvider(
    endpoint_name="llama-31-deployment-endpoint",
    region_name="us-east-1"
)

agent = Agent(
    name="llama-assistant",
    model=provider,
    system_prompt="You are a helpful AI assistant powered by Llama 3.1."
)

response = agent("What are the key benefits of deploying LLMs on SageMaker?")