Agent Tool Discovery
Why This Matters
Tool discovery is what transforms an agent from a static script into an adaptive problem-solver. Instead of hardcoding which tool to use for each situation, agents that can discover tools dynamically can handle novel tasks, adapt to new tool ecosystems, and even create new tools on the fly. This is the key to building agents that improve over time and handle the unexpected.
Real-World Analogy: Imagine a mechanic with a toolbox. A beginner mechanic has a fixed routine—use the wrench for bolts, the screwdriver for screws. An expert mechanic looks at the problem, considers all available tools, and sometimes creates a custom solution. Tool discovery gives agents that expert-level adaptability.
Tool Discovery Architecture
Tool Discovery Engine
import asyncio
import hashlib
import time
import logging
from dataclasses import dataclass, field
from typing import Any, Callable, Coroutine, Optional
from enum import Enum
import json
logger = logging.getLogger(__name__)
class ToolCapability(Enum):
READ = "read"
WRITE = "write"
COMPUTE = "compute"
NETWORK = "network"
ANALYZE = "analyze"
@dataclass
class ToolSignature:
name: str
description: str
parameters: dict[str, dict]
return_type: str
capabilities: list[ToolCapability]
examples: list[dict] = field(default_factory=list)
tags: list[str] = field(default_factory=list)
@dataclass
class ToolUsageRecord:
tool_name: str
task_description: str
parameters: dict
success: bool
duration: float
error_message: Optional[str] = None
timestamp: float = 0.0
class ToolDiscoveryEngine:
def __init__(self):
self.tools: dict[str, ToolSignature] = {}
self.usage_history: list[ToolUsageRecord] = []
self.success_rates: dict[str, float] = {}
self.task_tool_mapping: dict[str, list[str]] = {}
def register_tool(self, signature: ToolSignature, handler: Callable):
self.tools[signature.name] = signature
self.success_rates[signature.name] = 1.0
logger.info(f"Registered tool: {signature.name}")
async def discover_tools(
self,
task_description: str,
required_capabilities: list[ToolCapability] = None,
max_tools: int = 5,
) -> list[ToolSignature]:
candidates = []
for name, sig in self.tools.items():
score = self._compute_relevance_score(sig, task_description)
if required_capabilities:
cap_match = len(set(sig.capabilities) & set(required_capabilities))
score *= (cap_match / len(required_capabilities))
success_bonus = self.success_rates.get(name, 0.5)
score *= (0.5 + 0.5 * success_bonus)
candidates.append((sig, score))
candidates.sort(key=lambda x: x[1], reverse=True)
return [sig for sig, _ in candidates[:max_tools]]
def _compute_relevance_score(self, sig: ToolSignature, task: str) -> float:
task_words = set(task.lower().split())
desc_words = set(sig.description.lower().split())
tag_words = set(tag.lower() for tag in sig.tags)
word_overlap = len(task_words & (desc_words | tag_words))
word_score = word_overlap / max(len(task_words), 1)
example_score = 0
for example in sig.examples:
example_desc = example.get("description", "").lower()
if any(word in example_desc for word in task_words):
example_score += 0.2
return word_score + min(example_score, 0.3)
async def record_usage(self, record: ToolUsageRecord):
self.usage_history.append(record)
self._update_success_rate(record)
self._update_task_mapping(record)
def _update_success_rate(self, record: ToolUsageRecord):
tool_records = [r for r in self.usage_history if r.tool_name == record.tool_name]
if tool_records:
successes = sum(1 for r in tool_records if r.success)
self.success_rates[record.tool_name] = successes / len(tool_records)
def _update_task_mapping(self, record: ToolUsageRecord):
task_key = record.task_description[:50].lower()
if task_key not in self.task_tool_mapping:
self.task_tool_mapping[task_key] = []
if record.tool_name not in self.task_tool_mapping[task_key]:
self.task_tool_mapping[task_key].append(record.tool_name)
def get_tool_analytics(self) -> dict:
return {
"total_tools": len(self.tools),
"total_calls": len(self.usage_history),
"success_rates": self.success_rates.copy(),
"most_used": self._get_most_used_tools(),
"least_successful": self._get_least_successful(),
}
def _get_most_used_tools(self, top_n: int = 5) -> list[dict]:
tool_counts = {}
for record in self.usage_history:
tool_counts[record.tool_name] = tool_counts.get(record.tool_name, 0) + 1
sorted_tools = sorted(tool_counts.items(), key=lambda x: x[1], reverse=True)
return [{"tool": t, "count": c} for t, c in sorted_tools[:top_n]]
def _get_least_successful(self, top_n: int = 5) -> list[dict]:
sorted_rates = sorted(self.success_rates.items(), key=lambda x: x[1])
return [{"tool": t, "rate": r} for t, r in sorted_rates[:top_n]]
Dynamic Tool Selector
from dataclasses import dataclass
from typing import Any, Callable
import asyncio
import json
import logging
logger = logging.getLogger(__name__)
@dataclass
class ToolCandidate:
name: str
handler: Callable
signature: dict
confidence: float
estimated_cost: float = 0.0
estimated_latency: float = 1.0
class DynamicToolSelector:
def __init__(self, llm_client=None):
self.llm_client = llm_client
self.tool_cache: dict[str, list[ToolCandidate]] = {}
self.selection_history: list[dict] = []
async def select_tools(
self,
task: str,
available_tools: list[dict],
budget: float = None,
max_latency: float = None,
) -> list[ToolCandidate]:
if self.llm_client:
return await self._llm_based_selection(task, available_tools)
return await self._heuristic_selection(
task, available_tools, budget, max_latency
)
async def _llm_based_selection(
self, task: str, tools: list[dict]
) -> list[ToolCandidate]:
tool_descriptions = "\n".join([
f"- {t['name']}: {t['description']}" for t in tools
])
prompt = f"""Given the task: "{task}"
Available tools:
{tool_descriptions}
Select the most appropriate tools (as a JSON list of tool names, ordered by relevance)."""
response = await self.llm_client.complete(prompt)
try:
selected_names = json.loads(response)
candidates = []
for name in selected_names:
tool = next((t for t in tools if t["name"] == name), None)
if tool:
candidates.append(ToolCandidate(
name=tool["name"],
handler=tool.get("handler"),
signature=tool,
confidence=0.8,
))
return candidates
except json.JSONDecodeError:
return await self._heuristic_selection(task, tools, None, None)
async def _heuristic_selection(
self,
task: str,
tools: list[dict],
budget: float,
max_latency: float,
) -> list[ToolCandidate]:
candidates = []
task_words = set(task.lower().split())
for tool in tools:
desc_words = set(tool.get("description", "").lower().split())
overlap = len(task_words & desc_words)
confidence = overlap / max(len(task_words), 1)
cost = tool.get("estimated_cost", 0)
latency = tool.get("estimated_latency", 1)
if budget and cost > budget:
continue
if max_latency and latency > max_latency:
continue
candidates.append(ToolCandidate(
name=tool["name"],
handler=tool.get("handler"),
signature=tool,
confidence=confidence,
estimated_cost=cost,
estimated_latency=latency,
))
candidates.sort(key=lambda c: c.confidence, reverse=True)
return candidates[:5]
Tool Composition System
from dataclasses import dataclass, field
from typing import Any, Callable
import asyncio
import logging
logger = logging.getLogger(__name__)
@dataclass
class ToolChain:
name: str
steps: list[dict]
description: str = ""
estimated_cost: float = 0.0
estimated_latency: float = 0.0
class ToolComposer:
def __init__(self):
self.tools: dict[str, Callable] = {}
self.compositions: dict[str, ToolChain] = {}
def register_tool(self, name: str, handler: Callable):
self.tools[name] = handler
async def compose(
self,
task: str,
tool_sequence: list[dict],
name: str = None,
) -> Any:
chain = ToolChain(
name=name or f"chain_{len(self.compositions)}",
steps=tool_sequence,
)
self.compositions[chain.name] = chain
context = {"task": task}
for step in tool_sequence:
tool_name = step["tool"]
params = step.get("params", {})
if callable(params):
params = params(context)
handler = self.tools.get(tool_name)
if not handler:
raise ValueError(f"Tool not found: {tool_name}")
result = await handler(**params)
context[step.get("output_key", tool_name)] = result
return context
async def optimize_chain(
self,
chain: ToolChain,
test_cases: list[dict],
) -> ToolChain:
results = []
for test in test_cases:
try:
result = await self.compose(test["input"], chain.steps)
results.append({"success": True, "result": result, "test": test})
except Exception as e:
results.append({"success": False, "error": str(e), "test": test})
success_rate = sum(1 for r in results if r["success"]) / len(results)
logger.info(f"Chain '{chain.name}' success rate: {success_rate:.2%}")
return chain
Performance Considerations
| Method | Latency | Cost | Accuracy | Scalability |
|---|---|---|---|---|
| Keyword Match | Very Low | Low | Low | Very High |
| Semantic Match | Medium | Medium | High | High |
| LLM-based | High | High | Very High | Medium |
| Success Rate | Low | Low | Medium | High |
| Meta-Tools | High | High | Very High | Low |
Security Considerations
- Tool Validation: Validate tool signatures before execution to prevent code injection
- Sandboxing: Execute untrusted tools in isolated environments with limited permissions
- Rate Limiting: Limit tool call frequency to prevent abuse and resource exhaustion
- Input Sanitization: Sanitize parameters passed to tools to prevent injection attacks
- Audit Logging: Log all tool selections and executions for security monitoring
Mathematical Foundation
Tool Relevance Score:
Tool Selection Cost Function:
Bayesian Tool Selection:
Where is the likelihood of the tool succeeding on the task, and is the prior probability based on usage history.
Interview Questions
1. What is tool discovery and why is it important for agents?
Answer: Tool discovery is the ability of an agent to dynamically identify and select appropriate tools for a given task, rather than hardcoding tool selection. It's important because: 1) Agents face diverse tasks requiring different capabilities, 2) New tools can be added without modifying agent logic, 3) It enables adaptive behavior as tool ecosystems evolve, 4) It reduces manual configuration and improves scalability. Discovery uses semantic matching, parameter analysis, and historical performance to rank candidates.
2. How does semantic matching work for tool discovery?
Answer: Semantic matching compares the task description to tool descriptions using embedding similarity: 1) Encode task and tool descriptions into vector representations, 2) Compute cosine similarity between vectors, 3) Rank tools by similarity score. This captures meaning beyond keyword matching—for example, "find information" matches a "search" tool even without exact word overlap. Improvements: use domain-specific embeddings, include parameter descriptions, and incorporate usage context.
3. What are meta-tools and when should you use them?
Answer: Meta-tools are tools that operate on other tools: tool composers create new tools by combining existing ones, tool selectors dynamically choose tools for tasks, and tool creators generate new tools from descriptions. Use meta-tools when: 1) Tasks require complex tool combinations, 2) You need dynamic tool creation for novel tasks, 3) The tool ecosystem is too large for static selection, 4) You want agents to learn from usage patterns. Meta-tools add complexity but enable more adaptive agents.
4. How do you handle tool compatibility and dependencies?
Answer: Implement a dependency graph: 1) Declare input/output types for each tool, 2) Validate type compatibility at composition time, 3) Check for circular dependencies, 4) Handle optional vs. required parameters, 5) Support tool versioning for backward compatibility. At runtime: validate inputs before execution, handle missing dependencies gracefully, and provide clear error messages. For complex systems, use a service registry that tracks tool capabilities and compatibility matrices.
5. How would you implement learning from tool usage?
Answer: Collect usage data: 1) Record tool calls with task context, parameters, and outcomes, 2) Extract features from successful vs. failed calls, 3) Train a classifier to predict tool success given task features, 4) Update success rates and rankings based on outcomes, 5) A/B test selection strategies. Learning approaches: supervised learning from labeled examples, bandit algorithms for exploration/exploitation, and reinforcement learning for sequential tool selection. Regularly retrain models with new data.
6. What is the cold start problem in tool discovery?
Answer: The cold start problem occurs when new tools have no usage history, making it impossible to learn their effectiveness. Solutions: 1) Use metadata-based similarity to bootstrap, 2) Start with conservative exploration (try new tools occasionally), 3) Leverage tool documentation and examples, 4) Transfer learning from similar existing tools, 5) Human annotation of tool capabilities. Monitor new tools closely and update rankings as data accumulates.
7. How do you optimize tool selection for cost and latency?
Answer: Multi-objective optimization: 1) Define cost and latency budgets, 2) Filter tools exceeding constraints, 3) Rank remaining tools by relevance weighted by cost/latency, 4) Use Pareto optimization to find non-dominated solutions, 5) Implement caching for frequently used tools, 6) Prefer local tools over network calls when possible, 7) Batch independent tool calls. Monitor actual vs. predicted costs and adjust models. Consider user preferences—some tasks prioritize speed, others minimize cost.
8. How would you design a self-improving tool discovery system?
Answer: Implement a feedback loop: 1) Track all tool selections and outcomes, 2) Analyze patterns in successful vs. failed selections, 3) Update selection models based on outcomes, 4) Experiment with new selection strategies, 5) Automatically retire tools with consistently poor performance, 6) Discover new tool combinations through exploration, 7) Generate reports on tool ecosystem health. Use multi-armed bandit algorithms to balance exploration of new tools with exploitation of known good ones.
Common Pitfalls
| Pitfall | Solution |
|---|---|
| Poor tool matching | Use semantic embeddings, not just keywords |
| Cold start for new tools | Bootstrap with metadata and exploration |
| Tool compatibility errors | Validate types at composition time |
| Stale success rates | Implement decay factors and periodic retraining |
| High latency from discovery | Cache results and pre-filter candidates |
| Over-reliance on historical data | Include exploration in selection strategy |
| Tool version incompatibility | Version tools and check compatibility |
| Resource exhaustion from meta-tools | Set limits on tool composition depth |
Summary with Key Takeaways
- Tool discovery enables agents to dynamically select appropriate tools based on task requirements
- Semantic matching uses embeddings to find tools by meaning, not just keywords
- Meta-tools create, compose, and optimize tools dynamically for complex tasks
- Success rate tracking provides empirical evidence for tool reliability
- Learning from usage improves selection over time through feedback loops
- Cost and latency optimization balances relevance with resource constraints
- Cold start is addressed through metadata bootstrapping and exploration
- Self-improving systems continuously learn and adapt their tool selection strategies
KnowledgeCheck
-
What is the primary benefit of semantic matching for tool discovery?
- a) It's faster than keyword matching
- b) It finds tools by meaning, not just exact words
- c) It requires no training data
- d) It always finds the best tool
-
What is a meta-tool?
- a) A tool that operates on other tools
- b) The most important tool in the registry
- c) A tool with maximum capabilities
- d) A backup tool for failures
-
How does the cold start problem affect new tools?
- a) They cannot be registered
- b) No historical data to assess their effectiveness
- c) They consume more resources
- d) They always fail
-
What is the purpose of tracking tool success rates?
- a) To increase tool costs
- b) To rank tools by reliability
- c) To reduce tool availability
- d) To simplify the codebase
-
How should tool selection balance relevance and cost?
- a) Always choose the cheapest tool
- b) Always choose the most relevant tool
- c) Use multi-objective optimization
- d) Random selection
-
What is tool composition?
- a) Creating new tools from existing ones
- b) Deleting unused tools
- c) Updating tool versions
- d) Monitoring tool usage
Answers: 1-b, 2-a, 3-b, 4-b, 5-c, 6-a