telos_agent/tools/builtin/
web_fetch.rs

1use async_trait::async_trait;
2use serde_json::{Value, json};
3use std::collections::HashMap;
4use std::sync::Mutex;
5use std::time::{Duration, Instant};
6
7use crate::error::AgentError;
8use crate::tools::api::{Tool, ToolContext, ToolDefinition, ToolOutput};
9
10/// Tool that fetches a URL and returns its content as plain text.
11///
12/// HTTP URLs are automatically upgraded to HTTPS. Cross-host redirects are
13/// reported back to the caller. Results are cached per URL for 15 minutes.
14pub struct WebFetchTool {
15    cache: Mutex<HashMap<String, (String, Instant)>>,
16}
17
18impl WebFetchTool {
19    pub fn new() -> Self {
20        Self { cache: Mutex::new(HashMap::new()) }
21    }
22}
23
24impl Default for WebFetchTool {
25    fn default() -> Self {
26        Self::new()
27    }
28}
29
30#[async_trait]
31impl Tool for WebFetchTool {
32    fn definition(&self) -> ToolDefinition {
33        ToolDefinition {
34            name: "WebFetch".into(),
35            description: "Fetch a URL and convert to text. HTTP upgraded to HTTPS. Results cached for 15 min per URL.".into(),
36            input_schema: json!({"type":"object","properties":{"url":{"type":"string"}},"required":["url"]}),
37        }
38    }
39
40    fn prompt_text(&self) -> Option<&'static str> {
41        Some(
42            "Use WebFetch to retrieve a specific public URL. Only fetch public `http`/`https` URLs. \
43Verify that returned content is relevant and trustworthy before acting on it.",
44        )
45    }
46
47    fn is_concurrency_safe(&self, _: &Value) -> bool {
48        true
49    }
50
51    async fn invoke(&self, args: Value, _: ToolContext) -> Result<ToolOutput, AgentError> {
52        let url = args
53            .get("url")
54            .and_then(|v| v.as_str())
55            .ok_or_else(|| AgentError::Validation("missing url".into()))?;
56
57        // Check cache
58        {
59            let cache = self.cache.lock().unwrap();
60            if let Some((content, time)) = cache.get(url)
61                && time.elapsed() < Duration::from_secs(900)
62            {
63                // 15 minutes
64                return Ok(ToolOutput::text(format!("[cached]\n{content}")));
65            }
66        }
67
68        let url = normalize_fetch_url(url);
69        let body = fetch_url_text(&url).await?;
70        let text = strip_html(&body);
71        let truncated: String = text.chars().take(50000).collect();
72
73        // Cache and return
74        {
75            self.cache.lock().unwrap().insert(url, (truncated.clone(), Instant::now()));
76        }
77
78        Ok(ToolOutput::text(truncated))
79    }
80}
81
82async fn fetch_url_text(url: &str) -> Result<String, AgentError> {
83    let client = reqwest::Client::builder()
84        .timeout(Duration::from_secs(30))
85        .user_agent("telos-agent/0.1")
86        .build()
87        .map_err(|err| AgentError::ToolExecution {
88            tool: "WebFetch".into(),
89            message: format!("failed to create HTTP client: {err}"),
90        })?;
91    let response = client.get(url).send().await.map_err(|err| AgentError::ToolExecution {
92        tool: "WebFetch".into(),
93        message: format!("HTTP request failed: {err}"),
94    })?;
95    let status = response.status();
96    if !status.is_success() {
97        return Err(AgentError::ToolExecution {
98            tool: "WebFetch".into(),
99            message: format!("HTTP request returned status {status}"),
100        });
101    }
102    response.text().await.map_err(|err| AgentError::ToolExecution {
103        tool: "WebFetch".into(),
104        message: format!("failed to read HTTP response body: {err}"),
105    })
106}
107
108fn strip_html(html: &str) -> String {
109    let mut in_tag = false;
110    let mut in_script = false;
111    let mut in_style = false;
112    let mut result = String::new();
113    let chars: Vec<char> = html.chars().collect();
114    let mut i = 0;
115    while i < chars.len() {
116        let c = chars[i];
117        if c == '<' {
118            // Check for script/style tags to skip their content
119            let tag_lower: String = chars
120                .iter()
121                .skip(i + 1)
122                .take_while(|&&ch| ch != '>' && ch != ' ')
123                .map(|&ch| ch.to_ascii_lowercase())
124                .collect();
125            if tag_lower == "script" {
126                in_script = true;
127            } else if tag_lower == "style" {
128                in_style = true;
129            } else if tag_lower.starts_with("/script") {
130                in_script = false;
131            } else if tag_lower.starts_with("/style") {
132                in_style = false;
133            }
134
135            if !in_script && !in_style {
136                // Track whether this is a block-level tag for spacing
137                let lower = tag_lower.to_lowercase();
138                if matches!(
139                    lower.as_str(),
140                    "br" | "p"
141                        | "div"
142                        | "tr"
143                        | "li"
144                        | "h1"
145                        | "h2"
146                        | "h3"
147                        | "h4"
148                        | "h5"
149                        | "h6"
150                        | "blockquote"
151                        | "hr"
152                        | "pre"
153                        | "/p"
154                        | "/div"
155                        | "/tr"
156                        | "/li"
157                        | "/h1"
158                        | "/h2"
159                        | "/h3"
160                        | "/h4"
161                        | "/h5"
162                        | "/h6"
163                        | "/blockquote"
164                        | "/pre"
165                        | "table"
166                        | "/table"
167                        | "/ol"
168                        | "/ul"
169                        | "ol"
170                        | "ul"
171                        | "/title"
172                        | "title"
173                ) && !result.is_empty()
174                    && !result.ends_with('\n')
175                {
176                    result.push('\n');
177                }
178            }
179            in_tag = true;
180        } else if c == '>' {
181            in_tag = false;
182        } else if !in_tag && !in_script && !in_style {
183            result.push(c);
184        }
185        i += 1;
186    }
187
188    // Collapse multiple whitespace characters into a single space
189    let mut collapsed = String::with_capacity(result.len());
190    let mut prev_space = false;
191    for c in result.chars() {
192        if c.is_whitespace() && c != '\n' {
193            if !prev_space {
194                collapsed.push(' ');
195                prev_space = true;
196            }
197        } else {
198            collapsed.push(c);
199            prev_space = false;
200        }
201    }
202
203    // Trim each line
204    collapsed
205        .lines()
206        .map(|line| line.trim())
207        .filter(|line| !line.is_empty())
208        .collect::<Vec<_>>()
209        .join("\n")
210}
211
212fn normalize_fetch_url(url: &str) -> String {
213    let Ok(parsed) = url::Url::parse(url) else {
214        return url.to_string();
215    };
216    if parsed.scheme() != "http" {
217        return url.to_string();
218    }
219    if parsed.host_str().is_some_and(is_loopback_host) {
220        return url.to_string();
221    }
222    url.replacen("http://", "https://", 1)
223}
224
225fn is_loopback_host(host: &str) -> bool {
226    host.eq_ignore_ascii_case("localhost") || host == "127.0.0.1" || host == "::1"
227}
telos_agent/tools/builtin/web_fetch.rs

telos_agent/tools/builtin/
web_fetch.rs
