package builtin import ( "context" "encoding/json" "fmt" "io" "net/http" "net/url" "regexp" "strings" "time" "reasonix/internal/tool" ) func init() { tool.RegisterBuiltin(webFetch{}) } type webFetch struct{} const ( webFetchTimeout = 15 * time.Second webFetchMaxRead = 1 << 20 // 1 MiB cap before extraction ) func (webFetch) Name() string { return "web_fetch" } func (webFetch) Description() string { return "Fetch a URL over HTTPS/HTTP and return its text content. HTML pages are reduced to readable text (scripts, styles, tags stripped, whitespace collapsed); JSON / plain text / markdown bodies come back verbatim. Use to read documentation pages, API responses, or source files hosted somewhere the local filesystem can't reach." } func (webFetch) Schema() json.RawMessage { return json.RawMessage(`{ "type":"object", "properties":{ "url":{"type":"string","description":"Absolute URL beginning with http:// or https://"} }, "required":["url"] }`) } func (webFetch) ReadOnly() bool { return true } func (webFetch) Execute(ctx context.Context, args json.RawMessage) (string, error) { var p struct { URL string `json:"url"` } if err := json.Unmarshal(args, &p); err != nil { return "", fmt.Errorf("invalid args: %w", err) } if p.URL == "" { return "", fmt.Errorf("url is required") } u, err := url.Parse(p.URL) if err != nil || (u.Scheme != "http" && u.Scheme != "https") { return "", fmt.Errorf("url must be an absolute http(s) address") } reqCtx, cancel := context.WithTimeout(ctx, webFetchTimeout) defer cancel() req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, p.URL, nil) if err != nil { return "", fmt.Errorf("build request: %w", err) } // A plain UA + Accept tip the server toward returning text/HTML rather // than minified asset bundles or binary content. req.Header.Set("User-Agent", "reasonix-web-fetch/1.0") req.Header.Set("Accept", "text/html,text/plain,text/markdown,application/json,*/*;q=0.5") resp, err := http.DefaultClient.Do(req) if err != nil { return "", fmt.Errorf("fetch %s: %w", p.URL, err) } defer resp.Body.Close() body, err := io.ReadAll(io.LimitReader(resp.Body, webFetchMaxRead)) if err != nil { return "", fmt.Errorf("read body: %w", err) } ct := strings.ToLower(resp.Header.Get("Content-Type")) out := string(body) if strings.Contains(ct, "text/html") || looksLikeHTML(out) { out = htmlToText(out) } out = strings.TrimSpace(out) if out == "" { return fmt.Sprintf("(empty body — status %s)", resp.Status), nil } header := fmt.Sprintf("status %s · %s · %d bytes\n\n", resp.Status, contentTypeShort(ct), len(body)) return header + out, nil } // looksLikeHTML lets servers that misreport Content-Type still hit the HTML // reducer — GitHub raw pages and many docs sites lie about content type. func looksLikeHTML(s string) bool { head := s if len(head) > 512 { head = head[:512] } low := strings.ToLower(head) return strings.Contains(low, "]*>.*?`) htmlComment = regexp.MustCompile(`(?s)`) anyTag = regexp.MustCompile(`(?s)<[^>]+>`) multiBlank = regexp.MustCompile(`\n[\t ]*\n([\t ]*\n)+`) trailingWS = regexp.MustCompile(`[\t ]+\n`) ) // htmlToText strips