What You'll Build
A small Go client for CRW built on the standard library — no SDK needed. CRW's HTTP API is Firecrawl-compatible and simple enough that net/http + encoding/json is all you need. You'll scrape a page, crawl a site with job polling, map URLs, run a web search, and batch-scrape with a worker pool.
Step 1: Run CRW
docker run -p 3000:3000 ghcr.io/us/crw:latest
curl -s http://localhost:3000/healthStep 2: Project Setup
mkdir crw-go && cd crw-go
go mod init example.com/crw-goStep 3: A Tiny Client (client.go)
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
)
const (
baseURL = "http://localhost:3000" // or https://api.fastcrw.com
apiKey = "fc-YOUR-KEY"
)
var httpClient = &http.Client{Timeout: 90 * time.Second}
// mustJSON marshals or panics — fine for request bodies you construct.
func mustJSON(v any) []byte {
b, err := json.Marshal(v)
if err != nil {
panic(err)
}
return b
}
func post(path string, body any) (map[string]any, error) {
buf, err := json.Marshal(body)
if err != nil {
return nil, err
}
req, err := http.NewRequest(http.MethodPost, baseURL+path, bytes.NewReader(buf))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+apiKey)
resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
data, _ := io.ReadAll(resp.Body)
if resp.StatusCode >= 300 {
return nil, fmt.Errorf("crw %d: %s", resp.StatusCode, string(data))
}
var out map[string]any
if err := json.Unmarshal(data, &out); err != nil {
return nil, err
}
return out, nil
}Step 4: Scrape a Page (scrape.go)
func Scrape(url string) (string, error) {
res, err := post("/v1/scrape", map[string]any{
"url": url,
"formats": []string{"markdown"},
"onlyMainContent": true,
})
if err != nil {
return "", err
}
data, _ := res["data"].(map[string]any)
md, _ := data["markdown"].(string)
return md, nil
}Step 5: Crawl With Job Polling (crawl.go)
Crawl is asynchronous: start a job, then poll until it completes.
func Crawl(url string, limit int) ([]any, error) {
start, err := post("/v1/crawl", map[string]any{
"url": url,
"limit": limit,
"scrapeOptions": map[string]any{
"formats": []string{"markdown"},
"onlyMainContent": true,
},
})
if err != nil {
return nil, err
}
jobID, _ := start["id"].(string)
for {
time.Sleep(2 * time.Second)
st, err := getJSON("/v1/crawl/" + jobID)
if err != nil {
return nil, err
}
status, _ := st["status"].(string)
if status == "completed" {
data, _ := st["data"].([]any)
return data, nil
}
if status == "failed" {
return nil, fmt.Errorf("crawl job failed")
}
fmt.Printf("crawl status: %s\n", status)
}
}
func getJSON(path string) (map[string]any, error) {
req, _ := http.NewRequest(http.MethodGet, baseURL+path, nil)
req.Header.Set("Authorization", "Bearer "+apiKey)
resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
b, _ := io.ReadAll(resp.Body)
var out map[string]any
return out, json.Unmarshal(b, &out)
}Step 6: Map and Search
func MapSite(url string) ([]any, error) {
res, err := post("/v1/map", map[string]any{"url": url})
if err != nil {
return nil, err
}
links, _ := res["links"].([]any)
return links, nil
}
func Search(query string, limit int) ([]any, error) {
res, err := post("/v1/search", map[string]any{
"query": query, "limit": limit,
})
if err != nil {
return nil, err
}
data, _ := res["data"].([]any)
return data, nil
}Step 7: Worker-Pool Batch Scrape
Go's goroutines make a bounded worker pool natural — this scrapes many URLs with a fixed number of concurrent workers:
type result struct {
URL string
Chars int
Err error
}
func BatchScrape(urls []string, workers int) []result {
jobs := make(chan string)
out := make(chan result)
for i := 0; i < workers; i++ {
go func() {
for u := range jobs {
md, err := Scrape(u)
out <- result{URL: u, Chars: len(md), Err: err}
}
}()
}
go func() {
for _, u := range urls {
jobs <- u
}
close(jobs)
}()
results := make([]result, 0, len(urls))
for range urls {
results = append(results, <-out)
}
return results
}Step 8: main.go
func main() {
md, err := Scrape("https://example.com")
if err != nil {
panic(err)
}
fmt.Println(md[:min(300, len(md))])
res := BatchScrape([]string{
"https://example.com",
"https://example.org",
"https://example.net",
}, 3)
for _, r := range res {
fmt.Printf("%s -> %d chars (err=%v)\n", r.URL, r.Chars, r.Err)
}
}
func min(a, b int) int {
if a < b {
return a
}
return b
}go run .Typed Responses Instead of map[string]any
The minimal client uses map[string]any to stay short, but production Go code should decode into structs. Typed decoding fails loudly on a schema drift instead of silently returning a zero value three layers deep, and it documents the contract in code:
type ScrapeResponse struct {
Success bool `json:"success"`
Data struct {
Markdown string `json:"markdown"`
Metadata struct {
Title string `json:"title"`
SourceURL string `json:"sourceURL"`
StatusCode int `json:"statusCode"`
} `json:"metadata"`
} `json:"data"`
Error string `json:"error"`
}
func ScrapeTyped(url string) (*ScrapeResponse, error) {
req, _ := http.NewRequest(http.MethodPost, baseURL+"/v1/scrape",
bytes.NewReader(mustJSON(map[string]any{
"url": url, "formats": []string{"markdown"},
"onlyMainContent": true,
})))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+apiKey)
resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
var out ScrapeResponse
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
return nil, fmt.Errorf("decode: %w", err)
}
if !out.Success {
return nil, fmt.Errorf("crw error: %s", out.Error)
}
return &out, nil
}This is the version you actually ship. The map[string]any form is fine for a spike, but a typed struct turns "why is this field empty" debugging sessions into compile-time and decode-time errors.
Context, Timeouts, and Cancellation
A Go service that calls out to the web must respect cancellation, or a client that hangs up leaves your goroutine fetching a page nobody wants. Always thread a context.Context through the request so an upstream deadline or a cancelled request tears down the scrape immediately:
func ScrapeCtx(ctx context.Context, url string) (string, error) {
body := mustJSON(map[string]any{
"url": url, "formats": []string{"markdown"},
})
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
baseURL+"/v1/scrape", bytes.NewReader(body))
if err != nil {
return "", err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+apiKey)
resp, err := httpClient.Do(req) // cancels with ctx
if err != nil {
return "", err
}
defer resp.Body.Close()
// ... decode as before
return "", nil
}With http.NewRequestWithContext, a cancelled context aborts the in-flight HTTP call rather than leaking it. In an HTTP handler, pass the request's context straight through; in a worker, derive one with context.WithTimeout so a single slow target cannot stall a queue indefinitely. This is idiomatic Go and it composes cleanly with CRW because the API is just HTTP — there is no SDK abstraction hiding the request from your cancellation logic.
Why Go + CRW Is a Lean Pairing
A common Go reason to avoid scraping is that the ecosystem's options pull in a headless browser, which means shipping Chromium and its memory profile alongside a language chosen partly for small static binaries. Offloading the fetch-and-clean step to CRW preserves what makes Go appealing: your service stays a single static binary with predictable memory, and the heavy, messy work of rendering and content extraction happens in a separate process you can scale independently. The two binaries — your Go service and CRW's ~6MB Rust engine — are each small, fast, and operationally boring, which is exactly the property you want in infrastructure.
Why CRW for Go Services
- No SDK lock-in — the HTTP API is plain JSON; the standard library is enough.
- Pairs with Go's strengths — a Go service plus CRW (open-core Rust, ~6MB binary) is a lean, fast stack with no Python or browser baggage.
- Same API self-host or cloud — change
baseURLtohttps://api.fastcrw.comand nothing else changes. Lower-latency, local-first, AGPL-3.0 + Managed Cloud.
Wrapping It in an HTTP Handler
A Go scraper usually lives behind a route. Here is the production shape — input validation, the request's context threaded through for cancellation, and honest status codes:
func scrapeHandler(w http.ResponseWriter, r *http.Request) {
url := r.URL.Query().Get("url")
if url == "" || (!strings.HasPrefix(url, "http://") &&
!strings.HasPrefix(url, "https://")) {
http.Error(w, "valid url required", http.StatusBadRequest)
return
}
ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
defer cancel()
md, err := ScrapeCtx(ctx, url)
if err != nil {
if ctx.Err() != nil { // client gone or timed out
http.Error(w, "scrape cancelled", http.StatusGatewayTimeout)
return
}
http.Error(w, err.Error(), http.StatusBadGateway)
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]string{
"url": url, "markdown": md,
})
}
func main() {
http.HandleFunc("/scrape", scrapeHandler)
http.ListenAndServe(":8080", nil)
}This composes cleanly precisely because CRW is plain HTTP: the request's context flows straight into the outbound call, so a client that disconnects cancels the scrape immediately and you do not leak a goroutine fetching a page nobody is waiting for. Validate at the edge, propagate cancellation, map failures to the right status — the same discipline you would apply to any upstream dependency, with no SDK layer in the way.
Next Steps
- Read Migrate a Firecrawl SDK Script to CRW
- See Deploy CRW on Kubernetes to run it next to your Go services
Self-host CRW from GitHub for free, or use fastCRW for managed cloud scraping.