Google Search (Rob Pike)
Build a concurrent search engine step by step — from sequential to replicated with timeouts. Based on Rob Pike's 2012 Google I/O talk.
package main
import (
"fmt"
"math/rand"
"time"
)
type Result string
type Search func(query string) Result
func fakeSearch(kind string) Search {
return func(query string) Result {
time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond)
return Result(fmt.Sprintf("%s result for %q", kind, query))
}
}
// --- 1.0: Sequential ---
func Google1(query string) []Result {
return []Result{
Web(query),
Image(query),
Video(query),
}
}
// --- 2.0: Concurrent fan-out, fan-in ---
func Google2(query string) []Result {
c := make(chan Result)
go func() { c <- Web(query) }()
go func() { c <- Image(query) }()
go func() { c <- Video(query) }()
var results []Result
for i := 0; i < 3; i++ {
results = append(results, <-c)
}
return results
}
// --- 2.1: Add timeout ---
func Google21(query string) []Result {
c := make(chan Result)
go func() { c <- Web(query) }()
go func() { c <- Image(query) }()
go func() { c <- Video(query) }()
timeout := time.After(80 * time.Millisecond)
var results []Result
for i := 0; i < 3; i++ {
select {
case r := <-c:
results = append(results, r)
case <-timeout:
fmt.Println("timed out")
return results
}
}
return results
}
// --- 3.0: Replicated + timeout ---
// Send to multiple replicas, take the fastest response.
func First(query string, replicas ...Search) Result {
c := make(chan Result)
for i := range replicas {
go func(idx int) { c <- replicas[idx](query) }(i)
}
return <-c
}
var (
Web = fakeSearch("web")
Web2 = fakeSearch("web")
Image = fakeSearch("image")
Image2 = fakeSearch("image")
Video = fakeSearch("video")
Video2 = fakeSearch("video")
)
func Google3(query string) []Result {
c := make(chan Result)
go func() { c <- First(query, Web, Web2) }()
go func() { c <- First(query, Image, Image2) }()
go func() { c <- First(query, Video, Video2) }()
timeout := time.After(80 * time.Millisecond)
var results []Result
for i := 0; i < 3; i++ {
select {
case r := <-c:
results = append(results, r)
case <-timeout:
fmt.Println("timed out")
return results
}
}
return results
}
func main() {
rand.Seed(time.Now().UnixNano())
for _, search := range []struct {
name string
fn func(string) []Result
}{
{"1.0 Sequential", Google1},
{"2.0 Concurrent", Google2},
{"2.1 With Timeout", Google21},
{"3.0 Replicated", Google3},
} {
start := time.Now()
results := search.fn("golang")
fmt.Printf("\n--- %s (%v) ---\n", search.name, time.Since(start))
for _, r := range results {
fmt.Println(" ", r)
}
}
}