Go语言圣经-匿名函数习题2详解编程语言

package main 
 
import ( 
        "fmt" 
        "io" 
        "io/ioutil" 
        "links" 
        "log" 
        "net/http" 
        "net/url" 
        "os" 
) 
/* 
练习5.13： 修改crawl，使其能保存发现的页面，必要时，可以创建目录来保存这些页面。只保存来自原始域名下的页面。假设初始页面在golang.org下，就不要保存vimeo.com下的页面。 
*/ 
var sum int 
func main() { 
        breadthFirst(crawl, os.Args[1:]) 
} 
 
/* 
抓取页面的所有连接 
*/ 
func crawl(url string) []string { 
        sum++ 
 
        go save(url) 
        fmt.Printf("%d|%s/n", sum, url) 
        list, err := links.Extract(url) 
        if err != nil { 
                log.Print(err) 
        } 
        return list 
} 
 
/* 
保存页面到文件 
*/ 
func save(u string) bool { 
 
        urlObj, _ := url.Parse(u) 
        path := "/tmp/crawl/" + urlObj.Host 
        if urlObj.Path == "" || urlObj.Path == "/" { 
                urlObj.Path = "/index.html" 
        } 
        filename := path + urlObj.Path //重点注意文件名 
        fmt.Println(filename) 
        //打开文件 
        f, _ := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0755) 
        //读取链接 
        resp, geterr := http.Get(u) 
 
        if geterr != nil || resp.StatusCode != http.StatusOK { 
                //resp.Body.Close() 
                return false 
        } 
        body, _ := ioutil.ReadAll(resp.Body) 
        //fmt.Println(body) 
        //创建保存目录 
        _, err := os.Stat(path) 
        if err != nil { 
                os.MkdirAll(path, 0755) 
        } 
 
        io.WriteString(f, string(body)) 
        resp.Body.Close() 
        body = nil 
        return true 
} 
 
/* 
广度优先算法 
*/ 
// breadthFirst calls f for each item in the worklist. 
// Any items returned by f are added to the worklist. 
// f is called at most once for each item. 
func breadthFirst(f func(item string) []string, worklist []string) { 
        seen := make(map[string]bool) 
        for len(worklist) > 0 { 
                items := worklist 
                worklist = nil 
                for _, item := range items { 
                        if !seen[item] { 
                                seen[item] = true 
                                worklist = append(worklist, f(item)...) 
                        } 
                } 
        } 
}