Александър обнови решението на 03.01.2017 14:41 (преди над 1 година)
+package main
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "sync"
+)
+
+type fileReader struct {
+ err error
+ mut sync.Mutex
+ dataWrittenCond *sync.Cond
+ data []byte
+ readSize int
+ filledSize int
+}
+
+func (f *fileReader) Read(p []byte) (int, error) {
+ f.mut.Lock()
+ defer f.mut.Unlock()
+ if f.err != nil {
+ return 0, f.err
+ }
+
+ if len(f.data) == 0 {
+ // we still doesn't have valid size so we should wait
+ f.dataWrittenCond.Wait()
+ }
+
+ if f.readSize == len(f.data) {
+ if f.err != nil {
+ return 0, f.err
+ }
+ return 0, io.EOF
+ }
+
+ size := len(p)
+ leftSize := f.filledSize - f.readSize
+ for leftSize == 0 {
+ if f.err != nil {
+ return 0, f.err
+ }
+ f.dataWrittenCond.Wait()
+ leftSize = f.filledSize - f.readSize
+ }
+
+ if size > leftSize {
+ size = leftSize
+ }
+
+ copy(p, f.data[f.readSize:f.readSize+size])
+ f.readSize += size
+
+ return size, f.err
+}
+
+func DownloadFile(ctx context.Context, urls []string) io.Reader {
+ result := &fileReader{}
+ result.dataWrittenCond = sync.NewCond(&result.mut)
+ if len(urls) == 0 {
+ result.err = fmt.Errorf("no valid urls")
+ return result
+ }
+
+ go func() {
+ //TODO: test code remove me
+ transport := &http.Transport{}
+ client := http.Client{Transport: transport}
+
+ validUrls := make([]string, 0, len(urls))
+ var fileSize int64 = -1
+ // found out the file size
+ for _, url := range urls {
+ response, err := client.Head(url)
+ if err == nil && response.StatusCode < 400 {
+ validUrls = append(validUrls, url)
+ fileSize = response.ContentLength
+ }
+ }
+
+ if len(validUrls) == 0 {
+ result.mut.Lock()
+ result.err = fmt.Errorf("no valid urls")
+ result.mut.Unlock()
+ result.dataWrittenCond.Signal()
+ return
+ }
+
+ if fileSize <= 0 {
+ result.mut.Lock()
+ result.err = io.EOF
+ result.mut.Unlock()
+ result.dataWrittenCond.Signal()
+ return
+ }
+
+ // set the max data
+ result.mut.Lock()
+ result.data = make([]byte, fileSize)
+ result.mut.Unlock()
+
+ numUrls := int64(len(validUrls))
+
+ type fileRange struct {
+ start int64
+ end int64
+ urlToDelete string
+ }
+
+ rangeChan := make(chan fileRange, 1)
+ rangeChan <- fileRange{0, fileSize, ""}
+
+ for ran := range rangeChan {
+ if ran.urlToDelete != "" {
+ for i, url := range validUrls {
+ if url == ran.urlToDelete {
+ validUrls = append(validUrls[:i], validUrls[i+1:len(validUrls)]...)
+ break
+ }
+ }
+ }
+ numUrls = int64(len(validUrls))
+ if numUrls == 0 {
+ close(rangeChan)
+ result.mut.Lock()
+ result.err = fmt.Errorf("no valid urls")
+ result.mut.Unlock()
+ result.dataWrittenCond.Signal()
+ return
+ }
+ chunkSize := ran.end - ran.start
+ fileChunk := chunkSize / numUrls
+ extraBytes := chunkSize - (fileChunk * numUrls)
+ startByte := ran.start
+ for _, url := range validUrls {
+ requestSize := fileChunk
+ if extraBytes > 0 {
+ requestSize++
+ extraBytes--
+ }
+ go func(startIndex, size int64, url string) {
+ request, _ := http.NewRequest("GET", url, nil)
+ request.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", startIndex, startIndex+size-1))
+
+ for {
+ response, err := client.Do(request)
+ if err != nil || response.StatusCode >= 400 {
+ rangeChan <- fileRange{startIndex, startIndex + size, url}
+ return
+ }
+
+ if response.StatusCode == 200 || response.StatusCode == 206 {
+ d, _ := ioutil.ReadAll(response.Body)
+ response.Body.Close()
+ result.mut.Lock()
+ for int64(result.filledSize) != startIndex {
+ result.dataWrittenCond.Wait()
+ }
+
+ copy(result.data[startIndex:], d[:])
+ result.filledSize += len(d)
+ result.mut.Unlock()
+ result.dataWrittenCond.Broadcast()
+ if int64(result.filledSize) == fileSize {
+ close(rangeChan)
+ }
+
+ if int64(len(d)) != size {
+ startIndex += int64(len(d))
+ size -= int64(len(d))
+ request.Header.Del("Range")
+ request.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", startIndex, startIndex+size-1))
+ } else {
+ break
+ }
+ }
+ }
+ }(startByte, requestSize, url)
+
+ startByte += requestSize
+ }
+ }
+ }()
+
+ return result
+}