Програмиране с Go

Курс във Факултета по Математика и Информатика към СУ

Решение на HTTP сваляч от Добромир Иванов

Обратно към всички решения

Към профила на Добромир Иванов

Резултати

  • 1 точка от тестове
  • 0 бонус точки
  • 1 точка общо
  • 1 успешни тест(а)
  • 16 неуспешни тест(а)

Код

package main
import "io"
import "io/ioutil"
import "strconv"
import "errors"
import "context"
import "net/http"
type result struct {
bytes []byte
pos int
err error
}
type downloadRequest struct {
begin int
end int
url string
}
type downloadResult struct {
begin int
end int
body []byte
url string
urlInvalid bool
}
func (res *result) Read(p []byte) (n int, err error) {
i := 0
for ; res.pos < len(res.bytes) && i < len(p); i++ {
p[i] = res.bytes[res.pos]
res.pos++
}
if res.pos >= len(res.bytes) {
return i, io.EOF
}
return i, res.err
}
func filterAndGetFileSize(client *http.Client, urls []string) ([]string, int) {
result := make([]string, 0, len(urls))
size := 0
for _, url := range urls {
response, err := http.Head(url)
if err != nil {
continue
}
ioutil.ReadAll(response.Body)
response.Body.Close()
if response.StatusCode >= 400 {
continue
}
size = int(response.ContentLength)
result = append(result, url)
}
return result, size - 1
}
func download(limitChan chan struct{}, client *http.Client, request downloadRequest, resultChan chan<- downloadResult) {
if limitChan != nil {
limitChan <- struct{}{}
}
defer func() {
if limitChan != nil {
<-limitChan
}
}()
req, _ := http.NewRequest("GET", request.url, nil)
rangeHeader := strconv.Itoa(request.begin) + "-" + strconv.Itoa(request.end)
req.Header.Add("Range", rangeHeader)
resp, err := client.Do(req)
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
urlInvalid := false
if resp.StatusCode >= 400 || err != nil {
urlInvalid = true
}
resultChan <- downloadResult{request.begin, request.end, body, request.url, urlInvalid}
}
func downloader(ctx context.Context, client *http.Client, requestChan <-chan downloadRequest, resultChan chan downloadResult) {
limit := 0
var limitChan chan struct{} = nil
if ctx != nil && ctx.Value("max-connections") != nil {
limit = ctx.Value("max-connections").(int)
limitChan = make(chan struct{}, limit)
}
for request := range requestChan {
if ctx != nil {
select {
case <-ctx.Done():
return
default:
go download(limitChan, client, request, resultChan)
}
} else {
go download(limitChan, client, request, resultChan)
}
}
}
func balanceDownloads(begin, downloadSize int, urls []string, requestChan chan<- downloadRequest) {
end, originalSize := downloadSize/len(urls), downloadSize
for i := 0; i < len(urls) && begin <= originalSize; i++ {
byteCount := downloadSize / (len(urls) - i)
downloadSize -= byteCount
downloadRequest := downloadRequest{begin, end, urls[i]}
requestChan <- downloadRequest
begin = end + 1
end = begin + byteCount
}
}
func DownloadFile(ctx context.Context, urls []string) io.Reader {
client := &http.Client{Transport: &http.Transport{}}
urls, size := filterAndGetFileSize(client, urls)
requestChan := make(chan downloadRequest)
resultChan := make(chan downloadResult)
defer func() {
close(resultChan)
close(requestChan)
}()
if len(urls) == 0 {
return &result{[]byte{}, 0, errors.New("no valid urls.")}
} else if size <= 0 {
return &result{[]byte{}, 0, nil}
}
go downloader(ctx, client, requestChan, resultChan)
balanceDownloads(0, size, urls, requestChan)
bytesReceived := 0
content := make([]byte, 0)
for res := range resultChan {
bytesReceived += len(res.body)
content = append(content, res.body...)
expectedSize := res.end - res.begin + 1
if len(res.body) < expectedSize && !res.urlInvalid {
requestChan <- downloadRequest{len(res.body) + 1, res.end, res.url}
} else if res.urlInvalid {
for i, url := range urls {
if url == res.url {
urls = append(urls[:i], urls[i+1:]...)
}
}
if len(urls) == 0 {
return &result{content, 0, errors.New("no valid urls.")}
}
balanceDownloads(len(res.body)+1, size-len(res.body), urls, requestChan)
}
if bytesReceived >= size || len(urls) == 0 {
return &result{content, 0, nil}
}
}
return &result{content, 0, nil}
}

Лог от изпълнението

PASS
ok  	_/tmp/d20170109-30451-lnkl4u	0.008s
--- FAIL: TestSingleURLWithReturn (0.01s)
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x48 pc=0x4765fe]

goroutine 69 [running]:
panic(0x67ee20, 0xc4200120d0)
	/usr/local/go/src/runtime/panic.go:500 +0x1a1
_/tmp/d20170109-30451-lnkl4u.download(0x0, 0xc42006d050, 0x2, 0x24, 0xc4200d6700, 0x1c, 0xc420064d20)
	/tmp/d20170109-30451-lnkl4u/solution.go:84 +0x19e
created by _/tmp/d20170109-30451-lnkl4u.downloader
	/tmp/d20170109-30451-lnkl4u/solution.go:110 +0x1c5
exit status 2
FAIL	_/tmp/d20170109-30451-lnkl4u	0.021s
panic: test timed out after 1s

goroutine 18 [running]:
panic(0x66a0e0, 0xc4200d8030)
	/usr/local/go/src/runtime/panic.go:500 +0x1a1
testing.startAlarm.func1()
	/usr/local/go/src/testing/testing.go:918 +0x10b
created by time.goFunc
	/usr/local/go/src/time/sleep.go:154 +0x44

goroutine 1 [chan receive]:
testing.(*T).Run(0xc42007c0c0, 0x6d730a, 0x2a, 0x6f17f0, 0xc42004bd01)
	/usr/local/go/src/testing/testing.go:647 +0x316
testing.RunTests.func1(0xc42007c0c0)
	/usr/local/go/src/testing/testing.go:793 +0x6d
testing.tRunner(0xc42007c0c0, 0xc42004be20)
	/usr/local/go/src/testing/testing.go:610 +0x81
testing.RunTests(0x6f19f8, 0x805140, 0x11, 0x11, 0x7f8ba67f0000)
	/usr/local/go/src/testing/testing.go:799 +0x2f5
testing.(*M).Run(0xc42004bee8, 0x687fc0)
	/usr/local/go/src/testing/testing.go:743 +0x85
main.main()
	_/tmp/d20170109-30451-lnkl4u/_test/_testmain.go:86 +0xc6

goroutine 17 [syscall, locked to thread]:
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2086 +0x1

goroutine 6 [select]:
net/http.(*persistConn).roundTrip(0xc42000aa00, 0xc4200bea60, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:1840 +0x93b
net/http.(*Transport).RoundTrip(0xc4200c2000, 0xc4200c21e0, 0xc4200c2000, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:380 +0x4ee
net/http.send(0xc4200c21e0, 0x7e8ea0, 0xc4200c2000, 0x0, 0x0, 0x0, 0x8, 0xc42003b9b8, 0xc42002a060)
	/usr/local/go/src/net/http/client.go:256 +0x15f
net/http.(*Client).send(0x808100, 0xc4200c21e0, 0x0, 0x0, 0x0, 0xc42002a060, 0x0, 0x1)
	/usr/local/go/src/net/http/client.go:146 +0x102
net/http.(*Client).doFollowingRedirects(0x808100, 0xc4200c21e0, 0x6f1d70, 0x1c, 0x0, 0x0)
	/usr/local/go/src/net/http/client.go:528 +0x5e5
net/http.(*Client).Head(0x808100, 0xc4200be9c0, 0x1c, 0xc42000d1d0, 0x30, 0x30)
	/usr/local/go/src/net/http/client.go:635 +0x93
net/http.Head(0xc4200be9c0, 0x1c, 0x1, 0xc4200132f0, 0x0)
	/usr/local/go/src/net/http/client.go:619 +0x41
_/tmp/d20170109-30451-lnkl4u.filterAndGetFileSize(0xc42000d1d0, 0xc42004deb8, 0x1, 0x1, 0x7f8ba67f0000, 0x0, 0x18, 0xc4200be9a0)
	/tmp/d20170109-30451-lnkl4u/solution.go:50 +0xc4
_/tmp/d20170109-30451-lnkl4u.DownloadFile(0x7ed3a0, 0xc420012560, 0xc42004deb8, 0x1, 0x1, 0x0, 0x0)
	/tmp/d20170109-30451-lnkl4u/solution.go:134 +0x121
_/tmp/d20170109-30451-lnkl4u.TestSingleURLBlockUntilDownloadFileReturns(0xc42007c180)
	/tmp/d20170109-30451-lnkl4u/solution_test.go:135 +0x246
testing.tRunner(0xc42007c180, 0x6f17f0)
	/usr/local/go/src/testing/testing.go:610 +0x81
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:646 +0x2ec

goroutine 7 [IO wait]:
net.runtime_pollWait(0x7f8ba6795178, 0x72, 0x0)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420014370, 0x72, 0xc4200285e0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420014370, 0x7ea8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc420014310, 0x0, 0x7e94a0, 0xc4200bebc0)
	/usr/local/go/src/net/fd_unix.go:419 +0x238
net.(*TCPListener).accept(0xc42002a048, 0x43418e, 0xc420028690, 0x52d8cd)
	/usr/local/go/src/net/tcpsock_posix.go:132 +0x2e
net.(*TCPListener).Accept(0xc42002a048, 0x6f1bd8, 0xc42001a500, 0x7ed420, 0xc4200da060)
	/usr/local/go/src/net/tcpsock.go:222 +0x49
net/http.(*Server).Serve(0xc42001a300, 0x7ecba0, 0xc42002a048, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:2273 +0x1ce
net/http/httptest.(*Server).goServe.func1(0xc42005e4e0)
	/usr/local/go/src/net/http/httptest/server.go:235 +0x6d
created by net/http/httptest.(*Server).goServe
	/usr/local/go/src/net/http/httptest/server.go:236 +0x5c

goroutine 12 [chan receive]:
_/tmp/d20170109-30451-lnkl4u.TestSingleURLBlockUntilDownloadFileReturns.func1(0x7ecda0, 0xc420061ee0, 0xc4200c22d0)
	/tmp/d20170109-30451-lnkl4u/solution_test.go:107 +0x81
net/http.HandlerFunc.ServeHTTP(0xc42000d140, 0x7ecda0, 0xc420061ee0, 0xc4200c22d0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.serverHandler.ServeHTTP(0xc42001a300, 0x7ecda0, 0xc420061ee0, 0xc4200c22d0)
	/usr/local/go/src/net/http/server.go:2202 +0x7d
net/http.(*conn).serve(0xc42001a500, 0x7ed360, 0xc420018b40)
	/usr/local/go/src/net/http/server.go:1579 +0x4b7
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d

goroutine 10 [IO wait]:
net.runtime_pollWait(0x7f8ba67950b8, 0x72, 0x5)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420014450, 0x72, 0xc42003b9d0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420014450, 0x7ea8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4200143f0, 0xc420083000, 0x1000, 0x1000, 0x0, 0x7ea8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42002a068, 0xc420083000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc42000aa00, 0xc420083000, 0x1000, 0x1000, 0x53b963, 0xc420028b78, 0xc420028b88)
	/usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc42005ea20)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc42005ea20, 0x1, 0xc42005eba0, 0xc420028c60, 0xc420028bc8, 0x4106d8, 0x50)
	/usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc42000aa00)
	/usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 11 [select]:
net/http.(*persistConn).writeLoop(0xc42000aa00)
	/usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1063 +0x50e
exit status 2
FAIL	_/tmp/d20170109-30451-lnkl4u	1.017s
panic: test timed out after 1s

goroutine 4843 [running]:
panic(0x66a0e0, 0xc4203e1170)
	/usr/local/go/src/runtime/panic.go:500 +0x1a1
testing.startAlarm.func1()
	/usr/local/go/src/testing/testing.go:918 +0x10b
created by time.goFunc
	/usr/local/go/src/time/sleep.go:154 +0x44

goroutine 1 [chan receive]:
testing.(*T).Run(0xc42007c0c0, 0x6d8b5e, 0x32, 0x6f17f8, 0xc42004bd01)
	/usr/local/go/src/testing/testing.go:647 +0x316
testing.RunTests.func1(0xc42007c0c0)
	/usr/local/go/src/testing/testing.go:793 +0x6d
testing.tRunner(0xc42007c0c0, 0xc42011ce30)
	/usr/local/go/src/testing/testing.go:610 +0x81
testing.RunTests(0x6f19f8, 0x805140, 0x11, 0x11, 0x7f026e181000)
	/usr/local/go/src/testing/testing.go:799 +0x2f5
testing.(*M).Run(0xc42011cef8, 0x687fc0)
	/usr/local/go/src/testing/testing.go:743 +0x85
main.main()
	_/tmp/d20170109-30451-lnkl4u/_test/_testmain.go:86 +0xc6

goroutine 17 [syscall, locked to thread]:
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2086 +0x1

goroutine 6 [chan receive]:
_/tmp/d20170109-30451-lnkl4u.DownloadFile(0x7ed360, 0xc420018980, 0xc420013310, 0x1, 0x1, 0x0, 0x0)
	/tmp/d20170109-30451-lnkl4u/solution.go:155 +0x355
_/tmp/d20170109-30451-lnkl4u.TestSingleURLCancelContextAfterHalfBytesWereServed(0xc42007c180)
	/tmp/d20170109-30451-lnkl4u/solution_test.go:191 +0x296
testing.tRunner(0xc42007c180, 0x6f17f8)
	/usr/local/go/src/testing/testing.go:610 +0x81
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:646 +0x2ec

goroutine 7 [IO wait]:
net.runtime_pollWait(0x7f026e126178, 0x72, 0x0)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420014370, 0x72, 0xc4200285e0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420014370, 0x7ea8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc420014310, 0x0, 0x7e94a0, 0xc4200de3c0)
	/usr/local/go/src/net/fd_unix.go:419 +0x238
net.(*TCPListener).accept(0xc42002a048, 0x43418e, 0xc420028690, 0x52d8cd)
	/usr/local/go/src/net/tcpsock_posix.go:132 +0x2e
net.(*TCPListener).Accept(0xc42002a048, 0x6f1bd8, 0xc4200f6180, 0x7ed420, 0xc4200e00c0)
	/usr/local/go/src/net/tcpsock.go:222 +0x49
net/http.(*Server).Serve(0xc42001a300, 0x7ecba0, 0xc42002a048, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:2273 +0x1ce
net/http/httptest.(*Server).goServe.func1(0xc42005e4e0)
	/usr/local/go/src/net/http/httptest/server.go:235 +0x6d
created by net/http/httptest.(*Server).goServe
	/usr/local/go/src/net/http/httptest/server.go:236 +0x5c

goroutine 20 [IO wait]:
net.runtime_pollWait(0x7f026e125ff8, 0x72, 0x6)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4200f00d0, 0x72, 0xc4200377b0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4200f00d0, 0x7ea8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4200f0070, 0xc4200f8000, 0x1000, 0x1000, 0x0, 0x7ea8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4200e2008, 0xc4200f8000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
net/http.(*connReader).Read(0xc4200de120, 0xc4200f8000, 0x1000, 0x1000, 0xc420037918, 0x6d1e34, 0x19)
	/usr/local/go/src/net/http/server.go:586 +0x144
bufio.(*Reader).fill(0xc4200e61e0)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).ReadSlice(0xc4200e61e0, 0xa, 0x0, 0x1e, 0x6, 0x0, 0x0)
	/usr/local/go/src/bufio/bufio.go:330 +0xb5
bufio.(*Reader).ReadLine(0xc4200e61e0, 0xc4200fa0f0, 0xf0, 0xf0, 0x6c1e40, 0x4a03d3, 0x807d78)
	/usr/local/go/src/bufio/bufio.go:359 +0x37
net/textproto.(*Reader).readLineSlice(0xc4200e0180, 0xc420037aa8, 0xc420037aa8, 0x4106d8, 0xf0, 0x6c1e40)
	/usr/local/go/src/net/textproto/reader.go:55 +0x5e
net/textproto.(*Reader).ReadLine(0xc4200e0180, 0xc4200fa0f0, 0xc420037b20, 0x401863, 0xc420037c78)
	/usr/local/go/src/net/textproto/reader.go:36 +0x2f
net/http.readRequest(0xc4200e61e0, 0x0, 0xc4200fa0f0, 0x0, 0x0)
	/usr/local/go/src/net/http/request.go:793 +0xa5
net/http.(*conn).readRequest(0xc4200f6000, 0x7ed360, 0xc4200e42c0, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:765 +0x10d
net/http.(*conn).serve(0xc4200f6000, 0x7ed360, 0xc4200e42c0)
	/usr/local/go/src/net/http/server.go:1532 +0x3d3
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d

goroutine 18 [IO wait]:
net.runtime_pollWait(0x7f026e1260b8, 0x72, 0x5)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420014450, 0x72, 0xc42003b9d0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420014450, 0x7ea8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4200143f0, 0xc4200e8000, 0x1000, 0x1000, 0x0, 0x7ea8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4200e2000, 0xc4200e8000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc42000a900, 0xc4200e8000, 0x1000, 0x1000, 0x30, 0xc42003bb58, 0x43b23c)
	/usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4200e6000)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4200e6000, 0x1, 0x0, 0x1, 0x1, 0xc4200e6060, 0x0)
	/usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc42000a900)
	/usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 19 [select]:
net/http.(*persistConn).writeLoop(0xc42000a900)
	/usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 21 [chan receive]:
_/tmp/d20170109-30451-lnkl4u.downloader(0x7ed360, 0xc420018980, 0xc42000d110, 0xc4200e6300, 0xc4200e6360)
	/tmp/d20170109-30451-lnkl4u/solution.go:104 +0xa1
created by _/tmp/d20170109-30451-lnkl4u.DownloadFile
	/tmp/d20170109-30451-lnkl4u/solution.go:150 +0x248

goroutine 26 [runnable]:
net/textproto.MIMEHeader.Add(0xc4203fd9b0, 0x6cd993, 0xd, 0xc42041e250, 0xc)
	/usr/local/go/src/net/textproto/header.go:15 +0x12f
net/http.Header.Add(0xc4203fd9b0, 0x6cd993, 0xd, 0xc42041e250, 0xc)
	/usr/local/go/src/net/http/header.go:24 +0x53
_/tmp/d20170109-30451-lnkl4u.TestSingleURLCancelContextAfterHalfBytesWereServed.func1(0x7ecda0, 0xc4204221a0, 0xc420424000)
	/tmp/d20170109-30451-lnkl4u/solution_test.go:172 +0x629
net/http.HandlerFunc.ServeHTTP(0xc4200189c0, 0x7ecda0, 0xc4204221a0, 0xc420424000)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.serverHandler.ServeHTTP(0xc42001a300, 0x7ecda0, 0xc4204221a0, 0xc420424000)
	/usr/local/go/src/net/http/server.go:2202 +0x7d
net/http.(*conn).serve(0xc4200f6180, 0x7ed360, 0xc4200e4640)
	/usr/local/go/src/net/http/server.go:1579 +0x4b7
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d

goroutine 24 [IO wait]:
net.runtime_pollWait(0x7f026e125f38, 0x72, 0x7)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4200f01b0, 0x72, 0xc4200389d0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4200f01b0, 0x7ea8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4200f0150, 0xc420114000, 0x1000, 0x1000, 0x0, 0x7ea8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4200e2030, 0xc420114000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc42010a000, 0xc420114000, 0x1000, 0x1000, 0x30, 0xc420038b58, 0x43b23c)
	/usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4200e66c0)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4200e66c0, 0x1, 0x0, 0x1, 0x0, 0xc42040eba0, 0x0)
	/usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc42010a000)
	/usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 25 [select]:
net/http.(*persistConn).writeLoop(0xc42010a000)
	/usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 4842 [select]:
net/http.(*persistConn).roundTrip(0xc42010a000, 0xc420418560, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:1840 +0x93b
net/http.(*Transport).RoundTrip(0xc4200c20f0, 0xc42041a2d0, 0xc4200c20f0, 0x0, 0xc400000000)
	/usr/local/go/src/net/http/transport.go:380 +0x4ee
net/http.send(0xc42041a2d0, 0x7e8ea0, 0xc4200c20f0, 0x0, 0x0, 0x0, 0x8, 0xc420028d08, 0xc420216eb8)
	/usr/local/go/src/net/http/client.go:256 +0x15f
net/http.(*Client).send(0xc42000d110, 0xc42041a2d0, 0x0, 0x0, 0x0, 0xc420216eb8, 0x0, 0x1)
	/usr/local/go/src/net/http/client.go:146 +0x102
net/http.(*Client).doFollowingRedirects(0xc42000d110, 0xc42041a2d0, 0x6f1d70, 0x3, 0x51f401, 0xc420405860)
	/usr/local/go/src/net/http/client.go:528 +0x5e5
net/http.(*Client).Do(0xc42000d110, 0xc42041a2d0, 0x5, 0xc4203e1058, 0x4)
	/usr/local/go/src/net/http/client.go:184 +0x1ea
_/tmp/d20170109-30451-lnkl4u.download(0x0, 0xc42000d110, 0x1, 0x24, 0xc4200be9c0, 0x1c, 0xc4200e6360)
	/tmp/d20170109-30451-lnkl4u/solution.go:83 +0x18a
created by _/tmp/d20170109-30451-lnkl4u.downloader
	/tmp/d20170109-30451-lnkl4u/solution.go:110 +0x1c5
exit status 2
FAIL	_/tmp/d20170109-30451-lnkl4u	1.040s
--- FAIL: TestNoValidUrls (0.00s)
	solution_test.go:215: Expected to get error, but got none
FAIL
exit status 1
FAIL	_/tmp/d20170109-30451-lnkl4u	0.010s
--- FAIL: TestReturnOnly10Bytes (0.00s)
	solution_test.go:248: Wrote 0 not 10 as expected
	solution_test.go:251: Got error while writing response
	solution_test.go:265: Expected to read 37 bytes from simple download but got 0
	solution_test.go:269: Expected to get  error with message 'no valid urls', but got '%!s(<nil>)'
	solution_test.go:53: Expected result was '54686973204953207468' but got ''
FAIL
exit status 1
FAIL	_/tmp/d20170109-30451-lnkl4u	0.007s
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x48 pc=0x4765fe]

goroutine 41 [running]:
panic(0x67ee20, 0xc4200120d0)
	/usr/local/go/src/runtime/panic.go:500 +0x1a1
_/tmp/d20170109-30451-lnkl4u.download(0x0, 0xc42000d110, 0x2, 0x12, 0xc4200bea20, 0x1c, 0xc4200e44e0)
	/tmp/d20170109-30451-lnkl4u/solution.go:84 +0x19e
created by _/tmp/d20170109-30451-lnkl4u.downloader
	/tmp/d20170109-30451-lnkl4u/solution.go:110 +0x1c5
exit status 2
FAIL	_/tmp/d20170109-30451-lnkl4u	0.020s
panic: send on closed channel

goroutine 61 [running]:
panic(0x67ef40, 0xc4200e5f80)
	/usr/local/go/src/runtime/panic.go:500 +0x1a1
_/tmp/d20170109-30451-lnkl4u.download(0x0, 0xc42000d110, 0x2, 0x12, 0xc4200bea20, 0x1c, 0xc4200de360)
	/tmp/d20170109-30451-lnkl4u/solution.go:92 +0x2d2
created by _/tmp/d20170109-30451-lnkl4u.downloader
	/tmp/d20170109-30451-lnkl4u/solution.go:113 +0x237
exit status 2
FAIL	_/tmp/d20170109-30451-lnkl4u	0.014s
--- FAIL: TestTwoUrlsWithOneBroken (0.02s)
	solution_test.go:436: Expected to read 37 bytes from simple download but got 36
	solution_test.go:53: Expected result was '5468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e7365' but got '545454545454545454545454545454545454545454545454545454545454545454545454'
FAIL
exit status 1
FAIL	_/tmp/d20170109-30451-lnkl4u	0.027s
--- FAIL: TestTwoUrlsWithTheOtherOneBroken (0.01s)
	solution_test.go:481: Expected to read 37 bytes from simple download but got 36
	solution_test.go:53: Expected result was '5468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e7365' but got '545454545454545454545454545454545454545454545454545454545454545454545454'
FAIL
exit status 1
FAIL	_/tmp/d20170109-30451-lnkl4u	0.019s
panic: send on closed channel

goroutine 29 [running]:
panic(0x67ef40, 0xc4200d9370)
	/usr/local/go/src/runtime/panic.go:500 +0x1a1
_/tmp/d20170109-30451-lnkl4u.download(0x0, 0xc42000d1a0, 0x2, 0x24, 0xc4200be9e0, 0x1d, 0xc42005ed20)
	/tmp/d20170109-30451-lnkl4u/solution.go:92 +0x2d2
created by _/tmp/d20170109-30451-lnkl4u.downloader
	/tmp/d20170109-30451-lnkl4u/solution.go:110 +0x1c5
exit status 2
FAIL	_/tmp/d20170109-30451-lnkl4u	0.014s
panic: send on closed channel

goroutine 84 [running]:
panic(0x67ef40, 0xc420013d90)
	/usr/local/go/src/runtime/panic.go:500 +0x1a1
_/tmp/d20170109-30451-lnkl4u.download(0x0, 0xc42006d050, 0x2, 0x19, 0xc4200d66c0, 0x1d, 0xc420064e40)
	/tmp/d20170109-30451-lnkl4u/solution.go:92 +0x2d2
created by _/tmp/d20170109-30451-lnkl4u.downloader
	/tmp/d20170109-30451-lnkl4u/solution.go:110 +0x1c5
exit status 2
FAIL	_/tmp/d20170109-30451-lnkl4u	0.014s
--- FAIL: TestSingleURL1ByteARequest (0.01s)
	solution_test.go:690: Expected to read 37 bytes from simple download but got 36
	solution_test.go:53: Expected result was '5468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e7365' but got '545454545454545454545454545454545454545454545454545454545454545454545454'
FAIL
exit status 1
FAIL	_/tmp/d20170109-30451-lnkl4u	0.014s
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x48 pc=0x4765fe]

goroutine 66 [running]:
panic(0x67ee20, 0xc4200120d0)
	/usr/local/go/src/runtime/panic.go:500 +0x1a1
_/tmp/d20170109-30451-lnkl4u.download(0x0, 0xc42000d110, 0x2, 0x25, 0xc4200bea40, 0x1d, 0xc42005eea0)
	/tmp/d20170109-30451-lnkl4u/solution.go:84 +0x19e
created by _/tmp/d20170109-30451-lnkl4u.downloader
	/tmp/d20170109-30451-lnkl4u/solution.go:110 +0x1c5
exit status 2
FAIL	_/tmp/d20170109-30451-lnkl4u	0.017s
panic: send on closed channel

goroutine 2401 [running]:
panic(0x67ef40, 0xc420455e60)
	/usr/local/go/src/runtime/panic.go:500 +0x1a1
_/tmp/d20170109-30451-lnkl4u.download(0x0, 0xc42000d140, 0x2, 0xf3, 0xc4200bf8e0, 0x1a, 0xc420332300)
	/tmp/d20170109-30451-lnkl4u/solution.go:92 +0x2d2
created by _/tmp/d20170109-30451-lnkl4u.downloader
	/tmp/d20170109-30451-lnkl4u/solution.go:110 +0x1c5
exit status 2
FAIL	_/tmp/d20170109-30451-lnkl4u	0.536s
panic: test timed out after 1s

goroutine 21 [running]:
panic(0x66a0e0, 0xc420150880)
	/usr/local/go/src/runtime/panic.go:500 +0x1a1
testing.startAlarm.func1()
	/usr/local/go/src/testing/testing.go:918 +0x10b
created by time.goFunc
	/usr/local/go/src/time/sleep.go:154 +0x44

goroutine 1 [chan receive]:
testing.(*T).Run(0xc42007c0c0, 0x6d4711, 0x21, 0x6f1818, 0xc42004bd01)
	/usr/local/go/src/testing/testing.go:647 +0x316
testing.RunTests.func1(0xc42007c0c0)
	/usr/local/go/src/testing/testing.go:793 +0x6d
testing.tRunner(0xc42007c0c0, 0xc42004be20)
	/usr/local/go/src/testing/testing.go:610 +0x81
testing.RunTests(0x6f19f8, 0x805140, 0x11, 0x11, 0x7f7f9ea53000)
	/usr/local/go/src/testing/testing.go:799 +0x2f5
testing.(*M).Run(0xc42004bee8, 0x687fc0)
	/usr/local/go/src/testing/testing.go:743 +0x85
main.main()
	_/tmp/d20170109-30451-lnkl4u/_test/_testmain.go:86 +0xc6

goroutine 17 [syscall, locked to thread]:
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2086 +0x1

goroutine 6 [select]:
net/http.(*persistConn).roundTrip(0xc42000a900, 0xc42010bfa0, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:1840 +0x93b
net/http.(*Transport).RoundTrip(0xc4200c2000, 0xc42013bc20, 0xc4200c2000, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:380 +0x4ee
net/http.send(0xc42013bc20, 0x7e8ea0, 0xc4200c2000, 0x0, 0x0, 0x0, 0x8, 0xc42004d138, 0xc42002a580)
	/usr/local/go/src/net/http/client.go:256 +0x15f
net/http.(*Client).send(0x808100, 0xc42013bc20, 0x0, 0x0, 0x0, 0xc42002a580, 0x0, 0x1)
	/usr/local/go/src/net/http/client.go:146 +0x102
net/http.(*Client).doFollowingRedirects(0x808100, 0xc42013bc20, 0x6f1d70, 0x19, 0x0, 0x0)
	/usr/local/go/src/net/http/client.go:528 +0x5e5
net/http.(*Client).Head(0x808100, 0xc4200bf5c0, 0x19, 0xc420012d40, 0x0, 0x0)
	/usr/local/go/src/net/http/client.go:635 +0x93
net/http.Head(0xc4200bf5c0, 0x19, 0x0, 0x0, 0x200)
	/usr/local/go/src/net/http/client.go:619 +0x41
_/tmp/d20170109-30451-lnkl4u.filterAndGetFileSize(0xc42000d1d0, 0xc42004d918, 0x64, 0x64, 0x7f7f9ea53000, 0x0, 0x2, 0xc42001348a)
	/tmp/d20170109-30451-lnkl4u/solution.go:50 +0xc4
_/tmp/d20170109-30451-lnkl4u.DownloadFile(0x7ed420, 0xc42000d140, 0xc42004d918, 0x64, 0x64, 0x0, 0x0)
	/tmp/d20170109-30451-lnkl4u/solution.go:134 +0x121
_/tmp/d20170109-30451-lnkl4u.TestSlowLingchiWithMaxConnections(0xc42007c180)
	/tmp/d20170109-30451-lnkl4u/solution_test.go:897 +0x45d
testing.tRunner(0xc42007c180, 0x6f1818)
	/usr/local/go/src/testing/testing.go:610 +0x81
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:646 +0x2ec

goroutine 7 [IO wait]:
net.runtime_pollWait(0x7f7f9e9f8178, 0x72, 0x0)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420014370, 0x72, 0xc4200285e0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420014370, 0x7ea8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc420014310, 0x0, 0x7e94a0, 0xc4200dc100)
	/usr/local/go/src/net/fd_unix.go:419 +0x238
net.(*TCPListener).accept(0xc42002a048, 0x43418e, 0xc420028690, 0x52d8cd)
	/usr/local/go/src/net/tcpsock_posix.go:132 +0x2e
net.(*TCPListener).Accept(0xc42002a048, 0x6f1bd8, 0xc4200f4000, 0x7ed420, 0xc4200de0c0)
	/usr/local/go/src/net/tcpsock.go:222 +0x49
net/http.(*Server).Serve(0xc42001a300, 0x7ecba0, 0xc42002a048, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:2273 +0x1ce
net/http/httptest.(*Server).goServe.func1(0xc42005e4e0)
	/usr/local/go/src/net/http/httptest/server.go:235 +0x6d
created by net/http/httptest.(*Server).goServe
	/usr/local/go/src/net/http/httptest/server.go:236 +0x5c

goroutine 20 [sleep]:
time.Sleep(0x989680)
	/usr/local/go/src/runtime/time.go:59 +0xe1
_/tmp/d20170109-30451-lnkl4u.TestSlowLingchiWithMaxConnections.func1(0x7ecda0, 0xc420107d40, 0xc42013bb30)
	/tmp/d20170109-30451-lnkl4u/solution_test.go:858 +0xe7
net/http.HandlerFunc.ServeHTTP(0xc4200188c0, 0x7ecda0, 0xc420107d40, 0xc42013bb30)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.serverHandler.ServeHTTP(0xc42001a300, 0x7ecda0, 0xc420107d40, 0xc42013bb30)
	/usr/local/go/src/net/http/server.go:2202 +0x7d
net/http.(*conn).serve(0xc4200f4000, 0x7ed360, 0xc4200e22c0)
	/usr/local/go/src/net/http/server.go:1579 +0x4b7
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d

goroutine 18 [IO wait]:
net.runtime_pollWait(0x7f7f9e9f80b8, 0x72, 0x5)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420014450, 0x72, 0xc42003b9d0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420014450, 0x7ea8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4200143f0, 0xc4200e6000, 0x1000, 0x1000, 0x0, 0x7ea8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4200e0000, 0xc4200e6000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc42000a900, 0xc4200e6000, 0x1000, 0x1000, 0x30, 0xc42003bb58, 0x43b23c)
	/usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4200e4000)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4200e4000, 0x1, 0x0, 0x1, 0x1, 0xc42013d500, 0x0)
	/usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc42000a900)
	/usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 19 [select]:
net/http.(*persistConn).writeLoop(0xc42000a900)
	/usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1063 +0x50e
exit status 2
FAIL	_/tmp/d20170109-30451-lnkl4u	1.016s
panic: send on closed channel

goroutine 202 [running]:
panic(0x67ef40, 0xc420519990)
	/usr/local/go/src/runtime/panic.go:500 +0x1a1
_/tmp/d20170109-30451-lnkl4u.download(0xc420126b40, 0xc42000d140, 0x1e, 0x1f, 0xc4200bec00, 0x19, 0xc420126ae0)
	/tmp/d20170109-30451-lnkl4u/solution.go:92 +0x2d2
created by _/tmp/d20170109-30451-lnkl4u.downloader
	/tmp/d20170109-30451-lnkl4u/solution.go:110 +0x1c5
panic: send on closed channel

goroutine 190 [running]:
panic(0x67ef40, 0xc42059c700)
	/usr/local/go/src/runtime/panic.go:500 +0x1a1
_/tmp/d20170109-30451-lnkl4u.download(0xc420126b40, 0xc42000d140, 0x6, 0x7, 0xc4200bea80, 0x18, 0xc420126ae0)
	/tmp/d20170109-30451-lnkl4u/solution.go:92 +0x2d2
created by _/tmp/d20170109-30451-lnkl4u.downloader
	/tmp/d20170109-30451-lnkl4u/solution.go:110 +0x1c5
exit status 2
FAIL	_/tmp/d20170109-30451-lnkl4u	0.380s

История (2 версии и 0 коментара)

Добромир обнови решението на 03.01.2017 16:19 (преди над 1 година)

+package main
+
+import "io"
+import "io/ioutil"
+import "strconv"
+import "errors"
+import "context"
+import "net/http"
+
+type result struct {
+ bytes []byte
+ pos int
+ err error
+}
+
+type downloadRequest struct {
+ begin int
+ end int
+ url string
+}
+
+type downloadResult struct {
+ begin int
+ end int
+ body []byte
+ url string
+ urlInvalid bool
+}
+
+func (res *result) Read(p []byte) (n int, err error) {
+
+ i := 0
+ for ; res.pos < len(res.bytes) && i < len(p); i++ {
+ p[i] = res.bytes[res.pos]
+ res.pos++
+ }
+
+ if res.pos >= len(res.bytes) {
+ return i, io.EOF
+ }
+
+ return i, res.err
+}
+
+func filterAndGetFileSize(client *http.Client, urls []string) ([]string, int) {
+ result := make([]string, 0, len(urls))
+ size := 0
+
+ for _, url := range urls {
+ response, err := http.Head(url)
+ if err != nil {
+ continue
+ }
+
+ ioutil.ReadAll(response.Body)
+ response.Body.Close()
+
+ if response.StatusCode >= 400 {
+ continue
+ }
+
+ size = int(response.ContentLength)
+ result = append(result, url)
+ }
+
+ return result, size - 1
+}
+
+func download(limitChan chan struct{}, client *http.Client, request downloadRequest, resultChan chan<- downloadResult) {
+ if limitChan != nil {
+ limitChan <- struct{}{}
+ }
+ defer func() {
+ if limitChan != nil {
+ <-limitChan
+ }
+ }()
+
+ req, _ := http.NewRequest("GET", request.url, nil)
+ rangeHeader := strconv.Itoa(request.begin) + "-" + strconv.Itoa(request.end)
+ req.Header.Add("Range", rangeHeader)
+
+ resp, err := client.Do(req)
+ defer resp.Body.Close()
+ body, _ := ioutil.ReadAll(resp.Body)
+
+ urlInvalid := false
+ if resp.StatusCode >= 400 || err != nil {
+ urlInvalid = true
+ }
+
+ resultChan <- downloadResult{request.begin, request.end, body, request.url, urlInvalid}
+}
+
+func downloader(ctx context.Context, client *http.Client, requestChan <-chan downloadRequest, resultChan chan downloadResult) {
+
+ limit := 0
+ var limitChan chan struct{} = nil
+ if ctx != nil && ctx.Value("max-connections") != nil {
+ limit = ctx.Value("max-connections").(int)
+ limitChan = make(chan struct{}, limit)
+ }
+
+ for request := range requestChan {
+ if ctx != nil {
+ select {
+ case <-ctx.Done():
+ return
+ default:
+ go download(limitChan, client, request, resultChan)
+ }
+ } else {
+ go download(limitChan, client, request, resultChan)
+ }
+ }
+}
+
+func balanceDownloads(begin, downloadSize int, urls []string, requestChan chan<- downloadRequest) {
+ end, originalSize := downloadSize/len(urls), downloadSize
+ for i := 0; i < len(urls) && begin <= originalSize; i++ {
+ byteCount := downloadSize / (len(urls) - i)
+ downloadSize -= byteCount
+
+ downloadRequest := downloadRequest{begin, end, urls[i]}
+ requestChan <- downloadRequest
+
+ begin = end + 1
+ end = begin + byteCount
+ }
+}
+
+func DownloadFile(ctx context.Context, urls []string) io.Reader {
+ client := &http.Client{Transport: &http.Transport{}}
+ urls, size := filterAndGetFileSize(client, urls)
+
+ requestChan := make(chan downloadRequest)
+ resultChan := make(chan downloadResult)
+
+ defer func() {
+ close(resultChan)
+ close(requestChan)
+ }()
+
+ if len(urls) == 0 {
+ return &result{[]byte{}, 0, errors.New("no valid urls.")}
+ } else if size <= 0 {
+ return &result{[]byte{}, 0, nil}
+ }
+
+
+ go downloader(ctx, client, requestChan, resultChan)
+ balanceDownloads(0, size, urls, requestChan)
+
+ bytesReceived := 0
+ content := make([]byte, 0)
+ for res := range(resultChan) {
+ bytesReceived += len(res.body)
+ content = append(content, res.body...)
+
+ expectedSize := res.end - res.begin + 1
+ if len(res.body) < expectedSize && !res.urlInvalid {
+ requestChan <- downloadRequest{len(res.body) + 1, res.end, res.url}
+ } else if res.urlInvalid {
+ for i, url := range urls {
+ if url == res.url {
+ urls = append(urls[:i], urls[i+1:]...)
+ }
+ }
+
+ if len(urls) == 0 {
+ return &result{content, 0, errors.New("no valid urls.")}
+ }
+ balanceDownloads(len(res.body) + 1, size-len(res.body), urls, requestChan)
+ }
+
+ if bytesReceived >= size || len(urls) == 0 {
+ return &result{content, 0, nil}
+ }
+ }
+
+ return &result{content, 0, nil}
+}

Добромир обнови решението на 03.01.2017 16:20 (преди над 1 година)

package main
import "io"
import "io/ioutil"
import "strconv"
import "errors"
import "context"
import "net/http"
type result struct {
bytes []byte
pos int
err error
}
type downloadRequest struct {
begin int
end int
url string
}
type downloadResult struct {
- begin int
- end int
- body []byte
- url string
+ begin int
+ end int
+ body []byte
+ url string
urlInvalid bool
}
func (res *result) Read(p []byte) (n int, err error) {
i := 0
for ; res.pos < len(res.bytes) && i < len(p); i++ {
p[i] = res.bytes[res.pos]
res.pos++
}
if res.pos >= len(res.bytes) {
return i, io.EOF
}
return i, res.err
}
func filterAndGetFileSize(client *http.Client, urls []string) ([]string, int) {
result := make([]string, 0, len(urls))
size := 0
for _, url := range urls {
response, err := http.Head(url)
if err != nil {
continue
}
ioutil.ReadAll(response.Body)
response.Body.Close()
if response.StatusCode >= 400 {
continue
}
size = int(response.ContentLength)
result = append(result, url)
}
return result, size - 1
}
func download(limitChan chan struct{}, client *http.Client, request downloadRequest, resultChan chan<- downloadResult) {
if limitChan != nil {
limitChan <- struct{}{}
}
defer func() {
if limitChan != nil {
<-limitChan
}
}()
req, _ := http.NewRequest("GET", request.url, nil)
rangeHeader := strconv.Itoa(request.begin) + "-" + strconv.Itoa(request.end)
req.Header.Add("Range", rangeHeader)
resp, err := client.Do(req)
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
urlInvalid := false
if resp.StatusCode >= 400 || err != nil {
urlInvalid = true
}
resultChan <- downloadResult{request.begin, request.end, body, request.url, urlInvalid}
}
func downloader(ctx context.Context, client *http.Client, requestChan <-chan downloadRequest, resultChan chan downloadResult) {
limit := 0
var limitChan chan struct{} = nil
if ctx != nil && ctx.Value("max-connections") != nil {
limit = ctx.Value("max-connections").(int)
limitChan = make(chan struct{}, limit)
}
for request := range requestChan {
if ctx != nil {
select {
case <-ctx.Done():
return
default:
go download(limitChan, client, request, resultChan)
}
} else {
go download(limitChan, client, request, resultChan)
}
}
}
func balanceDownloads(begin, downloadSize int, urls []string, requestChan chan<- downloadRequest) {
end, originalSize := downloadSize/len(urls), downloadSize
for i := 0; i < len(urls) && begin <= originalSize; i++ {
byteCount := downloadSize / (len(urls) - i)
downloadSize -= byteCount
downloadRequest := downloadRequest{begin, end, urls[i]}
requestChan <- downloadRequest
begin = end + 1
end = begin + byteCount
}
}
func DownloadFile(ctx context.Context, urls []string) io.Reader {
client := &http.Client{Transport: &http.Transport{}}
urls, size := filterAndGetFileSize(client, urls)
requestChan := make(chan downloadRequest)
resultChan := make(chan downloadResult)
defer func() {
close(resultChan)
close(requestChan)
}()
if len(urls) == 0 {
return &result{[]byte{}, 0, errors.New("no valid urls.")}
} else if size <= 0 {
return &result{[]byte{}, 0, nil}
}
-
go downloader(ctx, client, requestChan, resultChan)
balanceDownloads(0, size, urls, requestChan)
bytesReceived := 0
content := make([]byte, 0)
- for res := range(resultChan) {
+ for res := range resultChan {
bytesReceived += len(res.body)
content = append(content, res.body...)
expectedSize := res.end - res.begin + 1
if len(res.body) < expectedSize && !res.urlInvalid {
requestChan <- downloadRequest{len(res.body) + 1, res.end, res.url}
} else if res.urlInvalid {
for i, url := range urls {
if url == res.url {
urls = append(urls[:i], urls[i+1:]...)
}
}
if len(urls) == 0 {
return &result{content, 0, errors.New("no valid urls.")}
}
- balanceDownloads(len(res.body) + 1, size-len(res.body), urls, requestChan)
+ balanceDownloads(len(res.body)+1, size-len(res.body), urls, requestChan)
}
if bytesReceived >= size || len(urls) == 0 {
return &result{content, 0, nil}
}
}
return &result{content, 0, nil}
-}
+}