Програмиране с Go

Курс във Факултета по Математика и Информатика към СУ

Решение на HTTP сваляч от Георги Костадинов

Обратно към всички решения

Към профила на Георги Костадинов

Резултати

  • 7 точки от тестове
  • 0 бонус точки
  • 7 точки общо
  • 12 успешни тест(а)
  • 5 неуспешни тест(а)

Код

package main
import (
"context"
"errors"
"io"
"io/ioutil"
"net/http"
"strconv"
"sync"
"time"
)
type Reader struct {
pipeReader io.PipeReader
pipeWriter io.PipeWriter
err error
}
func NewReader(pipeReader io.PipeReader, pipeWriter io.PipeWriter) *Reader {
return &Reader{pipeReader, pipeWriter, nil}
}
func (r *Reader) Read(p []byte) (n int, err error) {
n, err = r.pipeReader.Read(p)
if err == nil {
err = r.err
}
return n, err
}
func (r *Reader) Write(bytes []byte, err error) {
r.err = err
r.pipeWriter.Write(bytes)
}
func (r *Reader) SetError(err error) error {
r.err = err
return err
}
const maxUrls int = 128
type Url struct {
id int
url string
}
type Task struct {
url Url
rangeStart int
rangeEnd int
}
type TaskResult struct {
task *Task
taskSkipped bool
result []byte
}
type Queue []*Task
func (q *Queue) Push(n *Task) {
*q = append(*q, n)
}
func (q *Queue) Pop() (n *Task) {
n = (*q)[0]
*q = (*q)[1:]
return
}
func (q *Queue) Len() int {
return len(*q)
}
func getContentLength(urls []Url) int {
contentLength := -1
for _, url := range urls {
response, err := http.Head(url.url)
if err != nil {
continue
}
contentLength = int(response.ContentLength)
}
if contentLength < 0 {
contentLength = 0
}
return contentLength
}
func generateTasks(urls []Url, rangeStart int, rangeEnd int) Queue {
var tasks Queue
numberOfUrls := len(urls)
bucketSizes := make([]int, numberOfUrls)
evenLength := int(float32(rangeEnd-rangeStart+1) / float32(numberOfUrls))
for i := 0; i < numberOfUrls; i++ {
bucketSizes[i] = evenLength
}
surPlus := (rangeEnd - rangeStart + 1) % numberOfUrls
for i := 0; surPlus > 0; surPlus-- {
bucketSizes[i] += 1
i = (i + 1) % numberOfUrls
}
k := rangeStart
for i := 0; i < numberOfUrls && k <= rangeEnd; i++ {
rangeLow := k
rangeHigh := k + bucketSizes[i] - 1
k += bucketSizes[i]
url := urls[i]
tasks.Push(&Task{url, rangeLow, rangeHigh})
}
return tasks
}
func getRangedRequest(httpClient *http.Client, url string, rangeStart int, rangeEnd int) ([]byte, bool) {
request, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, false
}
request.Header.Add("Range", "bytes="+strconv.Itoa(rangeStart)+"-"+strconv.Itoa(rangeEnd))
response, err := httpClient.Do(request)
if err != nil {
return nil, false
}
defer response.Body.Close()
body, err := ioutil.ReadAll(response.Body)
if err != nil {
return body, true
}
statusCode := int(response.StatusCode)
if statusCode >= 300 && statusCode < 600 {
return nil, false
}
repeat := false
if rangeEnd != 1 && int(response.ContentLength)-1 != (rangeEnd-rangeStart) {
repeat = true
}
return body, repeat
}
func getValidUrls(urls []Url, invalidUrls [maxUrls]bool) []Url {
validUrls := make([]Url, 0)
for _, url := range urls {
if invalidUrls[url.id] {
continue
}
validUrls = append(validUrls[:], url)
}
return validUrls
}
func DownloadFile(ctx context.Context, urlStrings []string) io.Reader {
pr, pw := io.Pipe()
outputReader := NewReader(*pr, *pw)
numberOfUrls := len(urlStrings)
maxConcurrency := 0
ctxMaxConcurrency, ok := ctx.Value("max-connections").(int)
if ok {
maxConcurrency = ctxMaxConcurrency
}
if maxConcurrency == 0 || maxConcurrency >= numberOfUrls {
maxConcurrency = numberOfUrls
}
urls := make([]Url, 0)
for id, urlString := range urlStrings {
urls = append(urls[:], Url{id, urlString})
}
var invalidUrls [maxUrls]bool
go func(outputReader *Reader) {
var wg sync.WaitGroup
var errOnce sync.Once
cancel := func() {}
if ctx != nil {
ctx, cancel = context.WithCancel(ctx)
}
contentLength := getContentLength(urls)
rangeEnd := contentLength - 1
if contentLength-1 < 0 {
rangeEnd = 1
}
tasksQueue := generateTasks(urls, 0, rangeEnd)
httpClient := &http.Client{}
throttling := make(chan struct{}, maxConcurrency)
results := make(chan TaskResult, maxUrls)
justAdded := false
stopMainFor := false
for {
time.Sleep(20 * time.Millisecond)
if stopMainFor {
break
}
if tasksQueue.Len() == 0 {
continue
}
task := tasksQueue.Pop()
throttling <- struct{}{}
if !justAdded {
wg.Add(1)
}
if justAdded {
justAdded = false
}
go func(task *Task, wg *sync.WaitGroup) {
defer func() {
if !justAdded {
wg.Done()
}
}()
processTask := func() error {
if ctx != nil {
select {
case <-ctx.Done():
return outputReader.SetError(ctx.Err())
default:
}
}
var taskBytesOut []byte
rangeStart := task.rangeStart
rangeEnd := task.rangeEnd
for {
taskBytes, repeat := getRangedRequest(httpClient, task.url.url, rangeStart, rangeEnd)
if taskBytes == nil {
invalidUrls[task.url.id] = true
validUrls := getValidUrls(urls, invalidUrls)
if len(validUrls) > 0 {
newTasksQueue := generateTasks(validUrls, rangeStart, rangeEnd)
for newTasksQueue.Len() > 0 {
tasksQueue.Push(newTasksQueue.Pop())
justAdded = true
}
}
break
}
if len(taskBytesOut) == 0 {
taskBytesOut = taskBytes
} else {
for _, taskByte := range taskBytes {
taskBytesOut = append(taskBytesOut, taskByte)
}
}
if !repeat {
break
}
if (rangeStart + len(taskBytes)) > rangeEnd {
continue
} else {
rangeStart += len(taskBytes)
}
}
results <- TaskResult{task, invalidUrls[task.url.id], taskBytesOut}
return nil
}
if err := processTask(); err != nil {
errOnce.Do(func() {
cancel()
})
}
<-throttling
go func() {
if tasksQueue.Len() == 0 {
stopMainFor = true
}
}()
}(task, &wg)
}
go func(wg *sync.WaitGroup) {
defer pw.Close()
wg.Wait()
close(throttling)
close(results)
taskCount := 0
totalOutput := make([]byte, contentLength)
receivedBytes := 0
for result := range results {
for index, taskByte := range result.result {
totalOutput[index+result.task.rangeStart] = taskByte
receivedBytes += 1
}
taskCount += 1
}
var err error = nil
validUrls := getValidUrls(urls, invalidUrls)
if len(validUrls) == 0 {
err = errors.New("no valid urls")
}
if taskCount == 0 {
outputReader.Write(nil, err)
} else {
outputReader.Write(totalOutput[:receivedBytes], err)
}
}(&wg)
}(outputReader)
return outputReader
}

Лог от изпълнението

PASS
ok  	_/tmp/d20170109-30451-b1m5id	0.046s
PASS
ok  	_/tmp/d20170109-30451-b1m5id	0.046s
PASS
ok  	_/tmp/d20170109-30451-b1m5id	0.046s
panic: test timed out after 1s

goroutine 29 [running]:
panic(0x66b220, 0xc420151330)
	/usr/local/go/src/runtime/panic.go:500 +0x1a1
testing.startAlarm.func1()
	/usr/local/go/src/testing/testing.go:918 +0x10b
created by time.goFunc
	/usr/local/go/src/time/sleep.go:154 +0x44

goroutine 1 [chan receive]:
testing.(*T).Run(0xc42008c0c0, 0x6da4f6, 0x32, 0x6f3280, 0xc42004bd01)
	/usr/local/go/src/testing/testing.go:647 +0x316
testing.RunTests.func1(0xc42008c0c0)
	/usr/local/go/src/testing/testing.go:793 +0x6d
testing.tRunner(0xc42008c0c0, 0xc420454e30)
	/usr/local/go/src/testing/testing.go:610 +0x81
testing.RunTests(0x6f3470, 0x807140, 0x11, 0x11, 0x7f8a815914b0)
	/usr/local/go/src/testing/testing.go:799 +0x2f5
testing.(*M).Run(0xc420454ef8, 0x689360)
	/usr/local/go/src/testing/testing.go:743 +0x85
main.main()
	_/tmp/d20170109-30451-b1m5id/_test/_testmain.go:86 +0xc6

goroutine 17 [syscall, locked to thread]:
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2086 +0x1

goroutine 21 [semacquire]:
sync.runtime_notifyListWait(0xc42008c280, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42008c270)
	/usr/local/go/src/sync/cond.go:57 +0x80
io.(*pipe).read(0xc42008c240, 0xc4200ec000, 0x200, 0x200, 0x0, 0x0, 0x0)
	/usr/local/go/src/io/pipe.go:47 +0x102
io.(*PipeReader).Read(0xc4200d66c0, 0xc4200ec000, 0x200, 0x200, 0xc4200ec000, 0x200, 0x200)
	/usr/local/go/src/io/pipe.go:129 +0x4c
_/tmp/d20170109-30451-b1m5id.(*Reader).Read(0xc4200d66c0, 0xc4200ec000, 0x200, 0x200, 0xc420036eb8, 0xc4200d66e0, 0xc42008c250)
	/tmp/d20170109-30451-b1m5id/solution.go:25 +0x4b
bytes.(*Buffer).ReadFrom(0xc420036ef8, 0x7ea860, 0xc4200d66c0, 0x1, 0x1, 0x7ea860)
	/usr/local/go/src/bytes/buffer.go:176 +0x155
_/tmp/d20170109-30451-b1m5id.TestSingleURLCancelContextAfterHalfBytesWereServed(0xc42008c180)
	/tmp/d20170109-30451-b1m5id/solution_test.go:193 +0x2dd
testing.tRunner(0xc42008c180, 0x6f3280)
	/usr/local/go/src/testing/testing.go:610 +0x81
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:646 +0x2ec

goroutine 22 [IO wait]:
net.runtime_pollWait(0x7f8a81536178, 0x72, 0x0)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42005e290, 0x72, 0xc420023de0, 0xc420068080)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42005e290, 0x7ec8a0, 0xc420068080)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc42005e230, 0x0, 0x7eb4a0, 0xc4200d6860)
	/usr/local/go/src/net/fd_unix.go:419 +0x238
net.(*TCPListener).accept(0xc420076040, 0x43418e, 0xc420023e90, 0x52f07d)
	/usr/local/go/src/net/tcpsock_posix.go:132 +0x2e
net.(*TCPListener).Accept(0xc420076040, 0x6f3650, 0xc42008e500, 0x7ef420, 0xc42000c150)
	/usr/local/go/src/net/tcpsock.go:222 +0x49
net/http.(*Server).Serve(0xc42008e280, 0x7eeba0, 0xc420076040, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:2273 +0x1ce
net/http/httptest.(*Server).goServe.func1(0xc4200644e0)
	/usr/local/go/src/net/http/httptest/server.go:235 +0x6d
created by net/http/httptest.(*Server).goServe
	/usr/local/go/src/net/http/httptest/server.go:236 +0x5c

goroutine 23 [sleep]:
time.Sleep(0x1312d00)
	/usr/local/go/src/runtime/time.go:59 +0xe1
_/tmp/d20170109-30451-b1m5id.DownloadFile.func1(0xc4200691e0, 0xc4200d66e0, 0x1, 0x1, 0x1, 0xc42008e300, 0xc420076048, 0xc4200d66c0)
	/tmp/d20170109-30451-b1m5id/solution.go:211 +0x279
created by _/tmp/d20170109-30451-b1m5id.DownloadFile
	/tmp/d20170109-30451-b1m5id/solution.go:333 +0x48f

goroutine 27 [IO wait]:
net.runtime_pollWait(0x7f8a815360b8, 0x72, 0x5)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42005e370, 0x72, 0xc4200379d0, 0xc420068080)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42005e370, 0x7ec8a0, 0xc420068080)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc42005e310, 0xc4200ff000, 0x1000, 0x1000, 0x0, 0x7ec8a0, 0xc420068080)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420076078, 0xc4200ff000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc4200d8500, 0xc4200ff000, 0x1000, 0x1000, 0x30, 0xc420037b58, 0x43b23c)
	/usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc420064b40)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc420064b40, 0x1, 0x0, 0x1, 0x0, 0xc420289140, 0x0)
	/usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc4200d8500)
	/usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 26 [IO wait]:
net.runtime_pollWait(0x7f8a81535ff8, 0x72, 0x6)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42005e3e0, 0x72, 0xc4200387b0, 0xc420068080)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42005e3e0, 0x7ec8a0, 0xc420068080)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc42005e380, 0xc420099000, 0x1000, 0x1000, 0x0, 0x7ec8a0, 0xc420068080)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420076070, 0xc420099000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
net/http.(*connReader).Read(0xc4200d6880, 0xc420099000, 0x1000, 0x1000, 0xc420038918, 0x6d37cc, 0x19)
	/usr/local/go/src/net/http/server.go:586 +0x144
bufio.(*Reader).fill(0xc420064a80)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).ReadSlice(0xc420064a80, 0xa, 0x0, 0x1e, 0x6, 0x0, 0x0)
	/usr/local/go/src/bufio/bufio.go:330 +0xb5
bufio.(*Reader).ReadLine(0xc420064a80, 0xc42028eff0, 0xf0, 0xf0, 0x6c37e0, 0x4a1b83, 0x809d78)
	/usr/local/go/src/bufio/bufio.go:359 +0x37
net/textproto.(*Reader).readLineSlice(0xc4203c88a0, 0xc420038aa8, 0xc420038aa8, 0x4106d8, 0xf0, 0x6c37e0)
	/usr/local/go/src/net/textproto/reader.go:55 +0x5e
net/textproto.(*Reader).ReadLine(0xc4203c88a0, 0xc42028eff0, 0xc420038b20, 0x401863, 0xc420038c78)
	/usr/local/go/src/net/textproto/reader.go:36 +0x2f
net/http.readRequest(0xc420064a80, 0xc420038c00, 0xc42028eff0, 0x0, 0x0)
	/usr/local/go/src/net/http/request.go:793 +0xa5
net/http.(*conn).readRequest(0xc42008e500, 0x7ef360, 0xc420066b80, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:765 +0x10d
net/http.(*conn).serve(0xc42008e500, 0x7ef360, 0xc420066b80)
	/usr/local/go/src/net/http/server.go:1532 +0x3d3
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d

goroutine 28 [select]:
net/http.(*persistConn).writeLoop(0xc4200d8500)
	/usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 3 [GC assist wait]:
net/url.parse(0xc4200d66a0, 0x1c, 0x6cc900, 0x1, 0x6cca01, 0xc4200d66a0)
	/usr/local/go/src/net/url/url.go:454 +0x60
net/url.Parse(0xc4200d66a0, 0x1c, 0xc42004db01, 0x41104d, 0xc4200013a8)
	/usr/local/go/src/net/url/url.go:421 +0x8d
net/http.NewRequest(0x6ccaef, 0x3, 0xc4200d66a0, 0x1c, 0x0, 0x0, 0x0, 0xc42027f0ba, 0x2)
	/usr/local/go/src/net/http/request.go:676 +0x97
_/tmp/d20170109-30451-b1m5id.getRangedRequest(0xc42006d650, 0xc4200d66a0, 0x1c, 0x12, 0x24, 0x0, 0x0, 0x0, 0x0)
	/tmp/d20170109-30451-b1m5id/solution.go:126 +0x95
_/tmp/d20170109-30451-b1m5id.DownloadFile.func1.2.2(0x10, 0x6f3220)
	/tmp/d20170109-30451-b1m5id/solution.go:250 +0x127
_/tmp/d20170109-30451-b1m5id.DownloadFile.func1.2(0xc4200693f0, 0xc4200691e0, 0xc4200d66c0, 0xc42006d650, 0xc42008e300, 0xc4200d66e0, 0x1, 0x1, 0xc4200d6a20, 0xc420064de0, ...)
	/tmp/d20170109-30451-b1m5id/solution.go:288 +0x13f
created by _/tmp/d20170109-30451-b1m5id.DownloadFile.func1
	/tmp/d20170109-30451-b1m5id/solution.go:301 +0x415
exit status 2
FAIL	_/tmp/d20170109-30451-b1m5id	1.015s
PASS
ok  	_/tmp/d20170109-30451-b1m5id	0.051s
PASS
ok  	_/tmp/d20170109-30451-b1m5id	0.048s
PASS
ok  	_/tmp/d20170109-30451-b1m5id	0.068s
--- FAIL: TestTwoUrlsWithNilContext (0.00s)
panic: runtime error: invalid memory address or nil pointer dereference [recovered]
	panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x38 pc=0x476e2b]

goroutine 6 [running]:
panic(0x67ffc0, 0xc4200120d0)
	/usr/local/go/src/runtime/panic.go:500 +0x1a1
testing.tRunner.func1(0xc42007e180)
	/usr/local/go/src/testing/testing.go:579 +0x25d
panic(0x67ffc0, 0xc4200120d0)
	/usr/local/go/src/runtime/panic.go:458 +0x243
_/tmp/d20170109-30451-b1m5id.DownloadFile(0x0, 0x0, 0xc42003bea8, 0x2, 0x2, 0xc4200c0a40, 0x1d)
	/tmp/d20170109-30451-b1m5id/solution.go:174 +0x1cb
_/tmp/d20170109-30451-b1m5id.TestTwoUrlsWithNilContext(0xc42007e180)
	/tmp/d20170109-30451-b1m5id/solution_test.go:377 +0x2f3
testing.tRunner(0xc42007e180, 0x6f32c0)
	/usr/local/go/src/testing/testing.go:610 +0x81
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:646 +0x2ec
exit status 2
FAIL	_/tmp/d20170109-30451-b1m5id	0.015s
PASS
ok  	_/tmp/d20170109-30451-b1m5id	0.092s
PASS
ok  	_/tmp/d20170109-30451-b1m5id	0.173s
PASS
ok  	_/tmp/d20170109-30451-b1m5id	0.091s
PASS
ok  	_/tmp/d20170109-30451-b1m5id	0.129s
PASS
ok  	_/tmp/d20170109-30451-b1m5id	0.047s
PASS
ok  	_/tmp/d20170109-30451-b1m5id	0.067s
panic: test timed out after 1s

goroutine 97 [running]:
panic(0x66b220, 0xc42032d5b0)
	/usr/local/go/src/runtime/panic.go:500 +0x1a1
testing.startAlarm.func1()
	/usr/local/go/src/testing/testing.go:918 +0x10b
created by time.goFunc
	/usr/local/go/src/time/sleep.go:154 +0x44

goroutine 1 [chan receive]:
testing.(*T).Run(0xc42007c0c0, 0x6cea33, 0xb, 0x6f3248, 0xc42004bd01)
	/usr/local/go/src/testing/testing.go:647 +0x316
testing.RunTests.func1(0xc42007c0c0)
	/usr/local/go/src/testing/testing.go:793 +0x6d
testing.tRunner(0xc42007c0c0, 0xc42004be20)
	/usr/local/go/src/testing/testing.go:610 +0x81
testing.RunTests(0x6f3470, 0x807140, 0x11, 0x11, 0x7efe545c1000)
	/usr/local/go/src/testing/testing.go:799 +0x2f5
testing.(*M).Run(0xc42004bee8, 0x689360)
	/usr/local/go/src/testing/testing.go:743 +0x85
main.main()
	_/tmp/d20170109-30451-b1m5id/_test/_testmain.go:86 +0xc6

goroutine 17 [syscall, locked to thread]:
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2086 +0x1

goroutine 6 [semacquire]:
sync.runtime_notifyListWait(0xc42007c340, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42007c330)
	/usr/local/go/src/sync/cond.go:57 +0x80
io.(*pipe).read(0xc42007c300, 0xc420016400, 0x1, 0x200, 0x0, 0x0, 0x0)
	/usr/local/go/src/io/pipe.go:47 +0x102
io.(*PipeReader).Read(0xc4200e6820, 0xc420016400, 0x1, 0x200, 0xc4200188c0, 0x0, 0x0)
	/usr/local/go/src/io/pipe.go:129 +0x4c
_/tmp/d20170109-30451-b1m5id.(*Reader).Read(0xc4200e6820, 0xc420016400, 0x1, 0x200, 0xc420016400, 0x200, 0x200)
	/tmp/d20170109-30451-b1m5id/solution.go:25 +0x4b
testing/iotest.(*oneByteReader).Read(0xc4200139b0, 0xc420016400, 0x200, 0x200, 0xc4200d3f48, 0xc4200ee000, 0xc4200d1678)
	/usr/local/go/src/testing/iotest/reader.go:25 +0x64
bytes.(*Buffer).ReadFrom(0xc4200d1f98, 0x7eb6e0, 0xc4200139b0, 0x1f4, 0x1f4, 0x7ea860)
	/usr/local/go/src/bytes/buffer.go:176 +0x155
_/tmp/d20170109-30451-b1m5id.TestLingchi(0xc42007c180)
	/tmp/d20170109-30451-b1m5id/solution_test.go:819 +0x3ce
testing.tRunner(0xc42007c180, 0x6f3248)
	/usr/local/go/src/testing/testing.go:610 +0x81
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:646 +0x2ec

goroutine 7 [IO wait]:
net.runtime_pollWait(0x7efe54566178, 0x72, 0x0)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420014370, 0x72, 0xc4200285e0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420014370, 0x7ec8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc420014310, 0x0, 0x7eb4a0, 0xc4200da0a0)
	/usr/local/go/src/net/fd_unix.go:419 +0x238
net.(*TCPListener).accept(0xc42002a048, 0x43418e, 0xc420028690, 0x52f07d)
	/usr/local/go/src/net/tcpsock_posix.go:132 +0x2e
net.(*TCPListener).Accept(0xc42002a048, 0x6f3650, 0xc420102000, 0x7ef420, 0xc4200e2060)
	/usr/local/go/src/net/tcpsock.go:222 +0x49
net/http.(*Server).Serve(0xc42001a280, 0x7eeba0, 0xc42002a048, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:2273 +0x1ce
net/http/httptest.(*Server).goServe.func1(0xc42005e4e0)
	/usr/local/go/src/net/http/httptest/server.go:235 +0x6d
created by net/http/httptest.(*Server).goServe
	/usr/local/go/src/net/http/httptest/server.go:236 +0x5c

goroutine 8 [sleep]:
time.Sleep(0x1312d00)
	/usr/local/go/src/runtime/time.go:59 +0xe1
_/tmp/d20170109-30451-b1m5id.DownloadFile.func1(0xc420013990, 0xc4200ee000, 0x1f4, 0x200, 0x1f4, 0xc42001a300, 0xc42002a050, 0xc4200e6820)
	/tmp/d20170109-30451-b1m5id/solution.go:211 +0x279
created by _/tmp/d20170109-30451-b1m5id.DownloadFile
	/tmp/d20170109-30451-b1m5id/solution.go:333 +0x48f

goroutine 11 [IO wait]:
net.runtime_pollWait(0x7efe545660b8, 0x72, 0x5)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420014450, 0x72, 0xc42003b9d0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420014450, 0x7ec8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4200143f0, 0xc420083000, 0x1000, 0x1000, 0x0, 0x7ec8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42002a078, 0xc420083000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc42000a900, 0xc420083000, 0x1000, 0x1000, 0x544890, 0xc42003bb58, 0x40474d)
	/usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc42005eae0)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc42005eae0, 0x1, 0xc42003bbbd, 0x1, 0x0, 0xc420321740, 0x0)
	/usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc42000a900)
	/usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 12 [select]:
net/http.(*persistConn).writeLoop(0xc42000a900)
	/usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 18 [IO wait]:
net.runtime_pollWait(0x7efe54565ff8, 0x72, 0x6)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4200fe060, 0x72, 0xc4200367b0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4200fe060, 0x7ec8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4200fe000, 0xc42010a000, 0x1000, 0x1000, 0x0, 0x7ec8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420100000, 0xc42010a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
net/http.(*connReader).Read(0xc4200da0c0, 0xc42010a000, 0x1000, 0x1000, 0x521b39, 0x801a70, 0x0)
	/usr/local/go/src/net/http/server.go:586 +0x144
bufio.(*Reader).fill(0xc420108000)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).ReadSlice(0xc420108000, 0xa, 0x0, 0x1e, 0x6, 0x0, 0x0)
	/usr/local/go/src/bufio/bufio.go:330 +0xb5
bufio.(*Reader).ReadLine(0xc420108000, 0xc420340a50, 0xf0, 0xf0, 0x6c37e0, 0x4a1b83, 0x809d78)
	/usr/local/go/src/bufio/bufio.go:359 +0x37
net/textproto.(*Reader).readLineSlice(0xc4200e2150, 0xc420036aa8, 0xc420036aa8, 0x4106d8, 0xf0, 0x6c37e0)
	/usr/local/go/src/net/textproto/reader.go:55 +0x5e
net/textproto.(*Reader).ReadLine(0xc4200e2150, 0xc420340a50, 0xc420036b20, 0x401863, 0xc420036c78)
	/usr/local/go/src/net/textproto/reader.go:36 +0x2f
net/http.readRequest(0xc420108000, 0xc420036c00, 0xc420340a50, 0x0, 0x0)
	/usr/local/go/src/net/http/request.go:793 +0xa5
net/http.(*conn).readRequest(0xc420102000, 0x7ef360, 0xc4200d6140, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:765 +0x10d
net/http.(*conn).serve(0xc420102000, 0x7ef360, 0xc4200d6140)
	/usr/local/go/src/net/http/server.go:1532 +0x3d3
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d
exit status 2
FAIL	_/tmp/d20170109-30451-b1m5id	1.009s
panic: test timed out after 1s

goroutine 19 [running]:
panic(0x66b220, 0xc42015e340)
	/usr/local/go/src/runtime/panic.go:500 +0x1a1
testing.startAlarm.func1()
	/usr/local/go/src/testing/testing.go:918 +0x10b
created by time.goFunc
	/usr/local/go/src/time/sleep.go:154 +0x44

goroutine 1 [chan receive]:
testing.(*T).Run(0xc42007c0c0, 0x6d60a9, 0x21, 0x6f32a0, 0xc42004bd01)
	/usr/local/go/src/testing/testing.go:647 +0x316
testing.RunTests.func1(0xc42007c0c0)
	/usr/local/go/src/testing/testing.go:793 +0x6d
testing.tRunner(0xc42007c0c0, 0xc42004be20)
	/usr/local/go/src/testing/testing.go:610 +0x81
testing.RunTests(0x6f3470, 0x807140, 0x11, 0x11, 0x7f555815f000)
	/usr/local/go/src/testing/testing.go:799 +0x2f5
testing.(*M).Run(0xc42004bee8, 0x689360)
	/usr/local/go/src/testing/testing.go:743 +0x85
main.main()
	_/tmp/d20170109-30451-b1m5id/_test/_testmain.go:86 +0xc6

goroutine 17 [syscall, locked to thread]:
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2086 +0x1

goroutine 6 [semacquire]:
sync.runtime_notifyListWait(0xc42007c340, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42007c330)
	/usr/local/go/src/sync/cond.go:57 +0x80
io.(*pipe).read(0xc42007c300, 0xc420016400, 0x1, 0x200, 0x0, 0x0, 0x0)
	/usr/local/go/src/io/pipe.go:47 +0x102
io.(*PipeReader).Read(0xc4200bf640, 0xc420016400, 0x1, 0x200, 0xc420018880, 0x0, 0x0)
	/usr/local/go/src/io/pipe.go:129 +0x4c
_/tmp/d20170109-30451-b1m5id.(*Reader).Read(0xc4200bf640, 0xc420016400, 0x1, 0x200, 0xc420016400, 0x200, 0x200)
	/tmp/d20170109-30451-b1m5id/solution.go:25 +0x4b
testing/iotest.(*oneByteReader).Read(0xc4200134b0, 0xc420016400, 0x200, 0x200, 0xc42004df58, 0xc4200d6000, 0xc42004d580)
	/usr/local/go/src/testing/iotest/reader.go:25 +0x64
bytes.(*Buffer).ReadFrom(0xc42004d8a8, 0x7eb6e0, 0xc4200134b0, 0x64, 0x64, 0x7ea860)
	/usr/local/go/src/bytes/buffer.go:176 +0x155
_/tmp/d20170109-30451-b1m5id.TestSlowLingchiWithMaxConnections(0xc42007c180)
	/tmp/d20170109-30451-b1m5id/solution_test.go:900 +0x4fd
testing.tRunner(0xc42007c180, 0x6f32a0)
	/usr/local/go/src/testing/testing.go:610 +0x81
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:646 +0x2ec

goroutine 7 [IO wait]:
net.runtime_pollWait(0x7f5558104178, 0x72, 0x0)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420014370, 0x72, 0xc4200285e0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420014370, 0x7ec8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc420014310, 0x0, 0x7eb4a0, 0xc4200dc0a0)
	/usr/local/go/src/net/fd_unix.go:419 +0x238
net.(*TCPListener).accept(0xc42002a048, 0x43418e, 0xc420028690, 0x52f07d)
	/usr/local/go/src/net/tcpsock_posix.go:132 +0x2e
net.(*TCPListener).Accept(0xc42002a048, 0x6f3650, 0xc4200f4000, 0x7ef420, 0xc4200e2060)
	/usr/local/go/src/net/tcpsock.go:222 +0x49
net/http.(*Server).Serve(0xc42001a300, 0x7eeba0, 0xc42002a048, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:2273 +0x1ce
net/http/httptest.(*Server).goServe.func1(0xc42005e4e0)
	/usr/local/go/src/net/http/httptest/server.go:235 +0x6d
created by net/http/httptest.(*Server).goServe
	/usr/local/go/src/net/http/httptest/server.go:236 +0x5c

goroutine 8 [select]:
net/http.(*persistConn).roundTrip(0xc42000aa00, 0xc4201568c0, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:1840 +0x93b
net/http.(*Transport).RoundTrip(0xc4200c2000, 0xc4201641e0, 0xc4200c2000, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:380 +0x4ee
net/http.send(0xc4201641e0, 0x7eaea0, 0xc4200c2000, 0x0, 0x0, 0x0, 0x8, 0xc420047c10, 0xc42002a6f8)
	/usr/local/go/src/net/http/client.go:256 +0x15f
net/http.(*Client).send(0x80a100, 0xc4201641e0, 0x0, 0x0, 0x0, 0xc42002a6f8, 0x0, 0x1)
	/usr/local/go/src/net/http/client.go:146 +0x102
net/http.(*Client).doFollowingRedirects(0x80a100, 0xc4201641e0, 0x6f37e8, 0x19, 0x0, 0x0)
	/usr/local/go/src/net/http/client.go:528 +0x5e5
net/http.(*Client).Head(0x80a100, 0xc4200bf5e0, 0x19, 0xc420166000, 0x0, 0x0)
	/usr/local/go/src/net/http/client.go:635 +0x93
net/http.Head(0xc4200bf5e0, 0x19, 0xc420166000, 0x0, 0x0)
	/usr/local/go/src/net/http/client.go:619 +0x41
_/tmp/d20170109-30451-b1m5id.getContentLength(0xc4200d6000, 0x64, 0x80, 0xc420018980)
	/tmp/d20170109-30451-b1m5id/solution.go:82 +0x5e
_/tmp/d20170109-30451-b1m5id.DownloadFile.func1(0xc420013490, 0xc4200d6000, 0x64, 0x80, 0x14, 0xc42001a380, 0xc42002a050, 0xc4200bf640)
	/tmp/d20170109-30451-b1m5id/solution.go:197 +0xf0
created by _/tmp/d20170109-30451-b1m5id.DownloadFile
	/tmp/d20170109-30451-b1m5id/solution.go:333 +0x48f

goroutine 11 [IO wait]:
net.runtime_pollWait(0x7f55581040b8, 0x72, 0x5)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420014450, 0x72, 0xc42003b9d0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420014450, 0x7ec8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4200143f0, 0xc420083000, 0x1000, 0x1000, 0x0, 0x7ec8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42002a078, 0xc420083000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc42000aa00, 0xc420083000, 0x1000, 0x1000, 0x30, 0xc42003bb58, 0x43b23c)
	/usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc42005eae0)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc42005eae0, 0x1, 0x0, 0x1, 0x1, 0xc42015c5a0, 0x0)
	/usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc42000aa00)
	/usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 18 [sleep]:
time.Sleep(0x989680)
	/usr/local/go/src/runtime/time.go:59 +0xe1
_/tmp/d20170109-30451-b1m5id.TestSlowLingchiWithMaxConnections.func1(0x7eeda0, 0xc4201601a0, 0xc4201640f0)
	/tmp/d20170109-30451-b1m5id/solution_test.go:858 +0xe7
net/http.HandlerFunc.ServeHTTP(0xc4200188c0, 0x7eeda0, 0xc4201601a0, 0xc4201640f0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.serverHandler.ServeHTTP(0xc42001a300, 0x7eeda0, 0xc4201601a0, 0xc4201640f0)
	/usr/local/go/src/net/http/server.go:2202 +0x7d
net/http.(*conn).serve(0xc4200f4000, 0x7ef360, 0xc4200d4140)
	/usr/local/go/src/net/http/server.go:1579 +0x4b7
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d

goroutine 12 [select]:
net/http.(*persistConn).writeLoop(0xc42000aa00)
	/usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1063 +0x50e
exit status 2
FAIL	_/tmp/d20170109-30451-b1m5id	1.009s
panic: test timed out after 1s

goroutine 86 [running]:
panic(0x66b220, 0xc4201729e0)
	/usr/local/go/src/runtime/panic.go:500 +0x1a1
testing.startAlarm.func1()
	/usr/local/go/src/testing/testing.go:918 +0x10b
created by time.goFunc
	/usr/local/go/src/time/sleep.go:154 +0x44

goroutine 1 [chan receive]:
testing.(*T).Run(0xc42007c0c0, 0x6da91a, 0x34, 0x6f3298, 0xc42004bd01)
	/usr/local/go/src/testing/testing.go:647 +0x316
testing.RunTests.func1(0xc42007c0c0)
	/usr/local/go/src/testing/testing.go:793 +0x6d
testing.tRunner(0xc42007c0c0, 0xc42004be20)
	/usr/local/go/src/testing/testing.go:610 +0x81
testing.RunTests(0x6f3470, 0x807140, 0x11, 0x11, 0x7f8d8b74b000)
	/usr/local/go/src/testing/testing.go:799 +0x2f5
testing.(*M).Run(0xc42004bee8, 0x689360)
	/usr/local/go/src/testing/testing.go:743 +0x85
main.main()
	_/tmp/d20170109-30451-b1m5id/_test/_testmain.go:86 +0xc6

goroutine 17 [syscall, locked to thread]:
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2086 +0x1

goroutine 6 [semacquire]:
sync.runtime_notifyListWait(0xc42007c280, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42007c270)
	/usr/local/go/src/sync/cond.go:57 +0x80
io.(*pipe).read(0xc42007c240, 0xc420016400, 0x1, 0x200, 0x0, 0x0, 0x0)
	/usr/local/go/src/io/pipe.go:47 +0x102
io.(*PipeReader).Read(0xc4200bf000, 0xc420016400, 0x1, 0x200, 0xc420018940, 0x0, 0x0)
	/usr/local/go/src/io/pipe.go:129 +0x4c
_/tmp/d20170109-30451-b1m5id.(*Reader).Read(0xc4200bf000, 0xc420016400, 0x1, 0x200, 0xc420016400, 0x200, 0x200)
	/tmp/d20170109-30451-b1m5id/solution.go:25 +0x4b
testing/iotest.(*oneByteReader).Read(0xc420013460, 0xc420016400, 0x200, 0x200, 0xc42003bf68, 0xc4200da000, 0xc42003b958)
	/usr/local/go/src/testing/iotest/reader.go:25 +0x64
bytes.(*Buffer).ReadFrom(0xc42003bbd8, 0x7eb6e0, 0xc420013460, 0x32, 0x32, 0x7ea860)
	/usr/local/go/src/bytes/buffer.go:176 +0x155
_/tmp/d20170109-30451-b1m5id.TestSlowLingchiWithBothMaxConnectionsAndALotOfErrors(0xc42007c180)
	/tmp/d20170109-30451-b1m5id/solution_test.go:998 +0x593
testing.tRunner(0xc42007c180, 0x6f3298)
	/usr/local/go/src/testing/testing.go:610 +0x81
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:646 +0x2ec

goroutine 7 [IO wait]:
net.runtime_pollWait(0x7f8d8b6f0178, 0x72, 0x0)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420014370, 0x72, 0xc4200285e0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420014370, 0x7ec8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc420014310, 0x0, 0x7eb4a0, 0xc4200dc0a0)
	/usr/local/go/src/net/fd_unix.go:419 +0x238
net.(*TCPListener).accept(0xc42002a048, 0x43418e, 0xc420028690, 0x52f07d)
	/usr/local/go/src/net/tcpsock_posix.go:132 +0x2e
net.(*TCPListener).Accept(0xc42002a048, 0x6f3650, 0xc4200f6000, 0x7ef420, 0xc4200e2060)
	/usr/local/go/src/net/tcpsock.go:222 +0x49
net/http.(*Server).Serve(0xc42001a300, 0x7eeba0, 0xc42002a048, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:2273 +0x1ce
net/http/httptest.(*Server).goServe.func1(0xc42005e4e0)
	/usr/local/go/src/net/http/httptest/server.go:235 +0x6d
created by net/http/httptest.(*Server).goServe
	/usr/local/go/src/net/http/httptest/server.go:236 +0x5c

goroutine 8 [sleep]:
time.Sleep(0x1312d00)
	/usr/local/go/src/runtime/time.go:59 +0xe1
_/tmp/d20170109-30451-b1m5id.DownloadFile.func1(0xc420013440, 0xc4200da000, 0x32, 0x40, 0x14, 0xc42001a380, 0xc42002a050, 0xc4200bf000)
	/tmp/d20170109-30451-b1m5id/solution.go:211 +0x279
created by _/tmp/d20170109-30451-b1m5id.DownloadFile
	/tmp/d20170109-30451-b1m5id/solution.go:333 +0x48f

goroutine 11 [IO wait]:
net.runtime_pollWait(0x7f8d8b6f00b8, 0x72, 0x5)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420014450, 0x72, 0xc42003c9d0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420014450, 0x7ec8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4200143f0, 0xc420083000, 0x1000, 0x1000, 0x0, 0x7ec8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42002a078, 0xc420083000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc42000aa00, 0xc420083000, 0x1000, 0x1000, 0x30, 0xc42003cb58, 0x43b23c)
	/usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc42005eae0)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc42005eae0, 0x1, 0x0, 0x1, 0x0, 0xc420166de0, 0x0)
	/usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc42000aa00)
	/usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 12 [select]:
net/http.(*persistConn).writeLoop(0xc42000aa00)
	/usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 18 [IO wait]:
net.runtime_pollWait(0x7f8d8b6efff8, 0x72, 0x6)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4200f0060, 0x72, 0xc4200367b0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4200f0060, 0x7ec8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4200f0000, 0xc4200fc000, 0x1000, 0x1000, 0x0, 0x7ec8a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4200f2000, 0xc4200fc000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
net/http.(*connReader).Read(0xc4200dc0c0, 0xc4200fc000, 0x1000, 0x1000, 0xc420036918, 0x6d37cc, 0x19)
	/usr/local/go/src/net/http/server.go:586 +0x144
bufio.(*Reader).fill(0xc4200fa000)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).ReadSlice(0xc4200fa000, 0xa, 0x0, 0x1e, 0xa, 0x33, 0x0)
	/usr/local/go/src/bufio/bufio.go:330 +0xb5
bufio.(*Reader).ReadLine(0xc4200fa000, 0xc42015fa40, 0xf0, 0xf0, 0x6c37e0, 0x4a1b83, 0x809d78)
	/usr/local/go/src/bufio/bufio.go:359 +0x37
net/textproto.(*Reader).readLineSlice(0xc42000d740, 0xc420036aa8, 0xc420036aa8, 0x4106d8, 0xf0, 0x6c37e0)
	/usr/local/go/src/net/textproto/reader.go:55 +0x5e
net/textproto.(*Reader).ReadLine(0xc42000d740, 0xc42015fa40, 0xc420036b20, 0x401863, 0xc420036c78)
	/usr/local/go/src/net/textproto/reader.go:36 +0x2f
net/http.readRequest(0xc4200fa000, 0xc420036c00, 0xc42015fa40, 0x0, 0x0)
	/usr/local/go/src/net/http/request.go:793 +0xa5
net/http.(*conn).readRequest(0xc4200f6000, 0x7ef360, 0xc4200d4140, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:765 +0x10d
net/http.(*conn).serve(0xc4200f6000, 0x7ef360, 0xc4200d4140)
	/usr/local/go/src/net/http/server.go:1532 +0x3d3
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d
exit status 2
FAIL	_/tmp/d20170109-30451-b1m5id	1.009s

История (1 версия и 0 коментара)

Георги обнови решението на 03.01.2017 13:56 (преди над 1 година)

+package main
+
+import (
+ "context"
+ "errors"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "strconv"
+ "sync"
+ "time"
+)
+
+type Reader struct {
+ pipeReader io.PipeReader
+ pipeWriter io.PipeWriter
+ err error
+}
+
+func NewReader(pipeReader io.PipeReader, pipeWriter io.PipeWriter) *Reader {
+ return &Reader{pipeReader, pipeWriter, nil}
+}
+
+func (r *Reader) Read(p []byte) (n int, err error) {
+ n, err = r.pipeReader.Read(p)
+ if err == nil {
+ err = r.err
+ }
+
+ return n, err
+}
+
+func (r *Reader) Write(bytes []byte, err error) {
+ r.err = err
+ r.pipeWriter.Write(bytes)
+}
+
+func (r *Reader) SetError(err error) error {
+ r.err = err
+ return err
+}
+
+const maxUrls int = 128
+
+type Url struct {
+ id int
+ url string
+}
+
+type Task struct {
+ url Url
+ rangeStart int
+ rangeEnd int
+}
+
+type TaskResult struct {
+ task *Task
+ taskSkipped bool
+ result []byte
+}
+
+type Queue []*Task
+
+func (q *Queue) Push(n *Task) {
+ *q = append(*q, n)
+}
+
+func (q *Queue) Pop() (n *Task) {
+ n = (*q)[0]
+ *q = (*q)[1:]
+ return
+}
+
+func (q *Queue) Len() int {
+ return len(*q)
+}
+
+func getContentLength(urls []Url) int {
+ contentLength := -1
+
+ for _, url := range urls {
+ response, err := http.Head(url.url)
+ if err != nil {
+ continue
+ }
+ contentLength = int(response.ContentLength)
+ }
+ if contentLength < 0 {
+ contentLength = 0
+ }
+
+ return contentLength
+}
+
+func generateTasks(urls []Url, rangeStart int, rangeEnd int) Queue {
+ var tasks Queue
+
+ numberOfUrls := len(urls)
+ bucketSizes := make([]int, numberOfUrls)
+
+ evenLength := int(float32(rangeEnd-rangeStart+1) / float32(numberOfUrls))
+ for i := 0; i < numberOfUrls; i++ {
+ bucketSizes[i] = evenLength
+ }
+
+ surPlus := (rangeEnd - rangeStart + 1) % numberOfUrls
+ for i := 0; surPlus > 0; surPlus-- {
+ bucketSizes[i] += 1
+ i = (i + 1) % numberOfUrls
+ }
+
+ k := rangeStart
+ for i := 0; i < numberOfUrls && k <= rangeEnd; i++ {
+ rangeLow := k
+ rangeHigh := k + bucketSizes[i] - 1
+ k += bucketSizes[i]
+
+ url := urls[i]
+ tasks.Push(&Task{url, rangeLow, rangeHigh})
+ }
+
+ return tasks
+}
+
+func getRangedRequest(httpClient *http.Client, url string, rangeStart int, rangeEnd int) ([]byte, bool) {
+ request, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ return nil, false
+ }
+
+ request.Header.Add("Range", "bytes="+strconv.Itoa(rangeStart)+"-"+strconv.Itoa(rangeEnd))
+ response, err := httpClient.Do(request)
+
+ if err != nil {
+ return nil, false
+ }
+
+ defer response.Body.Close()
+ body, err := ioutil.ReadAll(response.Body)
+ if err != nil {
+ return body, true
+ }
+ statusCode := int(response.StatusCode)
+ if statusCode >= 300 && statusCode < 600 {
+ return nil, false
+ }
+
+ repeat := false
+ if rangeEnd != 1 && int(response.ContentLength)-1 != (rangeEnd-rangeStart) {
+ repeat = true
+ }
+
+ return body, repeat
+}
+
+func getValidUrls(urls []Url, invalidUrls [maxUrls]bool) []Url {
+ validUrls := make([]Url, 0)
+ for _, url := range urls {
+ if invalidUrls[url.id] {
+ continue
+ }
+ validUrls = append(validUrls[:], url)
+ }
+
+ return validUrls
+}
+
+func DownloadFile(ctx context.Context, urlStrings []string) io.Reader {
+ pr, pw := io.Pipe()
+ outputReader := NewReader(*pr, *pw)
+
+ numberOfUrls := len(urlStrings)
+ maxConcurrency := 0
+ ctxMaxConcurrency, ok := ctx.Value("max-connections").(int)
+ if ok {
+ maxConcurrency = ctxMaxConcurrency
+ }
+ if maxConcurrency == 0 || maxConcurrency >= numberOfUrls {
+ maxConcurrency = numberOfUrls
+ }
+
+ urls := make([]Url, 0)
+ for id, urlString := range urlStrings {
+ urls = append(urls[:], Url{id, urlString})
+ }
+
+ var invalidUrls [maxUrls]bool
+
+ go func(outputReader *Reader) {
+ var wg sync.WaitGroup
+ var errOnce sync.Once
+ cancel := func() {}
+ if ctx != nil {
+ ctx, cancel = context.WithCancel(ctx)
+ }
+
+ contentLength := getContentLength(urls)
+ rangeEnd := contentLength - 1
+ if contentLength-1 < 0 {
+ rangeEnd = 1
+ }
+ tasksQueue := generateTasks(urls, 0, rangeEnd)
+
+ httpClient := &http.Client{}
+ throttling := make(chan struct{}, maxConcurrency)
+ results := make(chan TaskResult, maxUrls)
+
+ justAdded := false
+ stopMainFor := false
+ for {
+ time.Sleep(20 * time.Millisecond)
+ if stopMainFor {
+ break
+ }
+
+ if tasksQueue.Len() == 0 {
+ continue
+ }
+
+ task := tasksQueue.Pop()
+ throttling <- struct{}{}
+
+ if !justAdded {
+ wg.Add(1)
+ }
+ if justAdded {
+ justAdded = false
+ }
+
+ go func(task *Task, wg *sync.WaitGroup) {
+ defer func() {
+ if !justAdded {
+ wg.Done()
+ }
+ }()
+
+ processTask := func() error {
+ if ctx != nil {
+ select {
+ case <-ctx.Done():
+ return outputReader.SetError(ctx.Err())
+ default:
+ }
+ }
+
+ var taskBytesOut []byte
+ rangeStart := task.rangeStart
+ rangeEnd := task.rangeEnd
+ for {
+ taskBytes, repeat := getRangedRequest(httpClient, task.url.url, rangeStart, rangeEnd)
+ if taskBytes == nil {
+ invalidUrls[task.url.id] = true
+ validUrls := getValidUrls(urls, invalidUrls)
+ if len(validUrls) > 0 {
+ newTasksQueue := generateTasks(validUrls, rangeStart, rangeEnd)
+ for newTasksQueue.Len() > 0 {
+ tasksQueue.Push(newTasksQueue.Pop())
+ justAdded = true
+ }
+ }
+ break
+ }
+
+ if len(taskBytesOut) == 0 {
+ taskBytesOut = taskBytes
+ } else {
+ for _, taskByte := range taskBytes {
+ taskBytesOut = append(taskBytesOut, taskByte)
+ }
+ }
+
+ if !repeat {
+ break
+ }
+
+ if (rangeStart + len(taskBytes)) > rangeEnd {
+ continue
+ } else {
+ rangeStart += len(taskBytes)
+ }
+ }
+
+ results <- TaskResult{task, invalidUrls[task.url.id], taskBytesOut}
+
+ return nil
+ }
+
+ if err := processTask(); err != nil {
+ errOnce.Do(func() {
+ cancel()
+ })
+ }
+
+ <-throttling
+
+ go func() {
+ if tasksQueue.Len() == 0 {
+ stopMainFor = true
+ }
+ }()
+ }(task, &wg)
+ }
+
+ go func(wg *sync.WaitGroup) {
+ defer pw.Close()
+ wg.Wait()
+ close(throttling)
+ close(results)
+
+ taskCount := 0
+
+ totalOutput := make([]byte, contentLength)
+ receivedBytes := 0
+ for result := range results {
+ for index, taskByte := range result.result {
+ totalOutput[index+result.task.rangeStart] = taskByte
+ receivedBytes += 1
+ }
+ taskCount += 1
+ }
+
+ var err error = nil
+ validUrls := getValidUrls(urls, invalidUrls)
+ if len(validUrls) == 0 {
+ err = errors.New("no valid urls")
+ }
+ if taskCount == 0 {
+ outputReader.Write(nil, err)
+ } else {
+ outputReader.Write(totalOutput[:receivedBytes], err)
+ }
+ }(&wg)
+ }(outputReader)
+
+ return outputReader
+}