Програмиране с Go

Курс във Факултета по Математика и Информатика към СУ

Решение на HTTP сваляч от Георги Иванов

Обратно към всички решения

Към профила на Георги Иванов

Резултати

  • 4 точки от тестове
  • 0 бонус точки
  • 4 точки общо
  • 6 успешни тест(а)
  • 11 неуспешни тест(а)

Код

package main
import (
"context"
"io"
"net/http"
"sync"
"strconv"
"io/ioutil"
"fmt"
)
type Reader interface {
Read(p []byte) (n int, err error)
}
type File struct {
resp []byte
done bool
lastInd int64
isValid bool
}
func (f *File) Read(p []byte) (n int, err error) {
if !f.isValid {
return 0, fmt.Errorf("no valid urls")
}
if f.done || len(f.resp) == 0 {
return 0, io.EOF
}
if int64(len(p)) >= int64(len(f.resp)) - f.lastInd {
n := int64(len(f.resp)) - f.lastInd
for i := f.lastInd; i < n; i++ {
p[i - f.lastInd] = f.resp[i]
}
f.done = true;
return int(n), io.EOF
} else {
n = len(p)
for i := 0; i < n; i++ {
p[i] = f.resp[f.lastInd]
f.lastInd++
}
return len(p), nil
}
}
func splitInterval(size int64, parts int64) [4]int64 {
lowerSize := size / parts
remainder := size - parts*lowerSize
return [4]int64{parts - remainder, lowerSize, remainder, lowerSize + 1}
}
func getLength(urls []string, isValid []bool, cntValid *int64) (int64, error) {
var len int64
for ind, url := range urls {
res, err := http.Head(url)
if err == nil {
isValid[ind] = true
*cntValid++
len = res.ContentLength
}
}
fmt.Println(len)
if *cntValid == 0 {
return 0, fmt.Errorf("no valid urls")
} else {
return len, nil
}
}
func DownloadFile(ctx context.Context, urls []string) io.Reader {
isValid := make([]bool, len(urls))
var cntValid int64
length, err := getLength(urls, isValid, &cntValid)
if err != nil {
return &File{resp: nil, done: true, lastInd: 0, isValid: false}
}
validIdx := make([]int, cntValid)
cnt := 0
for ind, val := range isValid {
if val {
validIdx[cnt] = ind
cnt++
}
}
splits := splitInterval(length, cntValid)
maxConnections := cntValid
responses := make([][]byte, cntValid)
chMaxConnections := make(chan struct{}, maxConnections)
var wg sync.WaitGroup
var mtx sync.Mutex
cntInvalidUrls := 0;
downloadRange := func(start int64, end int64, ind int64) {
defer wg.Done()
client := &http.Client{}
req, _ := http.NewRequest("GET", urls[ind], nil)
rangeHeader := "bytes=" + strconv.FormatInt(start, 10) + "-" + strconv.FormatInt(end-1, 10)
req.Header.Add("Range", rangeHeader)
resp, _ := client.Do(req)
defer resp.Body.Close()
if resp.StatusCode >= 400 && resp.StatusCode < 600 {
mtx.Lock()
cntInvalidUrls++
mtx.Unlock()
}
r, _ := ioutil.ReadAll(resp.Body)
mtx.Lock()
responses[ind] = r
mtx.Unlock()
<-chMaxConnections
}
//first size ranges
for ind := 0; int64(ind) < splits[0]; ind++ {
chMaxConnections <- struct{}{}
wg.Add(1)
go downloadRange(int64(ind) * splits[1], (int64(ind) + 1) * splits[1], int64(validIdx[ind]))
}
//second size ranges
for ind := 0; int64(ind) < splits[2]; ind++ {
chMaxConnections <- struct{}{}
wg.Add(1)
go downloadRange((splits[0] * splits[1]) + int64(ind) * splits[3],
(splits[0] * splits[1]) + (int64(ind) + 1) * splits[3],
int64(validIdx[splits[0] + int64(ind)]))
}
wg.Wait()
response := make([]byte, 0)
for ind := 0; int64(ind) < cntValid; ind++ {
response = append(response, responses[ind]...)
}
return &File{resp: response, done: false, lastInd: 0, isValid: true}
}

Лог от изпълнението

0
PASS
ok  	_/tmp/d20170109-30451-uabzou	0.006s
37
PASS
ok  	_/tmp/d20170109-30451-uabzou	0.008s
panic: test timed out after 1s

goroutine 19 [running]:
panic(0x668d80, 0xc4200d8090)
	/usr/local/go/src/runtime/panic.go:500 +0x1a1
testing.startAlarm.func1()
	/usr/local/go/src/testing/testing.go:918 +0x10b
created by time.goFunc
	/usr/local/go/src/time/sleep.go:154 +0x44

goroutine 1 [chan receive]:
testing.(*T).Run(0xc42007e0c0, 0x6d5ec2, 0x2a, 0x6ef528, 0xc42004bd01)
	/usr/local/go/src/testing/testing.go:647 +0x316
testing.RunTests.func1(0xc42007e0c0)
	/usr/local/go/src/testing/testing.go:793 +0x6d
testing.tRunner(0xc42007e0c0, 0xc42004be20)
	/usr/local/go/src/testing/testing.go:610 +0x81
testing.RunTests(0x6ef718, 0x802140, 0x11, 0x11, 0x7fd490dcd000)
	/usr/local/go/src/testing/testing.go:799 +0x2f5
testing.(*M).Run(0xc42004bee8, 0x686ba0)
	/usr/local/go/src/testing/testing.go:743 +0x85
main.main()
	_/tmp/d20170109-30451-uabzou/_test/_testmain.go:86 +0xc6

goroutine 17 [syscall, locked to thread]:
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2086 +0x1

goroutine 6 [select]:
net/http.(*persistConn).roundTrip(0xc42000a900, 0xc4200c0a60, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:1840 +0x93b
net/http.(*Transport).RoundTrip(0xc4200c4000, 0xc4200c40f0, 0xc4200c4000, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:380 +0x4ee
net/http.send(0xc4200c40f0, 0x7e5ea0, 0xc4200c4000, 0x0, 0x0, 0x0, 0x8, 0xc42003ba20, 0xc42002a060)
	/usr/local/go/src/net/http/client.go:256 +0x15f
net/http.(*Client).send(0x805100, 0xc4200c40f0, 0x0, 0x0, 0x0, 0xc42002a060, 0x0, 0x1)
	/usr/local/go/src/net/http/client.go:146 +0x102
net/http.(*Client).doFollowingRedirects(0x805100, 0xc4200c40f0, 0x6efa90, 0x1c, 0x0, 0x0)
	/usr/local/go/src/net/http/client.go:528 +0x5e5
net/http.(*Client).Head(0x805100, 0xc4200c09c0, 0x1c, 0x805f20, 0x7fd490dcd000, 0x0)
	/usr/local/go/src/net/http/client.go:635 +0x93
net/http.Head(0xc4200c09c0, 0x1c, 0xc42003bc80, 0x44046b, 0x1)
	/usr/local/go/src/net/http/client.go:619 +0x41
_/tmp/d20170109-30451-uabzou.getLength(0xc4200132f0, 0x1, 0x1, 0xc4200132d5, 0x1, 0x1, 0xc42004dd28, 0x0, 0x0, 0xc4200c09c0)
	/tmp/d20170109-30451-uabzou/solution.go:58 +0x5b
_/tmp/d20170109-30451-uabzou.DownloadFile(0x7ea3a0, 0xc420012528, 0xc4200132f0, 0x1, 0x1, 0xc4200c09c0, 0x1c)
	/tmp/d20170109-30451-uabzou/solution.go:77 +0xc6
_/tmp/d20170109-30451-uabzou.TestSingleURLBlockUntilDownloadFileReturns(0xc42007e180)
	/tmp/d20170109-30451-uabzou/solution_test.go:135 +0x248
testing.tRunner(0xc42007e180, 0x6ef528)
	/usr/local/go/src/testing/testing.go:610 +0x81
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:646 +0x2ec

goroutine 7 [IO wait]:
net.runtime_pollWait(0x7fd490d72178, 0x72, 0x0)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420014370, 0x72, 0xc4200285e0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420014370, 0x7e78a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc420014310, 0x0, 0x7e64a0, 0xc4200d40a0)
	/usr/local/go/src/net/fd_unix.go:419 +0x238
net.(*TCPListener).accept(0xc42002a048, 0x43413e, 0xc420028690, 0x52b99d)
	/usr/local/go/src/net/tcpsock_posix.go:132 +0x2e
net.(*TCPListener).Accept(0xc42002a048, 0x6ef8f8, 0xc4200ee000, 0x7ea420, 0xc4200da060)
	/usr/local/go/src/net/tcpsock.go:222 +0x49
net/http.(*Server).Serve(0xc42001a300, 0x7e9ba0, 0xc42002a048, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:2273 +0x1ce
net/http/httptest.(*Server).goServe.func1(0xc4200604e0)
	/usr/local/go/src/net/http/httptest/server.go:235 +0x6d
created by net/http/httptest.(*Server).goServe
	/usr/local/go/src/net/http/httptest/server.go:236 +0x5c

goroutine 10 [IO wait]:
net.runtime_pollWait(0x7fd490d720b8, 0x72, 0x5)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420014450, 0x72, 0xc42003b9d0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420014450, 0x7e78a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4200143f0, 0xc420085000, 0x1000, 0x1000, 0x0, 0x7e78a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42002a068, 0xc420085000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc42000a900, 0xc420085000, 0x1000, 0x1000, 0x539a33, 0xc420028b78, 0xc420028b88)
	/usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc420060a20)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc420060a20, 0x1, 0xc420060ba0, 0xc420028c60, 0x0, 0x0, 0x0)
	/usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc42000a900)
	/usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 11 [select]:
net/http.(*persistConn).writeLoop(0xc42000a900)
	/usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 18 [chan receive]:
_/tmp/d20170109-30451-uabzou.TestSingleURLBlockUntilDownloadFileReturns.func1(0x7e9da0, 0xc4200dc1a0, 0xc4200f6000)
	/tmp/d20170109-30451-uabzou/solution_test.go:107 +0x81
net/http.HandlerFunc.ServeHTTP(0xc42000d140, 0x7e9da0, 0xc4200dc1a0, 0xc4200f6000)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.serverHandler.ServeHTTP(0xc42001a300, 0x7e9da0, 0xc4200dc1a0, 0xc4200f6000)
	/usr/local/go/src/net/http/server.go:2202 +0x7d
net/http.(*conn).serve(0xc4200ee000, 0x7ea360, 0xc4200d2140)
	/usr/local/go/src/net/http/server.go:1579 +0x4b7
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d
exit status 2
FAIL	_/tmp/d20170109-30451-uabzou	1.008s
37
--- FAIL: TestSingleURLCancelContextAfterHalfBytesWereServed (0.00s)
	solution_test.go:199: Expected to get error context canceled, but got '%!s(<nil>)'
FAIL
exit status 1
FAIL	_/tmp/d20170109-30451-uabzou	0.007s
0
PASS
ok  	_/tmp/d20170109-30451-uabzou	0.010s
37
--- FAIL: TestReturnOnly10Bytes (0.00s)
	solution_test.go:269: Expected to get  error with message 'no valid urls', but got '%!s(<nil>)'
FAIL
exit status 1
FAIL	_/tmp/d20170109-30451-uabzou	0.007s
37
PASS
ok  	_/tmp/d20170109-30451-uabzou	0.009s
37
PASS
ok  	_/tmp/d20170109-30451-uabzou	0.007s
37
PASS
ok  	_/tmp/d20170109-30451-uabzou	0.011s
37
panic: runtime error: index out of range

goroutine 23 [running]:
panic(0x67da00, 0xc420012110)
	/usr/local/go/src/runtime/panic.go:500 +0x1a1
_/tmp/d20170109-30451-uabzou.DownloadFile.func1(0x0, 0x25, 0x1)
	/tmp/d20170109-30451-uabzou/solution.go:118 +0x400
created by _/tmp/d20170109-30451-uabzou.DownloadFile
	/tmp/d20170109-30451-uabzou/solution.go:128 +0x518
exit status 2
FAIL	_/tmp/d20170109-30451-uabzou	0.015s
37
--- FAIL: TestTwoUrlsOneStopRespondingAfter5bytes (0.00s)
	solution_test.go:540: Expected to read 37 bytes from simple download but got 24
	solution_test.go:53: Expected result was '5468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e7365' but got '5468697320706963206f6620616c6c20726573706f6e7365'
	solution_test.go:562: It was expected that url[1] would serve 32 but have served 19 
FAIL
exit status 1
FAIL	_/tmp/d20170109-30451-uabzou	0.007s
37
--- FAIL: TestThreeUrlsOneOfWhichReturns1byteAtATimeOneOfWhichBreaksAfter2bytes (0.00s)
	solution_test.go:623: Expected to read 37 bytes from simple download but got 15
	solution_test.go:53: Expected result was '5468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e7365' but got '54686d6f73742065706963206f6620'
	solution_test.go:645: It was expected that url[1] +url[2] would serve 35 but have served 12 
	solution_test.go:650: Both urls should have been used equally but 12 were downloaded from url[1] and 1 from url[2]
FAIL
exit status 1
FAIL	_/tmp/d20170109-30451-uabzou	0.008s
37
--- FAIL: TestSingleURL1ByteARequest (0.00s)
	solution_test.go:690: Expected to read 37 bytes from simple download but got 1
	solution_test.go:53: Expected result was '5468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e7365' but got '54'
FAIL
exit status 1
FAIL	_/tmp/d20170109-30451-uabzou	0.006s
37
--- FAIL: TestTwoUrlsWithOneByteReader (0.00s)
	solution_test.go:53: Expected result was '5468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e7365' but got '5468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e7300'
FAIL
exit status 1
FAIL	_/tmp/d20170109-30451-uabzou	0.009s
500
--- FAIL: TestLingchi (0.44s)
	solution_test.go:53: Expected result was '5468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f7374206570' but got '5468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d6f7374206500'
FAIL
exit status 1
FAIL	_/tmp/d20170109-30451-uabzou	0.453s
panic: test timed out after 1s

goroutine 19 [running]:
panic(0x668d80, 0xc420156280)
	/usr/local/go/src/runtime/panic.go:500 +0x1a1
testing.startAlarm.func1()
	/usr/local/go/src/testing/testing.go:918 +0x10b
created by time.goFunc
	/usr/local/go/src/time/sleep.go:154 +0x44

goroutine 1 [chan receive]:
testing.(*T).Run(0xc42007c0c0, 0x6d32c9, 0x21, 0x6ef550, 0xc42004bd01)
	/usr/local/go/src/testing/testing.go:647 +0x316
testing.RunTests.func1(0xc42007c0c0)
	/usr/local/go/src/testing/testing.go:793 +0x6d
testing.tRunner(0xc42007c0c0, 0xc42004be20)
	/usr/local/go/src/testing/testing.go:610 +0x81
testing.RunTests(0x6ef718, 0x802140, 0x11, 0x11, 0x7f545a45c000)
	/usr/local/go/src/testing/testing.go:799 +0x2f5
testing.(*M).Run(0xc42004bee8, 0x686ba0)
	/usr/local/go/src/testing/testing.go:743 +0x85
main.main()
	_/tmp/d20170109-30451-uabzou/_test/_testmain.go:86 +0xc6

goroutine 17 [syscall, locked to thread]:
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2086 +0x1

goroutine 6 [select]:
net/http.(*persistConn).roundTrip(0xc42000a900, 0xc42014cae0, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:1840 +0x93b
net/http.(*Transport).RoundTrip(0xc4200c2000, 0xc42015c1e0, 0xc4200c2000, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:380 +0x4ee
net/http.send(0xc42015c1e0, 0x7e5ea0, 0xc4200c2000, 0x0, 0x0, 0x0, 0x8, 0xc42004d7c0, 0xc42002a6e8)
	/usr/local/go/src/net/http/client.go:256 +0x15f
net/http.(*Client).send(0x805100, 0xc42015c1e0, 0x0, 0x0, 0x0, 0xc42002a6e8, 0x0, 0x1)
	/usr/local/go/src/net/http/client.go:146 +0x102
net/http.(*Client).doFollowingRedirects(0x805100, 0xc42015c1e0, 0x6efa90, 0x19, 0x0, 0x0)
	/usr/local/go/src/net/http/client.go:528 +0x5e5
net/http.(*Client).Head(0x805100, 0xc4200bf5e0, 0x19, 0xc420160000, 0x0, 0x0)
	/usr/local/go/src/net/http/client.go:635 +0x93
net/http.Head(0xc4200bf5e0, 0x19, 0xc420160000, 0x0, 0x0)
	/usr/local/go/src/net/http/client.go:619 +0x41
_/tmp/d20170109-30451-uabzou.getLength(0xc420066d80, 0x64, 0x64, 0xc420014380, 0x64, 0x64, 0xc42004dad8, 0x0, 0xc420013400, 0xc4200bf620)
	/tmp/d20170109-30451-uabzou/solution.go:58 +0x5b
_/tmp/d20170109-30451-uabzou.DownloadFile(0x7ea420, 0xc42000d140, 0xc420066d80, 0x64, 0x64, 0xc42001348a, 0x2)
	/tmp/d20170109-30451-uabzou/solution.go:77 +0xc6
_/tmp/d20170109-30451-uabzou.TestSlowLingchiWithMaxConnections(0xc42007c180)
	/tmp/d20170109-30451-uabzou/solution_test.go:897 +0x4ac
testing.tRunner(0xc42007c180, 0x6ef550)
	/usr/local/go/src/testing/testing.go:610 +0x81
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:646 +0x2ec

goroutine 7 [IO wait]:
net.runtime_pollWait(0x7f545a401178, 0x72, 0x0)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420014370, 0x72, 0xc4200285e0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420014370, 0x7e78a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc420014310, 0x0, 0x7e64a0, 0xc4200d40a0)
	/usr/local/go/src/net/fd_unix.go:419 +0x238
net.(*TCPListener).accept(0xc42002a048, 0x43413e, 0xc420028690, 0x52b99d)
	/usr/local/go/src/net/tcpsock_posix.go:132 +0x2e
net.(*TCPListener).Accept(0xc42002a048, 0x6ef8f8, 0xc4200e4000, 0x7ea420, 0xc4200da060)
	/usr/local/go/src/net/tcpsock.go:222 +0x49
net/http.(*Server).Serve(0xc42001a300, 0x7e9ba0, 0xc42002a048, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:2273 +0x1ce
net/http/httptest.(*Server).goServe.func1(0xc42005e4e0)
	/usr/local/go/src/net/http/httptest/server.go:235 +0x6d
created by net/http/httptest.(*Server).goServe
	/usr/local/go/src/net/http/httptest/server.go:236 +0x5c

goroutine 10 [IO wait]:
net.runtime_pollWait(0x7f545a4010b8, 0x72, 0x5)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4200144c0, 0x72, 0xc42003b9d0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4200144c0, 0x7e78a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420014460, 0xc420083000, 0x1000, 0x1000, 0x0, 0x7e78a0, 0xc4200121b0)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42002a068, 0xc420083000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc42000a900, 0xc420083000, 0x1000, 0x1000, 0x30, 0xc42003bb58, 0x43b1ec)
	/usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc42005ea20)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc42005ea20, 0x1, 0x0, 0x1, 0x1, 0xc4201543c0, 0x0)
	/usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc42000a900)
	/usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 11 [select]:
net/http.(*persistConn).writeLoop(0xc42000a900)
	/usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 18 [sleep]:
time.Sleep(0x989680)
	/usr/local/go/src/runtime/time.go:59 +0xe1
_/tmp/d20170109-30451-uabzou.TestSlowLingchiWithMaxConnections.func1(0x7e9da0, 0xc420158270, 0xc42015c0f0)
	/tmp/d20170109-30451-uabzou/solution_test.go:858 +0xe7
net/http.HandlerFunc.ServeHTTP(0xc4200188c0, 0x7e9da0, 0xc420158270, 0xc42015c0f0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.serverHandler.ServeHTTP(0xc42001a300, 0x7e9da0, 0xc420158270, 0xc42015c0f0)
	/usr/local/go/src/net/http/server.go:2202 +0x7d
net/http.(*conn).serve(0xc4200e4000, 0x7ea360, 0xc4200d0140)
	/usr/local/go/src/net/http/server.go:1579 +0x4b7
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d
exit status 2
FAIL	_/tmp/d20170109-30451-uabzou	1.009s
50
--- FAIL: TestSlowLingchiWithBothMaxConnectionsAndALotOfErrors (0.28s)
	solution_test.go:942: Got new request while already having 21
	solution_test.go:942: Got new request while already having 22
	solution_test.go:942: Got new request while already having 23
	solution_test.go:942: Got new request while already having 21
	solution_test.go:942: Got new request while already having 22
	solution_test.go:942: Got new request while already having 23
	solution_test.go:942: Got new request while already having 24
	solution_test.go:942: Got new request while already having 25
	solution_test.go:942: Got new request while already having 26
	solution_test.go:942: Got new request while already having 27
	solution_test.go:942: Got new request while already having 28
	solution_test.go:942: Got new request while already having 29
	solution_test.go:942: Got new request while already having 30
	solution_test.go:942: Got new request while already having 31
	solution_test.go:942: Got new request while already having 32
	solution_test.go:942: Got new request while already having 28
	solution_test.go:942: Got new request while already having 28
	solution_test.go:942: Got new request while already having 29
	solution_test.go:942: Got new request while already having 30
	solution_test.go:942: Got new request while already having 31
	solution_test.go:942: Got new request while already having 32
	solution_test.go:942: Got new request while already having 33
	solution_test.go:942: Got new request while already having 34
	solution_test.go:942: Got new request while already having 35
	solution_test.go:942: Got new request while already having 36
	solution_test.go:942: Got new request while already having 37
	solution_test.go:942: Got new request while already having 38
	solution_test.go:1000: Expected to read 50 bytes from simple download but got 25
	solution_test.go:53: Expected result was '5468697320495320746865206d6f73742065706963206f6620616c6c20726573706f6e73655468697320495320746865206d' but got '5469205374656d732070636f206c2065706e65687349206800'
	solution_test.go:1015: From url with index 0 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 2 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 4 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 6 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 8 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 10 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 12 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 14 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 16 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 18 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 20 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 22 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 24 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 26 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 28 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 30 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 32 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 34 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 36 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 38 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 40 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 42 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 44 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 46 were received 1 bytes was expected 2
	solution_test.go:1015: From url with index 48 were received 1 bytes was expected 2
FAIL
exit status 1
FAIL	_/tmp/d20170109-30451-uabzou	0.287s

История (2 версии и 3 коментара)

Георги обнови решението на 28.12.2016 20:33 (преди над 1 година)

+package main
+
+import (
+ "context"
+ "io"
+ "net/http"
+ //"fmt"
+ "sync"
+ "strconv"
+ "io/ioutil"
+ "fmt"
+)
+
+type Reader interface {
+ Read(p []byte) (n int, err error)
+}
+
+type File struct {
+ resp []byte
+ done bool
+ lastInd int64
+}
+
+func (f *File) Read(p []byte) (n int, err error) {
+ if f.done || len(f.resp) == 0 {
+ return 0, io.EOF
+ }
+ if int64(len(p)) >= int64(len(f.resp)) - f.lastInd {
+ n := int64(len(f.resp)) - f.lastInd
+ for i := f.lastInd; i < n; i++ {
+ p[i - f.lastInd] = f.resp[i]
+ }
+ f.done = true;
+
+ return int(n), io.EOF
+ } else {
+ n = len(p)
+ for i := 0; i < n; i++ {
+ p[i] = f.resp[f.lastInd]
+ f.lastInd++
+ }
+ return len(p), nil
+ }
+}
+
+func splitInterval(size int64, parts int64) [4]int64 {
+ lowerSize := size / parts
+ remainder := size - parts*lowerSize
+ return [4]int64{parts - remainder, lowerSize, remainder, lowerSize + 1}
+}
+
+func getLength(url string) int64 {
+ res, _ := http.Head(url)
+ length := res.ContentLength
+ return length
+}
+
+func DownloadFile(ctx context.Context, urls []string) io.Reader {
+ length := getLength(urls[0])
+ splits := splitInterval(length, int64(len(urls)))
+ maxConnections := len(urls)
+ responses := make([][]byte, len(urls))
+
+ chMaxConnections := make(chan struct{}, maxConnections)
+
+ var wg sync.WaitGroup
+ var mtx sync.Mutex
+
+ downloadRange := func(start int64, end int64, ind int64) {
+ defer wg.Done()
+ client := &http.Client{}
+ req, _ := http.NewRequest("GET", urls[ind], nil)
+ rangeHeader := "bytes=" + strconv.FormatInt(start, 10) + "-" + strconv.FormatInt(end-1, 10)
+ req.Header.Add("Range", rangeHeader)
+
+ resp, _ := client.Do(req)
+ defer resp.Body.Close()
+ //resp.ContentLength == 1, expected 37
+ r, _ := ioutil.ReadAll(resp.Body)
+ fmt.Println(len(r))
+
+ mtx.Lock()
+ responses[ind] = r
+ mtx.Unlock()
+
+ <-chMaxConnections
+ }
+
+ //first size ranges
+ for ind := 0; int64(ind) < splits[0]; ind++ {
+ chMaxConnections <- struct{}{}
+ wg.Add(1)
+ go downloadRange(int64(ind) * splits[1], (int64(ind) + 1) * splits[1], int64(ind))
+ }
+ //second size ranges
+ for ind := 0; int64(ind) < splits[2]; ind++ {
+ chMaxConnections <- struct{}{}
+ wg.Add(1)
+ go downloadRange((splits[0] + int64(ind)) * splits[2],
+ (splits[0] + int64(ind) + 1) * splits[2],
+ splits[0] + int64(ind))
+ }
+
+ wg.Wait()
+ //fmt.Println(responses[0])
+ cntUrls := len(urls)
+ response := make([]byte, 0)
+ for ind := 0; ind < cntUrls; ind++ {
+ response = append(response, responses[ind]...)
+ }
+ return &File{resp: response, done: false, lastInd: 0}
+}

Знам, че съм до никъде, и вероятно задавам доста глупав въпрос, но все пак. Може ли малко hint каква е причината при ред 78 в кода ми на теста TestSingleURLWithReturn дължината на респонса да ми е 1 при очаквана 37? Реално хваща само първата буква и не мога да разбера защо. Пробвах ръчно с URL с файл в интернет и дължината е колкото искам. Благодаря.

Георги обнови решението на 02.01.2017 22:49 (преди над 1 година)

package main
import (
"context"
"io"
"net/http"
- //"fmt"
"sync"
"strconv"
"io/ioutil"
"fmt"
)
type Reader interface {
Read(p []byte) (n int, err error)
}
type File struct {
resp []byte
done bool
lastInd int64
+ isValid bool
}
func (f *File) Read(p []byte) (n int, err error) {
+ if !f.isValid {
+ return 0, fmt.Errorf("no valid urls")
+ }
if f.done || len(f.resp) == 0 {
return 0, io.EOF
}
if int64(len(p)) >= int64(len(f.resp)) - f.lastInd {
n := int64(len(f.resp)) - f.lastInd
for i := f.lastInd; i < n; i++ {
p[i - f.lastInd] = f.resp[i]
}
f.done = true;
return int(n), io.EOF
} else {
n = len(p)
for i := 0; i < n; i++ {
p[i] = f.resp[f.lastInd]
f.lastInd++
}
return len(p), nil
}
}
func splitInterval(size int64, parts int64) [4]int64 {
lowerSize := size / parts
remainder := size - parts*lowerSize
return [4]int64{parts - remainder, lowerSize, remainder, lowerSize + 1}
}
-func getLength(url string) int64 {
- res, _ := http.Head(url)
- length := res.ContentLength
- return length
+func getLength(urls []string, isValid []bool, cntValid *int64) (int64, error) {
+ var len int64
+ for ind, url := range urls {
+ res, err := http.Head(url)
+
+ if err == nil {
+ isValid[ind] = true
+ *cntValid++
+ len = res.ContentLength
+ }
+ }
+ fmt.Println(len)
+ if *cntValid == 0 {
+ return 0, fmt.Errorf("no valid urls")
+ } else {
+ return len, nil
+ }
}
func DownloadFile(ctx context.Context, urls []string) io.Reader {
- length := getLength(urls[0])
- splits := splitInterval(length, int64(len(urls)))
- maxConnections := len(urls)
- responses := make([][]byte, len(urls))
+ isValid := make([]bool, len(urls))
+ var cntValid int64
+ length, err := getLength(urls, isValid, &cntValid)
+ if err != nil {
+ return &File{resp: nil, done: true, lastInd: 0, isValid: false}
+ }
+ validIdx := make([]int, cntValid)
+ cnt := 0
+ for ind, val := range isValid {
+ if val {
+ validIdx[cnt] = ind
+ cnt++
+ }
+ }
+
+ splits := splitInterval(length, cntValid)
+ maxConnections := cntValid
+ responses := make([][]byte, cntValid)
+
chMaxConnections := make(chan struct{}, maxConnections)
var wg sync.WaitGroup
var mtx sync.Mutex
+ cntInvalidUrls := 0;
+
downloadRange := func(start int64, end int64, ind int64) {
defer wg.Done()
client := &http.Client{}
req, _ := http.NewRequest("GET", urls[ind], nil)
rangeHeader := "bytes=" + strconv.FormatInt(start, 10) + "-" + strconv.FormatInt(end-1, 10)
req.Header.Add("Range", rangeHeader)
-
resp, _ := client.Do(req)
defer resp.Body.Close()
- //resp.ContentLength == 1, expected 37
+ if resp.StatusCode >= 400 && resp.StatusCode < 600 {
+ mtx.Lock()
+ cntInvalidUrls++
+ mtx.Unlock()
+ }
r, _ := ioutil.ReadAll(resp.Body)
- fmt.Println(len(r))
mtx.Lock()
responses[ind] = r
mtx.Unlock()
<-chMaxConnections
}
//first size ranges
for ind := 0; int64(ind) < splits[0]; ind++ {
chMaxConnections <- struct{}{}
wg.Add(1)
- go downloadRange(int64(ind) * splits[1], (int64(ind) + 1) * splits[1], int64(ind))
+ go downloadRange(int64(ind) * splits[1], (int64(ind) + 1) * splits[1], int64(validIdx[ind]))
}
//second size ranges
for ind := 0; int64(ind) < splits[2]; ind++ {
chMaxConnections <- struct{}{}
wg.Add(1)
- go downloadRange((splits[0] + int64(ind)) * splits[2],
- (splits[0] + int64(ind) + 1) * splits[2],
- splits[0] + int64(ind))
+ go downloadRange((splits[0] * splits[1]) + int64(ind) * splits[3],
+ (splits[0] * splits[1]) + (int64(ind) + 1) * splits[3],
+ int64(validIdx[splits[0] + int64(ind)]))
}
wg.Wait()
- //fmt.Println(responses[0])
- cntUrls := len(urls)
+
response := make([]byte, 0)
- for ind := 0; ind < cntUrls; ind++ {
+ for ind := 0; int64(ind) < cntValid; ind++ {
response = append(response, responses[ind]...)
}
- return &File{resp: response, done: false, lastInd: 0}
+ return &File{resp: response, done: false, lastInd: 0, isValid: true}
}