// Copyright 2018 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package storage import ( "context" "errors" "fmt" "io" "io/ioutil" "net/http" "strconv" "strings" "testing" "google.golang.org/api/option" ) const readData = "0123456789" func TestRangeReader(t *testing.T) { hc, close := newTestServer(handleRangeRead) defer close() ctx := context.Background() c, err := NewClient(ctx, option.WithHTTPClient(hc)) if err != nil { t.Fatal(err) } obj := c.Bucket("b").Object("o") for _, test := range []struct { offset, length int64 want string }{ {0, -1, readData}, {0, 10, readData}, {0, 5, readData[:5]}, {1, 3, readData[1:4]}, {6, -1, readData[6:]}, {4, 20, readData[4:]}, {-20, -1, readData}, {-6, -1, readData[4:]}, } { r, err := obj.NewRangeReader(ctx, test.offset, test.length) if err != nil { t.Errorf("%d/%d: %v", test.offset, test.length, err) continue } gotb, err := ioutil.ReadAll(r) if err != nil { t.Errorf("%d/%d: %v", test.offset, test.length, err) continue } if got := string(gotb); got != test.want { t.Errorf("%d/%d: got %q, want %q", test.offset, test.length, got, test.want) } } } func handleRangeRead(w http.ResponseWriter, r *http.Request) { rh := strings.TrimSpace(r.Header.Get("Range")) data := readData var from, to int if rh == "" { from = 0 to = len(data) } else { // assume "bytes=N-", "bytes=-N" or "bytes=N-M" var err error i := strings.IndexRune(rh, '=') j := strings.IndexRune(rh, '-') hasPositiveStartOffset := i+1 != j if hasPositiveStartOffset { // The case of "bytes=N-" from, err = strconv.Atoi(rh[i+1 : j]) } else { // The case of "bytes=-N" from, err = strconv.Atoi(rh[i+1:]) from += len(data) if from < 0 { from = 0 } } if err != nil { w.WriteHeader(500) return } to = len(data) if hasPositiveStartOffset && j+1 < len(rh) { // The case of "bytes=N-M" to, err = strconv.Atoi(rh[j+1:]) if err != nil { w.WriteHeader(500) return } to++ // Range header is inclusive, Go slice is exclusive } if from >= len(data) && to != from { w.WriteHeader(416) return } if from > len(data) { from = len(data) } if to > len(data) { to = len(data) } } data = data[from:to] if data != readData { w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", from, to-1, len(readData))) w.WriteHeader(http.StatusPartialContent) } if _, err := w.Write([]byte(data)); err != nil { panic(err) } } type http2Error string func (h http2Error) Error() string { return string(h) } func TestRangeReaderRetry(t *testing.T) { retryErr := http2Error("blah blah INTERNAL_ERROR") readBytes := []byte(readData) hc, close := newTestServer(handleRangeRead) defer close() ctx := context.Background() c, err := NewClient(ctx, option.WithHTTPClient(hc)) if err != nil { t.Fatal(err) } obj := c.Bucket("b").Object("o") for i, test := range []struct { offset, length int64 bodies []fakeReadCloser want string }{ { offset: 0, length: -1, bodies: []fakeReadCloser{ {data: readBytes, counts: []int{10}, err: io.EOF}, }, want: readData, }, { offset: 0, length: -1, bodies: []fakeReadCloser{ {data: readBytes, counts: []int{3}, err: retryErr}, {data: readBytes[3:], counts: []int{5, 2}, err: io.EOF}, }, want: readData, }, { offset: 0, length: -1, bodies: []fakeReadCloser{ {data: readBytes, counts: []int{5}, err: retryErr}, {data: readBytes[5:], counts: []int{1, 3}, err: retryErr}, {data: readBytes[9:], counts: []int{1}, err: io.EOF}, }, want: readData, }, { offset: 0, length: 5, bodies: []fakeReadCloser{ {data: readBytes, counts: []int{3}, err: retryErr}, {data: readBytes[3:], counts: []int{2}, err: io.EOF}, }, want: readData[:5], }, { offset: 1, length: 5, bodies: []fakeReadCloser{ {data: readBytes, counts: []int{3}, err: retryErr}, {data: readBytes[3:], counts: []int{2}, err: io.EOF}, }, want: readData[:5], }, { offset: 1, length: 3, bodies: []fakeReadCloser{ {data: readBytes[1:], counts: []int{1}, err: retryErr}, {data: readBytes[2:], counts: []int{2}, err: io.EOF}, }, want: readData[1:4], }, { offset: 4, length: -1, bodies: []fakeReadCloser{ {data: readBytes[4:], counts: []int{1}, err: retryErr}, {data: readBytes[5:], counts: []int{4}, err: retryErr}, {data: readBytes[9:], counts: []int{1}, err: io.EOF}, }, want: readData[4:], }, { offset: -4, length: -1, bodies: []fakeReadCloser{ {data: readBytes[6:], counts: []int{1}, err: retryErr}, {data: readBytes[7:], counts: []int{3}, err: io.EOF}, }, want: readData[6:], }, } { r, err := obj.NewRangeReader(ctx, test.offset, test.length) if err != nil { t.Errorf("#%d: %v", i, err) continue } r.body = &test.bodies[0] b := 0 r.reopen = func(int64) (*http.Response, error) { b++ return &http.Response{Body: &test.bodies[b]}, nil } buf := make([]byte, len(readData)/2) var gotb []byte for { n, err := r.Read(buf) gotb = append(gotb, buf[:n]...) if err == io.EOF { break } if err != nil { t.Fatalf("#%d: %v", i, err) } } if err != nil { t.Errorf("#%d: %v", i, err) continue } if got := string(gotb); got != test.want { t.Errorf("#%d: got %q, want %q", i, got, test.want) } if r.Attrs.Size != int64(len(readData)) { t.Errorf("#%d: got Attrs.Size=%q, want %q", i, r.Attrs.Size, len(readData)) } wantOffset := test.offset if wantOffset < 0 { wantOffset += int64(len(readData)) if wantOffset < 0 { wantOffset = 0 } } if got := r.Attrs.StartOffset; got != wantOffset { t.Errorf("#%d: got Attrs.Offset=%q, want %q", i, got, wantOffset) } } r, err := obj.NewRangeReader(ctx, -100, 10) if err == nil { t.Fatal("Expected a non-nil error with negative offset and positive length") } else if want := "storage: invalid offset"; !strings.HasPrefix(err.Error(), want) { t.Errorf("Error mismatch\nGot: %q\nWant prefix: %q\n", err.Error(), want) } if r != nil { t.Errorf("Expected nil reader") } } type fakeReadCloser struct { data []byte counts []int // how much of data to deliver on each read err error // error to return with last count d int // current position in data c int // current position in counts } func (f *fakeReadCloser) Close() error { return nil } func (f *fakeReadCloser) Read(buf []byte) (int, error) { i := f.c n := 0 if i < len(f.counts) { n = f.counts[i] } var err error if i >= len(f.counts)-1 { err = f.err } copy(buf, f.data[f.d:f.d+n]) if len(buf) < n { n = len(buf) f.counts[i] -= n err = nil } else { f.c++ } f.d += n return n, err } func TestFakeReadCloser(t *testing.T) { e := errors.New("") f := &fakeReadCloser{ data: []byte(readData), counts: []int{1, 2, 3}, err: e, } wants := []string{"0", "12", "345"} buf := make([]byte, 10) for i := 0; i < 3; i++ { n, err := f.Read(buf) if got, want := n, f.counts[i]; got != want { t.Fatalf("i=%d: got %d, want %d", i, got, want) } var wantErr error if i == 2 { wantErr = e } if err != wantErr { t.Fatalf("i=%d: got error %v, want %v", i, err, wantErr) } if got, want := string(buf[:n]), wants[i]; got != want { t.Fatalf("i=%d: got %q, want %q", i, got, want) } } }