plumbing/transport: add common tests and fixes. (#136)

* plumbing/transport: add common tests and fixes.

* add common test suite for different transport implementations.
* fix different behaviour on error handling for ssh and http.
  fixes issue #123.
* support detecting unexisting repositories with SSH + GitHub/Bitbucket
  (apparently, there is no standard for all SSH servers).
* remove ssh.NewClient (only DefaultClient makes sense at the moment).
* make ssh.Client and http.Client private.
* utils/ioutil: utilities to work with io interfaces.

* * transport: test actual objects fetched, not just packfile size.

* * fix doc typo.

* * improve UploadPackRequest.IsEmpty

Author: smola

plumbing/format/packp/advrefs/decoder.go

@@ -86,6 +86,12 @@ func decodePrefix(d *Decoder) decoderStateFn {
 		return nil
 	}
 
+	// If the repository is empty, we receive a flush here (SSH).
+	if isFlush(d.line) {
+		d.err = ErrEmpty
+		return nil
+	}
+
 	if isPrefix(d.line) {
 		tmp := make([]byte, len(d.line))
 		copy(tmp, d.line)
@@ -117,6 +123,12 @@ func isFlush(payload []byte) bool {
 // list-of-refs is comming, and the hash will be followed by the first
 // advertised ref.
 func decodeFirstHash(p *Decoder) decoderStateFn {
+	// If the repository is empty, we receive a flush here (HTTP).
+	if isFlush(p.line) {
+		p.err = ErrEmpty
+		return nil
+	}
+
 	if len(p.line) < hashSize {
 		p.error("cannot read hash, pkt-line too short")
 		return nil

plumbing/format/packp/advrefs/decoder_test.go

@@ -26,6 +26,32 @@ func (s *SuiteDecoder) TestEmpty(c *C) {
 	c.Assert(err, Equals, advrefs.ErrEmpty)
 }
 
+func (s *SuiteDecoder) TestEmptyFlush(c *C) {
+	ar := advrefs.New()
+	var buf bytes.Buffer
+	e := pktline.NewEncoder(&buf)
+	e.Flush()
+
+	d := advrefs.NewDecoder(&buf)
+
+	err := d.Decode(ar)
+	c.Assert(err, Equals, advrefs.ErrEmpty)
+}
+
+func (s *SuiteDecoder) TestEmptyPrefixFlush(c *C) {
+	ar := advrefs.New()
+	var buf bytes.Buffer
+	e := pktline.NewEncoder(&buf)
+	e.EncodeString("# service=git-upload-pack")
+	e.Flush()
+	e.Flush()
+
+	d := advrefs.NewDecoder(&buf)
+
+	err := d.Decode(ar)
+	c.Assert(err, Equals, advrefs.ErrEmpty)
+}
+
 func (s *SuiteDecoder) TestShortForHash(c *C) {
 	payloads := []string{
 		"6ecf0ef2c2dffb796",

plumbing/transport/client/client_test.go

@@ -22,14 +22,14 @@ func (s *ClientSuite) TestNewClientHTTP(c *C) {
 
 	output, err := NewClient(e)
 	c.Assert(err, IsNil)
-	c.Assert(typeAsString(output), Equals, "*http.Client")
+	c.Assert(typeAsString(output), Equals, "*http.client")
 
 	e, err = transport.NewEndpoint("https://github.com/src-d/go-git")
 	c.Assert(err, IsNil)
 
 	output, err = NewClient(e)
 	c.Assert(err, IsNil)
-	c.Assert(typeAsString(output), Equals, "*http.Client")
+	c.Assert(typeAsString(output), Equals, "*http.client")
 }
 
 func (s *ClientSuite) TestNewClientSSH(c *C) {
@@ -38,7 +38,7 @@ func (s *ClientSuite) TestNewClientSSH(c *C) {
 
 	output, err := NewClient(e)
 	c.Assert(err, IsNil)
-	c.Assert(typeAsString(output), Equals, "*ssh.Client")
+	c.Assert(typeAsString(output), Equals, "*ssh.client")
 }
 
 func (s *ClientSuite) TestNewClientUnknown(c *C) {

plumbing/transport/common.go

@@ -24,9 +24,12 @@ import (
 
 var (
 	ErrRepositoryNotFound     = errors.New("repository not found")
+	ErrEmptyRemoteRepository  = errors.New("remote repository is empty")
 	ErrAuthorizationRequired  = errors.New("authorization required")
 	ErrEmptyUploadPackRequest = errors.New("empty git-upload-pack given")
 	ErrInvalidAuthMethod      = errors.New("invalid auth method")
+
+	ErrAdvertistedReferencesAlreadyCalled = errors.New("cannot call AdvertisedReference twice")
 )
 
 const (

plumbing/transport/fetch_pack.go

@@ -33,7 +33,7 @@ func (i *UploadPackInfo) Decode(r io.Reader) error {
 	ar := advrefs.New()
 	if err := d.Decode(ar); err != nil {
 		if err == advrefs.ErrEmpty {
-			return plumbing.NewPermanentError(err)
+			return err
 		}
 		return plumbing.NewUnexpectedError(err)
 	}
@@ -128,6 +128,28 @@ func (r *UploadPackRequest) Have(h ...plumbing.Hash) {
 	r.Haves = append(r.Haves, h...)
 }
 
+func (r *UploadPackRequest) IsEmpty() bool {
+	return isSubset(r.Wants, r.Haves)
+}
+
+func isSubset(needle []plumbing.Hash, haystack []plumbing.Hash) bool {
+	for _, h := range needle {
+		found := false
+		for _, oh := range haystack {
+			if h == oh {
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			return false
+		}
+	}
+
+	return true
+}
+
 func (r *UploadPackRequest) String() string {
 	b, _ := ioutil.ReadAll(r.Reader())
 	return string(b)

plumbing/transport/fetch_pack_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/base64"
 
 	"gopkg.in/src-d/go-git.v4/plumbing"
+	"gopkg.in/src-d/go-git.v4/plumbing/format/packp/advrefs"
 
 	. "gopkg.in/check.v1"
 )
@@ -58,7 +59,7 @@ func (s *UploadPackSuite) TestUploadPackInfoEmpty(c *C) {
 
 	i := NewUploadPackInfo()
 	err := i.Decode(b)
-	c.Assert(err, ErrorMatches, "permanent.*empty.*")
+	c.Assert(err, Equals, advrefs.ErrEmpty)
 }
 
 func (s *UploadPackSuite) TestUploadPackEncode(c *C) {
@@ -94,3 +95,25 @@ func (s *UploadPackSuite) TestUploadPackRequest(c *C) {
 			"0009done\n",
 	)
 }
+
+func (s *UploadPackSuite) TestUploadPackRequest_IsEmpty(c *C) {
+	r := &UploadPackRequest{}
+	r.Want(plumbing.NewHash("d82f291cde9987322c8a0c81a325e1ba6159684c"))
+	r.Want(plumbing.NewHash("2b41ef280fdb67a9b250678686a0c3e03b0a9989"))
+	r.Have(plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"))
+
+	c.Assert(r.IsEmpty(), Equals, false)
+
+	r = &UploadPackRequest{}
+	r.Want(plumbing.NewHash("d82f291cde9987322c8a0c81a325e1ba6159684c"))
+	r.Want(plumbing.NewHash("2b41ef280fdb67a9b250678686a0c3e03b0a9989"))
+	r.Have(plumbing.NewHash("d82f291cde9987322c8a0c81a325e1ba6159684c"))
+
+	c.Assert(r.IsEmpty(), Equals, false)
+
+	r = &UploadPackRequest{}
+	r.Want(plumbing.NewHash("d82f291cde9987322c8a0c81a325e1ba6159684c"))
+	r.Have(plumbing.NewHash("d82f291cde9987322c8a0c81a325e1ba6159684c"))
+
+	c.Assert(r.IsEmpty(), Equals, true)
+}

plumbing/transport/http/common.go

@@ -9,33 +9,38 @@ import (
 	"gopkg.in/src-d/go-git.v4/plumbing/transport"
 )
 
-type Client struct {
+type client struct {
 	c *http.Client
 }
 
+// DefaultClient is the default HTTP client, which uses `http.DefaultClient`.
 var DefaultClient = NewClient(nil)
 
 // NewClient creates a new client with a custom net/http client.
 // See `InstallProtocol` to install and override default http client.
 // Unless a properly initialized client is given, it will fall back into
 // `http.DefaultClient`.
+//
+// Note that for HTTP client cannot distinguist between private repositories and
+// unexistent repositories on GitHub. So it returns `ErrAuthorizationRequired`
+// for both.
 func NewClient(c *http.Client) transport.Client {
 	if c == nil {
-		return &Client{http.DefaultClient}
+		return &client{http.DefaultClient}
 	}
 
-	return &Client{
+	return &client{
 		c: c,
 	}
 }
 
-func (c *Client) NewFetchPackSession(ep transport.Endpoint) (
+func (c *client) NewFetchPackSession(ep transport.Endpoint) (
 	transport.FetchPackSession, error) {
 
 	return newFetchPackSession(c.c, ep), nil
 }
 
-func (c *Client) NewSendPackSession(ep transport.Endpoint) (
+func (c *client) NewSendPackSession(ep transport.Endpoint) (
 	transport.SendPackSession, error) {
 
 	return newSendPackSession(c.c, ep), nil

plumbing/transport/http/common_test.go

@@ -20,16 +20,20 @@ var _ = Suite(&ClientSuite{})
 
 func (s *ClientSuite) SetUpSuite(c *C) {
 	var err error
-	s.Endpoint, err = transport.NewEndpoint("https://github.com/git-fixtures/basic")
+	s.Endpoint, err = transport.NewEndpoint(
+		"https://github.com/git-fixtures/basic",
+	)
 	c.Assert(err, IsNil)
 }
 
 func (s *FetchPackSuite) TestNewClient(c *C) {
-	roundTripper := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
-	client := &http.Client{Transport: roundTripper}
-	r := NewClient(client).(*Client)
-
-	c.Assert(r.c, Equals, client)
+	roundTripper := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	cl := &http.Client{Transport: roundTripper}
+	r, ok := NewClient(cl).(*client)
+	c.Assert(ok, Equals, true)
+	c.Assert(r.c, Equals, cl)
 }
 
 func (s *ClientSuite) TestNewBasicAuth(c *C) {
@@ -54,7 +58,8 @@ func (s *ClientSuite) TestNewErrNotFound(c *C) {
 }
 
 func (s *ClientSuite) TestNewHTTPError40x(c *C) {
-	s.testNewHTTPError(c, http.StatusPaymentRequired, "unexpected client error.*")
+	s.testNewHTTPError(c, http.StatusPaymentRequired,
+		"unexpected client error.*")
 }
 
 func (s *ClientSuite) testNewHTTPError(c *C, code int, msg string) {

plumbing/transport/http/fetch_pack.go

@@ -1,20 +1,22 @@
 package http
 
 import (
-	"bufio"
 	"bytes"
 	"fmt"
 	"io"
 	"net/http"
 	"strings"
 
 	"gopkg.in/src-d/go-git.v4/plumbing"
+	"gopkg.in/src-d/go-git.v4/plumbing/format/packp/advrefs"
 	"gopkg.in/src-d/go-git.v4/plumbing/format/packp/pktline"
 	"gopkg.in/src-d/go-git.v4/plumbing/transport"
+	"gopkg.in/src-d/go-git.v4/utils/ioutil"
 )
 
 type fetchPackSession struct {
 	*session
+	advRefsRun bool
 }
 
 func newFetchPackSession(c *http.Client,
@@ -31,6 +33,11 @@ func newFetchPackSession(c *http.Client,
 
 func (s *fetchPackSession) AdvertisedReferences() (*transport.UploadPackInfo,
 	error) {
+	if s.advRefsRun {
+		return nil, transport.ErrAdvertistedReferencesAlreadyCalled
+	}
+
+	defer func() { s.advRefsRun = true }()
 
 	url := fmt.Sprintf(
 		"%s/info/refs?service=%s",
@@ -50,15 +57,28 @@ func (s *fetchPackSession) AdvertisedReferences() (*transport.UploadPackInfo,
 	}
 
 	defer res.Body.Close()
+
 	if res.StatusCode == http.StatusUnauthorized {
 		return nil, transport.ErrAuthorizationRequired
 	}
 
 	i := transport.NewUploadPackInfo()
-	return i, i.Decode(res.Body)
+	if err := i.Decode(res.Body); err != nil {
+		if err == advrefs.ErrEmpty {
+			err = transport.ErrEmptyRemoteRepository
+		}
+
+		return nil, err
+	}
+
+	return i, nil
 }
 
 func (s *fetchPackSession) FetchPack(r *transport.UploadPackRequest) (io.ReadCloser, error) {
+	if r.IsEmpty() {
+		return nil, transport.ErrEmptyUploadPackRequest
+	}
+
 	url := fmt.Sprintf(
 		"%s/%s",
 		s.endpoint.String(), transport.UploadPackServiceName,
@@ -69,20 +89,21 @@ func (s *fetchPackSession) FetchPack(r *transport.UploadPackRequest) (io.ReadClo
 		return nil, err
 	}
 
-	reader := newBufferedReadCloser(res.Body)
-	if _, err := reader.Peek(1); err != nil {
-		if err == io.ErrUnexpectedEOF {
-			return nil, transport.ErrEmptyUploadPackRequest
-		}
+	reader, err := ioutil.NonEmptyReader(res.Body)
+	if err == ioutil.ErrEmptyReader || err == io.ErrUnexpectedEOF {
+		return nil, transport.ErrEmptyUploadPackRequest
+	}
 
+	if err != nil {
 		return nil, err
 	}
 
-	if err := discardResponseInfo(reader); err != nil {
+	rc := ioutil.NewReadCloser(reader, res.Body)
+	if err := discardResponseInfo(rc); err != nil {
 		return nil, err
 	}
 
-	return reader, nil
+	return rc, nil
 }
 
 // Close does nothing.
@@ -140,16 +161,3 @@ func (s *fetchPackSession) applyHeadersToRequest(req *http.Request, content *str
 		req.Header.Add("Content-Length", string(content.Len()))
 	}
 }
-
-type bufferedReadCloser struct {
-	*bufio.Reader
-	closer io.Closer
-}
-
-func newBufferedReadCloser(r io.ReadCloser) *bufferedReadCloser {
-	return &bufferedReadCloser{bufio.NewReader(r), r}
-}
-
-func (r *bufferedReadCloser) Close() error {
-	return r.closer.Close()
-}