Merge pull request src-d#219 from camathieu/1.2.1

1.2.1

Author: Charles-Antoine Mathieu

.travis.yml

@@ -3,7 +3,7 @@ language: go
 sudo: required
 
 go:
-  - 1.7.4
+  - 1.9.1
 
 before_install:
   # Update node_js version : https://github.com/travis-ci/travis-ci/issues/7108

Makefile

@@ -24,7 +24,7 @@
 # THE SOFTWARE.
 ###
 
-RELEASE_VERSION="1.2"
+RELEASE_VERSION="1.2.1"
 RELEASE_DIR="release/plik-$(RELEASE_VERSION)"
 RELEASE_TARGETS=darwin-386 darwin-amd64 freebsd-386 \
 freebsd-amd64 linux-386 linux-amd64 linux-arm openbsd-386 \

README.md

@@ -20,16 +20,16 @@ Plik is a scalable & friendly temporary file upload system ( wetransfer like ) i
    - Upload restriction : Source IP / Token
 
 ### Version
-1.2
+1.2.1
 
 ### Installation
 
 ##### From release
 To run plik, it's very simple :
 ```sh
-$ wget https://github.com/root-gg/plik/releases/download/1.2/plik-1.2-linux-64bits.tar.gz
-$ tar xzvf plik-1.2-linux-64bits.tar.gz
-$ cd plik-1.2/server
+$ wget https://github.com/root-gg/plik/releases/download/1.2.1/plik-1.2.1-linux-64bits.tar.gz
+$ tar xzvf plik-1.2.1-linux-64bits.tar.gz
+$ cd plik-1.2.1/server
 $ ./plikd
 ```
 Et voilà ! You now have a fully functional instance of plik running on http://127.0.0.1:8080.  

changelog/1.2.1

@@ -0,0 +1,35 @@
+Hi,
+
+We're happy to release Plik 1.2.1
+
+Here is the changelog :
+
+New :
+    - Configuration option to prevent anonymous uploads.
+    - Configuration option to limit google email domains
+    - Serve HTTP on a different root path
+    - Display release notes on client update
+    - Backend testing script using Docker
+    - Load client config from /etc/plik/plikrc
+
+Misc :
+    - Add mongodb connect timeout
+    - Migrating from godeps to govendor
+    - Enable race detector when testing
+    - Add a confirm dialog to delete an Upload
+    - update golang and node_js version in Travis
+    - Update npm / bower dependencies ( bump Angularjs from 1.2 to 1.6 )
+
+Fix :
+    - Fix uploaded file size calculus
+    - Don't log file upload request body in debug mode
+    - Clean url properly when deleting upload
+    - Fix NPD in swift authentication
+    - Add missing letters to randRunes
+    - add clean-frontend to clean-all makefile directive
+    - Webapp : Fix progress bar when uploading file fails
+
+The binaires are compiled with go 1.9.1.
+
+Faithfully,
+The Plik Team.

client/plik.go

@@ -294,13 +294,14 @@ func upload(uploadInfo *common.Upload, fileToUpload *config.FileToUpload, reader
 		fmt.Printf("%s\n", getFileCommand(uploadInfo, fileToUpload.File))
 	}
 
-	// TODO Handler error properly here
-	go func() error {
-
+	errCh := make(chan error)
+	go func(errCh chan error) {
 		part, err := multipartWriter.CreateFormFile("file", fileToUpload.Name)
 		if err != nil {
-			fmt.Println(err)
-			return pipeWriter.CloseWithError(err)
+			err = fmt.Errorf("Unable to create multipartWriter : %s", err)
+			pipeWriter.CloseWithError(err)
+			errCh <- err
+			return
 		}
 
 		var multiWriter io.Writer
@@ -322,20 +323,27 @@ func upload(uploadInfo *common.Upload, fileToUpload *config.FileToUpload, reader
 		if config.Config.Secure {
 			err = config.GetCryptoBackend().Encrypt(reader, multiWriter)
 			if err != nil {
-				fmt.Println(err)
-				return pipeWriter.CloseWithError(err)
+				pipeWriter.CloseWithError(err)
+				errCh <- err
+				return
 			}
 		} else {
 			_, err = io.Copy(multiWriter, reader)
 			if err != nil {
-				fmt.Println(err)
-				return pipeWriter.CloseWithError(err)
+				pipeWriter.CloseWithError(err)
+				errCh <- err
+				return
 			}
 		}
 
 		err = multipartWriter.Close()
-		return pipeWriter.CloseWithError(err)
-	}()
+		if err != nil {
+			err = fmt.Errorf("Unable to close multipartWriter : %s", err)
+		}
+
+		pipeWriter.CloseWithError(err)
+		errCh <- err
+	}(errCh)
 
 	mode := "file"
 	if uploadInfo.Stream {
@@ -345,13 +353,13 @@ func upload(uploadInfo *common.Upload, fileToUpload *config.FileToUpload, reader
 	var URL *url.URL
 	URL, err = url.Parse(config.Config.URL + "/" + mode + "/" + uploadInfo.ID + "/" + fileToUpload.ID + "/" + fileToUpload.Name)
 	if err != nil {
-		return
+		return nil, err
 	}
 
 	var req *http.Request
 	req, err = http.NewRequest("POST", URL.String(), pipeReader)
 	if err != nil {
-		return
+		return nil, err
 	}
 
 	req.Header.Set("Content-Type", multipartWriter.FormDataContentType())
@@ -363,24 +371,30 @@ func upload(uploadInfo *common.Upload, fileToUpload *config.FileToUpload, reader
 
 	resp, err := makeRequest(req)
 	if err != nil {
-		return
+		return nil, err
+	}
+
+	err = <-errCh
+	if err != nil {
+		return nil, err
 	}
 
 	defer resp.Body.Close()
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return
+		return nil, err
 	}
 
 	// Parse Json response
 	file = new(common.File)
 	err = json.Unmarshal(body, file)
 	if err != nil {
-		return
+		return nil, err
 	}
 
 	config.Debug(fmt.Sprintf("Uploaded %s : %s", file.Name, config.Sdump(file)))
-	return
+
+	return file, nil
 }
 
 func getFileCommand(upload *common.Upload, file *common.File) (command string) {

server/handlers/addFile.go

@@ -44,6 +44,13 @@ import (
 	"github.com/root-gg/utils"
 )
 
+type preprocessOutputReturn struct {
+	size     int64
+	md5sum   string
+	mimeType string
+	err      error
+}
+
 // AddFile add a file to an existing upload.
 func AddFile(ctx *juliet.Context, resp http.ResponseWriter, req *http.Request) {
 	log := common.GetLogger(ctx)
@@ -105,6 +112,7 @@ func AddFile(ctx *juliet.Context, resp http.ResponseWriter, req *http.Request) {
 	prefix := fmt.Sprintf("%s[%s]", log.Prefix, newFile.ID)
 	log.SetPrefix(prefix)
 
+	// Save file to the context
 	ctx.Set("file", newFile)
 
 	// Get file handle from multipart request
@@ -153,47 +161,11 @@ func AddFile(ctx *juliet.Context, resp http.ResponseWriter, req *http.Request) {
 
 	// Pipe file data from the request body to a preprocessing goroutine
 	//  - Guess content type
+	//  - Compute/Limit upload size
 	//  - Compute md5sum
-	//  - Limit upload size
 	preprocessReader, preprocessWriter := io.Pipe()
-	md5Hash := md5.New()
-	totalBytes := 0
-	go func() {
-		for {
-			buf := make([]byte, 1024)
-			bytesRead, err := file.Read(buf)
-			if err != nil {
-				if err != io.EOF {
-					log.Warningf("Unable to read data from request body : %s", err)
-				}
-
-				preprocessWriter.Close()
-				return
-			}
-
-			// Detect the content-type using the 512 first bytes
-			if totalBytes == 0 {
-				newFile.Type = http.DetectContentType(buf)
-			}
-
-			// Increment size
-			totalBytes += bytesRead
-
-			// Compute md5sum
-			md5Hash.Write(buf[:bytesRead])
-
-			// Check upload max size limit
-			if int64(totalBytes) > common.Config.MaxFileSize {
-				err = fmt.Errorf("File too big (limit is set to %d bytes)", common.Config.MaxFileSize)
-				log.Warning(err.Error())
-				preprocessWriter.CloseWithError(err)
-				return
-			}
-
-			// Pass file data to data backend
-			preprocessWriter.Write(buf[:bytesRead])
-		}
-	}()
+	preprocessOutputCh := make(chan preprocessOutputReturn)
+	go preprocessor(ctx, file, preprocessWriter, preprocessOutputCh)
 
 	// Save file in the data backend
 	var backend dataBackend.DataBackend
@@ -209,14 +181,24 @@ func AddFile(ctx *juliet.Context, resp http.ResponseWriter, req *http.Request) {
 		return
 	}
 
+	// Get preprocessor goroutine output
+	preprocessOutput := <-preprocessOutputCh
+	if preprocessOutput.err != nil {
+		log.Warningf("Unable to execute preprocessor : %s", err)
+		common.Fail(ctx, req, resp, "Unable to save file", 500)
+		return
+	}
+
 	// Fill-in file information
-	newFile.CurrentSize = int64(totalBytes)
+	newFile.Type = preprocessOutput.mimeType
+	newFile.CurrentSize = preprocessOutput.size
+	newFile.Md5 = preprocessOutput.md5sum
+
 	if upload.Stream {
 		newFile.Status = "downloaded"
 	} else {
 		newFile.Status = "uploaded"
 	}
-	newFile.Md5 = fmt.Sprintf("%x", md5Hash.Sum(nil))
 	newFile.UploadDate = time.Now().Unix()
 	newFile.BackendDetails = backendDetails
 
@@ -243,3 +225,74 @@ func AddFile(ctx *juliet.Context, resp http.ResponseWriter, req *http.Request) {
 		return
 	}
 }
+
+//  - Guess content type
+//  - Compute/Limit upload size
+//  - Compute md5sum
+func preprocessor(ctx *juliet.Context, file io.Reader, preprocessWriter io.WriteCloser, outputCh chan preprocessOutputReturn) {
+	log := common.GetLogger(ctx)
+
+	var err error
+	var totalBytes int64
+	var mimeType string
+	var md5sum string
+
+	md5Hash := md5.New()
+	buf := make([]byte, 1048)
+
+	eof := false
+	for !eof {
+		bytesRead, err := file.Read(buf)
+		if err == io.EOF {
+			eof = true
+			if bytesRead <= 0 {
+				break
+			}
+		} else if err != nil {
+			err = log.EWarningf("Unable to read data from request body : %s", err)
+			break
+		}
+
+		// Detect the content-type using the 512 first bytes
+		if totalBytes == 0 {
+			mimeType = http.DetectContentType(buf)
+		}
+
+		// Increment size
+		totalBytes += int64(bytesRead)
+
+		// Check upload max size limit
+		if int64(totalBytes) > common.Config.MaxFileSize {
+			err = log.EWarningf("File too big (limit is set to %d bytes)", common.Config.MaxFileSize)
+			break
+		}
+
+		// Compute md5sum
+		md5Hash.Write(buf[:bytesRead])
+
+		// Forward data to the data backend
+		bytesWritten, err := preprocessWriter.Write(buf[:bytesRead])
+		if err != nil {
+			log.Warning(err.Error())
+			break
+		}
+		if bytesWritten != bytesRead {
+			err = log.EWarningf("Invalid number of bytes written. Expected %d but got %d", bytesRead, bytesWritten)
+			break
+		}
+	}
+
+	err = preprocessWriter.Close()
+	if err != nil {
+		log.Warningf("Unable to close preprocessWriter : %s", err)
+	}
+
+	if err != nil {
+		outputCh <- preprocessOutputReturn{err: err}
+	} else {
+		md5sum = fmt.Sprintf("%x", md5Hash.Sum(nil))
+		outputCh <- preprocessOutputReturn{size: totalBytes, md5sum: md5sum, mimeType: mimeType}
+	}
+
+	close(outputCh)
+}

server/metadataBackend/mongo/mongo.go

@@ -62,6 +62,7 @@ func NewMongoMetadataBackend(config map[string]interface{}) (mmb *MetadataBacken
 	dialInfo := &mgo.DialInfo{}
 	dialInfo.Addrs = []string{mmb.config.URL}
 	dialInfo.Database = mmb.config.Database
+	dialInfo.Timeout = 5 * time.Second
 	if mmb.config.Username != "" && mmb.config.Password != "" {
 		dialInfo.Username = mmb.config.Username
 		dialInfo.Password = mmb.config.Password