Merge pull request #184 from ambroff/issue-2303

[RECREATED] Deal with different file encodings when submitting solutions

Author: kytrinyx

.gitignore

@@ -24,3 +24,6 @@ _testmain.go
 out/
 release/
 go-exercism
+
+# Intellij
+/.idea

.travis.yml

@@ -7,7 +7,7 @@ go:
 install: true
 
 before_install:
-  go get github.com/codegangsta/cli && go get github.com/stretchr/testify/assert
+  bin/deps
 
 script:
   - go test ./...

api/iteration.go

@@ -1,15 +1,24 @@
 package api
 
 import (
+	"bytes"
 	"errors"
 	"io/ioutil"
 	"path/filepath"
 	"strings"
+
+	"golang.org/x/net/html/charset"
+	"golang.org/x/text/transform"
+)
+
+const (
+	mimeType = "text/plain"
 )
 
 var (
 	errUnidentifiable = errors.New("unable to identify track and problem")
 	errNoFiles        = errors.New("no files submitted")
+	utf8BOM           = []byte{0xef, 0xbb, 0xbf}
 )
 
 // Iteration represents a version of a particular exercise.
@@ -53,12 +62,13 @@ func NewIteration(dir string, filenames []string) (*Iteration, error) {
 	iter.Problem = segments[2]
 
 	for _, filename := range filenames {
-		b, err := ioutil.ReadFile(filename)
+		fileContents, err := readFileAsUTF8String(filename)
 		if err != nil {
 			return nil, err
 		}
+
 		path := filename[len(iter.RelativePath()):]
-		iter.Solution[path] = string(b)
+		iter.Solution[path] = *fileContents
 	}
 	return iter, nil
 }
@@ -73,3 +83,28 @@ func (iter *Iteration) isValidFilepath(path string) bool {
 	}
 	return strings.HasPrefix(strings.ToLower(path), strings.ToLower(iter.Dir))
 }
+
+func readFileAsUTF8String(filename string) (*string, error) {
+	b, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	encoding, _, _ := charset.DetermineEncoding(b, mimeType)
+	decoder := encoding.NewDecoder()
+	decodedBytes, _, err := transform.Bytes(decoder, b)
+	if err != nil {
+		return nil, err
+	}
+
+	// Drop the UTF-8 BOM that may have been added. This isn't necessary, and
+	// it's going to be written into another UTF-8 buffer anyway once it's JSON
+	// serialized.
+	//
+	// The standard recommends omitting the BOM. See
+	// http://www.unicode.org/versions/Unicode5.0.0/ch02.pdf
+	decodedBytes = bytes.TrimPrefix(decodedBytes, utf8BOM)
+
+	s := string(decodedBytes)
+	return &s, nil
+}

api/iteration_test.go

@@ -3,7 +3,9 @@ package api
 import (
 	"path/filepath"
 	"runtime"
+	"strings"
 	"testing"
+	"unicode/utf8"
 )
 
 func TestNewIteration(t *testing.T) {
@@ -14,6 +16,8 @@ func TestNewIteration(t *testing.T) {
 		filepath.Join(dir, "python", "leap", "one.py"),
 		filepath.Join(dir, "python", "leap", "two.py"),
 		filepath.Join(dir, "python", "leap", "lib", "three.py"),
+		filepath.Join(dir, "python", "leap", "utf16le.py"),
+		filepath.Join(dir, "python", "leap", "utf16be.py"),
 	}
 
 	iter, err := NewIteration(dir, files)
@@ -28,18 +32,25 @@ func TestNewIteration(t *testing.T) {
 		t.Errorf("Expected problem to be leap, was %s", iter.Problem)
 	}
 
-	if len(iter.Solution) != 3 {
+	if len(iter.Solution) != 5 {
 		t.Fatalf("Expected solution to have 3 files, had %d", len(iter.Solution))
 	}
 
 	expected := map[string]string{
-		"one.py":       "# one\n",
-		"two.py":       "# two\n",
-		"lib/three.py": "# three\n",
+		"one.py": "# one",
+		"two.py": "# two",
+		filepath.Join("lib", "three.py"): "# three",
+		"utf16le.py":                     "# utf16le",
+		"utf16be.py":                     "# utf16be",
 	}
+
 	for filename, code := range expected {
-		if iter.Solution[filename] != code {
-			t.Errorf("Expected %s to contain %s, had %s", filename, code, iter.Solution[filename])
+		if !utf8.ValidString(iter.Solution[filename]) {
+			t.Errorf("Iteration content is not valid UTF-8 data: %s", iter.Solution[filename])
+		}
+
+		if !strings.HasPrefix(iter.Solution[filename], code) {
+			t.Errorf("Expected %s to contain `%s', had `%s'", filename, code, iter.Solution[filename])
 		}
 	}
 }

bin/deps

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+LIBRARIES="\
+  github.com/codegangsta/cli \
+  github.com/stretchr/testify/assert \
+  golang.org/x/net/html/charset \
+  golang.org/x/text/transform"
+
+go get $LIBRARIES