diff --git a/.gitignore b/.gitignore index 3fb2474e8..e849b2269 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ _testmain.go out/ release/ go-exercism + +# Intellij +/.idea \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 3ebe5609b..33dd30db6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ go: install: true before_install: - go get github.com/codegangsta/cli && go get github.com/stretchr/testify/assert + bin/deps script: - go test ./... diff --git a/api/iteration.go b/api/iteration.go index 49b833a62..d98503fa1 100644 --- a/api/iteration.go +++ b/api/iteration.go @@ -1,15 +1,24 @@ package api import ( + "bytes" "errors" "io/ioutil" "path/filepath" "strings" + + "golang.org/x/net/html/charset" + "golang.org/x/text/transform" +) + +const ( + mimeType = "text/plain" ) var ( errUnidentifiable = errors.New("unable to identify track and problem") errNoFiles = errors.New("no files submitted") + utf8BOM = []byte{0xef, 0xbb, 0xbf} ) // Iteration represents a version of a particular exercise. @@ -53,12 +62,13 @@ func NewIteration(dir string, filenames []string) (*Iteration, error) { iter.Problem = segments[2] for _, filename := range filenames { - b, err := ioutil.ReadFile(filename) + fileContents, err := readFileAsUTF8String(filename) if err != nil { return nil, err } + path := filename[len(iter.RelativePath()):] - iter.Solution[path] = string(b) + iter.Solution[path] = *fileContents } return iter, nil } @@ -73,3 +83,28 @@ func (iter *Iteration) isValidFilepath(path string) bool { } return strings.HasPrefix(strings.ToLower(path), strings.ToLower(iter.Dir)) } + +func readFileAsUTF8String(filename string) (*string, error) { + b, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + + encoding, _, _ := charset.DetermineEncoding(b, mimeType) + decoder := encoding.NewDecoder() + decodedBytes, _, err := transform.Bytes(decoder, b) + if err != nil { + return nil, err + } + + // Drop the UTF-8 BOM that may have been added. This isn't necessary, and + // it's going to be written into another UTF-8 buffer anyway once it's JSON + // serialized. + // + // The standard recommends omitting the BOM. See + // http://www.unicode.org/versions/Unicode5.0.0/ch02.pdf + decodedBytes = bytes.TrimPrefix(decodedBytes, utf8BOM) + + s := string(decodedBytes) + return &s, nil +} diff --git a/api/iteration_test.go b/api/iteration_test.go index 92a0e55b5..c0b6832cf 100644 --- a/api/iteration_test.go +++ b/api/iteration_test.go @@ -3,7 +3,9 @@ package api import ( "path/filepath" "runtime" + "strings" "testing" + "unicode/utf8" ) func TestNewIteration(t *testing.T) { @@ -14,6 +16,8 @@ func TestNewIteration(t *testing.T) { filepath.Join(dir, "python", "leap", "one.py"), filepath.Join(dir, "python", "leap", "two.py"), filepath.Join(dir, "python", "leap", "lib", "three.py"), + filepath.Join(dir, "python", "leap", "utf16le.py"), + filepath.Join(dir, "python", "leap", "utf16be.py"), } iter, err := NewIteration(dir, files) @@ -28,18 +32,25 @@ func TestNewIteration(t *testing.T) { t.Errorf("Expected problem to be leap, was %s", iter.Problem) } - if len(iter.Solution) != 3 { + if len(iter.Solution) != 5 { t.Fatalf("Expected solution to have 3 files, had %d", len(iter.Solution)) } expected := map[string]string{ - "one.py": "# one\n", - "two.py": "# two\n", - "lib/three.py": "# three\n", + "one.py": "# one", + "two.py": "# two", + filepath.Join("lib", "three.py"): "# three", + "utf16le.py": "# utf16le", + "utf16be.py": "# utf16be", } + for filename, code := range expected { - if iter.Solution[filename] != code { - t.Errorf("Expected %s to contain %s, had %s", filename, code, iter.Solution[filename]) + if !utf8.ValidString(iter.Solution[filename]) { + t.Errorf("Iteration content is not valid UTF-8 data: %s", iter.Solution[filename]) + } + + if !strings.HasPrefix(iter.Solution[filename], code) { + t.Errorf("Expected %s to contain `%s', had `%s'", filename, code, iter.Solution[filename]) } } } diff --git a/bin/deps b/bin/deps new file mode 100755 index 000000000..16c4c916a --- /dev/null +++ b/bin/deps @@ -0,0 +1,9 @@ +#!/bin/bash + +LIBRARIES="\ + github.com/codegangsta/cli \ + github.com/stretchr/testify/assert \ + golang.org/x/net/html/charset \ + golang.org/x/text/transform" + +go get $LIBRARIES diff --git a/fixtures/iteration/python/leap/utf16be.py b/fixtures/iteration/python/leap/utf16be.py new file mode 100644 index 000000000..316c25d2d Binary files /dev/null and b/fixtures/iteration/python/leap/utf16be.py differ diff --git a/fixtures/iteration/python/leap/utf16le.py b/fixtures/iteration/python/leap/utf16le.py new file mode 100644 index 000000000..dd06d31dd Binary files /dev/null and b/fixtures/iteration/python/leap/utf16le.py differ