From e481b224b4ab279b53a5507c383d843a9ae7f91b Mon Sep 17 00:00:00 2001 From: Kyle Ambroff Date: Fri, 3 Apr 2015 17:51:44 -0700 Subject: [PATCH 1/3] Ignore Intellij project directory. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 3fb2474e8..e849b2269 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ _testmain.go out/ release/ go-exercism + +# Intellij +/.idea \ No newline at end of file From b01c9c3a05446668b8e133fad9db4e430935ee66 Mon Sep 17 00:00:00 2001 From: Kyle Ambroff Date: Sat, 4 Apr 2015 14:39:06 -0700 Subject: [PATCH 2/3] Fixes #2303: cli cannot read file contents unless it's ascii or utf-8 Fixes issue #2303 in the exercism.io project. https://github.com/exercism/exercism.io/issues/2303 Attempt to detect the charset of a submission so that it can be properly transcoded to UTF-8. Added UTF-16 and UTF-16BE fixture files for the test suite. Changed the tests slightly to deal with different path and newline separators on Windows. Update travis config to include new dependencies. --- .travis.yml | 2 +- api/iteration.go | 39 ++++++++++++++++++++-- api/iteration_test.go | 23 +++++++++---- bin/deps | 9 +++++ fixtures/iteration/python/leap/utf16be.py | Bin 0 -> 20 bytes fixtures/iteration/python/leap/utf16le.py | Bin 0 -> 20 bytes 6 files changed, 64 insertions(+), 9 deletions(-) create mode 100644 bin/deps create mode 100644 fixtures/iteration/python/leap/utf16be.py create mode 100644 fixtures/iteration/python/leap/utf16le.py diff --git a/.travis.yml b/.travis.yml index 3ebe5609b..33dd30db6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ go: install: true before_install: - go get github.com/codegangsta/cli && go get github.com/stretchr/testify/assert + bin/deps script: - go test ./... diff --git a/api/iteration.go b/api/iteration.go index 49b833a62..d98503fa1 100644 --- a/api/iteration.go +++ b/api/iteration.go @@ -1,15 +1,24 @@ package api import ( + "bytes" "errors" "io/ioutil" "path/filepath" "strings" + + "golang.org/x/net/html/charset" + "golang.org/x/text/transform" +) + +const ( + mimeType = "text/plain" ) var ( errUnidentifiable = errors.New("unable to identify track and problem") errNoFiles = errors.New("no files submitted") + utf8BOM = []byte{0xef, 0xbb, 0xbf} ) // Iteration represents a version of a particular exercise. @@ -53,12 +62,13 @@ func NewIteration(dir string, filenames []string) (*Iteration, error) { iter.Problem = segments[2] for _, filename := range filenames { - b, err := ioutil.ReadFile(filename) + fileContents, err := readFileAsUTF8String(filename) if err != nil { return nil, err } + path := filename[len(iter.RelativePath()):] - iter.Solution[path] = string(b) + iter.Solution[path] = *fileContents } return iter, nil } @@ -73,3 +83,28 @@ func (iter *Iteration) isValidFilepath(path string) bool { } return strings.HasPrefix(strings.ToLower(path), strings.ToLower(iter.Dir)) } + +func readFileAsUTF8String(filename string) (*string, error) { + b, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + + encoding, _, _ := charset.DetermineEncoding(b, mimeType) + decoder := encoding.NewDecoder() + decodedBytes, _, err := transform.Bytes(decoder, b) + if err != nil { + return nil, err + } + + // Drop the UTF-8 BOM that may have been added. This isn't necessary, and + // it's going to be written into another UTF-8 buffer anyway once it's JSON + // serialized. + // + // The standard recommends omitting the BOM. See + // http://www.unicode.org/versions/Unicode5.0.0/ch02.pdf + decodedBytes = bytes.TrimPrefix(decodedBytes, utf8BOM) + + s := string(decodedBytes) + return &s, nil +} diff --git a/api/iteration_test.go b/api/iteration_test.go index 92a0e55b5..c0b6832cf 100644 --- a/api/iteration_test.go +++ b/api/iteration_test.go @@ -3,7 +3,9 @@ package api import ( "path/filepath" "runtime" + "strings" "testing" + "unicode/utf8" ) func TestNewIteration(t *testing.T) { @@ -14,6 +16,8 @@ func TestNewIteration(t *testing.T) { filepath.Join(dir, "python", "leap", "one.py"), filepath.Join(dir, "python", "leap", "two.py"), filepath.Join(dir, "python", "leap", "lib", "three.py"), + filepath.Join(dir, "python", "leap", "utf16le.py"), + filepath.Join(dir, "python", "leap", "utf16be.py"), } iter, err := NewIteration(dir, files) @@ -28,18 +32,25 @@ func TestNewIteration(t *testing.T) { t.Errorf("Expected problem to be leap, was %s", iter.Problem) } - if len(iter.Solution) != 3 { + if len(iter.Solution) != 5 { t.Fatalf("Expected solution to have 3 files, had %d", len(iter.Solution)) } expected := map[string]string{ - "one.py": "# one\n", - "two.py": "# two\n", - "lib/three.py": "# three\n", + "one.py": "# one", + "two.py": "# two", + filepath.Join("lib", "three.py"): "# three", + "utf16le.py": "# utf16le", + "utf16be.py": "# utf16be", } + for filename, code := range expected { - if iter.Solution[filename] != code { - t.Errorf("Expected %s to contain %s, had %s", filename, code, iter.Solution[filename]) + if !utf8.ValidString(iter.Solution[filename]) { + t.Errorf("Iteration content is not valid UTF-8 data: %s", iter.Solution[filename]) + } + + if !strings.HasPrefix(iter.Solution[filename], code) { + t.Errorf("Expected %s to contain `%s', had `%s'", filename, code, iter.Solution[filename]) } } } diff --git a/bin/deps b/bin/deps new file mode 100644 index 000000000..16c4c916a --- /dev/null +++ b/bin/deps @@ -0,0 +1,9 @@ +#!/bin/bash + +LIBRARIES="\ + github.com/codegangsta/cli \ + github.com/stretchr/testify/assert \ + golang.org/x/net/html/charset \ + golang.org/x/text/transform" + +go get $LIBRARIES diff --git a/fixtures/iteration/python/leap/utf16be.py b/fixtures/iteration/python/leap/utf16be.py new file mode 100644 index 0000000000000000000000000000000000000000..316c25d2d00fd77a540f60c47100f15ba944ad89 GIT binary patch literal 20 bcmezOpFx>HfuWS4gdvT=kim>0i6IpLJ%0qg literal 0 HcmV?d00001 diff --git a/fixtures/iteration/python/leap/utf16le.py b/fixtures/iteration/python/leap/utf16le.py new file mode 100644 index 0000000000000000000000000000000000000000..dd06d31ddd42b17ff54bd64e75a8de816e602881 GIT binary patch literal 20 bcmezWPnki1p_HM7A&tS1!Hgk?A(a6DKxhQW literal 0 HcmV?d00001 From d7d8d13467dde06fe873cdffdab466582214c412 Mon Sep 17 00:00:00 2001 From: Kyle Ambroff Date: Sun, 5 Apr 2015 13:03:28 -0700 Subject: [PATCH 3/3] bin/deps needs to be executable. --- bin/deps | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bin/deps diff --git a/bin/deps b/bin/deps old mode 100644 new mode 100755