Skip to content

Commit c3c395d

Browse files
authored
Merge pull request #97 from henry-megarry/master
adding key separator into spec to allow '.' in key values
2 parents d3a8cbb + 596b83c commit c3c395d

12 files changed

Lines changed: 140 additions & 35 deletions

File tree

kazaam_int_test.go

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,53 @@ func TestKazaamCoalesceTransformAndShift(t *testing.T) {
168168
}
169169
}
170170

171+
func TestKazaamCoalesceTransformAndShiftWithKeySeparator(t *testing.T) {
172+
spec := `[{
173+
"operation": "coalesce",
174+
"spec": {"foo": ["rating.foo", "rating.primary"]},
175+
"keySeparator": "."
176+
}, {
177+
"operation": "shift",
178+
"spec": {"rating.foo": "foo", "rating.example.value": "rating.primary.value"},
179+
"keySeparator": "."
180+
}]`
181+
jsonOut := `{"rating":{"foo":{"value":3},"example":{"value":3}}}`
182+
183+
kazaamTransform, _ := kazaam.NewKazaam(spec)
184+
kazaamOut, _ := kazaamTransform.TransformJSONStringToString(testJSONInput)
185+
areEqual, _ := checkJSONStringsEqual(kazaamOut, jsonOut)
186+
187+
if !areEqual {
188+
t.Error("Transformed data does not match expectation.")
189+
t.Log("Expected: ", jsonOut)
190+
t.Log("Actual: ", kazaamOut)
191+
t.FailNow()
192+
}
193+
}
194+
195+
func TestKazaamCoalesceTransformAndShiftWithKeySeparatorNonDefault(t *testing.T) {
196+
spec := `[{
197+
"operation": "coalesce",
198+
"spec": {"foo": ["rating>foo", "rating>primary"]},
199+
"keySeparator": ">"
200+
}, {
201+
"operation": "shift",
202+
"spec": {"rating.foo": "foo", "rating.example.value": "rating.primary.value"}
203+
}]`
204+
jsonOut := `{"rating":{"foo":{"value":3},"example":{"value":3}}}`
205+
206+
kazaamTransform, _ := kazaam.NewKazaam(spec)
207+
kazaamOut, _ := kazaamTransform.TransformJSONStringToString(testJSONInput)
208+
areEqual, _ := checkJSONStringsEqual(kazaamOut, jsonOut)
209+
210+
if !areEqual {
211+
t.Error("Transformed data does not match expectation.")
212+
t.Log("Expected: ", jsonOut)
213+
t.Log("Actual: ", kazaamOut)
214+
t.FailNow()
215+
}
216+
}
217+
171218
func TestKazaamShiftTransformWithTimestamp(t *testing.T) {
172219
spec := `[{
173220
"operation": "shift",

spec.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ func (s *spec) UnmarshalJSON(b []byte) (err error) {
3131
err = &Error{ErrMsg: "Spec must contain at least one element", ErrType: SpecError}
3232
return
3333
}
34+
if s.Config != nil && s.KeySeparator == "" {
35+
s.KeySeparator = "."
36+
}
3437
return
3538
}
3639
return

transform/coalesce.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,12 +60,12 @@ func Coalesce(spec *Config, data []byte) ([]byte, error) {
6060
var err error
6161

6262
// grab the data
63-
dataForV, err = getJSONRaw(data, v, false)
63+
dataForV, err = getJSONRaw(data, v, false, spec.KeySeparator)
6464
if err != nil {
6565
return nil, err
6666
}
6767
if !inArray(dataForV, ignoreSlice) {
68-
data, err = setJSONRaw(data, dataForV, k)
68+
data, err = setJSONRaw(data, dataForV, k, spec.KeySeparator)
6969
if err != nil {
7070
return nil, err
7171
}

transform/concat.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ func Concat(spec *Config, data []byte) ([]byte, error) {
3333
if !ok {
3434
path, ok := vItem.(map[string]interface{})["path"]
3535
if ok {
36-
zed, err := getJSONRaw(data, path.(string), spec.Require)
36+
zed, err := getJSONRaw(data, path.(string), spec.Require, spec.KeySeparator)
3737
switch {
3838
case err != nil && spec.Require == true:
3939
return nil, RequireError("Path does not exist")
@@ -63,7 +63,7 @@ func Concat(spec *Config, data []byte) ([]byte, error) {
6363

6464
applyDelim = true
6565
}
66-
data, err := setJSONRaw(data, bookend([]byte(outString), '"', '"'), targetPath.(string))
66+
data, err := setJSONRaw(data, bookend([]byte(outString), '"', '"'), targetPath.(string), spec.KeySeparator)
6767
if err != nil {
6868
return nil, err
6969
}

transform/default.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ func Default(spec *Config, data []byte) ([]byte, error) {
1313
if err != nil {
1414
return nil, ParseError(fmt.Sprintf("Warn: Unable to coerce element to json string: %v", v))
1515
}
16-
data, err = setJSONRaw(data, dataForV, k)
16+
data, err = setJSONRaw(data, dataForV, k, spec.KeySeparator)
1717
if err != nil {
1818
return nil, err
1919
}

transform/delete.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ func Delete(spec *Config, data []byte) ([]byte, error) {
2222
}
2323

2424
var err error
25-
data, err = delJSONRaw(data, path, spec.Require)
25+
data, err = delJSONRaw(data, path, spec.Require, spec.KeySeparator)
2626
if err != nil {
2727
return nil, err
2828
}

transform/extract.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ func Extract(spec *Config, data []byte) ([]byte, error) {
66
if !ok {
77
return nil, SpecError("Unable to get path")
88
}
9-
result, err := getJSONRaw(data, outPath.(string), spec.Require)
9+
result, err := getJSONRaw(data, outPath.(string), spec.Require, spec.KeySeparator)
1010
if err != nil {
1111
return nil, err
1212
}

transform/shift.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ func Shift(spec *Config, data []byte) ([]byte, error) {
4848
if v == "$" {
4949
dataForV = data
5050
} else {
51-
dataForV, err = getJSONRaw(data, v, spec.Require)
51+
dataForV, err = getJSONRaw(data, v, spec.Require, spec.KeySeparator)
5252
if err != nil {
5353
return nil, err
5454
}
@@ -65,7 +65,7 @@ func Shift(spec *Config, data []byte) ([]byte, error) {
6565
// Note: following pattern from current Shift() - if multiple elements are included in an array,
6666
// they will each successively overwrite each other and only the last element will be included
6767
// in the transformed data.
68-
outData, err = setJSONRaw(outData, dataForV, k)
68+
outData, err = setJSONRaw(outData, dataForV, k, spec.KeySeparator)
6969
if err != nil {
7070
return nil, err
7171
}

transform/timestamp.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ func Timestamp(spec *Config, data []byte) ([]byte, error) {
4848
inputFormat = time.RFC3339
4949
} else {
5050
// grab the data
51-
dataForV, err = getJSONRaw(data, k, spec.Require)
51+
dataForV, err = getJSONRaw(data, k, spec.Require, spec.KeySeparator)
5252
if err != nil {
5353
return nil, err
5454
}
@@ -65,7 +65,7 @@ func Timestamp(spec *Config, data []byte) ([]byte, error) {
6565
if err != nil {
6666
return nil, err
6767
}
68-
data, err = setJSONRaw(data, []byte(formattedItem), k)
68+
data, err = setJSONRaw(data, []byte(formattedItem), k, spec.KeySeparator)
6969
if err != nil {
7070
return nil, err
7171
}
@@ -84,7 +84,7 @@ func Timestamp(spec *Config, data []byte) ([]byte, error) {
8484
}
8585
// replacing the wildcard here feels hacky, but seems to be the
8686
// quickest way to achieve the outcome we want
87-
data, err = setJSONRaw(data, []byte(formattedItem), strings.Replace(k, "*", strconv.Itoa(idx), -1))
87+
data, err = setJSONRaw(data, []byte(formattedItem), strings.Replace(k, "*", strconv.Itoa(idx), -1), spec.KeySeparator)
8888
if err != nil {
8989
return nil, err
9090
}

transform/util.go

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,10 @@ func (s SpecError) Error() string {
3535
// Config contains the options that dictate the behavior of a transform. The internal
3636
// `spec` object can be an arbitrary json configuration for the transform.
3737
type Config struct {
38-
Spec *map[string]interface{} `json:"spec"`
39-
Require bool `json:"require,omitempty"`
40-
InPlace bool `json:"inplace,omitempty"`
38+
Spec *map[string]interface{} `json:"spec"`
39+
Require bool `json:"require,omitempty"`
40+
InPlace bool `json:"inplace,omitempty"`
41+
KeySeparator string `json:"keySeparator"`
4142
}
4243

4344
var (
@@ -46,8 +47,8 @@ var (
4647
)
4748

4849
// Given a json byte slice `data` and a kazaam `path` string, return the object at the path in data if it exists.
49-
func getJSONRaw(data []byte, path string, pathRequired bool) ([]byte, error) {
50-
objectKeys := strings.Split(path, ".")
50+
func getJSONRaw(data []byte, path string, pathRequired bool, keySeparator string) ([]byte, error) {
51+
objectKeys := strings.Split(path, keySeparator)
5152
numOfInserts := 0
5253
for element, k := range objectKeys {
5354
// check the object key to see if it also contains an array reference
@@ -64,7 +65,7 @@ func getJSONRaw(data []byte, path string, pathRequired bool) ([]byte, error) {
6465
// ArrayEach setup
6566
objectKeys[element+numOfInserts] = objKey
6667
beforePath := objectKeys[:element+numOfInserts+1]
67-
newPath := strings.Join(objectKeys[element+numOfInserts+1:], ".")
68+
newPath := strings.Join(objectKeys[element+numOfInserts+1:], keySeparator)
6869
var results [][]byte
6970

7071
// use jsonparser.ArrayEach to copy the array into results
@@ -82,7 +83,7 @@ func getJSONRaw(data []byte, path string, pathRequired bool) ([]byte, error) {
8283
// GetJSONRaw() the rest of path for each element in results
8384
if newPath != "" {
8485
for i, value := range results {
85-
intermediate, err := getJSONRaw(value, newPath, pathRequired)
86+
intermediate, err := getJSONRaw(value, newPath, pathRequired, keySeparator)
8687
if err == jsonparser.KeyPathNotFoundError {
8788
if pathRequired {
8889
return nil, NonExistentPath
@@ -137,11 +138,10 @@ func getJSONRaw(data []byte, path string, pathRequired bool) ([]byte, error) {
137138
}
138139

139140
// setJSONRaw sets the value at a key and handles array indexing
140-
func setJSONRaw(data, out []byte, path string) ([]byte, error) {
141+
func setJSONRaw(data, out []byte, path, keySeparator string) ([]byte, error) {
141142
var err error
142-
splitPath := strings.Split(path, ".")
143+
splitPath := strings.Split(path, keySeparator)
143144
numOfInserts := 0
144-
145145
for element, k := range splitPath {
146146
arrayRefs := jsonPathRe.FindAllStringSubmatch(k, -1)
147147
if arrayRefs != nil && len(arrayRefs) > 0 {
@@ -158,7 +158,7 @@ func setJSONRaw(data, out []byte, path string) ([]byte, error) {
158158
// ArrayEach setup
159159
splitPath[element+numOfInserts] = objKey
160160
beforePath := splitPath[:element+numOfInserts+1]
161-
afterPath := strings.Join(splitPath[element+numOfInserts+1:], ".")
161+
afterPath := strings.Join(splitPath[element+numOfInserts+1:], keySeparator)
162162
// use jsonparser.ArrayEach to count the number of items in the
163163
// array
164164
var arraySize int
@@ -175,18 +175,18 @@ func setJSONRaw(data, out []byte, path string) ([]byte, error) {
175175
// iterate through each item in the array by replacing the
176176
// wildcard with an int and joining the path back together
177177
newArrayKey := strings.Join([]string{"[", strconv.Itoa(i), "]"}, "")
178-
beforePathStr := strings.Join(beforePath, ".")
178+
beforePathStr := strings.Join(beforePath, keySeparator)
179179
beforePathArrayKeyStr := strings.Join([]string{beforePathStr, newArrayKey}, "")
180180
// if there's nothing that comes after the array index,
181181
// don't join so that we avoid trailing cruft
182182
if len(afterPath) > 0 {
183-
newPath = strings.Join([]string{beforePathArrayKeyStr, afterPath}, ".")
183+
newPath = strings.Join([]string{beforePathArrayKeyStr, afterPath}, keySeparator)
184184
} else {
185185
newPath = beforePathArrayKeyStr
186186
}
187187
// now call the function, but this time with an array index
188188
// instead of a wildcard
189-
data, err = setJSONRaw(data, out, newPath)
189+
data, err = setJSONRaw(data, out, newPath, keySeparator)
190190
if err != nil {
191191
return nil, err
192192
}
@@ -209,9 +209,9 @@ func setJSONRaw(data, out []byte, path string) ([]byte, error) {
209209
}
210210

211211
// delJSONRaw deletes the value at a path and handles array indexing
212-
func delJSONRaw(data []byte, path string, pathRequired bool) ([]byte, error) {
212+
func delJSONRaw(data []byte, path string, pathRequired bool, keySeparator string) ([]byte, error) {
213213
var err error
214-
splitPath := strings.Split(path, ".")
214+
splitPath := strings.Split(path, keySeparator)
215215
numOfInserts := 0
216216

217217
for element, k := range splitPath {

0 commit comments

Comments
 (0)