#62 Add Delete Transform (#63)

* Initial setup for delete transform

* Finish Delete transform and add more tests

README.md

@@ -46,6 +46,7 @@ Kazaam currently supports the following transforms:
 - uuid
 - default
 - pass
+- delete
 
 ### Shift
 The shift transform is the current Kazaam workhorse used for remapping of fields.
@@ -357,6 +358,39 @@ A default transform provides the ability to set a key's value explicitly. For ex
 would ensure that the output JSON message includes `{"type": "message"}`.
 
 
+### Delete
+A delete transform provides the ability to delete keys in place.
+```javascript
+{
+  "operation": "delete",
+  "spec": {
+    "paths": ["doc.uid", "doc.guidObjects[1]"]
+  }
+}
+```
+
+executed on a json message with format
+```javascript
+{
+  "doc": {
+    "uid": 12345,
+    "guid": ["guid0", "guid2", "guid4"],
+    "guidObjects": [{"id": "guid0"}, {"id": "guid2"}, {"id": "guid4"}]
+  }
+}
+```
+
+would result in
+```javascript
+{
+  "doc": {
+    "guid": ["guid0", "guid2", "guid4"],
+    "guidObjects": [{"id": "guid0"}, {"id": "guid4"}]
+  }
+}
+```
+
+
 ### Pass
 A pass transform, as the name implies, passes the input data unchanged to the output. This is used internally
 when a null transform spec is specified, but may also be useful for testing.

kazaam.go

@@ -31,6 +31,7 @@ func init() {
 		"shift":     transform.Shift,
 		"extract":   transform.Extract,
 		"default":   transform.Default,
+		"delete":    transform.Delete,
 		"concat":    transform.Concat,
 		"coalesce":  transform.Coalesce,
 		"timestamp": transform.Timestamp,

kazaam_int_test.go

@@ -415,3 +415,23 @@ func TestKazaamTransformTwoOpWithOverRequire(t *testing.T) {
 		t.FailNow()
 	}
 }
+
+func TestKazaamTransformDelete(t *testing.T) {
+	spec := `[{
+		"operation": "delete",
+		"spec": {"paths": ["doc.uid", "doc.guidObjects[1]"]}
+	}]`
+	jsonIn := `{"doc":{"uid":12345,"guid":["guid0","guid2","guid4"],"guidObjects":[{"id":"guid0"},{"id":"guid2"},{"id":"guid4"}]}}`
+	jsonOut := `{"doc":{"guid":["guid0","guid2","guid4"],"guidObjects":[{"id":"guid0"},{"id":"guid4"}]}}`
+
+	kazaamTransform, _ := kazaam.NewKazaam(spec)
+	kazaamOut, _ := kazaamTransform.TransformJSONStringToString(jsonIn)
+	areEqual, _ := checkJSONStringsEqual(kazaamOut, jsonOut)
+
+	if !areEqual {
+		t.Error("Transformed data does not match expectation.")
+		t.Log("Expected: ", jsonOut)
+		t.Log("Actual:   ", kazaamOut)
+		t.FailNow()
+	}
+}

kazaam_test.go

@@ -38,7 +38,7 @@ func TestReregisterKazaamTransform(t *testing.T) {
 }
 
 func TestDefaultTransformsSetCardinarily(t *testing.T) {
-	if len(validSpecTypes) != 8 {
+	if len(validSpecTypes) != 9 {
 		t.Error("Unexpected number of default transforms. Missing tests?")
 	}
 }

transform/delete.go

@@ -0,0 +1,31 @@
+package transform
+
+import (
+	"fmt"
+)
+
+// Delete deletes keys in-place from the provided data if they exist
+// keys are specified in an array under "keys" in the spec.
+func Delete(spec *Config, data []byte) ([]byte, error) {
+	paths, pathsOk := (*spec.Spec)["paths"]
+	if !pathsOk {
+		return nil, SpecError("Unable to get paths to delete")
+	}
+	pathSlice, sliceOk := paths.([]interface{})
+	if !sliceOk {
+		return nil, SpecError(fmt.Sprintf("paths should be a slice of strings: %v", paths))
+	}
+	for _, pItem := range pathSlice {
+		path, ok := pItem.(string)
+		if !ok {
+			return nil, SpecError(fmt.Sprintf("Error processing %v: path should be a string", pItem))
+		}
+
+		var err error
+		data, err = delJSONRaw(data, path, spec.Require)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return data, nil
+}

transform/delete_test.go

@@ -0,0 +1,150 @@
+package transform
+
+import "testing"
+
+func TestDelete(t *testing.T) {
+	spec := `{"paths": ["rating.example"]}`
+	jsonOut := `{"rating":{"primary":{"value":3}}}`
+
+	cfg := getConfig(spec, false)
+	kazaamOut, err := getTransformTestWrapper(Delete, cfg, testJSONInput)
+
+	if err != nil {
+		t.Error("Error in transform (simplejson).")
+		t.Log("Error: ", err.Error())
+		t.FailNow()
+	}
+
+	areEqual, _ := checkJSONBytesEqual(kazaamOut, []byte(jsonOut))
+	if !areEqual {
+		t.Error("Transformed data does not match expectation.")
+		t.Log("Expected:   ", jsonOut)
+		t.Log("Actual:     ", string(kazaamOut))
+		t.FailNow()
+	}
+}
+
+func TestDeleteSpecErrorNoPathsKey(t *testing.T) {
+	spec := `{"pathz": ["a.path"]}`
+	expectedErr := "Unable to get paths to delete"
+
+	cfg := getConfig(spec, false)
+	_, err := getTransformTestWrapper(Delete, cfg, testJSONInput)
+
+	if err == nil {
+		t.Error("Should have generated error for invalid paths")
+		t.Log("Spec:   ", spec)
+		t.FailNow()
+	}
+	e, ok := err.(SpecError)
+	if !ok {
+		t.Error("Unexpected error type")
+		t.FailNow()
+	}
+
+	if e.Error() != expectedErr {
+		t.Error("Unexpected error details")
+		t.Log("Expected:   ", expectedErr)
+		t.Log("Actual:     ", e.Error())
+		t.FailNow()
+	}
+}
+
+func TestDeleteSpecErrorInvalidPaths(t *testing.T) {
+	spec := `{"paths": false}`
+	expectedErr := "paths should be a slice of strings: false"
+
+	cfg := getConfig(spec, false)
+	_, err := getTransformTestWrapper(Delete, cfg, testJSONInput)
+
+	if err == nil {
+		t.Error("Should have generated error for invalid paths")
+		t.Log("Spec:   ", spec)
+		t.FailNow()
+	}
+	e, ok := err.(SpecError)
+	if !ok {
+		t.Error("Unexpected error type")
+		t.FailNow()
+	}
+
+	if e.Error() != expectedErr {
+		t.Error("Unexpected error details")
+		t.Log("Expected:   ", expectedErr)
+		t.Log("Actual:     ", e.Error())
+		t.FailNow()
+	}
+}
+
+func TestDeleteSpecErrorInvalidPathItem(t *testing.T) {
+	spec := `{"paths": ["foo", 42]}`
+	expectedErr := "Error processing 42: path should be a string"
+
+	cfg := getConfig(spec, false)
+	_, err := getTransformTestWrapper(Delete, cfg, testJSONInput)
+
+	if err == nil {
+		t.Error("Should have generated error for invalid paths")
+		t.Log("Spec:   ", spec)
+		t.FailNow()
+	}
+	e, ok := err.(SpecError)
+	if !ok {
+		t.Error("Unexpected error type")
+		t.FailNow()
+	}
+
+	if e.Error() != expectedErr {
+		t.Error("Unexpected error details")
+		t.Log("Expected:   ", expectedErr)
+		t.Log("Actual:     ", e.Error())
+		t.FailNow()
+	}
+}
+
+func TestDeleteSpecErrorWildcardNotSupported(t *testing.T) {
+	spec := `{"paths": ["ratings[*].value"]}`
+	jsonIn := `{"ratings: [{"value": 3, "user": "rick"}, {"value": 7, "user": "jerry"}]}`
+	expectedErr := "Array wildcard not supported for this operation."
+
+	cfg := getConfig(spec, false)
+	_, err := getTransformTestWrapper(Delete, cfg, jsonIn)
+
+	if err == nil {
+		t.Error("Should have generated error for invalid paths")
+		t.Log("Spec:   ", spec)
+		t.FailNow()
+	}
+	e, ok := err.(SpecError)
+	if !ok {
+		t.Error("Unexpected error type")
+		t.FailNow()
+	}
+
+	if e.Error() != expectedErr {
+		t.Error("Unexpected error details")
+		t.Log("Expected:   ", expectedErr)
+		t.Log("Actual:     ", e.Error())
+		t.FailNow()
+	}
+}
+
+func TestDeleteWithRequire(t *testing.T) {
+	spec := `{"paths": ["rating.examplez"]}`
+
+	cfg := getConfig(spec, true)
+	_, err := getTransformTestWrapper(Delete, cfg, testJSONInput)
+
+	if err == nil {
+		t.Error("Should have generated error for invalid paths")
+		t.Log("Spec:   ", spec)
+		t.FailNow()
+	}
+	_, ok := err.(RequireError)
+	if !ok {
+		t.Error("Unexpected error type")
+		t.Error(err.Error())
+		t.FailNow()
+	}
+
+}

transform/util.go

@@ -208,6 +208,50 @@ func setJSONRaw(data, out []byte, path string) ([]byte, error) {
 	return data, nil
 }
 
+// delJSONRaw deletes the value at a path and handles array indexing
+func delJSONRaw(data []byte, path string, pathRequired bool) ([]byte, error) {
+	var err error
+	splitPath := strings.Split(path, ".")
+	numOfInserts := 0
+
+	for element, k := range splitPath {
+		arrayRefs := jsonPathRe.FindAllStringSubmatch(k, -1)
+		if arrayRefs != nil && len(arrayRefs) > 0 {
+			objKey := arrayRefs[0][1]      // the key
+			arrayKeyStr := arrayRefs[0][2] // the array index
+			err = validateArrayKeyString(arrayKeyStr)
+			if err != nil {
+				return nil, err
+			}
+
+			// not currently supported
+			if arrayKeyStr == "*" {
+				return nil, SpecError("Array wildcard not supported for this operation.")
+			}
+
+			// if not a wildcard then piece that path back together with the
+			// array index as an entry in the splitPath slice
+			splitPath = makePathWithIndex(arrayKeyStr, objKey, splitPath, element+numOfInserts)
+			numOfInserts++
+		} else {
+			// no array reference, good to go
+			continue
+		}
+	}
+
+	if pathRequired {
+		_, _, _, err = jsonparser.Get(data, splitPath...)
+		if err == jsonparser.KeyPathNotFoundError {
+			return nil, NonExistentPath
+		} else if err != nil {
+			return nil, err
+		}
+	}
+
+	data = jsonparser.Delete(data, splitPath...)
+	return data, nil
+}
+
 // validateArrayKeyString is a helper function to make sure the array index is
 // legal
 func validateArrayKeyString(arrayKeyStr string) error {