-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* add remap transformation * rename * fix any arrays
- Loading branch information
Showing
6 changed files
with
298 additions
and
14 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| /* | ||
| * Copyright (c) 2024 Alexey Khokhlov | ||
| */ | ||
|
|
||
| package remap | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "errors" | ||
|
|
||
| "github.com/expr-lang/expr/vm" | ||
| "github.com/rs/zerolog" | ||
|
|
||
| "github.com/expr-lang/expr" | ||
|
|
||
| "github.com/pipelane/pipelaner" | ||
| ) | ||
|
|
||
| var ( | ||
| ErrInvalidDataType = errors.New("error invalid data type") | ||
| ) | ||
|
|
||
| type EnvMap struct { | ||
| Data any | ||
| } | ||
|
|
||
| type ExprConfig struct { | ||
| Code string `pipelane:"code"` | ||
| } | ||
|
|
||
| type Remap struct { | ||
| cfg *pipelaner.BaseLaneConfig | ||
| logger zerolog.Logger | ||
| program *vm.Program | ||
| } | ||
|
|
||
| func init() { | ||
| pipelaner.RegisterMap("remap", &Remap{}) | ||
| } | ||
|
|
||
| func (e *Remap) Init(ctx *pipelaner.Context) error { | ||
| e.cfg = ctx.LaneItem().Config() | ||
| e.logger = pipelaner.NewLogger() | ||
| v := &ExprConfig{} | ||
| err := e.cfg.ParseExtended(v) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| program, err := expr.Compile(v.Code, expr.Env(EnvMap{})) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| e.program = program | ||
| return nil | ||
| } | ||
|
|
||
| func (e *Remap) Map(_ *pipelaner.Context, val any) any { | ||
| var v any | ||
| switch value := val.(type) { | ||
| case map[string]any: | ||
| v = value | ||
| case map[string][]any: | ||
| v = value | ||
| case string: | ||
| b := []byte(value) | ||
| err := json.Unmarshal(b, &v) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| case []byte: | ||
| err := json.Unmarshal(value, &v) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| default: | ||
| return ErrInvalidDataType | ||
| } | ||
| output, err := expr.Run(e.program, EnvMap{Data: v}) | ||
| if err != nil { | ||
| e.logger.Err(err).Msg("Expr: output error") | ||
| return err | ||
| } | ||
| return output | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,200 @@ | ||
| /* | ||
| * Copyright (c) 2024 Alexey Khokhlov | ||
| */ | ||
|
|
||
| package remap | ||
|
|
||
| import ( | ||
| "context" | ||
| "testing" | ||
|
|
||
| "github.com/rs/zerolog" | ||
| "github.com/stretchr/testify/assert" | ||
| "github.com/stretchr/testify/require" | ||
|
|
||
| "github.com/pipelane/pipelaner" | ||
| ) | ||
|
|
||
| func newCfg( | ||
| itemType pipelaner.LaneTypes, | ||
| extended map[string]any, | ||
| ) *pipelaner.BaseLaneConfig { | ||
| c, err := pipelaner.NewBaseConfigWithTypeAndExtended( | ||
| itemType, | ||
| "test_maps_sinks", | ||
| extended, | ||
| ) | ||
| if err != nil { | ||
| return nil | ||
| } | ||
| return c | ||
| } | ||
|
|
||
| func TestExprLanguage_Map(t *testing.T) { | ||
| type args struct { | ||
| val any | ||
| ctx *pipelaner.Context | ||
| } | ||
| tests := []struct { | ||
| name string | ||
| args args | ||
| want any | ||
| }{ | ||
| { | ||
| name: "test expr maps return nil", | ||
| args: args{ | ||
| ctx: pipelaner.NewContext(context.Background(), | ||
| pipelaner.NewLaneItem(newCfg(pipelaner.MapType, | ||
| map[string]any{ | ||
| "code": "{ \"value_name\": Data.name, \"value_price\": Data.price}", | ||
| }), | ||
| ), | ||
| ), | ||
| val: map[string]any{ | ||
| "id": 1, | ||
| "name": "iPhone 12", | ||
| "price": 999, | ||
| "quantity": 1, | ||
| }, | ||
| }, | ||
| want: map[string]any{ | ||
| "value_name": "iPhone 12", | ||
| "value_price": 999, | ||
| }, | ||
| }, | ||
| } | ||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| e := &Remap{ | ||
| logger: zerolog.Nop(), | ||
| } | ||
| err := e.Init(tt.args.ctx) | ||
| require.Nil(t, err) | ||
| got := e.Map(tt.args.ctx, tt.args.val) | ||
| assert.Equal(t, got, tt.want) | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| func TestExprLanguage_String(t *testing.T) { | ||
| type args struct { | ||
| val any | ||
| ctx *pipelaner.Context | ||
| } | ||
| tests := []struct { | ||
| name string | ||
| args args | ||
| want any | ||
| }{ | ||
| { | ||
| name: "test remap string return nil", | ||
| args: args{ | ||
| ctx: pipelaner.NewContext(context.Background(), | ||
| pipelaner.NewLaneItem(newCfg(pipelaner.MapType, | ||
| map[string]any{ | ||
| "code": "{ \"value_name\": Data.name, \"value_price\": Data.price}", | ||
| }), | ||
| ), | ||
| ), | ||
| val: " {\"id\": 1,\"name\": \"iPhone 12\",\"price\": \"999\",\"quantity\": 1}", | ||
| }, | ||
| want: map[string]any{ | ||
| "value_name": "iPhone 12", | ||
| "value_price": "999", | ||
| }, | ||
| }, | ||
| } | ||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| e := &Remap{ | ||
| logger: zerolog.Nop(), | ||
| } | ||
| err := e.Init(tt.args.ctx) | ||
| require.Nil(t, err) | ||
| got := e.Map(tt.args.ctx, tt.args.val) | ||
| assert.Equal(t, got, tt.want) | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| func TestExprLanguage_StringArray(t *testing.T) { | ||
| type args struct { | ||
| val any | ||
| ctx *pipelaner.Context | ||
| } | ||
| tests := []struct { | ||
| name string | ||
| args args | ||
| want any | ||
| }{ | ||
| { | ||
| name: "test remap string array", | ||
| args: args{ | ||
| ctx: pipelaner.NewContext(context.Background(), | ||
| pipelaner.NewLaneItem(newCfg(pipelaner.MapType, | ||
| map[string]any{ | ||
| "code": "{ \"value_name\": Data[0].name, \"value_price\": Data[0].price}", | ||
| }), | ||
| ), | ||
| ), | ||
| val: "[{\"id\": 1,\"name\": \"iPhone 12\",\"price\": \"999\",\"quantity\": 1}, {\"id\": 2,\"name\": \"iPhone 13\",\"price\": \"999\",\"quantity\": 1}]", | ||
| }, | ||
| want: map[string]any{ | ||
| "value_name": "iPhone 12", | ||
| "value_price": "999", | ||
| }, | ||
| }, | ||
| } | ||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| e := &Remap{ | ||
| logger: zerolog.Nop(), | ||
| } | ||
| err := e.Init(tt.args.ctx) | ||
| require.Nil(t, err) | ||
| got := e.Map(tt.args.ctx, tt.args.val) | ||
| assert.Equal(t, got, tt.want) | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| func TestExprLanguage_Bytes(t *testing.T) { | ||
| type args struct { | ||
| val any | ||
| ctx *pipelaner.Context | ||
| } | ||
| tests := []struct { | ||
| name string | ||
| args args | ||
| want any | ||
| }{ | ||
| { | ||
| name: "test remap string return nil", | ||
| args: args{ | ||
| ctx: pipelaner.NewContext(context.Background(), | ||
| pipelaner.NewLaneItem(newCfg(pipelaner.MapType, | ||
| map[string]any{ | ||
| "code": "{ \"value_name\": Data.name, \"value_price\": Data.price}", | ||
| }), | ||
| ), | ||
| ), | ||
| val: []byte("{\"id\": 1,\"name\": \"iPhone 12\",\"price\": \"999\",\"quantity\": 1}"), | ||
| }, | ||
| want: map[string]any{ | ||
| "value_name": "iPhone 12", | ||
| "value_price": "999", | ||
| }, | ||
| }, | ||
| } | ||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| e := &Remap{ | ||
| logger: zerolog.Nop(), | ||
| } | ||
| err := e.Init(tt.args.ctx) | ||
| require.Nil(t, err) | ||
| got := e.Map(tt.args.ctx, tt.args.val) | ||
| assert.Equal(t, got, tt.want) | ||
| }) | ||
| } | ||
| } |