Directive migration

The following is the list of all the directives mapping from version 1.0 to version 2.0. Rewriter allows you to convert from old format to new. If the recipe doesn't include #pragma version 2.0 the recipe is treated as version 1.0 and the rewriter is invoked, else new parser is used.

Name

Directive - Version 1.0

Directive - Version 2.0

Name

Directive - Version 1.0

Directive - Version 2.0

SET

set column salary salary > 100 ? 100 : salary

set-column :salary exp:{salary > 100 ? 100 : salary};

SET

set columns fname,lname,address,city,state,country,zipcode

set-columns :fname,:lname,:address,:city,:state,:country,:zipcode;

RENAME

rename body_fname fname

rename :body_fname :fname;

SET-TYPE

set-type value int

set-type :value int;

DROP

drop fname,lname,address,zipcode,city

drop :fname,:lname,:address,:zipcode,:city;

MERGE

merge fname lname name ,

merge :fname :lname :name ',';

UPPERCASE

uppercase name

uppercase :name;

LOWERCASE

lowercase name

lowercase :name;

TITLECASE

titlecase name

titlecase :name;

INDEXSPLIT

indexsplit department 1 10 zone

indexsplit :department 1 10 :zone;

SPLIT

split name , fname lname

split :name ',' :fname :lname;

FILTER-ROW-IF-MATCHED

filter-row-if-matched address .*?

filter-row-if-matched :address '.*?';

FILTER-ROW-IF-NOT-MATCHED

filter-row-if-not-matched address .*?

filter-row-if-not-matched :address '.*?';

FILTER-ROW-IF-TRUE

filter-row-if-true age < 10.0

filter-row-if-true exp:{age < 10.0};

FILTER-ROW-IF-FALSE

filter-row-if-false age < 10.0 && gender == 'm'

filter-row-if-false exp:{age < 10.0 && gender == 'm'};

SET-VARIABLE

set-variable test count > 10

set-variable test exp:{count > 10};

INCREMENT-VARIABLE

increment-variable test 1 count > 10

increment-variable test 1 exp:{count > 10};

MASK-NUMBER

mask-number ssn xxx-xx-####

mask-number :ssn 'xxx-xx-####';

MASK-SHUFFLE

mask-shuffle address

mask-shuffle :address;

FORMAT-DATE

format-date startdate yyyy-MM-dd

format-date :startdate 'yyyy-MM-dd';

FORMAT-UNIX-TIMESTAMP

format-unix-timestamp timestamp yyyy/MM/dd

format-unix-timestamp :timestamp 'yyyy/MM/dd';

QUANTIZE

quantize col1 col2 1:2=test,3:4=test1

quantize :col1 :col2 1:2=test,3:4=test1;

FIND-AND-REPLACE

find-and-replace name s/test//g

find-and-replace :name 's/test//g';

PARSE-AS-CSV

parse-as-csv body , true

parse-as-csv :body ',' true;

PARSE-AS-JSON

parse-as-json body 1

parse-as-json :body 1;

PARSE-AS-PROTOBUF

parse-as-protobuf body test event 1.0

parse-as-protobuf :body test 'event' '1.0';

JSON-PATH

json-path source target a/b/c

json-path :source :target 'a/b/c';

SET-CHARSET

set-charset body utf-8

set-charset :body utf-8;

INVOKE-HTTP

invoke-http http://a.b/json fname,lname,address a=b,x=y

invoke-http 'http://a.b/json' :fname,:lname,:address 'a=b,x=y';

SET-RECORD-DELIM

set-record-delim body , 10

set-record-delim :body ',' 10;

PARSE-AS-FIXED-LENGTH

parse-as-fixed-length body 3,4,5,6,7,8 #

parse-as-fixed-length :body 3,4,5,6,7,8 '#';

SPLIT-TO-ROWS

split-to-rows body ,

split-to-rows :body ',';

SPLIT-TO-COLUMNS

split-to-columns body ,

split-to-columns :body ',';

PARSE-XML-TO-JSON

parse-xml-to-json body 1

parse-xml-to-json :body 1;

PARSE-AS-XML

parse-as-xml xml

parse-as-xml :xml;

PARSE-AS-EXCEL

parse-as-excel body 0

parse-as-excel :body '0';

XPATH

xpath name fname /items/item/first_name

xpath :name :fname '/items/item/first_name';

XPATH-ARRAY

xpath-array name fname /items/item/first_name

xpath-array :name :fname '/items/item/first_name';

FLATTEN

flatten a,b,c,d

flatten :a,:b,:c,:d;

COPY

copy source target true

copy :source :target true;

FILL-NULL-OR-EMPTY

fill-null-or-empty value ,

fill-null-or-empty :value ',';

CUT-CHARACTER

cut-character phone areacode 1-3

cut-character :phone :areacode '1-3';

GENERATE-UUID

generate-uuid ssn

generate-uuid :ssn;

URL-ENCODE

url-encode url

url-encode :url;

URL-DECODE

url-decode url

url-decode :url;

PARSE-AS-LOG

parse-as-log body %m-%y-%{HOSTNAME}

parse-as-log :body '%m-%y-%{HOSTNAME}';

PARSE-AS-DATE

parse-as-date date UTC

parse-as-date :date 'UTC';

PARSE-AS-SIMPLE-DATE

parse-as-simple-date date yyyy-MM-dd

parse-as-simple-date :date 'yyyy-MM-dd';

DIFF-DATE

diff-date date1 date2 diffdate

diff-date :date1 :date2 :diffdate;

KEEP

keep fname,lname,address,city,zipcode

keep :fname,:lname,:address,:city,:zipcode;

PARSE-AS-HL7

parse-as-hl7 body 1

parse-as-hl7 :body 1;

SPLIT-EMAIL

split-email email

split-email :email;

SWAP

swap col1 col2

swap :col1 :col2;

HASH

hash col SHA1 true

hash :col 'SHA1' true;

WRITE-AS-JSON-MAP

write-as-json-map output

write-as-json-map :output;

WRITE-AS-JSON-OBJECT

write-as-json-object output fname,lname,address

write-as-json-object :output :fname,:lname,:address;

WRITE-AS-CSV

write-as-csv output

write-as-csv :output;

FILTER-ROWS-ON

filter-rows-on condition-false output < 10

filter-rows-on condition-false exp:{output < 10};

FILTER-ROWS-ON

filter-rows-on condition-true output < 10

filter-rows-on condition-true exp:{output < 10};

FILTER-ROWS-ON

filter-rows-on empty-or-null-columns fname,lname

filter-rows-on empty-or-null-columns :fname,:lname;

FILTER-ROWS-ON

filter-rows-on regex-match col test*

filter-rows-on regex-match :col 'test*';

FILTER-ROWS-ON

filter-rows-on regex-not-match col test*

filter-rows-on regex-not-match :col 'test*';

PARSE-AS-AVRO-FILE

parse-as-avro-file body

parse-as-avro-file :body;

SEND-TO-ERROR

send-to-error Fare < 8.06

send-to-error exp:{Fare < 8.06};

FAIL

fail Fare < 8.06

fail exp:{Fare < 8.06};

TEXT-DISTANCE

text-distance abc col1 col2 output

text-distance abc :col1 :col2 :output;

TEXT-METRIC

text-metric abc col1 col2 output

text-metric abc :col1 :col2 :output;

CATALOG-LOOKUP

catalog-lookup ICD-9 value

catalog-lookup 'ICD-9' :value;

TABLE-LOOKUP

table-lookup value mylookup-table

table-lookup :value 'mylookup-table';

STEMMING

stemming text

stemming :text;

COLUMNS-REPLACE

columns-replace s/body_//g

columns-replace 's/body_//g';

EXTRACT-REGEX-GROUPS

extract-regex-groups body s/body_//g

extract-regex-groups :body 's/body_//g';

SPLIT-URL

split-url url

split-url :url;

CLEANSE-COLUMN-NAMES

cleanse-column-names

cleanse-column-names;

CHANGE-COLUMN-CASE

change-column-case upper

change-column-case upper;

SET-COLUMN

set-column value output > 10 ? 'test' : 'non-test'

set-column :value exp:{output > 10 ? 'test' : 'non-test'};

ENCODE

encode base64 binary

encode base64 :binary;

DECODE

decode base64 binary

decode base64 :binary;

TRIM

trim name

trim :name;

LTRIM

ltrim name

ltrim :name;

RTRIM

rtrim name

rtrim :name;

 

Created in 2020 by Google Inc.