...
Pipeline Studio. To create and manage pipelines through the CDAP Pipeline Studio, see the Data Pipeline User Guide.
Command line tools (such as
curlor the CDAP CLI).Lifecycle Microservices.
In order to create a pipeline application, a pipeline configuration (either as a file or in-memory) is required. It specifies the configuration of plugins to be used along with their properties.
...
| Code Block |
|---|
{
"name": "ETLApp",
"artifact": {
"name": "cdap-data-pipeline",
"version": "6.3.0",
"scope": "SYSYEM"
},
"config": {
"schedule": "* * * * *",
"engine": "spark",
"postActions": [
{
"name": "Email",
"plugin": {
"name": "Email",
"type": "postaction",
"artifact": {
"name": "core-plugins",
"version": "2.5.0",
"scope": "SYSTEM"
},
"properties": {
"runCondition": "completion",
"includeWorkflowToken": "false",
"recipients": "users@example.com",
"sender": "admin@example.com",
"subject": "Post-action Status",
"message": "Completed run."
}
}
}
],
"stages": [
{
"name": "myFileSource",
"plugin": {
"name": "File",
"type": "batchsource",
"artifact": {
"name": "core-plugins",
"version": "2.5.0",
"scope": "SYSTEM"
},
"properties": {
"name": "myFileSource",
"duration": "1m",
"format": "text",
"schema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"body\",\"type\":\"string\"}]}"
}
}
},
{
"name": "myFileSink",
"plugin": {
"name": "File",
"type": "batchsink",
"artifact": {
"name": "core-plugins",
"version": "2.5.0",
"scope": "SYSTEM"
},
"properties": {
"name": "myFileSink",
"schema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"ts\",\"type\":\"long\"},{\"name\":\"body\",\"type\":\"string\"}]}",
"format”: “csv”
}
}
}
],
"connections": [
{
"from": "myFileSource",
"to": "myFileSink"
}
]
}
} |
...