cdap-data-pipeline.json

{
    "artifact": {
        "name": "cdap-data-pipeline",
        "version": "6.10.1",
        "scope": "SYSTEM",
        "label": "Data Pipeline - Batch"
    },
    "description": "",
    "name": "",
    "change": {
        "description": ""
    },
    "parentVersion": "",
    "config": {
        "resources": {
            "memoryMB": 2048,
            "virtualCores": 1
        },
        "driverResources": {
            "memoryMB": 2048,
            "virtualCores": 1
        },
        "connections": [
            {
                "from": "File",
                "to": "GCS"
            }
        ],
        "comments": [],
        "postActions": [],
        "properties": {},
        "processTimingEnabled": true,
        "stageLoggingEnabled": false,
        "stages": [
            {
                "name": "File",
                "plugin": {
                    "name": "File",
                    "type": "batchsource",
                    "label": "File",
                    "artifact": {
                        "name": "core-plugins",
                        "version": "2.12.1",
                        "scope": "SYSTEM"
                    },
                    "properties": {
                        "sampleSize": "1000",
                        "enableQuotedValues": "false",
                        "skipHeader": "false",
                        "filenameOnly": "false",
                        "recursive": "false",
                        "ignoreNonExistingFolders": "false",
                        "fileEncoding": "UTF-8",
                        "schema": "{\"name\":\"fileRecord\",\"type\":\"record\",\"fields\":[{\"name\":\"offset\",\"type\":\"long\"},{\"name\":\"body\",\"type\":\"string\"}]}"
                    }
                },
                "outputSchema": "{\"name\":\"fileRecord\",\"type\":\"record\",\"fields\":[{\"name\":\"offset\",\"type\":\"long\"},{\"name\":\"body\",\"type\":\"string\"}]}",
                "id": "File"
            },
            {
                "name": "GCS",
                "plugin": {
                    "name": "GCS",
                    "type": "batchsink",
                    "label": "GCS",
                    "artifact": {
                        "name": "google-cloud",
                        "version": "0.23.1",
                        "scope": "SYSTEM"
                    },
                    "properties": {
                        "useConnection": "false",
                        "project": "auto-detect",
                        "serviceAccountType": "filePath",
                        "serviceFilePath": "auto-detect",
                        "suffix": "yyyy-MM-dd-HH-mm",
                        "writeHeader": "false",
                        "location": "us",
                        "contentType": "application/octet-stream"
                    }
                },
                "id": "GCS"
            }
        ],
        "schedule": "0 1 */1 * *",
        "engine": "spark",
        "numOfRecordsPreview": 100,
        "rangeRecordsPreview": {
            "min": 1,
            "max": "5000"
        },
        "maxConcurrentRuns": 1
    }
}