DataFusionQuickstart-cdap-data-pipeline.json

{
    "name": "DataFusionQuickstart",
    "description": "Data Pipeline Application",
    "artifact": {
        "name": "cdap-data-pipeline",
        "version": "6.12.0-SNAPSHOT",
        "scope": "SYSTEM"
    },
    "config": {
        "resources": {
            "memoryMB": 2048,
            "virtualCores": 1
        },
        "driverResources": {
            "memoryMB": 2048,
            "virtualCores": 1
        },
        "connections": [
            {
                "from": "NYT Best Sellers Raw Data",
                "to": "Parsing and Transformations"
            },
            {
                "from": "Parsing and Transformations",
                "to": "Top Rated Inexpensive Books"
            }
        ],
        "comments": [],
        "postActions": [],
        "properties": {},
        "processTimingEnabled": true,
        "stageLoggingEnabled": true,
        "stages": [
            {
                "name": "NYT Best Sellers Raw Data",
                "plugin": {
                    "name": "GCSFile",
                    "type": "batchsource",
                    "label": "NYT Best Sellers Raw Data",
                    "artifact": {
                        "name": "google-cloud",
                        "version": "0.25.0-SNAPSHOT",
                        "scope": "SYSTEM"
                    },
                    "properties": {
                        "filenameOnly": "false",
                        "copyHeader": "false",
                        "schema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"body\",\"type\":\"string\"}]}",
                        "path": "gs://nyt-bestsellers/nyt2.json",
                        "format": "text",
                        "recursive": "false",
                        "referenceName": "NYT_BestSellers_Raw",
                        "serviceFilePath": "auto-detect",
                        "project": "auto-detect"
                    }
                },
                "outputSchema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"body\",\"type\":\"string\"}]}",
                "id": "NYT-Best-Sellers-Raw-Data"
            },
            {
                "name": "Parsing and Transformations",
                "plugin": {
                    "name": "Wrangler",
                    "type": "transform",
                    "label": "Parsing and Transformations",
                    "artifact": {
                        "name": "wrangler-transform",
                        "version": "4.12.0-SNAPSHOT",
                        "scope": "SYSTEM"
                    },
                    "properties": {
                        "workspaceId": "b22b1b0c24b12acd45f834ac5e1dfffb",
                        "directives": "split-to-rows body \\n\nparse-as-json :body 1\nparse-as-json :body__id 1\nparse-as-json :body_weeks_on_list 1\nparse-as-json :body_rank_last_week 1\nparse-as-json :body_rank 1\nparse-as-json :body_price 1\nparse-as-json :body_published_date 2\nparse-as-json :body_bestsellers_date 2\ncolumns-replace s/^body_//g\ncleanse-column-names\nrename _id__oid id\nrename weeks_on_list__numberint weeks_on_list\nrename rank_last_week__numberint rank_last_week\nrename rank__numberint rank\nfill-null-or-empty :price__numberint '0'\nrename price__numberint price\nrename published_date__date__numberlong published_date\nrename bestsellers_date__date__numberlong bestsellers_date\nfill-null-or-empty :price__numberdouble '0'\nrename price__numberdouble price_other\nfill-null-or-empty :price '0'\nfill-null-or-empty :price_other '0'\nset-type :price float\nset-type :price_other float\nset-column :price_final price+price_other\ndrop price\ndrop price_other\nset-type :published_date long\nset-type :bestsellers_date long\nset-type :weeks_on_list integer\nset-type :rank_last_week integer\nset-type :rank long\nfilter-rows-on regex-not-match rank_last_week ^0$\ndrop rank_last_week\ndrop weeks_on_list\ndrop published_date\ndrop bestsellers_date\nfilter-rows-on condition-false rank <=3\nfilter-rows-on regex-match price_final ^0.0$\nfilter-rows-on condition-false price_final <25.0",
                        "schema": "{\"name\":\"etlSchemaBody\",\"type\":\"record\",\"fields\":[{\"name\":\"amazon_product_url\",\"type\":[\"string\",\"null\"]},{\"name\":\"author\",\"type\":[\"string\",\"null\"]},{\"name\":\"description\",\"type\":[\"string\",\"null\"]},{\"name\":\"publisher\",\"type\":[\"string\",\"null\"]},{\"name\":\"title\",\"type\":[\"string\",\"null\"]},{\"name\":\"id\",\"type\":[\"string\",\"null\"]},{\"name\":\"rank\",\"type\":[\"long\",\"null\"]},{\"name\":\"price_final\",\"type\":[\"double\",\"null\"]}]}",
                        "field": "body",
                        "precondition": "false"
                    }
                },
                "outputSchema": "{\"name\":\"etlSchemaBody\",\"type\":\"record\",\"fields\":[{\"name\":\"amazon_product_url\",\"type\":[\"string\",\"null\"]},{\"name\":\"author\",\"type\":[\"string\",\"null\"]},{\"name\":\"description\",\"type\":[\"string\",\"null\"]},{\"name\":\"publisher\",\"type\":[\"string\",\"null\"]},{\"name\":\"title\",\"type\":[\"string\",\"null\"]},{\"name\":\"id\",\"type\":[\"string\",\"null\"]},{\"name\":\"rank\",\"type\":[\"long\",\"null\"]},{\"name\":\"price_final\",\"type\":[\"double\",\"null\"]}]}",
                "inputSchema": [
                    {
                        "name": "NYT Best Sellers Raw Data",
                        "schema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"body\",\"type\":\"string\"}]}"
                    }
                ],
                "id": "Parsing-and-Transformations"
            },
            {
                "name": "Top Rated Inexpensive Books",
                "plugin": {
                    "name": "BigQueryTable",
                    "type": "batchsink",
                    "label": "Top Rated Inexpensive Books",
                    "artifact": {
                        "name": "google-cloud",
                        "version": "0.25.0-SNAPSHOT",
                        "scope": "SYSTEM"
                    },
                    "properties": {
                        "project": "auto-detect",
                        "serviceFilePath": "auto-detect",
                        "schema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"amazon_product_url\",\"type\":[\"string\",\"null\"]},{\"name\":\"author\",\"type\":[\"string\",\"null\"]},{\"name\":\"description\",\"type\":[\"string\",\"null\"]},{\"name\":\"publisher\",\"type\":[\"string\",\"null\"]},{\"name\":\"title\",\"type\":[\"string\",\"null\"]},{\"name\":\"id\",\"type\":[\"string\",\"null\"]},{\"name\":\"rank\",\"type\":[\"long\",\"null\"]},{\"name\":\"price_final\",\"type\":[\"double\",\"null\"]}]}",
                        "referenceName": "Top_Rated_Inexpensive_Books",
                        "dataset": "GCPQuickStart",
                        "allowSchemaRelaxation": "false",
                        "table": "top_rated_inexpensive"
                    }
                },
                "outputSchema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"amazon_product_url\",\"type\":[\"string\",\"null\"]},{\"name\":\"author\",\"type\":[\"string\",\"null\"]},{\"name\":\"description\",\"type\":[\"string\",\"null\"]},{\"name\":\"publisher\",\"type\":[\"string\",\"null\"]},{\"name\":\"title\",\"type\":[\"string\",\"null\"]},{\"name\":\"id\",\"type\":[\"string\",\"null\"]},{\"name\":\"rank\",\"type\":[\"long\",\"null\"]},{\"name\":\"price_final\",\"type\":[\"double\",\"null\"]}]}",
                "inputSchema": [
                    {
                        "name": "Parsing and Transformations",
                        "schema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"amazon_product_url\",\"type\":[\"string\",\"null\"]},{\"name\":\"author\",\"type\":[\"string\",\"null\"]},{\"name\":\"description\",\"type\":[\"string\",\"null\"]},{\"name\":\"publisher\",\"type\":[\"string\",\"null\"]},{\"name\":\"title\",\"type\":[\"string\",\"null\"]},{\"name\":\"id\",\"type\":[\"string\",\"null\"]},{\"name\":\"rank\",\"type\":[\"long\",\"null\"]},{\"name\":\"price_final\",\"type\":[\"double\",\"null\"]}]}"
                    }
                ],
                "id": "Top-Rated-Inexpensive-Books"
            }
        ],
        "schedule": "0 * * * *",
        "engine": "spark",
        "numOfRecordsPreview": 100,
        "rangeRecordsPreview": {
            "min": 1,
            "max": "5000"
        },
        "maxConcurrentRuns": 1
    },
    "version": "2c14bd63-ed09-11ef-aa0b-ee3795e5a1f1"
}