Merge pull request #495 from n0elleli/template

Template

templates/Copy data from ADLS Gen2 to a file/Copy data from ADLS Gen2 to a file.json

@@ -0,0 +1 @@
+{"$schema":"http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#","contentVersion":"1.0.0.0","parameters":{"testADLS2":{"type":"string"},"testLakehouse":{"type":"string"}},"variables":{},"resources":[{"name":"Copy data from ADLS Gen2 to Lakehouse file","type":"pipelines","apiVersion":"2018-06-01","properties":{"activities":[{"name":"Copy data from ADLS Gen2 to Lakehouse File","type":"Copy","dependsOn":[],"policy":{"timeout":"0.12:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":false,"secureInput":false},"typeProperties":{"source":{"type":"DelimitedTextSource","storeSettings":{"type":"AzureBlobFSReadSettings","recursive":true,"enablePartitionDiscovery":false},"formatSettings":{"type":"DelimitedTextReadSettings"},"datasetSettings":{"annotations":[],"type":"DelimitedText","typeProperties":{"location":{"type":"AzureBlobFSLocation","fileName":{"value":"@pipeline().parameters.FileName","type":"Expression"},"folderPath":{"value":"@pipeline().parameters.DirectoryName","type":"Expression"},"fileSystem":{"value":"@pipeline().parameters.FileSystemName","type":"Expression"}},"columnDelimiter":",","escapeChar":"\\","quoteChar":"\""},"schema":[],"externalReferences":{"connection":"[parameters('testADLS2')]"}}},"sink":{"type":"DelimitedTextSink","storeSettings":{"type":"LakehouseWriteSettings"},"formatSettings":{"type":"DelimitedTextWriteSettings","quoteAllText":true,"fileExtension":".txt"},"datasetSettings":{"annotations":[],"linkedService":{"name":"testLakehouse","properties":{"type":"Lakehouse","typeProperties":{"workspaceId":"33eca854-3c9b-498f-8cf4-9144c6661828","artifactId":"[parameters('testLakehouse')]","rootFolder":"Files"}}},"type":"DelimitedText","typeProperties":{"location":{"type":"LakehouseLocation","fileName":{"value":"@pipeline().parameters.LakehouseFileName","type":"Expression"},"folderPath":{"value":"@pipeline().parameters.LakehouseDirectoryName","type":"Expression"}},"columnDelimiter":",","escapeChar":"\\","quoteChar":"\""},"schema":[]}},"enableStaging":false,"translator":{"type":"TabularTranslator","typeConversion":true,"typeConversionSettings":{"allowDataTruncation":true,"treatBooleanAsNumber":false}}}}],"parameters":{"FileSystemName":{"type":"String"},"DirectoryName":{"type":"String"},"FileName":{"type":"String"},"LakehouseDirectoryName":{"type":"string"},"LakehouseFileName":{"type":"string"}},"annotations":[]},"dependsOn":[]}]}

templates/Copy data from ADLS Gen2 to a file/manifest.json

@@ -0,0 +1,27 @@
+{
+    "name": "Copy data from ADLS Gen2 to Lakehouse file",
+    "image": "<svg width=\"400px\" height=\"240px\" viewBox=\"0 0 400 240\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" style=\"width: 400px; height: 240px;\"><g transform=\"matrix(1, 0, 0, 1, 0, 0)\" clip-path=\"url(#mainClip49027)\"><g transform=\"matrix(0.8928571428571429, 0, 0, 0.8928571428571429, -296.42857142857144, -29.821428571428584)\"><g transform=\"matrix(1, 0, 0, 1, 453, 128)\"><g transform=\"matrix(1, 0, 0, 1, 1, 0)\"><g transform=\"matrix(1, 0, 0, 1, 0, 0)\"><path d=\"M 0,0 L 191,0 L 191,78.6 L 0,78.6 z\" fill=\"transparent\" stroke=\"none\" transform=\"matrix(1, 0, 0, 1, 0.5, 0.5)\"/><path d=\"M 0,0 L 191,0 L 191,78.6 L 0,78.6 z\" fill=\"none\" stroke=\"#e0e0e0\" stroke-width=\"1\" stroke-linecap=\"butt\" stroke-linejoin=\"miter\" stroke-miterlimit=\"10\" transform=\"matrix(1, 0, 0, 1, 0.5, 0.5)\"/><g transform=\"matrix(1, 0, 0, 1, 0, 0)\" clip-path=\"url(#CLIP49029)\"><g transform=\"matrix(1, 0, 0, 1, -0.7999999999999972, 0.5)\"><g transform=\"matrix(1, 0, 0, 1, 0.8, 0.8)\"><g transform=\"matrix(1, 0, 0, 1, 0, 0)\"><rect x=\"0\" y=\"0\" width=\"192\" height=\"77\" fill=\"#ffffff\" stroke=\"none\" transform=\"matrix(1, 0, 0, 1, 0, 0)\"/><g transform=\"matrix(1, 0, 0, 1, 0, 0)\" clip-path=\"url(#CLIP49031)\"><g transform=\"matrix(1, 0, 0, 1, 0, -1.5)\"><g transform=\"matrix(1, 0, 0, 1, 0, 0)\"><rect x=\"0\" y=\"0\" width=\"192\" height=\"25\" fill=\"#EBEBEB\" stroke=\"none\" transform=\"matrix(1, 0, 0, 1, 0, 0)\"/><g transform=\"matrix(1, 0, 0, 1, 0, 0)\" clip-path=\"url(#CLIP49033)\"><text x=\"0\" y=\"10.403173828125\" style=\"font: 12px Segoe UI\" text-anchor=\"start\" fill=\"black\" stroke=\"none\" transform=\"matrix(1, 0, 0, 1, 5, 6.5)\">Copy data</text></g><g transform=\"matrix(1, 0, 0, 1, 0, 0)\" clip-path=\"url(#CLIP49035)\"><g transform=\"matrix(1, 0, 0, 1, 161, -3)\"><path d=\"M 1.8409491661026458,0 L 3.6818983322052947,0 C 4.506119348655733,0 5.52284749830794,1.0167281496522078 5.52284749830794,1.8409491661026458 L 5.52284749830794,3.681898332205292 C 5.52284749830794,4.50611934865573 4.506119348655733,5.522847498307938 3.6818983322052947,5.522847498307938 L 1.8409491661026458,5.522847498307938 C 1.0167281496522078,5.522847498307938 0,4.50611934865573 0,3.681898332205292 L 0,1.8409491661026458 C 0,1.0167281496522078 1.0167281496522078,0 1.8409491661026458,0 z\" fill=\"#f5f5f5\" stroke=\"none\" transform=\"matrix(1, 0, 0, 1, 0.5, 0.5)\"/><path d=\"M 1.8409491661026458,0 L 3.6818983322052947,0 C 4.506119348655733,0 5.52284749830794,1.0167281496522078 5.52284749830794,1.8409491661026458 L 5.52284749830794,3.681898332205292 C 5.52284749830794,4.50611934865573 4.506119348655733,5.522847498307938 3.6818983322052947,5.522847498307938 L 1.8409491661026458,5.522847498307938 C 1.0167281496522078,5.522847498307938 0,4.50611934865573 0,3.681898332205292 L 0,1.8409491661026458 C 0,1.0167281496522078 1.0167281496522078,0 1.8409491661026458,0 z\" fill=\"none\" stroke=\"transparent\" stroke-width=\"1\" stroke-linecap=\"butt\" stroke-linejoin=\"miter\" stroke-miterlimit=\"10\" transform=\"matrix(1, 0, 0, 1, 0.5, 0.5)\"/></g></g></g></g></g><g transform=\"matrix(1, 0, 0, 1, 0, 22)\"><g transform=\"matrix(1, 0, 0, 1, 0, 0)\"><g transform=\"matrix(1, 0, 0, 1, 0, 0)\"><path d=\"M 0,0 L 192,0 L 192,40 L 0,40 z\" fill=\"#ffffff\" stroke=\"none\" transform=\"matrix(1, 0, 0, 1, 0, 0)\"/><g transform=\"matrix(1, 0, 0, 1, 0, 0)\" clip-path=\"url(#CLIP49037)\"><g transform=\"matrix(1, 0, 0, 1, 0, -22)\"><g transform=\"matrix(1, 0, 0, 1, 10, 34)\"><path d=\"M 11.4787875,6.307275 C 7.0275375,6.307275 3.4162875,5.1504 3.4162875,3.622275 L 3.4162875,17.92479375 C 3.4162875,19.3948125 6.9619125,20.587312500000003 11.3625375,20.6098125 L 11.475037500000001,20.6098125 C 15.924412499999999,20.6098125 19.5375,19.4529375 19.5375,17.92479375 L 19.5375,3.622275 C 19.5375,5.114775 15.928162500000001,6.307275 11.4787875,6.307275 z\" fill=\"url(#GRAD49038)\" stroke=\"none\" transform=\"matrix(1, 0, 0, 1, 0, 0)\"/><path d=\"M 19.5376875,3.6225 C 19.5376875,5.1225000000000005 15.92825625,6.3075 11.47513125,6.3075 C 7.0220062500000004,6.3075 3.41263125,5.150625 3.41263125,3.6225 C 3.41263125,2.094375 7.02763125,0.9375 11.47888125,0.9375 C 15.930131249999999,0.9375 19.5414375,2.094375 19.5414375,3.6225\" fill=\"#E8E8E8\" stroke=\"none\" transform=\"matrix(1, 0, 0, 1, 0, 0)\"/><path d=\"M 17.66445,3.4030875000000003 C 17.66445,4.3518375 14.8857,5.1149625 11.47695,5.1149625 C 8.068200000000001,5.1149625 5.28945,4.3518375 5.28945,3.4030875000000003 C 5.28945,2.4543375000000003 8.07196875,1.70059125 11.47884375,1.70059125 C 14.8857,1.70059125 17.666325,2.4655875000000003 17.666325,3.4030875000000003\" fill=\"#50E6FF\" stroke=\"none\" transform=\"matrix(1, 0, 0, 1, 0, 0)\"/><path d=\"M 11.47905,3.8304375 C 9.82389375,3.7934249999999996 8.173368750000002,4.019362500000001 6.58905,4.4998125 C 8.175393750000001,4.96708125 9.8262,5.17785 11.47905,5.1241875 C 13.135349999999999,5.16695625 14.78739375,4.937025 16.369049999999998,4.4435625 C 14.780175,3.98945625 13.13085,3.78264375 11.47905,3.8304375 z\" fill=\"#198AB3\" stroke=\"none\" transform=\"matrix(1, 0, 0, 1, 0, 0)\"/><path d=\"M 19.5376875,14.759887500000001 C 15.086475,14.759887500000001 11.475225,13.6011375 11.475225,12.074887499999999 L 11.475225,26.3773125 C 11.475225,27.8473125 15.02085,29.0398125 19.4195625,29.0623125 L 19.5339375,29.0623125 C 23.9833125,29.0623125 27.5964375,27.905437499999998 27.5964375,26.3773125 L 27.5964375,12.074887499999999 C 27.594562500000002,13.5673875 23.9870625,14.759887500000001 19.5376875,14.759887500000001 z\" fill=\"url(#GRAD49039)\" stroke=\"none\" transform=\"matrix(1, 0, 0, 1, 0, 0)\"/><path d=\"M 27.59475,12.0751125 C 27.59475,13.5751125 23.98725,14.7601125 19.532249999999998,14.7601125 C 15.07723125,14.7601125 11.46973125,13.6013625 11.46973125,12.0751125 C 11.46973125,10.5488625 15.07723125,9.3901125 19.532249999999998,9.3901125 C 23.98725,9.3901125 27.59475,10.5469875 27.59475,12.0751125 z\" fill=\"#E8E8E8\" stroke=\"none\" transform=\"matrix(1, 0, 0, 1, 0, 0)\"/><path d=\"M 25.721625,11.853825 C 25.721625,12.802575 22.942875,13.567575 19.534125,13.567575 C 16.125300000000003,13.567575 13.346549999999999,12.802575 13.346549999999999,11.853825 C 13.346549999999999,10.905075 16.125300000000003,10.1532 19.534125,10.1532 C 22.942875,10.1532 25.721625,10.916325 25.721625,11.853825 z\" fill=\"#50E6FF\" stroke=\"none\" transform=\"matrix(1, 0, 0, 1, 0, 0)\"/><path d=\"M 19.5380625,12.28335 C 17.88305625,12.24159375 16.23195,12.462468750000001 14.64615,12.937725 C 16.23331875,13.404093750000001 17.884593749999997,13.614862500000001 19.5380625,13.562100000000001 C 21.1936875,13.60685625 22.8455625,13.3801125 24.4280625,12.89085 C 22.839,12.4377375 21.1895625,12.23281875 19.5380625,12.28335 z\" fill=\"#198AB3\" stroke=\"none\" transform=\"matrix(1, 0, 0, 1, 0, 0)\"/></g><g transform=\"matrix(1, 0, 0, 1, 50, 36.285009765625)\"><g transform=\"matrix(1, 0, 0, 1, 0, 0)\"><text x=\"0\" y=\"9.53624267578125\" style=\"font: 11px Segoe UI\" text-anchor=\"start\" fill=\"black\" stroke=\"none\" transform=\"matrix(1, 0, 0, 1, 0, 0)\">Copy data from ADLS</text><text x=\"0\" y=\"22.251232910156247\" style=\"font: 11px Segoe UI\" text-anchor=\"start\" fill=\"black\" stroke=\"none\" transform=\"matrix(1, 0, 0, 1, 0, 0)\">Gen2 to Lakehouse File</text></g></g></g></g></g></g></g></g></g></g></g></g></g><g transform=\"matrix(1, 0, 0, 1, 0, 0)\" filter=\"url(#SHADOW49040)\"><filter id=\"SHADOW49040\" x=\"-100%\" y=\"-100%\" width=\"300%\" height=\"300%\"><feGaussianBlur in=\"SourceAlpha\" result=\"blur\" LA=\"1.5\"/><feFlood in=\"blur\" result=\"flood\" flood-color=\"#0078dA\"/><feComposite in=\"flood\" in2=\"blur\" operator=\"in\" result=\"comp\"/><feOffset in=\"comp\" result=\"offsetBlur\" dx=\"0\" dy=\"0\"/><feMerge><feMergeNode in=\"offsetBlur\"/><feMergeNode in=\"SourceGraphic\"/></feMerge></filter><g transform=\"matrix(1, 0, 0, 1, 194, 3.799999999999997)\"><g transform=\"matrix(1, 0, 0, 1, 0, 0)\"><g transform=\"matrix(1, 0, 0, 1, 0, 6)\"/></g><g transform=\"matrix(1, 0, 0, 1, 0, 18)\"><g transform=\"matrix(1, 0, 0, 1, 0, 6)\"><g transform=\"matrix(1, 0, 0, 1, 0, 0)\" filter=\"url(#SHADOW49041)\"><filter id=\"SHADOW49041\" x=\"-100%\" y=\"-100%\" width=\"300%\" height=\"300%\"><feGaussianBlur in=\"SourceAlpha\" result=\"blur\" LA=\"1.5\"/><feFlood in=\"blur\" result=\"flood\" flood-color=\"#0078dA\"/><feComposite in=\"flood\" in2=\"blur\" operator=\"in\" result=\"comp\"/><feOffset in=\"comp\" result=\"offsetBlur\" dx=\"0\" dy=\"0\"/><feMerge><feMergeNode in=\"offsetBlur\"/><feMergeNode in=\"SourceGraphic\"/></feMerge></filter><path d=\"M 0,0 L 12,0 L 12,12 L 0,12 z\" fill=\"#ffffff\" stroke=\"none\" transform=\"matrix(1, 0, 0, 1, 0, 0)\"/></g><path d=\"M 0.1080000000000001,4.0649999999999995 C 0.03399999999999981,3.9859999999999998 -0.0040000000000000036,3.8819999999999997 0,3.774 C 0.0040000000000000036,3.665 0.04999999999999982,3.5650000000000004 0.1299999999999999,3.49 L 0.9649999999999999,2.718 C 1.041,2.649 1.1379999999999995,2.6109999999999998 1.2389999999999994,2.6109999999999998 C 1.3509999999999995,2.6109999999999998 1.459,2.6580000000000004 1.5349999999999997,2.74 L 3.671,5.032 L 7.478010000000001,0.15700000000000003 C 7.555010000000001,0.05799999999999983 7.671010000000001,0 7.79701,0 C 7.88701,0 7.972010000000001,0.028999999999999915 8.04401,0.08399999999999963 L 8.95001,0.7830000000000004 C 9.12301,0.9109999999999996 9.16001,1.1600000000000001 9.03001,1.3369999999999997 L 4.161999999999999,7.57 C 3.9730000000000003,7.811999999999999 3.6119999999999997,7.827999999999999 3.4019999999999997,7.603 L 0.1080000000000001,4.0649999999999995 z\" fill=\"#107C10\" stroke=\"none\" transform=\"matrix(1, 0, 0, 1, 1.4849949999999996, 2.1985)\"/></g></g><g transform=\"matrix(1, 0, 0, 1, 0, 36)\"><g transform=\"matrix(1, 0, 0, 1, 0, 6)\"/></g><g transform=\"matrix(1, 0, 0, 1, 0, 54)\"><g transform=\"matrix(1, 0, 0, 1, 0, 6)\"/></g></g></g></g></g></g><clipPath id=\"mainClip49027\"><rect x=\"0\" y=\"0\" width=\"400px\" height=\"240px\"/></clipPath><clipPath id=\"CLIP49028\"><path d=\"M 0,0 L 192,0 L 192,79.6 L 0,79.6 z\" transform=\"matrix(1, 0, 0, 1, 0.5, 0.5)\"/></clipPath><clipPath id=\"CLIP49029\"><path d=\"M 0,0 L 192,0 L 192,79.6 L 0,79.6 z\" transform=\"matrix(1, 0, 0, 1, 0, 0)\"/></clipPath><clipPath id=\"CLIP49030\"><path d=\"M 0,0 L 192,0 L 192,23.5 L 0,23.5 z\" transform=\"matrix(1, 0, 0, 1, 0, 0)\"/></clipPath><clipPath id=\"CLIP49031\"><path d=\"M 0,0 L 192,0 L 192,23.5 L 0,23.5 z\" transform=\"matrix(1, 0, 0, 1, 0, 0)\"/></clipPath><clipPath id=\"CLIP49032\"><path d=\"M 5,6.5 L 192,6.5 L 192,25 L 5,25 z\" transform=\"matrix(1, 0, 0, 1, 0, 0)\"/></clipPath><clipPath id=\"CLIP49033\"><path d=\"M 5,6.5 L 192,6.5 L 192,25 L 5,25 z\" transform=\"matrix(1, 0, 0, 1, 0, 0)\"/></clipPath><clipPath id=\"CLIP49034\"><path d=\"M 161,0 L 167.52284749830795,0 L 167.52284749830795,3.5228474983079376 L 161,3.5228474983079376 z\" transform=\"matrix(1, 0, 0, 1, 0, 0)\"/></clipPath><clipPath id=\"CLIP49035\"><path d=\"M 161,0 L 167.52284749830795,0 L 167.52284749830795,3.5228474983079376 L 161,3.5228474983079376 z\" transform=\"matrix(1, 0, 0, 1, 0, 0)\"/></clipPath><clipPath id=\"CLIP49036\"><path d=\"M 0,0 L 192,0 L 192,40 L 0,40 z\" transform=\"matrix(1, 0, 0, 1, 0, 0)\"/></clipPath><clipPath id=\"CLIP49037\"><path d=\"M 0,0 L 192,0 L 192,40 L 0,40 z\" transform=\"matrix(1, 0, 0, 1, 0, 0)\"/></clipPath><linearGradient x1=\"35.63679075\" x2=\"203.58075000000002\" y1=\"133.17834129562502\" y2=\"133.17834129562502\" id=\"GRAD49038\" gradientUnits=\"userSpaceOnUse\"><stop offset=\"0\" stop-color=\"#005BA1\"/><stop offset=\"0.07\" stop-color=\"#0060A9\"/><stop offset=\"0.36\" stop-color=\"#0071C8\"/><stop offset=\"0.52\" stop-color=\"#0078D4\"/><stop offset=\"0.64\" stop-color=\"#0074CD\"/><stop offset=\"0.82\" stop-color=\"#006ABB\"/><stop offset=\"1\" stop-color=\"#005BA1\"/></linearGradient><linearGradient x1=\"1621.4866378312502\" x2=\"3660.8606094375004\" y1=\"115785.70611562501\" y2=\"115785.70611562501\" id=\"GRAD49039\" gradientUnits=\"userSpaceOnUse\"><stop offset=\"0\" stop-color=\"#005BA1\"/><stop offset=\"0.07\" stop-color=\"#0060A9\"/><stop offset=\"0.36\" stop-color=\"#0071C8\"/><stop offset=\"0.52\" stop-color=\"#0078D4\"/><stop offset=\"0.64\" stop-color=\"#0074CD\"/><stop offset=\"0.82\" stop-color=\"#006ABB\"/><stop offset=\"1\" stop-color=\"#005BA1\"/></linearGradient></svg>",
+    "icons": [
+        "Copy"
+    ],
+    "requires": {
+        "linkedservices": {
+            "testADLS2": {
+                "supportTypes": [
+                    "AzureBlobFS"
+                ]
+            },
+            "testLakehouse": {
+                "supportTypes": [
+                    "Lakehouse"
+                ]
+            }
+        }
+    },
+    "annotations": [],
+    "services": [],
+    "categories": [],
+    "compatibleWithPBI": true,
+    "descriptionForPBI": "Use this template to copy data from your Azure SQL database to a specified file location in your Lakehouse.\n\nIf you want to copy data from a number of files, please use the “Copy assistant” to create your pipeline.",
+    "scope":["PBI"] 
+}

templates/Copy data from ADLS Gen2 to table/Copy data from ADLS Gen2 to table.json

@@ -0,0 +1 @@
+{"$schema":"http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#","contentVersion":"1.0.0.0","parameters":{"testADLS2":{"type":"string"},"testLakehouse":{"type":"string"}},"variables":{},"resources":[{"name":"Copy file from ADLS Gen2 to Lakehouse Table","type":"pipelines","apiVersion":"2018-06-01","properties":{"activities":[{"name":"Copy data from ADLS Gen2 to Lakehouse","type":"Copy","dependsOn":[],"policy":{"timeout":"0.12:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":false,"secureInput":false},"typeProperties":{"source":{"type":"DelimitedTextSource","storeSettings":{"type":"AzureBlobFSReadSettings","recursive":true,"enablePartitionDiscovery":false},"formatSettings":{"type":"DelimitedTextReadSettings"},"datasetSettings":{"annotations":[],"type":"DelimitedText","typeProperties":{"location":{"type":"AzureBlobFSLocation","fileName":{"value":"@pipeline().parameters.FileName","type":"Expression"},"folderPath":{"value":"@pipeline().parameters.DirectoryName","type":"Expression"},"fileSystem":{"value":"@pipeline().parameters.FileSystemName","type":"Expression"}},"columnDelimiter":",","escapeChar":"\\","quoteChar":"\""},"schema":[],"externalReferences":{"connection":"[parameters('testADLS2')]"}}},"sink":{"type":"DeltaSink","storeSettings":{"type":"LakehouseWriteSettings"},"formatSettings":{"type":"DeltaWriteSettings","enableVertiParquet":true},"datasetSettings":{"annotations":[],"linkedService":{"name":"testLakehouse","properties":{"type":"Lakehouse","typeProperties":{"workspaceId":"33eca854-3c9b-498f-8cf4-9144c6661828","artifactId":"[parameters('testLakehouse')]","rootFolder":"Tables"}}},"type":"Delta","typeProperties":{"location":{"type":"LakehouseLocation","folderPath":{"value":"@pipeline().parameters.LHTableName","type":"Expression"}}},"schema":[]}},"enableStaging":false,"translator":{"type":"TabularTranslator","typeConversion":true,"typeConversionSettings":{"allowDataTruncation":true,"treatBooleanAsNumber":false}}}}],"parameters":{"FileSystemName":{"type":"String"},"DirectoryName":{"type":"String"},"FileName":{"type":"String"},"LHTableName":{"type":"string"}},"annotations":[]},"dependsOn":[]}]}