Merge pull request #1844 from rudderlabs/release/v1.100.0

chore(release): pull release/v1.100.0 into main
rudderlabs · Jan 8, 2025 · 2995574 · 2995574
2 parents f56ee8c + 9aabeb8
commit 2995574
Show file tree

Hide file tree

Showing 33 changed files with 399 additions and 197 deletions.
diff --git a/.github/workflows/raise-pr-for-constants.yml b/.github/workflows/raise-pr-for-constants.yml
@@ -33,10 +33,10 @@ jobs:
         id: compare_files
         run: |
           if cmp -s "${{ env.SRC_FILE }}" "dest-repo/${{ env.DEST_PATH }}"; then
-            echo "No changes detected." > "result.txt"
+            echo "No changes detected."
             echo "pr_required=false" >> $GITHUB_OUTPUT
           else
-            echo "Changes detected." > "result.txt"
+            echo "Changes detected."
             echo "pr_required=true" >> $GITHUB_OUTPUT
           fi
 
@@ -58,20 +58,21 @@ jobs:
           GH_TOKEN: ${{ secrets.PAT }}
         run: |
           cd dest-repo
-          EXISTING_PR=$(gh pr list --head ${{ env.BRANCH_NAME }} --json number --jq ".[0].number")
+          EXISTING_PR=$(gh pr list --head "$BRANCH_NAME" --json number --jq ".[0].number")
           if [ -z "$EXISTING_PR" ]; then
-            gh pr create \
-              --title "fix: update destination constants" \
-              --body "$(cat <<EOF
-                This PR updates the destination constants file.
+          # Create the PR body content in a variable
+          PR_BODY="This PR updates the destination constants file.
 
-                **Changes:**
-                - Updated \`Destinations.ts\` with latest constants
+          **Changes:**
+          - Updated \`Destinations.ts\` with latest constants
 
-                NOTE: This PR was automatically generated by GitHub Actions.
-                EOF
-                )" \
-              --label "automated,dependencies"
+          **NOTE:** This PR was automatically generated by GitHub Actions."
+
+          # Create a new PR
+          gh pr create \
+            --title "fix: update destination constants" \
+            --body "$PR_BODY" \
+            --label "github_actions,dependencies"
           else
             echo "PR already exists: $EXISTING_PR"
           fi
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,11 +7,18 @@ All notable changes to this project will be documented in this file. See [standa
 
 ### Features
 
+* add tags for sdk source types ([#1841](https://github.com/rudderlabs/rudder-config-schema/issues/1841)) ([1f4e1c5](https://github.com/rudderlabs/rudder-config-schema/commit/1f4e1c5edb6fb95be21e62d5671427a91a371a50))
+
+
+### Bug Fixes
+
+* pr body in constants pr workflow ([83d7b26](https://github.com/rudderlabs/rudder-config-schema/commit/83d7b2674f2b4bc209331d1dcd2b3c3bacafb4ab))
+* pr body in constants pr workflow ([3d227f7](https://github.com/rudderlabs/rudder-config-schema/commit/3d227f7126c96df9bf64fbd9587ac5d8f19fe17c))
+* pr labels in constants pr workflow ([e8c5167](https://github.com/rudderlabs/rudder-config-schema/commit/e8c5167cad5f9792e2621ddc7695fe85b7aa8fb4))
 * onboard topsort destination ([#1842](https://github.com/rudderlabs/rudder-config-schema/issues/1842)) ([#1846](https://github.com/rudderlabs/rudder-config-schema/issues/1846)) ([0ce45b6](https://github.com/rudderlabs/rudder-config-schema/commit/0ce45b6bc44a29226353b2ad8d929f7372eac235))
 
 ### [1.99.1](https://github.com/rudderlabs/rudder-config-schema/compare/v1.99.0...v1.99.1) (2024-12-12)
 
-
 ### Bug Fixes
 
 * **http:** update the default for propertiesMapping ([#1838](https://github.com/rudderlabs/rudder-config-schema/issues/1838)) ([5fc592f](https://github.com/rudderlabs/rudder-config-schema/commit/5fc592f4d52697737f6716a93e5b6372d3211e98))

diff --git a/scripts/deployToDB.py b/scripts/deployToDB.py
@@ -8,41 +8,67 @@
 from constants import CONFIG_DIR
 
 
+ALL_SELECTORS = ["destination", "source"]
+
+
 def get_command_line_arguments():
     parser = argparse.ArgumentParser(description="Script to deploy config files to DB.")
     parser.add_argument("control_plane_url", nargs="?", help="Control plane URL")
     parser.add_argument("username", nargs="?", help="Control plane admin username")
     parser.add_argument("password", nargs="?", help="Control plane admin password")
+    parser.add_argument(
+        "selector",
+        nargs="?",
+        help="Specify destination or source",
+        default=None,
+    )
+    parser.add_argument(
+        "item_name", nargs="?", help="Specific item name to update.", default=None
+    )
 
     args = parser.parse_args()
 
     control_plane_url = args.control_plane_url or os.getenv("CONTROL_PLANE_URL")
     username = args.username or os.getenv("API_USER")
     password = args.password or os.getenv("API_PASSWORD")
+    selector = args.selector or os.getenv("SELECTOR")
+    item_name = args.item_name or os.getenv("ITEM_NAME")
 
-    missing_args = []
+    invalid_args = []
 
     if control_plane_url is None:
-        missing_args.append(
-            "1st positional argument or CONTROL_PLANE_URL environment variable"
+        invalid_args.append(
+            "1st positional argument or CONTROL_PLANE_URL environment variable is missing"
         )
     if username is None:
-        missing_args.append("2nd positional argument or API_USER environment variable")
+        invalid_args.append(
+            "2nd positional argument or API_USER environment variable is missing"
+        )
     if password is None:
-        missing_args.append(
-            "3rd positional argument or API_PASSWORD environment variable"
+        invalid_args.append(
+            "3rd positional argument or API_PASSWORD environment variable is missing"
         )
+    if selector is None:
+        SELECTORS = ALL_SELECTORS
+    elif selector not in ALL_SELECTORS:
+        invalid_args.append(
+            "4th positional argument or SELECTOR environment variable is invalid"
+        )
+    else:
+        SELECTORS = [selector]
 
-    if missing_args:
-        print("Error: Missing the following arguments or environment variables:")
-        for arg in missing_args:
+    if invalid_args:
+        print("Error: The following arguments or environment variables are invalid:")
+        for arg in invalid_args:
             print(arg)
         sys.exit(1)
 
-    return control_plane_url, username, password
+    return control_plane_url, username, password, SELECTORS, item_name
 
 
-CONTROL_PLANE_URL, USERNAME, PASSWORD = get_command_line_arguments()
+CONTROL_PLANE_URL, USERNAME, PASSWORD, SELECTORS, ITEM_NAME = (
+    get_command_line_arguments()
+)
 
 # CONSTANTS
 HEADER = {"Content-Type": "application/json"}
@@ -139,11 +165,16 @@ def update_config(data_diff, selector):
     return json.dumps(results, indent=2)
 
 
-def update_diff_db(selector):
+def update_diff_db(selector, item_name=None):
     final_report = []
 
     ## data sets
-    current_items = os.listdir(f"./{CONFIG_DIR}/{selector}s")
+    if item_name:
+        current_items = [item_name]
+    else:
+        current_items = os.listdir(f"./{CONFIG_DIR}/{selector}s")
+
+    print(f"Current items: {current_items}")
 
     for item in current_items:
         # check if item is a directory
@@ -203,47 +234,18 @@ def get_stale_data(selector, report):
 
 
 if __name__ == "__main__":
-    print("\n")
-    print("#" * 50)
-    print("Running Destination Definitions Updates")
-    dest_final_report = update_diff_db("destination")
-
-    print("\n")
-    print("#" * 50)
-    print("Destination Definition Update Report")
-    print(get_formatted_json(dest_final_report))
-
-    print("\n")
-    print("#" * 50)
-    print("Stale Destinations Report")
-    print(get_formatted_json(get_stale_data("destination", dest_final_report)))
-
-    print("\n")
-    print("#" * 50)
-    print("Running Source Definitions Updates")
-    src_final_report = update_diff_db("source")
-
-    print("\n")
-    print("#" * 50)
-    print("Source Definition Update Report")
-    print(get_formatted_json(src_final_report))
-
-    print("\n")
-    print("#" * 50)
-    print("Stale Sources Report")
-    print(get_formatted_json(get_stale_data("source", src_final_report)))
-
-    print("\n")
-    print("#" * 50)
-    print("Running Wht Lib Project Definitions Updates")
-    wht_final_report = update_diff_db("wht-lib-project")
-
-    print("\n")
-    print("#" * 50)
-    print("Wht Lib Project Definition Update Report")
-    print(get_formatted_json(wht_final_report))
-
-    print("\n")
-    print("#" * 50)
-    print("Stale Wht Lib Projects Report")
-    print(get_formatted_json(get_stale_data("wht-lib-project", wht_final_report)))
+    for selector in SELECTORS:
+        print("\n")
+        print("#" * 50)
+        print("Running {} Definitions Updates".format(selector.capitalize()))
+        final_report = update_diff_db(selector, ITEM_NAME)
+
+        print("\n")
+        print("#" * 50)
+        print("{} Definition Update Report".format(selector.capitalize()))
+        print(get_formatted_json(final_report))
+
+        print("\n")
+        print("#" * 50)
+        print("Stale {}s Report".format(selector.capitalize()))
+        print(get_formatted_json(get_stale_data(selector, final_report)))
diff --git a/scripts/listJsonMapperDestinations.js b/scripts/listJsonMapperDestinations.js
@@ -0,0 +1,48 @@
+/* eslint-disable no-console */
+const fs = require('fs');
+const path = require('path');
+
+const destinationsDir = path.join(__dirname, '../src/configurations/destinations');
+
+function findJsonMapperDestinations() {
+  try {
+    if (!fs.existsSync(destinationsDir)) {
+      throw new Error(`Destinations directory not found: ${destinationsDir}`);
+    }
+    return fs
+      .readdirSync(destinationsDir)
+      .map((destination) => {
+        try {
+          const destinationsFilePath = path.join(destinationsDir, destination, 'db-config.json');
+          if (!fs.existsSync(destinationsFilePath)) {
+            console.warn(`Skipping ${destination}: Missing configuration file`);
+            return null;
+          }
+          const destinationsContent = fs.readFileSync(destinationsFilePath, 'utf8');
+          const destinationDefinition = JSON.parse(destinationsContent);
+          if (!destinationDefinition.name) {
+            console.warn(`Skipping ${destination}: Missing name`);
+            return null;
+          }
+          return {
+            name: destinationDefinition.name,
+            config: destinationDefinition.config,
+          };
+        } catch (err) {
+          console.error(`Error processing ${destination}:`, err.message);
+          return null;
+        }
+      })
+      .filter(Boolean)
+      .filter(
+        (destination) =>
+          !destination.config?.disableJsonMapper && !destination.config?.supportsVisualMapper,
+      )
+      .map((destination) => destination.name);
+  } catch (err) {
+    console.error('Failed to process destinations:', err.message);
+    return [];
+  }
+}
+
+console.log(findJsonMapperDestinations().join('\n'));
diff --git a/src/configurations/destinations/bingads_offline_conversions/db-config.json b/src/configurations/destinations/bingads_offline_conversions/db-config.json
@@ -20,13 +20,13 @@
       "warehouse": ["cloud"]
     },
     "destConfig": {
-      "defaultConfig": ["rudderAccountId", "customerAccountId", "customerId"],
       "warehouse": [
         "connectionMode",
         "oneTrustCookieCategories",
         "ketchConsentPurposes",
         "consentManagement"
-      ]
+      ],
+      "defaultConfig": ["rudderAccountId", "customerAccountId", "customerId", "isHashRequired"]
     }
   },
   "options": {

diff --git a/src/configurations/destinations/bingads_offline_conversions/schema.json b/src/configurations/destinations/bingads_offline_conversions/schema.json
@@ -12,6 +12,10 @@
         "type": "string",
         "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^[0-9]+$"
       },
+      "isHashRequired": {
+        "type": "boolean",
+        "default": false
+      },
       "oneTrustCookieCategories": {
         "type": "object",
         "properties": {

diff --git a/src/configurations/destinations/bingads_offline_conversions/ui-config.json b/src/configurations/destinations/bingads_offline_conversions/ui-config.json
@@ -58,6 +58,38 @@
         "title": "Configuration settings",
         "note": "Manage the settings for your destination",
         "sections": [
+          {
+            "id": "hash",
+            "title": "Destination settings",
+            "note": "Configure advanced destination-specific settings here",
+            "icon": "settings",
+            "groups": [
+              {
+                "title": "Bing ads offline conversion additional settings",
+                "fields": [
+                  {
+                    "type": "checkbox",
+                    "label": "Enable it, if you are not sending hashed data.",
+                    "note": [
+                      "After enabling this we will send",
+                      {
+                        "text": "hashed email",
+                        "link": "https://learn.microsoft.com/en-us/advertising/bulk-service/offline-conversion?view=bingads-13#hashedemailaddress"
+                      },
+                      "and",
+                      {
+                        "text": "hashed phone numbers",
+                        "link": "https://learn.microsoft.com/en-us/advertising/bulk-service/offline-conversion?view=bingads-13#hashedphonenumber"
+                      },
+                      "to Bingads Offline Conversion."
+                    ],
+                    "configKey": "isHashRequired",
+                    "default": false
+                  }
+                ]
+              }
+            ]
+          },
           {
             "id": "consentSettings",
             "title": "Consent settings",

diff --git a/src/configurations/destinations/http/db-config.json b/src/configurations/destinations/http/db-config.json
@@ -45,6 +45,7 @@
         "propertiesMapping",
         "queryParams",
         "headers",
+        "pathParams",
         "isBatchingEnabled",
         "maxBatchSize",
         "blacklistedEvents",

diff --git a/src/configurations/destinations/http/schema.json b/src/configurations/destinations/http/schema.json
@@ -38,7 +38,19 @@
             },
             "from": {
               "type": "string",
-              "pattern": "^\\$(?:\\.|(\\.(\\w+|\\*)|\\[\\d+\\]|\\[('\\w+'|\"\\w+\")\\]|\\[\\*\\]|\\.\\w+\\(\\))*)$|^(?!\\\\$)[A-Za-z0-9!#$%&'*+.^_`|~-]{1,100}$"
+              "pattern": "^\\$(?:\\.|(\\.(\\w+|\\*)|\\[\\d+\\]|\\[('\\w+'|\"\\w+\")\\]|\\[\\*\\]|\\.\\w+\\(\\))*)$|^(?!\\$)[A-Za-z0-9!#$%&'*+.^_`|~-]{1,100}$"
+            }
+          }
+        }
+      },
+      "pathParams": {
+        "type": "array",
+        "items": {
+          "type": "object",
+          "properties": {
+            "path": {
+              "type": "string",
+              "pattern": "^\\$(?:\\.|(\\.(\\w+|\\*)|\\[\\d+\\]|\\[('\\w+'|\"\\w+\")\\]|\\[\\*\\]|\\.\\w+\\(\\))*)$|^(?!\\$)[A-Za-z0-9!#$%&'*+.^_`|~-]{1,100}$"
             }
           }
         }
@@ -50,11 +62,11 @@
           "properties": {
             "to": {
               "type": "string",
-              "pattern": "^(?!\\\\$)[A-Za-z0-9!#$%&'*+.^_`|~-]{1,100}$"
+              "pattern": "^(?!\\$)[A-Za-z0-9!#$%&'*+.^_`|~-]{1,100}$"
             },
             "from": {
               "type": "string",
-              "pattern": "^\\$(?:\\.|(\\.(\\w+|\\*)|\\[\\d+\\]|\\[('\\w+'|\"\\w+\")\\]|\\[\\*\\]|\\.\\w+\\(\\))*)$|^(?!\\\\$)[A-Za-z0-9!#$%&'*+.^_`|~-]{1,100}$"
+              "pattern": "^\\$(?:\\.|(\\.(\\w+|\\*)|\\[\\d+\\]|\\[('\\w+'|\"\\w+\")\\]|\\[\\*\\]|\\.\\w+\\(\\))*)$|^(?!\\$)[A-Za-z0-9!#$%&'*+.^_`|~-]{1,100}$"
             }
           }
         }
@@ -66,11 +78,11 @@
           "properties": {
             "to": {
               "type": "string",
-              "pattern": "^(?!\\\\$)[A-Za-z0-9!#$%&'*+.^_`|~-]{1,100}$"
+              "pattern": "^(?!\\$)[A-Za-z0-9!#$%&'*+.^_`|~-]{1,100}$"
             },
             "from": {
               "type": "string",
-              "pattern": "^\\$(?:\\.|(\\.(\\w+|\\*)|\\[\\d+\\]|\\[('\\w+'|\"\\w+\")\\]|\\[\\*\\]|\\.\\w+\\(\\))*)$|^(?!\\\\$)[A-Za-z0-9!#$%&'*+.^_`|~-]{1,100}$"
+              "pattern": "^\\$(?:\\.|(\\.(\\w+|\\*)|\\[\\d+\\]|\\[('\\w+'|\"\\w+\")\\]|\\[\\*\\]|\\.\\w+\\(\\))*)$|^(?!\\$)[A-Za-z0-9!#$%&'*+.^_`|~-]{1,100}$"
             }
           }
         }