Merge branch 'master' into bootstrap-mcps-1

.github/workflows/metadata-io.yml

@@ -8,6 +8,7 @@ on:
       - "li-utils/**"
       - "metadata-models/**"
       - "metadata-io/**"
+      - ".github/workflows/metadata-io.yml"
   pull_request:
     branches:
       - "**"
@@ -16,6 +17,7 @@ on:
       - "li-utils/**"
       - "metadata-models/**"
       - "metadata-io/**"
+      - ".github/workflows/metadata-io.yml"
   release:
     types: [published]
 
@@ -52,6 +54,8 @@ jobs:
           sudo apt-get remove 'dotnet-*' azure-cli || true
           sudo rm -rf /usr/local/lib/android/ || true
           sudo docker image prune -a -f || true
+      - name: Disk Check
+        run: df -h . && docker images
       - uses: acryldata/sane-checkout-action@v3
       - name: Set up JDK 17
         uses: actions/setup-java@v4

build.gradle

@@ -392,7 +392,7 @@ subprojects {
       implementation externalDependency.annotationApi
       constraints {
         implementation("com.google.googlejavaformat:google-java-format:$googleJavaFormatVersion")
-        implementation('io.netty:netty-all:4.1.100.Final')
+        implementation('io.netty:netty-all:4.1.114.Final')
         implementation('org.apache.commons:commons-compress:1.26.0')
         implementation('org.apache.velocity:velocity-engine-core:2.3')
         implementation('org.hibernate:hibernate-validator:6.0.20.Final')

datahub-frontend/play.gradle

@@ -20,7 +20,7 @@ dependencies {
     play('com.nimbusds:nimbus-jose-jwt:8.18')
     play('com.typesafe.akka:akka-actor_2.12:2.6.20')
     play(externalDependency.jsonSmart)
-    play('io.netty:netty-all:4.1.86.Final')
+    play('io.netty:netty-all:4.1.114.Final')
     implementation(externalDependency.commonsText) {
       because("previous versions are vulnerable to CVE-2022-42889")
     }

docker/build.gradle

@@ -108,6 +108,8 @@ dockerCompose {
         environment.put "ACTIONS_EXTRA_PACKAGES", 'acryl-datahub-actions[executor] acryl-datahub-actions'
         environment.put "ACTIONS_CONFIG", 'https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml'
         environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally
+        // disabled for spark-lineage smoke-test
+        environment.put 'DATAHUB_LOCAL_COMMON_ENV', "${rootProject.project(':metadata-integration:java:spark-lineage-legacy').projectDir}/spark-smoke-test/smoke-gms.env"
 
         useComposeFiles = ['profiles/docker-compose.yml']
         projectName = 'datahub'

docs-website/adoptionStoriesIndexes.json

@@ -14,8 +14,6 @@
     {
       "name": "Visa",
       "slug": "visa",
-      "imageUrl": "/img/logos/companies/visa.png",
-      "imageSize": "large",
       "link": "https://blog.datahubproject.io/how-visa-uses-datahub-to-scale-data-governance-cace052d61c5",
       "linkType": "blog",
       "tagline": "How Visa uses DataHub to scale data governance",
@@ -56,6 +54,12 @@
       "category": "B2B & B2C",
       "description": "Pinterest adopted a DataHub project to enhance metadata management for its big data query platform, facilitating better data navigation and understanding."
     },
+    {
+      "name": "Snap, Inc.",
+      "slug": "snap",
+      "imageUrl": "/img/logos/companies/snap.png",
+      "imageSize": "small"
+    },
     {
       "name": "Airtel",
       "slug": "airtel",
@@ -89,6 +93,42 @@
       "category": "B2B & B2C",
       "description": "<i>“We looked around for data catalog tool, and DataHub was a clear winner.”</i> <br /> <br /> Zynga levels up data management using DataHub, highlighting its role in enhancing data management, tracing data lineage, and ensuring data quality."
     },
+    {
+      "name": "Miro",
+      "slug": "miro",
+      "imageUrl": "/img/logos/companies/miro.png",
+      "imageSize": "medium",
+      "link": "https://miro.com/careers/life-at-miro/tech/data-products-reliability-the-power-of-metadata/",
+      "linkType": "blog",
+      "tagline": "Data Products Reliability: The Power of Metadata",
+      "category": "B2B & B2C",
+      "platform": "cloud",
+      "description": "<i>\"Leveraging our Datahub catalog, we have centralized metadata access for all data products. This integration eliminates the dependency on Airflow metadata alone for defining contracts, enabling flexible definitions for both building blocks and business metrics.\"</i>"
+    },
+    {
+      "name": "Foursquare",
+      "slug": "foursquare",
+      "imageUrl": "/img/logos/companies/foursquare.png",
+      "imageSize": "medium",
+      "link": "https://location.foursquare.com/resources/blog/leadership/foursquare-data-platform-from-fragmentation-to-control-plane/",
+      "linkType": "blog",
+      "tagline": "Foursquare Data Platform: From Fragmentation to Control (Plane)",
+      "category": "B2B & B2C",
+      "platform": "cloud",
+      "description": "<i>\"After evaluating several options (...) we chose DataHub as the control plane for our data platform and partnered with Acryl Data, which offers DataHub Cloud, a premium hosted version of DataHub. (...) Another important thing that worked in DataHub’s favor is the rich and flexible taxonomy it offered for modeling the various aspects of a data platform.\"</i>"
+    },
+    {
+      "name": "Deutsche Telekom",
+      "slug": "deutsche-telekom",
+      "imageUrl": "/img/logos/companies/deutsche-telekom.png",
+      "imageSize": "medium",
+      "link": "https://karanjindal95.medium.com/from-chaos-to-clarity-how-datahub-transformed-our-data-utilization-5b5151efd34a",
+      "linkType": "blog",
+      "tagline": "From Chaos to Clarity: How DataHub Transformed our Data Utilization",
+      "category": "B2B & B2C",
+      "platform": "cloud",
+      "description": "<i>\"The DataHub data catalog significantly supported our AI/ML team’s efforts by offering seamless access to detailed column descriptions and table schemas through its APIs. This comprehensive data accessibility enabled the team to efficiently develop a text-to-SQL tool, which translates natural language queries into SQL commands.\"</i>"
+    },
     {
       "name": "Chime",
       "slug": "chime",
@@ -374,4 +414,4 @@
       "category": "And More"
     }
   ]
-}
+}

docs-website/src/pages/_components/CaseStudy/caseStudyContent.js

@@ -16,7 +16,7 @@ const caseStudyData = [
     tag: "Finance",
     backgroundImage:
       "https://miro.medium.com/v2/resize:fit:2000/format:webp/1*[email protected]",
-    image: "https://datahubproject.io/img/logos/companies/visa.png",
+    image: "/img/logos/companies/visa_text.png",
     link: "https://datahubproject.io/adoption-stories/#visa",
   },
   {

docs-website/src/pages/_components/Logos/index.js

@@ -171,7 +171,7 @@ export const CompanyLogos = () => (
       className={clsx("mySwiper", styles.companyWrapper)}
     >
       {companies
-        .filter((company) => company.imageUrl) // Filter companies with imageUrl
+        .filter((company) => company.imageUrl && company.link) // Filter companies with imageUrl and link
         .map((company, idx) => (
           <SwiperSlide key={idx}>
             {company.link ? (

docs-website/src/pages/index.js

@@ -21,7 +21,7 @@ import CloseButton from "@ant-design/icons/CloseCircleFilled";
 
 const companyIndexes = require("../../adoptionStoriesIndexes.json");
 const companies = companyIndexes.companies;
-const keyCompanySlugs = ["netflix", "visa", "pinterest", "airtel", "optum"];
+const keyCompanySlugs = ["netflix", "pinterest", "notion", "snap", "optum"]; //, "airtel"];
 const keyCompanies = keyCompanySlugs
   .map((slug) => companies.find((co) => co.slug === slug))
   .filter((isDefined) => isDefined);
@@ -63,7 +63,13 @@ function Home() {
         </div>
         <div className="company_logos_list_wrapper">
           {keyCompanies.map((company) => (
-            <a href={`/adoption-stories#${company.slug}`}>
+            <a
+              href={
+                company.slug != "snap"
+                  ? `/adoption-stories#${company.slug}`
+                  : undefined
+              }
+            >
               <img
                 src={useBaseUrl(company.imageUrl)}
                 alt={company.name}
@@ -72,6 +78,9 @@ function Home() {
               />
             </a>
           ))}
+          <a href="/adoption-stories" class="more_link">
+            + More
+          </a>
         </div>
         {/* <div style={{ textAlign: "center", margin: "1rem" }}>
           <Link

docs-website/src/styles/global.scss

@@ -399,7 +399,7 @@ div[class^="announcementBar"] {
   .text {
     padding-right: 24px;
     color: #777;
-    font-size: 1.25rem;
+    font-size: 1.2rem;
     line-height: 1.5rem;
   }
 
@@ -409,6 +409,16 @@ div[class^="announcementBar"] {
     align-items: center;
     flex-grow: 1;
     justify-content: space-around;
+
+
+    .more_link {
+      font-size: 1.25rem;
+      color: #bbb;
+      font-weight: 600;
+      text-decoration: none;
+      position: relative;
+      top: -.4rem;
+    }
   }
   .company_logo {
     max-width: 80px;
@@ -486,6 +496,9 @@ div[class^="announcementBar"] {
     }
     .company_logos_list_wrapper {
       width: 100%;
+      .more_link {
+        display: none;
+      }
     }
     .company_logo {
       max-width: 60px;

metadata-ingestion-modules/dagster-plugin/examples/advanced_ops_jobs.py

@@ -32,12 +32,12 @@ def extract():
     ins={
         "data": In(
             dagster_type=PythonObjectDagsterType(list),
-            metadata={"datahub.inputs": [DatasetUrn("snowflake", "tableA").urn]},
+            metadata={"datahub.inputs": [DatasetUrn("snowflake", "tableA").urn()]},
         )
     },
     out={
         "result": Out(
-            metadata={"datahub.outputs": [DatasetUrn("snowflake", "tableB").urn]}
+            metadata={"datahub.outputs": [DatasetUrn("snowflake", "tableB").urn()]}
         )
     },
 )
@@ -101,6 +101,5 @@ def asset_lineage_extractor(
     dagster_url="http://localhost:3000",
     asset_lineage_extractor=asset_lineage_extractor,
 )
-
 datahub_sensor = make_datahub_sensor(config=config)
 defs = Definitions(jobs=[do_stuff], sensors=[datahub_sensor])

metadata-ingestion-modules/dagster-plugin/examples/assets_job.py

@@ -7,6 +7,7 @@
     define_asset_job,
     multi_asset,
 )
+from datahub.ingestion.graph.config import DatahubClientConfig
 from datahub.utilities.urns.dataset_urn import DatasetUrn
 
 from datahub_dagster_plugin.sensors.datahub_sensors import (
@@ -18,7 +19,7 @@
 @multi_asset(
     outs={
         "extract": AssetOut(
-            metadata={"datahub.outputs": [DatasetUrn("snowflake", "tableD").urn]}
+            metadata={"datahub.outputs": [DatasetUrn("snowflake", "tableD").urn()]}
         ),
     }
 )
@@ -47,13 +48,9 @@ def transform(extract):
 
 assets_job = define_asset_job(name="assets_job")
 
-config = DatahubDagsterSourceConfig.parse_obj(
-    {
-        "rest_sink_config": {
-            "server": "http://localhost:8080",
-        },
-        "dagster_url": "http://localhost:3000",
-    }
+config = DatahubDagsterSourceConfig(
+    datahub_client_config=DatahubClientConfig(server="http://localhost:8080"),
+    dagster_url="http://localhost:3000",
 )
 
 datahub_sensor = make_datahub_sensor(config=config)

metadata-ingestion-modules/dagster-plugin/examples/ops_job.py

@@ -1,4 +1,5 @@
 from dagster import Definitions, In, Out, PythonObjectDagsterType, job, op
+from datahub.ingestion.graph.config import DatahubClientConfig
 from datahub.utilities.urns.dataset_urn import DatasetUrn
 
 from datahub_dagster_plugin.sensors.datahub_sensors import (
@@ -17,12 +18,12 @@ def extract():
     ins={
         "data": In(
             dagster_type=PythonObjectDagsterType(list),
-            metadata={"datahub.inputs": [DatasetUrn("snowflake", "tableA").urn]},
+            metadata={"datahub.inputs": [DatasetUrn("snowflake", "tableA").urn()]},
         )
     },
     out={
         "result": Out(
-            metadata={"datahub.outputs": [DatasetUrn("snowflake", "tableB").urn]}
+            metadata={"datahub.outputs": [DatasetUrn("snowflake", "tableB").urn()]}
         )
     },
 )
@@ -38,13 +39,9 @@ def do_stuff():
     transform(extract())
 
 
-config = DatahubDagsterSourceConfig.parse_obj(
-    {
-        "rest_sink_config": {
-            "server": "http://localhost:8080",
-        },
-        "dagster_url": "http://localhost:3000",
-    }
+config = DatahubDagsterSourceConfig(
+    datahub_client_config=DatahubClientConfig(server="http://localhost:8080"),
+    dagster_url="http://localhost:3000",
 )
 
 datahub_sensor = make_datahub_sensor(config=config)