pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>

  <parent>
    <groupId>org.gbif.pipelines</groupId>
    <artifactId>examples</artifactId>
    <version>3.0.1-SNAPSHOT</version>
    <relativePath>../pom.xml</relativePath>
  </parent>

  <artifactId>examples-metrics</artifactId>
  <packaging>jar</packaging>

  <name>Pipelines :: Examples :: Metrics</name>
  <description>The example demonstrates how to send Apache Beam SparkRunner metrics to ELK</description>


  <dependencies>

    <!-- This project -->
    <!-- Dependency to custom Beam Slf4J sink -->
    <dependency>
      <groupId>org.gbif.pipelines</groupId>
      <artifactId>beam-common</artifactId>
    </dependency>
    <!-- Dependency to custom Beam Options -->
    <dependency>
      <groupId>org.gbif.pipelines</groupId>
      <artifactId>ingest-gbif-beam</artifactId>
      <exclusions>
        <!-- Excluded to avoid problem with Spark standalone mode and Log4j logger -->
        <exclusion>
          <groupId>ch.qos.logback</groupId>
          <artifactId>logback-classic</artifactId>
        </exclusion>
      </exclusions>
    </dependency>

    <!-- Beam -->
    <dependency>
      <groupId>org.apache.beam</groupId>
      <artifactId>beam-sdks-java-core</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.beam</groupId>
      <artifactId>beam-runners-spark-3</artifactId>
    </dependency>

    <!-- Hadoop -->
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-common</artifactId>
      <scope>compile</scope>
    </dependency>

    <!-- Spark -->
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_2.12</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-streaming_2.12</artifactId>
    </dependency>

    <!-- Logging -->
    <dependency>
      <groupId>org.slf4j</groupId>
      <artifactId>slf4j-api</artifactId>
    </dependency>
    <!-- GELF adapter for Spark to send logs to ELK -->
    <dependency>
      <groupId>biz.paluch.logging</groupId>
      <artifactId>logstash-gelf</artifactId>
    </dependency>

  </dependencies>

  <profiles>
    <profile>
      <id>example-artifacts</id>
      <build>
        <plugins>
          <!-- Shade the project into an uber jar to send to Spark -->
          <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-shade-plugin</artifactId>
            <configuration>
              <createDependencyReducedPom>true</createDependencyReducedPom>
              <filters>
                <filter>
                  <artifact>*:*</artifact>
                  <excludes>
                    <exclude>META-INF/*.SF</exclude>
                    <exclude>META-INF/*.DSA</exclude>
                    <exclude>META-INF/*.RSA</exclude>
                  </excludes>
                </filter>
              </filters>
            </configuration>
            <executions>
              <execution>
                <phase>package</phase>
                <goals>
                  <goal>shade</goal>
                </goals>
                <configuration>
                  <shadedArtifactAttached>true</shadedArtifactAttached>
                  <shadedClassifierName>shaded</shadedClassifierName>
                  <transformers>
                    <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
                    <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                      <mainClass>org.gbif.pipelines.examples.MetricsPipeline</mainClass>
                    </transformer>
                  </transformers>
                  <relocations>
                    <!-- To avoid guava problem in Spark cluster mode -->
                    <relocation>
                      <pattern>com.google.common</pattern>
                      <shadedPattern>g20.com.google.common</shadedPattern>
                    </relocation>
                  </relocations>
                </configuration>
              </execution>
            </executions>
          </plugin>
        </plugins>
      </build>
    </profile>
  </profiles>

</project>