Skip to content

Commit

Permalink
LakeFSCatalog Initial Implementation
Browse files Browse the repository at this point in the history
  • Loading branch information
@N-o-Z
N-o-Z committed Feb 25, 2024
1 parent 8ae704c commit f86dcad
Show file tree
Hide file tree
Showing 16 changed files with 15,233 additions and 0 deletions.
76 changes: 76 additions & 0 deletions .github/workflows/tests.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
name: iceberg-tests
on:
pull_request:
push:
branches:
- main

jobs:
iceberg-test:
name: Test lakeFS iceberg catalog
runs-on: ubuntu-20.04
env:
LAKEFS_INSTALLATION_ACCESS_KEY_ID: AKIAIOSFDNN7EXAMPLEQ
LAKEFS_INSTALLATION_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
services:
lakefs:
image: "treeverse/lakefs:1"
ports:
- '8000:8000'
env:
LAKEFS_DATABASE_TYPE: local
LAKEFS_BLOCKSTORE_TYPE: s3
LAKEFS_INSTALLATION_ACCESS_KEY_ID: ${{ env.LAKEFS_INSTALLATION_ACCESS_KEY_ID }}
LAKEFS_INSTALLATION_SECRET_ACCESS_KEY: ${{ env.LAKEFS_INSTALLATION_SECRET_ACCESS_KEY }}
LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
LAKEFS_BLOCKSTORE_S3_CREDENTIALS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
LAKEFS_AUTH_ENCRYPT_SECRET_KEY: some random secret string
LAKEFS_STATS_ENABLED: false
LAKEFS_INSTALLATION_USER_NAME: admin

steps:
- name: Check-out code
uses: actions/checkout@v3

- name: Set up JDK 11
uses: actions/setup-java@v3
with:
distribution: "temurin"
java-version: "11"
cache: "maven"

- name: Generate uniquifying value
id: unique
run: echo "value=$RANDOM" >> $GITHUB_OUTPUT

- name: Build lakeFS Iceberg
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
STORAGE_NAMESPACE: s3://esti-system-testing/${{ github.run_number }}-iceberg-rest/${{ steps.unique.outputs.value }}
run: mvn clean install --batch-mode -U -P\!sign-artifacts

- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.9
cache: 'pip'
- run: pip install -r ./tests/requirements.txt

- name: Run Python Tests
run: pip install pytest pytest-md pytest-emoji
- uses: pavelzw/pytest-action@v2
env:
LAKECTL_CREDENTIALS_ACCESS_KEY_ID: ${{ env.LAKEFS_INSTALLATION_ACCESS_KEY_ID }}
LAKECTL_CREDENTIALS_SECRET_ACCESS_KEY: ${{ env.LAKEFS_INSTALLATION_SECRET_ACCESS_KEY }}
LAKECTL_SERVER_ENDPOINT_URL: http://localhost:8000
STORAGE_NAMESPACE: s3://esti-system-testing/${{ github.run_number }}-iceberg-rest/${{ steps.unique.outputs.value }}
with:
verbose: true
emoji: true
job-summary: true
custom-arguments: './tests -rP'
click-to-expand: true
report-title: 'lakeFS Iceberg Catalog System Tests Report'
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
target/
306 changes: 306 additions & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,306 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>io.lakefs</groupId>
<artifactId>lakefs-iceberg-rest</artifactId>
<version>0.1.0-SNAPSHOT</version>

<name>lakefs-iceberg-rest</name>
<url>https://lakefs.io</url>
<description>An Iceberg REST catalog implementation for lakeFS</description>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<iceberg.version>1.4.3</iceberg.version>
<hadoop.version>3.3.5</hadoop.version>
</properties>
<scm>
<connection>scm:git:[email protected]:treeverse/lakefs-iceberg-rest.git</connection>
<developerConnection>scm:git:[email protected]:treeverse/lakefs-iceberg-rest.git</developerConnection>
<url>https://github.com/treeverse/lakefs-iceberg-rest</url>
</scm>
<developers>
<developer>
<name>lakeFS</name>
<email>[email protected]</email>
<organization>lakefs.io</organization>
<organizationUrl>https://lakefs.io</organizationUrl>
</developer>
</developers>
<dependencies>
<dependency>
<groupId>io.lakefs</groupId>
<artifactId>hadoop-lakefs</artifactId>
<version>0.2.3-RC.0</version> <!-- Modify after releasing new lakeFSFS version -->
</dependency>
<dependency>
<groupId>io.lakefs</groupId>
<artifactId>hadoop-lakefs</artifactId>
<version>0.2.3-RC.0</version> <!-- Modify after releasing new lakeFSFS version -->
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-scala_3</artifactId>
<version>2.16.1</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.iceberg/iceberg-core -->
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-core</artifactId>
<version>${iceberg.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-spark-3.3_2.13</artifactId>
<version>${iceberg.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-spark-extensions-3.3_2.13</artifactId>
<version>${iceberg.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-spark-runtime-3.3_2.13</artifactId>
<version>${iceberg.version}</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.13</artifactId>
<version>3.3.4</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-api</artifactId>
<version>${iceberg.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.assertj/assertj-core -->
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>3.24.2</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.guava/guava -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>33.0.0-jre</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>${hadoop.version}</version>
<scope>provided</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client-api -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-api</artifactId>
<version>${hadoop.version}</version>
<scope>provided</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-api -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>2.0.9</version>
</dependency>
<!-- <dependency>--> <!-- Enable for the REST server-->
<!-- <groupId>org.apache.iceberg</groupId>-->
<!-- <artifactId>iceberg-core</artifactId>-->
<!-- <version>${iceberg.version}</version>-->
<!-- <type>test-jar</type>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>org.eclipse.jetty</groupId>-->
<!-- <artifactId>jetty-server</artifactId>-->
<!-- <version>9.4.51.v20230217</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>org.eclipse.jetty</groupId>-->
<!-- <artifactId>jetty-servlet</artifactId>-->
<!-- <version>9.4.51.v20230217</version>-->
<!-- </dependency>-->
<dependency>
<groupId>org.mock-server</groupId>
<artifactId>mockserver-junit-rule-no-dependencies</artifactId>
<version>5.14.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mock-server</groupId>
<artifactId>mockserver-core</artifactId>
<version>5.15.0</version>
<scope>test</scope>
</dependency>
</dependencies>
<profiles>
<profile>
<id>sign-artifacts</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-gpg-plugin</artifactId>
<version>1.6</version>
<configuration>
<keyname>408A5896A9B8EC94EE4F7B6472FF63CDF8B05FCF</keyname>
</configuration>
<executions>
<execution>
<id>sign-artifacts</id>
<configuration>
<gpgArguments>
<arg>--pinentry-mode</arg>
<arg>loopback</arg>
</gpgArguments>
</configuration>
<phase>verify</phase>
<goals>
<goal>sign</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.5.0</version>
<executions>
<execution>
<id>attach-javadocs</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>2.2.1</version>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar-no-fork</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>

<distributionManagement>
<snapshotRepository>
<id>ossrh</id>
<name>Nexus Snapshots Repo</name>
<url>https://s01.oss.sonatype.org/content/repositories/snapshots/</url>
</snapshotRepository>
<repository>
<id>ossrh</id>
<name>Nexus Staging Repo</name>
<url>https://s01.oss.sonatype.org/nexus/service/local/staging/deploy/maven2/</url>
</repository>
</distributionManagement>

<build>
<plugins>
<plugin>
<groupId>org.sonatype.plugins</groupId>
<artifactId>nexus-staging-maven-plugin</artifactId>
<version>1.6.8</version>
<extensions>true</extensions>
<configuration>
<serverId>ossrh</serverId>
<nexusUrl>https://s01.oss.sonatype.org/</nexusUrl>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.3.0</version>
<configuration>
<archive>
<manifest>
<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
<addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
</manifest>
</archive>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<relocations>
<relocation>
<pattern>okio</pattern>
<shadedPattern>com.shaded.okio</shadedPattern>
</relocation>
<relocation>
<pattern>okhttp3</pattern>
<shadedPattern>com.shaded.okhttp3</shadedPattern>
</relocation>
<relocation>
<pattern>com.google</pattern>
<shadedPattern>com.shaded.google</shadedPattern>
</relocation>
</relocations>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>log4j.properties</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
Loading

0 comments on commit f86dcad

Please sign in to comment.