Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Extract public #244

Merged
merged 1 commit into from
Sep 26, 2023
Merged

Extract public #244

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
182 changes: 182 additions & 0 deletions jflyte-utils/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright 2021 Flyte Authors.
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
Copyright 2021 Flyte Authors.
Copyright 2023 Flyte Authors.

Unless there is some specific requirement that it this license section is the exact same everywhere?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

flytekit-java/pom.xml

Line 415 in 04e6fdc

* Copyright 2021 Flyte Authors
this would fail the build and to update that, we need to upgrade every single file. :(

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I was afraid it was something like this. Thanks for confirming!


Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>org.flyte</groupId>
<artifactId>flytekit-parent</artifactId>
<version>0.4.23-SNAPSHOT</version>
</parent>

<artifactId>jflyte-utils</artifactId>

<name>JFlyte Utils - Utilities for jflyte</name>
<description>Primarily used by jflyte, but can also be used to extend or build a jflyte alternative</description>

<dependencies>
<!-- compile -->
<dependency>
<groupId>org.flyte</groupId>
<artifactId>flyteidl-protos</artifactId>
</dependency>
<dependency>
<groupId>org.flyte</groupId>
<artifactId>flytekit-api</artifactId>
</dependency>
<dependency>
<groupId>org.flyte</groupId>
<artifactId>flytekit-local-engine</artifactId>
</dependency>
<dependency>
<groupId>org.flyte</groupId>
<artifactId>jflyte-api</artifactId>
</dependency>
<dependency>
<groupId>org.flyte</groupId>
<artifactId>jflyte-aws</artifactId>
<!-- We want to have the dependency, so it is already built before building the docker image,
but we want to avoid the artifacts from the module to pollute jflyte -->
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.flyte</groupId>
<artifactId>jflyte-google-cloud</artifactId>
<!-- We want to have the dependency, so it is already built before building the docker image,
but we want to avoid the artifacts from the module to pollute jflyte -->
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>jul-to-slf4j</artifactId>
</dependency>
<dependency>
<groupId>info.picocli</groupId>
<artifactId>picocli</artifactId>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty</artifactId>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-protobuf</artifactId>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-stub</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-tcnative-boringssl-static</artifactId>
</dependency>

<!-- runtime -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<scope>runtime</scope>
</dependency>

<!-- provided -->
<dependency>
<groupId>com.google.auto.value</groupId>
<artifactId>auto-value-annotations</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.auto.service</groupId>
<artifactId>auto-service-annotations</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.errorprone</groupId>
<artifactId>error_prone_annotations</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
<scope>provided</scope>
</dependency>

<!-- test -->
<dependency>
<groupId>org.flyte</groupId>
<artifactId>flytekit-jackson</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.flyte</groupId>
<artifactId>flytekit-java</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-testing</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.github.marschall</groupId>
<artifactId>memoryfilesystem</artifactId>
<scope>test</scope>
</dependency>
<!-- ensure there is at least one slf4j implementation in test classpath so that we can test
org/slf4j/impl/StaticLoggerBinder.class is only discovered in child class loader -->
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>
2 changes: 1 addition & 1 deletion .../main/java/org/flyte/jflyte/Artifact.java → ...java/org/flyte/jflyte/utils/Artifact.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.flyte.jflyte;
package org.flyte.jflyte.utils;

import com.google.auto.value.AutoValue;
import org.flyte.jflyte.api.FileSystem;
Expand Down
6 changes: 3 additions & 3 deletions ...java/org/flyte/jflyte/ArtifactStager.java → ...rg/flyte/jflyte/utils/ArtifactStager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.flyte.jflyte;
package org.flyte.jflyte.utils;

import static com.google.common.base.Verify.verify;

Expand Down Expand Up @@ -56,7 +56,7 @@
* for directories as well, where directory would be unique and contain hash code of all of it's
* contents.
*/
class ArtifactStager {
public class ArtifactStager {
private static final Logger LOG = LoggerFactory.getLogger(ArtifactStager.class);

private final String stagingLocation;
Expand All @@ -69,7 +69,7 @@ class ArtifactStager {
this.executorService = executorService;
}

static ArtifactStager create(
public static ArtifactStager create(
Config config, Collection<ClassLoader> modules, ExecutorService executorService) {
try {
String stagingLocation = config.stagingLocation();
Expand Down
2 changes: 1 addition & 1 deletion ...flyte/AuthorizationHeaderInterceptor.java → ...utils/AuthorizationHeaderInterceptor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.flyte.jflyte;
package org.flyte.jflyte.utils;

import io.grpc.CallOptions;
import io.grpc.Channel;
Expand Down
6 changes: 3 additions & 3 deletions ...g/flyte/jflyte/ChildFirstClassLoader.java → ...e/jflyte/utils/ChildFirstClassLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.flyte.jflyte;
package org.flyte.jflyte.utils;

import com.google.errorprone.annotations.Var;
import java.io.IOException;
Expand All @@ -36,7 +36,7 @@
* between the code in parent class loader, and the code loaded in child class loaders, and we pass
* instances of these classes around.
*/
class ChildFirstClassLoader extends URLClassLoader {
public class ChildFirstClassLoader extends URLClassLoader {

// we have to load these classes in parent class loader
// it's base shared between all plugins and user code
Expand Down Expand Up @@ -66,7 +66,7 @@ public URL nextElement() {
}
}

ChildFirstClassLoader(URL[] urls) {
public ChildFirstClassLoader(URL[] urls) {
super(urls, ChildFirstClassLoader.class.getClassLoader());
}

Expand Down
16 changes: 8 additions & 8 deletions ...n/java/org/flyte/jflyte/ClassLoaders.java → .../org/flyte/jflyte/utils/ClassLoaders.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.flyte.jflyte;
package org.flyte.jflyte.utils;

import static org.flyte.jflyte.MoreCollectors.toUnmodifiableMap;
import static org.flyte.jflyte.utils.MoreCollectors.toUnmodifiableMap;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
Expand All @@ -33,29 +33,29 @@
import org.slf4j.LoggerFactory;

/** Utility to work with class loaders. */
class ClassLoaders {
public class ClassLoaders {

private static final Logger LOG = LoggerFactory.getLogger(ClassLoaders.class);

private ClassLoaders() {
throw new UnsupportedOperationException();
}

static Map<String, ClassLoader> forModuleDir(String dir) {
public static Map<String, ClassLoader> forModuleDir(String dir) {
return listDirectory(new File(dir)).stream()
.filter(File::isDirectory)
.map(subDir -> Maps.immutableEntry(subDir.getAbsolutePath(), forDirectory(subDir)))
.collect(toUnmodifiableMap());
}

static ClassLoader forDirectory(File dir) {
public static ClassLoader forDirectory(File dir) {
LOG.debug("Loading jars from [{}]", dir.getAbsolutePath());

return AccessController.doPrivileged(
(PrivilegedAction<ClassLoader>) () -> new ChildFirstClassLoader(getClassLoaderUrls(dir)));
}

static URL[] getClassLoaderUrls(File dir) {
private static URL[] getClassLoaderUrls(File dir) {
Preconditions.checkNotNull(dir, "dir is null");

return listDirectory(dir).stream()
Expand All @@ -73,7 +73,7 @@ static URL[] getClassLoaderUrls(File dir) {
.toArray(URL[]::new);
}

static List<File> listDirectory(File file) {
private static List<File> listDirectory(File file) {
if (!file.exists()) {
throw new RuntimeException(
String.format("Directory doesn't exist [%s]", file.getAbsolutePath()));
Expand All @@ -88,7 +88,7 @@ static List<File> listDirectory(File file) {
return ImmutableList.copyOf(files);
}

static <V> V withClassLoader(ClassLoader classLoader, Callable<V> callable) {
public static <V> V withClassLoader(ClassLoader classLoader, Callable<V> callable) {
ClassLoader originalContextClassLoader = Thread.currentThread().getContextClassLoader();
Thread.currentThread().setContextClassLoader(classLoader);

Expand Down
2 changes: 1 addition & 1 deletion .../org/flyte/jflyte/CompletableFutures.java → ...lyte/jflyte/utils/CompletableFutures.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.flyte.jflyte;
package org.flyte.jflyte.utils;

import java.util.ArrayList;
import java.util.List;
Expand Down
16 changes: 8 additions & 8 deletions ...rc/main/java/org/flyte/jflyte/Config.java → ...n/java/org/flyte/jflyte/utils/Config.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,52 +14,52 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.flyte.jflyte;
package org.flyte.jflyte.utils;

import com.google.auto.value.AutoValue;
import javax.annotation.Nullable;

/** Configuration file for jflyte. */
@AutoValue
abstract class Config {
public abstract class Config {

/**
* Get flyte platform URL.
*
* @return flyte platform URL.
*/
abstract String platformUrl();
public abstract String platformUrl();

/**
* Get flyte docker image.
*
* @return flyte docker image.
*/
abstract String image();
public abstract String image();

/**
* Get flyte staging location path.
*
* @return flyte staging location path.
*/
@Nullable
abstract String stagingLocation();
public abstract String stagingLocation();

/**
* Get flyte module dir path.
*
* @return flyte module dir path.
*/
abstract String moduleDir();
public abstract String moduleDir();

/**
* Get if the auth with the platform is secure or not.
*
* @return boolean indicating if platform is insecure or not.
*/
abstract boolean platformInsecure();
public abstract boolean platformInsecure();

static Config load() {
public static Config load() {
return Config.builder()
.platformUrl(getenv("FLYTE_PLATFORM_URL"))
.moduleDir(getenv("FLYTE_INTERNAL_MODULE_DIR"))
Expand Down
Loading
Loading