Skip to content

Commit c40b69d

Browse files
mahmoudhanafyMahmoud Hanafy
authored andcommitted
Remove squishPandaFromPace example
1 parent d276733 commit c40b69d

File tree

1 file changed

+2
-41
lines changed

1 file changed

+2
-41
lines changed

src/main/java/com/highperformancespark/examples/dataframe/JavaHappyPandas.java

Lines changed: 2 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,17 @@
11
package com.highperformancespark.examples.dataframe;
22

3-
import com.highperformancespark.examples.objects.JavaRawPanda;
43
import org.apache.spark.api.java.JavaRDD;
54
import org.apache.spark.api.java.JavaSparkContext;
65
import org.apache.spark.sql.Column;
76
import org.apache.spark.sql.DataFrame;
8-
import org.apache.spark.sql.Row;
97
import org.apache.spark.sql.SQLContext;
108
import org.apache.spark.sql.expressions.Window;
119
import org.apache.spark.sql.expressions.WindowSpec;
1210
import org.apache.spark.sql.hive.HiveContext;
1311
import org.apache.spark.sql.hive.thriftserver.HiveThriftServer2;
14-
import scala.collection.JavaConversions;
15-
import scala.collection.mutable.Buffer;
16-
import scala.reflect.api.TypeTags;
1712

18-
import java.util.Arrays;
1913
import java.util.HashMap;
20-
import java.util.List;
2114
import java.util.Map;
22-
import java.util.stream.Collectors;
2315

2416
import static org.apache.spark.sql.functions.*;
2517

@@ -97,37 +89,6 @@ public DataFrame minHappyPandas(DataFrame pandaInfo, int minHappyPandas) {
9789
return pandaInfo.filter(pandaInfo.col("happyPandas").geq(minHappyPandas));
9890
}
9991

100-
/**
101-
* Extra the panda info from panda places and compute the squisheness of the panda
102-
*/
103-
public DataFrame squishPandaFromPace(DataFrame pandaPlace) {
104-
Buffer<Column> inputCols = JavaConversions.asScalaBuffer(Arrays.asList(pandaPlace.col("pandas")));
105-
106-
TypeTags.TypeTag tag = null; // TODO don't know how to create Type Tag in java ??
107-
108-
DataFrame pandaInfo = pandaPlace.explode(inputCols.toList(), r -> {
109-
List<Row> pandas = r.getList(0);
110-
List<JavaRawPanda> rawPandasList = pandas
111-
.stream()
112-
.map(a -> {
113-
long id = a.getLong(0);
114-
String zip = a.getString(1);
115-
String pt = a.getString(2);
116-
boolean happy = a.getBoolean(3);
117-
List<Double> attrs = a.getList(4);
118-
return new JavaRawPanda(id, zip, pt, happy, attrs);
119-
}).collect(Collectors.toList());
120-
121-
return JavaConversions.asScalaBuffer(rawPandasList);
122-
}, tag);
123-
124-
DataFrame squishyness =
125-
pandaInfo.select((pandaInfo.col("attributes").apply(0).divide(pandaInfo.col("attributes")).apply(1))
126-
.as("squishyness"));
127-
128-
return squishyness;
129-
}
130-
13192
/**
13293
* Find pandas that are sad.
13394
*/
@@ -201,8 +162,8 @@ public void startJDBCServer(HiveContext sqlContext) {
201162

202163
/**
203164
* Orders pandas by size ascending and by age descending.
204-
* Pandas will be sorted by "size" first and if two pandas have the same "size"
205-
* will be sorted by "age".
165+
* Pandas will be sorted by "size" first and if two pandas
166+
* have the same "size" will be sorted by "age".
206167
*/
207168
public DataFrame orderPandas(DataFrame pandas) {
208169
return pandas.orderBy(pandas.col("pandaSize").asc(), pandas.col("age").desc());

0 commit comments

Comments
 (0)