From 51f136cc59dc35932bf915c31a68971fcb3f134e Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Wed, 17 Aug 2016 19:10:40 -0700 Subject: [PATCH] Update Example.java --- .../datasyslab/geospark/showcase/Example.java | 108 +----------------- 1 file changed, 4 insertions(+), 104 deletions(-) diff --git a/src/main/java/org/datasyslab/geospark/showcase/Example.java b/src/main/java/org/datasyslab/geospark/showcase/Example.java index cff8b98b31..1ed0c041af 100644 --- a/src/main/java/org/datasyslab/geospark/showcase/Example.java +++ b/src/main/java/org/datasyslab/geospark/showcase/Example.java @@ -88,11 +88,9 @@ public static void main(String[] args) { } conf=new SparkConf().setAppName(queryName+"+"+inputLocation+"+"+gridType+"+"+cores+"+"+numPartitions).setMaster(masterName) .set("spark.history.fs.logDirectory", "/home/ubuntu/sparkeventlog") - //.set("spark.history.fs.logDirectory", "/data1/jia/sparkeventlog") .set("spark.history.retainedApplications", "10000") .set("spark.eventLog.enabled", "true") .set("spark.eventLog.dir", "/home/ubuntu/sparkeventlog") - //.set("spark.eventLog.dir", "/data1/jia/sparkeventlog") .set("spark.executor.memory", "50g") .set("spark.core.connection.ack.wait.timeout","900") .set("spark.akka.timeout","900") @@ -114,14 +112,11 @@ public static void main(String[] args) { try { switch(queryName) { - case "formattransform": - formatTransform(args[0]); - break; case "pointrange": testSpatialRangeQuery(); break; case "pointrangeindex": - testSpatialRangeQueryUsingIndex(); + testSpatialRangeQueryUsingIndex(); break; case "pointknn": testSpatialKnnQuery(); @@ -135,9 +130,6 @@ public static void main(String[] args) { case "pointjoinindex": testSpatialJoinQueryUsingIndex(); break; - case "pointjoinspark": - testSpatialJoinQueryUsingSpark(); - break; default: throw new Exception("Query type is not recognized, "); } @@ -148,33 +140,6 @@ public static void main(String[] args) { TearDown(); } - public static void formatTransform(String outputname) throws Exception - { - PolygonRDD polygonRDD = new PolygonRDD(sc, inputLocation, offset, splitter); - JavaRDD result=polygonRDD.rawPolygonRDD.map(new Function() - { - @Override - public String call(Polygon v1) throws Exception { - // TODO Auto-generated method stub - Envelope envelope=v1.getEnvelopeInternal(); - String result=""; - result=envelope.getMinX()+"\t"+envelope.getMinY()+"\t"+envelope.getMaxX()+"\t"+envelope.getMaxY(); - return result; - } - - }); - FileWriter fw = new FileWriter(outputname+".new"); - BufferedWriter bw = new BufferedWriter(fw); - PrintWriter out = new PrintWriter(bw); - List output=result.collect(); - for(String str:output) - { - out.println(str); - } - out.close(); - bw.close(); - fw.close(); - } public static void testSpatialRangeQuery() throws Exception { Random random=new Random(); double randomNumber=random.nextInt(10)+random.nextDouble(); @@ -188,18 +153,7 @@ public static void testSpatialRangeQuery() throws Exception { } } - public static void testSpatialRangeQuery1() throws Exception { - Random random=new Random(); - double randomNumber=random.nextInt(10)+random.nextDouble(); - queryEnvelope=new Envelope (-90.01+randomNumber,-80.01+randomNumber,30.01+randomNumber,40.01+randomNumber); - PointRDD pointRDD = new PointRDD(sc, inputLocation, offset, splitter); - pointRDD.rawPointRDD.persist(StorageLevel.MEMORY_ONLY()); - for(int i=0;i-1; - } - } + public static void testSpatialRangeQueryUsingIndex() throws Exception { Random random=new Random(); @@ -245,13 +199,7 @@ public static void testSpatialJoinQuery() throws Exception { RectangleRDD rectangleRDD = new RectangleRDD(sc, inputLocation2, offset2, splitter2); //polygonRDD.rawPolygonRDD.unpersist(); RectangleRDD objectRDD; - if(gridType=="voronoi"){ - objectRDD = new RectangleRDD(sc, inputLocation, offset ,splitter,gridType,numPartitions2); - } - else - { - objectRDD = new RectangleRDD(sc, inputLocation, offset ,splitter,gridType,numPartitions); - } + objectRDD = new RectangleRDD(sc, inputLocation, offset ,splitter,gridType,numPartitions); objectRDD.gridRectangleRDD.persist(StorageLevel.MEMORY_ONLY()); JoinQuery joinQuery = new JoinQuery(sc,objectRDD,rectangleRDD); for(int i=0;i queryWindows=objectRDD.rawRectangleRDD.collect(); - final Broadcast> queryWindowsBroadcast=sc.broadcast(queryWindows); - for(int i=0;i joinRDD=rectangleRDD.rawRectangleRDD.cartesian(pointRDD.rawPointRDD); - JavaPairRDD filteredJoinRDD=objectRDD.rawRectangleRDD.flatMapToPair(new PairFlatMapFunction() - { - - @Override - public Iterable> call(Envelope t) - throws Exception { - // TODO Auto-generated method stub - List queryWindowSet=queryWindowsBroadcast.getValue(); - Iterator queryWindowIterator=queryWindowSet.iterator(); - final HashSet> result=new HashSet>(); - while(queryWindowIterator.hasNext()) - { - Envelope window=queryWindowIterator.next(); - if(window.contains(t)) - { - result.add(new Tuple2(window,t)); - } - - } - return result; - } - - }); - queryWindowsBroadcast.destroy(); - JavaPairRDD> resultRDD=filteredJoinRDD.groupByKey(); - long resultSize=resultRDD.count(); - assert resultSize>-1; - } - } - - - public static void TearDown() { sc.stop(); }