diff --git a/spark-submit.sh.template b/spark-submit.sh.template index 5f535d1e..aae82f7d 100755 --- a/spark-submit.sh.template +++ b/spark-submit.sh.template @@ -23,4 +23,4 @@ exec $SPARK/bin/spark-submit \ --packages org.rogach:scallop_2.11:latest.integration \ --jars lib/fastdoop-1.0.0.jar \ --master $MASTER \ - --class discount.spark.Discount target/scala-2.11/discount_2.11-1.1.0.jar $* + --class discount.spark.Discount target/scala-2.11/discount_2.11-1.2.0.jar $* diff --git a/submit-aws.sh.template b/submit-aws.sh.template index dd75dbfc..f87319bc 100755 --- a/submit-aws.sh.template +++ b/submit-aws.sh.template @@ -16,7 +16,7 @@ BUCKET=s3://my-bucket/discount #Copy jars and data files the first time only, after which the following lines can safely be commented out aws s3 cp lib/fastdoop-1.0.0.jar $BUCKET/ aws s3 cp PASHA/pasha_all_*.txt $BUCKET/PASHA/ -aws s3 cp target/scala-2.11/discount_2.11-1.1.0.jar $BUCKET/ +aws s3 cp target/scala-2.11/discount_2.11-1.2.0.jar $BUCKET/ #Max size of input splits in bytes. A smaller number reduces memory usage but increases the number of #partitions for the first stage. If this variable is unset, Spark's default of 128 MB will be used. @@ -30,7 +30,7 @@ PARTITIONS=##spark.sql.shuffle.partitions=4000 COMMAND=(--packages org.rogach:scallop_2.11:latest.integration \ --conf $SPLIT \ --conf $PARTITIONS \ - --jars $BUCKET/fastdoop-1.0.0.jar --class discount.spark.Discount $BUCKET/discount_2.11-1.1.0.jar $*) + --jars $BUCKET/fastdoop-1.0.0.jar --class discount.spark.Discount $BUCKET/discount_2.11-1.2.0.jar $*) RUNNER_ARGS="spark-submit" for PARAM in ${COMMAND[@]} diff --git a/submit-gcloud.sh.template b/submit-gcloud.sh.template index e2130dc9..53d9f7ee 100755 --- a/submit-gcloud.sh.template +++ b/submit-gcloud.sh.template @@ -43,6 +43,6 @@ SPLIT=##spark.hadoop.mapreduce.input.fileinputformat.split.maxsize=$((64 * 1024 PROPERTIES="^##^spark.jars.packages=org.rogach:scallop_2.11:latest.integration$PARTITIONS$MAXRES$OVERHEAD$EXECMEM$SPLIT" exec gcloud --verbosity=info dataproc jobs submit spark --region $REGION --cluster $CLUSTER \ - --class discount.spark.Discount --jars target/scala-2.11/discount_2.11-1.1.0.jar,lib/fastdoop-1.0.0.jar \ + --class discount.spark.Discount --jars target/scala-2.11/discount_2.11-1.2.0.jar,lib/fastdoop-1.0.0.jar \ --properties $PROPERTIES -- "$@"