Skip to content

Commit

Permalink
Set version to 2.1.0 in script templates
Browse files Browse the repository at this point in the history
  • Loading branch information
jtnystrom committed Oct 22, 2021
1 parent 12cdfaf commit e8c265b
Show file tree
Hide file tree
Showing 4 changed files with 5 additions and 5 deletions.
2 changes: 1 addition & 1 deletion spark-shell.sh.template
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,4 @@ SPARK=/set/spark/dir
exec $SPARK/bin/spark-shell \
--conf spark.driver.maxResultSize=2g \
--master $MASTER \
--jars target/scala-2.12/Discount-assembly-2.1.0-SNAPSHOT.jar
--jars target/scala-2.12/Discount-assembly-2.1.0.jar
2 changes: 1 addition & 1 deletion spark-submit.sh.template
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,4 @@ PARTITIONS="spark.sql.shuffle.partitions=400"
exec $SPARK/bin/spark-submit \
--conf spark.driver.maxResultSize=2g \
--master $MASTER \
--class com.jnpersson.discount.spark.Discount target/scala-2.12/Discount-assembly-2.1.0-SNAPSHOT.jar $*
--class com.jnpersson.discount.spark.Discount target/scala-2.12/Discount-assembly-2.1.0.jar $*
4 changes: 2 additions & 2 deletions submit-aws.sh.template
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ BUCKET=s3://my-bucket/discount
#Change 2.12 to 2.11 below if compiling for scala 2.11.

#Copy jars and data files the first time only, after which the following lines can safely be commented out
aws s3 cp target/scala-2.12/Discount-assembly-2.1.0-SNAPSHOT.jar $BUCKET/
aws s3 cp target/scala-2.12/Discount-assembly-2.1.0.jar $BUCKET/

for f in PASHA/pasha_all*txt
do
Expand All @@ -32,7 +32,7 @@ PARTITIONS=##spark.sql.shuffle.partitions=4000

COMMAND=( --conf $SPLIT \
--conf $PARTITIONS \
--class com.jnpersson.discount.spark.Discount $BUCKET/Discount-assembly-2.1.0-SNAPSHOT.jar $*)
--class com.jnpersson.discount.spark.Discount $BUCKET/Discount-assembly-2.1.0.jar $*)

#Turn off paging for output
export AWS_PAGER=""
Expand Down
2 changes: 1 addition & 1 deletion submit-gcloud.sh.template
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,6 @@ PROPERTIES="$PARTITIONS,$MAXRES,$SPLIT"

#Change 2.12 to 2.11 below if compiling for scala 2.11.
exec gcloud --verbosity=info dataproc jobs submit spark --region $REGION --cluster $CLUSTER \
--class com.jnpersson.discount.spark.Discount --jars target/scala-2.12/Discount-assembly-2.1.0-SNAPSHOT.jar \
--class com.jnpersson.discount.spark.Discount --jars target/scala-2.12/Discount-assembly-2.1.0.jar \
--properties $PROPERTIES -- "$@"

0 comments on commit e8c265b

Please sign in to comment.