Skip to content

Commit 8db8ed9

Browse files
dsergeevmaprekrivokonmapr
dsergeevmapr
authored and
ekrivokonmapr
committed
added configure.sh and warden.spark-thriftserver.conf
1 parent a7ed13c commit 8db8ed9

File tree

4 files changed

+374
-8
lines changed

4 files changed

+374
-8
lines changed

‎bin/configure.sh

Lines changed: 354 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,354 @@
1+
#!/usr/bin/env bash
2+
3+
#############################################################################
4+
# Copyright (c) 2009 & onwards. MapR Tech, Inc., All rights reserved
5+
#############################################################################
6+
#
7+
# Configure script for Spark
8+
#
9+
# This script is normally run by the core configure.sh to setup Spark during
10+
# install. If it is run standalone, need to correctly initialize the
11+
# variables that it normally inherits from the master configure.sh
12+
#
13+
#############################################################################
14+
# Import functions and variables from 'common-ecosystem.sh'
15+
#############################################################################
16+
#
17+
# Result codes
18+
#
19+
20+
RETURN_SUCCESS=0
21+
RETURN_ERR_MAPR_HOME=1
22+
RETURN_ERR_ARGS=2
23+
RETURN_ERR_MAPRCLUSTER=3
24+
RETURN_ERR_OTHER=4
25+
26+
#
27+
# Globals
28+
#
29+
30+
MAPR_HOME="${MAPR_HOME:-/opt/mapr}"
31+
. ${MAPR_HOME}/server/common-ecosystem.sh 2> /dev/null # prevent verbose output, set by 'set -x'
32+
if [ $? -ne 0 ]; then
33+
echo 'Error: Seems that MAPR_HOME is not correctly set or mapr-core is not installed.'
34+
exit 1
35+
fi 2> /dev/null
36+
{ set +x; } 2>/dev/null
37+
38+
initCfgEnv
39+
40+
MAPR_CONF_DIR=${MAPR_CONF_DIR:-"$MAPR_HOME/conf"}
41+
SPARK_VERSION="2.1.0"
42+
SPARK_HOME="$MAPR_HOME"/spark/spark-"$SPARK_VERSION"
43+
SPARK_BIN="$SPARK_HOME"/bin
44+
SPARK_LOGS="$SPARK_HOME"/logs
45+
DAEMON_CONF=${MAPR_HOME}/conf/daemon.conf
46+
47+
if [ -f $MAPR_HOME/MapRBuildVersion ]; then
48+
MAPR_MIN_VERSION=4.0
49+
MAPR_VERSION=`cat $MAPR_HOME/MapRBuildVersion | awk -F "." '{print $1"."$2}'`
50+
51+
#
52+
# If the MapR release >=4.0 (yarn beta) returns boolean 1, else returns boolean 0
53+
#
54+
if [ $(echo | awk -v cur=$MAPR_VERSION -v min=$MAPR_MIN_VERSION '{if (cur >= min) printf("1"); else printf ("0");}') -eq 0 ]; then
55+
rm -f "$SPARK_HOME"/lib/spark*hadoop2*.jar
56+
else
57+
rm -f "$SPARK_HOME"/lib/spark*hadoop1*.jar
58+
fi
59+
fi
60+
61+
ln -sfn "$SPARK_HOME" /usr/local/spark
62+
63+
#
64+
# Make the logs directory rwx, and set the sticky bit.
65+
#
66+
mkdir -p "$SPARK_HOME/logs"
67+
chmod a+rwxt "$SPARK_HOME"/logs
68+
69+
#
70+
# create tmp directory with rwx, and set the sticky bit.
71+
#
72+
mkdir -p "$SPARK_HOME/tmp"
73+
chmod a+rwxt "$SPARK_HOME"/tmp
74+
75+
#
76+
# Improved default logging level (WARN instead of INFO)
77+
#
78+
sed 's/rootCategory=INFO/rootCategory=WARN/' "$SPARK_HOME/conf/log4j.properties.template" > "$SPARK_HOME/conf/log4j.properties"
79+
80+
#
81+
# Add MapR customization to spark
82+
#
83+
if [ ! -e "$SPARK_HOME"/conf/spark-env.sh ]; then
84+
cp "$SPARK_HOME"/conf/spark-env.sh.template "$SPARK_HOME"/conf/spark-env.sh
85+
fi
86+
cat >> "$SPARK_HOME"/conf/spark-env.sh << EOM
87+
88+
89+
#########################################################################################################
90+
# Set MapR attributes and compute classpath
91+
#########################################################################################################
92+
93+
# Set the spark attributes
94+
if [ -d "$SPARK_HOME" ]; then
95+
export SPARK_HOME=$SPARK_HOME
96+
fi
97+
98+
# Load the hadoop version attributes
99+
source $SPARK_HOME/mapr-util/hadoop-version-picker.sh
100+
export HADOOP_HOME=\$hadoop_home_dir
101+
export HADOOP_CONF_DIR=\$hadoop_conf_dir
102+
103+
# Enable mapr impersonation
104+
export MAPR_IMPERSONATION_ENABLED=1
105+
106+
MAPR_HADOOP_CLASSPATH=\`mapr classpath\`
107+
MAPR_HADOOP_JNI_PATH=\`hadoop jnipath\`
108+
MAPR_SPARK_CLASSPATH="\$MAPR_HADOOP_CLASSPATH"
109+
110+
SPARK_MAPR_HOME=$MAPR_HOME
111+
112+
export SPARK_LIBRARY_PATH=\$MAPR_HADOOP_JNI_PATH
113+
export LD_LIBRARY_PATH="\$MAPR_HADOOP_JNI_PATH:\$LD_LIBRARY_PATH"
114+
115+
# Load the classpath generator script
116+
source $SPARK_HOME/mapr-util/generate-classpath.sh
117+
118+
# Calculate hive jars to include in classpath
119+
generate_compatible_classpath "spark" "$SPARK_VERSION" "hive"
120+
MAPR_HIVE_CLASSPATH=\${generated_classpath}
121+
if [ ! -z "\$MAPR_HIVE_CLASSPATH" ]; then
122+
MAPR_SPARK_CLASSPATH="\$MAPR_SPARK_CLASSPATH:\$MAPR_HIVE_CLASSPATH"
123+
fi
124+
125+
# Calculate hbase jars to include in classpath
126+
generate_compatible_classpath "spark" "$SPARK_VERSION" "hbase"
127+
MAPR_HBASE_CLASSPATH=\${generated_classpath}
128+
if [ ! -z "\$MAPR_HBASE_CLASSPATH" ]; then
129+
MAPR_SPARK_CLASSPATH="\$MAPR_SPARK_CLASSPATH:\$MAPR_HBASE_CLASSPATH"
130+
SPARK_SUBMIT_OPTS="\$SPARK_SUBMIT_OPTS -Dspark.driver.extraClassPath=\$MAPR_HBASE_CLASSPATH"
131+
fi
132+
133+
# Set executor classpath for MESOS. Uncomment following string if you want deploy spark jobs on Mesos
134+
#MAPR_MESOS_CLASSPATH=\$MAPR_SPARK_CLASSPATH
135+
SPARK_SUBMIT_OPTS="\$SPARK_SUBMIT_OPTS -Dspark.executor.extraClassPath=\$MAPR_HBASE_CLASSPATH:\$MAPR_MESOS_CLASSPATH"
136+
137+
# Set SPARK_DIST_CLASSPATH
138+
export SPARK_DIST_CLASSPATH=\$MAPR_SPARK_CLASSPATH
139+
140+
# Security status
141+
source $MAPR_HOME/conf/env.sh
142+
if [ "\$MAPR_SECURITY_STATUS" = "true" ]; then
143+
SPARK_SUBMIT_OPTS="\$SPARK_SUBMIT_OPTS -Dhadoop.login=hybrid -Dmapr_sec_enabled=true"
144+
fi
145+
146+
# scala
147+
export SCALA_VERSION=2.11
148+
export SPARK_SCALA_VERSION=\$SCALA_VERSION
149+
export SCALA_HOME=$SPARK_HOME/scala
150+
export SCALA_LIBRARY_PATH=\$SCALA_HOME/lib
151+
152+
# Use a fixed identifier for pid files
153+
export SPARK_IDENT_STRING="mapr"
154+
155+
#########################################################################################################
156+
# :::CAUTION::: DO NOT EDIT ANYTHING ON OR ABOVE THIS LINE
157+
#########################################################################################################
158+
159+
160+
#
161+
# MASTER HA SETTINGS
162+
#
163+
#export SPARK_DAEMON_JAVA_OPTS="-Dspark.deploy.recoveryMode=ZOOKEEPER -Dspark.deploy.zookeeper.url=<zookeerper1:5181,zookeeper2:5181,..> -Djava.security.auth.login.config=/opt/mapr/conf/mapr.login.conf -Dzookeeper.sasl.client=false"
164+
165+
166+
# MEMORY SETTINGS
167+
export SPARK_DAEMON_MEMORY=1g
168+
export SPARK_WORKER_MEMORY=16g
169+
170+
# Worker Directory
171+
export SPARK_WORKER_DIR=\$SPARK_HOME/tmp
172+
173+
# Environment variable for printing spark command everytime you run spark.Set to "1" to print.
174+
# export SPARK_PRINT_LAUNCH_COMMAND=1
175+
176+
177+
EOM
178+
179+
source $MAPR_HOME/conf/env.sh
180+
if [ "$MAPR_SECURITY_STATUS" = "true" ]; then
181+
sed -i '/# Security/,/# EndOfSecurityConfiguration/d' "$SPARK_HOME"/conf/spark-defaults.conf
182+
cat >> "$SPARK_HOME"/conf/spark-defaults.conf << EOM
183+
184+
# Security
185+
# - ACLS
186+
spark.acls.enable true
187+
spark.admin.acls mapr
188+
spark.admin.acls.groups mapr
189+
# - Authorization and Network Encryption
190+
spark.authenticate true
191+
# - - This secret will be used only by local/standalone modes. YARN will override this with its own secret
192+
spark.authenticate.secret changeMe
193+
spark.authenticate.enableSaslEncryption true
194+
spark.network.sasl.serverAlwaysEncrypt true
195+
# - IO Encryption
196+
spark.io.encryption.enabled true
197+
spark.io.encryption.keySizeBits 128
198+
# EndOfSecurityConfiguration
199+
EOM
200+
fi
201+
202+
#
203+
# If the spark version is greater than 1.0, remove
204+
# the any shark directory that is left over. Otherwise,
205+
# add MapR customization of shark.
206+
#
207+
SPARK_MIN_VERSION=1.0
208+
if [ $(echo | awk -v cur=$SPARK_VERSION -v min=$SPARK_MIN_VERSION '{if (cur >= min) printf("1"); else printf ("0");}') -eq 0 ]; then
209+
MAPR_SHARK_HOME=$(dir -d -1 $MAPR_HOME/shark/* 2> /dev/null | head -1)
210+
cp $MAPR_SHARK_HOME/conf/shark-env.sh.template $MAPR_SHARK_HOME/conf/shark-env.sh
211+
212+
cat >> $MAPR_SHARK_HOME/conf/shark-env.sh << EOM
213+
214+
# Load the hadoop version attributes
215+
source $SPARK_HOME/mapr-util/hadoop-version-picker.sh
216+
export HADOOP_HOME=\$hadoop_home_dir
217+
#export MASTER=SET MASTER URL(eg: spark://master-hostname:7077)
218+
export SPARK_HOME=$SPARK_HOME
219+
#export HIVE_CONF_DIR=
220+
221+
source \$SPARK_HOME/conf/spark-env.sh
222+
223+
EOM
224+
else
225+
rm -rf $MAPR_HOME/shark
226+
fi
227+
228+
#####################################
229+
# Functions warden/permission
230+
#####################################
231+
232+
#
233+
# Change permission
234+
#
235+
236+
function change_permissions() {
237+
if [ -f $DAEMON_CONF ]; then
238+
MAPR_USER=$( awk -F = '$1 == "mapr.daemon.user" { print $2 }' $DAEMON_CONF)
239+
MAPR_GROUP=$( awk -F = '$1 == "mapr.daemon.group" { print $2 }' $DAEMON_CONF)
240+
241+
if [ ! -z "$MAPR_USER" ]; then
242+
chown -R ${MAPR_USER} ${SPARK_HOME}
243+
fi
244+
245+
if [ ! -z "$MAPR_GROUP" ]; then
246+
chgrp -R ${MAPR_GROUP} ${SPARK_HOME}
247+
fi
248+
chmod -f u+x $SPARK_HOME/bin/*
249+
fi
250+
}
251+
252+
#
253+
# Add warden files
254+
#
255+
256+
function installWardenConfFile() {
257+
if checkNetworkPortAvailability 8080 2>/dev/null; then
258+
{ set +x; } 2>/dev/null
259+
#Register port for spark master
260+
registerNetworkPort spark_master 8080 2>/dev/null
261+
262+
cp "${SPARK_HOME}/warden/warden.spark-master.conf" "${MAPR_CONF_DIR}/conf.d/" 2>/dev/null || :
263+
logInfo 'Warden conf for Spark-master copied.'
264+
else
265+
{ set +x; } 2>/dev/null
266+
logErr 'Spark-master cannot start because its ports already has been taken.'
267+
exit $RETURN_ERR_MAPRCLUSTER
268+
fi
269+
270+
if checkNetworkPortAvailability 18080 2>/dev/null; then
271+
#Register port for spark historyserver
272+
{ set +x; } 2>/dev/null
273+
registerNetworkPort spark_historyserver 18080 2>/dev/null
274+
275+
cp "${SPARK_HOME}/warden/warden.spark-historyserver.conf" "${MAPR_CONF_DIR}/conf.d/" 2>/dev/null || :
276+
logInfo 'Warden conf for Spark-historyserver copied.'
277+
else
278+
{ set +x; } 2>/dev/null
279+
logErr 'Spark-historyserver cannot start because its ports already has been taken.'
280+
exit $RETURN_ERR_MAPRCLUSTER
281+
fi
282+
283+
284+
if checkNetworkPortAvailability 4040 2>/dev/null; then
285+
#Register port for spark thriftserver
286+
{ set +x; } 2>/dev/null
287+
registerNetworkPort spark_thriftserver 4040 2>/dev/null
288+
289+
cp "${SPARK_HOME}/warden/warden.spark-thriftserver.conf" "${MAPR_CONF_DIR}/conf.d/" 2>/dev/null || :
290+
logInfo 'Warden conf for Spark-thriftserver copied.'
291+
else
292+
{ set +x; } 2>/dev/null
293+
logErr 'Spark-thriftserver cannot start because its ports already has been taken.'
294+
exit $RETURN_ERR_MAPRCLUSTER
295+
fi
296+
}
297+
298+
function stopServicesForRestartByWarden() {
299+
#Stop spark master
300+
if [ -e ${MAPR_CONF_DIR}/conf.d/warden.spark-master.conf ]; then
301+
${SPARK_HOME}/sbin/stop-master.sh
302+
fi
303+
304+
#Stop spark historyserver
305+
if [ -e ${MAPR_CONF_DIR}/conf.d/warden.spark-historyserver.conf ]; then
306+
${SPARK_HOME}/sbin/stop-history-server.sh
307+
fi
308+
309+
#Stop spark thriftserver
310+
if [ -e ${MAPR_CONF_DIR}/conf.d/warden.spark-thriftserver.conf ]; then
311+
${SPARK_HOME}/sbin/stop-thriftserver.sh
312+
fi
313+
}
314+
#
315+
# Parse options
316+
#
317+
318+
USAGE="usage: $0 [-h] [-R] [-secure] [-unsecure]"
319+
320+
OPTS=`getopt -n "$0" -a -o h -l R -l EC: -l secure -l unsecure -- "$@"`
321+
322+
if [ $? != 0 ] || [ ${#} -lt 1 ] ; then
323+
echo "${USAGE}"
324+
exit $RETURN_ERR_ARGS
325+
fi
326+
327+
eval set -- "$OPTS"
328+
329+
for i ; do
330+
case "$i" in
331+
--secure)
332+
isSecure=1;
333+
shift 1;;
334+
--unsecure)
335+
isSecure=0;
336+
shift 1;;
337+
--help)
338+
echo "${USAGE}"
339+
exit $RETURN_SUCCESS
340+
;;
341+
--)
342+
shift;;
343+
*)
344+
# Invalid arguments passed
345+
echo "${USAGE}"
346+
exit $RETURN_ERR_ARGS
347+
esac
348+
done
349+
350+
change_permissions
351+
installWardenConfFile
352+
stopServicesForRestartByWarden
353+
354+
exit $RETURN_SUCCESS

‎ext-conf/warden.spark-historyserver.conf

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
services=spark-historyserver:1:cldb
22
service.displayname=SparkHistoryServer
3-
service.command.start=/opt/mapr/spark/spark-__VERSION__/sbin/spark-daemon.sh start org.apache.spark.deploy.history.HistoryServer 1
4-
service.command.stop=/opt/mapr/spark/spark-__VERSION__/sbin/spark-daemon.sh stop org.apache.spark.deploy.history.HistoryServer 1
5-
service.command.monitorcommand=/opt/mapr/spark/spark-__VERSION__/sbin/spark-daemon.sh status org.apache.spark.deploy.history.HistoryServer 1
3+
service.command.start=/opt/mapr/spark/spark-2.1.0/sbin/spark-daemon.sh start org.apache.spark.deploy.history.HistoryServer 1
4+
service.command.stop=/opt/mapr/spark/spark-2.1.0/sbin/spark-daemon.sh stop org.apache.spark.deploy.history.HistoryServer 1
5+
service.command.monitorcommand=/opt/mapr/spark/spark-2.1.0/sbin/spark-daemon.sh status org.apache.spark.deploy.history.HistoryServer 1
66
service.command.type=BACKGROUND
77
service.ui.port=18080
88
service.uri=/
9-
service.logs.location=/opt/mapr/spark/spark-__VERSION__/logs/
9+
service.logs.location=/opt/mapr/spark/spark-2.1.0/logs/
1010
service.process.type=JAVA
1111
service.alarm.tersename=nasparkh
1212
service.alarm.label=SPARKHISTORYDOWNALARM

‎ext-conf/warden.spark-master.conf

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
services=spark-master:all:cldb
22
service.displayname=SparkMaster
3-
service.command.start=/opt/mapr/spark/spark-__VERSION__/sbin/spark-daemon.sh start org.apache.spark.deploy.master.Master 1
4-
service.command.stop=/opt/mapr/spark/spark-__VERSION__/sbin/spark-daemon.sh stop org.apache.spark.deploy.master.Master 1
5-
service.command.monitorcommand=/opt/mapr/spark/spark-__VERSION__/sbin/spark-daemon.sh status org.apache.spark.deploy.master.Master 1
3+
service.command.start=/opt/mapr/spark/spark-2.1.0/sbin/spark-daemon.sh start org.apache.spark.deploy.master.Master 1
4+
service.command.stop=/opt/mapr/spark/spark-2.1.0/sbin/spark-daemon.sh stop org.apache.spark.deploy.master.Master 1
5+
service.command.monitorcommand=/opt/mapr/spark/spark-2.1.0/sbin/spark-daemon.sh status org.apache.spark.deploy.master.Master 1
66
service.command.type=BACKGROUND
77
service.port=7077
88
service.ui.port=8080
99
service.uri=/
10-
service.logs.location=/opt/mapr/spark/spark-__VERSION__/logs/
10+
service.logs.location=/opt/mapr/spark/spark-2.1.0/logs/
1111
service.process.type=JAVA
1212
service.alarm.tersename=naspark
1313
service.alarm.label=SPARKDOWNALARM

0 commit comments

Comments
 (0)