I am trying to run Spark and SparkSQL job from Alibaba E-MapReduce workflow console.
I am trying to run following command:
--master yarn-client --driver-memory 7g --num-executors 10 --executor-memory 5g --executor-cores 1 --jars ossref://emr/checklist/jars/emr-core-0.1.0.jar ossref://emr/checklist/python/wordcount.py oss://emr/checklist/data/kddb 5 32
Getting the following error:
Tue Jun 18 10:48:58 CST 2019 [LocalJobLauncherAM] INFO Starting emr flow launcher am ... =================PRINT RUNTIME ENV BEGIN=================
== System Properties ============
java.runtime.name=OpenJDK Runtime Environment
sun.boot.library.path=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.151-1.b12.el7_4.x86_64/jre/lib/amd64 java.vm.version=25.151-b12 java.vm.vendor=Oracle Corporation
java.vendor.url=http://java.oracle.com/ path.separator=:
java.vm.name=OpenJDK 64-Bit Server VM file.encoding.pkg=sun.io
user.country=US sun.java.launcher=SUN_STANDARD
sun.os.patch.level=unknown java.vm.specification.name=Java Virtual Machine Specification
user.dir=/mnt/disk1/flow-agent/local-rm/LocalApplication_1560824463304_2/container_1560824463304_2_01_000001 java.runtime.version=1.8.0_151-b12
java.awt.graphicsenv=sun.awt.X11GraphicsEnvironment
java.endorsed.dirs=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.151-1.b12.el7_4.x86_64/jre/lib/endorsed os.arch=amd64 java.io.tmpdir=tmp line.separator=java.vm.specification.vendor=Oracle Corporation os.name=Linux
sun.jnu.encoding=UTF-8 emr.flow.user=hadoop
java.library.path=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.151-1.b12.el7_4.x86_64/jre/lib/amd64/server:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.151-1.b12.el7_4.x86_64/jre/lib/amd64:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.151-1.b12.el7_4.x86_64/jre/../lib/amd64:/lib64:/usr/lib/hadoop-current/lib/native/::/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server:/usr/lib/hadoop-current/lib/native:/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib java.specification.name=Java Platform API Specification
java.class.version=52.0 sun.management.compiler=HotSpot 64-Bit Tiered Compilers os.version=3.10.0-693.2.2.el7.x86_64
user.home=/home/hadoop user.timezone=Asia/Shanghai
java.awt.printerjob=sun.print.PSPrinterJob file.encoding=UTF-8
java.specification.version=1.8 flow.job.id=FJI-F6D6115A3E436AAC_0
java.class.path=launcher.jar user.name=hadoop
flow.job.result.local.dir=/mnt/disk1/flow-agent/job-results
flow.job.launcher.class.name=com.aliyun.emr.flow.agent.jobs.launcher.impl.SparkSqlJobLauncherImpl java.vm.specification.version=1.8
sun.java.command=com.aliyun.emr.flow.agent.jobs.launcher.local.LocalJobLauncherAM java.home=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.151-1.b12.el7_4.x86_64/jre sun.arch.data.model=64 user.language=en
java.specification.vendor=Oracle Corporation
awt.toolkit=sun.awt.X11.XToolkit java.vm.info=mixed mode
java.version=1.8.0_151
java.ext.dirs=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.151-1.b12.el7_4.x86_64/jre/lib/ext:/usr/java/packages/lib/ext sun.boot.class.path=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.151-1.b12.el7_4.x86_64/jre/lib/resources.jar:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.151-1.b12.el7_4.x86_64/jre/lib/rt.jar:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.151-1.b12.el7_4.x86_64/jre/lib/sunrsasign.jar:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.151-1.b12.el7_4.x86_64/jre/lib/jsse.jar:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.151-1.b12.el7_4.x86_64/jre/lib/jce.jar:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.151-1.b12.el7_4.x86_64/jre/lib/charsets.jar:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.151-1.b12.el7_4.x86_64/jre/lib/jfr.jar:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.151-1.b12.el7_4.x86_64/jre/classes java.vendor=Oracle Corporation file.separator=/
java.vendor.url.bug=http://bugreport.sun.com/bugreport/
flow.job.meta.data.path=job.metadata
sun.io.unicode.encoding=UnicodeLittle sun.cpu.endian=little
sun.cpu.isalist=== System Env ===================
PATH=/usr/lib/spark-current/bin:/usr/lib/analytics-zoo/bin:/usr/lib/anaconda/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/usr/lib/flow-agent-current/bin:/usr/lib/hadoop-current/bin:/usr/lib/hadoop-current/sbin:/usr/lib/hadoop-current/bin:/usr/lib/hadoop-current/sbin:/usr/lib/hadoop-current/bin:/usr/lib/hadoop-current/sbin:/home/hadoop/.local/bin:/home/hadoop/bin HADOOP_CONF_DIR=/etc/ecm/hadoop-conf HISTCONTROL=ignoredups
FLOW_AGENT_HOME=/usr/lib/flow-agent-current
JAVA_LIBRARY_PATH=/usr/lib/hadoop-current/lib/native:/usr/lib/hadoop-current/lib/native: HISTSIZE=1000 JAVA_HOME=/usr/lib/jvm/java-1.8.0
ZOOCFGDIR=/etc/ecm/zookeeper-conf TERM=unknown
XFILESEARCHPATH=/usr/dt/app-defaults/%L/Dt
HADOOP_PID_DIR=/usr/lib/hadoop-current/pids LANG=en_US.UTF-8
XDG_SESSION_ID=c76
HADOOP_CLASSPATH=/opt/apps/extra-jars/:/usr/lib/spark-current/yarn/spark-2.3.2-yarn-shuffle.jar:/opt/apps/extra-jars/ MAIL=/var/spool/mail/hadoop SPARK_HOME=/usr/lib/spark-current
ANALYTICS_ZOO_HOME=/usr/lib/analytics-zoo
LD_LIBRARY_PATH=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.151-1.b12.el7_4.x86_64/jre/lib/amd64/server:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.151-1.b12.el7_4.x86_64/jre/lib/amd64:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.151-1.b12.el7_4.x86_64/jre/../lib/amd64:/lib64:/usr/lib/hadoop-current/lib/native/::/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server:/usr/lib/hadoop-current/lib/native YARN_LOG_DIR=/var/log/hadoop-yarn JVMFLAGS= -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=128M -Xloggc:/mnt/disk1/log/zookeeper/zookeeper-gc.log -javaagent:/var/lib/ecm-agent/data/jmxetric-1.0.8.jar=host=localhost,port=8649,mode=unicast,wireformat31x=true,process=ZOOKEEPER_ZOOKEEPER,config=/var/lib/ecm-agent/data/jmxetric.xml HADOOP_HDFS_HOME=/usr/lib/hadoop-current LOGNAME=hadoop
PWD=/mnt/disk1/flow-agent/local-rm/LocalApplication_1560824463304_2/container_1560824463304_2_01_000001 _=/usr/lib/jvm/java-1.8.0/bin/java LESSOPEN=||/usr/bin/lesspipe.sh %s SHELL=/bin/bash ANACONDA_HOME=/usr/lib/anaconda
OLDPWD=/mnt/disk1/flow-agent/local-rm/LocalApplication_1560824463304_2/container_1560824463304_2_01_000001 USER=hadoop YARN_PID_DIR=/usr/lib/hadoop-current/pids
HADOOP_MAPRED_PID_DIR=/usr/lib/hadoop-current/pids
SPARK_CONF_DIR=/etc/ecm/spark-conf
HOSTNAME=emr-header-1.cluster-121550
SPARK_PID_DIR=/usr/lib/spark-current/pids
ZOO_LOG_DIR=/mnt/disk1/log/zookeeper
NLSPATH=/usr/dt/lib/nls/msg/%L/%N.cat
HADOOP_MAPRED_LOG_DIR=/var/log/hadoop-mapred
HADOOP_HOME=/usr/lib/hadoop-current
HADOOP_LOG_DIR=/var/log/hadoop-hdfs
FLOW_AGENT_CONF_DIR=/etc/ecm/flow-agent-conf HOME=/home/hadoop
SHLVL=3 ZOOKEEPER_HOME=/usr/lib/zookeeper-current
SPARK_LOG_DIR=/mnt/disk1/log/spark== System Arguments =============
[] =================PRINT RUNTIME ENV END================= Tue Jun 18 10:48:58 CST 2019 [LocalJobLauncherAM] ERROR Job Launcher ClassName is not set, quit.
