I'm trying to use cgroup on yarn.
but I have some trouble.
I use 1 namenod and 3 datanodes.
I can't run container-executor.
I think I do something wrong set property, but I don't have any idea
could you help me?? If you need more information I will write comment.
yarn-site.xml
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<configuration>
<property>
<name>yarn.resourcemanager.scheduler.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
</property>
<property>
<name>yarn.scheduler.fair.allocation.file</name>
<value>/home/hadoop/tools/hadoop/etc/hadoop/fair-scheduler.xml</value>
</property>
<property>
<name>yarn.resourcemanager.hostname</name>
<value>em-name01</value>
</property>
<property>
<name>yarn.nodemanager.resource.memory-mb</name>
<value>8192</value>
</property>
<property>
<name>yarn.nodemanager.resource.cpu-vcores</name>
<value>4</value>
</property>
<property>
<name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
<value>100</value>
</property>
<property>
<name>yarn.scheduler.fair.allow-undeclared-pools</name>
<value>false</value>
</property>
<property>
<name>yarn.scheduler.fair.preemption</name>
<value>false</value>
</property>
<!-- cgroup config -->
<property>
<name>yarn.nodemanager.container-executor.class</name>
<value>org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor</value>
</property>
<property>
<name>yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users</name>
<value>false</value>
</property>
<property>
<name>yarn.nodemanager.linux-container-executor.resources-handler.class</name>
<value>org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler</value>
</property>
<property>
<name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name>
<value>/hadoop-yarn</value>
</property>
<property>
<name>yarn.nodemanager.linux-container-executor.cgroups.mount</name>
<value>true</value>
</property>
<property>
<name>yarn.nodemanager.linux-container-executor.cgroups.mount-path</name>
<value>/cgroup/cpu</value>
</property>
<property>
<name>yarn.nodemanager.linux-container-executor.group</name>
<value>hadoop</value>
</property>
<!--
-->
</configuration>
container-executor.cfg
yarn.nodemanager.linux-container-executor.group=hadoop#configured value of yarn.nodemanager.linux-container-executor.group yarn.nodemanager.local-dirs=/tmp/hadoop-hadoop/nm-local-dir banned.users=#comma separated list of users who can not run applications min.user.id=500#Prevent other super-users allowed.system.users=hadoop##comma separated list of system users who CAN run applications
container-executor
[hadoop@em-name01 hadoop]$ ll $HADOOP_HOME/bin/
---Sr-s--- 1 root hadoop 160351 Jan 26 00:20 container-executor
[hadoop@em-name01 hadoop]$ ll container-executor.cfg
-rw-r--r-- 1 root hadoop 389 May 27 02:38 container-executor.cfg
yarn-hadoop-nodemanager-em-data02.log (All datanodes have same log)
************************************************************/ 2016-05-27 10:45:31,586 INFO org.apache.hadoop.yarn.server.nodemanager.NodeManager: registered UNIX signal handlers for [TERM, HUP, INT] 2016-05-27 10:45:32,826 WARN org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor: yarn.nodemanager.lin ux-container-executor.nonsecure-mode.limit-users: impersonation without authentication enabled 2016-05-27 10:45:32,830 WARN org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor: Exit code from conta iner executor initialization is : 24 ExitCodeException exitCode=24: File /home/hadoop/tools/hadoop-2.7.2/etc/hadoop must be owned by root, but is owned by 500 at org.apache.hadoop.util.Shell.runCommand(Shell.java:545) at org.apache.hadoop.util.Shell.run(Shell.java:456) at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:722) at org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor.init(LinuxContainerExecutor.java:185) at org.apache.hadoop.yarn.server.nodemanager.NodeManager.serviceInit(NodeManager.java:216) at org.apache.hadoop.service.AbstractService.init(AbstractService.java:163) at org.apache.hadoop.yarn.server.nodemanager.NodeManager.initAndStartNodeManager(NodeManager.java:485) at org.apache.hadoop.yarn.server.nodemanager.NodeManager.main(NodeManager.java:533) 2016-05-27 10:45:32,832 INFO org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor: 2016-05-27 10:45:32,832 INFO org.apache.hadoop.service.AbstractService: Service NodeManager failed in state INITED; cause: org.apache.hadoop.yarn.exceptions.YarnRuntimeException: Failed to initialize container executor org.apache.hadoop.yarn.exceptions.YarnRuntimeException: Failed to initialize container executor at org.apache.hadoop.yarn.server.nodemanager.NodeManager.serviceInit(NodeManager.java:218) at org.apache.hadoop.service.AbstractService.init(AbstractService.java:163) at org.apache.hadoop.yarn.server.nodemanager.NodeManager.initAndStartNodeManager(NodeManager.java:485) at org.apache.hadoop.yarn.server.nodemanager.NodeManager.main(NodeManager.java:533) Caused by: java.io.IOException: Linux container executor not configured properly (error=24) at org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor.init(LinuxContainerExecutor.java:191) at org.apache.hadoop.yarn.server.nodemanager.NodeManager.serviceInit(NodeManager.java:216) ... 3 more Caused by: ExitCodeException exitCode=24: File /home/hadoop/tools/hadoop-2.7.2/etc/hadoop must be owned by root, but is owned by 500 at org.apache.hadoop.util.Shell.runCommand(Shell.java:545) at org.apache.hadoop.util.Shell.run(Shell.java:456) at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:722) at org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor.init(LinuxContainerExecutor.java:185) ... 4 more 2016-05-27 10:45:32,834 FATAL org.apache.hadoop.yarn.server.nodemanager.NodeManager: Error starting NodeManager org.apache.hadoop.yarn.exceptions.YarnRuntimeException: Failed to initialize container executor at org.apache.hadoop.yarn.server.nodemanager.NodeManager.serviceInit(NodeManager.java:218) at org.apache.hadoop.service.AbstractService.init(AbstractService.java:163) at org.apache.hadoop.yarn.server.nodemanager.NodeManager.initAndStartNodeManager(NodeManager.java:485) at org.apache.hadoop.yarn.server.nodemanager.NodeManager.main(NodeManager.java:533) Caused by: java.io.IOException: Linux container executor not configured properly (error=24) at org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor.init(LinuxContainerExecutor.java:191) at org.apache.hadoop.yarn.server.nodemanager.NodeManager.serviceInit(NodeManager.java:216) ... 3 more Caused by: ExitCodeException exitCode=24: File /home/hadoop/tools/hadoop-2.7.2/etc/hadoop must be owned by root, but is owned by 500 at org.apache.hadoop.util.Shell.runCommand(Shell.java:545) at org.apache.hadoop.util.Shell.run(Shell.java:456) at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:722) at org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor.init(LinuxContainerExecutor.java:185) ... 4 more 2016-05-27 10:45:32,838 INFO org.apache.hadoop.yarn.server.nodemanager.NodeManager: SHUTDOWN_MSG: /************************************************************ SHUTDOWN_MSG: Shutting down NodeManager at em-data02/58.181.37.179 ************************************************************/
