|NO.Z.00054|——————————|BigDataEnd|——|Hadoop&MapReduce.V26|——|Hadoop.v26|源码剖析|NameNode启动流程|
一、[源码剖析之NameNode启动流程]：Apache Hadoop 核心源码剖析
### --- 下载Apache Hadoop-2.9.2官方源码：https://hadoop.apache.org/releases.html
——> Source Deownload：源码
——> Binary download：二进制包
——> Release notes：正式版本发布记录：hadoop-2.9.2-src
——> 历史版本地址：https://archive.apache.org/dist/hadoop/common/
### --- 将源码导入idea中
~~~ 源码导入：Import Project——>
~~~ 等待下载和解决依赖完成，源码导入成功！！
二、NameNode 启动流程
### --- 命令启动Hdfs集群
~~~ 该命令会启动Hdfs的NameNode以及DataNode，
~~~ 启动NameNode主要是通过org.apache.hadoop.hdfs.server.namenode.NameNode类。
~~~ 我们重点关注NameNode在启动过程中做了哪些工作（偏离主线的技术细节不深究）
start-dfs.sh### --- 对于分析启动流程主要关注两部分代码：
public class NameNode extends ReconfigurableBase implements NameNodeStatusMXBean {
//该静态代码块主要是初始化一些HDFS的配置信息
static{
HdfsConfiguration.init();//进入之后发现方法是空的，没有任何操作？其实不是观察
HdfsConfiguration的静态代码块
}
//HdfsConfiguration的类以及静态代码块
public class HdfsConfiguration extends Configuration {
static {
addDeprecatedKeys();
// adds the default resources
Configuration.addDefaultResource("hdfs-default.xml");
Configuration.addDefaultResource("hdfs-site.xml");
}
....
//main方法
public static void main(String argv[]) throws Exception{
//分析传入的参数是否为帮助参数，如果是帮助的话打印帮助信息，并退出。
if(DFSUtil.parseHelpArgument(argv,NameNode.USAGE,System.out,true)){
System.exit(0);
}
try{
//格式化输出启动信息，并且创建hook（打印节点关闭信息）
StringUtils.startupShutdownMessage(NameNode.class,argv,LOG);
//创建namenode
NameNode namenode=createNameNode(argv,null);
if（namenode!=null){
//加入集群
namenode.join()
}
}catch(Throwable e){
//异常处理
LOG.error("Failed to start namenode.",e)
terminate(1,e);
}
}
----------------------------------------------------------------------
//关注createNameNode
public static NameNode createNameNode(String argv[], Configuration conf)
throws IOException {
LOG.info("createNameNode " + Arrays.asList(argv));
if (conf == null)
conf = new HdfsConfiguration();
// Parse out some generic args into Configuration.
GenericOptionsParser hParser = new GenericOptionsParser(conf, argv);
argv = hParser.getRemainingArgs();
// Parse the rest, NN specific args.
//解析启动的参数
StartupOption startOpt = parseArguments(argv);
if (startOpt == null) {
printUsage(System.err);
return null;
}
setStartupOption(conf, startOpt);
switch (startOpt) {
....
default: { //正常启动进入该分支
//初始化metric系统
DefaultMetricsSystem.initialize("NameNode");
//返回新的NameNode
return new NameNode(conf);
}
}
}
----------------------------------------------------------------------
//NameNode的构造
public NameNode(Configuration conf) throws IOException {
this(conf, NamenodeRole.NAMENODE);
}
...
protected NameNode(Configuration conf, NamenodeRole role)
throws IOException {
this.conf = conf;
this.role = role;
// 设置NameNode#clientNamenodeAddress为"hdfs://localhost:9000"
setClientNamenodeAddress(conf);
String nsId = getNameServiceId(conf);
String namenodeId = HAUtil.getNameNodeId(conf, nsId);
// HA相关
this.haEnabled = HAUtil.isHAEnabled(conf, nsId);
state = createHAState(getStartupOption(conf));
this.allowStaleStandbyReads = HAUtil.shouldAllowStandbyReads(conf);
this.haContext = createHAContext();
try {
initializeGenericKeys(conf, nsId, namenodeId);
// 完成实际的初始化工作
initialize(conf);
// HA相关
try {
haContext.writeLock();
state.prepareToEnterState(haContext);
state.enterState(haContext);
} finally {
haContext.writeUnlock();
}
} catch (IOException e) {
this.stop();
throw e;
} catch (HadoopIllegalArgumentException e) {
this.stop();
throw e;
}
}
//尽管本地没有开启HA（haEnabled=false**），**namenode依然拥有一个HAState，namenode
的HAState状态为active.
----------------------------------------------------------------------
// 完成实际的初始化工作
// initialize(conf);
protected void initialize(Configuration conf) throws IOException {
if (conf.get(HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS) == null) {
String intervals = conf.get(DFS_METRICS_PERCENTILES_INTERVALS_KEY);
if (intervals != null) {
conf.set(HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS,
intervals);
}
}
UserGroupInformation.setConfiguration(conf);
loginAsNameNodeUser(conf);
// 初始化metric
NameNode.initMetrics(conf, this.getRole());
StartupProgressMetrics.register(startupProgress);
// 启动httpServer
if (NamenodeRole.NAMENODE == role) {
startHttpServer(conf);
}
this.spanReceiverHost = SpanReceiverHost.getInstance(conf);
// 从namenode目录加载fsimage与editlog，初始化FsNamesystem、FsDirectory、
LeaseManager等
loadNamesystem(conf);
// 创建RpcServer，封装了NameNodeRpcServer clientRpcServer，支持
ClientNamenodeProtocol、DatanodeProtocolPB等协议
rpcServer = createRpcServer(conf);
if (clientNamenodeAddress == null) {
// This is expected for MiniDFSCluster. Set it now using
// the RPC server's bind address.
clientNamenodeAddress =
NetUtils.getHostPortString(rpcServer.getRpcAddress());
LOG.info("Clients are to use " + clientNamenodeAddress + " to access"
+ " this namenode/service.");
}
if (NamenodeRole.NAMENODE == role) {
httpServer.setNameNodeAddress(getNameNodeAddress());
httpServer.setFSImage(getFSImage());
}
// 启动JvmPauseMonitor等，反向监控JVM
pauseMonitor = new JvmPauseMonitor(conf);
pauseMonitor.start();
metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
// 启动执行多个非常重要工作的多个线程
startCommonServices(conf);
}
----------------------------------------------------------------------
private void startCommonServices(Configuration conf) throws IOException {
// 创建NameNodeResourceChecker、激活BlockManager等
namesystem.startCommonServices(conf, haContext);
registerNNSMXBean();
// 角色非`NamenodeRole.NAMENODE`的在此处启动HttpServer
if (NamenodeRole.NAMENODE != role) {
startHttpServer(conf);
httpServer.setNameNodeAddress(getNameNodeAddress());
httpServer.setFSImage(getFSImage());
}
// 启动RPCServer
rpcServer.start();
... // 启动各插件
LOG.info(getRole() + " RPC up at: " + rpcServer.getRpcAddress());
if (rpcServer.getServiceRpcAddress() != null) {
LOG.info(getRole() + " service RPC up at: "
+ rpcServer.getServiceRpcAddress());
}
}
--------------------------------------------------------------------------------
------
void startCommonServices(Configuration conf, HAContext haContext) throws
IOException {
this.registerMBean(); // register the MBean for the FSNamesystemState
writeLock();
this.haContext = haContext;
try {
// 创建NameNodeResourceChecker，并立即检查一次
nnResourceChecker = new NameNodeResourceChecker(conf);
checkAvailableResources();
assert safeMode != null && !isPopulatingReplQueues();
// 设置一些启动过程中的信息
StartupProgress prog = NameNode.getStartupProgress();
prog.beginPhase(Phase.SAFEMODE);
prog.setTotal(Phase.SAFEMODE, STEP_AWAITING_REPORTED_BLOCKS,
getCompleteBlocksTotal());
// 设置已完成的数据块总量
setBlockTotal();
// 激活BlockManager
blockManager.activate(conf);
} finally {
writeUnlock();
}
registerMXBean();
DefaultMetricsSystem.instance().register(this);
snapshotManager.registerMXBean();
}
//blockManager.activate(conf)激活BlockManager主要完成PendingReplicationMonitor、
DecommissionManager#Monitor、HeartbeatManager#Monitor、ReplicationMonitor
public void activate(Configuration conf) {
// 启动PendingReplicationMonitor
pendingReplications.start();
// 激活DatanodeManager：启动DecommissionManager--Monitor、HeartbeatManager--
Monitor
datanodeManager.activate(conf);
// 启动BlockManager--ReplicationMonitor
this.replicationThread.start();
} ~~~ namenode的主要责任是文件元信息与数据块映射的管理。
~~~ 相应的，namenode的启动流程需要关注与客户端、datanode通信的工作线程，
~~~ 文件元信息的管理机制，数据块的管理机制等。
~~~ 其中，RpcServer主要负责与客户端、datanode通信，FSDirectory主要负责管理文件元信息。Walter Savage Landor:strove with none,for none was worth my strife.Nature I loved and, next to Nature, Art:I warm’d both hands before the fire of life.It sinks, and I am ready to depart
——W.S.Landor
版权声明：本文为yanqi_vip原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。