hadoop Configuration 源码解读

hadoop Configuration 源码解读

一般 Configuration 使用方法：

Configuration conf = new Configuration();
String s = conf.get("fs.defaultFS");
System.out.println(s);
FileSystem fs = FileSystem.get(conf);
1
2
3
4

想必大家都对属性的加载有点疑惑，今天就来探究一番。

没有配置 hdfs 相关配置时候

Configuration conf = new Configuration();
String s = conf.get("fs.defaultFS");
System.out.println(s);
1
2
3

输出的 s 结果是：

file:///
1

因为 有一份 core-default.xml 在 hadoop-common-3.3.3.jar 的 resource 里面，Configuration 初始化的时候，会加载 它，
它里面的 fs.defaultFS 默认值就是 file:///

源码开始

new Configuration() 发生了什么

1. Configuration 类的 static 属性初始化

private static final Logger LOG = LoggerFactory.getLogger(Configuration.class);
private static final Logger LOG_DEPRECATION = LoggerFactory.getLogger("org.apache.hadoop.conf.Configuration.deprecation");
private static final Set TAGS = ConcurrentHashMap.newKeySet();
private static final String DEFAULT_STRING_CHECK = "testingforemptydefaultvalue";
private static boolean restrictSystemPropsDefault = false;
private static final WeakHashMap REGISTRY = new WeakHashMap(); //放置 configuration 对象
private static final CopyOnWriteArrayList defaultResources = new CopyOnWriteArrayList(); //放置默认的 资源文件名 list core-default.xml、core-site.xml、hadoop-site.xml
private static DeprecationDelta[] defaultDeprecations =   //已经废弃了的 conf key 旧key与新key mapping
    new DeprecationDelta[] {
      new DeprecationDelta("topology.script.file.name", 
        CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY),
      new DeprecationDelta("topology.script.number.args", 
        CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY),
      new DeprecationDelta("hadoop.configured.node.mapping", 
        CommonConfigurationKeys.NET_TOPOLOGY_CONFIGURED_NODE_MAPPING_KEY),
      new DeprecationDelta("topology.node.switch.mapping.impl", 
        CommonConfigurationKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY),
      new DeprecationDelta("dfs.df.interval", 
        CommonConfigurationKeys.FS_DF_INTERVAL_KEY),
      new DeprecationDelta("fs.default.name", 
        CommonConfigurationKeys.FS_DEFAULT_NAME_KEY),
      new DeprecationDelta("dfs.umaskmode",
        CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY),
      new DeprecationDelta("dfs.nfs.exports.allowed.hosts",
          CommonConfigurationKeys.NFS_EXPORTS_ALLOWED_HOSTS_KEY)
    };
//全局已经被 弃用的 key
private static AtomicReference deprecationContext =
      new AtomicReference(
          new DeprecationContext(null, defaultDeprecations));
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

2. Configuration 类的 static 代码块执行

static {
    // Add default resources
    addDefaultResource("core-default.xml"); //添加 core-default.xml 文件名到 defaultResources
    addDefaultResource("core-site.xml");    //添加 core-site.xml    文件名到 defaultResources；注意
    // hadoop-common jar 中内置了 core-site.xml 默认空文件

    // print deprecation warning if hadoop-site.xml is found in classpath
    ClassLoader cL = Thread.currentThread().getContextClassLoader();
    if (cL == null) {
      cL = Configuration.class.getClassLoader();
    }
    if (cL.getResource("hadoop-site.xml") != null) { //如果 classpath 里面患有 hadoop-site.xml
      LOG.warn("DEPRECATED: hadoop-site.xml found in the classpath. " +
          "Usage of hadoop-site.xml is deprecated. Instead use core-site.xml, "
          + "mapred-site.xml and hdfs-site.xml to override properties of " +
          "core-default.xml, mapred-default.xml and hdfs-default.xml " +
          "respectively");
      addDefaultResource("hadoop-site.xml");
    }
  }

public static synchronized void addDefaultResource(String name) { //添加文件名到 defaultResources
    if(!defaultResources.contains(name)) {
      defaultResources.add(name);
      for(Configuration conf : REGISTRY.keySet()) {//因为 REGISTRY size 是 0，所以这里不会执行
        if(conf.loadDefaults) {
          conf.reloadConfiguration();
        }
      }
    }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

3. Configuration 类的 构造方法

public Configuration() {
    this(true);
}

public Configuration(boolean loadDefaults) {
    this.loadDefaults = loadDefaults;

    synchronized(Configuration.class) { //类锁
      REGISTRY.put(this, null); //REGISTRY 中加入本身
    }
}
1
2
3
4
5
6
7
8
9
10
11

conf.get(“fs.defaultFS”); 发生了什么

private Properties properties;	//属性存放的地方
  public String get(String name) {
    String[] names = handleDeprecation(deprecationContext.get(), name); //处理 新旧 key
    String result = null;
    for(String n : names) {
      result = substituteVars(getProps().getProperty(n));
      //从已经读取了  core-default.xml 、 core-site.xml(如果有) 属性 到 properties ，从 properties 获取 属性值
    }
    return result;
}

  private String[] handleDeprecation(DeprecationContext deprecations,
                                     String name) {
    if (null != name) {
      name = name.trim(); //去除 两端 的 空格
    }
    // Initialize the return value with requested name
    String[] names = new String[]{name};
    // Deprecated keys are logged once and an updated names are returned 拿到已经废弃 conf 的 keys，这个 name 是否是 已经废弃的 key
    DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name);
    if (keyInfo != null) {
      if (!keyInfo.getAndSetAccessed()) {
        logDeprecation(keyInfo.getWarningMessage(name));
      }
      // Override return value for deprecated keys
      names = keyInfo.newKeys;  //替换为 新 key
    }

    // Update properties with deprecated key if already loaded and new
    // deprecation has been added
    updatePropertiesWithDeprecatedKeys(deprecations, names);  //设置新 key 对应 旧 key 的值；
    // 里面已经读取了  core-default.xml 、 core-site.xml(如果有) 属性 到 properties 了

    // If there are no overlay values we can return early
    Properties overlayProperties = getOverlay();  //是否有需要 覆盖的值
    if (overlayProperties.isEmpty()) {  //没有的话，直接返货 names
      return names;
    }
    // Update properties and overlays with reverse lookup values
    for (String n : names) {
      String deprecatedKey = deprecations.getReverseDeprecatedKeyMap().get(n);
      if (deprecatedKey != null && !overlayProperties.containsKey(n)) {
        String deprecatedValue = overlayProperties.getProperty(deprecatedKey);
        if (deprecatedValue != null) {
          getProps().setProperty(n, deprecatedValue);
          overlayProperties.setProperty(n, deprecatedValue);
        }
      }
    }
    return names;
}

private void updatePropertiesWithDeprecatedKeys(
      DeprecationContext deprecations, String[] newNames) {
    for (String newName : newNames) {
      String deprecatedKey = deprecations.getReverseDeprecatedKeyMap().get(newName);  //拿到 旧 conf key
      if (deprecatedKey != null && !getProps().containsKey(newName)) {
        String deprecatedValue = getProps().getProperty(deprecatedKey);
        if (deprecatedValue != null) {
          getProps().setProperty(newName, deprecatedValue); //旧key对应的新key设置旧值
        }
      }
    }
}

private void updatePropertiesWithDeprecatedKeys(
      DeprecationContext deprecations, String[] newNames) {
    for (String newName : newNames) {
      String deprecatedKey = deprecations.getReverseDeprecatedKeyMap().get(newName);  //拿到 旧 conf key
      if (deprecatedKey != null && !getProps().containsKey(newName)) {  //执行 getProps 方法
        String deprecatedValue = getProps().getProperty(deprecatedKey);
        if (deprecatedValue != null) {
          getProps().setProperty(newName, deprecatedValue); //旧key对应的新key设置旧值
        }
      }
    }
}

protected synchronized Properties getProps() { //获取 core-default.xml 、 core-site.xml(如果有) 属性 保存类 properties
    if (properties == null) { //首次可能为 null
      properties = new Properties();
      loadProps(properties, 0, true);//加载 core-default.xml 、 core-site.xml(如果有) 到 props 里面
    }
    return properties;
}


private synchronized void loadProps(final Properties props,
      final int startIdx, final boolean fullReload) {
    if (props != null) {
      Map backup =
          updatingResource != null  // 第一次执行该方法时，backup为null
              ? new ConcurrentHashMap<>(updatingResource) : null;
      //加载 core-default.xml 、 core-site.xml(如果有) 到 props 里面；resources目前size=0
      loadResources(props, resources, startIdx, fullReload, quietmode); //加载资源
      if (overlay != null) {
        props.putAll(overlay);
        if (backup != null) {
          for (Map.Entry item : overlay.entrySet()) {
            String key = (String) item.getKey();
            String[] source = backup.get(key);
            if (source != null) {
              updatingResource.put(key, source);
            }
          }
        }
      }
    }
}

private void loadResources(Properties properties,
                             ArrayList resources,
                             int startIdx,
                             boolean fullReload,
                             boolean quiet) {
    if(loadDefaults && fullReload) {
      for (String resource : defaultResources) { //从 放置默认的 资源文件名 list
        // 1. 加载 core-default.xml 属性 到 properties； 在 hadoop-common-3.3.3.jar 的 resource 里面
        // 2. 记载 core-site.xml 中的属性到 properties （如果classpath配置了这个文件）
        loadResource(properties, new Resource(resource, false), quiet);
      }
    }
    
    for (int i = startIdx; i < resources.size(); i++) {
      Resource ret = loadResource(properties, resources.get(i), quiet);
      if (ret != null) {
        resources.set(i, ret);
      }
    }
    this.addTags(properties);
}

private Resource loadResource(Properties properties,
                                Resource wrapper, boolean quiet) {
    String name = UNKNOWN_RESOURCE;
    try {
      Object resource = wrapper.getResource();
      name = wrapper.getName();
      boolean returnCachedProperties = false;
    // Resource类未继承任何父类，因此并不满足以下的逻辑判断
      if (resource instanceof InputStream) {
        returnCachedProperties = true;
      } else if (resource instanceof Properties) {
        overlay(properties, (Properties)resource);
      }
      //如果没有手动给classpath中添加core-default.xml、core-site.xml文件，会从/org/apache/hadoop/hadoop-common/3.3.3/hadoop-common-3.2.2.jar!/core-default.xml位置读取配置文件
      XMLStreamReader2 reader = getStreamReader(wrapper, quiet);
      if (reader == null) { //如果没有 core-site.xml 文件在 classpath，则会返回null
        if (quiet) {
          return null;
        }
        throw new RuntimeException(resource + " not found");
      }
      Properties toAddTo = properties;
      if(returnCachedProperties) {
        toAddTo = new Properties();
      }
      //读取到属性值后，将属性值依次赋值到toAddTo
      List items = new Parser(reader, wrapper, quiet).parse();
      for (ParsedItem item : items) {
        loadProperty(toAddTo, item.name, item.key, item.value,
            item.isFinal, item.sources);
      }
      reader.close();

      if (returnCachedProperties) {
        overlay(properties, toAddTo);
        return new Resource(toAddTo, name, wrapper.isParserRestricted());
      }
      return null;
    } catch (IOException e) {
      LOG.error("error parsing conf " + name, e);
      throw new RuntimeException(e);
    } catch (XMLStreamException e) {
      LOG.error("error parsing conf " + name, e);
      throw new RuntimeException(e);
    }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178