[KYUUBI apache#839]Fix load user specific defaults from properties file

hddong · ulysses-you · commit 8cef5c809a95 · 2021-07-21T12:13:46.000+08:00
### _Why are the changes needed?_  loadFromMap ignored user specific defaults that start with '___'. ### _How was this patch tested?_ - [X] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/tools/testing.html#running-tests) locally before make a pull request Closes apache#840 from hddong/kyuubi-839. Closes apache#839 ba3683e [hongdongdong] fix comments a775384 [hongdongdong] fix test 07f8266 [hongdongdong] fix test b2a4c3c [hongdongdong] [KYUUBI#839]Fix load user specific defaults from properties file Authored-by: hongdongdong <hongdongdong@cmss.chinamobile.com> Signed-off-by: ulysses-you <ulyssesyou18@gmail.com>
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -39,7 +39,9 @@ case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging {
   }
 
   private def loadFromMap(props: Map[String, String] = Utils.getSystemProperties): KyuubiConf = {
-    for ((key, value) <- props if key.startsWith("kyuubi.") || key.startsWith("spark.")) {
+    for ((key, value) <- props if key.startsWith("kyuubi.") || key.startsWith("spark.") ||
+        // for user specific defaults
+        key.startsWith("___")) {
       set(key, value)
     }
     this
diff --git a/kyuubi-common/src/test/resources/kyuubi-defaults.conf b/kyuubi-common/src/test/resources/kyuubi-defaults.conf
@@ -18,3 +18,7 @@
 kyuubi.yes yes
 spark.kyuubi.yes no
 # kyuubi.no no
+
+spark.user.test a
+___userb___.spark.user.test b
+___userc___.spark.user.test c
diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala
@@ -97,17 +97,10 @@ class KyuubiConfSuite extends KyuubiFunSuite {
 
 
   test("get user specific defaults") {
-    val conf = KyuubiConf(false)
-      .set("spark.user.test", "a")
-      .set("___userb___.spark.user.test", "b")
-      .set("___userc___.spark.user.test", "c")
-
-    val all1 = conf.getUserDefaults("kyuubi").getAll
-    assert(all1.size === 1)
-    assert(all1("spark.user.test") === "a")
-    val all2 = conf.getUserDefaults("userb").getAll
-    assert(all2.size === 1)
-    assert(all2("spark.user.test") === "b")
+    val conf = KyuubiConf().loadFileDefaults()
+
+    assert(conf.getUserDefaults("kyuubi").getOption("spark.user.test").get === "a")
+    assert(conf.getUserDefaults("userb").getOption("spark.user.test").get === "b")
     assert(conf.getUserDefaults("userc").getOption("spark.user.test").get === "c")
   }
 

Original file line number	Diff line number	Diff line change
`@@ -39,7 +39,9 @@ case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging {`
`39`	`39`	`}`
`40`	`40`
`41`	`41`	`private def loadFromMap(props: Map[String, String] = Utils.getSystemProperties): KyuubiConf = {`
`42`		`- for ((key, value) <- props if key.startsWith("kyuubi.") \|\| key.startsWith("spark.")) {`
	`42`	`+ for ((key, value) <- props if key.startsWith("kyuubi.") \|\| key.startsWith("spark.") \|\|`
	`43`	`+ // for user specific defaults`
	`44`	`+ key.startsWith("___")) {`
`43`	`45`	`set(key, value)`
`44`	`46`	`}`
`45`	`47`	`this`