diff --git a/.github/workflows/dep.yml b/.github/workflows/dep.yml index 5ea4447cc47..4e6969390f6 100644 --- a/.github/workflows/dep.yml +++ b/.github/workflows/dep.yml @@ -56,4 +56,6 @@ jobs: -DskipTests -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am - name: Check dependency list - run: build/dependency.sh + run: | + /usr/bin/mvn --version + build/dependency.sh diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 59bcdbd5885..8547e0e0de4 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -421,6 +421,7 @@ jobs: if: failure() run: | kubectl get pods + kubectl get pods | grep driver | awk -F " " '{print$1}' | xargs -I {} kubectl describe pods {} kubectl get pods | grep driver | awk -F " " '{print$1}' | xargs -I {} kubectl logs {} - name: Upload test logs if: failure() diff --git a/dev/dependencyList b/dev/dependencyList index 10933b9faf0..b01497b8513 100644 --- a/dev/dependencyList +++ b/dev/dependencyList @@ -16,13 +16,10 @@ # HikariCP/4.0.3//HikariCP-4.0.3.jar -ST4/4.3.4//ST4-4.3.4.jar animal-sniffer-annotations/1.21//animal-sniffer-annotations-1.21.jar annotations/4.1.1.4//annotations-4.1.1.4.jar -antlr-runtime/3.5.3//antlr-runtime-3.5.3.jar antlr4-runtime/4.9.3//antlr4-runtime-4.9.3.jar aopalliance-repackaged/2.6.1//aopalliance-repackaged-2.6.1.jar -automaton/1.11-8//automaton-1.11-8.jar classgraph/4.8.138//classgraph-4.8.138.jar commons-codec/1.15//commons-codec-1.15.jar commons-collections/3.2.2//commons-collections-3.2.2.jar @@ -37,7 +34,6 @@ error_prone_annotations/2.14.0//error_prone_annotations-2.14.0.jar failsafe/2.4.4//failsafe-2.4.4.jar failureaccess/1.0.1//failureaccess-1.0.1.jar fliptables/1.0.2//fliptables-1.0.2.jar -generex/1.0.2//generex-1.0.2.jar grpc-api/1.48.0//grpc-api-1.48.0.jar grpc-context/1.48.0//grpc-context-1.48.0.jar grpc-core/1.48.0//grpc-core-1.48.0.jar @@ -63,7 +59,6 @@ hk2-locator/2.6.1//hk2-locator-2.6.1.jar hk2-utils/2.6.1//hk2-utils-2.6.1.jar httpclient/4.5.14//httpclient-4.5.14.jar httpcore/4.4.16//httpcore-4.4.16.jar -httpmime/4.5.14//httpmime-4.5.14.jar j2objc-annotations/1.3//j2objc-annotations-1.3.jar jackson-annotations/2.14.1//jackson-annotations-2.14.1.jar jackson-core/2.14.1//jackson-core-2.14.1.jar @@ -104,27 +99,30 @@ jetty-util-ajax/9.4.50.v20221201//jetty-util-ajax-9.4.50.v20221201.jar jetty-util/9.4.50.v20221201//jetty-util-9.4.50.v20221201.jar jline/0.9.94//jline-0.9.94.jar jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar -kubernetes-client/5.12.1//kubernetes-client-5.12.1.jar -kubernetes-model-admissionregistration/5.12.1//kubernetes-model-admissionregistration-5.12.1.jar -kubernetes-model-apiextensions/5.12.1//kubernetes-model-apiextensions-5.12.1.jar -kubernetes-model-apps/5.12.1//kubernetes-model-apps-5.12.1.jar -kubernetes-model-autoscaling/5.12.1//kubernetes-model-autoscaling-5.12.1.jar -kubernetes-model-batch/5.12.1//kubernetes-model-batch-5.12.1.jar -kubernetes-model-certificates/5.12.1//kubernetes-model-certificates-5.12.1.jar -kubernetes-model-common/5.12.1//kubernetes-model-common-5.12.1.jar -kubernetes-model-coordination/5.12.1//kubernetes-model-coordination-5.12.1.jar -kubernetes-model-core/5.12.1//kubernetes-model-core-5.12.1.jar -kubernetes-model-discovery/5.12.1//kubernetes-model-discovery-5.12.1.jar -kubernetes-model-events/5.12.1//kubernetes-model-events-5.12.1.jar -kubernetes-model-extensions/5.12.1//kubernetes-model-extensions-5.12.1.jar -kubernetes-model-flowcontrol/5.12.1//kubernetes-model-flowcontrol-5.12.1.jar -kubernetes-model-metrics/5.12.1//kubernetes-model-metrics-5.12.1.jar -kubernetes-model-networking/5.12.1//kubernetes-model-networking-5.12.1.jar -kubernetes-model-node/5.12.1//kubernetes-model-node-5.12.1.jar -kubernetes-model-policy/5.12.1//kubernetes-model-policy-5.12.1.jar -kubernetes-model-rbac/5.12.1//kubernetes-model-rbac-5.12.1.jar -kubernetes-model-scheduling/5.12.1//kubernetes-model-scheduling-5.12.1.jar -kubernetes-model-storageclass/5.12.1//kubernetes-model-storageclass-5.12.1.jar +kubernetes-client-api/6.1.1//kubernetes-client-api-6.1.1.jar +kubernetes-client/6.1.1//kubernetes-client-6.1.1.jar +kubernetes-httpclient-okhttp/6.1.1//kubernetes-httpclient-okhttp-6.1.1.jar +kubernetes-model-admissionregistration/6.1.1//kubernetes-model-admissionregistration-6.1.1.jar +kubernetes-model-apiextensions/6.1.1//kubernetes-model-apiextensions-6.1.1.jar +kubernetes-model-apps/6.1.1//kubernetes-model-apps-6.1.1.jar +kubernetes-model-autoscaling/6.1.1//kubernetes-model-autoscaling-6.1.1.jar +kubernetes-model-batch/6.1.1//kubernetes-model-batch-6.1.1.jar +kubernetes-model-certificates/6.1.1//kubernetes-model-certificates-6.1.1.jar +kubernetes-model-common/6.1.1//kubernetes-model-common-6.1.1.jar +kubernetes-model-coordination/6.1.1//kubernetes-model-coordination-6.1.1.jar +kubernetes-model-core/6.1.1//kubernetes-model-core-6.1.1.jar +kubernetes-model-discovery/6.1.1//kubernetes-model-discovery-6.1.1.jar +kubernetes-model-events/6.1.1//kubernetes-model-events-6.1.1.jar +kubernetes-model-extensions/6.1.1//kubernetes-model-extensions-6.1.1.jar +kubernetes-model-flowcontrol/6.1.1//kubernetes-model-flowcontrol-6.1.1.jar +kubernetes-model-gatewayapi/6.1.1//kubernetes-model-gatewayapi-6.1.1.jar +kubernetes-model-metrics/6.1.1//kubernetes-model-metrics-6.1.1.jar +kubernetes-model-networking/6.1.1//kubernetes-model-networking-6.1.1.jar +kubernetes-model-node/6.1.1//kubernetes-model-node-6.1.1.jar +kubernetes-model-policy/6.1.1//kubernetes-model-policy-6.1.1.jar +kubernetes-model-rbac/6.1.1//kubernetes-model-rbac-6.1.1.jar +kubernetes-model-scheduling/6.1.1//kubernetes-model-scheduling-6.1.1.jar +kubernetes-model-storageclass/6.1.1//kubernetes-model-storageclass-6.1.1.jar libfb303/0.9.3//libfb303-0.9.3.jar libthrift/0.9.3//libthrift-0.9.3.jar log4j-1.2-api/2.19.0//log4j-1.2-api-2.19.0.jar diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/MiniKube.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/MiniKube.scala index cd373873a6a..ed6163e396a 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/MiniKube.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/MiniKube.scala @@ -17,7 +17,7 @@ package org.apache.kyuubi.kubernetes.test -import io.fabric8.kubernetes.client.{Config, DefaultKubernetesClient} +import io.fabric8.kubernetes.client.{Config, KubernetesClient, KubernetesClientBuilder} /** * This code copied from Aapache Spark @@ -44,7 +44,7 @@ object MiniKube { executeMinikube(true, "ip").head } - def getKubernetesClient: DefaultKubernetesClient = { + def getKubernetesClient: KubernetesClient = { // only the three-part version number is matched (the optional suffix like "-beta.0" is dropped) val versionArrayOpt = "\\d+\\.\\d+\\.\\d+".r .findFirstIn(minikubeVersionString.split(VERSION_PREFIX)(1)) @@ -66,6 +66,6 @@ object MiniKube { "non-numeric suffix is intentionally dropped)") } - new DefaultKubernetesClient(Config.autoConfigure("minikube")) + new KubernetesClientBuilder().withConfig(Config.autoConfigure("minikube")).build() } } diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/WithKyuubiServerOnKubernetes.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/WithKyuubiServerOnKubernetes.scala index ed9cbce09fe..09ba392cfc8 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/WithKyuubiServerOnKubernetes.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/WithKyuubiServerOnKubernetes.scala @@ -18,14 +18,15 @@ package org.apache.kyuubi.kubernetes.test import io.fabric8.kubernetes.api.model.Pod -import io.fabric8.kubernetes.client.DefaultKubernetesClient +import io.fabric8.kubernetes.client.KubernetesClient import org.apache.kyuubi.KyuubiFunSuite trait WithKyuubiServerOnKubernetes extends KyuubiFunSuite { protected def connectionConf: Map[String, String] = Map.empty - lazy val miniKubernetesClient: DefaultKubernetesClient = MiniKube.getKubernetesClient + private val miniKubernetesClient: KubernetesClient = MiniKube.getKubernetesClient + lazy val kyuubiPod: Pod = miniKubernetesClient.pods().withName("kyuubi-test").get() lazy val kyuubiServerIp: String = kyuubiPod.getStatus.getPodIP lazy val miniKubeIp: String = MiniKube.getIp diff --git a/kyuubi-server/pom.xml b/kyuubi-server/pom.xml index 4dd89e0e62c..1e5b6582046 100644 --- a/kyuubi-server/pom.xml +++ b/kyuubi-server/pom.xml @@ -91,6 +91,10 @@ io.fabric8 kubernetes-client + + io.fabric8 + kubernetes-httpclient-okhttp + org.apache.hive diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala index bee69b11762..868d2aca4ba 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala @@ -17,9 +17,8 @@ package org.apache.kyuubi.engine -import io.fabric8.kubernetes.api.model.{Pod, PodList} +import io.fabric8.kubernetes.api.model.Pod import io.fabric8.kubernetes.client.KubernetesClient -import io.fabric8.kubernetes.client.dsl.FilterWatchListDeletable import org.apache.kyuubi.Logging import org.apache.kyuubi.config.KyuubiConf @@ -57,23 +56,22 @@ class KubernetesApplicationOperation extends ApplicationOperation with Logging { if (kubernetesClient != null) { debug(s"Deleting application info from Kubernetes cluster by $tag tag") try { - // Need driver only - val operation = findDriverPodByTag(tag) - val podList = operation.list().getItems - if (podList.size() != 0) { - toApplicationState(podList.get(0).getStatus.getPhase) match { - case FAILED | UNKNOWN => - ( - false, - s"Target Pod ${podList.get(0).getMetadata.getName} is in FAILED or UNKNOWN status") - case _ => - ( - operation.delete(), - s"Operation of deleted appId: ${podList.get(0).getMetadata.getName} is completed") - } - } else { - // client mode - jpsOperation.killApplicationByTag(tag) + findDriverPodByTag(tag) match { + case Some(pod) => + val podName = pod.getMetadata.getName + toApplicationState(pod.getStatus.getPhase) match { + case FAILED | UNKNOWN => + (false, s"Target Driver Pod $podName is in FAILED or UNKNOWN status") + case _ => + // Imitate from fabric8io/kubernetes-client + // kubernetes-tests/CustomResourceTest.java#testDeleteNonExistentItem + ( + !kubernetesClient.pods().inNamespace(podName).withName( + pod.getMetadata.getName).delete().isEmpty, + s"Operation of deleted app: $podName is completed") + } + case None => + jpsOperation.killApplicationByTag(tag) } } catch { case e: Exception => @@ -88,21 +86,18 @@ class KubernetesApplicationOperation extends ApplicationOperation with Logging { if (kubernetesClient != null) { debug(s"Getting application info from Kubernetes cluster by $tag tag") try { - val operation = findDriverPodByTag(tag) - val podList = operation.list().getItems - if (podList.size() != 0) { - val pod = podList.get(0) - val info = ApplicationInfo( - // spark pods always tag label `spark-app-selector:` - id = pod.getMetadata.getLabels.get(SPARK_APP_ID_LABEL), - name = pod.getMetadata.getName, - state = KubernetesApplicationOperation.toApplicationState(pod.getStatus.getPhase), - error = Option(pod.getStatus.getReason)) - debug(s"Successfully got application info by $tag: $info") - info - } else { - // client mode - jpsOperation.getApplicationInfoByTag(tag) + findDriverPodByTag(tag) match { + case Some(pod) => + val info = ApplicationInfo( + // spark pods always tag label `spark-app-selector:` + id = pod.getMetadata.getLabels.get(SPARK_APP_ID_LABEL), + name = pod.getMetadata.getName, + state = KubernetesApplicationOperation.toApplicationState(pod.getStatus.getPhase), + error = Option(pod.getStatus.getReason)) + debug(s"Successfully got application info by $tag: $info") + info + case None => + jpsOperation.getApplicationInfoByTag(tag) } } catch { case e: Exception => @@ -114,14 +109,21 @@ class KubernetesApplicationOperation extends ApplicationOperation with Logging { } } - private def findDriverPodByTag(tag: String): FilterWatchListDeletable[Pod, PodList] = { + private def findDriverPodByTag(tag: String): Option[Pod] = { val operation = kubernetesClient.pods() .withLabel(KubernetesApplicationOperation.LABEL_KYUUBI_UNIQUE_KEY, tag) - val size = operation.list().getItems.size() - if (size != 1) { - warn(s"Get Tag: ${tag} Driver Pod In Kubernetes size: ${size}, we expect 1") + val podList = operation.list().getItems + val size = podList.size() + size match { + case 0 => + warn(s"Can't find Driver pod with tag $tag") + None + case 1 => + Some(podList.get(0)) + case _ => + warn(s"Get Tag: $tag Driver Pod In Kubernetes size: $size, we expect 1") + Some(podList.get(0)) } - operation } override def stop(): Unit = { diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala index 921aa04ae3c..c30c78905a0 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala @@ -22,7 +22,7 @@ import java.io.File import com.fasterxml.jackson.databind.ObjectMapper import com.google.common.base.Charsets import com.google.common.io.Files -import io.fabric8.kubernetes.client.{Config, ConfigBuilder, DefaultKubernetesClient, KubernetesClient} +import io.fabric8.kubernetes.client.{Config, ConfigBuilder, KubernetesClient, KubernetesClientBuilder} import io.fabric8.kubernetes.client.Config.autoConfigure import io.fabric8.kubernetes.client.okhttp.OkHttpClientFactory import okhttp3.{Dispatcher, OkHttpClient} @@ -93,7 +93,10 @@ object KubernetesUtils extends Logging { debug("Kubernetes client config: " + new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(config)) - Some(new DefaultKubernetesClient(factoryWithCustomDispatcher.createHttpClient(config), config)) + Some(new KubernetesClientBuilder() + .withConfig(config) + .withHttpClientFactory(factoryWithCustomDispatcher) + .build()) } implicit private class OptionConfigurableConfigBuilder(val configBuilder: ConfigBuilder) diff --git a/pom.xml b/pom.xml index 8a86d85018c..61da36bd1f3 100644 --- a/pom.xml +++ b/pom.xml @@ -166,7 +166,7 @@ 9.4.50.v20221201 0.9.94 4.13.1 - 5.12.1 + 6.1.1 1.15.0 6.0.5 2.19.0 @@ -587,6 +587,16 @@ kubernetes-client ${kubernetes-client.version} + + + io.fabric8 + kubernetes-httpclient-okhttp + ${kubernetes-client.version} +