Shulou(Shulou.com)06/03 Report--

Resource-managers\ pom.xml

Org.apache.spark.deploy.SparkSubmit.scala

< /tmp/java_opts.txt && if ! [ -z ${SPARK_MOUNTED_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_SUBMIT_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_SUBMIT_EXTRA_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_EXTRA_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_MOUNTED_FILES_DIR+x} ]; then cp -R \"$SPARK_MOUNTED_FILES_DIR/.\" .; fi && if ! [ -z ${SPARK_MOUNTED_FILES_FROM_SECRET_DIR} ]; then cp -R \"$SPARK_MOUNTED_FILES_FROM_SECRET_DIR/.\" .; fi && ${JAVA_HOME}/bin/java \"${SPARK_DRIVER_JAVA_OPTS[@]}\" -cp $SPARK_CLASSPATH -Xms$SPARK_DRIVER_MEMORY -Xmx$SPARK_DRIVER_MEMORY -Dspark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS $SPARK_DRIVER_CLASS $SPARK_DRIVER_ARGS" ], "State": { "Status": "running", "Running": true, "Paused": false, "Restarting": false, "OOMKilled": false, "Dead": false, "Pid": 832405, "ExitCode": 0, "Error": "", "StartedAt": "2018-09-11T03:47:47.066061613Z", "FinishedAt": "0001-01-01T00:00:00Z" }, "Image": "sha256:4ba1d64922f4ccfb682ca09d7d716252f41c55a650884b796598bafb03174805", "ResolvConfPath": "/var/lib/docker/containers/d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b/resolv.conf", "HostnamePath": "/var/lib/docker/containers/d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b/hostname", "HostsPath": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/etc-hosts", "LogPath": "/var/lib/docker/containers/2e96d8a55415dc58420d346243b83968a4c1a1098000ad76af9152f2ba904a0c/2e96d8a55415dc58420d346243b83968a4c1a1098000ad76af9152f2ba904a0c-json.log", "Name": "/k8s_spark-kubernetes-driver_iot-stream-analysis-driver-100_spark-cluster_64491f22-b575-11e8-910e-0050568848e8_0", "RestartCount": 0, "Driver": "overlay", "MountLabel": "", "ProcessLabel": "", "AppArmorProfile": "", "ExecIDs": null, "HostConfig": { "Binds": [ "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/spark-local-dir-0-spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58:/mnt/tmp/spark-local/spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58", "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-jars-volume:/var/spark-data/spark-jars", "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-files:/var/spark-data/spark-files", "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~secret/spark-token-kpkrp:/var/run/secrets/kubernetes.io/serviceaccount:ro", "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/etc-hosts:/etc/hosts", "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/containers/spark-kubernetes-driver/65c96924:/dev/termination-log" ], "ContainerIDFile": "", "LogConfig": { "Type": "json-file", "Config": { "max-file": "5", "max-size": "50m" } }, "NetworkMode": "container:d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b", "PortBindings": null, "RestartPolicy": { "Name": "", "MaximumRetryCount": 0 }, "AutoRemove": false, "VolumeDriver": "", "VolumesFrom": null, "CapAdd": null, "CapDrop": null, "Dns": null, "DnsOptions": null, "DnsSearch": null, "ExtraHosts": null, "GroupAdd": null, "IpcMode": "container:d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b", "Cgroup": "", "Links": null, "OomScoreAdj": 936, "PidMode": "", "Privileged": false, "PublishAllPorts": false, "ReadonlyRootfs": false, "SecurityOpt": [ "seccomp=unconfined" ], "UTSMode": "", "UsernsMode": "", "ShmSize": 67108864, "Runtime": "runc", "ConsoleSize": [ 0, 0 ], "Isolation": "", "CpuShares": 1024, "Memory": 1476395008, "NanoCpus": 0, "CgroupParent": "kubepods-burstable-pod64491f22_b575_11e8_910e_0050568848e8.slice", "BlkioWeight": 0, "BlkioWeightDevice": null, "BlkioDeviceReadBps": null, "BlkioDeviceWriteBps": null, "BlkioDeviceReadIOps": null, "BlkioDeviceWriteIOps": null, "CpuPeriod": 0, "CpuQuota": 0, "CpuRealtimePeriod": 0, "CpuRealtimeRuntime": 0, "CpusetCpus": "", "CpusetMems": "", "Devices": [], "DiskQuota": 0, "KernelMemory": 0, "MemoryReservation": 0, "MemorySwap": 2952790016, "MemorySwappiness": -1, "OomKillDisable": false, "PidsLimit": 0, "Ulimits": null, "CpuCount": 0, "CpuPercent": 0, "IOMaximumIOps": 0, "IOMaximumBandwidth": 0 }, "GraphDriver": { "Name": "overlay", "Data": { "LowerDir": "/var/lib/docker/overlay/e7d727c6b28f718f78192d5bc76f24dffcf4603fba6e6173bded398b95d4c578/root", "MergedDir": "/var/lib/docker/overlay/2644946a6d486a66c93455eb2220b15227386d9329576df23568bc0f5221e6de/merged", "UpperDir": "/var/lib/docker/overlay/2644946a6d486a66c93455eb2220b15227386d9329576df23568bc0f5221e6de/upper", "WorkDir": "/var/lib/docker/overlay/2644946a6d486a66c93455eb2220b15227386d9329576df23568bc0f5221e6de/work" } }, "Mounts": [ { "Type": "bind", "Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~secret/spark-token-kpkrp", "Destination": "/var/run/secrets/kubernetes.io/serviceaccount", "Mode": "ro", "RW": false, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/etc-hosts", "Destination": "/etc/hosts", "Mode": "", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/containers/spark-kubernetes-driver/65c96924", "Destination": "/dev/termination-log", "Mode": "", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/spark-local-dir-0-spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58", "Destination": "/mnt/tmp/spark-local/spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58", "Mode": "", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-jars-volume", "Destination": "/var/spark-data/spark-jars", "Mode": "", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-files", "Destination": "/var/spark-data/spark-files", "Mode": "", "RW": true, "Propagation": "" } ], "Config": { "Hostname": "iot-stream-analysis-driver-100", "Domainname": "", "User": "0", "AttachStdin": false, "AttachStdout": false, "AttachStderr": false, "Tty": false, "OpenStdin": false, "StdinOnce": false, "Env": [ "SPARK_DRIVER_BIND_ADDRESS=161.161.104.67", "SPARK_JAVA_OPT_9=-Dspark.kubernetes.initcontainer.docker.image=registry.dev161.com:5000/iot/spark-init:v2.2.0-kubernetes-0.5.1", "SPARK_JAVA_OPT_10=-Dspark.kubernetes.driver.pod.name=iot-stream-analysis-driver-100", "SPARK_JAVA_OPT_13=-Dspark.kubernetes.docker.image.pullPolicy=Always", "SPARK_JAVA_OPT_14=-Dspark.kubernetes.driver.docker.image=registry.dev161.com:5000/iot/spark-driver:v2.2.0-kubernetes-0.5.1", "SPARK_JAVA_OPT_29=-Dspark.kubernetes.executor.docker.image=registry.dev161.com:5000/iot/spark-executor:v2.2.0-kubernetes-0.5.1", "SPARK_DRIVER_MEMORY=1024m", "SPARK_JAVA_OPT_1=-Dspark.kubernetes.svc.domain=svc.dev161", "SPARK_JAVA_OPT_5=-Dspark.local.dir=/mnt/tmp/spark-local/spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58", "SPARK_JAVA_OPT_11=-Dspark.app.name=spark-job-on-k8s-100", "SPARK_JAVA_OPT_19=-Dspark.kubernetes.executor.podNamePrefix=spark-job-on-k8s-100-1536637851433", "SPARK_JAVA_OPT_22=-Dspark.driver.port=7078", "SPARK_JAVA_OPT_25=-Dspark.executor.memory=1024m", "SPARK_MOUNTED_FILES_DIR=/var/spark-data/spark-files", "SPARK_JAVA_OPT_0=-Dspark.driver.host=spark-job-on-k8s-100-1536637851433-driver-svc.spark-cluster.svc.dev161", "SPARK_JAVA_OPT_3=-Dspark.submit.deployMode=cluster", "SPARK_JAVA_OPT_7=-Dspark.kubernetes.resourceStagingServer.uri=http://10.110.20.161:31000", "SPARK_JAVA_OPT_27=-Dspark.app.id=spark-1619ed3f66b04bf9a54a7230a1f551a4", "SPARK_JAVA_OPT_20=-Dspark.driver.memory=1024m", "SPARK_JAVA_OPT_21=-Dspark.kubernetes.submission.waitAppCompletion=true", "SPARK_JAVA_OPT_26=-Dspark.ui.enabled=false", "SPARK_MOUNTED_CLASSPATH=/var/spark-data/spark-jars/iot-stream-app-1.3-SNAPSHOT.jar", "SPARK_JAVA_OPT_2=-Dspark.kubernetes.initcontainer.executor.configmapkey=download-submitted-files", "SPARK_JAVA_OPT_18=-Dspark.kubernetes.initcontainer.executor.stagingServerSecret.name=spark-job-on-k8s-100-1536637851433-init-secret", "SPARK_JAVA_OPT_23=-Dspark.kubernetes.initcontainer.executor.stagingServerSecret.mountDir=/mnt/secrets/spark-init", "SPARK_JAVA_OPT_28=-Dspark.executor.instances=1", "SPARK_JAVA_OPT_30=-Dspark.executor.cores=1", "SPARK_DRIVER_ARGS=--base64=true --rule=c2VsZWN0IHRpbWVTdGFtcCBBcyBrZXksIGNvbmNhdF93cygifCIsIHN0YXRlLnJlcG9ydGVkLnRlbXBlcmF0dXJlLCBjbGllbnRUb2tlbikgYXMgdmFsdWUgZnJvbSB0b3BpY3M= --sample={\"timeStamp\":1531381822,\"clientToken\":\"clientId_lamp\",\"state\":{\"reported\":{\"temperature\":23}}} --source-type=kafka --source={\"kafka.bootstrap.servers\":\"fnd-kafka-svc.default:9092\",\"subscribe\":\"sensor\"} --sink-type=console --verbose", "SPARK_JAVA_OPT_8=-Dspark.jars=/var/spark-data/spark-jars/iot-stream-app-1.3-SNAPSHOT.jar", "SPARK_JAVA_OPT_12=-Dspark.driver.cores=1", "SPARK_JAVA_OPT_15=-Dspark.kubernetes.driver.nodeport=31202", "SPARK_JAVA_OPT_17=-Dspark.kubernetes.namespace=spark-cluster", "SPARK_JAVA_OPT_4=-Dspark.kubernetes.authenticate.driver.serviceAccountName=spark", "SPARK_JAVA_OPT_6=-Dspark.kubernetes.initcontainer.executor.configmapname=spark-job-on-k8s-100-1536637851433-init-config", "SPARK_JAVA_OPT_16=-Dspark.driver.blockManager.port=7079", "SPARK_DRIVER_CLASS=com.inspur.iot.RuleEngine", "SPARK_JAVA_OPT_24=-Dspark.master=k8s://http://10.110.20.161:8080", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_HOST=161.161.13.152", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_PORT=4040", "KUBERNETES_SERVICE_PORT_HTTPS=443", "KUBERNETES_PORT_443_TCP_PROTO=tcp", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_ADDR=161.161.52.23", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_PORT_DRIVER_UI_PORT=4040", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP=tcp://161.161.13.152:4040", "KUBERNETES_PORT=tcp://161.161.0.1:443", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_PORT=4040", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_PROTO=tcp", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT=tcp://161.161.13.152:4040", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_PROTO=tcp", "KUBERNETES_SERVICE_PORT=443", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_HOST=161.161.52.23", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_PORT_DRIVER_UI_PORT=4040", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT=tcp://161.161.52.23:4040", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_PORT=4040", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_ADDR=161.161.13.152", "KUBERNETES_SERVICE_HOST=161.161.0.1", "KUBERNETES_PORT_443_TCP=tcp://161.161.0.1:443", "KUBERNETES_PORT_443_TCP_PORT=443", "KUBERNETES_PORT_443_TCP_ADDR=161.161.0.1", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP=tcp://161.161.52.23:4040", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_PORT=4040", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/jvm/java-1.8-openjdk/jre/bin:/usr/lib/jvm/java-1.8-openjdk/bin", "LANG=C.UTF-8", "JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk", "JAVA_VERSION=8u131", "JAVA_ALPINE_VERSION=8.131.11-r2", "SPARK_HOME=/opt/spark" ], "Cmd": [ "/bin/sh", "-c", "SPARK_CLASSPATH=\"${SPARK_HOME}/jars/*\" && env | grep SPARK_JAVA_OPT_ | sed 's/[^=]*=\\(.*\\)/\\1/g' >

/ tmp/java_opts.txt & & readarray-t SPARK_DRIVER_JAVA_OPTS

< /tmp/java_opts.txt && if ! [ -z ${SPARK_MOUNTED_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_SUBMIT_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_SUBMIT_EXTRA_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_EXTRA_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_MOUNTED_FILES_DIR+x} ]; then cp -R \"$SPARK_MOUNTED_FILES_DIR/.\" .; fi && if ! [ -z ${SPARK_MOUNTED_FILES_FROM_SECRET_DIR} ]; then cp -R \"$SPARK_MOUNTED_FILES_FROM_SECRET_DIR/.\" .; fi && ${JAVA_HOME}/bin/java \"${SPARK_DRIVER_JAVA_OPTS[@]}\" -cp $SPARK_CLASSPATH -Xms$SPARK_DRIVER_MEMORY -Xmx$SPARK_DRIVER_MEMORY -Dspark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS $SPARK_DRIVER_CLASS $SPARK_DRIVER_ARGS" ], "Healthcheck": { "Test": [ "NONE" ] }, "ArgsEscaped": true, "Image": "registry.dev161.com:5000/iot/spark-driver@sha256:6cf4ab9fd7ff1c21ddd1d66feb76107928c844cfaae912172bc618734942ff80", "Volumes": null, "WorkingDir": "/opt/spark/work-dir", "Entrypoint": [ "/opt/entrypoint.sh" ], "OnBuild": null, "Labels": { "annotation.io.kubernetes.container.hash": "57d51e36", "annotation.io.kubernetes.container.restartCount": "0", "annotation.io.kubernetes.container.terminationMessagePath": "/dev/termination-log", "annotation.io.kubernetes.container.terminationMessagePolicy": "File", "annotation.io.kubernetes.pod.terminationGracePeriod": "30", "io.kubernetes.container.logpath": "/var/log/pods/64491f22-b575-11e8-910e-0050568848e8/spark-kubernetes-driver_0.log", "io.kubernetes.container.name": "spark-kubernetes-driver", "io.kubernetes.docker.type": "container", "io.kubernetes.pod.name": "iot-stream-analysis-driver-100", "io.kubernetes.pod.namespace": "spark-cluster", "io.kubernetes.pod.uid": "64491f22-b575-11e8-910e-0050568848e8", "io.kubernetes.sandbox.id": "d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b" } }, "NetworkSettings": { "Bridge": "", "SandboxID": "", "HairpinMode": false, "LinkLocalIPv6Address": "", "LinkLocalIPv6PrefixLen": 0, "Ports": null, "SandboxKey": "", "SecondaryIPAddresses": null, "SecondaryIPv6Addresses": null, "EndpointID": "", "Gateway": "", "GlobalIPv6Address": "", "GlobalIPv6PrefixLen": 0, "IPAddress": "", "IPPrefixLen": 0, "IPv6Gateway": "", "MacAddress": "", "Networks": {} } }] executor: [ { "Id": "37f1bce3f1295156a691768c4d385903f7cb978d35e63b3e5931d32e2802c322", "Created": "2018-09-11T03:47:55.861508477Z", "Path": "/opt/entrypoint.sh", "Args": [ "/bin/sh", "-c", "SPARK_CLASSPATH=\"${SPARK_HOME}/jars/*\" && env | grep SPARK_JAVA_OPT_ | sed 's/[^=]*=\\(.*\\)/\\1/g' >

/ tmp/java_opts.txt & & readarray-t SPARK_EXECUTOR_JAVA_OPTS

< /tmp/java_opts.txt && if ! [ -z ${SPARK_MOUNTED_CLASSPATH}+x} ]; then SPARK_CLASSPATH=\"$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_EXECUTOR_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_EXECUTOR_EXTRA_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_EXTRA_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_MOUNTED_FILES_DIR+x} ]; then cp -R \"$SPARK_MOUNTED_FILES_DIR/.\" .; fi && if ! [ -z ${SPARK_MOUNTED_FILES_FROM_SECRET_DIR+x} ]; then cp -R \"$SPARK_MOUNTED_FILES_FROM_SECRET_DIR/.\" .; fi && ${JAVA_HOME}/bin/java \"${SPARK_EXECUTOR_JAVA_OPTS[@]}\" -Dspark.executor.port=$SPARK_EXECUTOR_PORT -Xms$SPARK_EXECUTOR_MEMORY -Xmx$SPARK_EXECUTOR_MEMORY -cp $SPARK_CLASSPATH org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url $SPARK_DRIVER_URL --executor-id $SPARK_EXECUTOR_ID --cores $SPARK_EXECUTOR_CORES --app-id $SPARK_APPLICATION_ID --hostname $SPARK_EXECUTOR_POD_IP" ], "State": { "Status": "running", "Running": true, "Paused": false, "Restarting": false, "OOMKilled": false, "Dead": false, "Pid": 11397, "ExitCode": 0, "Error": "", "StartedAt": "2018-09-11T03:47:55.982525484Z", "FinishedAt": "0001-01-01T00:00:00Z" }, "Image": "sha256:c45bb8d3fb02e8684a36f5cc02d00548ddeffe1102bd8e80e330ef715c470de9", "ResolvConfPath": "/var/lib/docker/containers/0c5997eecdb5c81783f79e70be703a607762ba813dff722e58971899dc4bda1b/resolv.conf", "HostnamePath": "/var/lib/docker/containers/0c5997eecdb5c81783f79e70be703a607762ba813dff722e58971899dc4bda1b/hostname", "HostsPath": "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/etc-hosts", "LogPath": "/var/lib/docker/containers/37f1bce3f1295156a691768c4d385903f7cb978d35e63b3e5931d32e2802c322/37f1bce3f1295156a691768c4d385903f7cb978d35e63b3e5931d32e2802c322-json.log", "Name": "/k8s_executor_spark-job-on-k8s-100-1536637851433-exec-1_spark-cluster_74a94609-b575-11e8-910e-0050568848e8_0", "RestartCount": 0, "Driver": "overlay", "MountLabel": "", "ProcessLabel": "", "AppArmorProfile": "", "ExecIDs": null, "HostConfig": { "Binds": [ "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/spark-local-dir-0-spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58:/mnt/tmp/spark-local/spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58", "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-jars-volume:/var/spark-data/spark-jars", "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-files:/var/spark-data/spark-files", "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~secret/default-token-jxg4k:/var/run/secrets/kubernetes.io/serviceaccount:ro", "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/etc-hosts:/etc/hosts", "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/containers/executor/664b5635:/dev/termination-log" ], "ContainerIDFile": "", "LogConfig": { "Type": "json-file", "Config": { "max-file": "5", "max-size": "50m" } }, "NetworkMode": "container:0c5997eecdb5c81783f79e70be703a607762ba813dff722e58971899dc4bda1b", "PortBindings": null, "RestartPolicy": { "Name": "", "MaximumRetryCount": 0 }, "AutoRemove": false, "VolumeDriver": "", "VolumesFrom": null, "CapAdd": null, "CapDrop": null, "Dns": null, "DnsOptions": null, "DnsSearch": null, "ExtraHosts": null, "GroupAdd": null, "IpcMode": "container:0c5997eecdb5c81783f79e70be703a607762ba813dff722e58971899dc4bda1b", "Cgroup": "", "Links": null, "OomScoreAdj": 936, "PidMode": "", "Privileged": false, "PublishAllPorts": false, "ReadonlyRootfs": false, "SecurityOpt": [ "seccomp=unconfined" ], "UTSMode": "", "UsernsMode": "", "ShmSize": 67108864, "Runtime": "runc", "ConsoleSize": [ 0, 0 ], "Isolation": "", "CpuShares": 1024, "Memory": 1476395008, "NanoCpus": 0, "CgroupParent": "kubepods-burstable-pod74a94609_b575_11e8_910e_0050568848e8.slice", "BlkioWeight": 0, "BlkioWeightDevice": null, "BlkioDeviceReadBps": null, "BlkioDeviceWriteBps": null, "BlkioDeviceReadIOps": null, "BlkioDeviceWriteIOps": null, "CpuPeriod": 0, "CpuQuota": 0, "CpuRealtimePeriod": 0, "CpuRealtimeRuntime": 0, "CpusetCpus": "", "CpusetMems": "", "Devices": [], "DiskQuota": 0, "KernelMemory": 0, "MemoryReservation": 0, "MemorySwap": 2952790016, "MemorySwappiness": -1, "OomKillDisable": false, "PidsLimit": 0, "Ulimits": null, "CpuCount": 0, "CpuPercent": 0, "IOMaximumIOps": 0, "IOMaximumBandwidth": 0 }, "GraphDriver": { "Name": "overlay", "Data": { "LowerDir": "/var/lib/docker/overlay/9e0164c7e2d049bbf3260f6a332b2995368d8af4d6f8357f8c70c18f8d4c7864/root", "MergedDir": "/var/lib/docker/overlay/482521c521bccd54709755eb4ffb21fe2a0f63171ca2ee576f4477736cb2b25c/merged", "UpperDir": "/var/lib/docker/overlay/482521c521bccd54709755eb4ffb21fe2a0f63171ca2ee576f4477736cb2b25c/upper", "WorkDir": "/var/lib/docker/overlay/482521c521bccd54709755eb4ffb21fe2a0f63171ca2ee576f4477736cb2b25c/work" } }, "Mounts": [ { "Type": "bind", "Source": "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-jars-volume", "Destination": "/var/spark-data/spark-jars", "Mode": "", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-files", "Destination": "/var/spark-data/spark-files", "Mode": "", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~secret/default-token-jxg4k", "Destination": "/var/run/secrets/kubernetes.io/serviceaccount", "Mode": "ro", "RW": false, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/etc-hosts", "Destination": "/etc/hosts", "Mode": "", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/containers/executor/664b5635", "Destination": "/dev/termination-log", "Mode": "", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/spark-local-dir-0-spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58", "Destination": "/mnt/tmp/spark-local/spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58", "Mode": "", "RW": true, "Propagation": "" } ], "Config": { "Hostname": "spark-job-on-k8s-100-1536637851433-exec-1", "Domainname": "", "User": "0", "AttachStdin": false, "AttachStdout": false, "AttachStderr": false, "Tty": false, "OpenStdin": false, "StdinOnce": false, "Env": [ "SPARK_EXECUTOR_ID=1", "SPARK_EXECUTOR_POD_IP=161.161.76.219", "SPARK_MOUNTED_FILES_DIR=/var/spark-data/spark-files", "SPARK_EXECUTOR_PORT=10000", "SPARK_DRIVER_URL=spark://CoarseGrainedScheduler@spark-job-on-k8s-100-1536637851433-driver-svc.spark-cluster.svc.dev161:7078", "SPARK_EXECUTOR_CORES=1", "SPARK_EXECUTOR_MEMORY=1024m", "SPARK_APPLICATION_ID=spark-1619ed3f66b04bf9a54a7230a1f551a4", "SPARK_MOUNTED_CLASSPATH=/var/spark-data/spark-jars/*", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_PORT_DRIVER_UI_PORT=4040", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT=tcp://161.161.52.23:4040", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_ADDR=161.161.52.23", "KUBERNETES_PORT_443_TCP_ADDR=161.161.0.1", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_PROTO=tcp", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_PORT=4040", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_PORT=4040", "KUBERNETES_PORT_443_TCP=tcp://161.161.0.1:443", "KUBERNETES_PORT_443_TCP_PROTO=tcp", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_HOST=161.161.13.152", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_PORT_DRIVER_UI_PORT=4040", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_HOST=161.161.52.23", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_PROTO=tcp", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_PORT=4040", "KUBERNETES_SERVICE_HOST=161.161.0.1", "KUBERNETES_SERVICE_PORT_HTTPS=443", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_PORT=4040", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_ADDR=161.161.13.152", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP=tcp://161.161.52.23:4040", "KUBERNETES_SERVICE_PORT=443", "KUBERNETES_PORT=tcp://161.161.0.1:443", "KUBERNETES_PORT_443_TCP_PORT=443", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT=tcp://161.161.13.152:4040", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP=tcp://161.161.13.152:4040", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/jvm/java-1.8-openjdk/jre/bin:/usr/lib/jvm/java-1.8-openjdk/bin", "LANG=C.UTF-8", "JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk", "JAVA_VERSION=8u131", "JAVA_ALPINE_VERSION=8.131.11-r2", "SPARK_HOME=/opt/spark" ], "Cmd": [ "/bin/sh", "-c", "SPARK_CLASSPATH=\"${SPARK_HOME}/jars/*\" && env | grep SPARK_JAVA_OPT_ | sed 's/[^=]*=\\(.*\\)/\\1/g' >

/ tmp/java_opts.txt & & readarray-t SPARK_EXECUTOR_JAVA_OPTS < / tmp/java_opts.txt & & if! [- z ${SPARK_MOUNTED_CLASSPATH} + x}]; then SPARK_CLASSPATH=\ "$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH\"; fi & & if! [- z ${SPARK_EXECUTOR_EXTRA_CLASSPATH+x}]; then SPARK_CLASSPATH=\ "$SPARK_EXECUTOR_EXTRA_CLASSPATH:$SPARK_CLASSPATH\" Fi & & if! [- z ${SPARK_EXTRA_CLASSPATH+x}]; then SPARK_CLASSPATH=\ "$SPARK_EXTRA_CLASSPATH:$SPARK_CLASSPATH\"; fi & & if! [- z ${SPARK_MOUNTED_FILES_DIR+x}]; then cp-R\ "$SPARK_MOUNTED_FILES_DIR/.\".; fi & if! [- z ${SPARK_MOUNTED_FILES_FROM_SECRET_DIR+x}] Then cp-R\ "$SPARK_MOUNTED_FILES_FROM_SECRET_DIR/.\". Fi & ${JAVA_HOME} / bin/java\ "${SPARK_EXECUTOR_JAVA_OPTS [@]}\"-Dspark.executor.port=$SPARK_EXECUTOR_PORT-Xms$SPARK_EXECUTOR_MEMORY-Xmx$SPARK_EXECUTOR_MEMORY-cp $SPARK_CLASSPATH org.apache.spark.executor.CoarseGrainedExecutorBackend-- driver-url $SPARK_DRIVER_URL-- executor-id $SPARK_EXECUTOR_ID-- cores $SPARK_EXECUTOR_CORES-- app-id $SPARK_APPLICATION_ID-- hostname $SPARK_EXECUTOR_POD_IP "] "Healthcheck": {"Test": ["NONE"]}, "ArgsEscaped": true, "Image": "registry.dev161.com:5000/iot/spark-executor@sha256:c8650704517e254751fcef052eff13e3e4e40cff4b02039aceb9ce050f77d9a3", "Volumes": null "WorkingDir": "/ opt/spark/work-dir", "Entrypoint": ["/ opt/entrypoint.sh"], "OnBuild": null, "Labels": {"annotation.io.kubernetes.container.hash": "3136f4c3" "annotation.io.kubernetes.container.ports": "[{\" name\ ":\" executor\ ",\" containerPort\ ": 10000,\" protocol\ ":\" TCP\ "}, {\" name\ ":\" blockmanager\ ",\" containerPort\ ": 7079,\" protocol\ ":\" TCP\ "]", "annotation.io.kubernetes.container.restartCount": "0" Annotation.io.kubernetes.container.terminationMessagePath: "/ dev/termination-log", "annotation.io.kubernetes.container.terminationMessagePolicy": "File", "annotation.io.kubernetes.pod.terminationGracePeriod": "30", "io.kubernetes.container.logpath": "/ var/log/pods/74a94609-b575-11e8-910e-0050568848e8/executor_0.log" "io.kubernetes.container.name": "executor", "io.kubernetes.docker.type": "container", "io.kubernetes.pod.name": "spark-job-on-k8s-100-1536637851433-exec-1", "io.kubernetes.pod.namespace": "spark-cluster" "io.kubernetes.pod.uid": "74a94609-b575-11e8-910e-0050568848e8", "io.kubernetes.sandbox.id": "0c5997eecdb5c81783f79e70be703a607762ba813dff722e58971899dc4bda1b"}}, "NetworkSettings": {"Bridge": "," SandboxID ":"," HairpinMode ": false," LinkLocalIPv6Address ":" "LinkLocalIPv6PrefixLen": 0, "Ports": null, "SandboxKey": "," SecondaryIPAddresses ": null," SecondaryIPv6Addresses ": null," EndpointID ":"," Gateway ":", "GlobalIPv6Address": "," GlobalIPv6PrefixLen ": 0," IPAddress ":" "IPPrefixLen": 0, "IPv6Gateway": "", "MacAddress": "," Networks ": {}]

Resource-staging-server:

ApiVersion: v1kind: Podmetadata: creationTimestamp: 2018-09-11T10:39:29Z generateName: spark-resource-staging-server-544f9dfdc7- labels: pod-template-hash: "1009589873" resource-staging-server-instance: default name: spark-resource-staging-server-544f9dfdc7- 6z9qm namespace: iop ownerReferences:-apiVersion: extensions/v1beta1 blockOwnerDeletion: true controller: true kind: ReplicaSet name: spark-resource-staging-server-544f9dfdc7 uid: f64cb936-b5ae-11e8-a973-6c92bf88f88e resourceVersion: "5271010" selfLink: / api/v1/namespaces/iop/pods/spark-resource-staging-server-544f9dfdc7-6z9qm uid: f64ec3f8-b5ae-11e8-a973-6c92bf88f88espec: containers:-args:-/ etc/spark-resource-staging-server/resource-staging-server.properties image: registry.inspurcloud.cn/service/bigdata/spark-resource-staging-server:v2.2.0-kubernetes-0.4.0 imagePullPolicy: IfNotPresent name: spark-resource-staging-server resources : limits: cpu: "1" memory: 2Gi requests: cpu: 100m memory: 256Mi terminationMessagePath: / dev/termination-log terminationMessagePolicy: File volumeMounts:-mountPath: / etc/spark-resource-staging-server name: resource-staging-server-properties-mountPath: / var/run/secrets/kubernetes.io/serviceaccount name: default-token-ztnhf readOnly: true dnsPolicy: ClusterFirst imagePullSecrets :-name: service-registry nodeName: slave1 restartPolicy: Always schedulerName: default-scheduler securityContext: {} serviceAccount: default serviceAccountName: default terminationGracePeriodSeconds: 30 volumes:-configMap: defaultMode: 420 name: spark-resource-staging-server-config name: resource-staging-server-properties-name: default-token-ztnhf secret: defaultMode: 420 secretName: default-token-ztnhfstatus: conditions:-lastProbeTime: null lastTransitionTime: 2018-09-11T10:39:29Z status: "True" type: Initialized-lastProbeTime: null lastTransitionTime: 2018-09-11T10:39:42Z status: "True" type: Ready-lastProbeTime: null lastTransitionTime: 2018-09-11T10:39:29Z status: "True" type: PodScheduled containerStatuses:-containerID: docker://433aec71282a7371c091c69f7f921a5612f2ac98ec26c758629836c4855ebb1f image: registry.inspurcloud.cn/service/bigdata/spark-resource-staging-server:v2.2.0-kubernetes-0.4.0 imageID: docker-pullable:/ / registry.inspurcloud.cn/service/bigdata/spark-resource-staging-server@sha256:9480a49fb7c1ff6d6da85dcdc034a2e0a0006fb3d3a424671bf174c055b538d3 lastState: {} name: spark-resource-staging-server ready: true restartCount: 0 state: running: startedAt: 2018-09-11T10:39:42Z hostIP: 10.100.0.17 phase: Running podIP: 214.213.140.247 qosClass: Burstable startTime: 2018-09-11T10:39:29Z

Welcome to subscribe "Shulou Technology Information " to get latest news, interesting things and hot topics in the IT industry, and controls the hottest and latest Internet news, technology news and IT industry trends.