如何解决Kubernetes Pod因卷连接问题而失败
我们看到在将卷安装/连接到吊舱时某些吊舱发生故障。这是间歇性发生的,并且在反弹kubelet服务吊舱后可以重新附加卷并成功。碰到吊舱时看到以下错误。
我创建了单个PV和相应的PVC,以用于吊舱的体积安装:
错误:
警告失败,挂载16m(x4超过43m)kubelet,ca1md-k8w03-02无法附加或装载卷:卸载卷= [workdir],未附加卷= [default-token-pwvpc podMetadata docker-sock workdir] :等待条件超时 警告FailedMount 7m32s(x5超过41m)kubelet,ca1md-k8w03-02无法附加或装载卷:卸载卷= [workdir],未连接卷= [docker-sock workdir default-token-pwvpc podMetadata]:超时等待健康)状况 警告FailedMount 3m2s(x10,距离45m)kubelet,ca1md-k8w03-02无法附加或装载卷:卸载的卷= [workdir],未连接的卷= [podMetadata docker-sock workdir default-token-pwvpc]:超时等待健康)状况 警告FailedMount 45s(x2超过21m)kubelet,ca1md-k8w03-02无法附加或安装卷:卸载的卷= [workdir],未连接的卷= [workdir default-token-pwvpc podMetadata docker-sock]:超时等待条件
版本:
Client Version: v1.17.2
Server Version: v1.17.2
主机操作系统:
Centos 7.7
CNI:
weave
apiVersion: v1
items:
- apiVersion: apps/v1
kind: StatefulSet
Metadata:
creationTimestamp: "2020-07-17T21:55:47Z"
generation: 1
labels:
logicmonitor.com/collectorset: kubernetes-03
managedFields:
- apiVersion: apps/v1
fieldsType: FieldsV1
fieldsV1:
f:Metadata:
f:labels:
.: {}
f:logicmonitor.com/collectorset: {}
f:spec:
f:podManagementPolicy: {}
f:replicas: {}
f:revisionHistoryLimit: {}
f:selector:
f:matchLabels:
.: {}
f:logicmonitor.com/collectorset: {}
f:template:
f:Metadata:
f:labels:
.: {}
f:logicmonitor.com/collectorset: {}
f:namespace: {}
f:spec:
f:affinity:
.: {}
f:podAntiAffinity:
.: {}
f:requiredDuringSchedulingIgnoredDuringExecution: {}
f:containers:
k:{"name":"collector"}:
.: {}
f:env:
.: {}
k:{"name":"COLLECTOR_IDS"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"access_id"}:
.: {}
f:name: {}
f:valueFrom:
.: {}
f:secretKeyRef:
.: {}
f:key: {}
f:name: {}
f:optional: {}
k:{"name":"access_key"}:
.: {}
f:name: {}
f:valueFrom:
.: {}
f:secretKeyRef:
.: {}
f:key: {}
f:name: {}
f:optional: {}
k:{"name":"account"}:
.: {}
f:name: {}
f:valueFrom:
.: {}
f:secretKeyRef:
.: {}
f:key: {}
f:name: {}
f:optional: {}
k:{"name":"collector_size"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"collector_version"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"kubernetes"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"use_ea"}:
.: {}
f:name: {}
f:value: {}
f:image: {}
f:imagePullPolicy: {}
f:name: {}
f:resources:
.: {}
f:limits:
.: {}
f:memory: {}
f:terminationMessagePath: {}
f:terminationMessagePolicy: {}
f:dnsPolicy: {}
f:restartPolicy: {}
f:schedulerName: {}
f:securityContext: {}
f:serviceAccount: {}
f:serviceAccountName: {}
f:terminationGracePeriodSeconds: {}
f:updateStrategy:
f:type: {}
f:status:
f:replicas: {}
manager: collectorset-controller
operation: Update
time: "2020-08-22T03:42:35Z"
name: kubernetes-03
namespace: default
resourceVersion: "10831902"
selfLink: /apis/apps/v1/namespaces/default/statefulsets/kubernetes-03
uid: 1296654b-77bc-4af8-9537-04f0a00bdd0c
spec:
podManagementPolicy: Parallel
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
logicmonitor.com/collectorset: kubernetes-03
serviceName: ""
template:
Metadata:
creationTimestamp: null
labels:
logicmonitor.com/collectorset: kubernetes-03
namespace: default
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
logicmonitor.com/collectorset: kubernetes-03
topologyKey: kubernetes.io/hostname
containers:
- env:
- name: account
valueFrom:
secretKeyRef:
key: account
name: collectorset-controller
optional: false
- name: access_id
valueFrom:
secretKeyRef:
key: accessID
name: collectorset-controller
optional: false
- name: access_key
valueFrom:
secretKeyRef:
key: accessKey
name: collectorset-controller
optional: false
- name: kubernetes
value: "true"
- name: collector_size
value: small
- name: collector_version
value: "0"
- name: use_ea
value: "false"
- name: COLLECTOR_IDS
value: "205"
image: logicmonitor/collector:latest
imagePullPolicy: Always
name: collector
resources:
limits:
memory: 2Gi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: collector
serviceAccountName: collector
terminationGracePeriodSeconds: 30
updateStrategy:
type: RollingUpdate
status:
collisionCount: 0
currentReplicas: 1
currentRevision: kubernetes-03-655b46ff69
observedGeneration: 1
readyReplicas: 1
replicas: 1
updateRevision: kubernetes-03-655b46ff69
updatedReplicas: 1
kind: List
Metadata:
resourceVersion: ""
selfLink: ""
kubectl get pv -o yaml
apiVersion: v1
items:
- apiVersion: v1
kind: PersistentVolume
Metadata:
annotations:
pv.kubernetes.io/bound-by-controller: "yes"
creationTimestamp: "2020-07-10T00:21:49Z"
finalizers:
- kubernetes.io/pv-protection
labels:
type: local
managedFields:
- apiVersion: v1
fieldsType: FieldsV1
fieldsV1:
f:Metadata:
f:labels:
.: {}
f:type: {}
f:spec:
f:accessModes: {}
f:capacity:
.: {}
f:storage: {}
f:nfs:
.: {}
f:path: {}
f:server: {}
f:persistentVolumeReclaimPolicy: {}
f:storageClassName: {}
f:volumeMode: {}
manager: kubectl
operation: Update
time: "2020-07-10T00:21:49Z"
- apiVersion: v1
fieldsType: FieldsV1
fieldsV1:
f:Metadata:
f:annotations:
.: {}
f:pv.kubernetes.io/bound-by-controller: {}
f:spec:
f:claimRef:
.: {}
f:apiVersion: {}
f:kind: {}
f:name: {}
f:namespace: {}
f:resourceVersion: {}
f:uid: {}
f:status:
f:phase: {}
manager: kube-controller-manager
operation: Update
time: "2020-07-10T00:21:52Z"
name: pv-modeldata
resourceVersion: "15764"
selfLink: /api/v1/persistentvolumes/pv-data
uid: 68521e84-0aa9-4643-a047-441a61451599
spec:
accessModes:
- ReadWriteMany
capacity:
storage: 5999Gi
claimRef:
apiVersion: v1
kind: PersistentVolumeClaim
name: pvc-modeldata
namespace: default
resourceVersion: "15762"
uid: e4a1309e-339d-4ed5-8fe1-9ed32f779ea7
nfs:
path: /k8fs03
server: kubemaster01.rms.com
persistentVolumeReclaimPolicy: Retain
storageClassName: manual
volumeMode: Filesystem
status:
phase: Bound
kind: List
Metadata:
resourceVersion: ""
selfLink: ""
# kubectl get pvc -o yaml
apiVersion: v1
items:
- apiVersion: v1
kind: PersistentVolumeClaim
Metadata:
annotations:
pv.kubernetes.io/bind-completed: "yes"
pv.kubernetes.io/bound-by-controller: "yes"
creationTimestamp: "2020-07-10T00:21:52Z"
finalizers:
- kubernetes.io/pvc-protection
managedFields:
- apiVersion: v1
fieldsType: FieldsV1
fieldsV1:
f:Metadata:
f:annotations:
.: {}
f:pv.kubernetes.io/bind-completed: {}
f:pv.kubernetes.io/bound-by-controller: {}
f:spec:
f:volumeName: {}
f:status:
f:accessModes: {}
f:capacity:
.: {}
f:storage: {}
f:phase: {}
manager: kube-controller-manager
operation: Update
time: "2020-07-10T00:21:52Z"
- apiVersion: v1
fieldsType: FieldsV1
fieldsV1:
f:spec:
f:accessModes: {}
f:resources:
f:requests:
.: {}
f:storage: {}
f:storageClassName: {}
f:volumeMode: {}
manager: kubectl
operation: Update
time: "2020-07-10T00:21:52Z"
name: pvc-modeldata
namespace: default
resourceVersion: "15766"
selfLink: /api/v1/namespaces/default/persistentvolumeclaims/pvc-data
uid: e4a1309e-339d-4ed5-8fe1-9ed32f779ea7
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 5999Gi
storageClassName: manual
volumeMode: Filesystem
volumeName: pv-data
status:
accessModes:
- ReadWriteMany
capacity:
storage: 5999Gi
phase: Bound
kind: List
Metadata:
resourceVersion: ""
selfLink: ""
# kubectl get sc -o yaml
apiVersion: v1
items: []
kind: List
Metadata:
resourceVersion: ""
selfLink: ""
谢谢, Chittu
版权声明:本文内容由互联网用户自发贡献,该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容, 请发送邮件至 dio@foxmail.com 举报,一经查实,本站将立刻删除。