Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: multus fails to reach api server when the old service account is out of data && update multus to v4 #4393

Merged
merged 1 commit into from
Dec 30, 2024
Merged

fix: multus fails to reach api server when the old service account is out of data && update multus to v4 #4393

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion charts/spiderpool/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ helm install spiderpool spiderpool/spiderpool --wait --namespace kube-system \
| `multus.multusCNI.image.repository` | the multus-CNI image repository | `k8snetworkplumbingwg/multus-cni` |
| `multus.multusCNI.image.pullPolicy` | the multus-CNI image pullPolicy | `IfNotPresent` |
| `multus.multusCNI.image.digest` | the multus-CNI image digest | `""` |
| `multus.multusCNI.image.tag` | the multus-CNI image tag | `v3.9.3` |
| `multus.multusCNI.image.tag` | the multus-CNI image tag | `v4.1.4` |
| `multus.multusCNI.image.imagePullSecrets` | the multus-CNI image imagePullSecrets | `[]` |
| `multus.multusCNI.defaultCniCRName` | if this value is empty, multus will automatically get default CNI according to the existed CNI conf file in /etc/cni/net.d/, if no cni files found in /etc/cni/net.d, A Spidermultusconfig CR named default will be created, please update the related SpiderMultusConfig for default CNI after installation. The namespace of defaultCniCRName follows with the release namespace of spdierpool | `""` |
| `multus.multusCNI.securityContext.privileged` | the securityContext privileged of multus-CNI daemonset pod | `true` |
Expand Down
152 changes: 145 additions & 7 deletions charts/spiderpool/templates/configmap.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ metadata:
{{- include "tplvalues.render" ( dict "value" .Values.global.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
data:
clusterNetwork: {{ .Values.multus.multusCNI.defaultCniCRName | quote }}
conf.yml: |
ipamUnixSocketPath: {{ .Values.global.ipamUNIXSocketHostPath }}
enableIPv4: {{ .Values.ipam.enableIPv4 }}
Expand All @@ -36,31 +37,168 @@ data:
kind: ConfigMap
apiVersion: v1
metadata:
name: {{ .Values.multus.multusCNI.name | trunc 63 | trimSuffix "-" }}
name: {{ .Values.multus.multusCNI.name | trunc 63 | trimSuffix "-" }}-entrypoint
namespace: {{ .Release.Namespace | quote }}
labels:
{{- include "spiderpool.multus.labels" . | nindent 4 }}
{{- if .Values.global.commonLabels }}
{{- include "tplvalues.render" ( dict "value" .Values.global.commonLabels "context" $ ) | nindent 4 }}
{{- end }}
data:
cni-conf.json: |
entrypoint.sh: |
#!/bin/bash
set -e

function log(){
echo "INFO: $(date --iso-8601=seconds) ${1}"
}
function error(){
log "ERR: {$1}"
}
function warn(){
log "WARN: {$1}"
}

function generateKubeConfig {
# Check if we're running as a k8s pod.
if [ -f "$SERVICE_ACCOUNT_TOKEN_PATH" ]; then
# We're running as a k8d pod - expect some variables.
if [ -z ${KUBERNETES_SERVICE_HOST} ]; then
error "KUBERNETES_SERVICE_HOST not set"; exit 1;
fi
if [ -z ${KUBERNETES_SERVICE_PORT} ]; then
error "KUBERNETES_SERVICE_PORT not set"; exit 1;
fi

if [ "$SKIP_TLS_VERIFY" == "true" ]; then
TLS_CFG="insecure-skip-tls-verify: true"
elif [ -f "$KUBE_CA_FILE" ]; then
TLS_CFG="certificate-authority-data: $(cat $KUBE_CA_FILE | base64 | tr -d '\n')"
fi

# Get the contents of service account token.
SERVICEACCOUNT_TOKEN=$(cat $SERVICE_ACCOUNT_TOKEN_PATH)

SKIP_TLS_VERIFY=${SKIP_TLS_VERIFY:-false}

# Write a kubeconfig file for the CNI plugin. Do this
# to skip TLS verification for now. We should eventually support
# writing more complete kubeconfig files. This is only used
# if the provided CNI network config references it.
touch $MULTUS_TEMP_KUBECONFIG
chmod ${KUBECONFIG_MODE:-600} $MULTUS_TEMP_KUBECONFIG
# Write the kubeconfig to a temp file first.
timenow=$(date)
cat > $MULTUS_TEMP_KUBECONFIG <<EOF
# Kubeconfig file for Multus CNI plugin.
# Generated at ${timenow}
apiVersion: v1
kind: Config
clusters:
- name: local
cluster:
server: ${KUBERNETES_SERVICE_PROTOCOL:-https}://[${KUBERNETES_SERVICE_HOST}]:${KUBERNETES_SERVICE_PORT}
$TLS_CFG
users:
- name: multus
user:
token: "${SERVICEACCOUNT_TOKEN}"
contexts:
- name: multus-context
context:
cluster: local
user: multus
current-context: multus-context
EOF

# Atomically move the temp kubeconfig to its permanent home.
mv -f $MULTUS_TEMP_KUBECONFIG $MULTUS_KUBECONFIG

# Keep track of the md5sum
LAST_SERVICEACCOUNT_MD5SUM=$(md5sum $SERVICE_ACCOUNT_TOKEN_PATH | awk '{print $1}')
LAST_KUBE_CA_FILE_MD5SUM=$(md5sum $KUBE_CA_FILE | awk '{print $1}')
else
warn "Doesn't look like we're running in a kubernetes environment (no serviceaccount token)"
fi
}

CNI_CONF_DIR="/host/etc/cni/net.d"
MULTUS_TEMP_CONFIG="/tmp/00-multus.conf"
touch $MULTUS_TEMP_CONFIG
MULTUS_TEMP_KUBECONFIG="/tmp/multus.kubeconfig"
mkdir -p $CNI_CONF_DIR/multus.d
MULTUS_KUBECONFIG=$CNI_CONF_DIR/multus.d/multus.kubeconfig
SERVICE_ACCOUNT_PATH=/var/run/secrets/kubernetes.io/serviceaccount
SERVICE_ACCOUNT_TOKEN_PATH=$SERVICE_ACCOUNT_PATH/token
KUBE_CA_FILE=${KUBE_CA_FILE:-$SERVICE_ACCOUNT_PATH/ca.crt}

LAST_SERVICEACCOUNT_MD5SUM=""
LAST_KUBE_CA_FILE_MD5SUM=""

cat > $MULTUS_TEMP_CONFIG << EOF
{
"cniVersion": "0.3.1",
"name": "multus-cni-network",
"type": "multus",
"confDir": "/etc/cni/net.d/" ,
"logLevel": "{{ .Values.multus.multusCNI.log.logLevel }}",
"logFile": "{{ .Values.multus.multusCNI.log.logFile }}",
"logLevel": "debug",
"logFile": "/var/log/multus.log",
"capabilities": {
"portMappings": true,
"bandwidth": true
},
"namespaceIsolation": false,
"clusterNetwork": "{{ .Values.multus.multusCNI.defaultCniCRName }}",
"clusterNetwork": "$MULTUS_CLUSTER_NETWORK",
"defaultNetworks": [],
"multusNamespace": "{{ .Release.Namespace }}",
"multusNamespace": "$MULTUS_NAMESPACE",
"systemNamespaces": [],
"kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
}
{{- end }}
EOF

if [ -z "${MULTUS_CLUSTER_NETWORK}" ]; then
log "ENV MULTUS_CLUSTER_NETWORK is empty, Detecting default cni in the ${CNI_CONF_DIR}"
DEFAULT_CNI_FILEPATH=$(ls -l ${CNI_CONF_DIR} | grep ^- | grep -v -i multus | awk '{print $9}' | grep -E '(*\.conf|*\.conflist|*\.json)' | head -n 1)
if [ -z "$DEFAULT_CNI_FILEPATH" ] ; then
error "No default cni file found in ${CNI_CONF_DIR}, please install your default cni in the cluster first" && exit 1
fi

log "Found the default-cni file: ${DEFAULT_CNI_FILEPATH}"
log "cat /host/etc/cni/net.d/${DEFAULT_CNI_FILEPATH}:"
cat /host/etc/cni/net.d/${DEFAULT_CNI_FILEPATH}

echo ""
DEFAULT_CNI_NAME=$(grep '"name":' ${CNI_CONF_DIR}/${DEFAULT_CNI_FILEPATH} | awk '{print $2}' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | tr -d ',' | tr -d '"')
if [ -z "$DEFAULT_CNI_NAME" ] ; then
error "The name fleid shouldn't be empty, please check the default cni: ${DEFAULT_CNI_FILEPATH}" && exit 1
fi

log "Updating the clusterNetwork of the multus-cni config to $DEFAULT_CNI_NAME"
sed -i "s?\"clusterNetwork\": \"\"?\"clusterNetwork\": \"${DEFAULT_CNI_NAME}\"?g" /tmp/00-multus.conf
else
log "User set multus ClusterNetwork: $MULTUS_CLUSTER_NETWORK"
fi

generateKubeConfig
log "multus kubeconfig is generated."

cp $MULTUS_TEMP_CONFIG /host/etc/cni/net.d
log "multus config file ${MULTUS_TEMP_CONFIG} is copied to ${CNI_CONF_DIR}."
log "cat ${CNI_CONF_DIR}/00-multus.conf"
cat ${CNI_CONF_DIR}/00-multus.conf

log "Entering watch loop..."
while true; do

# Check the md5sum of the service account token and ca.
svcaccountsum=$(md5sum $SERVICE_ACCOUNT_TOKEN_PATH | awk '{print $1}')
casum=$(md5sum $KUBE_CA_FILE | awk '{print $1}')
if [ "$svcaccountsum" != "$LAST_SERVICEACCOUNT_MD5SUM" ] || [ "$casum" != "$LAST_KUBE_CA_FILE_MD5SUM" ]; then
log "Detected service account or CA file change, regenerating kubeconfig..."
generateKubeConfig
fi

# todo: watch the default cni file is changed.
sleep 10
done
{{- end }}
63 changes: 46 additions & 17 deletions charts/spiderpool/templates/daemonset.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,22 @@ spec:
- name: cni-bin-path
mountPath: /host/opt/cni/bin
{{- end }}
{{- if .Values.multus.multusCNI.install }}
- name: install-multus-binary
image: {{ include "spiderpool.multus.image" . | quote }}
imagePullPolicy: IfNotPresent
command:
- /install_multus
args:
- --type
- thin
securityContext:
privileged: true
volumeMounts:
- mountPath: /host/opt/cni/bin
mountPropagation: Bidirectional
name: cni-bin-path
{{- end }}
containers:
- name: {{ .Values.spiderpoolAgent.name | trunc 63 | trimSuffix "-" }}
image: {{ include "spiderpool.spiderpoolAgent.image" . | quote }}
Expand Down Expand Up @@ -234,21 +250,30 @@ spec:
{{- end }}
{{- if .Values.multus.multusCNI.install }}
- name: multus-cni
imagePullPolicy: {{ .Values.multus.multusCNI.image.pullPolicy }}
image: {{ include "spiderpool.multus.image" . | quote }}
image: {{ include "spiderpool.spiderpoolAgent.image" . | quote }}
imagePullPolicy: {{ .Values.spiderpoolAgent.image.pullPolicy }}
command:
- "/bin/sh"
- "-c"
- |
ITEM="multus"
rm -f /host/opt/cni/bin/${ITEM}.old || true
( [ -f "/host/opt/cni/bin/${ITEM}" ] && mv /host/opt/cni/bin/${ITEM} /host/opt/cni/bin/${ITEM}.old ) || true
cp /usr/src/multus-cni/bin/${ITEM} /host/opt/cni/bin/${ITEM}
rm -f /host/opt/cni/bin/${ITEM}.old &>/dev/null || true
./entrypoint.sh --multus-conf-file=/tmp/multus-conf/00-multus.conf \
--cni-version=0.3.1
- "/home/entrypoint.sh"
securityContext:
privileged: true
env:
- name: MULTUS_CLUSTER_NETWORK
valueFrom:
configMapKeyRef:
key: clusterNetwork
name: spiderpool-conf
- name: MULTUS_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
resources:
limits:
cpu: 100m
memory: 50Mi
requests:
cpu: 100m
memory: 50Mi
{{- if .Values.multus.multusCNI.uninstall }}
lifecycle:
preStop:
Expand All @@ -264,11 +289,8 @@ spec:
volumeMounts:
- name: cni
mountPath: /host/etc/cni/net.d
- name: cni-bin-path
mountPath: /host/opt/cni/bin
mountPropagation: Bidirectional
- name: multus-cfg
mountPath: /tmp/multus-conf
- mountPath: /home
name: multus-entrypoint
{{- if .Values.multus.multusCNI.extraVolumes }}
{{- include "tplvalues.render" ( dict "value" .Values.multus.multusCNI.extraVolumeMounts "context" $ ) | nindent 12 }}
{{- end }}
Expand Down Expand Up @@ -304,6 +326,13 @@ spec:
items:
- key: cni-conf.json
path: 00-multus.conf
- name: multus-entrypoint
configMap:
name: {{ .Values.multus.multusCNI.name | trunc 63 | trimSuffix "-" }}-entrypoint
defaultMode: 511
items:
- key: entrypoint.sh
path: entrypoint.sh
{{- end }}
{{- if .Values.spiderpoolAgent.extraVolumeMounts }}
{{- include "tplvalues.render" ( dict "value" .Values.spiderpoolAgent.extraVolumeMounts "context" $ ) | nindent 6 }}
Expand Down
6 changes: 0 additions & 6 deletions charts/spiderpool/templates/pod.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,19 +80,13 @@ spec:
{{- end }}
- name: SPIDERPOOL_INIT_ENABLE_MULTUS_CONFIG
value: {{ .Values.multus.enableMultusConfig | quote }}
- name: SPIDERPOOL_INIT_INSTALL_MULTUS
value: {{ .Values.multus.multusCNI.install | quote }}
- name: SPIDERPOOL_INIT_DEFAULT_CNI_NAME
value: {{ .Values.multus.multusCNI.defaultCniCRName | quote }}
- name: SPIDERPOOL_INIT_DEFAULT_CNI_NAMESPACE
value: {{ .Release.Namespace | quote }}
- name: SPIDERPOOL_INIT_MULTUS_CONFIGMAP
value: {{ .Values.multus.multusCNI.name | trunc 63 | trimSuffix "-" | quote }}
{{- if eq .Values.multus.multusCNI.defaultCniCRName "" }}
- name: SPIDERPOOL_INIT_DEFAULT_CNI_DIR
value: {{ .Values.global.cniConfHostPath | quote }}
- name: SPIDERPOOL_INIT_READINESS_FILE
value: "/etc/spiderpool/ready"
volumeMounts:
- name: cni
mountPath: {{ .Values.global.cniConfHostPath }}
Expand Down
3 changes: 1 addition & 2 deletions charts/spiderpool/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,8 +233,7 @@ multus:
digest: ""

## @param multus.multusCNI.image.tag the multus-CNI image tag
tag: v3.9.3
# tag: v4.0.2-thick
tag: v4.1.4

## @param multus.multusCNI.image.imagePullSecrets the multus-CNI image imagePullSecrets
imagePullSecrets: []
Expand Down
2 changes: 1 addition & 1 deletion cmd/spiderpool-controller/cmd/crd_manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,6 @@ type _webhookHealthCheck struct{}
func (*_webhookHealthCheck) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
if request.Method == http.MethodGet {
writer.WriteHeader(http.StatusOK)
logger.Info("Webhook health check successful")
logger.Debug("Webhook health check successful")
}
}
7 changes: 0 additions & 7 deletions cmd/spiderpool-init/cmd/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,6 @@ type InitDefaultConfig struct {

// multuscniconfig
enableMultusConfig bool
installMultusCNI bool
DefaultCNIDir string
DefaultCNIName string
DefaultCNINamespace string
Expand Down Expand Up @@ -280,12 +279,6 @@ func parseENVAsDefault() InitDefaultConfig {
logger.Sugar().Fatalf("ENV %s: %s invalid: %v", ENVEnableMultusConfig, enableMultusConfig, err)
}

installMultusCNI := strings.ReplaceAll(os.Getenv(ENVInstallMultusCNI), "\"", "")
config.installMultusCNI, err = strconv.ParseBool(installMultusCNI)
if err != nil {
logger.Sugar().Fatalf("ENV %s: %s invalid: %v", ENVInstallMultusCNI, installMultusCNI, err)
}

config.DefaultCNIDir = strings.ReplaceAll(os.Getenv(ENVDefaultCNIDir), "\"", "")
if config.DefaultCNIDir != "" {
_, err = os.ReadDir(config.DefaultCNIDir)
Expand Down
Loading
Loading