Skip to content

Commit

Permalink
Merge pull request #350 from AkihiroSuda/dev
Browse files Browse the repository at this point in the history
CI: shrink test matrix, etc.
  • Loading branch information
AkihiroSuda authored Dec 10, 2024
2 parents e527663 + 8c9f147 commit 8dd5ad3
Show file tree
Hide file tree
Showing 7 changed files with 66 additions and 133 deletions.
45 changes: 25 additions & 20 deletions .github/workflows/main.yaml
Original file line number Diff line number Diff line change
@@ -1,37 +1,43 @@
---
name: Main
on: [push, pull_request]
env:
DOCKER_BUILDKIT: 1
KUBECONFIG: ./kubeconfig
jobs:
# This uses the reusable-single-node.yaml template
single-node:
name: "Single node with defaults"
uses: ./.github/workflows/reusable-single-node.yaml

single-node-custom-ports:
name: "Single node with custom service ports"
name: "Single node"
strategy:
fail-fast: false
matrix:
container_engine: [docker, nerdctl, podman]
uses: ./.github/workflows/reusable-single-node.yaml
with:
# Defaults to 6443
kube_apiserver_port: "8080"
# Defaults to 8472
flannel_port: "9072"
# Defaults to 10250
kubelet_port: "20250"
# Defaults to 2379
etcd_port: "9090"
container_engine: ${{ matrix.container_engine }}

# This uses the reusable-multi-node.yaml template
multi-node:
name: "Multi node with defaults"
name: "Multi node"
strategy:
fail-fast: false
matrix:
include:
- lima_template: template://ubuntu-24.04
container_engine: docker
- lima_template: template://ubuntu-24.04
container_engine: nerdctl
- lima_template: template://centos-stream-9
container_engine: podman
- lima_template: template://fedora
container_engine: podman
uses: ./.github/workflows/reusable-multi-node.yaml
with:
lima_template: ${{ matrix.lima_template }}
container_engine: ${{ matrix.container_engine }}

# TODO: this test should create multiple instances of Usernetes on each of the hosts
multi-node-custom-ports:
name: "Multi node with custom service ports"
uses: ./.github/workflows/reusable-multi-node.yaml
with:
lima_template: "template://ubuntu-24.04"
container_engine: "docker"
# Defaults to 6443
kube_apiserver_port: "8080"
# Defaults to 8472
Expand All @@ -40,4 +46,3 @@ jobs:
kubelet_port: "20250"
# Defaults to 2379
etcd_port: "9090"

25 changes: 10 additions & 15 deletions .github/workflows/reusable-multi-node.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,14 @@ on:
workflow_call:
# allow reuse of this workflow in other files here
inputs:
lima_template:
description: lima template
type: string
default: "template://ubuntu-24.04"
container_engine:
description: container engine
type: string
default: "docker"
kubelet_port:
description: kubelet serving port
type: string
Expand Down Expand Up @@ -30,26 +38,13 @@ jobs:
name: "Multi node (emulated using Lima)"
runs-on: ubuntu-24.04
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
include:
- lima-template: template://ubuntu-24.04
engine: docker
- lima-template: template://ubuntu-24.04
engine: nerdctl
- lima-template: template://centos-stream-9
engine: podman
- lima-template: template://fedora
engine: podman
env:
LIMA_TEMPLATE: "${{ matrix.lima-template }}"
CONTAINER_ENGINE: "${{ matrix.engine }}"
LIMA_TEMPLATE: "${{ matrix.lima_template }}"
CONTAINER_ENGINE: "${{ matrix.container_engine }}"
U7S_PORT_KUBE_APISERVER: "${{ inputs.kube_apiserver_port }}"
U7S_PORT_FLANNEL: "${{ inputs.flannel_port }}"
U7S_PORT_KUBELET: "${{ inputs.kubelet_port }}"
U7S_PORT_ETCD: "${{ inputs.etcd_port }}"
DOCKER_BUILDKIT: 1
KUBECONFIG: ./kubeconfig
steps:
- uses: actions/checkout@v4
Expand Down
17 changes: 8 additions & 9 deletions .github/workflows/reusable-single-node.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@ name: Single Node
on:
workflow_call:
inputs:
container_engine:
description: container engine
type: string
default: "docker"
kubelet_port:
description: kubelet serving port
type: string
Expand Down Expand Up @@ -29,17 +33,12 @@ jobs:
name: "Single node"
runs-on: ubuntu-24.04
timeout-minutes: 40
strategy:
fail-fast: false
matrix:
engine: [docker, nerdctl, podman]
env:
CONTAINER_ENGINE: "${{ matrix.engine }}"
CONTAINER_ENGINE: "${{ inputs.container_engine }}"
U7S_PORT_KUBE_APISERVER: "${{ inputs.kube_apiserver_port }}"
U7S_PORT_FLANNEL: "${{ inputs.flannel_port }}"
U7S_PORT_KUBELET: "${{ inputs.kubelet_port }}"
U7S_PORT_ETCD: "${{ inputs.etcd_port }}"
DOCKER_BUILDKIT: 1
KUBECONFIG: ./kubeconfig
steps:
- uses: actions/checkout@v4
Expand All @@ -55,7 +54,7 @@ jobs:
# Preinstalled Moby does not contain dockerd-rootless-setuptool.sh
run: sudo apt-get remove moby-engine-*
- name: Set up Rootless Docker
if: ${{ matrix.engine == 'docker' }}
if: ${{ inputs.container_engine == 'docker' }}
run: |
set -eux -o pipefail
curl https://get.docker.com | sudo sh
Expand All @@ -64,14 +63,14 @@ jobs:
dockerd-rootless-setuptool.sh install
docker info
- name: Set up Rootless nerdctl
if: ${{ matrix.engine == 'nerdctl' }}
if: ${{ inputs.container_engine == 'nerdctl' }}
run: |
set -eux -o pipefail
sudo ./init-host/init-host.root.d/install-nerdctl.sh
./init-host/init-host.rootless.sh
nerdctl info
- name: Set up Rootless Podman
if: ${{ matrix.engine == 'podman' }}
if: ${{ inputs.container_engine == 'podman' }}
run: |
set -eux -o pipefail
sudo apt-get update
Expand Down
19 changes: 15 additions & 4 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,25 +1,36 @@
ARG BASE_IMAGE=docker.io/kindest/node:v1.31.2@sha256:18fbefc20a7113353c7b75b5c869d7145a6abd6269154825872dc59c1329912e
ARG CNI_PLUGINS_VERSION=v1.6.1
ARG HELM_VERSION=v3.16.3
ARG FLANNEL_VERSION=v0.26.1
FROM ${BASE_IMAGE}
COPY Dockerfile.d/SHA256SUMS.d/ /tmp/SHA256SUMS.d
ARG CNI_PLUGINS_VERSION
ARG HELM_VERSION
ARG FLANNEL_VERSION
RUN arch="$(uname -m | sed -e s/x86_64/amd64/ -e s/aarch64/arm64/)" && \
fname="cni-plugins-linux-${arch}-${CNI_PLUGINS_VERSION}.tgz" && \
curl -o "${fname}" -fSL "https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/${fname}" && \
grep "${fname}" "/tmp/SHA256SUMS.d/cni-plugins-${CNI_PLUGINS_VERSION}" | sha256sum -c && \
mkdir -p /opt/cni/bin && \
tar xzf "${fname}" -C /opt/cni/bin && \
rm -f "${fname}" && \
curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 && \
chmod 700 get_helm.sh && ./get_helm.sh && rm -f get_helm.sh
fname="helm-${HELM_VERSION}-linux-${arch}.tar.gz" && \
curl -o "${fname}" -fSL "https://get.helm.sh/${fname}" && \
grep "${fname}" "/tmp/SHA256SUMS.d/helm-${HELM_VERSION}" | sha256sum -c && \
tar xzf "${fname}" -C /usr/local/bin --strip-components=1 -- "linux-${arch}/helm" && \
rm -f "${fname}" && \
fname="flannel.tgz" && \
curl -o "${fname}" -fSL "https://github.com/flannel-io/flannel/releases/download/${FLANNEL_VERSION}/${fname}" && \
grep "${fname}" "/tmp/SHA256SUMS.d/flannel-${FLANNEL_VERSION}" | sha256sum -c && \
tar xzf "${fname}" -C / && \
rm -f "${fname}"
# gettext-base: for `envsubst`
# moreutils: for `sponge`
# socat: for `socat` (to silence "[WARNING FileExisting-socat]" from kubeadm)
RUN apt-get update && apt-get install -y --no-install-recommends \
gettext-base \
moreutils \
socat \
git
socat
ADD Dockerfile.d/etc_udev_rules.d_90-flannel.rules /etc/udev/rules.d/90-flannel.rules
ADD Dockerfile.d/u7s-entrypoint.sh /
ENTRYPOINT ["/u7s-entrypoint.sh", "/usr/local/bin/entrypoint", "/sbin/init"]
1 change: 1 addition & 0 deletions Dockerfile.d/SHA256SUMS.d/flannel-v0.26.1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0289f15fb91b4a14bf5c66d34f8d0ac2aebc6e68e450dbecef4af9871884202b flannel.tgz
2 changes: 2 additions & 0 deletions Dockerfile.d/SHA256SUMS.d/helm-v3.16.3
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
f5355c79190951eed23c5432a3b920e071f4c00a64f75e077de0dd4cb7b294ea helm-v3.16.3-linux-amd64.tar.gz
5bd34ed774df6914b323ff84a0a156ea6ff2ba1eaf0113962fa773f3f9def798 helm-v3.16.3-linux-arm64.tar.gz
90 changes: 5 additions & 85 deletions Makefile.d/install-flannel.sh
Original file line number Diff line number Diff line change
@@ -1,92 +1,12 @@
#!/bin/bash
set -eu -o pipefail

function INFO() {
echo >&2 -e "\e[104m\e[97m[INFO]\e[49m\e[39m $@"
}
function WARNING() {
echo >&2 -e "\e[101m\e[97m[WARNING]\e[49m\e[39m $@"
}

function ERROR() {
echo >&2 -e "\e[101m\e[97m[ERROR]\e[49m\e[39m $@"
}

# See chart values, 0 indicates default for platform
# https://github.com/flannel-io/flannel/blob/v0.26.1/chart/kube-flannel/values.yaml
: "${U7S_PORT_FLANNEL:='0'}"
: "${U7S_PORT_ETCD:='2379'}"

INFO "Flannel port: ${U7S_PORT_FLANNEL}"
INFO "ETCD port: ${U7S_PORT_ETCD}"

# Check hard dependency commands
for cmd in helm kubectl git; do
if ! command -v "${cmd}" >/dev/null 2>&1; then
ERROR "Command \"${cmd}\" is not installed"
exit 1
fi
done

# We need to customize the values.yaml to expose the backendPort and args
flannel_root=$(mktemp -d -u -t flannel-XXXXXXX)
git clone --quiet --depth 1 --branch v0.26.1 https://github.com/flannel-io/flannel $flannel_root
cd $flannel_root/chart

# Write a new values.yaml that exposes what we need
cat <<EOF > ./new-values.yaml
---
global:
imagePullSecrets:
# - name: "a-secret-name"
# The IPv4 cidr pool to create on startup if none exists. Pod IPs will be
# chosen from this range.
podCidr: "10.244.0.0/16"
podCidrv6: ""
flannel:
# kube-flannel image
image:
repository: docker.io/flannel/flannel
tag: v0.26.1
image_cni:
repository: docker.io/flannel/flannel-cni-plugin
tag: v1.5.1-flannel2
# flannel command arguments
enableNFTables: false
args:
- "--ip-masq"
- "--kube-subnet-mgr"
# Disabled, but left here for awareness that it can be set.
# It is not used as kube-subnet-mgr is enabled:
# https://github.com/flannel-io/flannel/blob/v0.26.1/Documentation/configuration.md
# - "--etcd-endpoints=\"http://127.0.0.1:4001,https://${U7S_HOST_IP}:${U7S_PORT_ETCD},http://127.0.0.1:${U7S_PORT_ETCD}\""
# Backend for kube-flannel. Backend should not be changed
# at runtime. (vxlan, host-gw, wireguard, udp)
# Documentation at https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md
backend: "vxlan"
# Port used by the backend 0 means default value (VXLAN: 8472, Wireguard: 51821, UDP: 8285)
backendPort: ${U7S_PORT_FLANNEL}
tolerations:
- effect: NoExecute
operator: Exists
- effect: NoSchedule
operator: Exists
netpol:
enabled: false
EOF

mv ./new-values.yaml ./kube-flannel/values.yaml

# Run this first in case a failure with kubectl
kubectl get pods -n kube-flannel
# Fall back to warning so a re-install does not fail
kubectl create namespace kube-flannel || WARNING "kube-flannel namespace might have been already created"
kubectl label --overwrite namespace kube-flannel pod-security.kubernetes.io/enforce=privileged || true
# If the command is issued again, this cleanup is needed
helm delete flannel --namespace kube-flannel kube-flannel || true
# We could also do --set flannel.backendPort=<value> but it's the same to set as the default
helm install flannel --namespace kube-flannel kube-flannel
cd -
rm -rf $flannel_root
if ! helm -n kube-flannel list -q | grep flannel; then
kubectl create namespace kube-flannel
kubectl label --overwrite namespace kube-flannel pod-security.kubernetes.io/enforce=privileged
helm install flannel --namespace kube-flannel --set-json flannel.backendPort=${U7S_PORT_FLANNEL} /flannel
fi

0 comments on commit 8dd5ad3

Please sign in to comment.