k8s ci & cd pipeline

---

Overview

Setup nexus private registry

generate ssl certificate and restart nexus

export NEXUS_HOST=your_hostname
export NEXUS_DOMAIN=your_domain_name.com
export NEXUS_IP_ADDRESS=192.168.10.10
export NEXUS_SSL_PORT=7777

keytool -genkeypair -keystore keystore.jks -storepass keystorePass -keypass keystorePass -alias jetty -keyalg RSA -keysize 2048 -validity 5000 -dname "CN=$NEXUS_DOMAIN, OU=Example, O=Sonatype, L=Unspecified, ST=Unspecified, C=US" -ext "SAN=DNS:$NEXUS_DOMAIN,IP:$NEXUS_IP_ADDRESS" -ext "BC=ca:true"

openssl req -newkey rsa:4096 -nodes -sha256 -keyout ${nexus_home}/etc/ssl/nexus.key -x509 -days 3650 -out ${nexus_home}/etc/ssl/nexus.cert

ls ${nexus_home}/etc/ssl/
keystore.jks  nexus.cert  nexus.key

etc/nexus-default.properties

application-port-ssl=8444
nexus-args=${jetty.etc}/jetty.xml,${jetty.etc}/jetty-http.xml,${jetty.etc}/jetty-https.xml,${jetty.etc}/jetty-requestlog.xml

etc/nexus.properties

application-port-ssl=7777

restart nexus service

${nexus_home}/bin/nexus restart

create nexus repository and set Repository Connectors in nexus UI

#hosted type repository for docker push
HTTPS: 7777

#group type repository for docker pull
HTTPS: 8888

setup ssl trust in docker client node(not always necessary)

#remove old cert
cd /etc/pki/ca-trust/source/anchors
rm –f your_domain_name.crt

#client trust
export DOMAIN_NAME=your_domain_name
export TCP_PORT=7777
openssl s_client -connect $DOMAIN_NAME:$TCP_PORT -showcerts </dev/null 2>/dev/null | openssl x509 -outform PEM | tee /etc/pki/ca-trust/source/anchors/$DOMAIN_NAME.crt

update-ca-trust

configure /etc/docker/daemon.json and restart docker service

{
    "insecure-registries" : ["your_domain_name:7777","your_domain_name:8888"]
}

restart docker service and login

/bin/systemctl restart docker.service

#test docker login
docker login your_domain_name:7777
docker login your_domain_name:8888

Create k8s ha cluster with ansible-galaxy scrips using kubeadm

ansible inventory example

master1   ansible_host=192.168.10.101
master2   ansible_host=192.168.10.102
master3   ansible_host=192.168.10.103
worker1   ansible_host=192.168.10.104
worker2   ansible_host=192.168.10.105

[center]
worker2

[k8schaos]
master1

[k8smaster]
master2
master3

[k8sworker]
worker1
worker2

install haproxy in center node, configure 3 node master as backend server

haproxy_conf: |
    listen k8smaster
      bind *:6443
      option tcplog
      mode tcp
      balance roundrobin
      option tcp-check
      server master1 192.168.10.101:6443 check fall 3 rise 2
      server master2 192.168.10.102:6443 check fall 3 rise 2
      server master3 192.168.10.103:6443 check fall 3 rise 2

prepare k8s images, push to private registry and load images to nodes

install first k8schaos node with kubeadm

kubeadm init --config /tmp/kubeadm-config.yml --upload-certs --v=5

install two extra k8smaster node and join to cluster
install k8sworker node

Helm install jenkins service in k8s

prepare images and helm charts
load docker images to nodes first
install helm and tiller

kubectl apply -f rbactiller.yml
helm init --tiller-image registry.hub.docker.com/jessestuart/tiller:v2.14.2 --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts --service-account=tiller

install nfs-client-provisioner

helm install --name ndp --values values.yaml --namespace ndp ./nfs-client-provisioner

exec helm install for jenkins

helm install --name jenkins --values values.yaml --namespace jenkins ./1.9.2/jenkins

install jenkins plugins and adjust settings

update timezone and security settings:

1, System.setProperty(‘org.apache.commons.jelly.tags.fmt.timeZone’, ‘Asia/Shanghai’)
2, Configure Global Security -> Jenkins’ own user database -> Project-based Matrix Authorization Strategy

update slave settings:（Docker in Docker）
Host Path Volume:
/usr/bin/docker
/var/run/docker.sock

update default PersistentVolume recycle policy with

kubectl patch pv $(kubectl get pv,pvc -n jenkins | grep -v NAME | grep jenkins/jenkins | awk '{print $1}' | cut -d'/' -f2) -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}'

Create ci cd pipeline sample code in k8s (non-docker)

pipeline {
    agent { label "jenkins-jenkins-slave" } // jenkins slave with maven sonar-scanner tools
    ...
    options {
        buildDiscarder(logRotator(numToKeepStr: '5'))
        preserveStashes()
        { skipDefaultCheckout() }
    }

    stages {
        stage("Code Checkout") {
            steps { ... }
        }

        stage('Build') {
            steps {
                script{
                    try {
                        script {
                            try { ... } finally { }
                        }
                    } catch(error) {
                        echo "First build failed, let's retry if accepted"
                        retry(2) { // same code as above }
                    }
                }
            }
        }

        stage('Assemble & Test') { ... }

        stage('Scan & Upload') {
            parallel {
                stage('Archive Artifacts') {
                    steps {
                        archiveArtifacts "project_name-assembly-${PKG_VERSION}-SNAPSHOT.zip"
                    }
                }
                stage("Upload Artifacts to Nexus") {
                 steps {
                     nexusArtifactUploader ...
                 }
                }
                stage("SonarQube Scan") {
                    steps {
                        script {
                            if (env.ENABLE_SONAR == 'Y') {
                                withSonarQubeEnv('dev_sonarqube') { // defined in jenkins global setting
                                    sh ''' ... '''
                                }
                                sleep(120)
                                timeout(time: 1, unit: 'MINUTES') {
                                    def qg = waitForQualityGate()
                                    if (qg.status != 'SUCCESS') {
                                        echo "Pipeline aborted due to quality gate failure: ${qg.status}"
                                        // error "Pipeline aborted due to quality gate failure: ${qg.status}"
                                    }
                                }
                            } else { }
                        }
                    }
                }
            }
        }

        stage('Deploy DEV') {
            agent { label "jenkins-slave-deploy" } // slave with deploy tools installed, deploy dev by default
            steps {
                script { ... }
            }
        }

        stage('Deploy QA') {
            agent { label "jenkins-slave-deploy" }
            when {
                environment name: 'DEPLOY_QA', value: 'Y'
            }
            steps {
                script { // refer dev deploy script }
            }
        }
    }
}

Create ci cd pipeline in k8s (build docker image and push to private registry)

stage("Build Docker Image") {
    steps {
        script {
            def app

            stage('Build image') {
                withDockerRegistry(credentialsId: "${IMAGE_NEXUS_CREDENTIAL_ID}", url: "http://${REPO}/v1") {
                    app = docker.build("${REPO_PUSH}/project_name/app_name-service:${env.DOMAIN}", "--label project_name/app_name-service --no-cache --pull --force-rm --build-arg REPO=${REPO} --build-arg DOMAIN=${DOMAIN} --build-arg GROUP_ID=${DOMAIN} --build-arg LAUNCHER_VERSION=${LAUNCHER_VERSION} --build-arg PKG_VERSION=${IMAGE_VERSION} --build-arg NEXUS_REPO=${NEXUS_REPO} .")
                }
            }
            
            stage('Push image') {
                withDockerRegistry(credentialsId: "${IMAGE_NEXUS_CREDENTIAL_ID}", url: "http://${REPO_PUSH}/v1") {
                    app.push("${env.IMAGE_VERSION}-${env.BUILD_NUMBER}")
                    app.push("${env.IMAGE_VERSION}-latest")
                }
            }
        }
    }
}