HPA原理图

HPA的一些默认机制

• hpa默认每隔15秒校验一次指标调谐副本数
• 默认的HPA相关指标容差为10%
• 在HPA中默认的扩容冷却周期是3min
• 缩容冷却周期是5min
• 针对同一个资源，可以配置多个hpa同时生效，是or的关系，取扩容值最大的
• HPA调谐实例数算法

desiredReplicas = ceil[currentReplicas * ( currentMetricValue / desiredMetricValue )]

期望的副本数 = 向上取整 [ ( 当前指标值 ➗ 期望指标值 ) ✖️ 当前副本数 ]

配置模板

hpa.yml

apiVersion: autoscaling/v2beta1
kind: HorizontalPodAutoscaler
metadata:
  name: {ENV}-{PROJECT}-hpa
  namespace: {K8S_NAMESPACE}
  labels:
    qcloud-app: {ENV}-{PROJECT}-hpa 
spec:
  # HPA的伸缩对象描述，HPA会动态修改该对象的pod数量
  scaleTargetRef:
      apiVersion: apps/v1
      kind: Deployment
      name: {PROJECT}
  # HPA的最小pod数量和最大pod数量
  minReplicas: {MIN}
  maxReplicas: {MAX}
  # 监控的指标数组，支持多种类型的指标共存
  metrics:
  - type: Pods
    pods:
      # 更多https://cloud.tencent.com/document/product/457/38929
      metricName: {METRICNAME}
      targetAverageValue: {THRESHOLD}

pipeline关键配置

# 变量
MIN = 2
MAX = 3
METRICNAME = "k8s_pod_rate_mem_no_cache_limit"
THRESHOLD = 90
HPAYML = "hpa.yml"

# 应用HPA
echo "\033[46;30m************************************************ 配置HPA开始 ************************************************\033[0m"
sh "cp /data/template/k8s/${HPAYML} ."
sh "sed -i -e 's#{ENV}#${ENV}#g;s#{PROJECT}#${PROJECT}#g;s#{K8S_NAMESPACE}#${K8S_NAMESPACE}#g;s#{MIN}#${MIN}#g;s#{MAX}#${MAX}#g;s#{METRICNAME}#${METRICNAME}#g;s#{THRESHOLD}#${THRESHOLD}#g;' ${HPAYML}"
sh "kubectl --kubeconfig /data/kubecfg/${CLUSTER} apply -f ${HPAYML}"
echo "\033[46;30m************************************************ 配置HPA结束 ************************************************\033[0m"

完整pipeline

pipeline {
    agent {
        label 'master'
    }
    options {
        ansiColor('xterm')
        buildDiscarder(logRotator(daysToKeepStr: '1', numToKeepStr: '3')) 
    }
    tools {
        maven 'apache_maven_3.5.0'
        jdk 'jdk_1.8_202'
        git 'git_2.19.1'
        dockerTool 'docker_19.03.12'
    }
    parameters{
        booleanParam(name: 'DEPLOY', defaultValue: true, description: '发布应用到环境【如果去掉勾选，则只构建docker镜像,不发布到环境】')
        booleanParam(name: 'CHECK_CODE_QUALITY', defaultValue: false, description: '静态代码质量检查【勾选为检查，不勾选为不检查】')
    }
    environment {
        GIT = 'http://gitlab.demo.com/logistics/logistics-base.git'
        MIN = 1
        MAX = 4
        METRICNAME = "k8s_pod_rate_mem_no_cache_limit"
        THRESHOLD = 30
        HPAYML = "hpa.yml"
        IMAGE_GROUP = "logistics" //对应harbor镜像分组
        REPLICAS = 1
        TEMPLATE="deployment.yml"
        DOCKERFILE="Dockerfile-jmx"
        NODE_SELECTOR = " "
        LIMIT_MEM="2048Mi"
        LIMIT_CPU="1000m"
        JVM=""" ,"-Xms1400m","-Xmx1400m","-XX:+UseG1GC","-XX:MaxGCPauseMillis=300","-XX:MaxTenuringThreshold=10","-XX:+UnlockExperimentalVMOptions","-XX:+UseCGroupMemoryLimitForHeap","-XX:+HeapDumpOnOutOfMemoryError","-XX:HeapDumpPath=/dumps/oom","-XX:+ExitOnOutOfMemoryError","-XX:OnOutOfMemoryError=./dump-handler -k \$HOSTNAME -e \$ENV" """
        XDIAMOND=""" ,"-Dxdiamond.host=innerxdiamond.demo.com" """
        JMX=""" ,"-javaagent:jmx_prometheus_javaagent-0.14.0.jar=12345:jmx-cfg.yml" """
        ARGS="""["-jar"${JMX}${XDIAMOND},"-server"${JVM},"-Dprofile.active=${NEWENV}","-Dspring.profiles.active=${NEWENV}","-Dserver.port=8888","-Dport=8888","${PROJECT}.jar"]"""
        K8S_NAMESPACE = "${ENV}-${IMAGE_GROUP}"
        CLUSTER = sh(script: """echo ${JOB_BASE_NAME} | awk -F '-' '{if (\$1=="dev") {print "test-cluster"} else {print \$1"-cluster"}}' """, returnStdout: true).trim()
        PROJECT = sh(script: """echo ${GIT} | awk -F '/' '{print \$NF}' | awk -F '.' '{print \$1}' | awk -F '-' '{if (\$1=="${IMAGE_GROUP}") {print \$0} else {print "${IMAGE_GROUP}-"\$0}}'| tr "[:upper:]" "[:lower:]" """, returnStdout: true).trim()
        ENV = sh(script: "echo ${JOB_BASE_NAME} | awk -F '-' '{print \$1}'", returnStdout: true).trim()
        NEWENV = sh(script: """echo ${JOB_BASE_NAME} | awk -F '-' '{if (\$1=="test") {print "new"\$1} else {print \$1}}' """, returnStdout: true).trim()
        SKYWALKING_SERVER="skywalking.demo.com"
        SKYWALKING=""" ,"-javaagent:agent/skywalking-agent.jar","-Dskywalking.collector.backend_service=${SKYWALKING_SERVER}","-Dskywalking.agent.namespace=${ENV}","-Dskywalking.agent.service_name=${ENV}-${PROJECT}" """
        HARBOR_HOST = 'harbor.demo.com'
        DOCKER_IMAGE = "${IMAGE_GROUP}/${JOB_BASE_NAME}:${VERSION_VALUE}"
        CHECK_TAG = sh(script: "echo ${BRANCH_OR_TAG} | awk -F '/' '{if (\$3) print \$3; else print \$1}'", returnStdout: true).trim()  // 分支或tag
        VERSION_VALUE = "${CHECK_TAG}-${TIME}" // 分支或tag
        TIME = sh(script: "date '+%Y%m%d%H%M%S'", returnStdout: true).trim()
    }
    stages {
        stage ('代码获取') {
            steps {
              echo "\033[46;30m************************************************ 拉取代码开始 ************************************************\033[0m"
              deleteDir() // 清理工作目录
              git credentialsId: 'gitlab_username_password_credential', url: "${GIT}"
              sh '[ -n "${CHECK_TAG}" ] &&  git checkout ${CHECK_TAG} ||  { echo -e "切换至指定的tag的版本，tag：${CHECK_TAG} 不存在或为空，请检查输入的tag!" && exit 111; }'
              buildName "${CHECK_TAG}"
              echo "\033[46;30m************************************************ 拉取代码结束 ************************************************\033[0m"
            }
        }

        stage('代码静态检查') {
            when{
                expression {
                    params.CHECK_CODE_QUALITY == true
                }
            }
            steps {
                echo "\033[46;30m************************************************ 代码静态检查开始 ************************************************\033[0m"
                withSonarQubeEnv("sonar_server") {
                    sh "mvn sonar:sonar \
                          -Dsonar.projectKey=sonar-check \
                          -Dsonar.host.url=http://sonar.demo.com:9000 \
                          -Dsonar.login=32d06d4d9b19cedb8192b3abbafdd2a4dd15170a"
                }
                echo "\033[46;30m************************************************ 代码静态检查结束 ************************************************\033[0m"
            }
        }

        stage('检查结果分析') {
            when{
                expression {
                    params.CHECK_CODE_QUALITY == true
                }
            }
            steps {
                echo "\033[46;30m************************************************ 检查结果分析开始 ************************************************\033[0m"
                script {
                    timeout(10) { 
                        def qg = waitForQualityGate() 
                            if (qg.status != 'OK') {
                                echo "\033[0;37;41m ========== 未通过代码质量阈检查，请及时修改！检查失败: ${qg.status}  ==========\033[0m"
                            }
                        }
                }
                echo "\033[46;30m************************************************ 检查结果分析结束 ************************************************\033[0m"
            }
        }

        stage ('代码编译') {
            steps {
              echo "\033[46;30m************************************************ 编译打包开始 ************************************************\033[0m"
              sh 'mvn -version'
              sh 'mvn -U clean package -DskipTests'
              echo "\033[46;30m************************************************ 编译打包结束 ************************************************\033[0m"
            }
        }

        stage('镜像构建') {
            steps {
              echo "\033[46;30m************************************************ 镜像构建开始 ************************************************\033[0m"
              script {
                  sh "/usr/bin/cp -f /data/template/docker/${Dockerfile} Dockerfile"
                  sh "/usr/bin/cp -r -f /data/template/skyagent/agent ."
                  sh "/usr/bin/cp -r -f /data/template/tools/dump-handler ."
                  sh "/usr/bin/cp -r -f /data/template/tools/readyCheck ."
                  sh "/usr/bin/cp -r -f /data/template/jvm-metrics/{jmx_prometheus_javaagent-0.14.0.jar,jmx-cfg.yml} ."
                  sh "sed -i -e 's#{SW_AGENT_NAME:Your_ApplicationName}#${JOB_BASE_NAME}#g' agent/config/agent.config"
                  sh "sed -i 's/###PROJECT###/${PROJECT}/g' ./Dockerfile"
                  sh "docker build -t ${HARBOR_HOST}/${DOCKER_IMAGE} ."
                  sh "docker push ${HARBOR_HOST}/${DOCKER_IMAGE}"
                  sh "docker rmi ${HARBOR_HOST}/${DOCKER_IMAGE}"
              }
              echo "\033[46;30m************************************************ 镜像构建结束 ************************************************\033[0m"
            }
        }

        stage('发布服务至kubernetes集群') {
            when{
                expression {
                    params.DEPLOY == true
                }
            }
            steps {
                script {
                echo "\033[46;30m************************************************ 发布服务至kubernetes集群开始 ************************************************\033[0m"
                    sh "cp /data/template/k8s/${TEMPLATE} ${TEMPLATE}"
                    sh "sed -i -e 's#{IMAGE_URL}#${HARBOR_HOST}/${DOCKER_IMAGE}#g;s#{ENV}#${ENV}#g;s#{NODE_SELECTOR}#${NODE_SELECTOR}#g;s#{PROJECT}#${PROJECT}#g;s#{ARGS}#${ARGS}#g;s#{IMAGE_GROUP}#${IMAGE_GROUP}#g;s#{K8S_NAMESPACE}#${K8S_NAMESPACE}#g;s#{REPLICAS}#${REPLICAS}#g;s#{IMAGE_GROUP}#${IMAGE_GROUP}#g;s#{LIMIT_MEM}#${LIMIT_MEM}#g;s#{LIMIT_CPU}#${LIMIT_CPU}#g;' ${TEMPLATE}"
                    sh "kubectl --kubeconfig /data/kubecfg/${CLUSTER} cluster-info && kubectl --kubeconfig /data/kubecfg/${CLUSTER} get nodes"
                    sh "kubectl --kubeconfig /data/kubecfg/${CLUSTER} apply -f ${TEMPLATE} --namespace=${K8S_NAMESPACE}"
                    echo "\033[46;30m************************************************ 配置HPA开始 ************************************************\033[0m"
                    sh "cp /data/template/k8s/${HPAYML} ."
                    sh "sed -i -e 's#{ENV}#${ENV}#g;s#{PROJECT}#${PROJECT}#g;s#{K8S_NAMESPACE}#${K8S_NAMESPACE}#g;s#{MIN}#${MIN}#g;s#{MAX}#${MAX}#g;s#{METRICNAME}#${METRICNAME}#g;s#{THRESHOLD}#${THRESHOLD}#g;' ${HPAYML}"
                    sh "kubectl --kubeconfig /data/kubecfg/${CLUSTER} apply -f ${HPAYML}"
                    echo "\033[46;30m************************************************ 配置HPA结束 ************************************************\033[0m"
                echo "\033[46;30m************************************************ 发布服务至kubernetes集群结束 ************************************************\033[0m"
                }
            }
        }
    }
}