1.重新构建 jnlp-slave
# ls config docker-18.06.1-ce.tgz Dockerfile kubectl
A.创建 config
# vim config
apiVersion: v1
clusters:
- cluster:
certificate-authority: /etc/kubernetes/ssl/ca.pem
server: https://192.168.100.180:8443
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: admin
name: kubernetes
current-context: kubernetes
kind: Config
preferences: {}
users:
- name: admin
user:
client-certificate: /etc/kubernetes/ssl/admin.pem
client-key: /etc/kubernetes/ssl/admin-key.pemB.创建 Dockerfile
# vim Dockerfile
FROM jenkins/jnlp-slave:3.23-1-alpine
MAINTAINER zhi <wangzhijiansd@qq.com>
USER root
ARG DOCKER_GID=994
ENV DOCKER_VERSION=18.06.1-ce
ENV SSL_DIR /etc/kubernetes/ssl
# 提取 docker 二进制文件
COPY docker-${DOCKER_VERSION}.tgz /var/tmp/
RUN tar --strip-components=1 -xvzf /var/tmp/docker-${DOCKER_VERSION}.tgz -C /usr/local/bin \
&& rm -rf /var/tmp/docker-${DOCKER_VERSION}.tgz \
&& chmod -R 775 /usr/local/bin/docker
# 安装 kubectl
COPY kubectl /usr/local/bin/
# 此处文件均为空文件将在运行时由 ConfigMap 挂载为 Volume 填充真实证书文件
COPY config ${SSL_DIR}/
RUN mkdir -p /root/.kube/ && mkdir -p ${SSL_DIR} \
touch ${SSL_DIR}/ca.pem \
touch ${SSL_DIR}/admin.pem \
touch ${SSL_DIR}/admin-key.pem
RUN export KUBE_CONFIG==${SSL_DIR}/config && kubectl config view
RUN addgroup -g ${DOCKER_GID} docker && adduser jenkins docker
# 暴露证书文件所在文件夹，在运行时由 ConfigMap 挂载为 Volume 填充真实证书文件
VOLUME ${SSL_DIR}
USER jenkins:${DOCKER_GID}注: docker 和 kubectl 跟 kubernetes 集群的版本一致。同时说明一下，这里GitLab 、Jenkins都部署在kube-ops这个namespace下。而docker使用了docker in docker 挂载宿主机/var/run/docker.dock进行操作。
C.构建 jnlp-slave 并推送至私有镜像仓库
# docker build -t 192.168.100.100/jenkinsci/jnlp-slave:latest . # docker images|grep jnlp-slave 192.168.100.100/jenkinsci/jnlp-slave latest d3b287300a83 6 days ago 377MB # docker push 192.168.100.100/jenkinsci/jnlp-slave:latest
2.创建并查看 configmap
# kubectl create configmap kubectl-cert-cm --from-file=/etc/kubernetes/ssl -n kube-ops # kubectl describe configmap kubectl-cert-cm -n kube-ops
3.部署 Jenkins
注: 由于无法解决配置了专有 secret 而 Jenkins 依然去找 default 的 secret 的问题，这里重新配置了 RBAC 以使用 default 的 secret 并绑定了 cluster-admin 角色。
A.创建 RBAC
# vim jenkins-rbac.yaml kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: name: default namespace: kube-ops rules: - apiGroups: [""] resources: ["pods"] verbs: ["create","get","update","list","watch","patch","delete"] - apiGroups: [""] resources: ["pods/exec"] verbs: ["create","get","update","list","watch","patch","delete"] - apiGroups: [""] resources: ["pods/log"] verbs: ["get","list","watch"] - apiGroups: [""] resources: ["secrets"] verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: default namespace: kube-ops roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cluster-admin subjects: - kind: ServiceAccount name: default namespace: kube-ops --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: default namespace: kube-ops roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: default subjects: - kind: ServiceAccount name: default namespace: kube-ops
B.创建PVC
# vim jenkins-pvc.yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: jenkins-pv-claim namespace: kube-ops labels: app: jenkins spec: storageClassName: kube-ops accessModes: - ReadWriteOnce resources: requests: storage: "10Gi"
C.创建 deployment 和 service
# vim jenkins.yaml apiVersion: extensions/v1beta1 kind: Deployment metadata: name: jenkins namespace: kube-ops spec: replicas: 1 template: metadata: name: jenkins labels: name: jenkins spec: #serviceAccountName: jenkins serviceAccountName: default securityContext: runAsUser: 1000 fsGroup: 1000 containers: - name: jenkins image: 192.168.100.100/jenkinsci/jenkins:lts resources: requests: memory: "1Gi" cpu: "1000m" limits: memory: "2Gi" cpu: "2000m" env: - name: JAVA_OPTS value: "-Duser.timezone=Asia/Shanghai" ports: - name: web containerPort: 8080 - name: agent containerPort: 50000 volumeMounts: - mountPath: /var/jenkins_home name: jenkins-data volumeMounts: - mountPath: /var/run/docker.sock readOnly: false name: docker-sock volumes: - name: jenkins-data persistentVolumeClaim: claimName: jenkins-pv-claim - name: docker-sock hostPath: path: /var/run/docker.sock --- apiVersion: v1 kind: Service metadata: name: jenkins namespace: kube-ops labels: name: jenkins spec: type: LoadBalancer ports: - name: web port: 8080 targetPort: web - name: agent port: 50000 targetPort: agent selector: name: jenkins
注:这里就不在赘述Jenkins的部署。也不再赘述gitlab的部署，需要更改的地方大同小异。
4.配置Jenkins
5.新建项目
# git clone http://192.168.100.185:32030/wangzhijian/grafana.git 正克隆到 'grafana'... Username for 'http://192.168.100.185:32030': wangzhijian Password for 'http://wangzhijian@192.168.100.185:32030': remote: Counting objects: 37, done. remote: Compressing objects: 100% (21/21), done. remote: Total 37 (delta 7), reused 37 (delta 7) Unpacking objects: 100% (37/37), done. # ls grafana/ build grafana.yaml # ls grafana/build/ build.sh Dockerfile
注:该项非必须，只是想显示一下文件存放目录。本来我想使用本地添加hosts，然后使用域名来git，但是不知道是因为添加了traefik代理的原因还是traefik的SSL没有完全配置好的原因，总之这里没有解决使用域名来git的问题。
使用IP进行git操作的格式:git clone http://nodeip:port/yourname/project.git
A.配置Dockerfile
# vim Dockerfile
ARG GRAFANA_VERSION="latest"
FROM 192.168.100.100/grafana/grafana:${GRAFANA_VERSION}
USER grafana
ARG GF_INSTALL_PLUGINS=""
RUN if [ ! -z "${GF_INSTALL_PLUGINS}" ]; then \
OLDIFS=$IFS; \
IFS=','; \
for plugin in ${GF_INSTALL_PLUGINS}; do \
IFS=$OLDIFS; \
grafana-cli --pluginsDir "$GF_PATHS_PLUGINS" plugins install ${plugin}; \
done; \B.配置构建脚本
# vim build.sh #!/bin/bash IMAGE="192.168.100.100/grafana/grafana-plugins" # 编译镜像 docker build -f build/Dockerfile -t $IMAGE \ --build-arg "GRAFANA_VERSION=latest" \ --build-arg "GF_INSTALL_PLUGINS=grafana-worldmap-panel,grafana-clock-panel,grafana-piechart-panel,alexanderzobnin-zabbix-app,grafana-kubernetes-app,grafana-simple-json-datasource,michaeldmoore-annunciator-panel" . # 登录Harbor镜像仓库 docker login -u admin -p Harbor12345 192.168.100.100 # 上传镜像 docker push $IMAGE # 清理镜像 docker rmi $IMAGE
C.创建yaml文件
# vim grafana.yaml apiVersion: extensions/v1beta1 kind: Deployment metadata: name: grafana namespace: kube-system spec: replicas: 1 template: metadata: name: grafana labels: k8s-app: grafana spec: containers: - name: grafana image: 192.168.100.100/grafana/grafana-plugins:latest ports: - containerPort: 3000 protocol: TCP env: - name: GF_SERVER_HTTP_PORT value: "3000" - name: GF_AUTH_BASIC_ENABLED value: "true" - name: GF_AUTH_ANONYMOUS_ENABLED value: "false" - name: GF_SECURITY_ADMIN_USER value: "admin" - name: GF_SECURITY_ADMIN_PASSWORD value: "wangzhijian" - name: GF_SERVER_ROOT_URL value: /api/v1/namespaces/kube-system/services/monitoring-grafana/proxy value: / - name: GF_SMTP_ENABLED value: "true" - name: GF_SMTP_HOST value: smtp.qq.com:465 - name: GF_SMTP_USER value: wangzhijiansd@qq.com - name: GF_SMTP_PASSWORD value: ******** - name: GF_SMTP_FROM_ADDRESS value: wangzhijiansd@qq.com --- apiVersion: v1 kind: Service metadata: labels: kubernetes.io/cluster-service: 'true' kubernetes.io/name: Grafana name: grafana namespace: kube-system spec: type: LoadBalancer ports: - port: 80 targetPort: 3000 selector: k8s-app: grafana
6.使用Jenkins新建任务
7.构建输出
A.构建镜像并推送到私有镜像仓库
B.部署grafana
C.查看部署情况
# kubectl cluster-info|grep Grafana Grafana is running at https://192.168.100.180:8443/api/v1/namespaces/kube-system/services/grafana/proxy # kubectl -n kube-system get pod|grep grafana grafana-576fd64977-2kj6c 1/1 Running 0 1d # kubectl -n kube-system get service|grep grafana grafana LoadBalancer 10.244.249.26 <pending> 80:36902/TCP 1d
这里使用浏览器输入http://nodeip:port,使用变量配置的用户名(admin)和密码(wangzhijian)进行登录
构建镜像时添加的一些插件显示如下:
特别感谢:
为了将kubectl运行起来，试过来很多方法，也搜了很多资料，但是依然不得其所，感谢作者和他的这篇文章:
基于 IBM Cloud Private 的 DevOps 实践
转载于:https://blog.51cto.com/wangzhijian/2285861