释放双眼，带上耳机，听听看~！
- 使用Helm安装Mysql:
??上一篇博文谈到了如何使用Helm安装Redis和RabbitMQ，下来我们来聊聊如何用Helm安装mysql.
??本人对于Mysql数据库不是非常熟悉，因为我们公司的分工比较明确，数据库这块的工作主要由DBA负责，运维同学只负责应用的维护。
??按照我们前面博文的描述，首先是在官方文档查看helm安装mysql的书名: https://github.com/helm/charts/tree/master/stable/mysql
??我根据官方文档的描述配置的value.yaml文件如下：
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
2## ref: https://hub.docker.com/r/library/mysql/tags/
3##
4image: "k8s.harbor.maimaiti.site/system/mysql"
5imageTag: "5.7.14"
6
7busybox:
8 image: "k8s.harbor.maimaiti.site/system/busybox"
9 tag: "1.29.3"
10
11testFramework:
12 image: "k8s.harbor.maimaiti.site/system/bats"
13 tag: "0.4.0"
14
15## Specify password for root user
16##
17## Default: random 10 character string
18mysqlRootPassword: admin123
19
20## Create a database user
21##
22mysqlUser: test
23## Default: random 10 character string
24mysqlPassword: test123
25
26## Allow unauthenticated access, uncomment to enable
27##
28# mysqlAllowEmptyPassword: true
29
30## Create a database
31##
32mysqlDatabase: test
33
34## Specify an imagePullPolicy (Required)
35## It's recommended to change this to 'Always' if the image tag is 'latest'
36## ref: http://kubernetes.io/docs/user-guide/images/#updating-images
37##
38imagePullPolicy: IfNotPresent
39
40extraVolumes: |
41 # - name: extras
42 # emptyDir: {}
43
44extraVolumeMounts: |
45 # - name: extras
46 # mountPath: /usr/share/extras
47 # readOnly: true
48
49extraInitContainers: |
50 # - name: do-something
51 # image: busybox
52 # command: ['do', 'something']
53
54# Optionally specify an array of imagePullSecrets.
55# Secrets must be manually created in the namespace.
56# ref: https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod
57# imagePullSecrets:
58 # - name: myRegistryKeySecretName
59
60## Node selector
61## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
62nodeSelector: {}
63
64## Tolerations for pod assignment
65## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
66##
67tolerations: []
68
69livenessProbe:
70 initialDelaySeconds: 30
71 periodSeconds: 10
72 timeoutSeconds: 5
73 successThreshold: 1
74 failureThreshold: 3
75
76readinessProbe:
77 initialDelaySeconds: 5
78 periodSeconds: 10
79 timeoutSeconds: 1
80 successThreshold: 1
81 failureThreshold: 3
82
83## Persist data to a persistent volume
84persistence:
85 enabled: true
86 ## database data Persistent Volume Storage Class
87 ## If defined, storageClassName: <storageClass>
88 ## If set to "-", storageClassName: "", which disables dynamic provisioning
89 ## If undefined (the default) or set to null, no storageClassName spec is
90 ## set, choosing the default provisioner. (gp2 on AWS, standard on
91 ## GKE, AWS & OpenStack)
92 ##
93 storageClass: "dynamic"
94 accessMode: ReadWriteOnce
95 size: 8Gi
96 annotations: {}
97
98## Configure resource requests and limits
99## ref: http://kubernetes.io/docs/user-guide/compute-resources/
100##
101resources:
102 requests:
103 memory: 256Mi
104 cpu: 100m
105
106# Custom mysql configuration files used to override default mysql settings
107configurationFiles: {}
108# mysql.cnf: |-
109# [mysqld]
110# skip-name-resolve
111# ssl-ca=/ssl/ca.pem
112# ssl-cert=/ssl/server-cert.pem
113# ssl-key=/ssl/server-key.pem
114
115# Custom mysql init SQL files used to initialize the database
116initializationFiles: {}
117# first-db.sql: |-
118# CREATE DATABASE IF NOT EXISTS first DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
119# second-db.sql: |-
120# CREATE DATABASE IF NOT EXISTS second DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
121
122metrics:
123 enabled: true
124 image: k8s.harbor.maimaiti.site/system/mysqld-exporter
125 imageTag: v0.10.0
126 imagePullPolicy: IfNotPresent
127 resources: {}
128 annotations: {}
129 # prometheus.io/scrape: "true"
130 # prometheus.io/port: "9104"
131 livenessProbe:
132 initialDelaySeconds: 15
133 timeoutSeconds: 5
134 readinessProbe:
135 initialDelaySeconds: 5
136 timeoutSeconds: 1
137
138## Configure the service
139## ref: http://kubernetes.io/docs/user-guide/services/
140service:
141 annotations: {}
142 ## Specify a service type
143 ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services---service-types
144 type: ClusterIP
145 port: 3306
146 # nodePort: 32000
147
148ssl:
149 enabled: false
150 secret: mysql-ssl-certs
151 certificates:
152# - name: mysql-ssl-certs
153# ca: |-
154# -----BEGIN CERTIFICATE-----
155# ...
156# -----END CERTIFICATE-----
157# cert: |-
158# -----BEGIN CERTIFICATE-----
159# ...
160# -----END CERTIFICATE-----
161# key: |-
162# -----BEGIN RSA PRIVATE KEY-----
163# ...
164# -----END RSA PRIVATE KEY-----
165
166## Populates the 'TZ' system timezone environment variable
167## ref: https://dev.mysql.com/doc/refman/5.7/en/time-zone-support.html
168##
169## Default: nil (mysql will use image's default timezone, normally UTC)
170## Example: 'Australia/Sydney'
171# timezone:
172
173# To be added to the database server pod(s)
174podAnnotations: {}
175
176podLabels: {}
177
178## Set pod priorityClassName
179# priorityClassName: {}
180
181
??主要修改了如下几点的配置：
将所有的镜像都改为了私服镜像地址； 配置了mysql的初始化root密码，普通用户账户和密码，创建一个测试数据库； 配置了持久化存储；
??使用helm install的时候也可以自定义参数，具体参数如何使用要看官方文档；比如:
2
2
??查看安装好之后的mysql如何连接，有一个my-mysql的服务直接调用即可；
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
2LAST DEPLOYED: Thu Apr 25 15:08:27 2019
3NAMESPACE: kube-system
4STATUS: DEPLOYED
5
6RESOURCES:
7==> v1/Pod(related)
8NAME READY STATUS RESTARTS AGE
9my-mysql-5fd54bd9cb-948td 2/2 Running 3 6d22h
10
11==> v1/Secret
12NAME TYPE DATA AGE
13my-mysql Opaque 2 6d22h
14
15==> v1/ConfigMap
16NAME DATA AGE
17my-mysql-test 1 6d22h
18
19==> v1/PersistentVolumeClaim
20NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
21my-mysql Bound pvc-ed8a9252-6728-11e9-8b25-480fcf659569 8Gi RWO dynamic 6d22h
22
23==> v1/Service
24NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
25my-mysql ClusterIP 10.200.200.169 <none> 3306/TCP,9104/TCP 6d22h
26
27==> v1beta1/Deployment
28NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE
29my-mysql 1 1 1 1 6d22h
30
31NOTES:
32MySQL can be accessed via port 3306 on the following DNS name from within your cluster:
33my-mysql.kube-system.svc.cluster.local
34
35To get your root password run:
36
37 MYSQL_ROOT_PASSWORD=$(kubectl get secret --namespace kube-system my-mysql -o jsonpath="{.data.mysql-root-password}" | base64 --decode; echo)
38
39To connect to your database:
40
411. Run an Ubuntu pod that you can use as a client:
42
43 kubectl run -i --tty ubuntu --image=ubuntu:16.04 --restart=Never -- bash -il
44
452. Install the mysql client:
46
47 $ apt-get update && apt-get install mysql-client -y
48
493. Connect using the mysql cli, then provide your password:
50 $ mysql -h my-mysql -p
51
52To connect to your database directly from outside the K8s cluster:
53 MYSQL_HOST=127.0.0.1
54 MYSQL_PORT=3306
55
56 # Execute the following command to route the connection:
57 kubectl port-forward svc/my-mysql 3306
58
59 mysql -h ${MYSQL_HOST} -P${MYSQL_PORT} -u root -p${MYSQL_ROOT_PASSWORD}
60
61[root@master-01 mysql]#
62
2
2
2.1 在prometheus的configmap里面增加配置:
2
3
4
5
6
7
8
2 static_configs:
3 - targets: ['my-mysql:9104']
4- job_name: "redis"
5 static_configs:
6 - targets: ['my-redis-redis-ha:9121']
7
8
??然后重新删除和建立configmap,热更新prometheus的配置；
2
3
2curl -X POST "http://10.109.108.37:9090/-/reload"
3
2
2
??首先还是搜索一下jenkins的chart
2
3
4
5
2NAME CHART VERSION APP VERSION DESCRIPTION
3stable/jenkins 1.1.10 lts Open source continuous integration server. It supports mu...
4[root@master-01 k8sdemo2]#
5
?? 然后我们把它下载下来并解压
2
2
??在kubernetes集群里面使用jenkins的原理主要包括:
在k8s集群里面运行jenkins master的pod，使用持久化存储，保证jenkins pod重启或者迁移之后，jenkins的插件和页面的job配置不会丢失;
在K8S集群里面运行jenkins slave Pod,当有一个job运行的时候，就会启动一个jenkins slave的Pod，当job运行完成之后这个slave pod会自动销毁，可以节省资源；
因为jenkins主要是CI/CD工具，所以jenkins完成的任务包括调用gitlab下载代码—>使用maven编译打包—>使用sonar代码检查（可选）—>使用docker build构建镜像—>使用docker push上传镜像到私有仓库—>使用kubectl命令发布应用到k8s集群
我们发布一个k8s应用一般需要用到Dockerfile、k8s YAML清单配置文件、jenkins pipeline流水线配置文件；
Dockerfile文件的主要目的是构建镜像，将jenkins maven打包之后的war包或者jar包ADD到镜像内，然后定义镜像的启动命令，环境变量（比如JAVA_HOME)等；
YAML文件主要定义的K8S部署应用的规则，比如部署几个副本，应用的资源限制是多少，应用启动之后的健康检查是curl还是tcp端口检查；应用是否需要挂载存储等；除了配置应用的deployment文件之外，一般还要配置service清单文件，用于其他应用调用服务名来访问本应用，如果应用还需要对外提供访问，还需要配置Ingress文件，甚至还包括配置文件configmap需要创建，应用依赖的数据库，MQ账户信息等需要使用secrets配置清单文件等等，所以建议熟悉Helm的同学在jenkins里面还是调用helm命令部署应用是最好的；
pipeline文件就是jenkins的配置内容了，现在使用jenkins都是推荐使用流水线模式，因为流水线模式非常利于jenkins的迁移等；pipeline文件主要定义了step,包括上面描述的打包、构建、镜像制作、k8s应用发布等动作，这些动作的实现都是依靠jenkins slave这个POD。所以这个jenkins slave镜像的制作就是非常重要的了。
jenkins slave镜像主要包含JAVA命令、maven命令、docker命令、kubectl命令等；还要挂载docker.sock文件和kubectl config文件等；
??紧接着就是参考官方文档的说明修改value.yaml文件: https://github.com/helm/charts/tree/master/stable/jenkins
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
2# This is a YAML-formatted file.
3# Declare name/value pairs to be passed into your templates.
4# name: value
5
6## Overrides for generated resource names
7# See templates/_helpers.tpl
8# nameOverride:
9# fullnameOverride:
10
11master:
12 # Used for label app.kubernetes.io/component
13 componentName: "jenkins-master"
14 image: "k8s.harbor.maimaiti.site/system/jenkins"
15 imageTag: "lts"
16 imagePullPolicy: "Always"
17 imagePullSecretName:
18 # Optionally configure lifetime for master-container
19 lifecycle:
20 # postStart:
21 # exec:
22 # command:
23 # - "uname"
24 # - "-a"
25 numExecutors: 0
26 # configAutoReload requires UseSecurity is set to true:
27 useSecurity: true
28 # Allows to configure different SecurityRealm using Jenkins XML
29 securityRealm: |-
30 <securityRealm class="hudson.security.LegacySecurityRealm"/>
31 # Allows to configure different AuthorizationStrategy using Jenkins XML
32 authorizationStrategy: |-
33 <authorizationStrategy class="hudson.security.FullControlOnceLoggedInAuthorizationStrategy">
34 <denyAnonymousReadAccess>true</denyAnonymousReadAccess>
35 </authorizationStrategy>
36 hostNetworking: false
37 # When enabling LDAP or another non-Jenkins identity source, the built-in admin account will no longer exist.
38 # Since the AdminUser is used by configAutoReload, in order to use configAutoReload you must change the
39 # .master.adminUser to a valid username on your LDAP (or other) server. This user does not need
40 # to have administrator rights in Jenkins (the default Overall:Read is sufficient) nor will it be granted any
41 # additional rights. Failure to do this will cause the sidecar container to fail to authenticate via SSH and enter
42 # a restart loop. Likewise if you disable the non-Jenkins identity store and instead use the Jenkins internal one,
43 # you should revert master.adminUser to your preferred admin user:
44 adminUser: "admin"
45 adminPassword: maimaiti.cn@4321
46 # adminSshKey: <defaults to auto-generated>
47 # If CasC auto-reload is enabled, an SSH (RSA) keypair is needed. Can either provide your own, or leave unconfigured to allow a random key to be auto-generated.
48 # If you supply your own, it is recommended that the values file that contains your key not be committed to source control in an unencrypted format
49 rollingUpdate: {}
50 # Ignored if Persistence is enabled
51 # maxSurge: 1
52 # maxUnavailable: 25%
53 resources:
54 requests:
55 cpu: "2000m"
56 memory: "2048Mi"
57 limits:
58 cpu: "2000m"
59 memory: "4096Mi"
60 # Environment variables that get added to the init container (useful for e.g. http_proxy)
61 # initContainerEnv:
62 # - name: http_proxy
63 # value: "http://192.168.64.1:3128"
64 # containerEnv:
65 # - name: http_proxy
66 # value: "http://192.168.64.1:3128"
67 # Set min/max heap here if needed with:
68 # javaOpts: "-Xms512m -Xmx512m"
69 # jenkinsOpts: ""
70 # jenkinsUrl: ""
71 # If you set this prefix and use ingress controller then you might want to set the ingress path below
72 # jenkinsUriPrefix: "/jenkins"
73 # Enable pod security context (must be `true` if runAsUser or fsGroup are set)
74 usePodSecurityContext: true
75 # Set runAsUser to 1000 to let Jenkins run as non-root user 'jenkins' which exists in 'jenkins/jenkins' docker image.
76 # When setting runAsUser to a different value than 0 also set fsGroup to the same value:
77 # runAsUser: <defaults to 0>
78 # fsGroup: <will be omitted in deployment if runAsUser is 0>
79 servicePort: 8080
80 # For minikube, set this to NodePort, elsewhere use LoadBalancer
81 # Use ClusterIP if your setup includes ingress controller
82 serviceType: LoadBalancer
83 # Jenkins master service annotations
84 serviceAnnotations: {}
85 # Jenkins master custom labels
86 deploymentLabels: {}
87 # foo: bar
88 # bar: foo
89 # Jenkins master service labels
90 serviceLabels: {}
91 # service.beta.kubernetes.io/aws-load-balancer-backend-protocol: https
92 # Put labels on Jenkins master pod
93 podLabels: {}
94 # Used to create Ingress record (should used with ServiceType: ClusterIP)
95 # hostName: jenkins.cluster.local
96 # nodePort: <to set explicitly, choose port between 30000-32767
97 # Enable Kubernetes Liveness and Readiness Probes
98 # ~ 2 minutes to allow Jenkins to restart when upgrading plugins. Set ReadinessTimeout to be shorter than LivenessTimeout.
99 healthProbes: true
100 healthProbesLivenessTimeout: 90
101 healthProbesReadinessTimeout: 60
102 healthProbeReadinessPeriodSeconds: 10
103 healthProbeLivenessFailureThreshold: 12
104 slaveListenerPort: 50000
105 slaveHostPort:
106 disabledAgentProtocols:
107 - JNLP-connect
108 - JNLP2-connect
109 csrf:
110 defaultCrumbIssuer:
111 enabled: true
112 proxyCompatability: true
113 cli: false
114 # Kubernetes service type for the JNLP slave service
115 # slaveListenerServiceType is the Kubernetes Service type for the JNLP slave service,
116 # either 'LoadBalancer', 'NodePort', or 'ClusterIP'
117 # Note if you set this to 'LoadBalancer', you *must* define annotations to secure it. By default
118 # this will be an external load balancer and allowing inbound 0.0.0.0/0, a HUGE
119 # security risk: https://github.com/kubernetes/charts/issues/1341
120 slaveListenerServiceType: "ClusterIP"
121 slaveListenerServiceAnnotations: {}
122 slaveKubernetesNamespace:
123
124 # Example of 'LoadBalancer' type of slave listener with annotations securing it
125 # slaveListenerServiceType: LoadBalancer
126 # slaveListenerServiceAnnotations:
127 # service.beta.kubernetes.io/aws-load-balancer-internal: "True"
128 # service.beta.kubernetes.io/load-balancer-source-ranges: "172.0.0.0/8, 10.0.0.0/8"
129
130 # LoadBalancerSourcesRange is a list of allowed CIDR values, which are combined with ServicePort to
131 # set allowed inbound rules on the security group assigned to the master load balancer
132 loadBalancerSourceRanges:
133 - 0.0.0.0/0
134 # Optionally assign a known public LB IP
135 # loadBalancerIP: 1.2.3.4
136 # Optionally configure a JMX port
137 # requires additional javaOpts, ie
138 # javaOpts: >
139 # -Dcom.sun.management.jmxremote.port=4000
140 # -Dcom.sun.management.jmxremote.authenticate=false
141 # -Dcom.sun.management.jmxremote.ssl=false
142 # jmxPort: 4000
143 # Optionally configure other ports to expose in the master container
144 extraPorts:
145 # - name: BuildInfoProxy
146 # port: 9000
147
148 # List of plugins to be install during Jenkins master start
149 installPlugins:
150 - kubernetes:1.14.0
151 - workflow-job:2.31
152 - workflow-aggregator:2.6
153 - credentials-binding:1.17
154 - git:3.9.1
155
156 # Enable to always override the installed plugins with the values of 'master.installPlugins' on upgrade or redeployment.
157 # overwritePlugins: true
158 # Enable HTML parsing using OWASP Markup Formatter Plugin (antisamy-markup-formatter), useful with ghprb plugin.
159 # The plugin is not installed by default, please update master.installPlugins.
160 enableRawHtmlMarkupFormatter: false
161 # Used to approve a list of groovy functions in pipelines used the script-security plugin. Can be viewed under /scriptApproval
162 scriptApproval:
163 # - "method groovy.json.JsonSlurperClassic parseText java.lang.String"
164 # - "new groovy.json.JsonSlurperClassic"
165 # List of groovy init scripts to be executed during Jenkins master start
166 initScripts:
167 # - |
168 # print 'adding global pipeline libraries, register properties, bootstrap jobs...'
169 # Kubernetes secret that contains a 'credentials.xml' for Jenkins
170 # credentialsXmlSecret: jenkins-credentials
171 # Kubernetes secret that contains files to be put in the Jenkins 'secrets' directory,
172 # useful to manage encryption keys used for credentials.xml for instance (such as
173 # master.key and hudson.util.Secret)
174 # secretsFilesSecret: jenkins-secrets
175 # Jenkins XML job configs to provision
176 jobs:
177 # test: |-
178 # <<xml here>>
179
180 # Below is the implementation of Jenkins Configuration as Code. Add a key under configScripts for each configuration area,
181 # where each corresponds to a plugin or section of the UI. Each key (prior to | character) is just a label, and can be any value.
182 # Keys are only used to give the section a meaningful name. The only restriction is they may only contain RFC 1123 \ DNS label
183 # characters: lowercase letters, numbers, and hyphens. The keys become the name of a configuration yaml file on the master in
184 # /var/jenkins_home/casc_configs (by default) and will be processed by the Configuration as Code Plugin. The lines after each |
185 # become the content of the configuration yaml file. The first line after this is a JCasC root element, eg jenkins, credentials,
186 # etc. Best reference is https://<jenkins_url>/configuration-as-code/reference. The example below creates a welcome message:
187 JCasC:
188 enabled: false
189 pluginVersion: 1.5
190 supportPluginVersion: 1.5
191 configScripts:
192 welcome-message: |
193 jenkins:
194 systemMessage: Welcome to our CI\CD server. This Jenkins is configured and managed 'as code'.
195
196 # Optionally specify additional init-containers
197 customInitContainers: []
198 # - name: CustomInit
199 # image: "alpine:3.7"
200 # imagePullPolicy: Always
201 # command: [ "uname", "-a" ]
202
203 sidecars:
204 configAutoReload:
205 # If enabled: true, Jenkins Configuration as Code will be reloaded on-the-fly without a reboot. If false or not-specified,
206 # jcasc changes will cause a reboot and will only be applied at the subsequent start-up. Auto-reload uses the Jenkins CLI
207 # over SSH to reapply config when changes to the configScripts are detected. The admin user (or account you specify in
208 # master.adminUser) will have a random SSH private key (RSA 4096) assigned unless you specify adminSshKey. This will be saved to a k8s secret.
209 enabled: false
210 image: shadwell/k8s-sidecar:0.0.2
211 imagePullPolicy: IfNotPresent
212 resources:
213 # limits:
214 # cpu: 100m
215 # memory: 100Mi
216 # requests:
217 # cpu: 50m
218 # memory: 50Mi
219 # SSH port value can be set to any unused TCP port. The default, 1044, is a non-standard SSH port that has been chosen at random.
220 # Is only used to reload jcasc config from the sidecar container running in the Jenkins master pod.
221 # This TCP port will not be open in the pod (unless you specifically configure this), so Jenkins will not be
222 # accessible via SSH from outside of the pod. Note if you use non-root pod privileges (runAsUser & fsGroup),
223 # this must be > 1024:
224 sshTcpPort: 1044
225 # folder in the pod that should hold the collected dashboards:
226 folder: "/var/jenkins_home/casc_configs"
227 # If specified, the sidecar will search for JCasC config-maps inside this namespace.
228 # Otherwise the namespace in which the sidecar is running will be used.
229 # It's also possible to specify ALL to search in all namespaces:
230 # searchNamespace:
231
232 # Allows you to inject additional/other sidecars
233 other:
234 ## The example below runs the client for https://smee.io as sidecar container next to Jenkins,
235 ## that allows to trigger build behind a secure firewall.
236 ## https://jenkins.io/blog/2019/01/07/webhook-firewalls/#triggering-builds-with-webhooks-behind-a-secure-firewall
237 ##
238 ## Note: To use it you should go to https://smee.io/new and update the url to the generete one.
239 # - name: smee
240 # image: docker.io/twalter/smee-client:1.0.2
241 # args: ["--port", "{{ .Values.master.servicePort }}", "--path", "/github-webhook/", "--url", "https://smee.io/new"]
242 # resources:
243 # limits:
244 # cpu: 50m
245 # memory: 128Mi
246 # requests:
247 # cpu: 10m
248 # memory: 32Mi
249 # Node labels and tolerations for pod assignment
250 # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
251 # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#taints-and-tolerations-beta-feature
252 nodeSelector: {}
253 tolerations: []
254 # Leverage a priorityClass to ensure your pods survive resource shortages
255 # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
256 # priorityClass: system-cluster-critical
257 podAnnotations: {}
258
259 # The below two configuration-related values are deprecated and replaced by Jenkins Configuration as Code (see above
260 # JCasC key). They will be deleted in an upcoming version.
261 customConfigMap: false
262 # By default, the configMap is only used to set the initial config the first time
263 # that the chart is installed. Setting `overwriteConfig` to `true` will overwrite
264 # the jenkins config with the contents of the configMap every time the pod starts.
265 # This will also overwrite all init scripts
266 overwriteConfig: false
267
268 # By default, the Jobs Map is only used to set the initial jobs the first time
269 # that the chart is installed. Setting `overwriteJobs` to `true` will overwrite
270 # the jenkins jobs configuration with the contents of Jobs every time the pod starts.
271 overwriteJobs: false
272
273 ingress:
274 enabled: true
275 # For Kubernetes v1.14+, use 'networking.k8s.io/v1beta1'
276 apiVersion: "extensions/v1beta1"
277 labels: {}
278 annotations:
279 kubernetes.io/ingress.class: traefik
280 # kubernetes.io/tls-acme: "true"
281 # Set this path to jenkinsUriPrefix above or use annotations to rewrite path
282 # path: "/jenkins"
283 hostName: k8s.jenkins.maimaiti.site
284 tls:
285 # - secretName: jenkins.cluster.local
286 # hosts:
287 # - jenkins.cluster.local
288
289 # Openshift route
290 route:
291 enabled: false
292 labels: {}
293 annotations: {}
294 # path: "/jenkins"
295
296 additionalConfig: {}
297
298 # master.hostAliases allows for adding entries to Pod /etc/hosts:
299 # https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/
300 hostAliases: []
301 # - ip: 192.168.50.50
302 # hostnames:
303 # - something.local
304 # - ip: 10.0.50.50
305 # hostnames:
306 # - other.local
307
308agent:
309 enabled: true
310 image: "10.83.74.102/jenkins/jnlp"
311 imageTag: "v11"
312 customJenkinsLabels: []
313 # name of the secret to be used for image pulling
314 imagePullSecretName:
315 componentName: "jenkins-slave"
316 privileged: false
317 resources:
318 requests:
319 cpu: "2000m"
320 memory: "4096Mi"
321 limits:
322 cpu: "2000m"
323 memory: "4096Mi"
324 # You may want to change this to true while testing a new image
325 alwaysPullImage: false
326 # Controls how slave pods are retained after the Jenkins build completes
327 # Possible values: Always, Never, OnFailure
328 podRetention: "Never"
329 # You can define the volumes that you want to mount for this container
330 # Allowed types are: ConfigMap, EmptyDir, HostPath, Nfs, Pod, Secret
331 # Configure the attributes as they appear in the corresponding Java class for that type
332 # https://github.com/jenkinsci/kubernetes-plugin/tree/master/src/main/java/org/csanchez/jenkins/plugins/kubernetes/volumes
333 # Pod-wide ennvironment, these vars are visible to any container in the slave pod
334 envVars:
335 # - name: PATH
336 # value: /usr/local/bin
337 volumes:
338 - type: HostPath
339 hostPath: /var/run/docker.sock
340 mountPath: /var/run/docker.sock
341 - type: HostPath
342 hostPath: /root/.kube
343 mountPath: /root/.kube
344 - type: Nfs
345 mountPath: /root/.m2
346 serverAddress: 10.83.32.224
347 serverPath: /data/m2
348 # - type: Secret
349 # secretName: mysecret
350 # mountPath: /var/myapp/mysecret
351 nodeSelector: {}
352 # Key Value selectors. Ex:
353 # jenkins-agent: v1
354
355 # Executed command when side container gets started
356 command:
357 args:
358 # Side container name
359 sideContainerName: "jnlp"
360 # Doesn't allocate pseudo TTY by default
361 TTYEnabled: false
362 # Max number of spawned agent
363 containerCap: 10
364 # Pod name
365 podName: "jenkins-slave"
366
367persistence:
368 enabled: true
369 ## A manually managed Persistent Volume and Claim
370 ## Requires persistence.enabled: true
371 ## If defined, PVC must be created manually before volume will be bound
372 existingClaim:
373 ## jenkins data Persistent Volume Storage Class
374 ## If defined, storageClassName: <storageClass>
375 ## If set to "-", storageClassName: "", which disables dynamic provisioning
376 ## If undefined (the default) or set to null, no storageClassName spec is
377 ## set, choosing the default provisioner. (gp2 on AWS, standard on
378 ## GKE, AWS & OpenStack)
379 ##
380 storageClass: "dynamic"
381 annotations: {}
382 accessMode: "ReadWriteOnce"
383 size: "8Gi"
384 volumes:
385 # - name: nothing
386 # emptyDir: {}
387 mounts:
388 # - mountPath: /var/nothing
389 # name: nothing
390 # readOnly: true
391
392networkPolicy:
393 # Enable creation of NetworkPolicy resources.
394 enabled: false
395 # For Kubernetes v1.4, v1.5 and v1.6, use 'extensions/v1beta1'
396 # For Kubernetes v1.7, use 'networking.k8s.io/v1'
397 apiVersion: networking.k8s.io/v1
398
399## Install Default RBAC roles and bindings
400rbac:
401 create: true
402
403serviceAccount:
404 create: true
405 # The name of the service account is autogenerated by default
406 name:
407 annotations: {}
408
409## Backup cronjob configuration
410## Ref: https://github.com/nuvo/kube-tasks
411backup:
412 # Backup must use RBAC
413 # So by enabling backup you are enabling RBAC specific for backup
414 enabled: false
415 # Used for label app.kubernetes.io/component
416 componentName: "backup"
417 # Schedule to run jobs. Must be in cron time format
418 # Ref: https://crontab.guru/
419 schedule: "0 2 * * *"
420 annotations:
421 # Example for authorization to AWS S3 using kube2iam
422 # Can also be done using environment variables
423 iam.amazonaws.com/role: "jenkins"
424 image:
425 repository: "nuvo/kube-tasks"
426 tag: "0.1.2"
427 # Additional arguments for kube-tasks
428 # Ref: https://github.com/nuvo/kube-tasks#simple-backup
429 extraArgs: []
430 # Add additional environment variables
431 env:
432 # Example environment variable required for AWS credentials chain
433 - name: "AWS_REGION"
434 value: "us-east-1"
435 resources:
436 requests:
437 memory: 1Gi
438 cpu: 1
439 limits:
440 memory: 1Gi
441 cpu: 1
442 # Destination to store the backup artifacts
443 # Supported cloud storage services: AWS S3, Minio S3, Azure Blob Storage
444 # Additional support can added. Visit this repository for details
445 # Ref: https://github.com/nuvo/skbn
446 destination: "s3://nuvo-jenkins-data/backup"
447checkDeprecation: true
448
449
??我主要修改了value.yaml以下几个配置参数，主要包括：
修改了镜像的参数为私服仓库的镜像。这里有一个特别注意的点就是jenkins slave镜像，这个镜像如果只是使用官方的镜像还是不行的，需要自己制作镜像。镜像里面要包含kubectl命令、docker命令、mvn打包命令等；
配置了jenkins的登录密码；
配置了资源限制情况，这里要特别注意一点，jenkins slave镜像的默认资源限制太小，经常会因为这个资源不足导致jinkins slave Pod终端；所以需要将jenkins slave镜像的资源限制调高一点；
再就是配置ingress,因为我们需要通过k8s集群外访问jenkins应用；
最主要的是agent这一块的配置，这里的镜像要自己制作，并且要把资源限额调大，并且要挂载docker.sock和kubectl的配置文件，将jenkins slaves的.m2目录配置成nfs存储。这样的话，重新启动的jenkins slave构建的时候就不用每次都在apache官网下载依赖的jar包了
配置持久化存储，因为持久化存储需要存储插件和jenkins的页面配置；
??在jenkins这个应用部署到k8s集群过程中，我踩过了几个坑，现在把这些坑都总结一下，给大家参考:
我在测试jenkins的时候，删除一个chart。由于一直习惯于使用命令
2
2
来删除应用，结果我使用了这个命令之后，再次使用
2
2
当我登录jenkins的时候，发现所有的配置都没有了。包括我已经配置的job、安装的插件等等；所以大家在使用helm delete的时候一定要记得–purge慎用，加了–purge就代表pvc也会一并删除；
我在制作jenkins slave镜像的时候，运行起来的slave pod，当执行docker命令的时候，总是提示如下报错:
2
3
4
5
6
7
8
9
10
11
12
13
2Client:
3 Version: 18.06.0-ce
4 API version: 1.38
5 Go version: go1.10.3
6 Git commit: 0ffa825
7 Built: Wed Jul 18 19:04:39 2018
8 OS/Arch: linux/amd64
9 Experimental: false
10Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.38/version: dial unix /var/run/docker.sock: connect: permission denied
11Build step 'Execute shell' marked build as failure
12Finished: FAILURE
13
其实这个报错的原因是因为官方的jenkins slave镜像是已jenkins(id为10010)的用户启动的，但是我们映射到slave镜像的/var/run/docker.sock文件，必须要root用户才有权限读取。所以就产生了这个问题；解决方案有两种:
把所有的k8s宿主机的/var/run/docker.sock文件权限修改为chmod -R 777 /var/run/docker.sock 再mount进去，因为不安全不推荐这样; 在jenkins slave镜像中设置jenkins用户可以使用sudo命令执行root权限的命令
2
2
2
2
??我这里选择了第三种方案；其实第二种方案是最安全的，只是因为我的pipeline文件里面调用了多个docker插件，来实现docker build和docker login等，不知道插件里面怎么实现sudo命令；自定义jenkins slave镜像的Dockerfile可以这样写:
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
2MAINTAINER Yang Gao "gaoyang@maimaiti.cn"
3USER root
4ADD jdk /root/
5ADD maven /root/maven
6ENV JAVA_HOME /root/jdk/
7ENV MAVEN_HOME /root/maven/
8ENV PATH $PATH:$JAVA_HOME/bin:$MAVEN_HOME/bin
9RUN echo "deb http://apt.dockerproject.org/repo debian-jessie main" \
10 > /etc/apt/sources.list.d/docker.list \
11 && apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 \
12 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D \
13 && apt-get update \
14 && apt-get install -y apt-transport-https \
15 && apt-get install -y sudo \
16 && apt-get install -y docker-engine \
17 && rm -rf /var/lib/apt/lists/*
18RUN echo "jenkins ALL=NOPASSWD: ALL" >> /etc/sudoers
19RUN curl -L https://github.com/docker/compose/releases/download/1.8.0/\
20docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose; \
21chmod +x /usr/local/bin/docker-compose
22
??然后使用docker build命令构建镜像
2
3
2docker push harbor.k8s.maimaiti.site/system/jenkins-jnlp:v11
3
jenkins slave pod模板可以在两个地方配置，一个就是helm chart value.yaml文件里面定义，包括slave镜像的版本，挂载等；还有一种就是在jenkins页面安装了kubernetes插件之后有个云的配置，里面也可以配置jenkins slave pod模板。但是有个问题，如果你使用了helm upgrade 命令修改了helm里面的参数，包括slave的版本等，实际上是不生效的。因为jenkins还是会使用管理页面的那个配置。当使用helm install jenkins的时候，第一次登陆jenkins页面，这个jenkins的helm默认就安装了kubernetes插件，所以可以直接到系统管理—系统配置里面找云的配置，默认就有了slave pod的配置，这个配置和helm value.yaml里面是一样的。但是后续再helm upgrade jenkins的时候，实际上是不能更新jenkins页面的这个地方的slave 配置，所以会发现一直更改jenkins slave不生效。 
??讲完了我探究jenkins in kubernetes遇到的坑，再看看一个k8s应用发布，到底都需要配置哪些jenkins的设置。这里我列举了一个我们公司的内部项目其中一些地方做了脱敏处理，大家只需要关注流程和架构即可，具体的配置已各公司的应用不同会有所不通；
登录jenkins,配置插件；系统管理—插件管理—available插件—安装docker、kubernetes插件等
配置凭据：配置包括gitlab和harbor的账户和密码，后面再pipeline里面会使用到
创建一个pipeline的job，配置包括: 配置发布应用的模块变量 配置发布应用的版本分支名称； 配置pipeline的内容；
k8s应用清单文件：
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
2 try {
3 stage('代码拉取') {
4 git branch: "${BranchName}", credentialsId: 'k8sgitlab', url: 'http://k8s.gitlab.test.site/root/test.git'
5 }
6 stage('项目构建') {
7 if ("${MODULE}".contains('test-ui')){
8 dir('test-ui'){
9 sh "npm i"
10 sh "npm run sit"
11 }
12 }else {
13 dir('test-parent'){
14 sh "mvn clean install -Psit"
15 }
16 }
17 }
18 def regPrefix = 'k8s.harbor.test.site/test/'
19 stage('构建镜像'){
20 docker.withRegistry('http://k8s.harbor.test.site/','k8sharbor'){
21 if ("${MODULE}".contains('test-admin')){
22 dir('test-parent/test-admin/target') {
23 sh "cp ../Dockerfile . && cp -rf ../BOOT-INF ./ &&cp -rf ../../pinpoint-agent ./"
24 sh "jar -uvf admin.jar BOOT-INF/classes/application.yml"
25 def imageName = docker.build("${regPrefix}admin:V1.0-${env.BUILD_ID}")
26 imageName.push("V1.0-${env.BUILD_ID}")
27 //imageName.push("latest")
28 sh "/usr/bin/docker rmi ${regPrefix}admin:V1.0-${env.BUILD_ID}"
29 }
30 }
31
32 if ("${MODULE}".contains('test-eureka')){
33 dir('test-parent/test-eureka/target') {
34 sh "cp ../Dockerfile . && cp -rf ../BOOT-INF ./ &&cp -rf ../../pinpoint-agent ./"
35 sh "jar -uvf testEurekaServer.jar BOOT-INF/classes/application.yml"
36 def imageName = docker.build("${regPrefix}eureka:V1.0-${env.BUILD_ID}")
37 imageName.push("V1.0-${env.BUILD_ID}")
38 //imageName.push("latest")
39 sh "/usr/bin/docker rmi ${regPrefix}eureka:V1.0-${env.BUILD_ID}"
40 }
41 }
42 if ("${MODULE}".contains('test-quality')){
43 dir('test-parent/test-quality/target') {
44 sh "cp ../Dockerfile . && cp -rf ../BOOT-INF ./ &&cp -rf ../../pinpoint-agent ./"
45 sh "jar -uvf quality.jar BOOT-INF/classes/application.yml"
46 def imageName = docker.build("${regPrefix}quality:V1.0-${env.BUILD_ID}")
47 imageName.push("V1.0-${env.BUILD_ID}")
48 //imageName.push("latest")
49 sh "/usr/bin/docker rmi ${regPrefix}quality:V1.0-${env.BUILD_ID}"
50 }
51 }
52 if ("${MODULE}".contains('test-schedule')){
53 dir('test-parent/test-schedule/target') {
54 sh "cp ../Dockerfile . && cp -rf ../BOOT-INF ./ &&cp -rf ../../pinpoint-agent ./"
55 sh "jar -uvf schedule.jar BOOT-INF/classes/application.yml "
56 def imageName = docker.build("${regPrefix}schedule:V1.0-${env.BUILD_ID}")
57 imageName.push("V1.0-${env.BUILD_ID}")
58 //imageName.push("latest")
59 sh "/usr/bin/docker rmi ${regPrefix}schedule:V1.0-${env.BUILD_ID}"
60 }
61 }
62 if ("${MODULE}".contains('test-zuul')){
63 dir('test-parent/test-zuul/target') {
64 sh "cp ../Dockerfile . && cp -rf ../BOOT-INF ./ &&cp -rf ../../pinpoint-agent ./"
65 sh "jar -uvf test-api.jar BOOT-INF/classes/application.yml "
66 def imageName = docker.build("${regPrefix}zuul:V1.0-${env.BUILD_ID}")
67 imageName.push("V1.0-${env.BUILD_ID}")
68 //imageName.push("latest")
69 sh "/usr/bin/docker rmi ${regPrefix}zuul:V1.0-${env.BUILD_ID}"
70 }
71 }
72 }
73 }
74 stage('重启应用'){
75 if ("${MODULE}".contains('test-admin')){
76 sh "sed -i \'s/latest/V1.0-${env.BUILD_ID}/g\' test-parent/DockerCompose/test-admin.yml "
77 sh "/usr/local/bin/kubectl --kubeconfig=test-parent/DockerCompose/config apply -f test-parent/DockerCompose/test-admin.yml --record "
78 }
79 if ("${MODULE}".contains('test-eureka')){
80 sh "sed -i \'s/latest/V1.0-${env.BUILD_ID}/g\' test-parent/DockerCompose/test-eureka.yml "
81 sh "/usr/local/bin/kubectl apply -f test-parent/DockerCompose/test-eureka.yml --record "
82 }
83 if ("${MODULE}".contains('test-quality')){
84 sh "sed -i \'s/latest/V1.0-${env.BUILD_ID}/g\' test-parent/DockerCompose/test-quality.yml "
85 sh "/usr/local/bin/kubectl apply -ftest-parent/DockerCompose/test-quality.yml --record "
86 }
87 if ("${MODULE}".contains('test-schedule')){
88 sh "sed -i \'s/latest/V1.0-${env.BUILD_ID}/g\' test-parent/DockerCompose/test-schedule.yml "
89 sh "/usr/local/bin/kubectl apply -f test-parent/DockerCompose/test-schedule.yml --record "
90 }
91 if ("${MODULE}".contains('test-zuul')){
92 sh "sed -i \'s/latest/V1.0-${env.BUILD_ID}/g\' test-parent/DockerCompose/test-zuul.yml "
93 sh "/usr/local/bin/kubectl apply -f test-parent/DockerCompose/test-zuul.yml --record "
94 }
95 }
96
97 }catch (any) {
98 currentBuild.result = 'FAILURE'
99 throw any}
100
101}
102
103
??Dockerfile的内容主要如下：
2
3
4
5
6
7
8
9
2ENV TZ=Asia/Shanghai
3VOLUME /tmp
4
5ADD admin.jar /app/test/admin.jar
6ADD skywalking-agent/ /app/icm/skywalking-agent
7ENTRYPOINT ["java","-javaagent:/app/test/skywalking-agent/skywalking-agent.jar","-Djava.security.egd=file:/dev/./urandom","-XX:+UnlockExperimentalVMOptions","-XX:+UseCGroupMemoryLimitForHeap","-jar","/app/test/admin.jar"]
8
9
??Yaml清单文件的内容主要如下:
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
2kind: Deployment
3metadata:
4 name: test-admin
5 namespace: sit
6 labels:
7 k8s-app: test-admin
8
9spec:
10 replicas: 3
11 revisionHistoryLimit: 3
12 #滚动升级时70s后认为该pod就绪
13 minReadySeconds: 70
14 strategy:
15 ##由于replicas为3,则整个升级,pod个数在2-4个之间
16 rollingUpdate:
17 #滚动升级时会先启动1个pod
18 maxSurge: 1
19 #滚动升级时允许的最大Unavailable的pod个数
20 maxUnavailable: 1
21 selector:
22 matchLabels:
23 k8s-app: test-admin
24 template:
25 metadata:
26 labels:
27 k8s-app: test-admin
28 spec:
29 containers:
30 - name: test-admin
31 image: k8s.harbor.test.site/test/admin:latest
32 resources:
33 # need more cpu upon initialization, therefore burstable class
34 #limits:
35 # memory: 1024Mi
36 # cpu: 200m
37 #requests:
38 # cpu: 100m
39 # memory: 256Mi
40 ports:
41 #容器的端口
42 - containerPort: 8281
43 #name: ui
44 protocol: TCP
45 livenessProbe:
46 httpGet:
47 path: /admin/health
48 port: 8281
49 scheme: HTTP
50 initialDelaySeconds: 180
51 timeoutSeconds: 5
52 periodSeconds: 15
53 successThreshold: 1
54 failureThreshold: 2
55 #volumeMounts:
56 #- mountPath: "/download"
57 # name: data
58 #volumes:
59 #- name: data
60 # persistentVolumeClaim:
61 # claimName: download-pvc
62---
63apiVersion: v1
64kind: Service
65metadata:
66 name: test-admin
67 namespace: sit
68 labels:
69 k8s-app: test-admin
70
71spec:
72 type: NodePort
73 ports:
74 #集群IP的端口
75 - port: 8281
76 protocol: TCP
77 #容器的端口
78 targetPort: 8281
79 #nodePort: 28281
80 selector:
81 k8s-app: test-admin
82
