上一篇是写的storm基于物理机的安装
首先我们先来编写一下Dockerfile
FROM ip:80/bigdata/centos7-jdk:1.8
#config storm env ENV STORM_USER=storm \ STORM_INSTALL_BASE=/home/frank/platform \ STORM_HOME=/home/frank/platform/storm \ STORM_CONF_DIR=/home/frank/conf/storm \ STORM_DIR=/home/frank/hard_disk/storm \ STORM_BLOBS=/home/frank/hard_disk/storm/blobs \ STORM_NIMBUS=/home/frank/hard_disk/storm/nimbus \ STORM_LOG_DIR=/home/frank/logs/storm \ STORM_ROOT=/home/frank \ STORM_LOG=/home/frank/logs ENV PATH $PATH:$STORM_HOME/bin #add FLINK user and make dirs RUN set -x \ && adduser "$STORM_USER" \ && mkdir -p "$STORM_INSTALL_BASE" "$STORM_CONF_DIR" "$STORM_LOG" "$STORM_LOG_DIR" "$STORM_BLOBS" "$STORM_NIMBUS" #install storm libs ADD apache-storm-1.2.2.tar.gz $STORM_INSTALL_BASE RUN cd $STORM_INSTALL_BASE && ln -s -f apache-storm-1.2.2 storm && mkdir -p /home/stormRUN chown -R "$STORM_USER:$STORM_USER" "$STORM_ROOT"
COPY docker-entrypoint.sh /home/storm RUN chmod 755 /home/storm/docker-entrypoint.sh USER storm ENTRYPOINT ["/home/storm/docker-entrypoint.sh"]其中的脚本如下:
#!/bin/sh
# If unspecified, the hostname of the container is taken as the JobManager address ACTION_CMD="$1" source /etc/profile export STORM_CONF_DIR=/home/frank/conf/storm export STORM_CONF_FILE=/home/frank/conf/storm/storm.yaml sed -i -e "s/storm.zookeeper.servers: \[localhost\]/storm.zookeeper.servers: \[${STORM_ZK_QUORUM}\]/" /home/frank/conf/storm/storm.yaml sed -i -e "s/storm.zookeeper.root: \"\/storm\"/storm.zookeeper.root: \/${STORM_ZK_ROOT}/" /home/frank/conf/storm/storm.yaml # if use cluster model, pod ${JOB_CLUSTER_NAME}-0,${JOB_CLUSTER_NAME}-1 as jobmanager if [ ${ACTION_CMD} == "cluster" ]; then nimbuses=(${NIMBUS_HOSTS//,/ }) ACTION_CMD="supervisor" for i in ${!nimbuses[@]} do if [ "$(hostname -s)" == "${nimbuses[i]}" ]; then NIMBUS_ADDRESS=${NIMBUS_ADDRESS:-$(hostname -f)} ACTION_CMD="nimbus" sed -i -e "s/nimbus.seeds: \[localhost\]/nimbus.seeds: \["${NIMBUS_ADDRESS}"\]/" /home/frank/conf/storm/storm.yaml echo "pod hostname match nimbus config host, change action to nimbus." fi done fi # if ha model, replace ha configuration if [ ${ACTION_CMD} == "help" ]; then echo "Usage: $(basename "$0") (cluster|nimbus|supervisor|ui|logviewer|help)" exit 0 elif [ ${ACTION_CMD} == "nimbus" ]; then echo "Starting nimbus ui logviewer supervisor" nohup /home/frank/platform/storm/bin/storm nimbus >> /tmp/nimbus.log 2>&1 & \ nohup /home/frank/platform/storm/bin/storm ui >> /tmp/ui.log 2>&1 & \ /home/frank/platform/storm/bin/storm logviewer elif [ ${ACTION_CMD} == "supervisor" ]; then echo "Starting supervisor" /home/frank/platform/storm/bin/storm supervisor fi镜像构建命令如下:
docker build --network=host -f Dockerfile -t ip:80/bigdata/storm:1.2.2 .
下面我们提供一下service的创建文件
apiVersion: v1
kind: Service metadata: name: storm-cluster labels: app: storm-cluster namespace: storm spec: clusterIP: None ports: - port: 8080 name: ui selector: app: storm-clusterapiVersion: apps/v1
kind: StatefulSet metadata: name: storm-cluster namespace: storm spec: selector: matchLabels: app: storm-cluster serviceName: storm-cluster replicas: 4 podManagementPolicy: Parallel template: metadata: labels: app: storm-cluster spec: terminationGracePeriodSeconds: 2 containers: - name: storm-cluster imagePullPolicy: Always image: ip:80/bigdata/storm:1.2.2 command: ["/bin/sleep"] args: ["30d"] volumeMounts: - name: storm-conf mountPath: /home/conf/storm - name: storm-log mountPath: /home/logs/storm resources: requests: memory: "4096Mi" cpu: 4 limits: memory: "4096Mi" cpu: 4 env: - name: NIMBUS_HOSTS value: "storm-cluster-2,storm-cluster-3" - name: STORM_ZK_QUORUM value: "ip" - name: STORM_ZK_ROOT value: "stormNew" ports: - containerPort: 6627 name: nimbus - containerPort: 8000 name: logviewer - containerPort: 8080 name: storm-ui - containerPort: 6700 name: sv0 - containerPort: 6701 name: sv1 - containerPort: 6702 name: sv2 - containerPort: 6703 name: sv3 - containerPort: 22 name: ssh volumes: - name: storm-conf configMap: name: storm-conf - name: storm-log hostPath: path: /stormtmp type: Directory service的创建脚本如下:kubectl create -f storm-svc.yaml
statefulSet的创建脚本如下:
kubectl create -f storm-statefulSet.yaml
这个创建脚本要注意如下问题:
command:/bin/sleep
程序需要手动启动
/home/vipshop/hard_disk/storm
drwxr-xr-x 2 storm storm 4096 Jul 12 12:20 blobs
drwxr-xr-x 2 storm storm 4096 Jul 12 12:20 nimbus drwxrwxr-x 5 storm storm 4096 Jul 29 00:00 supervisor 这个hard_disk目录需要有storm权限注:nimbus.thrift.max_buffer_size: 10485760
这个参数需要是2的倍数