Flink kubernates部署

#Build The Flink Docker Image

##Dockerfile

FROM java:8-jre-alpine
    
    # Install requirements
    RUN apk add --no-cache bash snappy
    
    # Flink environment variables
    ENV FLINK_INSTALL_PATH=/opt
    ENV FLINK_HOME $FLINK_INSTALL_PATH/flink
    ENV FLINK_LIB_DIR $FLINK_HOME/lib
    ENV PATH $PATH:$FLINK_HOME/bin
    
    # flink-dist can point to a directory or a tarball on the local system
    ARG flink_dist=NOT_SET
    
    # Install build dependencies and flink
    ADD $flink_dist $FLINK_INSTALL_PATH
    
    RUN set -x && \
      ln -s $FLINK_INSTALL_PATH/flink-* $FLINK_HOME && \
      ln -s $FLINK_INSTALL_PATH/job.jar $FLINK_LIB_DIR && \
      addgroup -S flink && adduser -D -S -H -G flink -h $FLINK_HOME flink && \
      chown -R flink:flink $FLINK_INSTALL_PATH/flink-* && \
      chown -h flink:flink $FLINK_HOME
    
    COPY docker-entrypoint.sh /
    
    USER flink
    EXPOSE 8081 6123
    ENTRYPOINT ["/docker-entrypoint.sh"]
    CMD ["help"]
    
    
---

FROM centos:centos7.5.1804

# Install requirements
    RUN yum install -y snappy wget telnet unzip curl
    
    # Flink environment variables
    ENV LANG=en_US.UTF-8
    ENV FLINK_INSTALL_PATH=/opt
    ENV FLINK_HOME $FLINK_INSTALL_PATH/flink
    ENV FLINK_LIB_DIR $FLINK_HOME/lib
    ENV JAVA_HOME /opt/jdk1.8.0_191
    ENV PATH $PATH:$FLINK_HOME/bin:$JAVA_HOME/bin
    
    # flink-dist can point to a directory or a tarball on the local system
    ARG flink_dist=NOT_SET
    
    # Install build dependencies and flink
    ADD $flink_dist $FLINK_INSTALL_PATH
    ADD jdk1.8.tar.gz $FLINK_INSTALL_PATH
    
    RUN set -x && \
      ln -s $FLINK_INSTALL_PATH/flink-* $FLINK_HOME && \
      ln -s $FLINK_INSTALL_PATH/job.jar $FLINK_LIB_DIR && \
      groupadd flink && adduser -g flink -d $FLINK_HOME flink && \
      chown -R flink:flink $FLINK_INSTALL_PATH/flink-* && \
      chown -h flink:flink $FLINK_HOME
    
    COPY docker-entrypoint.sh /
    
    USER flink
    EXPOSE 8081 6123
    ENTRYPOINT ["/docker-entrypoint.sh"]
    CMD ["help"]

##entrypoint

#!/bin/sh
    # If unspecified, the hostname of the container is taken as the JobManager address
    JOB_MANAGER_RPC_ADDRESS=${JOB_MANAGER_RPC_ADDRESS:-$(hostname -f)}
    
    drop_privs_cmd() {
        if [ -x /sbin/su-exec ]; then
            # Alpine
            echo su-exec
        else
            # Others
            echo gosu
        fi
    }
    
    if [ "$1" = "help" ]; then
        echo "Usage: $(basename "$0") (jobmanager|taskmanager|help)"
        exit 0
    elif [ "$1" = "jobmanager" ]; then
        shift 1
        echo "Starting Job Manager"
        sed -i -e "s/jobmanager.rpc.address: localhost/jobmanager.rpc.address: ${JOB_MANAGER_RPC_ADDRESS}/g" "$FLINK_HOME/conf/flink-conf.yaml"
        echo "blob.server.port: 6124" >> "$FLINK_HOME/conf/flink-conf.yaml"
        echo "query.server.port: 6125" >> "$FLINK_HOME/conf/flink-conf.yaml"
    
        echo "config file: " && grep '^[^\n#]' "$FLINK_HOME/conf/flink-conf.yaml"
        exec "$FLINK_HOME/bin/jobmanager.sh" start-foreground "$@"
    elif [ "$1" = "taskmanager" ]; then
        TASK_MANAGER_NUMBER_OF_TASK_SLOTS=${TASK_MANAGER_NUMBER_OF_TASK_SLOTS:-$(grep -c ^processor /proc/cpuinfo)}
    
        sed -i -e "s/jobmanager.rpc.address: localhost/jobmanager.rpc.address: ${JOB_MANAGER_RPC_ADDRESS}/g" "$FLINK_HOME/conf/flink-conf.yaml"
        sed -i -e "s/taskmanager.numberOfTaskSlots: 1/taskmanager.numberOfTaskSlots: $TASK_MANAGER_NUMBER_OF_TASK_SLOTS/g" "$FLINK_HOME/conf/flink-conf.yaml"
        echo "blob.server.port: 6124" >> "$FLINK_HOME/conf/flink-conf.yaml"
        echo "query.server.port: 6125" >> "$FLINK_HOME/conf/flink-conf.yaml"
    
        echo "Starting Task Manager"
        echo "config file: " && grep '^[^\n#]' "$FLINK_HOME/conf/flink-conf.yaml"
        exec "$FLINK_HOME/bin/taskmanager.sh" start-foreground
    fi
    
    exec "$@"
    
##Build Image

tar zcvf flink.tgz flink-1.6.1
    docker build --build-arg flink_dist="flink.tgz" -t "flink-jdk8-centos7:v1.6.1" .
    
    dokcer login
    docker tag flink-jdk8-centos7:v1.6.1 louyj/flink-jdk8-centos7:v1.6.1
    docker push  louyj/flink-jdk8-centos7:v1.6.1

#Flink session cluster on Kubernetes

A Flink session cluster is executed as a long-running Kubernetes Deployment. Note that you can run multiple Flink jobs on a session cluster. Each job needs to be submitted to the cluster after the cluster has been deployed.

A basic Flink session cluster deployment in Kubernetes has three components:

- Deployment/Job which runs the JobManager
- Deployment for a pool of TaskManagers
- Service exposing the JobManager’s REST and UI ports

#Deploy JobManger Service

vi jobmanager-service.yaml

apiVersion: v1
    kind: Service
    metadata:
      name: flink-jobmanager
    spec:
      ports:
      - name: rpc
        port: 6123
      - name: blob
        port: 6124
      - name: query
        port: 6125
      - name: ui
        port: 8081
      selector:
        app: flink
        component: jobmanager

kubectl create -f jobmanager-service.yaml

#Deploy JobManger

vi jobmanager-deployment.yaml

apiVersion: extensions/v1beta1
    kind: Deployment
    metadata:
      name: flink-jobmanager
    spec:
      replicas: 1
      template:
        metadata:
          labels:
            app: flink
            component: jobmanager
        spec:
          containers:
          - name: jobmanager
            image: louyj/flink:v1.6.1
            args:
            - jobmanager
            ports:
            - containerPort: 6123
              name: rpc
            - containerPort: 6124
              name: blob
            - containerPort: 6125
              name: query
            - containerPort: 8081
              name: ui
            env:
            - name: JOB_MANAGER_RPC_ADDRESS
              value: flink-jobmanager
            - name: TZ
              value: Asia/Shanghai
            volumeMounts:
            - mountPath: /opt/flink/checkpoints
              name: flink-checkpoints
            - mountPath: /opt/flink/completedjobs
              name: flink-completedjobs
            - mountPath: /opt/flink/statebackend
              name: flink-statebackend
            - mountPath: /opt/flink/target
              name: flink-target
          volumes:
          - name: flink-checkpoints
            hostPath:
              path: /mnt/nfs/172.17.36.38/flink-kube/checkpoints
          - name: flink-completedjobs
            hostPath:
              path: /mnt/nfs/172.17.36.38/flink-kube/completedjobs
          - name: flink-statebackend
            hostPath:
              path: /mnt/nfs/172.17.36.38/flink-kube/statebackend
          - name: flink-target
            hostPath:
              path: /mnt/nfs/172.17.36.38/flink-kube/target

kubectl create -f jobmanager-deployment.yaml

#Deploy TaskManager

apiVersion: extensions/v1beta1
    kind: Deployment
    metadata:
      name: flink-taskmanager
    spec:
      replicas: 10
      template:
        metadata:
          labels:
            app: flink
            component: taskmanager
        spec:
          containers:
          - name: taskmanager
            image: louyj/flink:v1.6.1
            args:
            - taskmanager
            ports:
            - containerPort: 6121
              name: data
            - containerPort: 6122
              name: rpc
            - containerPort: 6125
              name: query
            env:
            - name: JOB_MANAGER_RPC_ADDRESS
              value: flink-jobmanager
            - name: TZ
              value: Asia/Shanghai
            volumeMounts:
            - mountPath: /mnt/nfs
              name: nfs-files
            - mountPath: /opt/flink/checkpoints
              name: flink-checkpoints
            - mountPath: /opt/flink/completedjobs
              name: flink-completedjobs
            - mountPath: /opt/flink/statebackend
              name: flink-statebackend
            - mountPath: /opt/flink/target
              name: flink-target
          volumes:
          - name: nfs-files
            hostPath:
              path: /mnt/nfs
          - name: flink-checkpoints
            hostPath:
              path: /mnt/nfs/172.17.36.38/flink-kube/checkpoints
      - name: flink-completedjobs
        hostPath:
          path: /mnt/nfs/172.17.36.38/flink-kube/completedjobs
      - name: flink-statebackend
        hostPath:
          path: /mnt/nfs/172.17.36.38/flink-kube/statebackend
      - name: flink-target
        hostPath:
          path: /mnt/nfs/172.17.36.38/flink-kube/target
          
          
kubectl create -f taskmanager-deployment.yaml

#Visit WebUI

You can then access the Flink UI via kubectl proxy:

- Run kubectl proxy in a terminal
- Navigate to http://localhost:8001/api/v1/namespaces/default/services/flink-jobmanager:ui/proxy in your browser

http://localhost:8001/api/v1/namespaces/default/services/flink-jobmanager:ui/proxy

#Terminate Flink Cluster

In order to terminate the Flink session cluster, use kubectl:

kubectl delete -f jobmanager-deployment.yaml
    kubectl delete -f taskmanager-deployment.yaml
    kubectl delete -f jobmanager-service.yaml

Navigation

Recent Posts

Friend Links