tensorflowdistributedtrainingwithdefaultscheduledgpu.yaml

# For the full spec setting and usage, see ./pkg/apis/frameworkcontroller/v1/types.go
# For the full frameworkbarrier usage, see ./pkg/barrier/barrier.go

############################### Prerequisite ###################################
# See "[PREREQUISITE]" in this file.
################################################################################
apiVersion: frameworkcontroller.microsoft.com/v1
kind: Framework
metadata:
  name: tensorflowdistributedtrainingwithdefaultscheduledgpu
spec:
  executionType: Start
  retryPolicy:
    fancyRetryPolicy: true
    maxRetryCount: 2
  taskRoles:
  - name: ps
    taskNumber: 2
    frameworkAttemptCompletionPolicy:
      minFailedTaskCount: 1
      minSucceededTaskCount: -1
    task:
      retryPolicy:
        fancyRetryPolicy: false
        maxRetryCount: 0
      pod:
        spec:
          restartPolicy: Never
          # [PREREQUISITE]
          # User needs to setup the k8s cluster networking model and aware the
          # potential network overhead, if he want to disable the hostNetwork to
          # avoid the coordination of the containerPort usage.
          # And for this example, if the hostNetwork is disabled, it only needs
          # at least 1 node, otherwise, it needs at least 3 nodes since all the
          # 3 workers are specified with the same containerPort.
          # See https://kubernetes.io/docs/concepts/cluster-administration/networking
          hostNetwork: false
          containers:
          - name: tensorflow
            # Using official image to demonstrate this example.
            # The image contains and only contains tensorflow official code.
            image: frameworkcontroller/tensorflow-examples:gpu
            # For the tf_cnn_benchmarks usage, see
            # https://github.com/tensorflow/benchmarks/tree/master/scripts/tf_cnn_benchmarks
            workingDir: /tensorflow/benchmarks/scripts/tf_cnn_benchmarks
            # Using /mnt/frameworkbarrier/injector.sh to inject environment variables
            # without the need for image invasion and k8s DNS:
            # FB_{UpperCase({TaskRoleName})}_ADDRESSES=
            #   {Task[0].PodIP}:${FB_{UpperCase({TaskRoleName})}_PORT},...,
            #   {Task[TaskRole.TaskNumber-1].PodIP}:${FB_{UpperCase({TaskRoleName})}_PORT}
            # See more in ./example/framework/extension/frameworkbarrier.yaml
            command: [
            "sh", "-c",
            "FB_PS_PORT=4001 FB_WORKER_PORT=5001 . /mnt/frameworkbarrier/injector.sh &&
            python tf_cnn_benchmarks.py --job_name=ps --task_index=${FC_TASK_INDEX}
            --ps_hosts=${FB_PS_ADDRESSES} --worker_hosts=${FB_WORKER_ADDRESSES}
            --variable_update=parameter_server --cross_replica_sync=false
            --model=alexnet --batch_size=8 --num_batches=10
            --device=gpu --local_parameter_device=gpu --num_gpus=1 --data_format=NCHW
            --data_name=cifar10 --data_dir=/mnt/data/cifar-10-batches-py
            --train_dir=/mnt/data/${FC_FRAMEWORK_NAME}/output"]
            ports:
            - containerPort: 4001
            resources:
              limits:
                # [PREREQUISITE]
                # User needs to setup GPU for the k8s cluster.
                # See https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus
                nvidia.com/gpu: 1
            volumeMounts:
            - name: frameworkbarrier-volume
              mountPath: /mnt/frameworkbarrier
            - name: data-volume
              mountPath: /mnt/data
          # [PREREQUISITE]
          # User needs to create a service account for frameworkbarrier, if the
          # k8s cluster enforces authorization.
          # See more in ./example/framework/extension/frameworkbarrier.yaml
          serviceAccountName: frameworkbarrier
          initContainers:
          - name: frameworkbarrier
            # Using official image to demonstrate this example.
            image: frameworkcontroller/frameworkbarrier
            # Using k8s inClusterConfig, so usually, no need to specify
            # KUBE_APISERVER_ADDRESS or KUBECONFIG
            #env:
            #- name: KUBE_APISERVER_ADDRESS
            #  value: {http[s]://host:port}
            #- name: KUBECONFIG
            #  value: {Pod Local KubeConfig File Path}
            volumeMounts:
            - name: frameworkbarrier-volume
              mountPath: /mnt/frameworkbarrier
          volumes:
          - name: frameworkbarrier-volume
            emptyDir: {}
          - name: data-volume
            # [PREREQUISITE]
            # User needs to specify his own data-volume for input data and
            # output model.
            # The data-volume must be a distributed shared file system, so that
            # data can be "handed off" between Pods, such as nfs, cephfs or
            # glusterfs, etc.
            # See https://kubernetes.io/docs/concepts/storage/volumes.
            #
            # And then he needs to download and extract the example input data
            # from:
            #   https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz
            # to:
            #   {Volume Shared Directory}/cifar-10-batches-py
            #
            # For example:
            #nfs:
            #  server: {NFS Server Host}
            #  path: {NFS Shared Directory}
  - name: worker
    taskNumber: 3
    frameworkAttemptCompletionPolicy:
      minFailedTaskCount: 1
      # Succeed the FrameworkAttempt immediately if worker's all Tasks succeeded.
      minSucceededTaskCount: 3
    task:
      retryPolicy:
        fancyRetryPolicy: false
        maxRetryCount: 0
      pod:
        spec:
          restartPolicy: Never
          # [PREREQUISITE]
          # Same as ps TaskRole.
          hostNetwork: false
          containers:
          - name: tensorflow
            image: frameworkcontroller/tensorflow-examples:gpu
            workingDir: /tensorflow/benchmarks/scripts/tf_cnn_benchmarks
            command: [
            "sh", "-c",
            "FB_PS_PORT=4001 FB_WORKER_PORT=5001 . /mnt/frameworkbarrier/injector.sh &&
            python tf_cnn_benchmarks.py --job_name=worker --task_index=${FC_TASK_INDEX}
            --ps_hosts=${FB_PS_ADDRESSES} --worker_hosts=${FB_WORKER_ADDRESSES}
            --variable_update=parameter_server --cross_replica_sync=false
            --model=alexnet --batch_size=8 --num_batches=10
            --device=gpu --local_parameter_device=gpu --num_gpus=1 --data_format=NCHW
            --data_name=cifar10 --data_dir=/mnt/data/cifar-10-batches-py
            --train_dir=/mnt/data/${FC_FRAMEWORK_NAME}/output"]
            ports:
            - containerPort: 5001
            resources:
              limits:
                # [PREREQUISITE]
                # Same as ps TaskRole.
                nvidia.com/gpu: 1
            volumeMounts:
            - name: frameworkbarrier-volume
              mountPath: /mnt/frameworkbarrier
            - name: data-volume
              mountPath: /mnt/data
          # [PREREQUISITE]
          # Same as ps TaskRole.
          serviceAccountName: frameworkbarrier
          initContainers:
          - name: frameworkbarrier
            image: frameworkcontroller/frameworkbarrier
            #env:
            #- name: KUBE_APISERVER_ADDRESS
            #  value: {http[s]://host:port}
            #- name: KUBECONFIG
            #  value: {Pod Local KubeConfig File Path}
            volumeMounts:
            - name: frameworkbarrier-volume
              mountPath: /mnt/frameworkbarrier
          volumes:
          - name: frameworkbarrier-volume
            emptyDir: {}
          - name: data-volume
            # [PREREQUISITE]
            # Same as ps TaskRole.
            #nfs:
            #  server: {NFS Server Host}
            #  path: {NFS Shared Directory}