helm repo add kuberay https://ray-project.github.io/kuberay-helm/
helm install kuberay-operator kuberay/kuberay-operator --version 1.5.1 \
  --namespace kuberay-system --create-namespace

image:
  repository: quay.io/kuberay/operator
  tag: v1.5.1

# Namespace scoping
watchNamespace: []                    # empty = all namespaces
singleNamespaceInstall: false         # true = Role instead of ClusterRole

# RBAC
rbacEnable: true
crNamespacedRbacEnable: true          # false for GitOps tools like ArgoCD

# Feature gates
featureGates:
- name: RayClusterStatusConditions
  enabled: true

# Operator tuning
reconcileConcurrency: 1               # increase for many CRs
batchScheduler: ""                     # "volcano" or "yunikorn"

# Leader election (for HA)
leaderElection:
  enabled: true

apiVersion: ray.io/v1
kind: RayCluster
metadata:
  name: gpu-cluster
spec:
  rayVersion: "2.54.0"
  enableInTreeAutoscaling: true
  autoscalerOptions:
    upscalingMode: Default             # Default | Conservative | Aggressive
    idleTimeoutSeconds: 60
  headGroupSpec:
    serviceType: ClusterIP
    rayStartParams:
      dashboard-host: "0.0.0.0"
      num-cpus: "0"                    # prevent workloads on head
    template:
      spec:
        containers:
        - name: ray-head
          image: rayproject/ray-ml:2.54.0-gpu
          resources:
            limits: { cpu: "4", memory: 16Gi }
            requests: { cpu: "4", memory: 16Gi }
          env:
          - name: NVIDIA_VISIBLE_DEVICES
            value: void                # head doesn't need GPU
  workerGroupSpecs:
  - groupName: gpu-a100
    replicas: 2
    minReplicas: 0
    maxReplicas: 8
    rayStartParams: {}
    template:
      spec:
        containers:
        - name: ray-worker
          image: rayproject/ray-ml:2.54.0-gpu
          resources:
            limits: { cpu: "8", memory: 64Gi, nvidia.com/gpu: "1" }
            requests: { cpu: "8", memory: 64Gi, nvidia.com/gpu: "1" }
        tolerations:
        - key: nvidia.com/gpu
          operator: Exists
          effect: NoSchedule
        nodeSelector:
          nvidia.com/gpu.product: A100

headGroupSpec:
  rayStartParams: {}               # no longer need label/resource hacks here
  resources:
    custom_accelerator: "4"        # Ray logical resource (replaces JSON string in rayStartParams)
  labels:
    ray.io/zone: us-west-2a        # also mirrors into pod labels
    ray.io/region: us-west-2
workerGroupSpecs:
- groupName: gpu-workers
  rayStartParams: {}
  resources:
    custom_accelerator: "4"
  labels:
    ray.io/zone: us-west-2b
  template:
    # ...

rayStartParams:
  resources: '"{\"custom_accelerator\": 4}"'

rayStartParams:
  resources: '"{\"TPU\": 4, \"custom_resource\": 1}"'  # JSON string of custom resources

apiVersion: ray.io/v1
kind: RayJob
metadata:
  name: training-job
spec:
  entrypoint: python /home/ray/train.py --epochs 10
  runtimeEnvYAML: |
    pip:
      - torch==2.5.0
      - transformers
    env_vars:
      WANDB_API_KEY: "secret"
    working_dir: "https://github.com/org/repo/archive/main.zip"
  shutdownAfterJobFinishes: true
  ttlSecondsAfterFinished: 300
  activeDeadlineSeconds: 7200          # max total runtime
  backoffLimit: 0                      # ⚠️ each retry = NEW full cluster (see warning below)
  submissionMode: K8sJobMode           # see submission modes below
  suspend: false                       # true for Kueue integration
  rayClusterSpec:
    rayVersion: "2.54.0"
    headGroupSpec:
      rayStartParams:
        dashboard-host: "0.0.0.0"
      template:
        spec:
          containers:
          - name: ray-head
            image: rayproject/ray-ml:2.54.0-gpu
            resources:
              limits:
                cpu: "4"
                memory: 16Gi
    workerGroupSpecs:
    - groupName: gpu-workers
      replicas: 4
      template:
        spec:
          containers:
          - name: ray-worker
            image: rayproject/ray-ml:2.54.0-gpu
            resources:
              limits:
                cpu: "8"
                memory: 64Gi
                nvidia.com/gpu: "1"

spec:
  backoffLimit: 0              # never recreate the entire cluster on failure
  submitterConfig:
    backoffLimit: 3            # retry submission if dashboard temporarily unreachable

from ray.train import RunConfig, FailureConfig

run_config = RunConfig(
    failure_config=FailureConfig(max_failures=3),  # retry workers, keep cluster
)

spec:
  clusterSelector:
    ray.io/cluster: my-existing-cluster
  # Do NOT include rayClusterSpec

spec:
  enableInTreeAutoscaling: true
  autoscalerOptions:
    upscalingMode: Default           # Default | Aggressive | Conservative
    idleTimeoutSeconds: 60           # seconds before removing idle workers
    resources:
      limits:
        cpu: "500m"
        memory: 512Mi

spec:
  headGroupSpec:
    rayStartParams:
      redis-password: "${REDIS_PASSWORD}"
    template:
      metadata:
        annotations:
          ray.io/ft-enabled: "true"
      spec:
        containers:
        - name: ray-head
          env:
          - name: RAY_REDIS_ADDRESS
            value: "redis:6379"
          - name: RAY_gcs_rpc_server_reconnect_timeout_s
            value: "120"             # worker reconnect timeout (default 60s)

apiVersion: ray.io/v1
kind: RayJob
metadata:
  name: queued-training
  labels:
    kueue.x-k8s.io/queue-name: user-queue
spec:
  suspend: true    # Kueue controls unsuspension
  # ... rest of spec

export HEAD_POD=$(kubectl get pods --selector=ray.io/node-type=head -o custom-columns=POD:metadata.name --no-headers)
kubectl port-forward $HEAD_POD 8265:8265
# Open http://localhost:8265

kubectl exec -it $HEAD_POD -- ray status       # cluster resources
kubectl exec -it $HEAD_POD -- ray list actors   # actor states
kubectl exec -it $HEAD_POD -- ray summary actors

kubectl describe raycluster <name>   # events: CreatedService, CreatedHeadPod, CreatedWorkerPod
kubectl describe rayjob <name>       # events: job submission, completion, failure

# List all Ray resources
kubectl get rayclusters,rayjobs,rayservices -A

# Ray pods
kubectl get pods -l ray.io/is-ray-node=yes
kubectl get pods -l ray.io/node-type=head
kubectl get pods -l ray.io/node-type=worker

# Head pod logs
kubectl logs $HEAD_POD -c ray-head

# Autoscaler logs (sidecar)
kubectl logs $HEAD_POD -c autoscaler

# Worker init container (if stuck)
kubectl logs <worker-pod> -c wait-gcs-ready

# Operator logs
kubectl logs -n kuberay-system deploy/kuberay-operator

# Ray internal logs
kubectl exec -it $HEAD_POD -- ls /tmp/ray/session_latest/logs/
kubectl exec -it $HEAD_POD -- cat /tmp/ray/session_latest/logs/gcs_server.out