cephalobox - a podman cephadm dev env

.env.example

@@ -49,6 +49,9 @@ CEPH2_HOST_PORT=11002
 CEPH2_CUSTOM_BUILD_DIR=
 CEPH2_E2E_IMAGE=rhcsdashboard/ceph-e2e:nautilus
 
+# Cephalobox
+CEPHALOBOX_IMAGE=rhcsdashboard/cephalobox:main
+
 # Development options
 CEPH_DEBUG=0
 DASHBOARD_DEV_SERVER=1

.github/workflows/main.yml

@@ -162,3 +162,28 @@ jobs:
         docker logout
     - name: Docker Logout
       run: docker logout
+
+  build-cephalobox:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        branch: [main, umbrella, tentacle]
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@master
+
+    - name: Pull ceph-ci image
+      run: |
+        sudo docker pull quay.ceph.io/ceph-ci/ceph:${{ matrix.branch }}
+        sudo docker save quay.ceph.io/ceph-ci/ceph:${{ matrix.branch }} -o ./docker/cephalobox/ceph-image.tar
+
+    - name: Build
+      run: sudo docker build -t docker.io/rhcsdashboard/cephalobox:${{ matrix.branch }} -f docker/cephalobox/Dockerfile ./docker/cephalobox --build-arg TARGETCEPH=${{ matrix.branch }}
+
+    - name: Push
+      run: |
+        echo ${{ secrets.DOCKER_PASSWORD }} | sudo docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
+        sudo docker push docker.io/rhcsdashboard/cephalobox:${{ matrix.branch }}
+        sudo docker logout

README.md

@@ -450,3 +450,48 @@ curl http://localhost:9080
 
 <?xml version="1.0" encoding="UTF-8"?><ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>anonymous</ID></Owner><Buckets></Buckets></ListAllMyBucketsResult>%
 ```
+
+# Cephalobox
+
+A podman based cephadm deployment for deploying ceph. Instead of a full podman in podman, it leverages podman directly on container's host with privileged
+capabilities, so that cephadm can natively deploy and manage services.
+
+## Quirks
+
+* OSDs are not managed by cephadm. instead deployed using ceph-osd binary.
+
+## Deployment
+
+Strongly advised to use podman compose. docker compose is too much..
+
+1. Build cephadm binary in your local. Since this favors mostly local development, generate the cephadm binary in your `src/cephadm` directory inside ceph.
+
+2. Also build the front-end assets inside the `src/pybind/mgr/dashboard/frontend` folder in ceph.
+
+3. Populate the `CEPH_REPO_DIR` in the .env file so that it reads your local code.
+
+4. use the image you want, supports `main`, `umbrella` and `tentacle`. Prebundling the image so that we don't need to pull it everytime we start a cluster.
+Provide the correct image in .env file's `CEPHALOBOX_IMAGE`. defaults to `rhcsdashboard/cephalobox:main`.
+
+5. Start the container
+
+```bash
+podman compose up -d cephalobox
+```
+
+6. Check the logs. By default, it pushes out cephadm's bootstrap log and also the ceph-mgr logs.
+
+```bash
+podman compose logs -f cephalobox
+```
+
+7. to destroy
+
+```bash
+podman compose down -v
+```
+
+## Limitation
+
+* Impossible to run nvmeof service because of the way it is built.
+* Can't create osds at all. Maybe possible to add hosts, but not tested.

docker-compose.yml

@@ -219,6 +219,39 @@ services:
         - default
         profiles: ['kafka']
 
+    cephalobox:
+        image: ${CEPHALOBOX_IMAGE:-rhcsdashboard/cephalobox:main}
+        container_name: cephalobox
+        hostname: cephalobox
+        tty: true
+        stdin_open: true
+        ports:
+            - 8443:8443
+            - 3000:3000
+        networks:
+        - default
+        volumes:
+        - /run/udev:/run/udev:ro
+        - /tmp:/sys/class/dmi/id:ro
+        - ${CEPH_REPO_DIR}:/ceph
+        restart: unless-stopped
+        # focused privileges so that we don't need to give full
+        # privilege to container. SYS_ADMIN is still needed to load kernel but
+        # disallows permissions to network and raw devices.
+        # if it proves tricky, maybe eventually go on with privileged: true
+        # privileged: true
+        cap_add:
+        - SYS_ADMIN
+        security_opt:
+        - seccomp:unconfined
+        - label=disable
+        tmpfs:
+        - /run
+        - /run/lock
+        - /tmp
+        devices:
+        - /dev/fuse:/dev/fuse
+
 volumes:
     empty_volume:
     grafana_data:

docker/cephalobox/Dockerfile

@@ -0,0 +1,32 @@
+FROM almalinux:9
+
+RUN dnf install -y epel-release && \
+    dnf install -y python3 podman lvm2 \
+        systemd openssh-server openssh-clients \
+        nano procps-ng crun skopeo util-linux \
+        iproute iptables iptables-nft which
+
+# to prevent port conflict with host
+RUN mkdir -p /root/.ssh && \
+    echo -e "Host *\n    StrictHostKeyChecking no\n" > /root/.ssh/config && \
+    sed -i 's/^#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config
+
+COPY ceph-image.tar /opt/ceph-image.tar
+
+COPY wrappers/ /opt/wrappers/
+
+COPY cephalobox.service /etc/systemd/system/cephalobox.service
+COPY cephalobox.py /usr/local/bin/cephalobox.py
+COPY cephalobox.sh /usr/local/bin/cephalobox.sh
+COPY osd.sh /usr/local/bin/osd.sh
+
+RUN chmod +x /opt/wrappers/.
+RUN chmod +x /usr/local/bin/cephalobox.py /usr/local/bin/cephalobox.sh /usr/local/bin/osd.sh
+
+RUN bash /opt/wrappers/podman.sh
+RUN bash /opt/wrappers/logger.sh
+RUN bash /opt/wrappers/environment.sh
+
+RUN systemctl enable cephalobox.service
+
+CMD ["/sbin/init"]

docker/cephalobox/cephalobox.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+import os
+import sys
+import subprocess
+import socket
+
+SHARED_CEPH_FOLDER = "/ceph"
+
+def execute_shell_command_safely(command_string: str, capture_output: bool = True, exit_on_failure: bool = True) -> str:
+    print(f"\n[EXEC] {command_string}")
+    result = subprocess.run(command_string, shell=True, text=True, capture_output=capture_output)
+    if result.returncode != 0 and exit_on_failure:
+        print(f"\n[FATAL ERROR] Command failed with exit code {result.returncode}")
+        if capture_output:
+            print(f"[ERROR OUTPUT]\n{result.stderr.strip()}")
+        sys.exit(1)
+    return result.stdout.strip() if capture_output else ""
+
+def get_primary_routable_ip_address() -> str:
+    temporary_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    try:
+        temporary_socket.connect(('10.255.255.255', 1))
+        primary_ip = temporary_socket.getsockname()[0]
+    except Exception:
+        primary_ip = '127.0.0.1'
+    finally:
+        temporary_socket.close()
+    return primary_ip
+
+def prepare_local_cephadm_binary():
+    """Locates cephadm within the local shared source directory, sets up bin/, and makes it executable."""
+    print("\n=== Phase 2: Preparing Local Cephadm Binary ===")
+    local_cephadm_path = os.path.join(SHARED_CEPH_FOLDER, "src/cephadm/cephadm")
+
+    if not os.path.exists(local_cephadm_path):
+        print(f"\n[FATAL ERROR] Local cephadm binary not found at expected path: {local_cephadm_path}")
+        print(f"Please verify that your source code folder is properly mounted at: {SHARED_CEPH_FOLDER} ")
+        sys.exit(1)
+
+    os.makedirs("bin", exist_ok=True)
+    bin_cephadm_target = os.path.join("bin", "cephadm")
+
+    for target in [bin_cephadm_target, "./cephadm"]:
+        if os.path.exists(target):
+            os.remove(target)
+
+    execute_shell_command_safely(f"cp {local_cephadm_path} {bin_cephadm_target}")
+    execute_shell_command_safely(f"cp {local_cephadm_path} ./cephadm")
+    execute_shell_command_safely("chmod +x bin/cephadm ./cephadm")
+    print(f"Successfully configured local cephadm binary from {local_cephadm_path} into bin/ and root.")
+
+def create_initial_cluster_configuration_file():
+    """Generates the initial-ceph.conf file with size-one and deletion overrides."""
+    print("\n=== Phase 3: Creating Initial Cluster Config ===")
+    config_content = """[global]
+osd_pool_default_min_size=1
+osd_pool_default_size=1
+public_network=172.20.0.0/24
+
+[mon]
+mon_allow_pool_size_one=true
+mon_allow_pool_delete=true
+mon_data_avail_crit=1
+mon_data_avail_warn=1
+"""
+    with open("initial-ceph.conf", "w") as config_file:
+        config_file.write(config_content)
+    print("Successfully generated initial-ceph.conf")
+
+def load_ceph_image():
+    print("\nLoading cached images....")
+    tarball_path = "/opt/ceph-image.tar"
+    if os.path.exists(tarball_path):
+        print(f"Loading pre-bundled image from {tarball_path}...")
+        result = subprocess.run(["podman", "load", "-i", tarball_path])
+        if result.returncode == 0:
+            print("Successfully loaded cached image.")
+            return None
+
+    print("No cached images... going to retry on pulling the images..")
+    return None
+
+def bootstrap_initial_ceph_cluster(monitor_ip_address: str):
+    print(f"\n=== Phase 3: Bootstrapping Cluster on {monitor_ip_address} ===")
+    custom_image = os.environ.get("CEPH_IMAGE")
+    image_flag = f"--image {custom_image} " if custom_image else ""
+    bootstrap_command = (
+        f"cephadm {image_flag}bootstrap "
+        f"--mon-ip {monitor_ip_address} "
+        f"--allow-overwrite "
+        f"--skip-mon-network "
+        f"--config initial-ceph.conf "
+        f"--dashboard-password-noupdate "
+        f"--shared_ceph_folder /ceph "
+        f"--initial-dashboard-password admin "
+        f"--allow-fqdn-hostname"
+    )
+    execute_shell_command_safely(bootstrap_command, capture_output=False)
+
+def initialize_cephalobox():
+    monitor_ip_address = get_primary_routable_ip_address()
+    prepare_local_cephadm_binary()
+    create_initial_cluster_configuration_file()
+
+    load_ceph_image()
+
+    bootstrap_initial_ceph_cluster(monitor_ip_address)
+
+    print("\nCephalabox has done its job!!!!")
+
+
+if __name__ == '__main__':
+    initialize_cephalobox()

docker/cephalobox/cephalobox.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Cephalobox 
+After=network.target sshd.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+KillMode=process
+ExecStartPre=/bin/sleep 3
+ExecStart=/bin/bash /usr/local/bin/cephalobox.sh
+StandardOutput=journal+console
+
+[Install]
+WantedBy=multi-user.target

docker/cephalobox/cephalobox.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+echo "=== CephaloBox Orchestrator Wrapper Started ==="
+
+# Ensure standard binaries and your custom build path are fully accessible in PATH
+export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/ceph/build/bin
+
+# Execute the main Python automation script
+python3 -u /usr/local/bin/cephalobox.py
+/bin/bash /usr/local/bin/osd.sh
+
+echo "=== CephaloBox Orchestrator Finished Successfully ==="
+

docker/cephalobox/osd.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+set -e
+
+BASE_DIRECTORY_FOR_OSDS="/var/lib/ceph/cephalobox-osds"
+OSD_FILE_BACKING_SIZE="20G"
+CEPH_INTERNAL_USER_ID="167"
+CEPH_INTERNAL_GROUP_ID="167"
+
+mkdir -p "$BASE_DIRECTORY_FOR_OSDS"
+
+CONFIG_PATH="$BASE_DIRECTORY_FOR_OSDS/ceph.conf"
+cephadm shell -- cat /etc/ceph/ceph.conf > "$CONFIG_PATH"
+
+CONTAINER_IMAGE=$(podman images -q quay.ceph.io/ceph-ci/ceph | head -n 1)
+
+if [ -z "$CONTAINER_IMAGE" ]; then
+    echo "[FATAL] No pre-cached Ceph image found!"
+    exit 1
+fi
+
+for i in {0..2}
+do
+    echo -e "\n=== Phase 4.$i: Provisioning Standalone OSD ==="
+
+    ASSIGNED_OSD_ID=$(cephadm shell -- ceph osd create)
+    echo "[EXEC] Created OSD ID: $ASSIGNED_OSD_ID"
+
+    OSD_SPECIFIC_DIRECTORY="$BASE_DIRECTORY_FOR_OSDS/osd-$ASSIGNED_OSD_ID"
+    mkdir -p "$OSD_SPECIFIC_DIRECTORY"
+
+    KEYRING_OUTPUT_PATH="$OSD_SPECIFIC_DIRECTORY/keyring"
+    cephadm shell -- ceph auth get-or-create osd.$ASSIGNED_OSD_ID \
+        mon 'allow profile osd' mgr 'allow profile osd' osd 'allow *' > "$KEYRING_OUTPUT_PATH"
+
+    BLOCK_DEVICE_FILE_PATH="$OSD_SPECIFIC_DIRECTORY/block"
+    truncate -s "$OSD_FILE_BACKING_SIZE" "$BLOCK_DEVICE_FILE_PATH"
+    chown -R "$CEPH_INTERNAL_USER_ID":"$CEPH_INTERNAL_GROUP_ID" "$OSD_SPECIFIC_DIRECTORY"
+
+    # format OSD using the image's built-in binary
+    echo "[EXEC] Formatting OSD $ASSIGNED_OSD_ID with BlueStore..."
+    podman run --rm --net=host \
+        -v "$CONFIG_PATH":/etc/ceph/ceph.conf:z \
+        -v "$OSD_SPECIFIC_DIRECTORY":/var/lib/ceph/osd/ceph-$ASSIGNED_OSD_ID:z \
+        "$CONTAINER_IMAGE" ceph-osd -i "$ASSIGNED_OSD_ID" --mkfs
+
+    # start OSD Daemon in background using the image's built-in binary
+    echo "[EXEC] Starting background OSD container: cephalobox-osd-$ASSIGNED_OSD_ID"
+    podman run -d --name cephalobox-osd-$ASSIGNED_OSD_ID --net=host \
+        -v "$CONFIG_PATH":/etc/ceph/ceph.conf:z \
+        -v "$OSD_SPECIFIC_DIRECTORY":/var/lib/ceph/osd/ceph-$ASSIGNED_OSD_ID:z \
+        "$CONTAINER_IMAGE" ceph-osd -i "$ASSIGNED_OSD_ID" -f
+done
+
+echo -e "\n=== CephaloBox OSD Provisioning Complete! ==="
+cephadm shell -- ceph -s

docker/cephalobox/wrappers/environment.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# missing dir for cephadm
+mkdir -p /run/udev/data
+mkdir -p /dev/cpu
+
+rm -rf /etc/sysctl.d
+mkdir -p /etc/sysctl.d
+
+# spoofing time for chrony service
+if [ ! -f "/usr/bin/systemctl.real" ]; then
+    mv /usr/bin/systemctl /usr/bin/systemctl.real
+fi
+
+cat <<'EOF' > /usr/bin/systemctl
+#!/bin/bash
+args=("$@")
+TIME_DAEMONS=("chronyd" "chrony.service" "chrony" "systemd-timesyncd" "ntpd" "ntp" "timemaster")
+
+for arg in "${args[@]}"; do
+    for daemon in "${TIME_DAEMONS[@]}"; do
+        if [[ "$arg" == *"$daemon"* ]]; then
+            if [[ "${args[0]}" == "is-active" || "${args[0]}" == "status" ]]; then
+                echo "active"
+                exit 0
+            fi
+            exit 0
+        fi
+    done
+done
+
+exec /usr/bin/systemctl.real "${args[@]}"
+EOF
+
+chmod +x /usr/bin/systemctl
+
+ssh-keygen -A
+systemctl enable sshd