cephalobox: support custom registry and images

.env.example

@@ -51,6 +51,16 @@ CEPH2_E2E_IMAGE=rhcsdashboard/ceph-e2e:nautilus
 
 # Cephalobox
 CEPHALOBOX_IMAGE=rhcsdashboard/cephalobox:main
+CEPHADM_IMAGE=
+SHARED_CEPH_REPO_DIR=1 # set to 0 to disable shared_ceph_folder in cephadm
+REGISTRY_URL=
+REGISTRY_USERNAME=
+REGISTRY_PASSWORD=
+CONTAINER_IMAGE_PROMETHEUS=
+CONTAINER_IMAGE_NODE_EXPORTER=
+CONTAINER_IMAGE_ALERTMANAGER=
+CONTAINER_IMAGE_GRAFANA=
+# End of cephalobox options
 
 # Development options
 CEPH_DEBUG=0

.github/workflows/main.yml

@@ -10,6 +10,7 @@ on:
     - docker/ceph/aws/aws-cli-configure.sh
     - 'docker/ceph/**/Dockerfile'
     - docker/ceph/rpm/set-ceph-repo.sh
+    - 'docker/cephalobox/**'
   pull_request:
     branches:
       - main
@@ -18,6 +19,7 @@ on:
     - docker/ceph/aws/aws-cli-configure.sh
     - 'docker/ceph/**/Dockerfile'
     - docker/ceph/rpm/set-ceph-repo.sh
+    - 'docker/cephalobox/**'
   schedule:
     - cron:  '0 2 * * *'
 

README.md

@@ -491,6 +491,96 @@ podman compose logs -f cephalobox
 podman compose down -v
 ```
 
+### Deploying a cluster with custom image
+
+1. Set `CEPHADM_IMAGE` in the .env file.
+
+```bash
+CEPHADM_IMAGE=my.registry/ceph/ceph
+```
+
+Optionally, if the registry is behind auth, provide the registry details in its env variables in .env
+
+```bash
+REGISTRY_URL=
+REGISTRY_USERNAME=
+REGISTRY_PASSWORD=
+```
+
+You can also set CONTAINER_IMAGE_* for each of the images incase you want to override the defaults
+
+```bash
+CONTAINER_IMAGE_PROMETHEUS=
+CONTAINER_IMAGE_NODE_EXPORTER=
+CONTAINER_IMAGE_ALERTMANAGER=
+CONTAINER_IMAGE_GRAFANA=
+```
+
+2. Once done all of those, start the cluster as
+
+```bash
+podman pull cephalobox
+
+podman compose up --attach-dependencies cephalobox
+```
+
+3. The image pulling happens in its own service which can be logged as
+
+```bash
+podman compose logs -f ceph-image-puller
+```
+
+### Disable Shared Ceph Folder
+
+1. Set `SHARED_CEPH_REPO_DIR` to 0 to disable cephadm from mounting your local directory.
+
+2. Start the cluster
+
+```bash
+podman compose up --attach-dependencies cephalobox
+```
+
+This will pull the images locally and then mount them to the cephalobox container so that the image pulling
+is more faster.
+
+### Starting multiple clusters
+
+1. Upto 5 clusters are defined in the docker-compose.yml template. This can be started by
+
+```bash
+podman compose up -d cephalobox cephalobox2 cephalobox3 cephalobox4 cephalobox5
+```
+
+2. You can view individual logs by doing
+
+```bash
+pdoamn compose logs -f cephalobox4
+```
+
+Or just all the logs by
+
+```bash
+podman compose logs -f
+```
+
+3. You can also add more clusters there by following the template defined there. Just make sure the port is mapped correctly.
+For eg. inside docker-compose.yml
+
+```yml
+    cephalobox6:
+        <<: *cephalobox-base
+        container_name: cephalobox6
+        hostname: cephalobox6
+        ports:
+            - "8448:8443"
+            - "3004:3000"
+```
+and then start as
+
+```bash
+podman compose up -d cephalobox6
+```
+
 ## Limitation
 
 * Impossible to run nvmeof service because of the way it is built.

docker-compose.yml

@@ -219,21 +219,47 @@ services:
         - default
         profiles: ['kafka']
 
-    cephalobox:
+    ceph-image-puller:
+        image: docker:cli
+        container_name: ceph-image-puller
+        security_opt:
+        - label=disable
+        environment:
+        - CEPHADM_TARGET_IMAGE_REFERENCE=${CEPHADM_IMAGE:-quay.ceph.io/ceph-ci/ceph:main}
+        - REGISTRY_URL=${REGISTRY_URL}
+        - REGISTRY_USERNAME=${REGISTRY_USERNAME}
+        - REGISTRY_PASSWORD=${REGISTRY_PASSWORD}
+        volumes:
+        - ${XDG_RUNTIME_DIR:-/run/user/1000}/podman/podman.sock:/var/run/docker.sock
+        - ceph_image:/opt:z
+        - ./docker/cephalobox/ceph-image-puller:/ceph-image-puller
+        entrypoint: /bin/sh
+        command: ["/ceph-image-puller/image-puller.sh"]
+
+    x-cephalobox-template: &cephalobox-base
         image: ${CEPHALOBOX_IMAGE:-rhcsdashboard/cephalobox:main}
-        container_name: cephalobox
-        hostname: cephalobox
         tty: true
         stdin_open: true
-        ports:
-            - 8443:8443
-            - 3000:3000
+        depends_on:
+            ceph-image-puller:
+                condition: service_completed_successfully
+        environment:
+        - CEPHADM_IMAGE=${CEPHADM_IMAGE:-quay.ceph.io/ceph-ci/ceph:main}
+        - SHARED_CEPH_REPO_DIR=${SHARED_CEPH_REPO_DIR:-0}
+        - REGISTRY_URL=${REGISTRY_URL}
+        - REGISTRY_USERNAME=${REGISTRY_USERNAME}
+        - REGISTRY_PASSWORD=${REGISTRY_PASSWORD}
+        - CONTAINER_IMAGE_PROMETHEUS=${CONTAINER_IMAGE_PROMETHEUS}
+        - CONTAINER_IMAGE_NODE_EXPORTER=${CONTAINER_IMAGE_NODE_EXPORTER}
+        - CONTAINER_IMAGE_ALERTMANAGER=${CONTAINER_IMAGE_ALERTMANAGER}
+        - CONTAINER_IMAGE_GRAFANA=${CONTAINER_IMAGE_GRAFANA}
         networks:
         - default
         volumes:
         - /run/udev:/run/udev:ro
         - /tmp:/sys/class/dmi/id:ro
         - ${CEPH_REPO_DIR}:/ceph
+        - ceph_image:/opt
         restart: unless-stopped
         # focused privileges so that we don't need to give full
         # privilege to container. SYS_ADMIN is still needed to load kernel but
@@ -242,6 +268,7 @@ services:
         # privileged: true
         cap_add:
         - SYS_ADMIN
+        - NET_ADMIN
         security_opt:
         - seccomp:unconfined
         - label=disable
@@ -252,9 +279,50 @@ services:
         devices:
         - /dev/fuse:/dev/fuse
 
+    cephalobox:
+        <<: *cephalobox-base
+        container_name: cephalobox
+        hostname: cephalobox
+        ports:
+            - "8443:8443"
+            - "3000:3000"
+
+    cephalobox2:
+        <<: *cephalobox-base
+        container_name: cephalobox2
+        hostname: cephalobox2
+        ports:
+            - "8444:8443"
+            - "3001:3000"
+
+    cephalobox3:
+        <<: *cephalobox-base
+        container_name: cephalobox3
+        hostname: cephalobox3
+        ports:
+            - "8445:8443"
+            - "3002:3000"
+
+    cephalobox4:
+        <<: *cephalobox-base
+        container_name: cephalobox4
+        hostname: cephalobox4
+        ports:
+            - "8446:8443"
+            - "3003:3000"
+
+    cephalobox5:
+        <<: *cephalobox-base
+        container_name: cephalobox5
+        hostname: cephalobox5
+        ports:
+            - "8447:8443"
+            - "3004:3000"
+
 volumes:
     empty_volume:
     grafana_data:
+    ceph_image:
 
 networks:
     kafka-network:

docker/cephalobox/Dockerfile

@@ -1,10 +1,10 @@
-FROM almalinux:9
+FROM almalinux:10
 
 RUN dnf install -y epel-release && \
     dnf install -y python3 podman lvm2 \
         systemd openssh-server openssh-clients \
         nano procps-ng crun skopeo util-linux \
-        iproute iptables iptables-nft which
+        iproute iptables iptables-nft which fuse-overlayfs
 
 # to prevent port conflict with host
 RUN mkdir -p /root/.ssh && \
@@ -20,12 +20,15 @@ COPY cephalobox.py /usr/local/bin/cephalobox.py
 COPY cephalobox.sh /usr/local/bin/cephalobox.sh
 COPY osd.sh /usr/local/bin/osd.sh
 
+COPY conf/fuse.conf /etc/containers/storage.conf.d/fuse.conf
+
 RUN chmod +x /opt/wrappers/.
 RUN chmod +x /usr/local/bin/cephalobox.py /usr/local/bin/cephalobox.sh /usr/local/bin/osd.sh
 
 RUN bash /opt/wrappers/podman.sh
 RUN bash /opt/wrappers/logger.sh
 RUN bash /opt/wrappers/environment.sh
+RUN bash /opt/wrappers/sysctl.sh
 
 RUN systemctl enable cephalobox.service
 

docker/cephalobox/ceph-image-puller/image-puller.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+echo "Target image requested: $CEPHADM_TARGET_IMAGE_REFERENCE"
+
+case "$CEPHADM_TARGET_IMAGE_REFERENCE" in
+    "quay.ceph.io/ceph-ci/ceph:main" | \
+    "quay.ceph.io/ceph-ci/ceph:umbrella" | \
+    "quay.ceph.io/ceph-ci/ceph:tentacle")
+        echo "[SKIP] $CEPHADM_TARGET_IMAGE_REFERENCE is a pre-bundled default."
+        echo "[SKIP] Cephalobox already has this internally. Shutting down sidecar."
+        exit 0
+        ;;
+esac
+
+# if registry, then login
+if [ -n "$REGISTRY_URL" ] && [ -n "$REGISTRY_USERNAME" ] && [ -n "$REGISTRY_PASSWORD" ]; then
+    echo "[NETWORK] Logging into registry $REGISTRY_URL..."
+    docker login "$REGISTRY_URL" -u "$REGISTRY_USERNAME" -p "$REGISTRY_PASSWORD"
+fi
+
+echo "[NETWORK] Custom image detected. Pulling via host..."
+docker pull "$CEPHADM_TARGET_IMAGE_REFERENCE"
+
+echo "[DISK] Generating offline tarball archive at /opt/ceph-image.tar..."
+docker save "$CEPHADM_TARGET_IMAGE_REFERENCE" -o /opt/ceph-image.tar
+
+echo "Custom image puller routine completed successfully."