diff --git a/.env.example b/.env.example index 1e38873382..a3f1fa8002 100644 --- a/.env.example +++ b/.env.example @@ -51,6 +51,16 @@ CEPH2_E2E_IMAGE=rhcsdashboard/ceph-e2e:nautilus # Cephalobox CEPHALOBOX_IMAGE=rhcsdashboard/cephalobox:main +CEPHADM_IMAGE= +SHARED_CEPH_REPO_DIR=1 # set to 0 to disable shared_ceph_folder in cephadm +REGISTRY_URL= +REGISTRY_USERNAME= +REGISTRY_PASSWORD= +CONTAINER_IMAGE_PROMETHEUS= +CONTAINER_IMAGE_NODE_EXPORTER= +CONTAINER_IMAGE_ALERTMANAGER= +CONTAINER_IMAGE_GRAFANA= +# End of cephalobox options # Development options CEPH_DEBUG=0 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3de5d58da1..f8f4f87577 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,6 +10,7 @@ on: - docker/ceph/aws/aws-cli-configure.sh - 'docker/ceph/**/Dockerfile' - docker/ceph/rpm/set-ceph-repo.sh + - 'docker/cephalobox/**' pull_request: branches: - main @@ -18,6 +19,7 @@ on: - docker/ceph/aws/aws-cli-configure.sh - 'docker/ceph/**/Dockerfile' - docker/ceph/rpm/set-ceph-repo.sh + - 'docker/cephalobox/**' schedule: - cron: '0 2 * * *' diff --git a/README.md b/README.md index 79a6c92d0a..acd511291c 100644 --- a/README.md +++ b/README.md @@ -491,6 +491,96 @@ podman compose logs -f cephalobox podman compose down -v ``` +### Deploying a cluster with custom image + +1. Set `CEPHADM_IMAGE` in the .env file. + +```bash +CEPHADM_IMAGE=my.registry/ceph/ceph +``` + +Optionally, if the registry is behind auth, provide the registry details in its env variables in .env + +```bash +REGISTRY_URL= +REGISTRY_USERNAME= +REGISTRY_PASSWORD= +``` + +You can also set CONTAINER_IMAGE_* for each of the images incase you want to override the defaults + +```bash +CONTAINER_IMAGE_PROMETHEUS= +CONTAINER_IMAGE_NODE_EXPORTER= +CONTAINER_IMAGE_ALERTMANAGER= +CONTAINER_IMAGE_GRAFANA= +``` + +2. Once done all of those, start the cluster as + +```bash +podman pull cephalobox + +podman compose up --attach-dependencies cephalobox +``` + +3. The image pulling happens in its own service which can be logged as + +```bash +podman compose logs -f ceph-image-puller +``` + +### Disable Shared Ceph Folder + +1. Set `SHARED_CEPH_REPO_DIR` to 0 to disable cephadm from mounting your local directory. + +2. Start the cluster + +```bash +podman compose up --attach-dependencies cephalobox +``` + +This will pull the images locally and then mount them to the cephalobox container so that the image pulling +is more faster. + +### Starting multiple clusters + +1. Upto 5 clusters are defined in the docker-compose.yml template. This can be started by + +```bash +podman compose up -d cephalobox cephalobox2 cephalobox3 cephalobox4 cephalobox5 +``` + +2. You can view individual logs by doing + +```bash +pdoamn compose logs -f cephalobox4 +``` + +Or just all the logs by + +```bash +podman compose logs -f +``` + +3. You can also add more clusters there by following the template defined there. Just make sure the port is mapped correctly. +For eg. inside docker-compose.yml + +```yml + cephalobox6: + <<: *cephalobox-base + container_name: cephalobox6 + hostname: cephalobox6 + ports: + - "8448:8443" + - "3004:3000" +``` +and then start as + +```bash +podman compose up -d cephalobox6 +``` + ## Limitation * Impossible to run nvmeof service because of the way it is built. diff --git a/docker-compose.yml b/docker-compose.yml index 283d637b6c..d606ca402e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -219,21 +219,47 @@ services: - default profiles: ['kafka'] - cephalobox: + ceph-image-puller: + image: docker:cli + container_name: ceph-image-puller + security_opt: + - label=disable + environment: + - CEPHADM_TARGET_IMAGE_REFERENCE=${CEPHADM_IMAGE:-quay.ceph.io/ceph-ci/ceph:main} + - REGISTRY_URL=${REGISTRY_URL} + - REGISTRY_USERNAME=${REGISTRY_USERNAME} + - REGISTRY_PASSWORD=${REGISTRY_PASSWORD} + volumes: + - ${XDG_RUNTIME_DIR:-/run/user/1000}/podman/podman.sock:/var/run/docker.sock + - ceph_image:/opt:z + - ./docker/cephalobox/ceph-image-puller:/ceph-image-puller + entrypoint: /bin/sh + command: ["/ceph-image-puller/image-puller.sh"] + + x-cephalobox-template: &cephalobox-base image: ${CEPHALOBOX_IMAGE:-rhcsdashboard/cephalobox:main} - container_name: cephalobox - hostname: cephalobox tty: true stdin_open: true - ports: - - 8443:8443 - - 3000:3000 + depends_on: + ceph-image-puller: + condition: service_completed_successfully + environment: + - CEPHADM_IMAGE=${CEPHADM_IMAGE:-quay.ceph.io/ceph-ci/ceph:main} + - SHARED_CEPH_REPO_DIR=${SHARED_CEPH_REPO_DIR:-0} + - REGISTRY_URL=${REGISTRY_URL} + - REGISTRY_USERNAME=${REGISTRY_USERNAME} + - REGISTRY_PASSWORD=${REGISTRY_PASSWORD} + - CONTAINER_IMAGE_PROMETHEUS=${CONTAINER_IMAGE_PROMETHEUS} + - CONTAINER_IMAGE_NODE_EXPORTER=${CONTAINER_IMAGE_NODE_EXPORTER} + - CONTAINER_IMAGE_ALERTMANAGER=${CONTAINER_IMAGE_ALERTMANAGER} + - CONTAINER_IMAGE_GRAFANA=${CONTAINER_IMAGE_GRAFANA} networks: - default volumes: - /run/udev:/run/udev:ro - /tmp:/sys/class/dmi/id:ro - ${CEPH_REPO_DIR}:/ceph + - ceph_image:/opt restart: unless-stopped # focused privileges so that we don't need to give full # privilege to container. SYS_ADMIN is still needed to load kernel but @@ -242,6 +268,7 @@ services: # privileged: true cap_add: - SYS_ADMIN + - NET_ADMIN security_opt: - seccomp:unconfined - label=disable @@ -252,9 +279,50 @@ services: devices: - /dev/fuse:/dev/fuse + cephalobox: + <<: *cephalobox-base + container_name: cephalobox + hostname: cephalobox + ports: + - "8443:8443" + - "3000:3000" + + cephalobox2: + <<: *cephalobox-base + container_name: cephalobox2 + hostname: cephalobox2 + ports: + - "8444:8443" + - "3001:3000" + + cephalobox3: + <<: *cephalobox-base + container_name: cephalobox3 + hostname: cephalobox3 + ports: + - "8445:8443" + - "3002:3000" + + cephalobox4: + <<: *cephalobox-base + container_name: cephalobox4 + hostname: cephalobox4 + ports: + - "8446:8443" + - "3003:3000" + + cephalobox5: + <<: *cephalobox-base + container_name: cephalobox5 + hostname: cephalobox5 + ports: + - "8447:8443" + - "3004:3000" + volumes: empty_volume: grafana_data: + ceph_image: networks: kafka-network: diff --git a/docker/cephalobox/Dockerfile b/docker/cephalobox/Dockerfile index 329916bf41..e869a69d22 100644 --- a/docker/cephalobox/Dockerfile +++ b/docker/cephalobox/Dockerfile @@ -1,10 +1,10 @@ -FROM almalinux:9 +FROM almalinux:10 RUN dnf install -y epel-release && \ dnf install -y python3 podman lvm2 \ systemd openssh-server openssh-clients \ nano procps-ng crun skopeo util-linux \ - iproute iptables iptables-nft which + iproute iptables iptables-nft which fuse-overlayfs # to prevent port conflict with host RUN mkdir -p /root/.ssh && \ @@ -20,12 +20,15 @@ COPY cephalobox.py /usr/local/bin/cephalobox.py COPY cephalobox.sh /usr/local/bin/cephalobox.sh COPY osd.sh /usr/local/bin/osd.sh +COPY conf/fuse.conf /etc/containers/storage.conf.d/fuse.conf + RUN chmod +x /opt/wrappers/. RUN chmod +x /usr/local/bin/cephalobox.py /usr/local/bin/cephalobox.sh /usr/local/bin/osd.sh RUN bash /opt/wrappers/podman.sh RUN bash /opt/wrappers/logger.sh RUN bash /opt/wrappers/environment.sh +RUN bash /opt/wrappers/sysctl.sh RUN systemctl enable cephalobox.service diff --git a/docker/cephalobox/ceph-image-puller/image-puller.sh b/docker/cephalobox/ceph-image-puller/image-puller.sh new file mode 100755 index 0000000000..260328dc9c --- /dev/null +++ b/docker/cephalobox/ceph-image-puller/image-puller.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +echo "Target image requested: $CEPHADM_TARGET_IMAGE_REFERENCE" + +case "$CEPHADM_TARGET_IMAGE_REFERENCE" in + "quay.ceph.io/ceph-ci/ceph:main" | \ + "quay.ceph.io/ceph-ci/ceph:umbrella" | \ + "quay.ceph.io/ceph-ci/ceph:tentacle") + echo "[SKIP] $CEPHADM_TARGET_IMAGE_REFERENCE is a pre-bundled default." + echo "[SKIP] Cephalobox already has this internally. Shutting down sidecar." + exit 0 + ;; +esac + +# if registry, then login +if [ -n "$REGISTRY_URL" ] && [ -n "$REGISTRY_USERNAME" ] && [ -n "$REGISTRY_PASSWORD" ]; then + echo "[NETWORK] Logging into registry $REGISTRY_URL..." + docker login "$REGISTRY_URL" -u "$REGISTRY_USERNAME" -p "$REGISTRY_PASSWORD" +fi + +echo "[NETWORK] Custom image detected. Pulling via host..." +docker pull "$CEPHADM_TARGET_IMAGE_REFERENCE" + +echo "[DISK] Generating offline tarball archive at /opt/ceph-image.tar..." +docker save "$CEPHADM_TARGET_IMAGE_REFERENCE" -o /opt/ceph-image.tar + +echo "Custom image puller routine completed successfully." diff --git a/docker/cephalobox/cephalobox.py b/docker/cephalobox/cephalobox.py index 31db7a4998..dd09ef898e 100644 --- a/docker/cephalobox/cephalobox.py +++ b/docker/cephalobox/cephalobox.py @@ -29,25 +29,53 @@ def get_primary_routable_ip_address() -> str: def prepare_local_cephadm_binary(): """Locates cephadm within the local shared source directory, sets up bin/, and makes it executable.""" - print("\n=== Phase 2: Preparing Local Cephadm Binary ===") - local_cephadm_path = os.path.join(SHARED_CEPH_FOLDER, "src/cephadm/cephadm") - - if not os.path.exists(local_cephadm_path): - print(f"\n[FATAL ERROR] Local cephadm binary not found at expected path: {local_cephadm_path}") - print(f"Please verify that your source code folder is properly mounted at: {SHARED_CEPH_FOLDER} ") - sys.exit(1) + print("\n=== Phase 2: Preparing Cephadm Binary ===") - os.makedirs("bin", exist_ok=True) - bin_cephadm_target = os.path.join("bin", "cephadm") + global_target_sbin = "/usr/sbin/cephadm" + global_target_bin = "/usr/bin/cephadm" - for target in [bin_cephadm_target, "./cephadm"]: + for target in [global_target_sbin, global_target_bin]: if os.path.exists(target): - os.remove(target) + try: + os.remove(target) + except OSError: + pass + + cephadm_image = os.environ.get("CEPHADM_IMAGE") + + # extract from the container image + if cephadm_image and cephadm_image != "None": + print(f"Extracting cephadm binary directly from container image: {cephadm_image}...") + + with open(global_target_sbin, "w") as out_file: + extract_result = subprocess.run( + ["podman", "run", "--rm", "--net=host", "--entrypoint=cat", cephadm_image, "/usr/sbin/cephadm"], + stdout=out_file, + text=True + ) + + if extract_result.returncode != 0: + print(f"\n[FATAL ERROR] Failed to extract cephadm from image {cephadm_image}.") + sys.exit(1) - execute_shell_command_safely(f"cp {local_cephadm_path} {bin_cephadm_target}") - execute_shell_command_safely(f"cp {local_cephadm_path} ./cephadm") - execute_shell_command_safely("chmod +x bin/cephadm ./cephadm") - print(f"Successfully configured local cephadm binary from {local_cephadm_path} into bin/ and root.") + print(f"Successfully extracted cephadm from {cephadm_image}") + + # copy from local shared source directory + else: + print("No container image specified. Falling back to local shared source directory...") + local_cephadm_path = os.path.join(SHARED_CEPH_FOLDER, "src/cephadm/cephadm") + + if not os.path.exists(local_cephadm_path): + print(f"\n[FATAL ERROR] Local cephadm binary not found at expected path: {local_cephadm_path}") + sys.exit(1) + + execute_shell_command_safely(f"cp {local_cephadm_path} {global_target_sbin}") + print(f"Successfully copied local cephadm binary from {local_cephadm_path}") + + execute_shell_command_safely(f"cp {global_target_sbin} {global_target_bin}") + execute_shell_command_safely(f"chmod +x {global_target_sbin} {global_target_bin}") + + print("Successfully made cephadm binaries executable globally.") def create_initial_cluster_configuration_file(): """Generates the initial-ceph.conf file with size-one and deletion overrides.""" @@ -63,6 +91,21 @@ def create_initial_cluster_configuration_file(): mon_data_avail_crit=1 mon_data_avail_warn=1 """ + mgr_configs = [] + prefix = "CONTAINER_IMAGE_" + + for env_key, env_value in os.environ.items(): + # looking for envs like CONTAINER_IMAGE_* that have a value + if env_key.startswith(prefix) and env_value.strip(): + component_name = env_key[len(prefix):].lower() + + mgr_configs.append(f"mgr/cephadm/container_image_{component_name} = {env_value.strip()}") + print(f"Found custom image for {component_name}: {env_value.strip()}") + + if mgr_configs: + config_content += "\n[mgr]\n" + config_content += "\n".join(mgr_configs) + "\n" + with open("initial-ceph.conf", "w") as config_file: config_file.write(config_content) print("Successfully generated initial-ceph.conf") @@ -80,30 +123,45 @@ def load_ceph_image(): print("No cached images... going to retry on pulling the images..") return None +def get_registry_credentials(): + registry_url = os.environ.get("REGISTRY_URL") + registry_username = os.environ.get("REGISTRY_USERNAME") + registry_password = os.environ.get("REGISTRY_PASSWORD") + + if not all([registry_url, registry_username, registry_password]): + print("\n[INFO] Registry credentials not fully provided. Skipping login.") + return "" + + return f" --registry-url {registry_url} --registry-username {registry_username} --registry-password {registry_password}" + def bootstrap_initial_ceph_cluster(monitor_ip_address: str): print(f"\n=== Phase 3: Bootstrapping Cluster on {monitor_ip_address} ===") - custom_image = os.environ.get("CEPH_IMAGE") + custom_image = os.environ.get("CEPHADM_IMAGE") + shared_repo_toggle = os.environ.get("SHARED_CEPH_REPO_DIR", "0") + + shared_repo_flag = "--shared_ceph_folder /ceph " if shared_repo_toggle == "1" else "" image_flag = f"--image {custom_image} " if custom_image else "" + bootstrap_command = ( - f"cephadm {image_flag}bootstrap " + f"yes \"yes\" | cephadm {image_flag}bootstrap " f"--mon-ip {monitor_ip_address} " f"--allow-overwrite " f"--skip-mon-network " f"--config initial-ceph.conf " f"--dashboard-password-noupdate " - f"--shared_ceph_folder /ceph " + f"{shared_repo_flag}" f"--initial-dashboard-password admin " f"--allow-fqdn-hostname" - ) + f"{get_registry_credentials()}" + ).strip() execute_shell_command_safely(bootstrap_command, capture_output=False) def initialize_cephalobox(): monitor_ip_address = get_primary_routable_ip_address() + load_ceph_image() prepare_local_cephadm_binary() create_initial_cluster_configuration_file() - load_ceph_image() - bootstrap_initial_ceph_cluster(monitor_ip_address) print("\nCephalabox has done its job!!!!") diff --git a/docker/cephalobox/cephalobox.sh b/docker/cephalobox/cephalobox.sh index 6038245c55..749227e83c 100644 --- a/docker/cephalobox/cephalobox.sh +++ b/docker/cephalobox/cephalobox.sh @@ -3,7 +3,21 @@ set -e echo "=== CephaloBox Orchestrator Wrapper Started ===" -# Ensure standard binaries and your custom build path are fully accessible in PATH +# some services require port mapping in the inner containers +# so this will bypass container engine's read-only lock for /proc/sys +# and allows netavark can config bridge routings +mount --bind /proc/sys/net /proc/sys/net +mount -o remount,rw,bind /proc/sys/net + +# to make sure env vars from host are available inside containers +if [ -f /proc/1/environ ]; then + while IFS= read -r -d '' env_var; do + if [[ "$env_var" == *=* ]]; then + export "$env_var" + fi + done < /proc/1/environ +fi + export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/ceph/build/bin # Execute the main Python automation script @@ -11,4 +25,3 @@ python3 -u /usr/local/bin/cephalobox.py /bin/bash /usr/local/bin/osd.sh echo "=== CephaloBox Orchestrator Finished Successfully ===" - diff --git a/docker/cephalobox/conf/fuse.conf b/docker/cephalobox/conf/fuse.conf new file mode 100644 index 0000000000..0b1afb08b1 --- /dev/null +++ b/docker/cephalobox/conf/fuse.conf @@ -0,0 +1,5 @@ +[storage] +driver = "overlay" + +[storage.options.overlay] +mount_program = "/usr/bin/fuse-overlayfs" diff --git a/docker/cephalobox/osd.sh b/docker/cephalobox/osd.sh index 994799487c..f3948cc0d7 100644 --- a/docker/cephalobox/osd.sh +++ b/docker/cephalobox/osd.sh @@ -11,7 +11,12 @@ mkdir -p "$BASE_DIRECTORY_FOR_OSDS" CONFIG_PATH="$BASE_DIRECTORY_FOR_OSDS/ceph.conf" cephadm shell -- cat /etc/ceph/ceph.conf > "$CONFIG_PATH" -CONTAINER_IMAGE=$(podman images -q quay.ceph.io/ceph-ci/ceph | head -n 1) +if [ -n "$CEPHADM_IMAGE" ] && [ "$CEPHADM_IMAGE" != "None" ]; then + CONTAINER_IMAGE="$CEPHADM_IMAGE" +else + echo "No CEPHADM_IMAGE specified. getting it from local podman images..." + CONTAINER_IMAGE=$(podman images --format "{{.Repository}}:{{.Tag}}" | grep ceph | head -n 1) +fi if [ -z "$CONTAINER_IMAGE" ]; then echo "[FATAL] No pre-cached Ceph image found!" diff --git a/docker/cephalobox/wrappers/podman.sh b/docker/cephalobox/wrappers/podman.sh index 8ca318acf1..454ebf6279 100644 --- a/docker/cephalobox/wrappers/podman.sh +++ b/docker/cephalobox/wrappers/podman.sh @@ -3,21 +3,28 @@ mv /usr/bin/podman /usr/bin/podman.real && \ cat <<'EOF' > /usr/bin/podman #!/bin/bash -args=("$@") +intercepted_execution_arguments=("$@") # Prevent network pulls if the image is already cached locally -if [[ "${args[0]}" == "pull" ]]; then - for arg in "${args[@]}"; do - if [[ "$arg" == *"quay.ceph.io/ceph-ci/ceph"* ]]; then - if /usr/bin/podman.real images -q "$arg" | grep -q .; then - echo "Fake pull successful. Image already cached locally." +if [[ "${intercepted_execution_arguments[0]}" == "pull" ]]; then + + # Iterate through arguments to find the target container image reference + for current_argument_string in "${intercepted_execution_arguments[@]}"; do + + # Ignore the 'pull' command itself and any flags (like --quiet or --tls-verify) + if [[ "$current_argument_string" != -* && "$current_argument_string" != "pull" ]]; then + + # Utilize native Podman validation to check the local registry cache + if /usr/bin/podman.real image exists "$current_argument_string"; then + echo "[Podman Intercept]: Target container image reference '$current_argument_string' is already cached locally. Bypassing network operation." exit 0 fi + fi done fi -exec /usr/bin/podman.real "${args[@]}" +exec /usr/bin/podman.real "${intercepted_execution_arguments[@]}" EOF chmod +x /usr/bin/podman diff --git a/docker/cephalobox/wrappers/sysctl.sh b/docker/cephalobox/wrappers/sysctl.sh new file mode 100644 index 0000000000..3b8e55f3bd --- /dev/null +++ b/docker/cephalobox/wrappers/sysctl.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +if [ ! -f /usr/sbin/sysctl.real ]; then + mv /usr/sbin/sysctl /usr/sbin/sysctl.real +fi + +cat <<'EOF' > /usr/sbin/sysctl +#!/bin/bash + +# A. Intercept write/apply commands and fake a success instantly +if [[ "$*" == *"--system"* ]] || [[ "$*" == *"-w"* ]] || [[ "$*" == *"-p"* ]]; then + exit 0 +fi + +# B. Pass all read commands (like -a) to the real binary. +# We pipe stderr to /dev/null just in case reading a restricted key throws a warning. +exec /usr/sbin/sysctl.real "$@" 2>/dev/null +EOF + +chmod +x /usr/sbin/sysctl