Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 26 additions & 2 deletions alts/scheduler/scheduling.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
import logging
import random
import re
import threading
import time
import urllib.parse
import uuid
from typing import List
from typing import List, Optional

import requests

Expand Down Expand Up @@ -70,6 +71,28 @@ def get_task_priority(
return TaskPriority.SMALL_BUILD
return TaskPriority.LARGE_BUILD

@staticmethod
def get_package_queue_suffix(package_name: str) -> Optional[str]:
"""
Returns the queue suffix configured for the package in
`package_queue_suffixes`, or None when no regex matches it.
"""
for pattern, suffix in CONFIG.package_queue_suffixes.items():
if re.fullmatch(pattern, package_name):
return suffix
return None

@staticmethod
def get_queue_suffix(runner_class, payload: TaskRequestPayload) -> str:
"""
Returns the last part of the `<runner>-<arch>-<suffix>` queue name:
the package's configured suffix, or the runner's cost otherwise.
"""
suffix = TestsScheduler.get_package_queue_suffix(payload.package_name)
if suffix:
return suffix
return str(runner_class.get_cost(payload.release_build))

def schedule_test_task(
self,
payload: TaskRequestPayload,
Expand Down Expand Up @@ -149,7 +172,8 @@ def schedule_test_task(
repo_counter += 1
repositories.append({'url': repository.baseurl, 'name': repo_name})

queue_name = f'{runner_type}-{queue_arch}-{runner_class.get_cost(payload.release_build)}'
queue_suffix = self.get_queue_suffix(runner_class, payload)
queue_name = f'{runner_type}-{queue_arch}-{queue_suffix}'
task_id = str(uuid.uuid4())
task_params = payload.model_dump()
task_params['task_id'] = task_id
Expand Down
25 changes: 24 additions & 1 deletion alts/shared/models.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import os
import re
import ssl
from logging import Logger
from typing import (
Expand All @@ -11,7 +12,7 @@
Set,
)

from pydantic import BaseModel, ConfigDict, computed_field
from pydantic import BaseModel, ConfigDict, computed_field, field_validator

from alts.shared import constants
from alts.shared.utils.path_utils import get_abspath
Expand Down Expand Up @@ -460,3 +461,25 @@ class SchedulerConfig(CeleryConfig):
# Builds handed out with at most this many test tasks are queued ahead
# of larger ones, so a few tests are not stuck behind a big backlog.
small_build_max_tasks: int = 10
# Maps a package name regex to the queue suffix its test tasks use in
# place of the runner cost, e.g. `{'acme-.*': 'acme'}` sends them
# to `<runner>-<arch>-acme`, served by dedicated workers. The regex
# must match the whole name; the first matching entry wins.
package_queue_suffixes: Dict[str, str] = {}

@field_validator('package_queue_suffixes')
@classmethod
def validate_package_queue_suffixes(
cls,
value: Dict[str, str],
) -> Dict[str, str]:
for pattern, suffix in value.items():
try:
re.compile(pattern)
except re.error as exc:
raise ValueError(
f'Invalid package regex {pattern!r}: {exc}'
) from exc
if not suffix:
raise ValueError(f'Empty queue suffix for {pattern!r}')
return value
6 changes: 6 additions & 0 deletions alts/worker/runners/docker.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,12 @@ class DockerRunner(BaseRunner):
TYPE = 'docker'
TF_MAIN_FILE = 'docker.tf'
TEMPFILE_PREFIX = 'docker_test_runner_'
COST_STANDARD = 0
COST_RELEASE_BUILD = 1

@classmethod
def get_cost(cls, release_build: bool = False) -> int:
return cls.COST_RELEASE_BUILD if release_build else cls.COST_STANDARD

def __init__(
self,
Expand Down
106 changes: 100 additions & 6 deletions tests/scheduler/test_scheduling.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,20 +15,25 @@
from alts.shared.constants import TaskPriority


def _payload(index: int, release_build: bool = False) -> dict:
def _payload(
index: int,
release_build: bool = False,
package_name: str = 'pkg',
runner_type: str = 'opennebula',
) -> dict:
return {
'bs_task_id': f'task-{index}',
'runner_type': 'opennebula',
'runner_type': runner_type,
'dist_name': 'almalinux',
'dist_version': '9',
'dist_arch': 'x86_64',
'package_name': 'pkg',
'package_name': package_name,
'release_build': release_build,
}


def _run_once(test_tasks: list) -> list:
"""Run one scheduler cycle and return the published priorities."""
def _run_once(test_tasks: list, field: str = 'priority') -> list:
"""Run one scheduler cycle and return the published `field` values."""
terminated = threading.Event()
graceful = threading.Event()

Expand All @@ -44,7 +49,7 @@ def stop(_seconds):
patch.object(scheduling.time, 'sleep', side_effect=stop):
scheduler.run()
return [
call.kwargs['priority']
call.kwargs[field]
for call in run_tests.apply_async.call_args_list
]

Expand Down Expand Up @@ -105,3 +110,92 @@ def test_redis_priority_steps_ignore_configured_max_priority():
})
transport = config.get_celery_config_dict()['broker_transport_options']
assert transport['priority_steps'] == [int(p) for p in TaskPriority]


PACKAGE_QUEUE_SUFFIXES = {
'acme-foo.*': 'foo',
'acme-bar[0-9]+(-.*)?': 'bar',
# Never reached for acme-foo/acme-bar: the first matching entry wins
'acme-.*': 'acme',
}


@pytest.fixture
def package_queue_suffixes(monkeypatch):
monkeypatch.setattr(
CONFIG, 'package_queue_suffixes', PACKAGE_QUEUE_SUFFIXES,
)


@pytest.mark.usefixtures('package_queue_suffixes')
@pytest.mark.parametrize('runner_type', ['docker', 'opennebula'])
@pytest.mark.parametrize('release_build', [False, True])
@pytest.mark.parametrize('package_name, suffix', [
('acme-foo1', 'foo'),
('acme-foo2-extra', 'foo'),
('acme-bar1', 'bar'),
('acme-bar1-extra', 'bar'),
('acme-baz', 'acme'),
])
def test_package_goes_to_its_queue(runner_type, release_build,
package_name, suffix):
task = _payload(0, release_build, package_name, runner_type)
assert _run_once([task], 'queue') == [f'{runner_type}-x86_64-{suffix}']


@pytest.mark.usefixtures('package_queue_suffixes')
@pytest.mark.parametrize('package_name', [
'pkg',
'other-foo1',
'foo-acme',
# The regex must match the whole name, case-sensitively
'my-acme-foo1',
'ACME-FOO1',
])
def test_unmatched_packages_keep_cost_queue(package_name):
task = _payload(0, package_name=package_name)
assert _run_once([task], 'queue') == ['opennebula-x86_64-1']


def test_no_package_queue_suffixes_by_default():
assert CONFIG.model_fields['package_queue_suffixes'].default == {}
task = _payload(0, package_name='acme-foo1')
assert _run_once([task], 'queue') == ['opennebula-x86_64-1']


@pytest.mark.parametrize('suffixes', [
{'acme-foo[': 'foo'},
{'acme-foo.*': ''},
])
def test_invalid_package_queue_suffixes_are_rejected(suffixes):
with pytest.raises(ValueError):
CONFIG.model_validate({
**CONFIG.model_dump(),
'package_queue_suffixes': suffixes,
})


@pytest.mark.parametrize('runner_type, release_build, queue', [
('docker', False, 'docker-x86_64-0'),
('docker', True, 'docker-x86_64-1'),
('opennebula', False, 'opennebula-x86_64-1'),
('opennebula', True, 'opennebula-x86_64-2'),
])
def test_queue_cost_comes_from_runner(runner_type, release_build, queue):
tasks = [
_payload(i, release_build, runner_type=runner_type)
for i in range(CONFIG.small_build_max_tasks + 1)
]
assert _run_once(tasks, 'queue') == [queue] * len(tasks)


def test_package_queue_gets_priority_support():
# Workers do not declare the package queues: Celery creates them on
# publish and must still give them the priority argument
from alts.worker.app import celery_app
assert celery_app.conf.task_create_missing_queues
queue = celery_app.amqp.queues['docker-x86_64-foo']
assert queue.routing_key == 'docker-x86_64-foo'
assert queue.queue_arguments == {
'x-max-priority': CONFIG.task_queue_max_priority,
}
Loading