diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..8e76fe5 --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ + + +[flake8] +exclude=src/txacme/_version.py,src/txacme/interfaces.py +ignore_names=setUp,_setUp,tearDown,startService,stopService diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3ecf82f..abe1aa9 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -8,48 +8,55 @@ on: pull_request: branches: [ master ] +permissions: + contents: read + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/trunk' }} defaults: run: shell: bash +env: + # The pebble tests needs to be over localhost as for testing we will use a + # name that resolved to 127.0.0.1. + PEBBLE_URL: 'https://localhost:14000/dir' jobs: testing: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 name: ${{ matrix.python-version }}-linux strategy: fail-fast: false matrix: - python-version: ["2.7", "3.10", "pypy-3.7"] + python-version: ["3.9", "3.13"] env: # As of April 2021 GHA VM have 2 CPUs - Azure Standard_DS2_v2 # Trial distributed jobs enabled to speed up the CI jobs. TRIAL_ARGS: "-j 4" + # Run pebble as a service container to help with end to end testing. + services: + # Label used to access the service container + pebble: + # Docker Hub image + image: ghcr.io/letsencrypt/pebble:latest + ports: + # Public API. There is also 15000 admin api but we don't need it. + - 14000:14000 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + with: + # We use full dept for branch diff coverage. + fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: - ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml', 'setup.py', - 'setup.cfg') }} - restore-keys: | - ${{ runner.os }}-pip- - - uses: twisted/python-info-action@v1 - name: Install dependencies run: | @@ -71,7 +78,12 @@ jobs: python -m coverage report --skip-covered ls -al - - uses: codecov/codecov-action@v2 + # Check branch coverage. + diff-cover --markdown-report coverage-report.md --compare-branch origin/master coverage.xml + cat coverage-report.md >> $GITHUB_STEP_SUMMARY + + - name: Publish to codecov.io + uses: codecov/codecov-action@v4 if: ${{ !cancelled() }} with: files: coverage.xml diff --git a/.gitignore b/.gitignore index d534da9..ecc0324 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.egg-info/ *.pyc .coverage +coverage.xml .hypothesis/ .testrepository/ .tox/ diff --git a/pyproject.toml b/pyproject.toml index 0ff5314..788bc65 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,12 @@ +[tool.isort] + +default_section = 'THIRDPARTY' +known_first_party = 'txacme' +multi_line_output = 4 +lines_after_imports = 2 +balanced_wrapping = true +order_by_type = false + [tool.towncrier] package = 'txacme' package_dir = 'src/' diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index b1a1775..0000000 --- a/setup.cfg +++ /dev/null @@ -1,14 +0,0 @@ -[wheel] -universal = 1 - -[isort] -default_section=THIRDPARTY -known_first_party=txacme -multi_line_output=4 -lines_after_imports=2 -balanced_wrapping=True -order_by_type=False - -[flake8] -exclude=src/txacme/_version.py,src/txacme/interfaces.py -ignore_names=setUp,_setUp,tearDown,startService,stopService diff --git a/setup.py b/setup.py index e01c598..d0a293c 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,7 @@ def read(*parts): packages=find_packages(where='src'), package_dir={'': 'src'}, zip_safe=True, + python_requires='>=3.9.2', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', @@ -31,35 +32,28 @@ def read(*parts): 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Software Development :: Libraries :: Python Modules', ], install_requires=[ - 'acme>=1.0.0', + 'acme>=4.0.0', 'attrs>=17.4.0', 'eliot>=0.8.0', - 'josepy', + 'josepy>=2', 'pem>=16.1.0', 'treq>=15.1.0', 'twisted[tls]>=16.2.0', 'txsni', - 'pyopenssl>=17.1.0', ], extras_require={ - 'libcloud': [ - 'apache-libcloud', - ], 'dev': [ 'coverage', + 'diff-cover', 'eliot-tree', + 'build', + 'pyOpenSSL', ], }, ) diff --git a/src/integration/__init__.py b/src/integration/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/integration/test_client.py b/src/integration/test_client.py deleted file mode 100644 index 850e0de..0000000 --- a/src/integration/test_client.py +++ /dev/null @@ -1,214 +0,0 @@ -""" -Integration tests for :mod:`acme.client`. -""" -from __future__ import print_function - -from functools import partial -from os import getenv - -from josepy.jwk import JWKRSA -from acme.messages import NewRegistration, STATUS_PENDING -from cryptography.hazmat.primitives import serialization -from eliot import start_action -from eliot.twisted import DeferredContext -from twisted.internet import reactor -from twisted.internet.defer import succeed -from twisted.internet.endpoints import serverFromString -from twisted.python.filepath import FilePath -from twisted.trial.unittest import TestCase -from twisted.web.resource import Resource -from twisted.web.server import Site -from txsni.snimap import SNIMap -from txsni.tlsendpoint import TLSEndpoint - -from txacme.client import ( - answer_challenge, Client, fqdn_identifier, poll_until_valid) -from txacme.messages import CertificateRequest -from txacme.testing import FakeClient, NullResponder -from txacme.urls import LETSENCRYPT_STAGING_DIRECTORY -from txacme.util import csr_for_names, generate_private_key, tap - - -try: - from txacme.challenges import LibcloudDNSResponder -except ImportError: - pass - - -class ClientTestsMixin(object): - """ - Integration tests for the ACME client. - """ - def _test_create_client(self): - with start_action(action_type=u'integration:create_client').context(): - self.key = JWKRSA(key=generate_private_key('rsa')) - return ( - DeferredContext(self._create_client(self.key)) - .addActionFinish()) - - def _test_register(self, new_reg=None): - with start_action(action_type=u'integration:register').context(): - return ( - DeferredContext(self.client.register(new_reg)) - .addActionFinish()) - - def _test_agree_to_tos(self, reg): - with start_action(action_type=u'integration:agree_to_tos').context(): - return ( - DeferredContext(self.client.agree_to_tos(reg)) - .addActionFinish()) - - def _test_request_challenges(self, host): - action = start_action( - action_type=u'integration:request_challenges', - host=host) - with action.context(): - return ( - DeferredContext( - self.client.request_challenges(fqdn_identifier(host))) - .addActionFinish()) - - def _test_poll_pending(self, auth): - action = start_action(action_type=u'integration:poll_pending') - with action.context(): - return ( - DeferredContext(self.client.poll(auth)) - .addCallback( - lambda auth: - self.assertEqual(auth[0].body.status, STATUS_PENDING)) - .addActionFinish()) - - def _test_answer_challenge(self, responder): - action = start_action(action_type=u'integration:answer_challenge') - with action.context(): - self.responder = responder - return ( - DeferredContext( - answer_challenge( - self.authzr, self.client, [responder])) - .addActionFinish()) - - def _test_poll(self, auth): - action = start_action(action_type=u'integration:poll') - with action.context(): - return ( - DeferredContext(poll_until_valid(auth, reactor, self.client)) - .addActionFinish()) - - def _test_issue(self, name): - def got_cert(certr): - key_bytes = self.issued_key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption()) - FilePath('issued.crt').setContent(certr.body) - FilePath('issued.key').setContent(key_bytes) - return certr - - action = start_action(action_type=u'integration:issue') - with action.context(): - self.issued_key = generate_private_key('rsa') - csr = csr_for_names([name], self.issued_key) - return ( - DeferredContext( - self.client.request_issuance(CertificateRequest(csr=csr))) - .addCallback(got_cert) - .addActionFinish()) - - def _test_chain(self, certr): - action = start_action(action_type=u'integration:chain') - with action.context(): - return ( - DeferredContext(self.client.fetch_chain(certr)) - .addActionFinish()) - - def _test_registration(self): - return ( - DeferredContext(self._test_create_client()) - .addCallback(partial(setattr, self, 'client')) - .addCallback(lambda _: self._test_register()) - .addCallback(tap( - lambda reg1: self.assertEqual(reg1.body.contact, ()))) - .addCallback(tap( - lambda reg1: - self._test_register( - NewRegistration.from_data(email=u'example@example.com')) - .addCallback(tap( - lambda reg2: self.assertEqual(reg1.uri, reg2.uri))) - .addCallback(lambda reg2: self.assertEqual( - reg2.body.contact, (u'mailto:example@example.com',))))) - .addCallback(self._test_agree_to_tos) - .addCallback( - lambda _: self._test_request_challenges(self.HOST)) - .addCallback(partial(setattr, self, 'authzr')) - .addCallback(lambda _: self._create_responder()) - .addCallback(tap(lambda _: self._test_poll_pending(self.authzr))) - .addCallback(self._test_answer_challenge) - .addCallback(tap(lambda _: self._test_poll(self.authzr))) - .addCallback(lambda stop_responding: stop_responding()) - .addCallback(lambda _: self._test_issue(self.HOST)) - .addCallback(self._test_chain) - .addActionFinish()) - - def test_issuing(self): - action = start_action(action_type=u'integration') - with action.context(): - return self._test_registration() - - -def _getenv(name, default=None): - """ - Sigh. - """ - value = getenv(name) - if value is None: - return default - return value - - -class LetsEncryptStagingLibcloudTests(ClientTestsMixin, TestCase): - """ - Tests using the real ACME client against the Let's Encrypt staging - environment, and the dns-01 challenge. - - You must set $ACME_HOST to a hostname that will be used for the challenge, - and $LIBCLOUD_PROVIDER, $LIBCLOUD_USERNAME, $LIBCLOUD_PASSWORD, and - $LIBCLOUD_ZONE to the appropriate values for the DNS provider to complete - the challenge with. - """ - HOST = _getenv(u'ACME_HOST') - PROVIDER = _getenv(u'LIBCLOUD_PROVIDER') - USERNAME = _getenv(u'LIBCLOUD_USERNAME') - PASSWORD = _getenv(u'LIBCLOUD_PASSWORD') - ZONE = _getenv(u'LIBCLOUD_ZONE') - - if None in (HOST, PROVIDER, USERNAME, PASSWORD): - skip = 'Must provide $ACME_HOST and $LIBCLOUD_*' - - def _create_client(self, key): - return Client.from_url(reactor, LETSENCRYPT_STAGING_DIRECTORY, key=key) - - def _create_responder(self): - with start_action(action_type=u'integration:create_responder'): - return LibcloudDNSResponder.create( - reactor, - self.PROVIDER, - self.USERNAME, - self.PASSWORD, - self.ZONE) - - -class FakeClientTests(ClientTestsMixin, TestCase): - """ - Tests against our verified fake. - """ - HOST = u'example.com' - - def _create_client(self, key): - return succeed(FakeClient(key, reactor)) - - def _create_responder(self): - return succeed(NullResponder(u'http-01')) - - -__all__ = ['FakeClientTests'] diff --git a/src/txacme/challenges/__init__.py b/src/txacme/challenges/__init__.py index c8d3091..2078d97 100644 --- a/src/txacme/challenges/__init__.py +++ b/src/txacme/challenges/__init__.py @@ -1,11 +1,3 @@ from ._http import HTTP01Responder - -try: - from ._libcloud import LibcloudDNSResponder -except ImportError: - # libcloud may not be installed - pass - - -__all__ = ['HTTP01Responder', 'LibcloudDNSResponder'] +__all__ = ['HTTP01Responder'] diff --git a/src/txacme/challenges/_libcloud.py b/src/txacme/challenges/_libcloud.py deleted file mode 100644 index 1299663..0000000 --- a/src/txacme/challenges/_libcloud.py +++ /dev/null @@ -1,186 +0,0 @@ -import hashlib -import time -from threading import Thread - -import attr -from josepy.b64 import b64encode -from libcloud.dns.providers import get_driver -from twisted._threads import pool -from twisted.internet.defer import Deferred -from twisted.python.failure import Failure -from zope.interface import implementer - -from txacme.errors import NotInZone, ZoneNotFound -from txacme.interfaces import IResponder -from txacme.util import const - - -def _daemon_thread(*a, **kw): - """ - Create a `threading.Thread`, but always set ``daemon``. - """ - thread = Thread(*a, **kw) - thread.daemon = True - return thread - - -def _defer_to_worker(deliver, worker, work, *args, **kwargs): - """ - Run a task in a worker, delivering the result as a ``Deferred`` in the - reactor thread. - """ - deferred = Deferred() - - def wrapped_work(): - try: - result = work(*args, **kwargs) - except BaseException: - f = Failure() - deliver(lambda: deferred.errback(f)) - else: - deliver(lambda: deferred.callback(result)) - worker.do(wrapped_work) - return deferred - - -def _split_zone(server_name, zone_name): - """ - Split the zone portion off from a DNS label. - - :param str server_name: The full DNS label. - :param str zone_name: The zone name suffix. - """ - server_name = server_name.rstrip(u'.') - zone_name = zone_name.rstrip(u'.') - if not (server_name == zone_name or - server_name.endswith(u'.' + zone_name)): - raise NotInZone(server_name=server_name, zone_name=zone_name) - return server_name[:-len(zone_name)].rstrip(u'.') - - -def _get_existing(driver, zone_name, server_name, validation): - """ - Get existing validation records. - """ - if zone_name is None: - zones = sorted( - (z for z - in driver.list_zones() - if server_name.rstrip(u'.') - .endswith(u'.' + z.domain.rstrip(u'.'))), - key=lambda z: len(z.domain), - reverse=True) - if len(zones) == 0: - raise NotInZone(server_name=server_name, zone_name=None) - else: - zones = [ - z for z - in driver.list_zones() - if z.domain == zone_name] - if len(zones) == 0: - raise ZoneNotFound(zone_name=zone_name) - zone = zones[0] - subdomain = _split_zone(server_name, zone.domain) - existing = [ - record for record - in zone.list_records() - if record.name == subdomain and - record.type == 'TXT' and - record.data == validation] - return zone, existing, subdomain - - -def _validation(response): - """ - Get the validation value for a challenge response. - """ - h = hashlib.sha256(response.key_authorization.encode("utf-8")) - return b64encode(h.digest()).decode() - - -@attr.s(hash=False) -@implementer(IResponder) -class LibcloudDNSResponder(object): - """ - A ``dns-01`` challenge responder using libcloud. - - .. warning:: Some libcloud backends are broken with regard to TXT records - at the time of writing; the Route 53 backend, for example. This makes - them unusable with this responder. - - .. note:: This implementation relies on invoking libcloud in a thread, so - may not be entirely production quality. - """ - challenge_type = u'dns-01' - - _reactor = attr.ib() - _thread_pool = attr.ib() - _driver = attr.ib() - zone_name = attr.ib() - settle_delay = attr.ib() - - @classmethod - def create(cls, reactor, driver_name, username, password, zone_name=None, - settle_delay=60.0): - """ - Create a responder. - - :param reactor: The Twisted reactor to use for threading support. - :param str driver_name: The name of the libcloud DNS driver to use. - :param str username: The username to authenticate with (the meaning of - this is driver-specific). - :param str password: The username to authenticate with (the meaning of - this is driver-specific). - :param str zone_name: The zone name to respond in, or ``None`` to - automatically detect zones. Usually auto-detection should be fine, - unless restricting responses to a single specific zone is desired. - :param float settle_delay: The time, in seconds, to allow for the DNS - provider to propagate record changes. - """ - return cls( - reactor=reactor, - thread_pool=pool(const(1), threadFactory=_daemon_thread), - driver=get_driver(driver_name)(username, password), - zone_name=zone_name, - settle_delay=settle_delay) - - def _defer(self, f): - """ - Run a function in our private thread pool. - """ - return _defer_to_worker( - self._reactor.callFromThread, self._thread_pool, f) - - def start_responding(self, server_name, challenge, response): - """ - Install a TXT challenge response record. - """ - validation = _validation(response) - full_name = challenge.validation_domain_name(server_name) - _driver = self._driver - - def _go(): - zone, existing, subdomain = _get_existing( - _driver, self.zone_name, full_name, validation) - if len(existing) == 0: - zone.create_record(name=subdomain, type='TXT', data=validation) - time.sleep(self.settle_delay) - return self._defer(_go) - - def stop_responding(self, server_name, challenge, response): - """ - Remove a TXT challenge response record. - """ - validation = _validation(response) - full_name = challenge.validation_domain_name(server_name) - _driver = self._driver - - def _go(): - zone, existing, subdomain = _get_existing( - _driver, self.zone_name, full_name, validation) - for record in existing: - record.delete() - return self._defer(_go) - - -__all__ = ['LibcloudDNSResponder'] diff --git a/src/txacme/client.py b/src/txacme/client.py index b9001ae..f31d34b 100644 --- a/src/txacme/client.py +++ b/src/txacme/client.py @@ -80,6 +80,7 @@ from acme.crypto_util import make_csr from acme.jws import JWS, Header from acme.messages import ( + CertificateRequest, STATUS_PENDING, STATUS_VALID, STATUS_INVALID, @@ -89,7 +90,7 @@ from josepy.jwa import RS256 from josepy.errors import DeserializationError -import OpenSSL +from cryptography import x509 from cryptography.hazmat.primitives import serialization from eliot.twisted import DeferredContext @@ -104,8 +105,6 @@ from txacme import __version__ from txacme.logging import ( LOG_ACME_ANSWER_CHALLENGE, - LOG_ACME_CONSUME_DIRECTORY, - LOG_ACME_REGISTER, LOG_HTTP_PARSE_LINKS, LOG_JWS_ADD_NONCE, LOG_JWS_CHECK_RESPONSE, @@ -157,22 +156,18 @@ def _parse_header_links(response): return links -def _default_client(jws_client, reactor, key, alg, directory, timeout): +def _default_jws_client(jws_client, reactor, key, alg, timeout): """ Make a client if we didn't get one. """ if jws_client is None: pool = HTTPConnectionPool(reactor) agent = Agent(reactor, pool=pool) - jws_d = JWSClient.from_directory(agent, key, alg, directory) - else: - jws_d = defer.succeed(jws_client) + jws_client = JWSClient(agent, key, alg) - def set_timeout(jws_client): - jws_client.timeout = timeout - return jws_client + jws_client.timeout = timeout + return jws_client - return jws_d.addCallback(set_timeout) def fqdn_identifier(fqdn): @@ -190,21 +185,6 @@ def fqdn_identifier(fqdn): typ=messages.IDENTIFIER_FQDN, value=fqdn) -@messages.Directory.register -class Finalize(jose.JSONObjectWithFields): - """ - ACME order finalize request. - - This is here as acme.messages.CertificateRequest does not work with - pebble in --strict mode. - - :ivar josepy.util.ComparableX509 csr: - `OpenSSL.crypto.X509Req` wrapped in `.ComparableX509` - """ - resource_type = 'finalize' - csr = jose.Field('csr', decoder=jose.decode_csr, encoder=jose.encode_csr) - - class Client(object): """ ACME client interface. @@ -246,21 +226,15 @@ def from_url( :return: The constructed client. :rtype: Deferred[`Client`] """ - action = LOG_ACME_CONSUME_DIRECTORY( - url=url, key_type=key.typ, alg=alg.name) - with action.context(): + @defer.inlineCallbacks + def setup_client(): check_directory_url_type(url) - directory = url.asText() - return ( - DeferredContext(jws_client=_default_client( - jws_client, reactor, key, alg, directory, timeout - )) - .addCallback( - tap(lambda jws_client: - action.add_success_fields(directory=directory))) - .addCallback(lambda jws_client: cls(reactor, key, jws_client)) - .addActionFinish() - ) + client = _default_jws_client( + jws_client, reactor, key, alg, timeout) + directory = yield client.start(url.asText()) + return cls(directory, reactor, key, client) + + return setup_client() def stop(self): """ @@ -272,33 +246,35 @@ def stop(self): """ return self._client.stop() - def register(self, email=None): + @defer.inlineCallbacks + def start(self): """ - Create a new registration with the ACME server or update - an existing account. - - It should be called before doing any ACME requests. + Prepare the client for communicating with the ACME server. - :param str: Comma separated contact emails used by the account. + If there is no account for the key, it creates a new registration with + the ACME server. :return: The registration resource. :rtype: Deferred[`~acme.messages.RegistrationResource`] """ uri = self.directory.newAccount new_reg = messages.Registration.from_data( - email=email, terms_of_service_agreed=True, ) - action = LOG_ACME_REGISTER(registration=new_reg) - with action.context(): - return ( - DeferredContext( - self._client.post(uri, new_reg)) - .addCallback(self._cb_check_existing_account, new_reg) - .addCallback(self._cb_check_registration) - .addCallback( - tap(lambda r: action.add_success_fields(registration=r))) - .addActionFinish()) + response = yield self._client.post(uri, new_reg) + + registration = yield self._parse_registration_response(response) + + if registration.body.key != self.key.public_key(): + # This is a response for another key. + raise errors.UnexpectedUpdate(registration) + + if registration.body.status != 'valid': + raise errors.UnexpectedUpdate(registration) + + self._client.kid = registration.uri + + return registration def stop(self): """ @@ -321,22 +297,7 @@ def _maybe_location(cls, response, uri=None): return location.decode('ascii') return uri - def _cb_check_existing_account(self, response, request): - """ - Get the response from the account registration and see if the - account is already registered and do an update in that case. - """ - if response.code == 200 and request.contact: - # Account already exists and we email address to update. - # I don't know how to remove a contact. - uri = self._maybe_location(response) - deferred = self._client.post(uri, request, kid=uri) - deferred.addCallback(self._cb_parse_registration_response, uri=uri) - return deferred - - return self._cb_parse_registration_response(response) - - def _cb_parse_registration_response(self, response, uri=None): + def _parse_registration_response(self, response, uri=None): """ Parse a new or update registration response from the server. """ @@ -354,22 +315,6 @@ def _cb_parse_registration_response(self, response, uri=None): terms_of_service=terms_of_service)) ) - def _cb_check_registration(self, regr): - """ - Check that a registration response contains the registration we were - expecting. - """ - if regr.body.key != self.key.public_key(): - # This is a response for another key. - raise errors.UnexpectedUpdate(regr) - - if regr.body.status != 'valid': - raise errors.UnexpectedUpdate(regr) - - self._client.kid = regr.uri - - return regr - @defer.inlineCallbacks def submit_order(self, key, names): """ @@ -544,10 +489,8 @@ def finalize(self, order): :rtype: Deferred[`acme.messages.OrderResource`] :return: The issued certificate. """ - csr = OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_PEM, order.csr_pem - ) - request = Finalize(csr=jose.ComparableX509(csr)) + csr = x509.load_pem_x509_csr(order.csr_pem) + request = CertificateRequest(csr=csr) response = yield self._client.post( order.body.finalize, obj=request ) @@ -824,7 +767,7 @@ class JWSClient(object): """ timeout = _DEFAULT_TIMEOUT - def __init__(self, agent, key, alg, new_nonce_url, kid, + def __init__(self, agent, key, alg, user_agent=u'txacme/{}'.format(__version__).encode('ascii')): self._treq = HTTPClient(agent=agent) self._agent = agent @@ -834,37 +777,51 @@ def __init__(self, agent, key, alg, new_nonce_url, kid, self._user_agent = user_agent self._nonces = set() - self._new_nonce = new_nonce_url - self._kid = kid + # URL from where a new nonce can be obtained. + # This is set at start time. + self._new_nonce = None + self._kid = None - @classmethod - def from_directory(cls, agent, key, alg, directory): - """ - Prepare for ACME operations based on 'directory' url. + @property + def kid(self): + return self._kid - :param str directory: The URL to the ACME v2 directory. + @kid.setter + def kid(self, value): + self._kid = value - :return: When operation is done. - :rtype: Deferred[None] + def _cb_wrap_in_jws(self, nonce, obj, url, kid=None): """ - # Provide invalid new_nonce_url & kid, but don't expose it to the - # caller. - self = cls(agent, key, alg, None, None) + Callback to wrap ``JSONDeSerializable`` object in ACME JWS. - def cb_extract_new_nonce(directory): - try: - self._new_nonce = directory.newNonce - except AttributeError: - raise errors.ClientError( - 'Directory has no newNonce URL', directory) + :param ~josepy.interfaces.JSONDeSerializable obj: + :param bytes nonce: + :param bytes url: URL to the request for which we wrap the payload. - return directory - return ( - self.get(directory) - .addCallback(json_content) - .addCallback(messages.Directory.from_json) - .addCallback(cb_extract_new_nonce) - ) + :rtype: `bytes` + :return: JSON-encoded data + """ + if kid is None: + kid = self._kid + + with LOG_JWS_SIGN(key_type=self._key.typ, alg=self._alg.name, + nonce=nonce, kid=kid): + if obj is None: + jobj = b'' + else: + jobj = obj.json_dumps().encode() + result = ( + JWS.sign( + payload=jobj, + key=self._key, + alg=self._alg, + nonce=nonce, + url=url, + kid=kid, + ) + .json_dumps() + .encode()) + return result @classmethod def _check_response(cls, response, content_type=JSON_CONTENT_TYPE): @@ -959,6 +916,30 @@ def cb_request_done(result): b'content-type', [None])[0]))) .addActionFinish()) + def start(self, directory): + """ + Prepare for ACME operations based on 'directory' url. + + :param str directory: The URL to the ACME v2 directory. + + :return: When operation is done. + :rtype: Deferred[None] + """ + def cb_extract_new_nonce(directory): + try: + self._new_nonce = directory.newNonce + except AttributeError: + raise errors.ClientError( + 'Directory has no newNonce URL', directory) + + return directory + return ( + self.get(directory) + .addCallback(json_content) + .addCallback(messages.Directory.from_json) + .addCallback(cb_extract_new_nonce) + ) + def stop(self): """ Stops the operation. @@ -1078,32 +1059,12 @@ def _post( if kid is None: kid = self._kid - def cb_wrap_in_jws(nonce): - with LOG_JWS_SIGN(key_type=self._key.typ, alg=self._alg.name, - nonce=nonce): - if obj is None: - jobj = b'' - else: - jobj = obj.json_dumps().encode() - result = ( - JWS.sign( - payload=jobj, - key=self._key, - alg=self._alg, - nonce=nonce, - url=url, - kid=kid, - ) - .json_dumps() - .encode()) - return result - with LOG_JWS_POST().context(): headers = kwargs.setdefault('headers', Headers()) headers.setRawHeaders(b'content-type', [JOSE_CONTENT_TYPE]) return ( DeferredContext(self._get_nonce(url)) - .addCallback(cb_wrap_in_jws) + .addCallback(self._cb_wrap_in_jws, obj, url, kid) .addCallback( lambda data: self._send_request( u'POST', url, data=data, **kwargs)) diff --git a/src/txacme/messages.py b/src/txacme/messages.py deleted file mode 100644 index dc676f6..0000000 --- a/src/txacme/messages.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -ACME protocol messages. - -This module provides supplementary message implementations that are not already -provided by the `acme` library. - -.. seealso:: `acme.messages` -""" -from acme.fields import Resource -from josepy import Field, JSONObjectWithFields - -from txacme.util import decode_csr, encode_csr - - -class CertificateRequest(JSONObjectWithFields): - """ - ACME new-cert request. - - Differs from the upstream version because it wraps a Cryptography CSR - object instead of a PyOpenSSL one. - - .. seealso:: `acme.messages.CertificateRequest`, - `cryptography.x509.CertificateSigningRequest` - """ - resource_type = 'new-cert' - resource = Resource(resource_type) - csr = Field('csr', decoder=decode_csr, encoder=encode_csr) - - -__all__ = ['CertificateRequest'] diff --git a/src/txacme/newsfragments/163.feature b/src/txacme/newsfragments/163.feature new file mode 100644 index 0000000..b60b964 --- /dev/null +++ b/src/txacme/newsfragments/163.feature @@ -0,0 +1,2 @@ +Code was updated to work with recent version of `acme` and `josepy`. +The `OpenSSL` dependency was removed. diff --git a/src/txacme/service.py b/src/txacme/service.py index cf36191..7a7f319 100644 --- a/src/txacme/service.py +++ b/src/txacme/service.py @@ -3,7 +3,6 @@ import attr from cryptography import x509 -from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization import pem from twisted.application.internet import TimerService @@ -107,7 +106,7 @@ def check(certs): for o in filter( lambda o: isinstance(o, pem.Certificate), objects): cert = x509.load_pem_x509_certificate( - o.as_bytes(), default_backend()) + o.as_bytes()) until_expiry = cert.not_valid_after - self._now() if until_expiry <= self.panic_interval: panicing.add(server_names) diff --git a/src/txacme/test/test_challenges.py b/src/txacme/test/test_challenges.py index 6ebfaed..0268996 100644 --- a/src/txacme/test/test_challenges.py +++ b/src/txacme/test/test_challenges.py @@ -1,12 +1,8 @@ """ Tests for `txacme.challenges`. """ -from operator import methodcaller - from acme import challenges -from josepy.b64 import b64encode from treq.testing import StubTreq -from twisted._threads import createMemoryWorker from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -15,9 +11,8 @@ from zope.interface.verify import verifyObject from txacme.challenges import HTTP01Responder -from txacme.errors import NotInZone, ZoneNotFound from txacme.interfaces import IResponder -from txacme.test.test_client import RSA_KEY_512, RSA_KEY_512_RAW +from txacme.test.test_client import RSA_TEST_KEY # A random example token for the challenge tests that need one @@ -45,7 +40,7 @@ def test_stop_responding_already_stopped(self): """ token = EXAMPLE_TOKEN challenge = challenges.HTTP01(token=token) - response = challenge.response(RSA_KEY_512) + response = challenge.response(RSA_TEST_KEY) responder = HTTP01Responder() yield responder.stop_responding( @@ -60,7 +55,7 @@ def test_start_responding(self): """ token = b'BWYcfxzmOha7-7LoxziqPZIUr99BCz3BfbN9kzSFnrU' challenge = challenges.HTTP01(token=token) - response = challenge.response(RSA_KEY_512) + response = challenge.response(RSA_TEST_KEY) responder = HTTP01Responder() diff --git a/src/txacme/test/test_client.py b/src/txacme/test/test_client.py index cc8f3d8..8fd677e 100644 --- a/src/txacme/test/test_client.py +++ b/src/txacme/test/test_client.py @@ -1,86 +1,59 @@ +import os import json +import unittest from contextlib import contextmanager from operator import attrgetter, methodcaller import attr -from josepy.jwa import RS256, RS384 +from josepy.jwa import RS256 from josepy.jwk import JWKRSA from josepy.jws import JWS -from josepy.b64 import b64encode, b64decode +from josepy.b64 import b64decode -from acme import challenges, errors, messages -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import serialization +from acme import errors, messages from cryptography.hazmat.primitives.asymmetric import rsa -from treq.client import HTTPClient from treq.testing import RequestSequence as treq_RequestSequence -from treq.testing import ( - _SynchronousProducer, RequestTraversalAgent, StringStubbingResource) from twisted.internet import defer, reactor -from twisted.internet.defer import Deferred, CancelledError, fail, succeed -from twisted.internet.error import ConnectionClosed -from twisted.internet.task import Clock +from twisted.internet.interfaces import IOpenSSLClientConnectionCreator from twisted.python.url import URL -from twisted.test.proto_helpers import MemoryReactor -from twisted.web import http, server -from twisted.web.resource import Resource +from twisted.web import http +from twisted.web.client import Agent, BrowserLikePolicyForHTTPS from twisted.web.http_headers import Headers from twisted.trial.unittest import TestCase from zope.interface import implementer +from OpenSSL import SSL from txacme.client import ( - _default_client, _find_supported_challenge, _parse_header_links, - answer_challenge, AuthorizationFailed, Client, DER_CONTENT_TYPE, - fqdn_identifier, JSON_CONTENT_TYPE, JOSE_CONTENT_TYPE, - JSON_ERROR_CONTENT_TYPE, JWSClient, NoSupportedChallenges, ServerError, - get_certificate + _parse_header_links, + Client, + fqdn_identifier, + JSON_CONTENT_TYPE, + JSON_ERROR_CONTENT_TYPE, + JWSClient, + ServerError, ) from txacme.interfaces import IResponder -from txacme.messages import CertificateRequest -from txacme.testing import NullResponder -from txacme.util import ( - csr_for_names, generate_private_key -) + + +# URL to the pebble directory. +PEBBLE_URL = os.environ.get('PEBBLE_URL', '') +if PEBBLE_URL: + PEBBLE_URL = URL.from_text(PEBBLE_URL) def failed_with(matcher): return failed(AfterPreprocessing(attrgetter('value'), matcher)) -# from cryptography: - -RSA_KEY_512_RAW = rsa.RSAPrivateNumbers( - p=int( - "d57846898d5c0de249c08467586cb458fa9bc417cdf297f73cfc52281b787cd9", 16 - ), - q=int( - "d10f71229e87e010eb363db6a85fd07df72d985b73c42786191f2ce9134afb2d", 16 - ), - d=int( - "272869352cacf9c866c4e107acc95d4c608ca91460a93d28588d51cfccc07f449" - "18bbe7660f9f16adc2b4ed36ca310ef3d63b79bd447456e3505736a45a6ed21", 16 - ), - dmp1=int( - "addff2ec7564c6b64bc670d250b6f24b0b8db6b2810099813b7e7658cecf5c39", 16 - ), - dmq1=int( - "463ae9c6b77aedcac1397781e50e4afc060d4b216dc2778494ebe42a6850c81", 16 - ), - iqmp=int( - "54deef8548f65cad1d411527a32dcb8e712d3e128e4e0ff118663fae82a758f4", 16 - ), - public_numbers=rsa.RSAPublicNumbers( - e=65537, - n=int( - "ae5411f963c50e3267fafcf76381c8b1e5f7b741fdb2a544bcf48bd607b10c991" - "90caeb8011dc22cf83d921da55ec32bd05cac3ee02ca5e1dbef93952850b525", - 16 - ), - ) -).private_key(default_backend()) - -RSA_KEY_512 = JWKRSA(key=RSA_KEY_512_RAW) +# We generate a new RSA key for each test run. +# This will make sure that we don't already have an account on the +# ACME server. +# Let's Encrypt staging only supports keys of minimum 2048 +RSA_TEST_KEY = JWKRSA(key=rsa.generate_private_key( + public_exponent=65537, + key_size=2048, + )) class RequestSequence(treq_RequestSequence): @@ -145,7 +118,7 @@ class TestResponse(object): code = attr.ib(default=http.OK) content_type = attr.ib(default=JSON_CONTENT_TYPE) nonce = attr.ib(default=None) - json = attr.ib(default=lambda: succeed({})) + json = attr.ib(default=lambda: defer.succeed({})) links = attr.ib(default=None) @property @@ -184,7 +157,7 @@ def test_directory_url_type(self): """ with self.assertRaises(TypeError): yield Client.from_url( - reactor, '/wrong/kind/of/directory', key=RSA_KEY_512) + reactor, '/wrong/kind/of/directory', key=RSA_TEST_KEY) def test_fqdn_identifier(self): """ @@ -239,7 +212,7 @@ def test_check_valid_error(self): response = TestResponse( code=http.FORBIDDEN, content_type=JSON_ERROR_CONTENT_TYPE, - json=lambda: succeed({ + json=lambda: defer.succeed({ u'type': u'unauthorized', u'detail': u'blah blah blah'})) @@ -273,4 +246,108 @@ def test_rfc_example1(self): result) -__all__ = ['ClientTests', 'ExtraCoverageTests', 'LinkParsingTests'] +@unittest.skipIf(not PEBBLE_URL, 'Pebble tests enabled') +class PebbleTests(TestCase): + """ + :class:`.Client` end to end test using Pebble over localhost. + """ + + @defer.inlineCallbacks + def test_directory_lets_encrypt_staging(self): + """ + Can start the client with the public Let's Encrypt staging URL. + """ + client = yield Client.from_url( + reactor, + URL.from_text('https://acme-staging-v02.api.letsencrypt.org/directory'), + key=RSA_TEST_KEY, + ) + registration = yield client.start() + + self.assertIn( + 'https://acme-staging-v02.api.letsencrypt.org/acme/acct/', + registration.uri) + + # Close any cached connection. + yield client.stop() + + @defer.inlineCallbacks + def test_directory_pebble_testing(self): + """ + Can start the client with the public Let's Encrypt staging URL. + """ + agent = Agent(reactor, contextFactory=UnsafePolicyForHTTPS()) + jws_client = JWSClient(agent, key=RSA_TEST_KEY, alg=RS256) + client = yield Client.from_url( + reactor, + PEBBLE_URL, + key=RSA_TEST_KEY, + jws_client=jws_client, + ) + # This will register the new account. + registration = yield client.start() + + # Minimal checks for the new account. + account_uri = registration.uri + self.assertIn('/my-account/', registration.uri) + + # Stop can be triggered multiple times. + yield client.stop() + yield client.stop() + + agent = Agent(reactor, contextFactory=UnsafePolicyForHTTPS()) + jws_client = JWSClient(agent, key=RSA_TEST_KEY, alg=RS256) + client = yield Client.from_url( + reactor, + PEBBLE_URL, + key=RSA_TEST_KEY, + jws_client=jws_client, + ) + + registration = yield client.start() + self.assertEqual(account_uri, registration.uri) + + # Trigger the closing of TCP connections. + yield client.stop() + + +class UnsafePolicyForHTTPS(BrowserLikePolicyForHTTPS): + """ + Policy to help with testing. + Doesn't validated the server certificate. + + This is to be used with the pebble server. + """ + def __init__(self): + self._ssl_context = SSL.Context(SSL.SSLv23_METHOD) + + def creatorForNetloc(self, hostname, port): + """ + Create a L{client connection creator + } for a + given network location. + """ + return UnsafeClientTLSOptions( + hostname=hostname.decode("ascii"), + ctx=self._ssl_context, + ) + + +@implementer(IOpenSSLClientConnectionCreator) +class UnsafeClientTLSOptions: + """ + Client creator for TLS with SNI but without server validation + """ + + def __init__(self, hostname, ctx): + self._hostname = hostname + self._ctx = ctx + + def clientConnectionForTLS(self, tlsProtocol): + """ + Create a TLS connection for a client. + """ + connection = SSL.Connection(self._ctx, None) + server_name = self._hostname.encode('utf-8') + connection.set_tlsext_host_name(server_name) + return connection diff --git a/src/txacme/test/test_util.py b/src/txacme/test/test_util.py index e6d2566..c96ffbb 100644 --- a/src/txacme/test/test_util.py +++ b/src/txacme/test/test_util.py @@ -1,21 +1,7 @@ -from codecs import decode - -import attr -from OpenSSL import crypto -from acme import challenges -from josepy.b64 import b64encode -from josepy.errors import DeserializationError -from cryptography import x509 -from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography.x509.oid import NameOID -from service_identity.pyopenssl import verify_hostname from twisted.trial.unittest import TestCase -from txacme.test.test_client import RSA_KEY_512, RSA_KEY_512_RAW -from txacme.util import ( - const, csr_for_names, decode_csr, encode_csr, - generate_private_key) +from txacme.util import (generate_private_key) class GeneratePrivateKeyTests(TestCase):