mirror of https://github.com/watcha-fr/synapse
Erik Johnston
6 years ago
commit
2a8edbaf74
@ -0,0 +1 @@ |
||||
Synapse can now automatically provision TLS certificates via ACME (the protocol used by CAs like Let's Encrypt). |
||||
@ -0,0 +1 @@ |
||||
Move SRV logic into the Agent layer |
||||
@ -0,0 +1 @@ |
||||
Apply a unique index to the user_ips table, preventing duplicates. |
||||
@ -0,0 +1 @@ |
||||
debian package: symlink to explicit python version |
||||
@ -0,0 +1 @@ |
||||
Add a metric for tracking event stream position of the user directory. |
||||
@ -0,0 +1 @@ |
||||
Add infrastructure to support different event formats |
||||
@ -0,0 +1,147 @@ |
||||
# -*- coding: utf-8 -*- |
||||
# Copyright 2019 New Vector Ltd |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
import logging |
||||
|
||||
import attr |
||||
from zope.interface import implementer |
||||
|
||||
from twisted.internet import defer |
||||
from twisted.internet.endpoints import serverFromString |
||||
from twisted.python.filepath import FilePath |
||||
from twisted.python.url import URL |
||||
from twisted.web import server, static |
||||
from twisted.web.resource import Resource |
||||
|
||||
logger = logging.getLogger(__name__) |
||||
|
||||
try: |
||||
from txacme.interfaces import ICertificateStore |
||||
|
||||
@attr.s |
||||
@implementer(ICertificateStore) |
||||
class ErsatzStore(object): |
||||
""" |
||||
A store that only stores in memory. |
||||
""" |
||||
|
||||
certs = attr.ib(default=attr.Factory(dict)) |
||||
|
||||
def store(self, server_name, pem_objects): |
||||
self.certs[server_name] = [o.as_bytes() for o in pem_objects] |
||||
return defer.succeed(None) |
||||
|
||||
|
||||
except ImportError: |
||||
# txacme is missing |
||||
pass |
||||
|
||||
|
||||
class AcmeHandler(object): |
||||
def __init__(self, hs): |
||||
self.hs = hs |
||||
self.reactor = hs.get_reactor() |
||||
|
||||
@defer.inlineCallbacks |
||||
def start_listening(self): |
||||
|
||||
# Configure logging for txacme, if you need to debug |
||||
# from eliot import add_destinations |
||||
# from eliot.twisted import TwistedDestination |
||||
# |
||||
# add_destinations(TwistedDestination()) |
||||
|
||||
from txacme.challenges import HTTP01Responder |
||||
from txacme.service import AcmeIssuingService |
||||
from txacme.endpoint import load_or_create_client_key |
||||
from txacme.client import Client |
||||
from josepy.jwa import RS256 |
||||
|
||||
self._store = ErsatzStore() |
||||
responder = HTTP01Responder() |
||||
|
||||
self._issuer = AcmeIssuingService( |
||||
cert_store=self._store, |
||||
client_creator=( |
||||
lambda: Client.from_url( |
||||
reactor=self.reactor, |
||||
url=URL.from_text(self.hs.config.acme_url), |
||||
key=load_or_create_client_key( |
||||
FilePath(self.hs.config.config_dir_path) |
||||
), |
||||
alg=RS256, |
||||
) |
||||
), |
||||
clock=self.reactor, |
||||
responders=[responder], |
||||
) |
||||
|
||||
well_known = Resource() |
||||
well_known.putChild(b'acme-challenge', responder.resource) |
||||
responder_resource = Resource() |
||||
responder_resource.putChild(b'.well-known', well_known) |
||||
responder_resource.putChild(b'check', static.Data(b'OK', b'text/plain')) |
||||
|
||||
srv = server.Site(responder_resource) |
||||
|
||||
listeners = [] |
||||
|
||||
for host in self.hs.config.acme_bind_addresses: |
||||
logger.info( |
||||
"Listening for ACME requests on %s:%s", host, self.hs.config.acme_port |
||||
) |
||||
endpoint = serverFromString( |
||||
self.reactor, "tcp:%s:interface=%s" % (self.hs.config.acme_port, host) |
||||
) |
||||
listeners.append(endpoint.listen(srv)) |
||||
|
||||
# Make sure we are registered to the ACME server. There's no public API |
||||
# for this, it is usually triggered by startService, but since we don't |
||||
# want it to control where we save the certificates, we have to reach in |
||||
# and trigger the registration machinery ourselves. |
||||
self._issuer._registered = False |
||||
yield self._issuer._ensure_registered() |
||||
|
||||
# Return a Deferred that will fire when all the servers have started up. |
||||
yield defer.DeferredList(listeners, fireOnOneErrback=True, consumeErrors=True) |
||||
|
||||
@defer.inlineCallbacks |
||||
def provision_certificate(self): |
||||
|
||||
logger.warning("Reprovisioning %s", self.hs.hostname) |
||||
|
||||
try: |
||||
yield self._issuer.issue_cert(self.hs.hostname) |
||||
except Exception: |
||||
logger.exception("Fail!") |
||||
raise |
||||
logger.warning("Reprovisioned %s, saving.", self.hs.hostname) |
||||
cert_chain = self._store.certs[self.hs.hostname] |
||||
|
||||
try: |
||||
with open(self.hs.config.tls_private_key_file, "wb") as private_key_file: |
||||
for x in cert_chain: |
||||
if x.startswith(b"-----BEGIN RSA PRIVATE KEY-----"): |
||||
private_key_file.write(x) |
||||
|
||||
with open(self.hs.config.tls_certificate_file, "wb") as certificate_file: |
||||
for x in cert_chain: |
||||
if x.startswith(b"-----BEGIN CERTIFICATE-----"): |
||||
certificate_file.write(x) |
||||
except Exception: |
||||
logger.exception("Failed saving!") |
||||
raise |
||||
|
||||
defer.returnValue(True) |
||||
@ -0,0 +1,124 @@ |
||||
# -*- coding: utf-8 -*- |
||||
# Copyright 2019 New Vector Ltd |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
import logging |
||||
|
||||
from zope.interface import implementer |
||||
|
||||
from twisted.internet import defer |
||||
from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS |
||||
from twisted.web.client import URI, Agent, HTTPConnectionPool |
||||
from twisted.web.iweb import IAgent |
||||
|
||||
from synapse.http.endpoint import parse_server_name |
||||
from synapse.http.federation.srv_resolver import SrvResolver, pick_server_from_list |
||||
from synapse.util.logcontext import make_deferred_yieldable |
||||
|
||||
logger = logging.getLogger(__name__) |
||||
|
||||
|
||||
@implementer(IAgent) |
||||
class MatrixFederationAgent(object): |
||||
"""An Agent-like thing which provides a `request` method which will look up a matrix |
||||
server and send an HTTP request to it. |
||||
|
||||
Doesn't implement any retries. (Those are done in MatrixFederationHttpClient.) |
||||
|
||||
Args: |
||||
reactor (IReactor): twisted reactor to use for underlying requests |
||||
|
||||
tls_client_options_factory (ClientTLSOptionsFactory|None): |
||||
factory to use for fetching client tls options, or none to disable TLS. |
||||
|
||||
srv_resolver (SrvResolver|None): |
||||
SRVResolver impl to use for looking up SRV records. None to use a default |
||||
implementation. |
||||
""" |
||||
|
||||
def __init__( |
||||
self, reactor, tls_client_options_factory, _srv_resolver=None, |
||||
): |
||||
self._reactor = reactor |
||||
self._tls_client_options_factory = tls_client_options_factory |
||||
if _srv_resolver is None: |
||||
_srv_resolver = SrvResolver() |
||||
self._srv_resolver = _srv_resolver |
||||
|
||||
self._pool = HTTPConnectionPool(reactor) |
||||
self._pool.retryAutomatically = False |
||||
self._pool.maxPersistentPerHost = 5 |
||||
self._pool.cachedConnectionTimeout = 2 * 60 |
||||
|
||||
@defer.inlineCallbacks |
||||
def request(self, method, uri, headers=None, bodyProducer=None): |
||||
""" |
||||
Args: |
||||
method (bytes): HTTP method: GET/POST/etc |
||||
|
||||
uri (bytes): Absolute URI to be retrieved |
||||
|
||||
headers (twisted.web.http_headers.Headers|None): |
||||
HTTP headers to send with the request, or None to |
||||
send no extra headers. |
||||
|
||||
bodyProducer (twisted.web.iweb.IBodyProducer|None): |
||||
An object which can generate bytes to make up the |
||||
body of this request (for example, the properly encoded contents of |
||||
a file for a file upload). Or None if the request is to have |
||||
no body. |
||||
|
||||
Returns: |
||||
Deferred[twisted.web.iweb.IResponse]: |
||||
fires when the header of the response has been received (regardless of the |
||||
response status code). Fails if there is any problem which prevents that |
||||
response from being received (including problems that prevent the request |
||||
from being sent). |
||||
""" |
||||
|
||||
parsed_uri = URI.fromBytes(uri) |
||||
server_name_bytes = parsed_uri.netloc |
||||
host, port = parse_server_name(server_name_bytes.decode("ascii")) |
||||
|
||||
# XXX disabling TLS is really only supported here for the benefit of the |
||||
# unit tests. We should make the UTs cope with TLS rather than having to make |
||||
# the code support the unit tests. |
||||
if self._tls_client_options_factory is None: |
||||
tls_options = None |
||||
else: |
||||
tls_options = self._tls_client_options_factory.get_options(host) |
||||
|
||||
if port is not None: |
||||
target = (host, port) |
||||
else: |
||||
server_list = yield self._srv_resolver.resolve_service(server_name_bytes) |
||||
if not server_list: |
||||
target = (host, 8448) |
||||
logger.debug("No SRV record for %s, using %s", host, target) |
||||
else: |
||||
target = pick_server_from_list(server_list) |
||||
|
||||
class EndpointFactory(object): |
||||
@staticmethod |
||||
def endpointForURI(_uri): |
||||
logger.info("Connecting to %s:%s", target[0], target[1]) |
||||
ep = HostnameEndpoint(self._reactor, host=target[0], port=target[1]) |
||||
if tls_options is not None: |
||||
ep = wrapClientTLS(tls_options, ep) |
||||
return ep |
||||
|
||||
agent = Agent.usingEndpointFactory(self._reactor, EndpointFactory(), self._pool) |
||||
res = yield make_deferred_yieldable( |
||||
agent.request(method, uri, headers, bodyProducer) |
||||
) |
||||
defer.returnValue(res) |
||||
@ -0,0 +1,183 @@ |
||||
# -*- coding: utf-8 -*- |
||||
# Copyright 2019 New Vector Ltd |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
import logging |
||||
|
||||
from mock import Mock |
||||
|
||||
import treq |
||||
|
||||
from twisted.internet import defer |
||||
from twisted.internet.protocol import Factory |
||||
from twisted.protocols.tls import TLSMemoryBIOFactory |
||||
from twisted.test.ssl_helpers import ServerTLSContext |
||||
from twisted.web.http import HTTPChannel |
||||
|
||||
from synapse.crypto.context_factory import ClientTLSOptionsFactory |
||||
from synapse.http.federation.matrix_federation_agent import MatrixFederationAgent |
||||
from synapse.util.logcontext import LoggingContext |
||||
|
||||
from tests.server import FakeTransport, ThreadedMemoryReactorClock |
||||
from tests.unittest import TestCase |
||||
|
||||
logger = logging.getLogger(__name__) |
||||
|
||||
|
||||
class MatrixFederationAgentTests(TestCase): |
||||
def setUp(self): |
||||
self.reactor = ThreadedMemoryReactorClock() |
||||
|
||||
self.mock_resolver = Mock() |
||||
|
||||
self.agent = MatrixFederationAgent( |
||||
reactor=self.reactor, |
||||
tls_client_options_factory=ClientTLSOptionsFactory(None), |
||||
_srv_resolver=self.mock_resolver, |
||||
) |
||||
|
||||
def _make_connection(self, client_factory): |
||||
"""Builds a test server, and completes the outgoing client connection |
||||
|
||||
Returns: |
||||
HTTPChannel: the test server |
||||
""" |
||||
|
||||
# build the test server |
||||
server_tls_protocol = _build_test_server() |
||||
|
||||
# now, tell the client protocol factory to build the client protocol (it will be a |
||||
# _WrappingProtocol, around a TLSMemoryBIOProtocol, around an |
||||
# HTTP11ClientProtocol) and wire the output of said protocol up to the server via |
||||
# a FakeTransport. |
||||
# |
||||
# Normally this would be done by the TCP socket code in Twisted, but we are |
||||
# stubbing that out here. |
||||
client_protocol = client_factory.buildProtocol(None) |
||||
client_protocol.makeConnection(FakeTransport(server_tls_protocol, self.reactor)) |
||||
|
||||
# tell the server tls protocol to send its stuff back to the client, too |
||||
server_tls_protocol.makeConnection(FakeTransport(client_protocol, self.reactor)) |
||||
|
||||
# finally, give the reactor a pump to get the TLS juices flowing. |
||||
self.reactor.pump((0.1,)) |
||||
|
||||
# fish the test server back out of the server-side TLS protocol. |
||||
return server_tls_protocol.wrappedProtocol |
||||
|
||||
@defer.inlineCallbacks |
||||
def _make_get_request(self, uri): |
||||
""" |
||||
Sends a simple GET request via the agent, and checks its logcontext management |
||||
""" |
||||
with LoggingContext("one") as context: |
||||
fetch_d = self.agent.request(b'GET', uri) |
||||
|
||||
# Nothing happened yet |
||||
self.assertNoResult(fetch_d) |
||||
|
||||
# should have reset logcontext to the sentinel |
||||
_check_logcontext(LoggingContext.sentinel) |
||||
|
||||
try: |
||||
fetch_res = yield fetch_d |
||||
defer.returnValue(fetch_res) |
||||
finally: |
||||
_check_logcontext(context) |
||||
|
||||
def test_get(self): |
||||
""" |
||||
happy-path test of a GET request |
||||
""" |
||||
self.reactor.lookups["testserv"] = "1.2.3.4" |
||||
test_d = self._make_get_request(b"matrix://testserv:8448/foo/bar") |
||||
|
||||
# Nothing happened yet |
||||
self.assertNoResult(test_d) |
||||
|
||||
# Make sure treq is trying to connect |
||||
clients = self.reactor.tcpClients |
||||
self.assertEqual(len(clients), 1) |
||||
(host, port, client_factory, _timeout, _bindAddress) = clients[0] |
||||
self.assertEqual(host, '1.2.3.4') |
||||
self.assertEqual(port, 8448) |
||||
|
||||
# make a test server, and wire up the client |
||||
http_server = self._make_connection(client_factory) |
||||
|
||||
self.assertEqual(len(http_server.requests), 1) |
||||
request = http_server.requests[0] |
||||
self.assertEqual(request.method, b'GET') |
||||
self.assertEqual(request.path, b'/foo/bar') |
||||
self.assertEqual( |
||||
request.requestHeaders.getRawHeaders(b'host'), |
||||
[b'testserv:8448'] |
||||
) |
||||
content = request.content.read() |
||||
self.assertEqual(content, b'') |
||||
|
||||
# Deferred is still without a result |
||||
self.assertNoResult(test_d) |
||||
|
||||
# send the headers |
||||
request.responseHeaders.setRawHeaders(b'Content-Type', [b'application/json']) |
||||
request.write('') |
||||
|
||||
self.reactor.pump((0.1,)) |
||||
|
||||
response = self.successResultOf(test_d) |
||||
|
||||
# that should give us a Response object |
||||
self.assertEqual(response.code, 200) |
||||
|
||||
# Send the body |
||||
request.write('{ "a": 1 }'.encode('ascii')) |
||||
request.finish() |
||||
|
||||
self.reactor.pump((0.1,)) |
||||
|
||||
# check it can be read |
||||
json = self.successResultOf(treq.json_content(response)) |
||||
self.assertEqual(json, {"a": 1}) |
||||
|
||||
|
||||
def _check_logcontext(context): |
||||
current = LoggingContext.current_context() |
||||
if current is not context: |
||||
raise AssertionError( |
||||
"Expected logcontext %s but was %s" % (context, current), |
||||
) |
||||
|
||||
|
||||
def _build_test_server(): |
||||
"""Construct a test server |
||||
|
||||
This builds an HTTP channel, wrapped with a TLSMemoryBIOProtocol |
||||
|
||||
Returns: |
||||
TLSMemoryBIOProtocol |
||||
""" |
||||
server_factory = Factory.forProtocol(HTTPChannel) |
||||
# Request.finish expects the factory to have a 'log' method. |
||||
server_factory.log = _log_request |
||||
|
||||
server_tls_factory = TLSMemoryBIOFactory( |
||||
ServerTLSContext(), isClient=False, wrappedFactory=server_factory, |
||||
) |
||||
|
||||
return server_tls_factory.buildProtocol(None) |
||||
|
||||
|
||||
def _log_request(request): |
||||
"""Implements Factory.log, which is expected by Request.finish""" |
||||
logger.info("Completed request %s", request) |
||||
Loading…
Reference in new issue