mirror of https://github.com/watcha-fr/synapse
Merge pull request #2854 from matrix-org/erikj/event_create_worker
Create a worker for event creationpull/4/merge
Erik Johnston
7 years ago
committed by
GitHub
commit
c0c9327fe0
@ -0,0 +1,170 @@ |
||||
#!/usr/bin/env python |
||||
# -*- coding: utf-8 -*- |
||||
# Copyright 2018 New Vector Ltd |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
import logging |
||||
import sys |
||||
|
||||
import synapse |
||||
from synapse import events |
||||
from synapse.app import _base |
||||
from synapse.config._base import ConfigError |
||||
from synapse.config.homeserver import HomeServerConfig |
||||
from synapse.config.logger import setup_logging |
||||
from synapse.crypto import context_factory |
||||
from synapse.http.server import JsonResource |
||||
from synapse.http.site import SynapseSite |
||||
from synapse.metrics.resource import METRICS_PREFIX, MetricsResource |
||||
from synapse.replication.slave.storage._base import BaseSlavedStore |
||||
from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore |
||||
from synapse.replication.slave.storage.client_ips import SlavedClientIpStore |
||||
from synapse.replication.slave.storage.devices import SlavedDeviceStore |
||||
from synapse.replication.slave.storage.events import SlavedEventStore |
||||
from synapse.replication.slave.storage.registration import SlavedRegistrationStore |
||||
from synapse.replication.slave.storage.room import RoomStore |
||||
from synapse.replication.tcp.client import ReplicationClientHandler |
||||
from synapse.rest.client.v1.room import RoomSendEventRestServlet |
||||
from synapse.server import HomeServer |
||||
from synapse.storage.engines import create_engine |
||||
from synapse.util.httpresourcetree import create_resource_tree |
||||
from synapse.util.logcontext import LoggingContext |
||||
from synapse.util.manhole import manhole |
||||
from synapse.util.versionstring import get_version_string |
||||
from twisted.internet import reactor |
||||
from twisted.web.resource import Resource |
||||
|
||||
logger = logging.getLogger("synapse.app.event_creator") |
||||
|
||||
|
||||
class EventCreatorSlavedStore( |
||||
SlavedDeviceStore, |
||||
SlavedClientIpStore, |
||||
SlavedApplicationServiceStore, |
||||
SlavedEventStore, |
||||
SlavedRegistrationStore, |
||||
RoomStore, |
||||
BaseSlavedStore, |
||||
): |
||||
pass |
||||
|
||||
|
||||
class EventCreatorServer(HomeServer): |
||||
def setup(self): |
||||
logger.info("Setting up.") |
||||
self.datastore = EventCreatorSlavedStore(self.get_db_conn(), self) |
||||
logger.info("Finished setting up.") |
||||
|
||||
def _listen_http(self, listener_config): |
||||
port = listener_config["port"] |
||||
bind_addresses = listener_config["bind_addresses"] |
||||
site_tag = listener_config.get("tag", port) |
||||
resources = {} |
||||
for res in listener_config["resources"]: |
||||
for name in res["names"]: |
||||
if name == "metrics": |
||||
resources[METRICS_PREFIX] = MetricsResource(self) |
||||
elif name == "client": |
||||
resource = JsonResource(self, canonical_json=False) |
||||
RoomSendEventRestServlet(self).register(resource) |
||||
resources.update({ |
||||
"/_matrix/client/r0": resource, |
||||
"/_matrix/client/unstable": resource, |
||||
"/_matrix/client/v2_alpha": resource, |
||||
"/_matrix/client/api/v1": resource, |
||||
}) |
||||
|
||||
root_resource = create_resource_tree(resources, Resource()) |
||||
|
||||
_base.listen_tcp( |
||||
bind_addresses, |
||||
port, |
||||
SynapseSite( |
||||
"synapse.access.http.%s" % (site_tag,), |
||||
site_tag, |
||||
listener_config, |
||||
root_resource, |
||||
) |
||||
) |
||||
|
||||
logger.info("Synapse event creator now listening on port %d", port) |
||||
|
||||
def start_listening(self, listeners): |
||||
for listener in listeners: |
||||
if listener["type"] == "http": |
||||
self._listen_http(listener) |
||||
elif listener["type"] == "manhole": |
||||
_base.listen_tcp( |
||||
listener["bind_addresses"], |
||||
listener["port"], |
||||
manhole( |
||||
username="matrix", |
||||
password="rabbithole", |
||||
globals={"hs": self}, |
||||
) |
||||
) |
||||
else: |
||||
logger.warn("Unrecognized listener type: %s", listener["type"]) |
||||
|
||||
self.get_tcp_replication().start_replication(self) |
||||
|
||||
def build_tcp_replication(self): |
||||
return ReplicationClientHandler(self.get_datastore()) |
||||
|
||||
|
||||
def start(config_options): |
||||
try: |
||||
config = HomeServerConfig.load_config( |
||||
"Synapse event creator", config_options |
||||
) |
||||
except ConfigError as e: |
||||
sys.stderr.write("\n" + e.message + "\n") |
||||
sys.exit(1) |
||||
|
||||
assert config.worker_app == "synapse.app.event_creator" |
||||
|
||||
assert config.worker_replication_http_port is not None |
||||
|
||||
setup_logging(config, use_worker_options=True) |
||||
|
||||
events.USE_FROZEN_DICTS = config.use_frozen_dicts |
||||
|
||||
database_engine = create_engine(config.database_config) |
||||
|
||||
tls_server_context_factory = context_factory.ServerContextFactory(config) |
||||
|
||||
ss = EventCreatorServer( |
||||
config.server_name, |
||||
db_config=config.database_config, |
||||
tls_server_context_factory=tls_server_context_factory, |
||||
config=config, |
||||
version_string="Synapse/" + get_version_string(synapse), |
||||
database_engine=database_engine, |
||||
) |
||||
|
||||
ss.setup() |
||||
ss.get_handlers() |
||||
ss.start_listening(config.worker_listeners) |
||||
|
||||
def start(): |
||||
ss.get_state_handler().start_caching() |
||||
ss.get_datastore().start_profiling() |
||||
|
||||
reactor.callWhenRunning(start) |
||||
|
||||
_base.start_worker_reactor("synapse-event-creator", config) |
||||
|
||||
|
||||
if __name__ == '__main__': |
||||
with LoggingContext("main"): |
||||
start(sys.argv[1:]) |
||||
@ -0,0 +1,31 @@ |
||||
# -*- coding: utf-8 -*- |
||||
# Copyright 2018 New Vector Ltd |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
|
||||
import send_event |
||||
|
||||
from synapse.http.server import JsonResource |
||||
|
||||
|
||||
REPLICATION_PREFIX = "/_synapse/replication" |
||||
|
||||
|
||||
class ReplicationRestResource(JsonResource): |
||||
def __init__(self, hs): |
||||
JsonResource.__init__(self, hs, canonical_json=False) |
||||
self.register_servlets(hs) |
||||
|
||||
def register_servlets(self, hs): |
||||
send_event.register_servlets(hs, self) |
||||
@ -0,0 +1,108 @@ |
||||
# -*- coding: utf-8 -*- |
||||
# Copyright 2018 New Vector Ltd |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
from twisted.internet import defer |
||||
|
||||
from synapse.events import FrozenEvent |
||||
from synapse.events.snapshot import EventContext |
||||
from synapse.http.servlet import RestServlet, parse_json_object_from_request |
||||
from synapse.util.metrics import Measure |
||||
from synapse.types import Requester |
||||
|
||||
import logging |
||||
import re |
||||
|
||||
logger = logging.getLogger(__name__) |
||||
|
||||
|
||||
def send_event_to_master(client, host, port, requester, event, context): |
||||
"""Send event to be handled on the master |
||||
|
||||
Args: |
||||
client (SimpleHttpClient) |
||||
host (str): host of master |
||||
port (int): port on master listening for HTTP replication |
||||
requester (Requester) |
||||
event (FrozenEvent) |
||||
context (EventContext) |
||||
""" |
||||
uri = "http://%s:%s/_synapse/replication/send_event" % (host, port,) |
||||
|
||||
payload = { |
||||
"event": event.get_pdu_json(), |
||||
"internal_metadata": event.internal_metadata.get_dict(), |
||||
"rejected_reason": event.rejected_reason, |
||||
"context": context.serialize(), |
||||
"requester": requester.serialize(), |
||||
} |
||||
|
||||
return client.post_json_get_json(uri, payload) |
||||
|
||||
|
||||
class ReplicationSendEventRestServlet(RestServlet): |
||||
"""Handles events newly created on workers, including persisting and |
||||
notifying. |
||||
|
||||
The API looks like: |
||||
|
||||
POST /_synapse/replication/send_event |
||||
|
||||
{ |
||||
"event": { .. serialized event .. }, |
||||
"internal_metadata": { .. serialized internal_metadata .. }, |
||||
"rejected_reason": .., // The event.rejected_reason field |
||||
"context": { .. serialized event context .. }, |
||||
"requester": { .. serialized requester .. }, |
||||
} |
||||
""" |
||||
PATTERNS = [re.compile("^/_synapse/replication/send_event$")] |
||||
|
||||
def __init__(self, hs): |
||||
super(ReplicationSendEventRestServlet, self).__init__() |
||||
|
||||
self.event_creation_handler = hs.get_event_creation_handler() |
||||
self.store = hs.get_datastore() |
||||
self.clock = hs.get_clock() |
||||
|
||||
@defer.inlineCallbacks |
||||
def on_POST(self, request): |
||||
with Measure(self.clock, "repl_send_event_parse"): |
||||
content = parse_json_object_from_request(request) |
||||
|
||||
event_dict = content["event"] |
||||
internal_metadata = content["internal_metadata"] |
||||
rejected_reason = content["rejected_reason"] |
||||
event = FrozenEvent(event_dict, internal_metadata, rejected_reason) |
||||
|
||||
requester = Requester.deserialize(self.store, content["requester"]) |
||||
context = EventContext.deserialize(self.store, content["context"]) |
||||
|
||||
if requester.user: |
||||
request.authenticated_entity = requester.user.to_string() |
||||
|
||||
logger.info( |
||||
"Got event to send with ID: %s into room: %s", |
||||
event.event_id, event.room_id, |
||||
) |
||||
|
||||
yield self.event_creation_handler.handle_new_client_event( |
||||
requester, event, context, |
||||
) |
||||
|
||||
defer.returnValue((200, {})) |
||||
|
||||
|
||||
def register_servlets(hs, http_server): |
||||
ReplicationSendEventRestServlet(hs).register(http_server) |
||||
Loading…
Reference in new issue