mirror of https://github.com/watcha-fr/synapse
Mark Haines
9 years ago
commit
5201c66108
@ -0,0 +1,93 @@ |
||||
# -*- coding: utf-8 -*- |
||||
# Copyright 2015 OpenMarket Ltd |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
from twisted.internet import defer |
||||
|
||||
from ._base import BaseHandler |
||||
|
||||
from synapse.api.constants import Membership |
||||
from synapse.api.errors import SynapseError |
||||
from synapse.events.utils import serialize_event |
||||
|
||||
import logging |
||||
|
||||
|
||||
logger = logging.getLogger(__name__) |
||||
|
||||
|
||||
class SearchHandler(BaseHandler): |
||||
|
||||
def __init__(self, hs): |
||||
super(SearchHandler, self).__init__(hs) |
||||
|
||||
@defer.inlineCallbacks |
||||
def search(self, user, content): |
||||
"""Performs a full text search for a user. |
||||
|
||||
Args: |
||||
user (UserID) |
||||
content (dict): Search parameters |
||||
|
||||
Returns: |
||||
dict to be returned to the client with results of search |
||||
""" |
||||
|
||||
try: |
||||
search_term = content["search_categories"]["room_events"]["search_term"] |
||||
keys = content["search_categories"]["room_events"].get("keys", [ |
||||
"content.body", "content.name", "content.topic", |
||||
]) |
||||
except KeyError: |
||||
raise SynapseError(400, "Invalid search query") |
||||
|
||||
# TODO: Search through left rooms too |
||||
rooms = yield self.store.get_rooms_for_user_where_membership_is( |
||||
user.to_string(), |
||||
membership_list=[Membership.JOIN], |
||||
# membership_list=[Membership.JOIN, Membership.LEAVE, Membership.Ban], |
||||
) |
||||
room_ids = set(r.room_id for r in rooms) |
||||
|
||||
# TODO: Apply room filter to rooms list |
||||
|
||||
rank_map, event_map = yield self.store.search_msgs(room_ids, search_term, keys) |
||||
|
||||
allowed_events = yield self._filter_events_for_client( |
||||
user.to_string(), event_map.values() |
||||
) |
||||
|
||||
# TODO: Filter allowed_events |
||||
# TODO: Add a limit |
||||
|
||||
time_now = self.clock.time_msec() |
||||
|
||||
results = { |
||||
e.event_id: { |
||||
"rank": rank_map[e.event_id], |
||||
"result": serialize_event(e, time_now) |
||||
} |
||||
for e in allowed_events |
||||
} |
||||
|
||||
logger.info("Found %d results", len(results)) |
||||
|
||||
defer.returnValue({ |
||||
"search_categories": { |
||||
"room_events": { |
||||
"results": results, |
||||
"count": len(results) |
||||
} |
||||
} |
||||
}) |
||||
@ -0,0 +1,123 @@ |
||||
# Copyright 2015 OpenMarket Ltd |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
import logging |
||||
|
||||
from synapse.storage.prepare_database import get_statements |
||||
from synapse.storage.engines import PostgresEngine, Sqlite3Engine |
||||
|
||||
import ujson |
||||
|
||||
logger = logging.getLogger(__name__) |
||||
|
||||
|
||||
POSTGRES_SQL = """ |
||||
CREATE TABLE event_search ( |
||||
event_id TEXT, |
||||
room_id TEXT, |
||||
key TEXT, |
||||
vector tsvector |
||||
); |
||||
|
||||
INSERT INTO event_search SELECT |
||||
event_id, room_id, 'content.body', |
||||
to_tsvector('english', json::json->'content'->>'body') |
||||
FROM events NATURAL JOIN event_json WHERE type = 'm.room.message'; |
||||
|
||||
INSERT INTO event_search SELECT |
||||
event_id, room_id, 'content.name', |
||||
to_tsvector('english', json::json->'content'->>'name') |
||||
FROM events NATURAL JOIN event_json WHERE type = 'm.room.name'; |
||||
|
||||
INSERT INTO event_search SELECT |
||||
event_id, room_id, 'content.topic', |
||||
to_tsvector('english', json::json->'content'->>'topic') |
||||
FROM events NATURAL JOIN event_json WHERE type = 'm.room.topic'; |
||||
|
||||
|
||||
CREATE INDEX event_search_fts_idx ON event_search USING gin(vector); |
||||
CREATE INDEX event_search_ev_idx ON event_search(event_id); |
||||
CREATE INDEX event_search_ev_ridx ON event_search(room_id); |
||||
""" |
||||
|
||||
|
||||
SQLITE_TABLE = ( |
||||
"CREATE VIRTUAL TABLE event_search USING fts3 ( event_id, room_id, key, value)" |
||||
) |
||||
|
||||
|
||||
def run_upgrade(cur, database_engine, *args, **kwargs): |
||||
if isinstance(database_engine, PostgresEngine): |
||||
run_postgres_upgrade(cur) |
||||
return |
||||
|
||||
if isinstance(database_engine, Sqlite3Engine): |
||||
run_sqlite_upgrade(cur) |
||||
return |
||||
|
||||
|
||||
def run_postgres_upgrade(cur): |
||||
for statement in get_statements(POSTGRES_SQL.splitlines()): |
||||
cur.execute(statement) |
||||
|
||||
|
||||
def run_sqlite_upgrade(cur): |
||||
cur.execute(SQLITE_TABLE) |
||||
|
||||
rowid = -1 |
||||
while True: |
||||
cur.execute( |
||||
"SELECT rowid, json FROM event_json" |
||||
" WHERE rowid > ?" |
||||
" ORDER BY rowid ASC LIMIT 100", |
||||
(rowid,) |
||||
) |
||||
|
||||
res = cur.fetchall() |
||||
|
||||
if not res: |
||||
break |
||||
|
||||
events = [ |
||||
ujson.loads(js) |
||||
for _, js in res |
||||
] |
||||
|
||||
rowid = max(rid for rid, _ in res) |
||||
|
||||
rows = [] |
||||
for ev in events: |
||||
if ev["type"] == "m.room.message": |
||||
rows.append(( |
||||
ev["event_id"], ev["room_id"], "content.body", |
||||
ev["content"]["body"] |
||||
)) |
||||
if ev["type"] == "m.room.name": |
||||
rows.append(( |
||||
ev["event_id"], ev["room_id"], "content.name", |
||||
ev["content"]["name"] |
||||
)) |
||||
if ev["type"] == "m.room.topic": |
||||
rows.append(( |
||||
ev["event_id"], ev["room_id"], "content.topic", |
||||
ev["content"]["topic"] |
||||
)) |
||||
|
||||
if rows: |
||||
logger.info(rows) |
||||
cur.executemany( |
||||
"INSERT INTO event_search (event_id, room_id, key, value)" |
||||
" VALUES (?,?,?,?)", |
||||
rows |
||||
) |
||||
@ -0,0 +1,93 @@ |
||||
# -*- coding: utf-8 -*- |
||||
# Copyright 2015 OpenMarket Ltd |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
from twisted.internet import defer |
||||
|
||||
from _base import SQLBaseStore |
||||
from synapse.storage.engines import PostgresEngine, Sqlite3Engine |
||||
|
||||
|
||||
class SearchStore(SQLBaseStore): |
||||
@defer.inlineCallbacks |
||||
def search_msgs(self, room_ids, search_term, keys): |
||||
"""Performs a full text search over events with given keys. |
||||
|
||||
Args: |
||||
room_ids (list): List of room ids to search in |
||||
search_term (str): Search term to search for |
||||
keys (list): List of keys to search in, currently supports |
||||
"content.body", "content.name", "content.topic" |
||||
|
||||
Returns: |
||||
2-tuple of (dict event_id -> rank, dict event_id -> event) |
||||
""" |
||||
clauses = [] |
||||
args = [] |
||||
|
||||
clauses.append( |
||||
"room_id IN (%s)" % (",".join(["?"] * len(room_ids)),) |
||||
) |
||||
args.extend(room_ids) |
||||
|
||||
local_clauses = [] |
||||
for key in keys: |
||||
local_clauses.append("key = ?") |
||||
args.append(key) |
||||
|
||||
clauses.append( |
||||
"(%s)" % (" OR ".join(local_clauses),) |
||||
) |
||||
|
||||
if isinstance(self.database_engine, PostgresEngine): |
||||
sql = ( |
||||
"SELECT ts_rank_cd(vector, query) AS rank, event_id" |
||||
" FROM plainto_tsquery('english', ?) as query, event_search" |
||||
" WHERE vector @@ query" |
||||
) |
||||
elif isinstance(self.database_engine, Sqlite3Engine): |
||||
sql = ( |
||||
"SELECT 0 as rank, event_id FROM event_search" |
||||
" WHERE value MATCH ?" |
||||
) |
||||
else: |
||||
# This should be unreachable. |
||||
raise Exception("Unrecognized database engine") |
||||
|
||||
for clause in clauses: |
||||
sql += " AND " + clause |
||||
|
||||
# We add an arbitrary limit here to ensure we don't try to pull the |
||||
# entire table from the database. |
||||
sql += " ORDER BY rank DESC LIMIT 500" |
||||
|
||||
results = yield self._execute( |
||||
"search_msgs", self.cursor_to_dict, sql, *([search_term] + args) |
||||
) |
||||
|
||||
events = yield self._get_events([r["event_id"] for r in results]) |
||||
|
||||
event_map = { |
||||
ev.event_id: ev |
||||
for ev in events |
||||
} |
||||
|
||||
defer.returnValue(( |
||||
{ |
||||
r["event_id"]: r["rank"] |
||||
for r in results |
||||
if r["event_id"] in event_map |
||||
}, |
||||
event_map |
||||
)) |
||||
@ -1,71 +0,0 @@ |
||||
# -*- coding: utf-8 -*- |
||||
# Copyright 2014, 2015 OpenMarket Ltd |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
""" This module allows you to send out emails. |
||||
""" |
||||
import email.utils |
||||
import smtplib |
||||
import twisted.python.log |
||||
from email.mime.text import MIMEText |
||||
from email.mime.multipart import MIMEMultipart |
||||
|
||||
import logging |
||||
|
||||
logger = logging.getLogger(__name__) |
||||
|
||||
|
||||
class EmailException(Exception): |
||||
pass |
||||
|
||||
|
||||
def send_email(smtp_server, from_addr, to_addr, subject, body): |
||||
"""Sends an email. |
||||
|
||||
Args: |
||||
smtp_server(str): The SMTP server to use. |
||||
from_addr(str): The address to send from. |
||||
to_addr(str): The address to send to. |
||||
subject(str): The subject of the email. |
||||
body(str): The plain text body of the email. |
||||
Raises: |
||||
EmailException if there was a problem sending the mail. |
||||
""" |
||||
if not smtp_server or not from_addr or not to_addr: |
||||
raise EmailException("Need SMTP server, from and to addresses. Check" |
||||
" the config to set these.") |
||||
|
||||
msg = MIMEMultipart('alternative') |
||||
msg['Subject'] = subject |
||||
msg['From'] = from_addr |
||||
msg['To'] = to_addr |
||||
plain_part = MIMEText(body) |
||||
msg.attach(plain_part) |
||||
|
||||
raw_from = email.utils.parseaddr(from_addr)[1] |
||||
raw_to = email.utils.parseaddr(to_addr)[1] |
||||
if not raw_from or not raw_to: |
||||
raise EmailException("Couldn't parse from/to address.") |
||||
|
||||
logger.info("Sending email to %s on server %s with subject %s", |
||||
to_addr, smtp_server, subject) |
||||
|
||||
try: |
||||
smtp = smtplib.SMTP(smtp_server) |
||||
smtp.sendmail(raw_from, raw_to, msg.as_string()) |
||||
smtp.quit() |
||||
except Exception as origException: |
||||
twisted.python.log.err() |
||||
ese = EmailException() |
||||
ese.cause = origException |
||||
raise ese |
||||
@ -0,0 +1,15 @@ |
||||
# -*- coding: utf-8 -*- |
||||
# Copyright 2014 OpenMarket Ltd |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
@ -0,0 +1,114 @@ |
||||
# -*- coding: utf-8 -*- |
||||
# Copyright 2015 OpenMarket Ltd |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
|
||||
from tests import unittest |
||||
|
||||
from synapse.events.builder import EventBuilder |
||||
from synapse.crypto.event_signing import add_hashes_and_signatures |
||||
|
||||
from unpaddedbase64 import decode_base64 |
||||
|
||||
import nacl.signing |
||||
|
||||
|
||||
# Perform these tests using given secret key so we get entirely deterministic |
||||
# signatures output that we can test against. |
||||
SIGNING_KEY_SEED = decode_base64( |
||||
"YJDBA9Xnr2sVqXD9Vj7XVUnmFZcZrlw8Md7kMW+3XA1" |
||||
) |
||||
|
||||
KEY_ALG = "ed25519" |
||||
KEY_VER = 1 |
||||
KEY_NAME = "%s:%d" % (KEY_ALG, KEY_VER) |
||||
|
||||
HOSTNAME = "domain" |
||||
|
||||
|
||||
class EventSigningTestCase(unittest.TestCase): |
||||
|
||||
def setUp(self): |
||||
self.signing_key = nacl.signing.SigningKey(SIGNING_KEY_SEED) |
||||
self.signing_key.alg = KEY_ALG |
||||
self.signing_key.version = KEY_VER |
||||
|
||||
def test_sign_minimal(self): |
||||
builder = EventBuilder( |
||||
{ |
||||
'event_id': "$0:domain", |
||||
'origin': "domain", |
||||
'origin_server_ts': 1000000, |
||||
'signatures': {}, |
||||
'type': "X", |
||||
'unsigned': {'age_ts': 1000000}, |
||||
}, |
||||
) |
||||
|
||||
add_hashes_and_signatures(builder, HOSTNAME, self.signing_key) |
||||
|
||||
event = builder.build() |
||||
|
||||
self.assertTrue(hasattr(event, 'hashes')) |
||||
self.assertIn('sha256', event.hashes) |
||||
self.assertEquals( |
||||
event.hashes['sha256'], |
||||
"6tJjLpXtggfke8UxFhAKg82QVkJzvKOVOOSjUDK4ZSI", |
||||
) |
||||
|
||||
self.assertTrue(hasattr(event, 'signatures')) |
||||
self.assertIn(HOSTNAME, event.signatures) |
||||
self.assertIn(KEY_NAME, event.signatures["domain"]) |
||||
self.assertEquals( |
||||
event.signatures[HOSTNAME][KEY_NAME], |
||||
"2Wptgo4CwmLo/Y8B8qinxApKaCkBG2fjTWB7AbP5Uy+" |
||||
"aIbygsSdLOFzvdDjww8zUVKCmI02eP9xtyJxc/cLiBA", |
||||
) |
||||
|
||||
def test_sign_message(self): |
||||
builder = EventBuilder( |
||||
{ |
||||
'content': { |
||||
'body': "Here is the message content", |
||||
}, |
||||
'event_id': "$0:domain", |
||||
'origin': "domain", |
||||
'origin_server_ts': 1000000, |
||||
'type': "m.room.message", |
||||
'room_id': "!r:domain", |
||||
'sender': "@u:domain", |
||||
'signatures': {}, |
||||
'unsigned': {'age_ts': 1000000}, |
||||
} |
||||
) |
||||
|
||||
add_hashes_and_signatures(builder, HOSTNAME, self.signing_key) |
||||
|
||||
event = builder.build() |
||||
|
||||
self.assertTrue(hasattr(event, 'hashes')) |
||||
self.assertIn('sha256', event.hashes) |
||||
self.assertEquals( |
||||
event.hashes['sha256'], |
||||
"onLKD1bGljeBWQhWZ1kaP9SorVmRQNdN5aM2JYU2n/g", |
||||
) |
||||
|
||||
self.assertTrue(hasattr(event, 'signatures')) |
||||
self.assertIn(HOSTNAME, event.signatures) |
||||
self.assertIn(KEY_NAME, event.signatures["domain"]) |
||||
self.assertEquals( |
||||
event.signatures[HOSTNAME][KEY_NAME], |
||||
"Wm+VzmOUOz08Ds+0NTWb1d4CZrVsJSikkeRxh6aCcUw" |
||||
"u6pNC78FunoD7KNWzqFn241eYHYMGCA5McEiVPdhzBA" |
||||
) |
||||
Loading…
Reference in new issue