mirror of https://github.com/watcha-fr/synapse
Erik Johnston
9 years ago
commit
d5e081c7ae
@ -0,0 +1,142 @@ |
||||
#! /usr/bin/python |
||||
|
||||
import ast |
||||
import yaml |
||||
|
||||
class DefinitionVisitor(ast.NodeVisitor): |
||||
def __init__(self): |
||||
super(DefinitionVisitor, self).__init__() |
||||
self.functions = {} |
||||
self.classes = {} |
||||
self.names = {} |
||||
self.attrs = set() |
||||
self.definitions = { |
||||
'def': self.functions, |
||||
'class': self.classes, |
||||
'names': self.names, |
||||
'attrs': self.attrs, |
||||
} |
||||
|
||||
def visit_Name(self, node): |
||||
self.names.setdefault(type(node.ctx).__name__, set()).add(node.id) |
||||
|
||||
def visit_Attribute(self, node): |
||||
self.attrs.add(node.attr) |
||||
for child in ast.iter_child_nodes(node): |
||||
self.visit(child) |
||||
|
||||
def visit_ClassDef(self, node): |
||||
visitor = DefinitionVisitor() |
||||
self.classes[node.name] = visitor.definitions |
||||
for child in ast.iter_child_nodes(node): |
||||
visitor.visit(child) |
||||
|
||||
def visit_FunctionDef(self, node): |
||||
visitor = DefinitionVisitor() |
||||
self.functions[node.name] = visitor.definitions |
||||
for child in ast.iter_child_nodes(node): |
||||
visitor.visit(child) |
||||
|
||||
|
||||
def non_empty(defs): |
||||
functions = {name: non_empty(f) for name, f in defs['def'].items()} |
||||
classes = {name: non_empty(f) for name, f in defs['class'].items()} |
||||
result = {} |
||||
if functions: result['def'] = functions |
||||
if classes: result['class'] = classes |
||||
names = defs['names'] |
||||
uses = [] |
||||
for name in names.get('Load', ()): |
||||
if name not in names.get('Param', ()) and name not in names.get('Store', ()): |
||||
uses.append(name) |
||||
uses.extend(defs['attrs']) |
||||
if uses: result['uses'] = uses |
||||
result['names'] = names |
||||
result['attrs'] = defs['attrs'] |
||||
return result |
||||
|
||||
|
||||
def definitions_in_code(input_code): |
||||
input_ast = ast.parse(input_code) |
||||
visitor = DefinitionVisitor() |
||||
visitor.visit(input_ast) |
||||
definitions = non_empty(visitor.definitions) |
||||
return definitions |
||||
|
||||
|
||||
def definitions_in_file(filepath): |
||||
with open(filepath) as f: |
||||
return definitions_in_code(f.read()) |
||||
|
||||
|
||||
def defined_names(prefix, defs, names): |
||||
for name, funcs in defs.get('def', {}).items(): |
||||
names.setdefault(name, {'defined': []})['defined'].append(prefix + name) |
||||
defined_names(prefix + name + ".", funcs, names) |
||||
|
||||
for name, funcs in defs.get('class', {}).items(): |
||||
names.setdefault(name, {'defined': []})['defined'].append(prefix + name) |
||||
defined_names(prefix + name + ".", funcs, names) |
||||
|
||||
|
||||
def used_names(prefix, defs, names): |
||||
for name, funcs in defs.get('def', {}).items(): |
||||
used_names(prefix + name + ".", funcs, names) |
||||
|
||||
for name, funcs in defs.get('class', {}).items(): |
||||
used_names(prefix + name + ".", funcs, names) |
||||
|
||||
for used in defs.get('uses', ()): |
||||
if used in names: |
||||
names[used].setdefault('used', []).append(prefix.rstrip('.')) |
||||
|
||||
|
||||
if __name__ == '__main__': |
||||
import sys, os, argparse, re |
||||
|
||||
parser = argparse.ArgumentParser(description='Find definitions.') |
||||
parser.add_argument( |
||||
"--unused", action="store_true", help="Only list unused definitions" |
||||
) |
||||
parser.add_argument( |
||||
"--ignore", action="append", metavar="REGEXP", help="Ignore a pattern" |
||||
) |
||||
parser.add_argument( |
||||
"--pattern", action="append", metavar="REGEXP", |
||||
help="Search for a pattern" |
||||
) |
||||
parser.add_argument( |
||||
"directories", nargs='+', metavar="DIR", |
||||
help="Directories to search for definitions" |
||||
) |
||||
args = parser.parse_args() |
||||
|
||||
definitions = {} |
||||
for directory in args.directories: |
||||
for root, dirs, files in os.walk(directory): |
||||
for filename in files: |
||||
if filename.endswith(".py"): |
||||
filepath = os.path.join(root, filename) |
||||
definitions[filepath] = definitions_in_file(filepath) |
||||
|
||||
names = {} |
||||
for filepath, defs in definitions.items(): |
||||
defined_names(filepath + ":", defs, names) |
||||
|
||||
for filepath, defs in definitions.items(): |
||||
used_names(filepath + ":", defs, names) |
||||
|
||||
patterns = [re.compile(pattern) for pattern in args.pattern or ()] |
||||
ignore = [re.compile(pattern) for pattern in args.ignore or ()] |
||||
|
||||
result = {} |
||||
for name, definition in names.items(): |
||||
if patterns and not any(pattern.match(name) for pattern in patterns): |
||||
continue |
||||
if ignore and any(pattern.match(name) for pattern in ignore): |
||||
continue |
||||
if args.unused and definition.get('used'): |
||||
continue |
||||
result[name] = definition |
||||
|
||||
yaml.dump(result, sys.stdout, default_flow_style=False) |
||||
@ -0,0 +1,22 @@ |
||||
/* Copyright 2015 OpenMarket Ltd |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
-- Should only ever contain one row |
||||
CREATE TABLE IF NOT EXISTS stats_reporting( |
||||
-- The stream ordering token which was most recently reported as stats |
||||
reported_stream_token INTEGER, |
||||
-- The time (seconds since epoch) stats were most recently reported |
||||
reported_time BIGINT |
||||
); |
||||
@ -0,0 +1,81 @@ |
||||
# -*- coding: utf-8 -*- |
||||
# Copyright 2015 OpenMarket Ltd |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
|
||||
from tests import unittest |
||||
from twisted.internet import defer |
||||
|
||||
from synapse.api.constants import EventTypes, Membership |
||||
from synapse.types import UserID, RoomID |
||||
|
||||
from tests.utils import setup_test_homeserver |
||||
|
||||
from mock import Mock |
||||
|
||||
|
||||
class EventInjector: |
||||
def __init__(self, hs): |
||||
self.hs = hs |
||||
self.store = hs.get_datastore() |
||||
self.message_handler = hs.get_handlers().message_handler |
||||
self.event_builder_factory = hs.get_event_builder_factory() |
||||
|
||||
@defer.inlineCallbacks |
||||
def create_room(self, room): |
||||
builder = self.event_builder_factory.new({ |
||||
"type": EventTypes.Create, |
||||
"room_id": room.to_string(), |
||||
"content": {}, |
||||
}) |
||||
|
||||
event, context = yield self.message_handler._create_new_client_event( |
||||
builder |
||||
) |
||||
|
||||
yield self.store.persist_event(event, context) |
||||
|
||||
@defer.inlineCallbacks |
||||
def inject_room_member(self, room, user, membership): |
||||
builder = self.event_builder_factory.new({ |
||||
"type": EventTypes.Member, |
||||
"sender": user.to_string(), |
||||
"state_key": user.to_string(), |
||||
"room_id": room.to_string(), |
||||
"content": {"membership": membership}, |
||||
}) |
||||
|
||||
event, context = yield self.message_handler._create_new_client_event( |
||||
builder |
||||
) |
||||
|
||||
yield self.store.persist_event(event, context) |
||||
|
||||
defer.returnValue(event) |
||||
|
||||
@defer.inlineCallbacks |
||||
def inject_message(self, room, user, body): |
||||
builder = self.event_builder_factory.new({ |
||||
"type": EventTypes.Message, |
||||
"sender": user.to_string(), |
||||
"state_key": user.to_string(), |
||||
"room_id": room.to_string(), |
||||
"content": {"body": body, "msgtype": u"message"}, |
||||
}) |
||||
|
||||
event, context = yield self.message_handler._create_new_client_event( |
||||
builder |
||||
) |
||||
|
||||
yield self.store.persist_event(event, context) |
||||
@ -0,0 +1,116 @@ |
||||
# -*- coding: utf-8 -*- |
||||
# Copyright 2015 OpenMarket Ltd |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
import uuid |
||||
from mock.mock import Mock |
||||
from synapse.types import RoomID, UserID |
||||
|
||||
from tests import unittest |
||||
from twisted.internet import defer |
||||
from tests.storage.event_injector import EventInjector |
||||
|
||||
from tests.utils import setup_test_homeserver |
||||
|
||||
|
||||
class EventsStoreTestCase(unittest.TestCase): |
||||
|
||||
@defer.inlineCallbacks |
||||
def setUp(self): |
||||
self.hs = yield setup_test_homeserver( |
||||
resource_for_federation=Mock(), |
||||
http_client=None, |
||||
) |
||||
self.store = self.hs.get_datastore() |
||||
self.db_pool = self.hs.get_db_pool() |
||||
self.message_handler = self.hs.get_handlers().message_handler |
||||
self.event_injector = EventInjector(self.hs) |
||||
|
||||
@defer.inlineCallbacks |
||||
def test_count_daily_messages(self): |
||||
self.db_pool.runQuery("DELETE FROM stats_reporting") |
||||
|
||||
self.hs.clock.now = 100 |
||||
|
||||
# Never reported before, and nothing which could be reported |
||||
count = yield self.store.count_daily_messages() |
||||
self.assertIsNone(count) |
||||
count = yield self.db_pool.runQuery("SELECT COUNT(*) FROM stats_reporting") |
||||
self.assertEqual([(0,)], count) |
||||
|
||||
# Create something to report |
||||
room = RoomID.from_string("!abc123:test") |
||||
user = UserID.from_string("@raccoonlover:test") |
||||
yield self.event_injector.create_room(room) |
||||
|
||||
self.base_event = yield self._get_last_stream_token() |
||||
|
||||
yield self.event_injector.inject_message(room, user, "Raccoons are really cute") |
||||
|
||||
# Never reported before, something could be reported, but isn't because |
||||
# it isn't old enough. |
||||
count = yield self.store.count_daily_messages() |
||||
self.assertIsNone(count) |
||||
self._assert_stats_reporting(1, self.hs.clock.now) |
||||
|
||||
# Already reported yesterday, two new events from today. |
||||
yield self.event_injector.inject_message(room, user, "Yeah they are!") |
||||
yield self.event_injector.inject_message(room, user, "Incredibly!") |
||||
self.hs.clock.now += 60 * 60 * 24 |
||||
count = yield self.store.count_daily_messages() |
||||
self.assertEqual(2, count) # 2 since yesterday |
||||
self._assert_stats_reporting(3, self.hs.clock.now) # 3 ever |
||||
|
||||
# Last reported too recently. |
||||
yield self.event_injector.inject_message(room, user, "Who could disagree?") |
||||
self.hs.clock.now += 60 * 60 * 22 |
||||
count = yield self.store.count_daily_messages() |
||||
self.assertIsNone(count) |
||||
self._assert_stats_reporting(4, self.hs.clock.now) |
||||
|
||||
# Last reported too long ago |
||||
yield self.event_injector.inject_message(room, user, "No one.") |
||||
self.hs.clock.now += 60 * 60 * 26 |
||||
count = yield self.store.count_daily_messages() |
||||
self.assertIsNone(count) |
||||
self._assert_stats_reporting(5, self.hs.clock.now) |
||||
|
||||
# And now let's actually report something |
||||
yield self.event_injector.inject_message(room, user, "Indeed.") |
||||
yield self.event_injector.inject_message(room, user, "Indeed.") |
||||
yield self.event_injector.inject_message(room, user, "Indeed.") |
||||
# A little over 24 hours is fine :) |
||||
self.hs.clock.now += (60 * 60 * 24) + 50 |
||||
count = yield self.store.count_daily_messages() |
||||
self.assertEqual(3, count) |
||||
self._assert_stats_reporting(8, self.hs.clock.now) |
||||
|
||||
@defer.inlineCallbacks |
||||
def _get_last_stream_token(self): |
||||
rows = yield self.db_pool.runQuery( |
||||
"SELECT stream_ordering" |
||||
" FROM events" |
||||
" ORDER BY stream_ordering DESC" |
||||
" LIMIT 1" |
||||
) |
||||
if not rows: |
||||
defer.returnValue(0) |
||||
else: |
||||
defer.returnValue(rows[0][0]) |
||||
|
||||
@defer.inlineCallbacks |
||||
def _assert_stats_reporting(self, messages, time): |
||||
rows = yield self.db_pool.runQuery( |
||||
"SELECT reported_stream_token, reported_time FROM stats_reporting" |
||||
) |
||||
self.assertEqual([(self.base_event + messages, time,)], rows) |
||||
Loading…
Reference in new issue