|
|
|
|
@ -14,12 +14,13 @@ |
|
|
|
|
# See the License for the specific language governing permissions and |
|
|
|
|
# limitations under the License. |
|
|
|
|
|
|
|
|
|
import fnmatch |
|
|
|
|
import imp |
|
|
|
|
import logging |
|
|
|
|
import os |
|
|
|
|
import re |
|
|
|
|
|
|
|
|
|
import attr |
|
|
|
|
|
|
|
|
|
from synapse.storage.engines.postgres import PostgresEngine |
|
|
|
|
|
|
|
|
|
logger = logging.getLogger(__name__) |
|
|
|
|
@ -54,6 +55,10 @@ def prepare_database(db_conn, database_engine, config): |
|
|
|
|
application config, or None if we are connecting to an existing |
|
|
|
|
database which we expect to be configured already |
|
|
|
|
""" |
|
|
|
|
|
|
|
|
|
# For now we only have the one datastore. |
|
|
|
|
data_stores = ["main"] |
|
|
|
|
|
|
|
|
|
try: |
|
|
|
|
cur = db_conn.cursor() |
|
|
|
|
version_info = _get_or_create_schema_state(cur, database_engine) |
|
|
|
|
@ -68,10 +73,16 @@ def prepare_database(db_conn, database_engine, config): |
|
|
|
|
raise UpgradeDatabaseException("Database needs to be upgraded") |
|
|
|
|
else: |
|
|
|
|
_upgrade_existing_database( |
|
|
|
|
cur, user_version, delta_files, upgraded, database_engine, config |
|
|
|
|
cur, |
|
|
|
|
user_version, |
|
|
|
|
delta_files, |
|
|
|
|
upgraded, |
|
|
|
|
database_engine, |
|
|
|
|
config, |
|
|
|
|
data_stores=data_stores, |
|
|
|
|
) |
|
|
|
|
else: |
|
|
|
|
_setup_new_database(cur, database_engine) |
|
|
|
|
_setup_new_database(cur, database_engine, data_stores=data_stores) |
|
|
|
|
|
|
|
|
|
# check if any of our configured dynamic modules want a database |
|
|
|
|
if config is not None: |
|
|
|
|
@ -84,7 +95,7 @@ def prepare_database(db_conn, database_engine, config): |
|
|
|
|
raise |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _setup_new_database(cur, database_engine): |
|
|
|
|
def _setup_new_database(cur, database_engine, data_stores): |
|
|
|
|
"""Sets up the database by finding a base set of "full schemas" and then |
|
|
|
|
applying any necessary deltas. |
|
|
|
|
|
|
|
|
|
@ -115,48 +126,65 @@ def _setup_new_database(cur, database_engine): |
|
|
|
|
current_dir = os.path.join(dir_path, "schema", "full_schemas") |
|
|
|
|
directory_entries = os.listdir(current_dir) |
|
|
|
|
|
|
|
|
|
valid_dirs = [] |
|
|
|
|
pattern = re.compile(r"^\d+(\.sql)?$") |
|
|
|
|
|
|
|
|
|
if isinstance(database_engine, PostgresEngine): |
|
|
|
|
specific = "postgres" |
|
|
|
|
else: |
|
|
|
|
specific = "sqlite" |
|
|
|
|
|
|
|
|
|
specific_pattern = re.compile(r"^\d+(\.sql." + specific + r")?$") |
|
|
|
|
# First we find the highest full schema version we have |
|
|
|
|
valid_versions = [] |
|
|
|
|
|
|
|
|
|
for filename in directory_entries: |
|
|
|
|
match = pattern.match(filename) or specific_pattern.match(filename) |
|
|
|
|
abs_path = os.path.join(current_dir, filename) |
|
|
|
|
if match and os.path.isdir(abs_path): |
|
|
|
|
ver = int(match.group(0)) |
|
|
|
|
if ver <= SCHEMA_VERSION: |
|
|
|
|
valid_dirs.append((ver, abs_path)) |
|
|
|
|
else: |
|
|
|
|
logger.debug("Ignoring entry '%s' in 'full_schemas'", filename) |
|
|
|
|
try: |
|
|
|
|
ver = int(filename) |
|
|
|
|
except ValueError: |
|
|
|
|
continue |
|
|
|
|
|
|
|
|
|
if not valid_dirs: |
|
|
|
|
if ver <= SCHEMA_VERSION: |
|
|
|
|
valid_versions.append(ver) |
|
|
|
|
|
|
|
|
|
if not valid_versions: |
|
|
|
|
raise PrepareDatabaseException( |
|
|
|
|
"Could not find a suitable base set of full schemas" |
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
max_current_ver, sql_dir = max(valid_dirs, key=lambda x: x[0]) |
|
|
|
|
max_current_ver = max(valid_versions) |
|
|
|
|
|
|
|
|
|
logger.debug("Initialising schema v%d", max_current_ver) |
|
|
|
|
|
|
|
|
|
directory_entries = os.listdir(sql_dir) |
|
|
|
|
# Now lets find all the full schema files, both in the global schema and |
|
|
|
|
# in data store schemas. |
|
|
|
|
directories = [os.path.join(current_dir, str(max_current_ver))] |
|
|
|
|
directories.extend( |
|
|
|
|
os.path.join( |
|
|
|
|
dir_path, |
|
|
|
|
"data_stores", |
|
|
|
|
data_store, |
|
|
|
|
"schema", |
|
|
|
|
"full_schemas", |
|
|
|
|
str(max_current_ver), |
|
|
|
|
) |
|
|
|
|
for data_store in data_stores |
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
directory_entries = [] |
|
|
|
|
for directory in directories: |
|
|
|
|
directory_entries.extend( |
|
|
|
|
_DirectoryListing(file_name, os.path.join(directory, file_name)) |
|
|
|
|
for file_name in os.listdir(directory) |
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
if isinstance(database_engine, PostgresEngine): |
|
|
|
|
specific = "postgres" |
|
|
|
|
else: |
|
|
|
|
specific = "sqlite" |
|
|
|
|
|
|
|
|
|
for filename in sorted( |
|
|
|
|
fnmatch.filter(directory_entries, "*.sql") |
|
|
|
|
+ fnmatch.filter(directory_entries, "*.sql." + specific) |
|
|
|
|
): |
|
|
|
|
sql_loc = os.path.join(sql_dir, filename) |
|
|
|
|
logger.debug("Applying schema %s", sql_loc) |
|
|
|
|
executescript(cur, sql_loc) |
|
|
|
|
directory_entries.sort() |
|
|
|
|
for entry in directory_entries: |
|
|
|
|
if entry.file_name.endswith(".sql") or entry.file_name.endswith( |
|
|
|
|
".sql." + specific |
|
|
|
|
): |
|
|
|
|
logger.debug("Applying schema %s", entry.absolute_path) |
|
|
|
|
executescript(cur, entry.absolute_path) |
|
|
|
|
|
|
|
|
|
cur.execute( |
|
|
|
|
database_engine.convert_param_style( |
|
|
|
|
"INSERT INTO schema_version (version, upgraded)" " VALUES (?,?)" |
|
|
|
|
"INSERT INTO schema_version (version, upgraded) VALUES (?,?)" |
|
|
|
|
), |
|
|
|
|
(max_current_ver, False), |
|
|
|
|
) |
|
|
|
|
@ -168,6 +196,7 @@ def _setup_new_database(cur, database_engine): |
|
|
|
|
upgraded=False, |
|
|
|
|
database_engine=database_engine, |
|
|
|
|
config=None, |
|
|
|
|
data_stores=data_stores, |
|
|
|
|
is_empty=True, |
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
@ -179,6 +208,7 @@ def _upgrade_existing_database( |
|
|
|
|
upgraded, |
|
|
|
|
database_engine, |
|
|
|
|
config, |
|
|
|
|
data_stores, |
|
|
|
|
is_empty=False, |
|
|
|
|
): |
|
|
|
|
"""Upgrades an existing database. |
|
|
|
|
@ -248,24 +278,51 @@ def _upgrade_existing_database( |
|
|
|
|
for v in range(start_ver, SCHEMA_VERSION + 1): |
|
|
|
|
logger.info("Upgrading schema to v%d", v) |
|
|
|
|
|
|
|
|
|
delta_dir = os.path.join(dir_path, "schema", "delta", str(v)) |
|
|
|
|
# We need to search both the global and per data store schema |
|
|
|
|
# directories for schema updates. |
|
|
|
|
|
|
|
|
|
try: |
|
|
|
|
directory_entries = os.listdir(delta_dir) |
|
|
|
|
except OSError: |
|
|
|
|
logger.exception("Could not open delta dir for version %d", v) |
|
|
|
|
raise UpgradeDatabaseException( |
|
|
|
|
"Could not open delta dir for version %d" % (v,) |
|
|
|
|
# First we find the directories to search in |
|
|
|
|
delta_dir = os.path.join(dir_path, "schema", "delta", str(v)) |
|
|
|
|
directories = [delta_dir] |
|
|
|
|
for data_store in data_stores: |
|
|
|
|
directories.append( |
|
|
|
|
os.path.join( |
|
|
|
|
dir_path, "data_stores", data_store, "schema", "delta", str(v) |
|
|
|
|
) |
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
# Now find which directories have anything of interest. |
|
|
|
|
directory_entries = [] |
|
|
|
|
for directory in directories: |
|
|
|
|
logger.debug("Looking for schema deltas in %s", directory) |
|
|
|
|
try: |
|
|
|
|
file_names = os.listdir(directory) |
|
|
|
|
directory_entries.extend( |
|
|
|
|
_DirectoryListing(file_name, os.path.join(directory, file_name)) |
|
|
|
|
for file_name in file_names |
|
|
|
|
) |
|
|
|
|
except FileNotFoundError: |
|
|
|
|
# Data stores can have empty entries for a given version delta. |
|
|
|
|
pass |
|
|
|
|
except OSError: |
|
|
|
|
logger.exception("Could not open delta dir for version %d", v) |
|
|
|
|
raise UpgradeDatabaseException( |
|
|
|
|
"Could not open delta dir for version %d" % (v,) |
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
if not directory_entries: |
|
|
|
|
continue |
|
|
|
|
|
|
|
|
|
directory_entries.sort() |
|
|
|
|
for file_name in directory_entries: |
|
|
|
|
for entry in directory_entries: |
|
|
|
|
file_name = entry.file_name |
|
|
|
|
relative_path = os.path.join(str(v), file_name) |
|
|
|
|
absolute_path = entry.absolute_path |
|
|
|
|
|
|
|
|
|
logger.debug("Found file: %s", relative_path) |
|
|
|
|
if relative_path in applied_delta_files: |
|
|
|
|
continue |
|
|
|
|
|
|
|
|
|
absolute_path = os.path.join(dir_path, "schema", "delta", relative_path) |
|
|
|
|
root_name, ext = os.path.splitext(file_name) |
|
|
|
|
if ext == ".py": |
|
|
|
|
# This is a python upgrade module. We need to import into some |
|
|
|
|
@ -448,3 +505,13 @@ def _get_or_create_schema_state(txn, database_engine): |
|
|
|
|
return current_version, applied_deltas, upgraded |
|
|
|
|
|
|
|
|
|
return None |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@attr.s() |
|
|
|
|
class _DirectoryListing(object): |
|
|
|
|
"""Helper class to store schema file name and the |
|
|
|
|
absolute path to it. |
|
|
|
|
""" |
|
|
|
|
|
|
|
|
|
file_name = attr.ib() |
|
|
|
|
absolute_path = attr.ib() |
|
|
|
|
|
Erik Johnston
5 years ago