from json import loads from subprocess import run, PIPE from argparse import ArgumentParser, ArgumentTypeError from datetime import datetime, timedelta, timezone from sys import stdout as sys_stdout from re import compile as re_compile, Pattern def run_command(args: list, remote: str = None, ssh_args: list = None, check: bool = True, stdout: int = PIPE, stderr: int = PIPE): if remote: args = ["ssh", *ssh_args, remote] + args return run(args, check=check, stdout=stdout, stderr=stderr) def occ(php: str, path: str, args: list, remote: str = None, ssh_args: list = None, json: bool = True): base_args = ["sudo", "-u", "www-data", "PHP_MEMORY_LIMIT=1G", php, f"{path}/occ", "--no-interaction", "--no-ansi", "--no-warnings"] if json: base_args += ["--output=json"] out = run_command(base_args + args, remote, ssh_args).stdout.decode().strip() if json: return loads(out) return out def get_users(php: str, path: str, remote: str = None, ssh_args: list = None, last_seen_limit: int = None, exclude_users: Pattern = None) -> list: users = {} offset = 0 while True: new_users = occ(php, path, ["user:list", "--info", "--offset", str(offset)], remote=remote, ssh_args=ssh_args) if not new_users: break users.update(new_users) offset = len(users) print(f"Got {offset} users") users = {k: v for k, v in users.items() if v["email"]} print(f"Got {len(users)} email filtered users") if exclude_users: users = {k: v for k, v in users.items() if not exclude_users.findall(k)} print(f"Got {len(users)} filtered users") if last_seen_limit: users = {k: v for k, v in users.items() if datetime.fromisoformat(v["last_seen"]) >= last_seen_limit} print(f"Got {len(users)} time filtered users") return users def main(src_path: str, dest_path: str, src_php: str, dest_php: str, src_remote: str = None, dest_remote: str = None, ssh_args: list = None, rsync_path: str = None, days: int = 0, exclude_users: str = None): if days != 0: last_seen_limit = datetime.now(timezone.utc)\ .replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(days=days) else: last_seen_limit = None if exclude_users: exclude_users = re_compile(exclude_users) src_data = occ(src_php, src_path, ["config:list", "--private"], remote=src_remote, ssh_args=ssh_args)["system"]["datadirectory"] dest_data = occ(dest_php, dest_path, ["config:list", "--private"], remote=dest_remote, ssh_args=ssh_args)["system"]["firstrunmigrate_dir"] users = get_users(src_php, src_path, src_remote, ssh_args, last_seen_limit, exclude_users) max_users = len(users) for idx, user in enumerate(users): src_user_files = f"{src_data}/{user}/files/" print(f"{'='*20}[{user} ({idx+1}/{max_users})]{'='*20}") sys_stdout.flush() # Check if user folder exist if run_command(["sudo", "-u", "www-data", "test", "-d", src_user_files], src_remote, ssh_args, check=False).returncode != 0: print(f"No files... ({src_user_files})") continue run(["rsync", "-avzPh", "--del", *((f"--rsync-path={rsync_path}",) if rsync_path else ()), "-e", f"ssh {' '.join(ssh_args)}", f"{f'{src_remote}:' if src_remote else ''}{src_user_files}", f"{f'{dest_remote}:' if dest_remote else ''}{dest_data}/{users[user]['email']}/"], check=True, stdout=None, stderr=None) if __name__ == '__main__': parser = ArgumentParser(prog="user_data_migrate") parser.add_argument("--src-path", default="/var/www/nextcloud") parser.add_argument("--dest-path", default="/var/www/nextcloud") parser.add_argument("--src-php", default="php7.4") parser.add_argument("--dest-php", default="php7.4") parser.add_argument("--src-remote", default=None) parser.add_argument("--dest-remote", default=None) parser.add_argument("--ssh-args", default=[]) parser.add_argument("--rsync-path", default=None) parser.add_argument("--days", "-d", type=int, default="0") parser.add_argument("--exclude-users", "-e", default=None) parser_args = parser.parse_args() if parser_args.src_remote and parser_args.dest_remote: raise ArgumentTypeError("Can't have two remote (rsync limitation)") if parser_args.ssh_args: parser_args.ssh_args = parser_args.ssh_args.split(" ") main(parser_args.src_path, parser_args.dest_path, parser_args.src_php, parser_args.dest_php, parser_args.src_remote, parser_args.dest_remote, parser_args.ssh_args, parser_args.rsync_path, parser_args.days, parser_args.exclude_users)