From 942d8c8b5302f7d38cb7dcb9274f5e79c21bbbae Mon Sep 17 00:00:00 2001 From: Nate Harris Date: Wed, 20 May 2020 16:29:38 -0400 Subject: [PATCH 1/4] Crypt password --- rclone_sa_magic.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/rclone_sa_magic.py b/rclone_sa_magic.py index 2f64c8b..0313e09 100644 --- a/rclone_sa_magic.py +++ b/rclone_sa_magic.py @@ -111,6 +111,9 @@ def parse_args(): parser.add_argument('--cache', action="store_true", help="for test: cache the remote destination.") + parser.add_argument('--password', type=str, default='autorclone', + help='the password for crypt') + args = parser.parse_args() return args @@ -183,14 +186,16 @@ def gen_rclone_cfg(args): # For crypt destination if args.crypt: + password = subprocess.Popen(['rclone', 'obscure', args.password], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) remote_name = '{}{:03d}'.format('dst', i + 1) try: fp.write('[{}_crypt]\n' 'type = crypt\n' 'remote = {}:\n' 'filename_encryption = standard\n' - 'password = hfSJiSRFrgyeQ_xNyx-rwOpsN2P2ZHZV\n' - 'directory_name_encryption = true\n\n'.format(remote_name, remote_name)) + 'password = {}\n' + 'directory_name_encryption = true\n\n'.format(remote_name, remote_name, password)) except: sys.exit("failed to write {} to {}".format(args.destination_id, output_of_config_file)) From f0bb773d031ee31005c2e7bfda911d4d161d930b Mon Sep 17 00:00:00 2001 From: Nate Harris Date: Wed, 20 May 2020 16:34:03 -0400 Subject: [PATCH 2/4] Crypt password --- rclone_sa_magic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rclone_sa_magic.py b/rclone_sa_magic.py index 0313e09..038edb6 100644 --- a/rclone_sa_magic.py +++ b/rclone_sa_magic.py @@ -111,7 +111,7 @@ def parse_args(): parser.add_argument('--cache', action="store_true", help="for test: cache the remote destination.") - parser.add_argument('--password', type=str, default='autorclone', + parser.add_argument('-ps', '--password', type=str, default='autorclone', help='the password for crypt') args = parser.parse_args() From 41a2ff4787ff6eb16454ea52df5862d01549c617 Mon Sep 17 00:00:00 2001 From: Nate Harris Date: Wed, 20 May 2020 18:12:51 -0400 Subject: [PATCH 3/4] Fixed error with password obscuring, crypt working now --- rclone_sa_magic.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/rclone_sa_magic.py b/rclone_sa_magic.py index 038edb6..e88e48a 100644 --- a/rclone_sa_magic.py +++ b/rclone_sa_magic.py @@ -41,6 +41,8 @@ # especially for tasks with a lot of small files TPSLIMIT = 3 TRANSFERS = 3 + + # =================modify here================= @@ -186,8 +188,7 @@ def gen_rclone_cfg(args): # For crypt destination if args.crypt: - password = subprocess.Popen(['rclone', 'obscure', args.password], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + password = obscure(args.password) remote_name = '{}{:03d}'.format('dst', i + 1) try: fp.write('[{}_crypt]\n' @@ -240,6 +241,15 @@ def check_path(path): sys.exit(str(error)) +def obscure(text): + try: + ret = subprocess.check_output('rclone obscure {}'.format(text), + shell=True) + return ret.decode('utf-8').replace('\0', '') + except subprocess.SubprocessError as error: + sys.exit(str(error)) + + def main(): signal(SIGINT, handler) @@ -306,7 +316,8 @@ def main(): if args.dry_run: rclone_cmd += "--dry-run " # --fast-list is default adopted in the latest rclone - rclone_cmd += "--drive-server-side-across-configs --rc --rc-addr=\"localhost:{}\" -vv --ignore-existing ".format(args.port) + rclone_cmd += "--drive-server-side-across-configs --rc --rc-addr=\"localhost:{}\" -vv --ignore-existing ".format( + args.port) rclone_cmd += "--tpslimit {} --transfers {} --drive-chunk-size 32M ".format(TPSLIMIT, TRANSFERS) if args.disable_list_r: rclone_cmd += "--disable ListR " @@ -335,7 +346,8 @@ def main(): already_start = False try: - response = subprocess.check_output('rclone rc --rc-addr="localhost:{}" core/pid'.format(args.port), shell=True) + response = subprocess.check_output('rclone rc --rc-addr="localhost:{}" core/pid'.format(args.port), + shell=True) pid = json.loads(response.decode('utf-8').replace('\0', ''))['pid'] if args.test_only: print('\npid is: {}\n'.format(pid)) @@ -390,7 +402,8 @@ def main(): # except: # print("have some encoding problem to print info") if already_start: - print("%s %dGB Done @ %fMB/s | checks: %d files" % (dst_label, size_GB_done, speed_now, checks_done), end="\r") + print("%s %dGB Done @ %fMB/s | checks: %d files" % (dst_label, size_GB_done, speed_now, checks_done), + end="\r") else: print("%s reading source/destination | checks: %d files" % (dst_label, checks_done), end="\r") @@ -455,4 +468,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() From eea2e38e3eda136c170c48cff6732295829c9547 Mon Sep 17 00:00:00 2001 From: Nate Harris Date: Tue, 4 Aug 2020 21:48:50 -0400 Subject: [PATCH 4/4] Option for custom name for generated rclone file Different from --rclone_config_file, which uses a pre-existing rclone file. A custom file name (rather than just the rclone.conf) allows for mutliple autorclone instances without conflicting config files --- rclone_sa_magic.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/rclone_sa_magic.py b/rclone_sa_magic.py index e88e48a..a1e4ec6 100644 --- a/rclone_sa_magic.py +++ b/rclone_sa_magic.py @@ -99,6 +99,8 @@ def parse_args(): parser.add_argument('-c', '--rclone_config_file', type=str, help='config file path of rclone') + parser.add_argument('-n', '--custom_config_name', type=str, + help='custom name for generated rclone config file') parser.add_argument('-test', '--test_only', action="store_true", help='for test: make rclone print some more information.') parser.add_argument('-t', '--dry_run', action="store_true", @@ -120,9 +122,12 @@ def parse_args(): return args -def gen_rclone_cfg(args): +def gen_rclone_cfg(args, custom_name): sa_files = glob.glob(os.path.join(args.service_account, '*.json')) - output_of_config_file = './rclone.conf' + if custom_name: + output_of_config_file = './{}.conf'.format(custom_name) + else: + output_of_config_file = './rclone.conf' if len(sa_files) == 0: sys.exit('No json files found in ./{}'.format(args.service_account)) @@ -264,7 +269,7 @@ def main(): config_file = args.rclone_config_file if config_file is None: print('generating rclone config file.') - config_file, end_id = gen_rclone_cfg(args) + config_file, end_id = gen_rclone_cfg(args, args.custom_config_name) print('rclone config file generated.') else: return print('not supported yet.')