-
Notifications
You must be signed in to change notification settings - Fork 0
/
syncer.py
122 lines (110 loc) · 4.5 KB
/
syncer.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
# recursively upload files in image folders to S3(-compatible services)
# (optional: ) and delete source files upon upload success.
# TODO: upload (compressed) data to S3.
import boto3
import os
from configs.config import S3Config as S3Conf
from concurrent.futures import ThreadPoolExecutor
import argparse
from itertools import repeat
session = boto3.session.Session()
client = session.client(
's3',
region_name = S3Conf.region_name,
endpoint_url = S3Conf.endpoint_url,
aws_access_key_id = S3Conf.RW_key,
aws_secret_access_key = S3Conf.RW_secret
)
extra_args_img = {
'ContentType': 'image/jpeg',
'ContentDisposition': 'inline',
'ACL': 'public-read'
}
extra_args_db = {
'ContentType': 'application/octet-stream',
'ContentDisposition': 'attachment',
'ACL': 'public-read'
}
extra_args_log = {
'ContentType': 'text/plain',
'ContentDisposition': 'attachment',
'ACL': 'public-read'
}
# recursively upload files
# 这脚本上传刚写入的图片,故使用写目录。
# TODO: optimize path-joining readability
# Q: why not using os.path.join? A: delimiter between *nix and windows are different, causing mix.
def generateFilePath(dir, having):
for root, _, file_names in os.walk(dir):
# root: 'proj_root\\path1\\path2\\...', delimeter in ['\\', '/'], we normalize to '/'.
root_norm = root.replace('\\','/') # 'proj_root/root/path1/path2/...'
sub_root_norm = '/'.join(root_norm.split('/')[1:]) # 'path1/path2/...'
for file_name in file_names:
local = '/'.join([root_norm, file_name])
remote = '/'.join([sub_root_norm, file_name])
if having in local: yield local, remote
else: continue
#print(local, ' | ', remote)