summaryrefslogtreecommitdiff
path: root/linaro-cp.py
blob: 46a921c398d98c9f8e906cc749e108f4eaa4b21d (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
#!/usr/bin/env python

import argparse
import atexit
import mimetypes
import os
import pprint
import re
import requests
import sys
import tempfile
import time
from multiprocessing.pool import ThreadPool

# Public artifacts BUILD-INFO.txt
build_info = 'Format-Version: 0.5\n\nFiles-Pattern: *\nLicense-Type: open\n'


class API_v1(object):
    def __init__(self, server, build_info, api_key):
        self.server = server
        self.api_base = server
        self.build_info = build_info
        self.api_key = api_key

    def _upload_data(self, url, data=None, files=None, headers=None, retry_count=5):
        for x in range(retry_count):
            try:
                resp = requests.post(url, headers=headers, data=data, files=files, timeout=61)
                if resp.status_code in (200, 201):
                    return resp
                else:
                    print("Unsuccessful status:", resp.status_code)
            except Exception as e:
                print(e)
            print('Upload failed for %s, retry attempt %s' % (url, x))
            time.sleep(2 * x + 1)
            retry_count -= 1

        raise Exception("LLP request failed")

    def upload_file(self, url, filename):
        with open(filename, 'rb') as f:
            self._upload_data(url, files={'file': f}, data={'key': self.api_key})

    def upload_transfer_queue(self, transfer_queue):
        p = ThreadPool(10)
        transfer_failures = []

        results = [p.apply_async(self.upload_file, (transfer_item, transfer_queue[transfer_item])) for transfer_item in
                   transfer_queue]

        for res in results:
            try:
                res.get()
            except Exception as e:
                transfer_failures.append(str(e))
        return transfer_failures

    def get_transfer_queue(self, src, dst, options):
        if self.api_base[-1] != '/' and dst[0] != '/':
            # one of these needs to be a slash to produce a url
            dst = '/' + dst

        transfer_queue = {}
        src_dir = os.path.abspath(src)

        if not os.path.isdir(src_dir):
            transfer_queue[self.api_base + dst + "/" + os.path.basename(src)] = src_dir
            return transfer_queue

        manifest = None
        if options.manifest:
            manifest = tempfile.NamedTemporaryFile(prefix="MANIFEST", delete=False)

        for root, dirs, files in os.walk(src_dir):
            rel_dir = root[len(src_dir) + 1:]
            for f in files:
                rel_path = os.path.join(rel_dir, f)
                # If there's at least one include pattern, skip unless
                # some pattern matches
                skip = len(options.include) > 0
                for pat in options.include:
                    if re.search(pat, rel_path):
                        skip = False
                        break
                if not skip:
                    dst_file = '%s%s/%s' % (self.api_base, dst, rel_path)
                    transfer_queue[dst_file] = os.path.join(root, f)
                    if manifest:
                        manifest.write(rel_path + "\n")
            if not options.no_build_info:
                if rel_dir and rel_dir[-1] != '/':
                    rel_dir += '/'
                build_info_file = os.path.join(root, 'BUILD-INFO.txt')
                if not os.path.exists(build_info_file):
                    dst_file = '%s%s/%s%s' % (
                        self.api_base, dst, rel_dir, 'BUILD-INFO.txt')
                    transfer_queue[dst_file] = self.build_info
        if manifest:
            manifest.close()
            dst_file = '%s%s/%s' % (
                self.api_base, dst, 'MANIFEST')
            transfer_queue[dst_file] = manifest.name
        return transfer_queue

    def upload(self, src, dst, options):
        transfer_queue = self.get_transfer_queue(src, dst, options)
        if not transfer_queue:
            print("Warning: no files to publish")
        if options.verbose:
            pprint.pprint(transfer_queue)
        if not options.dry_run:
            return self.upload_transfer_queue(transfer_queue)
        else:
            return []


class API_v2(API_v1):
    def __init__(self, server, build_info, api_key):
        super(API_v2, self).__init__(server, build_info, api_key)
        self.api_base = server + '/api/v2/publish/'

    def upload_file(self, url, filename):
        with open(filename, 'rb') as f:
            self._upload_data(url, files={'file': f}, headers={'AuthToken': self.api_key})

    def link_latest(self, dst):
        url = self.server + '/api/v2/link_latest/' + dst
        self._upload_data(url, headers={'AuthToken': self.api_key})


class API_v3(API_v1):
    def __init__(self, server, build_info, api_key):
        super(API_v3, self).__init__(server, build_info, api_key)
        self.api_base = server + '/api/v3/publish/'

    def _put_s3(self, url, filename, mtype, retry_count=5):
        size = os.path.getsize(filename)
        headers = {'Content-Type': mtype, 'Content-Length': str(size)}
        for x in range(retry_count):
            try:
                with open(filename, 'rb') as f:
                    resp = requests.put(url, headers=headers, data=f, timeout=61)
                if resp.status_code in (200, 201):
                    return
                else:
                    print("Unsuccessful status:", resp.status_code)
            except Exception as e:
                print(e)
            print('Upload failed for %s, retry attempt %s' % (url, x))
            time.sleep(2 * x + 1)
            retry_count -= 1

        raise Exception("S3 upload failed")

    def upload_file(self, url, filename):
        # ask llp for an s3 tempurl:
        mtype = mimetypes.guess_type(filename)[0]
        if not mtype:
            mtype = 'other'

        data = {'Content-Type': mtype}
        headers = {'AuthToken': self.api_key}

        error = None
        for x in range(3):
            try:
                # get the tempurl
                resp = self._upload_data(url, data=data, headers=headers)
                location = resp.headers['location']
                # publish file
                self._put_s3(location, filename, mtype)
                return
            except Exception as e:
                time.sleep(3)
                error = e
        raise error

    def link_latest(self, dst):
        url = self.server + '/api/v3/link_latest/' + dst
        self._upload_data(url, headers={'AuthToken': self.api_key})


def main():
    parser = argparse.ArgumentParser(
        description='Copy file(s) from source to destination')
    parser.add_argument('-k', '--key', help='key used for the copy')
    parser.add_argument('-a', '--api_version', choices=('3'), default='3',
                        help='API version to use. default=%(default)s')
    parser.add_argument('--server', default='http://snapshots.linaro.org/',
                        help='Publishing API server. default=%(default)s')
    parser.add_argument('-i', '--include', action='append', default=[],
                        help='Include regex for files')
    parser.add_argument('--no-build-info', action='store_true',
                        help="Don't auto-generate BUILD-INFO.txt")
    parser.add_argument('-b', '--build-info', help='Custom build-info file')
    parser.add_argument('--link-latest', action='store_true',
                        help='''Create symlink for "latest" to point to this
                             build.''')
    parser.add_argument('--split-job-owner', action='store_true',
                        help='Split Jenkins job owner in dst (owner_job -> ~owner/job)')
    parser.add_argument('--manifest', action='store_true',
                        help='Generate MANIFEST file with list of all files published')
    parser.add_argument('--verbose', action='store_true',
                        help='Verbose operation')
    parser.add_argument('--dry-run', action='store_true',
                        help="Don't actually publish files")
    g = parser.add_mutually_exclusive_group(required=True)
    g.add_argument('--make-link', action='store_true', required=False,
                   help='Don\'t publish files, just create "latest"')
    g.add_argument('src', nargs='?', help='source directory with files to publish')
    parser.add_argument('dst', help='destination to publish to')

    arguments = parser.parse_args()
    # Publish key is required. Fallback to PUBLISH_KEY environment
    # variable when it isn't passed as an argument
    if arguments.key:
        key = arguments.key
    else:
        key = os.environ.get('PUBLISH_TOKEN')
        if key:
            if arguments.api_version == '1':
                # Default to api v2 if not specified
                arguments.api_version = '2'
        else:
            key = os.environ.get('PUBLISH_KEY')
            if key is None:
                sys.exit('Error: Key is not defined.')

    if not arguments.build_info:
        fd, arguments.build_info = tempfile.mkstemp(prefix='BUILD-INFO.txt')
        atexit.register(os.unlink, arguments.build_info)
        os.write(fd, build_info)

    cls = globals()['API_v' + arguments.api_version]
    api = cls(arguments.server, arguments.build_info, key)

    if arguments.split_job_owner:
        # Rewrite job path .../owner_jobname/123 -> .../~owner/jobname/123 ,
        # as required for android-build publishing.
        arguments.dst = re.sub(r"^(.+?)/([^/]+?)_([^/]+?)/([0-9]+)/?$", r"\1/~\2/\3/\4/", arguments.dst)

    transfer_failures = []
    if not arguments.make_link:
        transfer_failures = api.upload(arguments.src, arguments.dst, arguments)

    if arguments.link_latest or arguments.make_link:
        err = api.link_latest(arguments.dst)
        if err:
            transfer_failures.append('unable to create symlink: ' + err)

    if len(transfer_failures) > 0:
        sys.exit('Error: Failed to transfer:\n  %s' % '\n  '.join(transfer_failures))


if __name__ == '__main__':
    main()