1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
|
#!/usr/bin/python3
# Checks the master manifest and ensures the local manifest is up-to-date
import codecs
import contextlib
import gzip
import json
import time
import urllib.request
@contextlib.contextmanager
def local_manifest(manifile):
reader = codecs.getreader("utf-8")
with gzip.open(manifile) as f:
manifest = json.load(reader(f))
yield manifest
def remote_manifest(url):
reader = codecs.getreader("utf-8")
tries = 3
for x in range(tries):
try:
remote = urllib.request.urlopen(url)
with gzip.open(remote) as f:
return json.load(reader(f))
except:
if x + 1 == tries:
raise
time.sleep(x+1)
def _handle_deletes(lrepos, rrepos):
# we don't know when repos are deleted, so we use a local temp file to
# track deletes we've missed, we allow one miss before issuing a warning
try:
with open('/tmp/grok-check-manifest-deletes.json') as f:
to_del_local = json.load(f)
except:
to_del_local = []
deletes = lrepos - rrepos
with open('/tmp/grok-check-manifest-deletes.json', 'w') as f:
json.dump(list(deletes), f)
to_del = []
for r in deletes:
if r in to_del_local:
to_del.append(r) # we should have deleted by now
return to_del
def compare_manifests(local, remote):
# ensure they have the same repos defined
lrepos = set(sorted(local.keys()))
rrepos = set(sorted(remote.keys()))
to_del = _handle_deletes(lrepos, rrepos)
to_add = []
for r in rrepos - lrepos:
# give newly added repos a little slack for cloning over
elapsed = time.time() - remote[r]['modified']
if elapsed > 120:
to_add.append(r)
# test fingerprints match
bad = []
for repo in lrepos & rrepos:
rem_fp = remote[repo]['fingerprint']
local_fp = local[repo]['fingerprint']
if rem_fp != local_fp:
elapsed = time.time() - remote[repo]['modified']
threshold = 90 # normally 90 seconds is enough
if 'kernel.git' in repo or 'linux.git' in repo:
threshold = 240 # give more time for kernel syncs
if elapsed > threshold:
bad.append('%s rem=%s us=%s' % (repo, rem_fp, local_fp))
return to_add, to_del, bad
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser(
description='''Verify local repositories are in sync by checking
remote server's manifest''')
parser.add_argument('--local', default='/srv/repositories/manifest.js.gz',
help='Local manifest to analyze. default=%(default)s')
parser.add_argument('manifest_url', help='''Manifest url. eg
http://git-us.linaro.org/manifest.js.gz''')
args = parser.parse_args()
remote = remote_manifest(args.manifest_url)
with local_manifest(args.local) as local:
to_add, to_del, to_sync = compare_manifests(local, remote)
if to_add:
print('Missing repos:\n %s' % '\n '.join(to_add))
if to_del:
print('Delete repos:\n %s' % '\n '.join(to_del))
if to_sync:
print('Repos out sync:')
print(' ' + '\n '.join(to_sync))
|