path: root/bin/glance

#!/usr/bin/env python3

'''
glance - JIRA to JSON and JSON data extraction

Prerequisites (ubuntu:16.04):

 * sudo apt -y install python3 python3-pip
 * sudo apt -y install python3-iso8601 python3-keyring
 * sudo pip3 install jira

'''

import argparse
from jira.client import JIRA
import json
import iso8601
import os
import re
import textwrap
import sys

import toys.collect as collect
import toys.config as config
import toys.date as date

# Let's make the plotting tools optional (for those who only use glimpse
# for their weekly report)
try:
	import toys.chart as chart
	import matplotlib.pyplot as plt
except:
	pass

# If it's installed we'd rather use IPython for interaction...
try:
	import IPython
	interact = IPython.embed
except:
	import pdb
	interact = pdb.set_trace

#
# Collection of find/replace strings to massage a summary for
#  readability.
#
# Ideally all we would do with hacks is strip our redundant idioms
# such as including the issuetype in the summary.
#
# However for now we go a bit deeper and try to remove other
# redundant information from the summary (such as member name) or
# excessive use of verbs.
#
hacks = (
	('WIFI', 'WiFi'),
	('wifi', 'WiFi'),
)

component_aliases = {
	'Android-upstreaming' : 'Linaro'
}

component_categories = (
	'BSP Analysis', 'Engineering works', 'Member Build', 'LAVA', 'Training', 'Upstream Consultancy'
)

maybe_component_categories = (
	'96Boards', 'LDTS', 'Metis', 'Linaro'
)

def warn(issue, msg):
	lines = textwrap.wrap('{} {}'.format(issue['url'], msg),
			initial_indent=   'WARNING: ',
			subsequent_indent='         ')
	print('\n'.join(lines), file=sys.stderr)

class Issue(dict):
	@staticmethod
	def wrap(jira, issue):
		'''Factory method to turn a JIRA issue into a JSON one'''
		self = Issue(issue.raw)

		summary = self['fields']['summary']
		for (old, new) in hacks:
			summary = summary.replace(old, new)
		self['summary'] = summary.strip()

		# This should probably transition into a method
		self['url'] = 'https://projects.linaro.org/browse/' + self['key']

		if self.is_story():
			self['parent'] = self['fields']['customfield_10005']
		#elif self.is_subtask():
		#	# Canonical form for parent: <JIRA Issue: key=u'KWG-56', id=u'13944'>
		#	self['parent'] = issue['fieldsl#].parent.key


		# Extract and attach the worklogs for this issue
		self['worklog'] = [ w.raw for w in jira.worklogs(self['key']) ]
		for w in self['worklog']:
			for (old, new) in hacks:
				w['comment'] = w['comment'].replace(old, new)

		return self

	@staticmethod
	def fetch(since, constraint=None):
		cfg = config.get_config()
		password = config.get_password(cfg, 'jira')

		query = 'project = "Support and Solutions Engineering"'
		query += ' AND (statusCategory != Done OR updatedDate >= "{}")'.format(since.strftime("%Y/%m/%d %H:%M"))
		if constraint:
			query += 'AND ({})'.format(constraint)

		jira = JIRA(options={'server': cfg['jira']['server']},
			basic_auth=(cfg['jira']['username'], password))

		issues = []
		new_issues = [ Issue.wrap(jira, i) \
				for i in jira.search_issues(query) ]

		while len(new_issues):
			issues += new_issues
			new_issues = []

			keys = set([ i['key'] for i in issues ])

			for issue in list(issues):
				if issue['key'].startswith('PSE'):
					for link in issue['fields']['issuelinks']:
						if 'outwardIssue' in link:
							rel = link['outwardIssue']
						else:
							rel = link['inwardIssue']
						if rel['key'] not in keys:
							j = jira.issue(rel['key'])
							new_issues.append(Issue.wrap(jira, j))
							keys.add(rel['key'])
				if 'parent' in issue:
					if issue['parent'] and issue['parent'] not in keys:
						try:
							j = jira.issue(issue['parent'])
							new_issues.append(Issue.wrap(jira, j))
							keys.add(issue['parent'])
						except:
							warn(issue, "cannot fetch parent '{}'".format(issue['parent']))

		return issues

	@staticmethod
	def load(obj):
		if not obj:
			data = json.load(sys.stdin)
		elif isinstance(obj, str):
			with open(obj, 'r') as f:
				data = json.load(f)
		else:
			data = json.load(obj)
		return [ Issue(i) for i in data ]


	def is_story(self):
		return self['fields']['issuetype']['name'] == 'Story'

	def is_subtask(self):
		'''A sub-task is similar to a blueprint but has a proper
		parent-child relationship with the engineering card.
		'''
		return self['fields']['issuetype']['name'] == 'Sub-task'

	def is_epic(self):
		# PSE uses 'Epic' but OCE uses 'Initiative' :-(
		return self['fields']['issuetype']['name'] == 'Epic' or \
	               self['fields']['issuetype']['name'] == 'Initiative'

	def categorize(self):
		lookup = {
			'Open': (),
			'TODO': ('Plan',),
			'In Progress' : ('Plan', 'Progress'),
			'Resolved' : ('Progress',),
			'Closed' : ('Progress',),
		}

		if self['fields']['status']['name'] in lookup:
			return set(lookup[self['fields']['status']['name']])

		warn(self, 'has bad status ({})'.format(self['fields']['status']['name']))
		return set()

	def date(self, key):
		if key not in self['fields']:
			return None

		if not self['fields'][key]:
			return None

		return iso8601.parse_date(self['fields'][key])

	def field(self, f):
		'''Searches both top-level and fields automatically.'''
		if f in self:
			return self[f]
		else:
			return self['fields'][f]

	def components(self):
		components = []
		if not self['key'].startswith('PSE-'):
			components += ('Linaro', 'Engineering works')
		for c in self['fields']['components']:
			name = c['name']
			if name in component_aliases:
				name = component_aliases[name]
			components.append(name)
		return components

	def has_component(self, needle, strict=False):
		haystack = self.components()
		if not strict:
			needle = needle.lower()
			haystack = [ hay.lower() for hay in haystack ]
		return needle in haystack

	def get_component(self):
		components = self.components()
		for p in component_categories + maybe_component_categories:
			if p in components:
				return p

		warn(self, 'does not match the component categories')
		return self['key'] + ': No category tag'

	def get_member(self):
		components = self.components()
		for p in component_categories:
			if p in components:
				components.remove(p)
		for p in maybe_component_categories:
			if len(components) < 2:
				break
			if p in components:
				components.remove(p)

		if len(components) == 0:
			warn(self, 'has no member')
			return self['key'] + ': No member tag'
		elif len(components) >= 2:
			warn(self, 'has too many members '+
					'(incomplete category list?)')
			return self['key'] + ': Too many member tags'

		return components[0]

class Worklog(dict):
	re_progress = re.compile('^(h[123456]\.|#+)?\s*[Pp]rogress\s*')
	re_plans = re.compile('^(h[123456]\.|#+)?\s*[Pp]lans\s*')
	re_split_worklog = re.compile('\n+ *[*+-]+  *|^ *[*+-]+  *|\n\n+')

	def date(self, key):
		if key not in self:
			return None

		if not self[key]:
			return None

		return iso8601.parse_date(self[key])

	def parse(self):
		progress = []
		plans = []
		active = progress

		comment = self['comment'].replace('\r', '')

		for ln in re.split(self.re_split_worklog, self['comment'].replace('\r', '')):
			ln = ln.replace('\n', ' ').strip()

			m = re.match(self.re_progress, ln)
			if m:
				ln = ln[m.end():]

			m = re.match(self.re_plans, ln)
			if m:
				active = plans
				ln = ln[m.end():]

			if ln:
				active.append(ln)

		return (progress, plans)


class Report(object):
	def __init__(self, issues=[]):
		self.issues = {}
		self.cards = {}
		self.blueprints = {}
		self.members = {}

		for i in issues:
			self.add(i)
		self.link_blueprints()

	def add(self, issue):
		self.issues[issue['key']] = issue

		if issue.is_story() or issue.is_subtask():
			self.blueprints[issue['key']] = issue
		elif issue.is_epic():
			self.cards[issue['key']] = issue

			components = []
			for c in issue['fields']['components']:
				name = c['name']
				if name in component_aliases:
					name = component_aliases[name]
				components.append(name)

			name = issue.get_member()
			if name not in self.members:
				self.members[name] = []
			self.members[name].append(issue)

			issue['blueprints'] = []
		else:
			warn(issue, 'has unexpected issuetype {}'.format(
				issue['fields']['issuetype']['name']))

	def link_blueprints(self):
		'''Iterate over the blueprints and link them to their cards.'''
		for b in self.blueprints.values():
			if 'parent' not in b:
				warn(b, 'is not linked to an EPIC')
				continue
			if b['parent'] not in self.cards:
				warn(b, 'is linked to non-existant {}'.format(b['parent']))
				continue

			card = self.cards[b['parent']]
			card['blueprints'].append(b)

	def assignees(self, key, recurse=False):
		issues = [ self.issues[key], ]
		if recurse:
			issues += issues[0]['blueprints']

		assignees = set()
		for i in issues:
			if i['fields']['assignee']:
				assignees.add(i['fields']['assignee']['displayName'])
			else:
				assignees.add("Noone")

		return [ a for a in sorted(assignees) ]

	def worklog(self, key=None, recurse=False):
		'''Fetch worklogs held within the report.'''
		if key:
			issues = [ self.issues[key], ]
			if recurse:
				issues += issues[0]['blueprints']
		else:
			issues = self.issues.values()

		worklog = []
		for i in issues:
			for w in i['worklog']:
				# This does alter the data but the change is
				# additive so it is harmless if we emit the
				# resulting structures.
				w['issue'] = i['key']
			worklog += i['worklog']

		return [ Worklog(w) for w in sorted(worklog, key=lambda x: x['started']) ]

def do_chart(args):
	issues = Issue.load(args.json)
	report = Report(issues)
	worklog = report.worklog()

	if not args.barchart and not args.piechart:
		args.barchart = True

	# Functions to parse worklog data
	collate_by_week = lambda w: w.date('started').strftime('%Y-%U')
	collate_by_month = lambda w: w.date('started').strftime('%Y-%m')
	collate_by_engineer = lambda w: w['author']['displayName']
	collate_by_member = lambda w: report.issues[w['issue']].get_member()
	collate_by_component = lambda w: report.issues[w['issue']].get_component()
	count_effort = lambda w: w['timeSpentSeconds'] / 3600

	# No barchart variant for --count-by-member because the collation is
	# rather difficult (need to keep all worklogs and count once (and only
	# once) for worklogs in a time interval. This means neither traversing
	# by issue nor traversing by worklog can give us the count we want.
	if args.count_by_member and args.piechart:
		# Second lambda counts 1 is the card has a worklog, 0 otherwise.
		data = collect.accumulate(issues,
				lambda i: i.get_member(),
				lambda i: int(bool(len(i['worklog']))))
		chart.piechart(data, args.count_by_member)

	if args.effort_by_engineer and args.barchart:
		data = collect.accumulate_2d(worklog,
				collate_by_week, collate_by_engineer, count_effort)
		chart.stacked_barchart(data, args.effort_by_engineer,
				title = 'Effort by week and assigned engineer',
				xlabel = 'Year and week number',
				ylabel = 'Effort (man/hours)')

	if args.effort_by_engineer and args.piechart:
		data = collect.accumulate(worklog, collate_by_engineer,
				count_effort)
		chart.piechart(data, args.effort_by_engineer)


	if args.effort_by_member and args.barchart:
		data = collect.accumulate_2d(worklog,
				collate_by_month, collate_by_member, count_effort)
		chart.stacked_barchart(data, args.effort_by_member,
				title = 'Effort by month and member',
				ylabel = 'Effort (man/hours)')

	if args.effort_by_member and args.piechart:
		data = collect.accumulate(worklog, collate_by_member, count_effort)
		chart.piechart(data, args.effort_by_member)

	if args.effort_by_component and args.barchart:
		data = collect.accumulate_2d(worklog,
				collate_by_month, collate_by_component, count_effort)
		chart.stacked_barchart(data, args.effort_by_component,
				title = 'Effort by month and component',
				ylabel = 'Effort (man/hours)')

	if args.effort_by_component and args.piechart:
		data = collect.accumulate(worklog, collate_by_component, count_effort)
		chart.piechart(data, args.effort_by_component)

	if args.card_tracker:
		since = date.smart_parse(args.since)
		until = date.smart_parse(args.until)

		all_cards = report.issues.values()
		closed_cards = [ c for c in all_cards if c.date('resolutiondate') ]

		created_by_month = collect.accumulate(all_cards,
				lambda c: c.date('created').strftime('%Y-%m'))
		resolved_by_month = collect.accumulate(closed_cards,
				lambda c: c.date('resolutiondate').strftime('%Y-%m'))

		labels = []
		d = since
		while d < until:
			labels.append(d.strftime('%Y-%m'))
			d = d.replace(month=d.month + 1) if d.month < 12 else d.replace(d.year + 1, 1)

		created_bys = [ created_by_month[l] for l in labels ]
		resolved_bys = [ resolved_by_month[l] for l in labels ]

		count = len(all_cards) - len(closed_cards)
		num_open = []
		for c, r in reversed(list(zip(created_bys, resolved_bys))):
			num_open.insert(0, count)
			count += r - c

		fig, ax = plt.subplots()
		bar_width = 0.25
		index = [x+bar_width for x in range(len(labels))]
		opacity = 0.4

		plt.plot([x+bar_width for x in index], num_open, color='b', linestyle='--', marker='o', label='Work to be completed')
		rects1 = plt.bar(index, created_bys, bar_width,alpha=opacity, color='b',label='New work')
		rects2 = plt.bar([x+bar_width for x in index], resolved_bys, bar_width,alpha=opacity,color='r',label='Completed work')

		#plt.xlabel('Time')
		plt.ylabel('Issues (epics & stories)')
		plt.title('Work items tracked per-month')
		plt.grid(zorder=0)
		plt.xticks([x+bar_width for x in index], labels, rotation=90)
		lgd = plt.legend(bbox_to_anchor=(1.05, 1), loc=2)
		plt.savefig(args.card_tracker, bbox_extra_artists=(lgd,), bbox_inches='tight')
		plt.close()

def do_count(args):
	issues = Issue.load(args.json)

	if args.worklog:
		report = Report(issues)
		print(len(report.worklog()))
		return

	print(len(issues))

def do_fetch(args):
	since = date.smart_parse(args.since)

	if args.constraint:
		# Substitute @ symbols if the arguments looks like it might be an
		# e-mail address (by containing a .)
		constraint = [ x.replace('@', '\\u0040') if '.' in x else x
					for x in args.constraint ]
		constraint = ' '.join(constraint)
	else:
		constraint = None

	issues = sorted(Issue.fetch(since, constraint), key=lambda i: i['key'])
	json.dump(issues, sys.stdout)

def do_filter(args):
	issues = Issue.load(args.json)
	by_key = {}
	for i in issues:
		by_key[i['key']] = i

	def filter_by(person, issue, needle):
		field = issue['fields'][person]
		if not field:
			return False

		return needle in field['displayName'] or needle in field['emailAddress']

	if args.assignee:
		issues = [ i for i in issues if filter_by('assignee', i, args.assignee) ]

	if args.component:
		issues = [ i for i in issues if i.has_component(args.component, args.strict) ]

	if args.since:
		since = date.smart_parse(args.since)
		issues = [ i for i in issues if i.date('updated') >= since ]

	if args.worklog_since:
		since = date.smart_parse(args.worklog_since)
		for i in issues:
			i['worklog'] = [ w for w in i['worklog'] if
					iso8601.parse_date(w['started']) >= since ]

	if args.worklog_until:
		until = date.smart_parse(args.worklog_until)
		for i in issues:
			i['worklog'] = [ w for w in i['worklog'] if
					iso8601.parse_date(w['started']) <= until ]

	if args.no_worklog:
		issues = [ i for i in issues if i['worklog'] ]

	# Go though the issues and ensure we "unfilter" any parent tickets
	# since the Report class may go looking for them.
	if not args.no_keep_parent:
		keys = set([ i['key'] for i in issues ])
		for i in list(issues):
			if 'parent' in i:
				if i['parent'] in keys or i['parent'] not in by_key:
					continue
				keys.add(i['parent'])
				issues.append(by_key[i['parent']])
		issues = sorted(issues, key=lambda i: i['key'])

	json.dump(issues, sys.stdout)

def do_format(args):
	for i in Issue.load(args.json):
		ln = args.template
		for m in re.finditer('{([^}:]+)([^}]*)}', args.template):
			field = m.group(1)
			fmt = m.group(2)

			try:
				if 'member' == field:
					val = i.get_member()
				elif 'component' == field:
					val = i.get_component()
				elif '-' in field:
					(field, attr) = field.split('-', 1)
					if isinstance(i.field(field), str):
						val = i.field(field)[0:int(attr)]
					else:
						val = i.field(field)[attr]
				else:
					val = i.field(field)
			except KeyError:
				continue

			ln = ln.replace(m.group(0), '{{{}}}'.format(fmt).format(val))
		print(ln)

def do_interact(args):
	'''Directly interaction with the JSON data'''
	issues = Issue.load(args.json)
	issue = issues[-1]

	interact()

def do_monthly(args):
	issues = Issue.load(args.json)
	report = Report(issues)

	print('''\
<html>
<head>
<title>Monthly report</title>
</head>
<body>
<table border=2>
<tr>
<td>Member</td>
<td>Accomplishments</td>
<td>Next Month's Plans</td>
</tr>
''')

	for member in sorted(report.members.keys()):
		# Make sure we don't double-report support activity
		if member == "Support":
			continue

		print('<tr><td>{}</td>'.format(member))
		print('<td><ul>')
		for card in sorted(report.members[member], key=lambda c: c['key']):
			just_in_time = False
			for log in report.worklog(card['key'], recurse=True):
				(progress, plans) = log.parse()

				# Abridge long reports (either leave the ellipsis or summarize manually)
				if len(progress) > 7:
					progress = progress[:4] + ['...',]

				for bullet in progress:
					if not just_in_time:
						print('<li>{} (<a href="{}">{}</a>)<ul>'.format(
							card['summary'], card['url'], card['key']))
						just_in_time = True

					print('<li>{}</li>'.format(bullet))
			if just_in_time:
				print('</ul></li>')
		print('</ul></td>')
		print('<td><ul>')
		for card in sorted(report.members[member], key=lambda c: c['key']):
			just_in_time = False
			for log in report.worklog(card['key'], recurse=True):
				(progress, plans) = log.parse()
				for bullet in plans:
					if not just_in_time:
						print('<li>{} (<a href="{}">{}</a>)<ul>'.format(
							card['summary'], card['url'], card['key']))
						just_in_time = True
					print('<li>{}</li>'.format(bullet))
			if just_in_time:
				print('</ul></li>')
		print('</ul></td>')
		print('</tr>')
	print ('''\
</table>
</body>
''')



def do_passwd(args):
	cfg = config.get_config()
	password = config.set_password(cfg, 'jira')

def do_weekly(args):
	issues = Issue.load(args.json)
	report = Report(issues)

	progress = [ ('## Progress', 0) ]
	plans = [ ('', 0), ('## Plans', 0) ]

	for member in sorted(report.members.keys()):
		for card in sorted(report.members[member], key=lambda c: c['key']):
			key = card['key']
			assignees = ', '.join([ a.split()[0] for a in report.assignees(key, recurse=True) ])
			summary = '{}: {} [{}] ({})'.format(member, card['summary'], assignees, key)

			progress.append(('\n', 0))
			progress.append((summary, 1))
			plans += progress[-2:]

			for log in report.worklog(card['key'], recurse=True):
				(prog, plan) = log.parse()
				progress += [ (p, 2) for p in prog ]
				plans += [ (p, 2) for p in plan ]

	wrappers = (
		textwrap.TextWrapper(),
		textwrap.TextWrapper(initial_indent=' * ', subsequent_indent='   '),
		textwrap.TextWrapper(initial_indent='   - ', subsequent_indent='     ')
	)
	for (msg, level) in progress + plans:
		print('\n'.join(wrappers[level].wrap(msg)))

def do_worklog(args):
	issues = Issue.load(args.json)
	report = Report(issues)

	# Sort by time order (glimpse used to sort by issue number here)
	for w in report.worklog():
		card = report.issues[w['issue']]
		if 'parent' in card:
			if card['parent'] in report.issues:
				epic = report.issues[card['parent']]
			else:
				epic = card
		else:
			epic = card
		print('"{}","{}","{}","{}","{}","{}","{}","{}"'.format(
			card['key'], card['summary'], epic['key'],
			card.get_member(), card.get_component(),
			w['started'], w['timeSpentSeconds'],
			w['author']['displayName']))

def main(argv):
	parser = argparse.ArgumentParser()
	subparsers = parser.add_subparsers(dest='sub-command')
	subparsers.required = True      # Can't be set using named arguments (yet)

	s = subparsers.add_parser('chart',
			help='Draw charts from the data')
	s.add_argument('--card-tracker', metavar='PNGFILE')
	s.add_argument('--effort-by-component', metavar='PNGFILE')
	s.add_argument('--effort-by-engineer', metavar='PNGFILE')
	s.add_argument('--effort-by-member', metavar='PNGFILE')
	s.add_argument('--count-by-member', metavar='PNGFILE')
	s.add_argument('--barchart', action='store_true')
	s.add_argument('--piechart', action='store_true')
	s.add_argument('--since',
		help='Date from which to chart data (applies to card graphs only)')
	s.add_argument('--until',
		help='Only chart data before this date (applies to card graphs only)')
	s.add_argument('json', nargs='?')
	s.set_defaults(func=do_chart)

	s = subparsers.add_parser('count',
			help='Generate summarized statistics')
	s.add_argument('--worklog', action='store_true')
	s.add_argument('json', nargs='?')
	s.set_defaults(func=do_count)

	s = subparsers.add_parser('fetch',
			help='Download status from projects.linaro.org')
	s.add_argument('--since', default='2012-01-01',
                        help='When to gather information from')
	s.add_argument("constraint", nargs="*",
			help="Any additional JQL contraints")
	s.set_defaults(func=do_fetch)

	s = subparsers.add_parser('filter',
			help='Filter cards and worklogs')
	s.add_argument('--assignee')
	s.add_argument('--component')
	s.add_argument('--since')
	s.add_argument('--strict', action='store_true')
	s.add_argument('--no-keep-parent', action='store_true')
	s.add_argument('--no-worklog', action='store_true')
	s.add_argument('--worklog-since')
	s.add_argument('--worklog-until')
	s.add_argument('json', nargs='?')
	s.set_defaults(func=do_filter)

	s = subparsers.add_parser('format',
			help='Summarize each card using a template')
	s.add_argument('--template',
			default='{key}: {summary} ({member})')
	s.add_argument('json', nargs='?')
	s.set_defaults(func=do_format)

	s = subparsers.add_parser('interact',
			help='Interact with report data via REPL')
	s.add_argument('json')
	s.set_defaults(func=do_interact)

	s = subparsers.add_parser('monthly',
			help='Generate a (template) monthly report')
	s.add_argument('json', nargs='?')
	s.set_defaults(func=do_monthly)

	s = subparsers.add_parser('summary',
			help='Generate a quick summary')
	s.add_argument('--template',
			default='{created-10}: {key}: {summary} ({member})')
	s.add_argument('json', nargs='?')
	s.set_defaults(func=do_format)

	s = subparsers.add_parser('passwd',
			help='Store your JIRA password to the keyring')
	s.set_defaults(func=do_passwd)

	s = subparsers.add_parser('weekly',
			help='Generate a (template) weekly report')
	s.add_argument('json', nargs='?')
	s.set_defaults(func=do_weekly)

	s = subparsers.add_parser('worklog',
			help='Summarize each worklog entry')
	s.add_argument('json', nargs='?')
	s.set_defaults(func=do_worklog)

	args = parser.parse_args(argv[1:])
	args.func(args)

if __name__ == '__main__':
        try:
                sys.exit(main(sys.argv))
        except KeyboardInterrupt:
                sys.exit(1)
        sys.exit(127)