96btool: pull: Better progress reporting

This is a fairly big change but only really does a couple of things:

  * --debug provokes a much richer log of activity
  * Breadcrumbs left during verbose operation give more insight into
    error responses.

1 files changed, 57 insertions, 35 deletions

diff --git a/bin/96btool b/bin/96btool
index c7d51be..ca29e72 100755
--- a/bin/96btool
+++ b/bin/96btool
@@ -136,8 +136,10 @@ class Post(dict):
 	def created_at(self):
 		return iso8601.parse_date(self['created_at'])
 
-	def get_user(self, client):
+	def get_user(self, client, quick=False):
 		if self['username'] not in Post.user_db:
+			if quick:
+				return None
 			Post.user_db[self['username']] = client.user(self['username'])
 		self['user'] = Post.user_db[self['username']]
 		return self['user']
@@ -167,53 +169,69 @@ class Post(dict):
 		return False
 
 	@staticmethod
-	def fetch(client, from_id, to_id, verbose=False):
+	def fetch(client, from_id, to_id, verbose=False, debug=False):
+		def log(verbose_txt, debug_txt):
+			if debug:
+				sys.stdout.write(debug_txt)
+			elif verbose:
+				sys.stdout.write(verbose_txt)
+			if debug or verbose:
+				sys.stdout.flush()
+
 		for post_id in range(from_id, to_id):
 			try:
-				post = client._get('/posts/{}.json'.format(post_id))
+				stem = '/posts/{}.json'.format(post_id)
+				log('', 'GET {} ... '.format(stem))
+				post = client._get(stem)
 				post = Post(post)
-				if post.lint():
-					if verbose:
-						sys.stdout.write('P')
-						sys.stdout.flush()
-					continue
-
-			except pydiscourse.exceptions.DiscourseClientError:
-				if verbose:
-					sys.stdout.write('E')
-					sys.stdout.flush()
+			except pydiscourse.exceptions.DiscourseClientError as e:
+				if e.response.status_code == 403:
+					# Permissions error (other peoples PMs)
+					log('p', '{}\n'.format(e))
+				elif e.response.status_code == 404:
+					# Not found
+					log('N', '{}\n'.format(e))
+				elif e.response.status_code == 429:
+					# Rate limiter kicked in
+					log('R', '{}\n'.format(e))
+				else:
+					# Unexpected error
+					log('E{}'.format(e.response.status_code),
+					    '{} ({})\n'.format(e, e.response.status_code))
 				continue
 
+			log('', 'ok\n')
 			result = '.'
 
 			if post['topic_id'] not in Post.topic_db:
 				try:
-					topic = client._get(
-						'/t/{}.json'.format(
-							post['topic_id']))
-				except pydiscourse.exceptions.DiscourseClientError:
-					if verbose:
-						sys.stdout.write('T')
-						sys.stdout.flush()
+					stem = '/t/{}.json'.format(post['topic_id'])
+					log('', 'GET {} ...  '.format(stem))
+					topic = client._get(stem)
+					log('', 'ok\n')
+					Post.topic_db[post['topic_id']] = topic
+					result = 'o'
+				except pydiscourse.exceptions.DiscourseClientError as e:
+					log('T', '{}\n'.format(e))
 					continue
-				result = 'o'
 			else:
 				topic = Post.topic_db[post['topic_id']]
 			post['topic'] = topic
 
-			try:
-				post.get_user(client)
-			except pydiscourse.exceptions.DiscourseClientError:
-				if verbose:
-					sys.stdout.write('U')
-					sys.stdout.flush()
-
-			post.lint()
-			post.index(post)
 
-			if verbose:
-				sys.stdout.write(result)
-				sys.stdout.flush()
+			if None == post.get_user(client, quick=True):
+				try:
+					log('', 'GET user {} ...  '.format(post['username']))
+					post.get_user(client)
+					log('', 'ok\n')
+				except pydiscourse.exceptions.DiscourseClientError as e:
+					log('U', '{}\n'.format(e))
+
+			if post.lint():
+				log('P', 'Discarding private message {}\n'.format(post_id))
+				continue
+			Post.index(post)
+			log(result, '')
 
 	@staticmethod
 	def index(posts):
@@ -662,6 +680,8 @@ def do_pull(args):
 		sys.stdout.write('Looking at the latest topics ...')
 		sys.stdout.flush()
 	latest_post_id = Post.latest(client)
+	if args.verbose:
+		sys.stdout.write('{}\n'.format(latest_post_id))
 
 	if len(Post.post_db) and not args.refresh:
 		max_post_id = max([ i for i in Post.post_db.keys() ])
@@ -672,14 +692,16 @@ def do_pull(args):
 
 	# Start grabbing the posts
 	if args.verbose:
-		sys.stdout.write(' ok\nFetching posts from #{} to #{} .'.format(
+		sys.stdout.write('\nFetching posts from #{} to #{} .'.format(
 				max_post_id + 1, latest_post_id))
 		sys.stdout.flush()
+		if args.debug:
+			sys.stdout.write('.. working\n')
 	try:
 		# fetch automatically indexes as it works so we can ignore
 		# the return value here...
 		Post.fetch(client, max_post_id+1, latest_post_id+1,
-				verbose=args.verbose)
+				verbose=args.verbose, debug=args.debug)
 		retval = 0
 
 	except BaseException as e: