llvm/utils/Reviewing/find_interesting_reviews.py

#!/usr/bin/env python

import argparse
import email.mime.multipart
import email.mime.text
import logging
import os.path
import pickle
import re
import smtplib
import subprocess
import sys
from datetime import datetime, timedelta
from phabricator import Phabricator

# Setting up a virtualenv to run this script can be done by running the
# following commands:
# $ virtualenv venv
# $ . ./venv/bin/activate
# $ pip install Phabricator

GIT_REPO_METADATA = (("llvm", "https://llvm.org/git/llvm.git"), )

# The below PhabXXX classes represent objects as modelled by Phabricator.
# The classes can be serialized to disk, to try and make sure that we don't
# needlessly have to re-fetch lots of data from Phabricator, as that would
# make this script unusably slow.


class PhabObject:
    OBJECT_KIND = None

    def __init__(self, id):
        self.id = id


class PhabObjectCache:
    def __init__(self, PhabObjectClass):
        self.PhabObjectClass = PhabObjectClass
        self.most_recent_info = None
        self.oldest_info = None
        self.id2PhabObjects = {}

    def get_name(self):
        return self.PhabObjectClass.OBJECT_KIND + "sCache"

    def get(self, id):
        if id not in self.id2PhabObjects:
            self.id2PhabObjects[id] = self.PhabObjectClass(id)
        return self.id2PhabObjects[id]

    def get_ids_in_cache(self):
        return self.id2PhabObjects.keys()

    def get_objects(self):
        return self.id2PhabObjects.values()

    DEFAULT_DIRECTORY = "PhabObjectCache"

    def _get_pickle_name(self, directory):
        file_name = "Phab" + self.PhabObjectClass.OBJECT_KIND + "s.pickle"
        return os.path.join(directory, file_name)

    def populate_cache_from_disk(self, directory=DEFAULT_DIRECTORY):
        """
        FIXME: consider if serializing to JSON would bring interoperability
        advantages over serializing to pickle.
        """
        try:
            f = open(self._get_pickle_name(directory), "rb")
        except IOError as err:
            print("Could not find cache. Error message: {0}. Continuing..."
                  .format(err))
        else:
            with f:
                try:
                    d = pickle.load(f)
                    self.__dict__.update(d)
                except EOFError as err:
                    print("Cache seems to be corrupt. " +
                          "Not using cache. Error message: {0}".format(err))

    def write_cache_to_disk(self, directory=DEFAULT_DIRECTORY):
        if not os.path.exists(directory):
            os.makedirs(directory)
        with open(self._get_pickle_name(directory), "wb") as f:
            pickle.dump(self.__dict__, f)
        print("wrote cache to disk, most_recent_info= {0}".format(
            datetime.fromtimestamp(self.most_recent_info)
            if self.most_recent_info is not None else None))


class PhabReview(PhabObject):
    OBJECT_KIND = "Review"

    def __init__(self, id):
        PhabObject.__init__(self, id)

    def update(self, title, dateCreated, dateModified, author):
        self.title = title
        self.dateCreated = dateCreated
        self.dateModified = dateModified
        self.author = author

    def setPhabDiffs(self, phabDiffs):
        self.phabDiffs = phabDiffs


class PhabUser(PhabObject):
    OBJECT_KIND = "User"

    def __init__(self, id):
        PhabObject.__init__(self, id)

    def update(self, phid, realName):
        self.phid = phid
        self.realName = realName


class PhabHunk:
    def __init__(self, rest_api_hunk):
        self.oldOffset = int(rest_api_hunk["oldOffset"])
        self.oldLength = int(rest_api_hunk["oldLength"])
        # self.actual_lines_changed_offset will contain the offsets of the
        # lines that were changed in this hunk.
        self.actual_lines_changed_offset = []
        offset = self.oldOffset
        inHunk = False
        hunkStart = -1
        contextLines = 3
        for line in rest_api_hunk["corpus"].split("\n"):
            if line.startswith("+"):
                # line is a new line that got introduced in this patch.
                # Do not record it as a changed line.
                if inHunk is False:
                    inHunk = True
                    hunkStart = max(self.oldOffset, offset - contextLines)
                continue
            if line.startswith("-"):
                # line was changed or removed from the older version of the
                # code. Record it as a changed line.
                if inHunk is False:
                    inHunk = True
                    hunkStart = max(self.oldOffset, offset - contextLines)
                offset += 1
                continue
            # line is a context line.
            if inHunk is True:
                inHunk = False
                hunkEnd = offset + contextLines
                self.actual_lines_changed_offset.append((hunkStart, hunkEnd))
            offset += 1
        if inHunk is True:
            hunkEnd = offset + contextLines
            self.actual_lines_changed_offset.append((hunkStart, hunkEnd))

        # The above algorithm could result in adjacent or overlapping ranges
        # being recorded into self.actual_lines_changed_offset.
        # Merge the adjacent and overlapping ranges in there:
        t = []
        lastRange = None
        for start, end in self.actual_lines_changed_offset + \
                [(sys.maxsize, sys.maxsize)]:
            if lastRange is None:
                lastRange = (start, end)
            else:
                if lastRange[1] >= start:
                    lastRange = (lastRange[0], end)
                else:
                    t.append(lastRange)
                    lastRange = (start, end)
        self.actual_lines_changed_offset = t


class PhabChange:
    def __init__(self, rest_api_change):
        self.oldPath = rest_api_change["oldPath"]
        self.hunks = [PhabHunk(h) for h in rest_api_change["hunks"]]


class PhabDiff(PhabObject):
    OBJECT_KIND = "Diff"

    def __init__(self, id):
        PhabObject.__init__(self, id)

    def update(self, rest_api_results):
        self.revisionID = rest_api_results["revisionID"]
        self.dateModified = int(rest_api_results["dateModified"])
        self.dateCreated = int(rest_api_results["dateCreated"])
        self.changes = [PhabChange(c) for c in rest_api_results["changes"]]


class ReviewsCache(PhabObjectCache):
    def __init__(self):
        PhabObjectCache.__init__(self, PhabReview)


class UsersCache(PhabObjectCache):
    def __init__(self):
        PhabObjectCache.__init__(self, PhabUser)


reviews_cache = ReviewsCache()
users_cache = UsersCache()


def init_phab_connection():
    phab = Phabricator()
    phab.update_interfaces()
    return phab


def update_cached_info(phab, cache, phab_query, order, record_results,
                       max_nr_entries_per_fetch, max_nr_days_to_cache):
    q = phab
    LIMIT = max_nr_entries_per_fetch
    for query_step in phab_query:
        q = getattr(q, query_step)
    results = q(order=order, limit=LIMIT)
    most_recent_info, oldest_info = record_results(cache, results, phab)
    oldest_info_to_fetch = datetime.fromtimestamp(most_recent_info) - \
        timedelta(days=max_nr_days_to_cache)
    most_recent_info_overall = most_recent_info
    cache.write_cache_to_disk()
    after = results["cursor"]["after"]
    print("after: {0!r}".format(after))
    print("most_recent_info: {0}".format(
        datetime.fromtimestamp(most_recent_info)))
    while (after is not None
           and datetime.fromtimestamp(oldest_info) > oldest_info_to_fetch):
        need_more_older_data = \
            (cache.oldest_info is None or
             datetime.fromtimestamp(cache.oldest_info) > oldest_info_to_fetch)
        print(("need_more_older_data={0} cache.oldest_info={1} " +
               "oldest_info_to_fetch={2}").format(
                   need_more_older_data,
                   datetime.fromtimestamp(cache.oldest_info)
                   if cache.oldest_info is not None else None,
                   oldest_info_to_fetch))
        need_more_newer_data = \
            (cache.most_recent_info is None or
             cache.most_recent_info < most_recent_info)
        print(("need_more_newer_data={0} cache.most_recent_info={1} " +
               "most_recent_info={2}")
              .format(need_more_newer_data, cache.most_recent_info,
                      most_recent_info))
        if not need_more_older_data and not need_more_newer_data:
            break
        results = q(order=order, after=after, limit=LIMIT)
        most_recent_info, oldest_info = record_results(cache, results, phab)
        after = results["cursor"]["after"]
        print("after: {0!r}".format(after))
        print("most_recent_info: {0}".format(
            datetime.fromtimestamp(most_recent_info)))
        cache.write_cache_to_disk()
    cache.most_recent_info = most_recent_info_overall
    if after is None:
        # We did fetch all records. Mark the cache to contain all info since
        # the start of time.
        oldest_info = 0
    cache.oldest_info = oldest_info
    cache.write_cache_to_disk()


def record_reviews(cache, reviews, phab):
    most_recent_info = None
    oldest_info = None
    for reviewInfo in reviews["data"]:
        if reviewInfo["type"] != "DREV":
            continue
        id = reviewInfo["id"]
        # phid = reviewInfo["phid"]
        dateModified = int(reviewInfo["fields"]["dateModified"])
        dateCreated = int(reviewInfo["fields"]["dateCreated"])
        title = reviewInfo["fields"]["title"]
        author = reviewInfo["fields"]["authorPHID"]
        phabReview = cache.get(id)
        if "dateModified" not in phabReview.__dict__ or \
           dateModified > phabReview.dateModified:
            diff_results = phab.differential.querydiffs(revisionIDs=[id])
            diff_ids = sorted(diff_results.keys())
            phabDiffs = []
            for diff_id in diff_ids:
                diffInfo = diff_results[diff_id]
                d = PhabDiff(diff_id)
                d.update(diffInfo)
                phabDiffs.append(d)
            phabReview.update(title, dateCreated, dateModified, author)
            phabReview.setPhabDiffs(phabDiffs)
            print("Updated D{0} modified on {1} ({2} diffs)".format(
                id, datetime.fromtimestamp(dateModified), len(phabDiffs)))

        if most_recent_info is None:
            most_recent_info = dateModified
        elif most_recent_info < dateModified:
            most_recent_info = dateModified

        if oldest_info is None:
            oldest_info = dateModified
        elif oldest_info > dateModified:
            oldest_info = dateModified
    return most_recent_info, oldest_info


def record_users(cache, users, phab):
    most_recent_info = None
    oldest_info = None
    for info in users["data"]:
        if info["type"] != "USER":
            continue
        id = info["id"]
        phid = info["phid"]
        dateModified = int(info["fields"]["dateModified"])
        # dateCreated = int(info["fields"]["dateCreated"])
        realName = info["fields"]["realName"]
        phabUser = cache.get(id)
        phabUser.update(phid, realName)
        if most_recent_info is None:
            most_recent_info = dateModified
        elif most_recent_info < dateModified:
            most_recent_info = dateModified
        if oldest_info is None:
            oldest_info = dateModified
        elif oldest_info > dateModified:
            oldest_info = dateModified
    return most_recent_info, oldest_info


PHABCACHESINFO = ((reviews_cache, ("differential", "revision", "search"),
                   "updated", record_reviews, 5, 7),
                  (users_cache, ("user", "search"), "newest", record_users,
                   100, 1000))


def load_cache():
    for cache, phab_query, order, record_results, _, _ in PHABCACHESINFO:
        cache.populate_cache_from_disk()
        print("Loaded {0} nr entries: {1}".format(
            cache.get_name(), len(cache.get_ids_in_cache())))
        print("Loaded {0} has most recent info: {1}".format(
            cache.get_name(),
            datetime.fromtimestamp(cache.most_recent_info)
            if cache.most_recent_info is not None else None))


def update_cache(phab):
    load_cache()
    for cache, phab_query, order, record_results, max_nr_entries_per_fetch, \
            max_nr_days_to_cache in PHABCACHESINFO:
        update_cached_info(phab, cache, phab_query, order, record_results,
                           max_nr_entries_per_fetch, max_nr_days_to_cache)
        ids_in_cache = cache.get_ids_in_cache()
        print("{0} objects in {1}".format(len(ids_in_cache), cache.get_name()))
        cache.write_cache_to_disk()


def get_most_recent_reviews(days):
    newest_reviews = sorted(
        reviews_cache.get_objects(), key=lambda r: -r.dateModified)
    if len(newest_reviews) == 0:
        return newest_reviews
    most_recent_review_time = \
        datetime.fromtimestamp(newest_reviews[0].dateModified)
    cut_off_date = most_recent_review_time - timedelta(days=days)
    result = []
    for review in newest_reviews:
        if datetime.fromtimestamp(review.dateModified) < cut_off_date:
            return result
        result.append(review)
    return result


# All of the above code is about fetching data from Phabricator and caching it
# on local disk. The below code contains the actual "business logic" for this
# script.

_userphid2realname = None


def get_real_name_from_author(user_phid):
    global _userphid2realname
    if _userphid2realname is None:
        _userphid2realname = {}
        for user in users_cache.get_objects():
            _userphid2realname[user.phid] = user.realName
    return _userphid2realname.get(user_phid, "unknown")


def print_most_recent_reviews(phab, days, filter_reviewers):
    msgs = []

    def add_msg(msg):
        msgs.append(msg)
        print(msg)

    newest_reviews = get_most_recent_reviews(days)
    add_msg(u"These are the reviews that look interesting to be reviewed. " +
            u"The report below has 2 sections. The first " +
            u"section is organized per review; the second section is organized "
            + u"per potential reviewer.\n")
    oldest_review = newest_reviews[-1] if len(newest_reviews) > 0 else None
    oldest_datetime = \
        datetime.fromtimestamp(oldest_review.dateModified) \
        if oldest_review else None
    add_msg((u"The report below is based on analyzing the reviews that got " +
             u"touched in the past {0} days (since {1}). " +
             u"The script found {2} such reviews.\n").format(
                 days, oldest_datetime, len(newest_reviews)))
    reviewer2reviews_and_scores = {}
    for i, review in enumerate(newest_reviews):
        matched_reviewers = find_reviewers_for_review(review)
        matched_reviewers = filter_reviewers(matched_reviewers)
        if len(matched_reviewers) == 0:
            continue
        add_msg((u"{0:>3}. https://reviews.llvm.org/D{1} by {2}\n     {3}\n" +
                 u"     Last updated on {4}").format(
                     i, review.id,
                     get_real_name_from_author(review.author), review.title,
                     datetime.fromtimestamp(review.dateModified)))
        for reviewer, scores in matched_reviewers:
            add_msg(u"    potential reviewer {0}, score {1}".format(
                reviewer,
                "(" + "/".join(["{0:.1f}%".format(s) for s in scores]) + ")"))
            if reviewer not in reviewer2reviews_and_scores:
                reviewer2reviews_and_scores[reviewer] = []
            reviewer2reviews_and_scores[reviewer].append((review, scores))

    # Print out a summary per reviewer.
    for reviewer in sorted(reviewer2reviews_and_scores.keys()):
        reviews_and_scores = reviewer2reviews_and_scores[reviewer]
        reviews_and_scores.sort(key=lambda rs: rs[1], reverse=True)
        add_msg(u"\n\nSUMMARY FOR {0} (found {1} reviews):".format(
            reviewer, len(reviews_and_scores)))
        for review, scores in reviews_and_scores:
            add_msg(u"[{0}] https://reviews.llvm.org/D{1} '{2}' by {3}".format(
                "/".join(["{0:.1f}%".format(s) for s in scores]), review.id,
                review.title, get_real_name_from_author(review.author)))
    return "\n".join(msgs)


def get_git_cmd_output(cmd):
    output = None
    try:
        logging.debug(cmd)
        output = subprocess.check_output(
            cmd, shell=True, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
        logging.debug(str(e))
    if output is None:
        return None
    return output.decode("utf-8", errors='ignore')


reAuthorMail = re.compile("^author-mail <([^>]*)>.*$")


def parse_blame_output_line_porcelain(blame_output):
    email2nr_occurences = {}
    if blame_output is None:
        return email2nr_occurences
    for line in blame_output.split('\n'):
        m = reAuthorMail.match(line)
        if m:
            author_email_address = m.group(1)
            if author_email_address not in email2nr_occurences:
                email2nr_occurences[author_email_address] = 1
            else:
                email2nr_occurences[author_email_address] += 1
    return email2nr_occurences


def find_reviewers_for_diff_heuristic(diff):
    # Heuristic 1: assume good reviewers are the ones that touched the same
    # lines before as this patch is touching.
    # Heuristic 2: assume good reviewers are the ones that touched the same
    # files before as this patch is touching.
    reviewers2nr_lines_touched = {}
    reviewers2nr_files_touched = {}
    # Assume last revision before diff was modified is the revision the diff
    # applies to.
    git_repo = "git_repos/llvm"
    cmd = 'git -C {0} rev-list -n 1 --before="{1}" master'.format(
        git_repo,
        datetime.fromtimestamp(
            diff.dateModified).strftime("%Y-%m-%d %H:%M:%s"))
    base_revision = get_git_cmd_output(cmd).strip()
    logging.debug("Base revision={0}".format(base_revision))
    for change in diff.changes:
        path = change.oldPath
        # Compute heuristic 1: look at context of patch lines.
        for hunk in change.hunks:
            for start_line, end_line in hunk.actual_lines_changed_offset:
                # Collect git blame results for authors in those ranges.
                cmd = ("git -C {0} blame --encoding=utf-8 --date iso -f -e " +
                       "-w --line-porcelain -L {1},{2} {3} -- {4}").format(
                           git_repo, start_line, end_line, base_revision, path)
                blame_output = get_git_cmd_output(cmd)
                for reviewer, nr_occurences in \
                        parse_blame_output_line_porcelain(blame_output).items():
                    if reviewer not in reviewers2nr_lines_touched:
                        reviewers2nr_lines_touched[reviewer] = 0
                    reviewers2nr_lines_touched[reviewer] += nr_occurences
        # Compute heuristic 2: don't look at context, just at files touched.
        # Collect git blame results for authors in those ranges.
        cmd = ("git -C {0} blame --encoding=utf-8 --date iso -f -e -w " +
               "--line-porcelain {1} -- {2}").format(git_repo, base_revision,
                                                     path)
        blame_output = get_git_cmd_output(cmd)
        for reviewer, nr_occurences in parse_blame_output_line_porcelain(
                blame_output).items():
            if reviewer not in reviewers2nr_files_touched:
                reviewers2nr_files_touched[reviewer] = 0
            reviewers2nr_files_touched[reviewer] += 1

    # Compute "match scores"
    total_nr_lines = sum(reviewers2nr_lines_touched.values())
    total_nr_files = len(diff.changes)
    reviewers_matchscores = \
        [(reviewer,
          (reviewers2nr_lines_touched.get(reviewer, 0)*100.0/total_nr_lines
           if total_nr_lines != 0 else 0,
           reviewers2nr_files_touched[reviewer]*100.0/total_nr_files
           if total_nr_files != 0 else 0))
         for reviewer, nr_lines
         in reviewers2nr_files_touched.items()]
    reviewers_matchscores.sort(key=lambda i: i[1], reverse=True)
    return reviewers_matchscores


def find_reviewers_for_review(review):
    # Process the newest diff first.
    diffs = sorted(
        review.phabDiffs, key=lambda d: d.dateModified, reverse=True)
    if len(diffs) == 0:
        return
    diff = diffs[0]
    matched_reviewers = find_reviewers_for_diff_heuristic(diff)
    # Show progress, as this is a slow operation:
    sys.stdout.write('.')
    sys.stdout.flush()
    logging.debug(u"matched_reviewers: {0}".format(matched_reviewers))
    return matched_reviewers


def update_git_repos():
    git_repos_directory = "git_repos"
    for name, url in GIT_REPO_METADATA:
        dirname = os.path.join(git_repos_directory, name)
        if not os.path.exists(dirname):
            cmd = "git clone {0} {1}".format(url, dirname)
            output = get_git_cmd_output(cmd)
        cmd = "git -C {0} pull --rebase".format(dirname)
        output = get_git_cmd_output(cmd)


def send_emails(email_addresses, sender, msg):
    s = smtplib.SMTP()
    s.connect()
    for email_address in email_addresses:
        email_msg = email.mime.multipart.MIMEMultipart()
        email_msg['From'] = sender
        email_msg['To'] = email_address
        email_msg['Subject'] = 'LLVM patches you may be able to review.'
        email_msg.attach(email.mime.text.MIMEText(msg.encode('utf-8'), 'plain'))
        # python 3.x: s.send_message(email_msg)
        s.sendmail(email_msg['From'], email_msg['To'], email_msg.as_string())
    s.quit()


def filter_reviewers_to_report_for(people_to_look_for):
    # The below is just an example filter, to only report potential reviews
    # to do for the people that will receive the report email.
    return lambda potential_reviewers: [r for r in potential_reviewers
                                        if r[0] in people_to_look_for]


def main():
    parser = argparse.ArgumentParser(
        description='Match open reviews to potential reviewers.')
    parser.add_argument(
        '--no-update-cache',
        dest='update_cache',
        action='store_false',
        default=True,
        help='Do not update cached Phabricator objects')
    parser.add_argument(
        '--email-report',
        dest='email_report',
        nargs='*',
        default="",
        help="A email addresses to send the report to.")
    parser.add_argument(
        '--sender',
        dest='sender',
        default="",
        help="The email address to use in 'From' on messages emailed out.")
    parser.add_argument(
        '--email-addresses',
        dest='email_addresses',
        nargs='*',
        help="The email addresses (as known by LLVM git) of " +
        "the people to look for reviews for.")
    parser.add_argument('--verbose', '-v', action='count')

    args = parser.parse_args()

    if args.verbose >= 1:
        logging.basicConfig(level=logging.DEBUG)

    people_to_look_for = [e.decode('utf-8') for e in args.email_addresses]
    logging.debug("Will look for reviews that following contributors could " +
                  "review: {}".format(people_to_look_for))
    logging.debug("Will email a report to: {}".format(args.email_report))

    phab = init_phab_connection()

    if args.update_cache:
        update_cache(phab)

    load_cache()
    update_git_repos()
    msg = print_most_recent_reviews(
        phab,
        days=1,
        filter_reviewers=filter_reviewers_to_report_for(people_to_look_for))

    if args.email_report != []:
        send_emails(args.email_report, args.sender, msg)


if __name__ == "__main__":
    main()