<commit_before>import os
import re
import subprocess
import sys

from reviewboard.diffviewer.parser import DiffParser
from reviewboard.scmtools.core import SCMTool, HEAD, PRE_CREATION
from reviewboard.scmtools.errors import SCMError, FileNotFoundError

# This specific import is necessary to handle the paths for
# cygwin enabled machines.
if (sys.platform.startswith('win') or sys.platform.startswith('cygwin')):
    import ntpath as cpath
else:
    import posixpath as cpath

# This is a workaround for buggy Python 2.7.x and Windows 7.
# A console window would pop up every time popen is invoked unless shell=true.
# Original issue was described at http://reviews.reviewboard.org/r/3804/
# Note:
#   - later versions of Windows may probably be impacted too
#   - Python 2.7.x is the only one known to get this issue
import platform

if (sys.version_info[:2] == (2, 7) and
    platform.system() == "Windows" and
    platform.release() == "7"):
    _popen_shell = True
else:
    _popen_shell = False

class ClearCaseTool(SCMTool):
    name = 'ClearCase'
    uses_atomic_revisions = False
    supports_authentication = False
    field_help_text = {
        'path': 'The absolute path to the VOB.',
    }
    dependencies = {
        'executables': ['cleartool'],
    }

    # This regular expression can extract from extended_path
    # pure system path. It is construct from two main parts.
    # First match everything from beginning of line to first
    # occurence of /. Second match parts between /main and
    # numbers (file version).
    # This patch assume each branch present in extended_path
    # was derived from /main and there is no file or directory
    # called "main" in path.
    UNEXTENDED = re.compile(r'^(.+?)/|/?(.+?)/main/?.*?/([0-9]+|CHECKEDOUT)')

    VIEW_SNAPSHOT, VIEW_DYNAMIC, VIEW_UNKNOWN = range(3)

    def __init__(self, repository):
        self.repopath = repository.path

        SCMTool.__init__(self, repository)

        self.viewtype = self._get_view_type(self.repopath)

        if self.viewtype == self.VIEW_SNAPSHOT:
            self.client = ClearCaseSnapshotViewClient(self.repopath)
        elif self.viewtype == self.VIEW_DYNAMIC:
            self.client = ClearCaseDynamicViewClient(self.repopath)
        else:
            raise SCMError('Unsupported view type.')

    def unextend_path(self, extended_path):
        """Remove ClearCase revision and branch informations from path.

        ClearCase paths contain additional informations about branch
        and file version preceded by @@. This function remove this
        parts from ClearCase path to make it more readable
        For example this function convert extended path::

            /vobs/comm@@/main/122/network@@/main/55/sntp
            @@/main/4/src@@/main/1/sntp.c@@/main/8

        to the the to regular path::

            /vobs/comm/network/sntp/src/sntp.c
        """
        if not '@@' in extended_path:
            return HEAD, extended_path

        # Result of regular expression search result is list of tuples.
        # We must flat this to one list. The best way is use list comprehension.
        # b is first because it frequently occure in tuples.
        # Before that remove @@ from path.
        unextended_chunks = [
            b or a
            for a, b, foo in self.UNEXTENDED.findall(extended_path.replace('@@', ''))
        ]

        if sys.platform.startswith('win'):
            # Properly handle full (with drive letter) and UNC paths
            if unextended_chunks[0].endswith(':'):
                unextended_chunks[0] = '%s\\' % unextended_chunks[0]
            elif unextended_chunks[0] == '/' or unextended_chunks[0] == os.sep:
                unextended_chunks[0] = '\\\\'

        # Purpose of realpath is remove parts like /./ generated by
        # ClearCase when vobs branch was fresh created
        unextended_path = cpath.realpath(
            cpath.join(*unextended_chunks)
        )

        revision = extended_path.rsplit('@@', 1)[1]
        if revision.endswith('CHECKEDOUT'):
            revision = HEAD

        return (revision, unextended_path)

    @classmethod
    def relpath(cls, path, start):
        """Wrapper for os.path.relpath for Python 2.4.

        Python 2.4 doesn't have the os.path.relpath function, so this
        approximates it well enough for our needs.

        ntpath.relpath() overflows and throws TypeError for paths containing
        atleast 520 characters (not that hard to encounter in UCM
        repository).
        """
        try:
            return cpath.relpath(path, start)
        except (AttributeError, TypeError):
            if start[-1] != os.sep:
                start += os.sep

            return path[len(start):]

    def normalize_path_for_display(self, filename):
        """Return display friendly path without revision informations.

        In path construct for only display purpuse we don't need
        information about branch, version or even repository path
        so we return unextended path relative to repopath (view)
        """
        return self.relpath(self.unextend_path(filename)[1], self.repopath)

    def get_repository_info(self):
        vobstag = self._get_vobs_tag(self.repopath)
        return {
            'repopath': self.repopath,
            'uuid': self._get_vobs_uuid(vobstag)
        }

    def _get_view_type(self, repopath):
        cmdline = ["cleartool", "lsview", "-full", "-properties", "-cview"]
        p = subprocess.Popen(
            cmdline,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=repopath,
            shell=_popen_shell)

        (res, error) = p.communicate()
        failure = p.poll()

        if failure:
            raise SCMError(error)

        for line in res.splitlines(True):
            splitted = line.split(' ')
            if splitted[0] == 'Properties:':
                if 'snapshot' in splitted:
                    return self.VIEW_SNAPSHOT
                elif 'dynamic' in splitted:
                    return self.VIEW_DYNAMIC

        return self.VIEW_UNKNOWN

    def _get_vobs_tag(self, repopath):
        cmdline = ["cleartool", "describe", "-short", "vob:."]
        p = subprocess.Popen(
            cmdline,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=self.repopath,
            shell=_popen_shell)

        (res, error) = p.communicate()
        failure = p.poll()

        if failure:
            raise SCMError(error)

        return res.rstrip()

    def _get_vobs_uuid(self, vobstag):
        cmdline = ["cleartool", "lsvob", "-long", vobstag]
        p = subprocess.Popen(
            cmdline,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=self.repopath,
            shell=_popen_shell)

        (res, error) = p.communicate()
        failure = p.poll()

        if failure:
            raise SCMError(error)

        for line  in res.splitlines(True):
            if line.startswith('Vob family uuid:'):
                return line.split(' ')[-1].rstrip()

        raise SCMError("Can't find familly uuid for vob: %s" % vobstag)

    def _get_object_kind(self, extended_path):
        cmdline = ["cleartool", "desc", "-fmt", "%m", extended_path]
        p = subprocess.Popen(
            cmdline,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=self.repopath,
            shell=_popen_shell)

        (res, error) = p.communicate()
        failure = p.poll()

        if failure:
            raise SCMError(error)

        return res.strip()

    def get_file(self, extended_path, revision=HEAD):
        """Return content of file or list content of directory"""
        if not extended_path:
            raise FileNotFoundError(extended_path, revision)

        if revision == PRE_CREATION:
            return ''

        if self.viewtype == self.VIEW_SNAPSHOT:
            # Get the path to (presumably) file element (remove version)
            # The '@@' at the end of file_path is required.
            file_path = extended_path.rsplit('@@', 1)[0] + '@@'
            okind = self._get_object_kind(file_path)

            if okind == 'directory element':
                raise SCMError('Directory elements are unsupported.')
            elif okind == 'file element':
                output = self.client.cat_file(extended_path, revision)
            else:
                raise FileNotFoundError(extended_path, revision)
        else:
            if cpath.isdir(extended_path):
                output = self.client.list_dir(extended_path, revision)
            elif cpath.exists(extended_path):
                output = self.client.cat_file(extended_path, revision)
            else:
                raise FileNotFoundError(extended_path, revision)

        return output

    def parse_diff_revision(self, extended_path, revision_str, *args, **kwargs):
        """Guess revision based on extended_path.

        Revision is part of file path, called extended-path,
        revision_str contains only modification's timestamp.
        """

        if extended_path.endswith(os.path.join(os.sep, 'main', '0')):
            revision = PRE_CREATION
        elif (extended_path.endswith('CHECKEDOUT')
            or not '@@' in extended_path):
            revision = HEAD
        else:
            revision = extended_path.rsplit('@@', 1)[1]

        return extended_path, revision

    def get_fields(self):
        return ['basedir', 'diff_path']

    def get_parser(self, data):
        return ClearCaseDiffParser(data, self.repopath)

class ClearCaseDiffParser(DiffParser):
    """
    Special parsing for diffs created with the post-review for ClearCase.
    """

    SPECIAL_REGEX = re.compile(r'^==== (\S+) (\S+) ====$')

    def __init__(self, data, repopath):
        self.repopath = repopath
        super(ClearCaseDiffParser, self).__init__(data)

    def parse_diff_header(self, linenum, info):
        """Obtain correct clearcase file paths.

        Paths for the same file may differ from paths in developer view
        because it depends from configspec and this is custom so we
        translate oids, attached by post-review, to filenames to get paths
        working well inside clearcase view on reviewboard side.
        """

        # Because ==== oid oid ==== is present after each header
        # parse standard +++ and --- headers at the first place
        linenum = super(ClearCaseDiffParser, self).parse_diff_header(linenum, info)
        m = self.SPECIAL_REGEX.match(self.lines[linenum])

        if m:
            info['origFile'] = self._oid2filename(m.group(1))
            info['newFile'] = self._oid2filename(m.group(2))
            linenum += 1
            if (linenum < len(self.lines) and
                (self.lines[linenum].startswith("Binary files ") or
                 self.lines[linenum].startswith("Files "))):

                # To consider filenames translated from oids
                # origInfo and newInfo keys must exists.
                # Other files already contain this values field
                # by timestamp from +++/--- diff header.
                info['origInfo'] = ''
                info['newInfo'] = ''

                # Binary files need add origInfo and newInfo manally
                # because they don't have diff's headers (only oids).
                info['binary'] = True
                linenum += 1

        return linenum

    def _oid2filename(self, oid):
        cmdline = ["cleartool", "describe", "-fmt", "%En@@%Vn", "oid:%s" % oid]
        p = subprocess.Popen(
            cmdline,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=self.repopath,
            shell=_popen_shell)

        (res, error) = p.communicate()
        failure = p.poll()

        if failure:
            raise SCMError(error)

        drive = os.path.splitdrive(self.repopath)[0]
        if drive:
            res = os.path.join(drive, res)

        return ClearCaseTool.relpath(res, self.repopath)

class ClearCaseDynamicViewClient(object):
    def __init__(self, path):
        self.path = path

    def cat_file(self, filename, revision):
        f = open(filename, 'r')
        lines = f.readlines()
        f.close()
        return ''.join(lines)

    def list_dir(self, path, revision):
        return ''.join([
            '%s\n' % s
            for s in sorted(os.listdir(path))
        ])

class ClearCaseSnapshotViewClient(object):
    def __init__(self, path):
        self.path = path

    def cat_file(self, extended_path, revision):
        import tempfile
        # Use tempfile to generate temporary filename
        temp = tempfile.NamedTemporaryFile()
        # Remove the file, so cleartool can write to it
        temp.close()

        cmdline = ["cleartool", "get", "-to", temp.name, extended_path]
        p = subprocess.Popen(
            cmdline,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            shell=_popen_shell)

        (res, error) = p.communicate()
        failure = p.poll()

        if failure:
            raise FileNotFoundError(extended_path, revision)

        try:
            fp = open(temp.name, 'r')
            data = fp.read()
            fp.close()
            return data
        except:
            raise FileNotFoundError(extended_path, revision)
<commit_msg>Clearcase often fails in multi-site mode because oids are not yet available on server<commit_after>import logging
import os
import re
import subprocess
import sys

from reviewboard.diffviewer.parser import DiffParser
from reviewboard.scmtools.core import SCMTool, HEAD, PRE_CREATION
from reviewboard.scmtools.errors import SCMError, FileNotFoundError

# This specific import is necessary to handle the paths for
# cygwin enabled machines.
if (sys.platform.startswith('win') or sys.platform.startswith('cygwin')):
    import ntpath as cpath
else:
    import posixpath as cpath

# This is a workaround for buggy Python 2.7.x and Windows 7.
# A console window would pop up every time popen is invoked unless shell=true.
# Original issue was described at http://reviews.reviewboard.org/r/3804/
# Note:
#   - later versions of Windows may probably be impacted too
#   - Python 2.7.x is the only one known to get this issue
import platform

if (sys.version_info[:2] == (2, 7) and
    platform.system() == "Windows" and
    platform.release() == "7"):
    _popen_shell = True
else:
    _popen_shell = False

class ClearCaseTool(SCMTool):
    name = 'ClearCase'
    uses_atomic_revisions = False
    supports_authentication = False
    field_help_text = {
        'path': 'The absolute path to the VOB.',
    }
    dependencies = {
        'executables': ['cleartool'],
    }

    # This regular expression can extract from extended_path
    # pure system path. It is construct from two main parts.
    # First match everything from beginning of line to first
    # occurence of /. Second match parts between /main and
    # numbers (file version).
    # This patch assume each branch present in extended_path
    # was derived from /main and there is no file or directory
    # called "main" in path.
    UNEXTENDED = re.compile(r'^(.+?)/|/?(.+?)/main/?.*?/([0-9]+|CHECKEDOUT)')

    VIEW_SNAPSHOT, VIEW_DYNAMIC, VIEW_UNKNOWN = range(3)

    def __init__(self, repository):
        self.repopath = repository.path

        SCMTool.__init__(self, repository)

        self.viewtype = self._get_view_type(self.repopath)

        if self.viewtype == self.VIEW_SNAPSHOT:
            self.client = ClearCaseSnapshotViewClient(self.repopath)
        elif self.viewtype == self.VIEW_DYNAMIC:
            self.client = ClearCaseDynamicViewClient(self.repopath)
        else:
            raise SCMError('Unsupported view type.')

    def unextend_path(self, extended_path):
        """Remove ClearCase revision and branch informations from path.

        ClearCase paths contain additional informations about branch
        and file version preceded by @@. This function remove this
        parts from ClearCase path to make it more readable
        For example this function convert extended path::

            /vobs/comm@@/main/122/network@@/main/55/sntp
            @@/main/4/src@@/main/1/sntp.c@@/main/8

        to the the to regular path::

            /vobs/comm/network/sntp/src/sntp.c
        """
        if not '@@' in extended_path:
            return HEAD, extended_path

        # Result of regular expression search result is list of tuples.
        # We must flat this to one list. The best way is use list comprehension.
        # b is first because it frequently occure in tuples.
        # Before that remove @@ from path.
        unextended_chunks = [
            b or a
            for a, b, foo in self.UNEXTENDED.findall(extended_path.replace('@@', ''))
        ]

        if sys.platform.startswith('win'):
            # Properly handle full (with drive letter) and UNC paths
            if unextended_chunks[0].endswith(':'):
                unextended_chunks[0] = '%s\\' % unextended_chunks[0]
            elif unextended_chunks[0] == '/' or unextended_chunks[0] == os.sep:
                unextended_chunks[0] = '\\\\'

        # Purpose of realpath is remove parts like /./ generated by
        # ClearCase when vobs branch was fresh created
        unextended_path = cpath.realpath(
            cpath.join(*unextended_chunks)
        )

        revision = extended_path.rsplit('@@', 1)[1]
        if revision.endswith('CHECKEDOUT'):
            revision = HEAD

        return (revision, unextended_path)

    @classmethod
    def relpath(cls, path, start):
        """Wrapper for os.path.relpath for Python 2.4.

        Python 2.4 doesn't have the os.path.relpath function, so this
        approximates it well enough for our needs.

        ntpath.relpath() overflows and throws TypeError for paths containing
        atleast 520 characters (not that hard to encounter in UCM
        repository).
        """
        try:
            return cpath.relpath(path, start)
        except (AttributeError, TypeError):
            if start[-1] != os.sep:
                start += os.sep

            return path[len(start):]

    def normalize_path_for_display(self, filename):
        """Return display friendly path without revision informations.

        In path construct for only display purpuse we don't need
        information about branch, version or even repository path
        so we return unextended path relative to repopath (view)
        """
        return self.relpath(self.unextend_path(filename)[1], self.repopath)

    def get_repository_info(self):
        vobstag = self._get_vobs_tag(self.repopath)
        return {
            'repopath': self.repopath,
            'uuid': self._get_vobs_uuid(vobstag)
        }

    def _get_view_type(self, repopath):
        cmdline = ["cleartool", "lsview", "-full", "-properties", "-cview"]
        p = subprocess.Popen(
            cmdline,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=repopath,
            shell=_popen_shell)

        (res, error) = p.communicate()
        failure = p.poll()

        if failure:
            raise SCMError(error)

        for line in res.splitlines(True):
            splitted = line.split(' ')
            if splitted[0] == 'Properties:':
                if 'snapshot' in splitted:
                    return self.VIEW_SNAPSHOT
                elif 'dynamic' in splitted:
                    return self.VIEW_DYNAMIC

        return self.VIEW_UNKNOWN

    def _get_vobs_tag(self, repopath):
        cmdline = ["cleartool", "describe", "-short", "vob:."]
        p = subprocess.Popen(
            cmdline,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=self.repopath,
            shell=_popen_shell)

        (res, error) = p.communicate()
        failure = p.poll()

        if failure:
            raise SCMError(error)

        return res.rstrip()

    def _get_vobs_uuid(self, vobstag):
        cmdline = ["cleartool", "lsvob", "-long", vobstag]
        p = subprocess.Popen(
            cmdline,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=self.repopath,
            shell=_popen_shell)

        (res, error) = p.communicate()
        failure = p.poll()

        if failure:
            raise SCMError(error)

        for line  in res.splitlines(True):
            if line.startswith('Vob family uuid:'):
                return line.split(' ')[-1].rstrip()

        raise SCMError("Can't find familly uuid for vob: %s" % vobstag)

    def _get_object_kind(self, extended_path):
        cmdline = ["cleartool", "desc", "-fmt", "%m", extended_path]
        p = subprocess.Popen(
            cmdline,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=self.repopath,
            shell=_popen_shell)

        (res, error) = p.communicate()
        failure = p.poll()

        if failure:
            raise SCMError(error)

        return res.strip()

    def get_file(self, extended_path, revision=HEAD):
        """Return content of file or list content of directory"""
        if not extended_path:
            raise FileNotFoundError(extended_path, revision)

        if revision == PRE_CREATION:
            return ''

        if self.viewtype == self.VIEW_SNAPSHOT:
            # Get the path to (presumably) file element (remove version)
            # The '@@' at the end of file_path is required.
            file_path = extended_path.rsplit('@@', 1)[0] + '@@'
            okind = self._get_object_kind(file_path)

            if okind == 'directory element':
                raise SCMError('Directory elements are unsupported.')
            elif okind == 'file element':
                output = self.client.cat_file(extended_path, revision)
            else:
                raise FileNotFoundError(extended_path, revision)
        else:
            if cpath.isdir(extended_path):
                output = self.client.list_dir(extended_path, revision)
            elif cpath.exists(extended_path):
                output = self.client.cat_file(extended_path, revision)
            else:
                raise FileNotFoundError(extended_path, revision)

        return output

    def parse_diff_revision(self, extended_path, revision_str, *args, **kwargs):
        """Guess revision based on extended_path.

        Revision is part of file path, called extended-path,
        revision_str contains only modification's timestamp.
        """

        if extended_path.endswith(os.path.join(os.sep, 'main', '0')):
            revision = PRE_CREATION
        elif (extended_path.endswith('CHECKEDOUT')
            or not '@@' in extended_path):
            revision = HEAD
        else:
            revision = extended_path.rsplit('@@', 1)[1]

        return extended_path, revision

    def get_fields(self):
        return ['basedir', 'diff_path']

    def get_parser(self, data):
        return ClearCaseDiffParser(data,
                                   self.repopath,
                                   self._get_vobs_tag(self.repopath))

class ClearCaseDiffParser(DiffParser):
    """
    Special parsing for diffs created with the post-review for ClearCase.
    """

    SPECIAL_REGEX = re.compile(r'^==== (\S+) (\S+) ====$')

    def __init__(self, data, repopath, vobstag):
        self.repopath = repopath
        self.vobstag = vobstag
        super(ClearCaseDiffParser, self).__init__(data)

    def parse_diff_header(self, linenum, info):
        """Obtain correct clearcase file paths.

        Paths for the same file may differ from paths in developer view
        because it depends from configspec and this is custom so we
        translate oids, attached by post-review, to filenames to get paths
        working well inside clearcase view on reviewboard side.
        """

        # Because ==== oid oid ==== is present after each header
        # parse standard +++ and --- headers at the first place
        linenum = super(ClearCaseDiffParser, self).parse_diff_header(linenum, info)
        m = self.SPECIAL_REGEX.match(self.lines[linenum])

        if m:
            # When using ClearCase in multi-site mode, data replication takes
            # much time, including oid. As said above, oid is used to retrieve
            # filename path independent of developer view.
            # When an oid is not found on server side an exception is thrown
            # and review request submission fails.
            # However at this time origFile and newFile info have already been
            # filled by super.parse_diff_header and contain client side paths,
            # client side paths are enough to start reviewing.
            # So we can safely catch exception and restore client side paths
            # if not found.
            currentFilename = info['origFile']
            try:
                info['origFile'] = self._oid2filename(m.group(1))
            except:
                logging.debug("oid (%s) not found, get filename from client",
                              m.group(1))
                info['origFile'] = self.client_relpath(currentFilename)

            currentFilename = info['newFile']
            try:
               info['newFile'] = self._oid2filename(m.group(2))
            except:
                logging.debug("oid (%s) not found, get filename from client",
                              m.group(2))
                info['newFile'] = self.client_relpath(currentFilename)

            linenum += 1
            if (linenum < len(self.lines) and
                (self.lines[linenum].startswith("Binary files ") or
                 self.lines[linenum].startswith("Files "))):

                # To consider filenames translated from oids
                # origInfo and newInfo keys must exists.
                # Other files already contain this values field
                # by timestamp from +++/--- diff header.
                info['origInfo'] = ''
                info['newInfo'] = ''

                # Binary files need add origInfo and newInfo manally
                # because they don't have diff's headers (only oids).
                info['binary'] = True
                linenum += 1

        return linenum

    def _oid2filename(self, oid):
        cmdline = ["cleartool", "describe", "-fmt", "%En@@%Vn", "oid:%s" % oid]
        p = subprocess.Popen(
            cmdline,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=self.repopath,
            shell=_popen_shell)

        (res, error) = p.communicate()
        failure = p.poll()

        if failure:
            raise SCMError(error)

        drive = os.path.splitdrive(self.repopath)[0]
        if drive:
            res = os.path.join(drive, res)

        return ClearCaseTool.relpath(res, self.repopath)

    def client_relpath(self, filename):
        """Normalize any path sent from client view and return relative path
        against vobtag
        """
	path, revision = filename.split("@@", 1)
        relpath = ""
        logging.debug("vobstag: %s, path: %s", self.vobstag, path)
        while True:
            # An error should be raised if vobstag cannot be reached.
            if path == "/":
                logging.debug("vobstag not found in path, use client filename")
                return filename
            # Vobstag reach, relpath can be returned.
            if path.endswith(self.vobstag):
                break
            path, basename = os.path.split(path)
            # Init relpath with basename.
            if len(relpath) == 0:
                relpath = basename
            else:
                relpath = os.path.join(basename, relpath)

        logging.debug("relpath: %s", relpath)
        return relpath + "@@" + revision

class ClearCaseDynamicViewClient(object):
    def __init__(self, path):
        self.path = path

    def cat_file(self, filename, revision):
        f = open(filename, 'r')
        lines = f.readlines()
        f.close()
        return ''.join(lines)

    def list_dir(self, path, revision):
        return ''.join([
            '%s\n' % s
            for s in sorted(os.listdir(path))
        ])

class ClearCaseSnapshotViewClient(object):
    def __init__(self, path):
        self.path = path

    def cat_file(self, extended_path, revision):
        import tempfile
        # Use tempfile to generate temporary filename
        temp = tempfile.NamedTemporaryFile()
        # Remove the file, so cleartool can write to it
        temp.close()

        cmdline = ["cleartool", "get", "-to", temp.name, extended_path]
        p = subprocess.Popen(
            cmdline,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            shell=_popen_shell)

        (res, error) = p.communicate()
        failure = p.poll()

        if failure:
            raise FileNotFoundError(extended_path, revision)

        try:
            fp = open(temp.name, 'r')
            data = fp.read()
            fp.close()
            return data
        except:
            raise FileNotFoundError(extended_path, revision)
<|endoftext|>


<commit_before>## @file
# This tool encodes and decodes GUIDed FFS sections or FMP capsule for a GUID type of
# EFI_CERT_TYPE_RSA2048_SHA256_GUID defined in the UEFI 2.4 Specification as
#   {0xa7717414, 0xc616, 0x4977, {0x94, 0x20, 0x84, 0x47, 0x12, 0xa7, 0x35, 0xbf}}
# This tool has been tested with OpenSSL 1.0.1e 11 Feb 2013
#
# Copyright (c) 2013 - 2017, Intel Corporation. All rights reserved.

# This program and the accompanying materials
# are licensed and made available under the terms and conditions of the BSD License
# which accompanies this distribution.  The full text of the license may be found at
# http://opensource.org/licenses/bsd-license.php
#
# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
#

'''
Rsa2048Sha256Sign
'''

import os
import sys
import argparse
import subprocess
import uuid
import struct
import collections
from Common.BuildVersion import gBUILD_VERSION

#
# Globals for help information
#
__prog__      = 'Rsa2048Sha256Sign'
__version__   = '%s Version %s' % (__prog__, '0.9 ' + gBUILD_VERSION)
__copyright__ = 'Copyright (c) 2013 - 2016, Intel Corporation. All rights reserved.'
__usage__     = '%s -e|-d [options] <input_file>' % (__prog__)

#
# GUID for SHA 256 Hash Algorithm from UEFI Specification
#
EFI_HASH_ALGORITHM_SHA256_GUID = uuid.UUID('{51aa59de-fdf2-4ea3-bc63-875fb7842ee9}')

#
# Structure defintion to unpack EFI_CERT_BLOCK_RSA_2048_SHA256 from UEFI 2.4 Specification
#
#   typedef struct _EFI_CERT_BLOCK_RSA_2048_SHA256 {
#     EFI_GUID HashType;
#     UINT8 PublicKey[256];
#     UINT8 Signature[256];
#   } EFI_CERT_BLOCK_RSA_2048_SHA256;
#
EFI_CERT_BLOCK_RSA_2048_SHA256        = collections.namedtuple('EFI_CERT_BLOCK_RSA_2048_SHA256', ['HashType','PublicKey','Signature'])
EFI_CERT_BLOCK_RSA_2048_SHA256_STRUCT = struct.Struct('16s256s256s')

#
# Filename of test signing private key that is stored in same directory as this tool
#
TEST_SIGNING_PRIVATE_KEY_FILENAME = 'TestSigningPrivateKey.pem'

if __name__ == '__main__':
  #
  # Create command line argument parser object
  #
  parser = argparse.ArgumentParser(prog=__prog__, version=__version__, usage=__usage__, description=__copyright__, conflict_handler='resolve')
  group = parser.add_mutually_exclusive_group(required=True)
  group.add_argument("-e", action="store_true", dest='Encode', help='encode file')
  group.add_argument("-d", action="store_true", dest='Decode', help='decode file')
  parser.add_argument("-o", "--output", dest='OutputFile', type=str, metavar='filename', help="specify the output filename", required=True)
  parser.add_argument("--monotonic-count", dest='MonotonicCountStr', type=str, help="specify the MonotonicCount in FMP capsule.")
  parser.add_argument("--private-key", dest='PrivateKeyFile', type=argparse.FileType('rb'), help="specify the private key filename.  If not specified, a test signing key is used.")
  parser.add_argument("-v", "--verbose", dest='Verbose', action="store_true", help="increase output messages")
  parser.add_argument("-q", "--quiet", dest='Quiet', action="store_true", help="reduce output messages")
  parser.add_argument("--debug", dest='Debug', type=int, metavar='[0-9]', choices=range(0,10), default=0, help="set debug level")
  parser.add_argument(metavar="input_file", dest='InputFile', type=argparse.FileType('rb'), help="specify the input filename")

  #
  # Parse command line arguments
  #
  args = parser.parse_args()

  #
  # Generate file path to Open SSL command
  #
  OpenSslCommand = 'openssl'
  try:
    OpenSslPath = os.environ['OPENSSL_PATH']
    OpenSslCommand = os.path.join(OpenSslPath, OpenSslCommand)
    if ' ' in OpenSslCommand:
      OpenSslCommand = '"' + OpenSslCommand + '"'
  except:
    pass

  #
  # Verify that Open SSL command is available
  #
  try:
    Process = subprocess.Popen('%s version' % (OpenSslCommand), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
  except:
    print 'ERROR: Open SSL command not available.  Please verify PATH or set OPENSSL_PATH'
    sys.exit(1)

  Version = Process.communicate()
  if Process.returncode <> 0:
    print 'ERROR: Open SSL command not available.  Please verify PATH or set OPENSSL_PATH'
    sys.exit(Process.returncode)
  print Version[0]

  #
  # Read input file into a buffer and save input filename
  #
  args.InputFileName   = args.InputFile.name
  args.InputFileBuffer = args.InputFile.read()
  args.InputFile.close()

  #
  # Save output filename and check if path exists
  #
  OutputDir = os.path.dirname(args.OutputFile)
  if not os.path.exists(OutputDir):
    print 'ERROR: The output path does not exist: %s' % OutputDir
    sys.exit(1)
  args.OutputFileName = args.OutputFile

  #
  # Save private key filename and close private key file
  #
  try:
    args.PrivateKeyFileName = args.PrivateKeyFile.name
    args.PrivateKeyFile.close()
  except:
    try:
      #
      # Get path to currently executing script or executable
      #
      if hasattr(sys, 'frozen'):
          RsaToolPath = sys.executable
      else:
          RsaToolPath = sys.argv[0]
      if RsaToolPath.startswith('"'):
          RsaToolPath = RsaToolPath[1:]
      if RsaToolPath.endswith('"'):
          RsaToolPath = RsaToolPath[:-1]
      args.PrivateKeyFileName = os.path.join(os.path.dirname(os.path.realpath(RsaToolPath)), TEST_SIGNING_PRIVATE_KEY_FILENAME)
      args.PrivateKeyFile = open(args.PrivateKeyFileName, 'rb')
      args.PrivateKeyFile.close()
    except:
      print 'ERROR: test signing private key file %s missing' % (args.PrivateKeyFileName)
      sys.exit(1)

  #
  # Extract public key from private key into STDOUT
  #
  Process = subprocess.Popen('%s rsa -in "%s" -modulus -noout' % (OpenSslCommand, args.PrivateKeyFileName), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
  PublicKeyHexString = Process.communicate()[0].split('=')[1].strip()
  PublicKey = ''
  while len(PublicKeyHexString) > 0:
    PublicKey = PublicKey + chr(int(PublicKeyHexString[0:2],16))
    PublicKeyHexString=PublicKeyHexString[2:]
  if Process.returncode <> 0:
    sys.exit(Process.returncode)

  if args.MonotonicCountStr:
    try:
      if args.MonotonicCountStr.upper().startswith('0X'):
        args.MonotonicCountValue = (long)(args.MonotonicCountStr, 16)
      else:
        args.MonotonicCountValue = (long)(args.MonotonicCountStr)
    except:
        pass

  if args.Encode:
    FullInputFileBuffer = args.InputFileBuffer
    if args.MonotonicCountStr:
      format = "%dsQ" % len(args.InputFileBuffer)
      FullInputFileBuffer = struct.pack(format, args.InputFileBuffer, args.MonotonicCountValue)
    #
    # Sign the input file using the specified private key and capture signature from STDOUT
    #
    Process = subprocess.Popen('%s sha1 -sha256 -sign "%s"' % (OpenSslCommand, args.PrivateKeyFileName), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
    Signature = Process.communicate(input=FullInputFileBuffer)[0]
    if Process.returncode <> 0:
      sys.exit(Process.returncode)

    #
    # Write output file that contains hash GUID, Public Key, Signature, and Input data
    #
    args.OutputFile = open(args.OutputFileName, 'wb')
    args.OutputFile.write(EFI_HASH_ALGORITHM_SHA256_GUID.get_bytes_le())
    args.OutputFile.write(PublicKey)
    args.OutputFile.write(Signature)
    args.OutputFile.write(args.InputFileBuffer)
    args.OutputFile.close()

  if args.Decode:
    #
    # Parse Hash Type, Public Key, and Signature from the section header
    #
    Header = EFI_CERT_BLOCK_RSA_2048_SHA256._make(EFI_CERT_BLOCK_RSA_2048_SHA256_STRUCT.unpack_from(args.InputFileBuffer))
    args.InputFileBuffer = args.InputFileBuffer[EFI_CERT_BLOCK_RSA_2048_SHA256_STRUCT.size:]

    #
    # Verify that the Hash Type matches the expected SHA256 type
    #
    if uuid.UUID(bytes_le = Header.HashType) <> EFI_HASH_ALGORITHM_SHA256_GUID:
      print 'ERROR: unsupport hash GUID'
      sys.exit(1)

    #
    # Verify the public key
    #
    if Header.PublicKey <> PublicKey:
      print 'ERROR: Public key in input file does not match public key from private key file'
      sys.exit(1)

    FullInputFileBuffer = args.InputFileBuffer
    if args.MonotonicCountStr:
      format = "%dsQ" % len(args.InputFileBuffer)
      FullInputFileBuffer = struct.pack(format, args.InputFileBuffer, args.MonotonicCountValue)

    #
    # Write Signature to output file
    #
    open(args.OutputFileName, 'wb').write(Header.Signature)

    #
    # Verify signature
    #
    Process = subprocess.Popen('%s sha1 -sha256 -prverify "%s" -signature %s' % (OpenSslCommand, args.PrivateKeyFileName, args.OutputFileName), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
    Process.communicate(input=FullInputFileBuffer)
    if Process.returncode <> 0:
      print 'ERROR: Verification failed'
      os.remove (args.OutputFileName)
      sys.exit(Process.returncode)

    #
    # Save output file contents from input file
    #
    open(args.OutputFileName, 'wb').write(args.InputFileBuffer)
<commit_msg>BaseTools: Update Rsa2048Sha256Sign to use openssl dgst option<commit_after>## @file
# This tool encodes and decodes GUIDed FFS sections or FMP capsule for a GUID type of
# EFI_CERT_TYPE_RSA2048_SHA256_GUID defined in the UEFI 2.4 Specification as
#   {0xa7717414, 0xc616, 0x4977, {0x94, 0x20, 0x84, 0x47, 0x12, 0xa7, 0x35, 0xbf}}
# This tool has been tested with OpenSSL 1.0.1e 11 Feb 2013
#
# Copyright (c) 2013 - 2018, Intel Corporation. All rights reserved.

# This program and the accompanying materials
# are licensed and made available under the terms and conditions of the BSD License
# which accompanies this distribution.  The full text of the license may be found at
# http://opensource.org/licenses/bsd-license.php
#
# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
#

'''
Rsa2048Sha256Sign
'''

import os
import sys
import argparse
import subprocess
import uuid
import struct
import collections
from Common.BuildVersion import gBUILD_VERSION

#
# Globals for help information
#
__prog__      = 'Rsa2048Sha256Sign'
__version__   = '%s Version %s' % (__prog__, '0.9 ' + gBUILD_VERSION)
__copyright__ = 'Copyright (c) 2013 - 2016, Intel Corporation. All rights reserved.'
__usage__     = '%s -e|-d [options] <input_file>' % (__prog__)

#
# GUID for SHA 256 Hash Algorithm from UEFI Specification
#
EFI_HASH_ALGORITHM_SHA256_GUID = uuid.UUID('{51aa59de-fdf2-4ea3-bc63-875fb7842ee9}')

#
# Structure defintion to unpack EFI_CERT_BLOCK_RSA_2048_SHA256 from UEFI 2.4 Specification
#
#   typedef struct _EFI_CERT_BLOCK_RSA_2048_SHA256 {
#     EFI_GUID HashType;
#     UINT8 PublicKey[256];
#     UINT8 Signature[256];
#   } EFI_CERT_BLOCK_RSA_2048_SHA256;
#
EFI_CERT_BLOCK_RSA_2048_SHA256        = collections.namedtuple('EFI_CERT_BLOCK_RSA_2048_SHA256', ['HashType','PublicKey','Signature'])
EFI_CERT_BLOCK_RSA_2048_SHA256_STRUCT = struct.Struct('16s256s256s')

#
# Filename of test signing private key that is stored in same directory as this tool
#
TEST_SIGNING_PRIVATE_KEY_FILENAME = 'TestSigningPrivateKey.pem'

if __name__ == '__main__':
  #
  # Create command line argument parser object
  #
  parser = argparse.ArgumentParser(prog=__prog__, version=__version__, usage=__usage__, description=__copyright__, conflict_handler='resolve')
  group = parser.add_mutually_exclusive_group(required=True)
  group.add_argument("-e", action="store_true", dest='Encode', help='encode file')
  group.add_argument("-d", action="store_true", dest='Decode', help='decode file')
  parser.add_argument("-o", "--output", dest='OutputFile', type=str, metavar='filename', help="specify the output filename", required=True)
  parser.add_argument("--monotonic-count", dest='MonotonicCountStr', type=str, help="specify the MonotonicCount in FMP capsule.")
  parser.add_argument("--private-key", dest='PrivateKeyFile', type=argparse.FileType('rb'), help="specify the private key filename.  If not specified, a test signing key is used.")
  parser.add_argument("-v", "--verbose", dest='Verbose', action="store_true", help="increase output messages")
  parser.add_argument("-q", "--quiet", dest='Quiet', action="store_true", help="reduce output messages")
  parser.add_argument("--debug", dest='Debug', type=int, metavar='[0-9]', choices=range(0,10), default=0, help="set debug level")
  parser.add_argument(metavar="input_file", dest='InputFile', type=argparse.FileType('rb'), help="specify the input filename")

  #
  # Parse command line arguments
  #
  args = parser.parse_args()

  #
  # Generate file path to Open SSL command
  #
  OpenSslCommand = 'openssl'
  try:
    OpenSslPath = os.environ['OPENSSL_PATH']
    OpenSslCommand = os.path.join(OpenSslPath, OpenSslCommand)
    if ' ' in OpenSslCommand:
      OpenSslCommand = '"' + OpenSslCommand + '"'
  except:
    pass

  #
  # Verify that Open SSL command is available
  #
  try:
    Process = subprocess.Popen('%s version' % (OpenSslCommand), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
  except:
    print 'ERROR: Open SSL command not available.  Please verify PATH or set OPENSSL_PATH'
    sys.exit(1)

  Version = Process.communicate()
  if Process.returncode <> 0:
    print 'ERROR: Open SSL command not available.  Please verify PATH or set OPENSSL_PATH'
    sys.exit(Process.returncode)
  print Version[0]

  #
  # Read input file into a buffer and save input filename
  #
  args.InputFileName   = args.InputFile.name
  args.InputFileBuffer = args.InputFile.read()
  args.InputFile.close()

  #
  # Save output filename and check if path exists
  #
  OutputDir = os.path.dirname(args.OutputFile)
  if not os.path.exists(OutputDir):
    print 'ERROR: The output path does not exist: %s' % OutputDir
    sys.exit(1)
  args.OutputFileName = args.OutputFile

  #
  # Save private key filename and close private key file
  #
  try:
    args.PrivateKeyFileName = args.PrivateKeyFile.name
    args.PrivateKeyFile.close()
  except:
    try:
      #
      # Get path to currently executing script or executable
      #
      if hasattr(sys, 'frozen'):
          RsaToolPath = sys.executable
      else:
          RsaToolPath = sys.argv[0]
      if RsaToolPath.startswith('"'):
          RsaToolPath = RsaToolPath[1:]
      if RsaToolPath.endswith('"'):
          RsaToolPath = RsaToolPath[:-1]
      args.PrivateKeyFileName = os.path.join(os.path.dirname(os.path.realpath(RsaToolPath)), TEST_SIGNING_PRIVATE_KEY_FILENAME)
      args.PrivateKeyFile = open(args.PrivateKeyFileName, 'rb')
      args.PrivateKeyFile.close()
    except:
      print 'ERROR: test signing private key file %s missing' % (args.PrivateKeyFileName)
      sys.exit(1)

  #
  # Extract public key from private key into STDOUT
  #
  Process = subprocess.Popen('%s rsa -in "%s" -modulus -noout' % (OpenSslCommand, args.PrivateKeyFileName), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
  PublicKeyHexString = Process.communicate()[0].split('=')[1].strip()
  PublicKey = ''
  while len(PublicKeyHexString) > 0:
    PublicKey = PublicKey + chr(int(PublicKeyHexString[0:2],16))
    PublicKeyHexString=PublicKeyHexString[2:]
  if Process.returncode <> 0:
    sys.exit(Process.returncode)

  if args.MonotonicCountStr:
    try:
      if args.MonotonicCountStr.upper().startswith('0X'):
        args.MonotonicCountValue = (long)(args.MonotonicCountStr, 16)
      else:
        args.MonotonicCountValue = (long)(args.MonotonicCountStr)
    except:
        pass

  if args.Encode:
    FullInputFileBuffer = args.InputFileBuffer
    if args.MonotonicCountStr:
      format = "%dsQ" % len(args.InputFileBuffer)
      FullInputFileBuffer = struct.pack(format, args.InputFileBuffer, args.MonotonicCountValue)
    #
    # Sign the input file using the specified private key and capture signature from STDOUT
    #
    Process = subprocess.Popen('%s dgst -sha256 -sign "%s"' % (OpenSslCommand, args.PrivateKeyFileName), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
    Signature = Process.communicate(input=FullInputFileBuffer)[0]
    if Process.returncode <> 0:
      sys.exit(Process.returncode)

    #
    # Write output file that contains hash GUID, Public Key, Signature, and Input data
    #
    args.OutputFile = open(args.OutputFileName, 'wb')
    args.OutputFile.write(EFI_HASH_ALGORITHM_SHA256_GUID.get_bytes_le())
    args.OutputFile.write(PublicKey)
    args.OutputFile.write(Signature)
    args.OutputFile.write(args.InputFileBuffer)
    args.OutputFile.close()

  if args.Decode:
    #
    # Parse Hash Type, Public Key, and Signature from the section header
    #
    Header = EFI_CERT_BLOCK_RSA_2048_SHA256._make(EFI_CERT_BLOCK_RSA_2048_SHA256_STRUCT.unpack_from(args.InputFileBuffer))
    args.InputFileBuffer = args.InputFileBuffer[EFI_CERT_BLOCK_RSA_2048_SHA256_STRUCT.size:]

    #
    # Verify that the Hash Type matches the expected SHA256 type
    #
    if uuid.UUID(bytes_le = Header.HashType) <> EFI_HASH_ALGORITHM_SHA256_GUID:
      print 'ERROR: unsupport hash GUID'
      sys.exit(1)

    #
    # Verify the public key
    #
    if Header.PublicKey <> PublicKey:
      print 'ERROR: Public key in input file does not match public key from private key file'
      sys.exit(1)

    FullInputFileBuffer = args.InputFileBuffer
    if args.MonotonicCountStr:
      format = "%dsQ" % len(args.InputFileBuffer)
      FullInputFileBuffer = struct.pack(format, args.InputFileBuffer, args.MonotonicCountValue)

    #
    # Write Signature to output file
    #
    open(args.OutputFileName, 'wb').write(Header.Signature)

    #
    # Verify signature
    #
    Process = subprocess.Popen('%s dgst -sha256 -prverify "%s" -signature %s' % (OpenSslCommand, args.PrivateKeyFileName, args.OutputFileName), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
    Process.communicate(input=FullInputFileBuffer)
    if Process.returncode <> 0:
      print 'ERROR: Verification failed'
      os.remove (args.OutputFileName)
      sys.exit(Process.returncode)

    #
    # Save output file contents from input file
    #
    open(args.OutputFileName, 'wb').write(args.InputFileBuffer)
<|endoftext|>


<commit_before>import threading
from lib.irc_connect import IRCConnection
from lib.api_connect import APIConnection
from lib.db_connect import NoSQLConnection
from stream import Stream
import time

print('connecting to noSQL database')
con = NoSQLConnection()

print('connecting to IRC server')
irc = IRCConnection()

print('creating API connection')
api = APIConnection()

# Constants
#  the time to live for the joining user elements in database
join_ttl = 600
#  this is the number of seconds before leave elements are
#  removed
leave_ttl = 600

#  if the number of users from IRC is less then 100 then
# the result is double checked with an API call
irc_min_users = 20

# the limit for joining and leaving entries to be considered related
related_limit = 300

# the limit for two migrations to be stored in the database
# if there is another migration where the user leaves and joins the same
# streams within leaving_overlap_limit seconds of the current migration
# then the current migration won't be added
leaving_overlap_limit = 300
joining_overlap_limit = 300

# This controls the delay in thread start in order to more evenly distribute
# work
thread_start_delay = 0.2

print_lock = threading.RLock()
irc_lock = threading.RLock()
api_lock = threading.RLock()

watching = []

def main():
    global watching

    start_time = time.time()

    print("""
    -------------------------------------------------------
    streams updated now starting from beginning
    -------------------------------------------------------
    """)

    streams = get_monitored_streams()
    streams = [stream['streamname'] for stream in streams[0]['streams']]

    # eliminate any Streams in watching where stream.name is not in streams
    watching = list(filter(lambda s: s.name in streams, watching))
    # add any streams in streams but not in watching
    # by creating a new Stream object
    for stream in streams:
        if stream not in list(map(lambda s: s.name, watching)):
            watching.append(Stream(stream))

    # at this point the streams in watching should be the same streams in
    # streams

    threads = []
    # print('\n{}\n'.format(threading.active_count()))
    for stream in streams:
        t = threading.Thread(target=update_stream, name=stream + '-thread',
                             args=(stream,))
        # print('{} threads created'.format(len(streams)), end='\r')
        threads.append(t)
        t.start()
        time.sleep(thread_start_delay)

    for t in threads:
        t.join()
        print(('{} threads created : {} threads remain')
              .format(len(streams), threading.active_count()), end='\r')
    print('all threads joined back into main thread')

    run_time = time.time() - start_time
    print('\nIt took {0} seconds to run on {1} streams'.format(run_time, len(streams)))

def update_stream(streamname):
    global watching

    for i in range(10):
        #  check that this stream is in the list of streams to watch
        stream = list(filter(lambda s: s.name == streamname, watching))
        if len(stream) != 1:
            print(('{} is not in watching. Something went ' +
                   'wrong').format(streamname))
            return
        stream = stream[0]

        # with print_lock:
        #     print('Entering thread: ' + stream.name)
        users = get_users(stream.name)
        #  if the users list is empty something wen't wrong because there should
        #  always be at least one watcher
        if len(users) == 0:
            return

        #  TODO possibly include remove_stale_joiners and remove_stale_leavers in
        #  the update_watching function
        stream.update_watching(set(users))
        stream.remove_stale_joiners(join_ttl)
        stream.remove_stale_leavers(leave_ttl)

        #  replace this stream in watching
        for i, s in enumerate(watching):
            if s.name == streamname:
                watching[i] = stream

        record_viewcount(stream)

        for j in watching:
            if j.name == streamname:
                continue
            # only record migrations from this stream to all other streams
            record_migrations(stream, j)

        # with print_lock:
        #     print('Exiting thread: ' + stream.name)
        #     print('\n\n{}\n\n'.format(threading.active_count()))

def record_viewcount(stream):
    """
    records the current viewcount of stream in the database
    """
    #  record current viewcount
    con.db[con.viewercount_collection].insert_one(
        {
            'streamname': stream.name,
            'user_count': len(stream.watching),
            'time': time.time()
        })

def record_migrations(from_stream, to_stream):
    """
    finds any users who have moved left from_stream and joined to_stream and records these
    in the database
    @return: doesn't return anything, instead stores it in the database
    """
    #  for every user that has left l and joined j
    for user in set(from_stream.leaving.keys()) & set(to_stream.joining.keys()):
        # the difference between leaving and joining time
        migration_time = abs(from_stream.leaving[user] - to_stream.joining[user])
        if migration_time < related_limit:
            # # selects any documents with the same user moving from
            # matching_migration = con.db[con.migration_collection].find_one(
            #     {
            #         '$or': [
            #             {
            #                 'username': user,
            #                 'from_stream': from_stream.name,
            #                 'to_stream': to_stream.name,
            #                 'leave_time': {
            #                     '$gt': from_stream.leaving[user] - leaving_overlap_limit
            #                 }
            #             },
            #             {
            #                 'username': user,
            #                 'from_stream': from_stream.name,
            #                 'to_stream': to_stream.name,
            #                 'join_time': {
            #                     '$gt': to_stream.joining[user] - joining_overlap_limit
            #                 }
            #             }
            #         ]
            #     })
            print(('user {0} left stream {1} and went to stream ' +
                   '{2}').format(user, from_stream.name, to_stream.name))
            con.db[con.migration_collection].insert_one(
                {
                    'username': user,
                    'from_stream': from_stream.name,
                    'to_stream': to_stream.name,
                    'leave_time': from_stream.leaving[user],
                    'join_time': to_stream.joining[user]
                })

def get_users(channel):
    """
    takes in a channel name and gathers the list of users currently signed into
    twitch and watching that channel
    @return: returns an array of strings that are users watching this channel
    """
    global headers
    global irc_lock
    global api_lock

    with irc_lock:
        # print(threading.current_thread().name + ' getting users for: ' +
        #       channel)
        users = irc.get_channel_users(channel)
        # print((threading.current_thread().name + ' length of users gotten ' +
        #       'from IRC is {0}').format(len(users)))

    #  print random string to make reading terminal easier
    # print(''.join(random.choice(string.ascii_uppercase + string.digits) for _
    #       in range(5)))

    if len(users) < irc_min_users:
        with api_lock:
            # print(('IRC user count is less than {0}, now double checking ' +
            #       'with API call').format(irc_min_users))
            users = api.get_users(channel)

    return users

def get_monitored_streams():
    #  get list of channels that are being monitored
    return con.db[con.monitoring_collection].find(
        {
            'list_category': 'main_list',
        },
        {
            '_id': False,
            'streams': True
        },
    )

if __name__ == '__main__':
    main()
<commit_msg>Changed db structure to make finding duplicate migrations easier instead of each migration being on document, each from stream and to stream pair gets a document and an array of migrations that contain the users name and the leave and join time. By putting an index on the from and to streams, and on the leaving and joining times this speeds up how long it takes to check for a duplicate migration so that the entire time is close to how it was before.<commit_after>import threading
from lib.irc_connect import IRCConnection
from lib.api_connect import APIConnection
from lib.db_connect import NoSQLConnection
from stream import Stream
import time

print('connecting to noSQL database')
con = NoSQLConnection()

print('connecting to IRC server')
irc = IRCConnection()

print('creating API connection')
api = APIConnection()

# Constants
#  the time to live for the joining user elements in database
join_ttl = 600
#  this is the number of seconds before leave elements are
#  removed
leave_ttl = 600

#  if the number of users from IRC is less then 100 then
# the result is double checked with an API call
irc_min_users = 20

# the limit for joining and leaving entries to be considered related
related_limit = 300

# the limit for two migrations to be stored in the database
# if there is another migration where the user leaves and joins the same
# streams within leaving_overlap_limit seconds of the current migration
# then the current migration won't be added
leaving_overlap_limit = 300
joining_overlap_limit = 300

# This controls the delay in thread start in order to more evenly distribute
# work
thread_start_delay = 0.2

print_lock = threading.RLock()
irc_lock = threading.RLock()
api_lock = threading.RLock()

watching = []

def main():
    global watching

    start_time = time.time()

    print("""
    -------------------------------------------------------
    streams updated now starting from beginning
    -------------------------------------------------------
    """)

    streams = get_monitored_streams()
    streams = [stream['streamname'] for stream in streams[0]['streams']]

    # eliminate any Streams in watching where stream.name is not in streams
    watching = list(filter(lambda s: s.name in streams, watching))
    # add any streams in streams but not in watching
    # by creating a new Stream object
    for stream in streams:
        if stream not in list(map(lambda s: s.name, watching)):
            watching.append(Stream(stream))

    # at this point the streams in watching should be the same streams in
    # streams

    threads = []
    # print('\n{}\n'.format(threading.active_count()))
    for stream in streams:
        t = threading.Thread(target=update_stream, name=stream + '-thread',
                             args=(stream,))
        # print('{} threads created'.format(len(streams)), end='\r')
        threads.append(t)
        t.start()
        time.sleep(thread_start_delay)

    for t in threads:
        t.join()
        print(('{} threads created : {} threads remain')
              .format(len(streams), threading.active_count()), end='\r')
    print('all threads joined back into main thread')

    run_time = time.time() - start_time
    print('\nIt took {0} seconds to run on {1} streams'.format(run_time, len(streams)))

def update_stream(streamname):
    global watching

    for i in range(10):
        #  check that this stream is in the list of streams to watch
        stream = list(filter(lambda s: s.name == streamname, watching))
        if len(stream) != 1:
            print(('{} is not in watching. Something went ' +
                   'wrong').format(streamname))
            return
        stream = stream[0]

        # with print_lock:
        #     print('Entering thread: ' + stream.name)

        users = get_users(stream.name)
        #  if the users list is empty something wen't wrong because there should
        #  always be at least one watcher
        if len(users) == 0:
            return

        #  TODO possibly include remove_stale_joiners and remove_stale_leavers in
        #  the update_watching function
        stream.update_watching(set(users))
        stream.remove_stale_joiners(join_ttl)
        stream.remove_stale_leavers(leave_ttl)

        #  replace this stream in watching
        for i, s in enumerate(watching):
            if s.name == streamname:
                watching[i] = stream

        record_viewcount(stream)

        for j in watching:
            if j.name == streamname:
                continue
            # only record migrations from this stream to all other streams
            record_migrations(stream, j)

        # with print_lock:
        #     print('Exiting thread: ' + stream.name)
        #     print('\n\n{}\n\n'.format(threading.active_count()))

def record_viewcount(stream):
    """
    records the current viewcount of stream in the database
    """
    #  record current viewcount
    con.db[con.viewercount_collection].insert_one(
        {
            'streamname': stream.name,
            'user_count': len(stream.watching),
            'time': time.time()
        })

def record_migrations(from_stream, to_stream):
    """
    finds any users who have moved left from_stream and joined to_stream and records these
    in the database
    @return: doesn't return anything, instead stores it in the database
    """
    #  for every user that has left l and joined j
    for user in set(from_stream.leaving.keys()) & set(to_stream.joining.keys()):
        # the difference between leaving and joining time
        migration_time = abs(from_stream.leaving[user] - to_stream.joining[user])
        if migration_time < related_limit:

            print(('user {0} left stream {1} and went to stream ' +
                   '{2}').format(user, from_stream.name, to_stream.name))
            add_migration_to_db(from_stream.name, to_stream.name, user,
                                from_stream.leaving[user],
                                to_stream.joining[user])

def add_migration_to_db(from_stream, to_stream, user, leave_time, join_time):
    """

    """
    # If there is no document for migrations from from_stream to to_stream
    # create it
    con.db[con.migration_collection].update_one(
        {
            'from_stream': from_stream,
            'to_stream': to_stream
        },
        {
            '$set': {
                'from_stream': from_stream,
                'to_stream': to_stream
            }
        },
        upsert=True)

    # Insert this new migration of user from from_stream to to_stream into the
    # list of migrations.
    # If there is already a migration with either the leaving time or joining time
    con.db[con.migration_collection].update_one(
        {
            'from_stream': from_stream,
            'to_stream': to_stream,
            'migrations': {
                '$elemMatch': {
                    '$or': [
                        {
                            'username': user,
                            'leave_time': {
                                '$lt': leave_time - leaving_overlap_limit
                            }
                        },
                        {
                            'username': user,
                            'join_time': {
                                '$lt': join_time - joining_overlap_limit
                            }
                        }
                    ]
                }
            }
        },
        {
            '$push': {
                'migrations': {
                    'username': user,
                    'leave_time': leave_time,
                    'join_time': join_time
                }
            }
        },
        upsert=False)

def get_users(channel):
    """
    takes in a channel name and gathers the list of users currently signed into
    twitch and watching that channel
    @return: returns an array of strings that are users watching this channel
    """
    global headers
    global irc_lock
    global api_lock

    with irc_lock:
        # print(threading.current_thread().name + ' getting users for: ' +
        #       channel)
        users = irc.get_channel_users(channel)
        # print((threading.current_thread().name + ' length of users gotten ' +
        #       'from IRC is {0}').format(len(users)))

    #  print random string to make reading terminal easier
    # print(''.join(random.choice(string.ascii_uppercase + string.digits) for _
    #       in range(5)))

    if len(users) < irc_min_users:
        with api_lock:
            # print(('IRC user count is less than {0}, now double checking ' +
            #       'with API call').format(irc_min_users))
            users = api.get_users(channel)

    return users

def get_monitored_streams():
    #  get list of channels that are being monitored
    return con.db[con.monitoring_collection].find(
        {
            'list_category': 'main_list',
        },
        {
            '_id': False,
            'streams': True
        },
    )

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

from bs4 import BeautifulSoup
import phabricator
import json
import datetime
import dateutil.parser
import os
import string
import pickle

MWCORETEAM_PHID = "PHID-PROJ-oft3zinwvih7bgdhpfgj"
WORKBOARD_HTML_CACHE = '/home/robla/2014/phabworkboard-data/html'
WORKBOARD_PICKLE_CACHE = '/home/robla/2014/phabworkboard-data/pickles'

# Scrape the HTML for a Phabricator workboard, and return a simple dict
# that represents the workboard.  The HTML is pre-retrieved via cron job
# that snarfs the HTML as much as hourly.  This function accesses the
# cache via timestamp, which is rounded to the nearest hour in the file
# name.
def parse_workboard_html(wbtime):
    # File name will look something like workboard-2014-12-22T00.html,
    # which corresponds to midnight on 2014-12-22
    filename = 'workboard-{:%Y-%m-%dT%H%Z}.html'.format(wbtime)
    htmlhandle = open(os.path.join(WORKBOARD_HTML_CACHE, filename))
    soup = BeautifulSoup(htmlhandle)
    columns = soup.find_all(class_="phui-workpanel-view")
    retval = {}
    for col in columns:
        state = col.find(class_="phui-action-header-title").strings.next()
        objnames = col.find_all(class_="phui-object-item-objname")
        for objname in objnames:
            retval[objname.string] = state
    return retval

# Take data structures representing the task states in two workboards,
# and return a dict that contains the tasks for which the state changed
# between the two workboards.  The contents of each item in the dict
# should be a tuple with the old state and the new state.
def get_workboard_diff(old_workboard, new_workboard):
    allkeys = list(set(old_workboard.keys()).union(new_workboard.keys()))
    diff = {}
    for key in allkeys:
        oldvalue = old_workboard.get(key)
        newvalue = new_workboard.get(key)
        if oldvalue != newvalue:
            diff[key] = (oldvalue, newvalue)
    return diff

def get_activity_for_tasks(phab, cachedir, tasks):
    tasknums = [int(string.lstrip(x, "T")) for x in tasks]
    activityquery = lambda: phab.maniphest.gettasktransactions(ids=tasknums)
    activity = call_phab_via_cache(cachedir, "gettasktransactions", activityquery)
    return activity

# Really lame overzealous caching implementation that's only currently
# useful as a developer convenience.  It stores queries to Phabricator
# based on named token.  Purging is entirely manual, even if the query
# changes (no signature checking).
def call_phab_via_cache(cachedir, key, apicall):
    picklefile = os.path.join(cachedir, key + ".pickle")
    try:
        result = pickle.load(open(picklefile))
    except IOError:
        result = apicall()
        pickle.dump(result, open(picklefile, "wb"))
    return result

# Return an item if it's relevant to our current search, or {} if it isn't.
# Also return any PHIDs that need to be resolved.
def get_filtered_transactions_for_task(taskfeed):
    transactions = []
    phids = set()
    for tact in taskfeed:
        item = {}
        item['transactionType'] = tact["transactionType"]
        item['timestamp'] = int(tact['dateCreated'])
        item['authorPHID'] = tact['authorPHID']
        phids.add(item['authorPHID'])
        if (tact["transactionType"] == "status"):
            item['oldValue'] = tact['oldValue']
            item['newValue'] = tact['newValue']
        elif (tact["transactionType"] == "reassign"):
            item['oldValue'] = tact['oldValue']
            if tact['oldValue']:
                phids.add(item['oldValue'])
            item['newValue'] = tact['newValue']
            if tact['newValue']:
                phids.add(item['newValue'])
        elif (tact["transactionType"] == "projectcolumn" and
                tact["oldValue"]["projectPHID"] == MWCORETEAM_PHID):
            oldvalphids = tact['oldValue']['columnPHIDs']
            if isinstance(oldvalphids, dict):
                item['oldValue'] = oldvalphids.values()[0]
                phids.add(item['oldValue'])
            else:
                item['oldValue'] = None
            item['newValue'] = tact['newValue']['columnPHIDs'][0]
            phids.add(item['newValue'])
        else:
            item = {}
        if item:
            transactions.append(item)
    return transactions, phids

def process_transactions(transactions, start, end):
    taskstate = {}
    taskstate['actorset'] = set()
    for tact in transactions:
        ttime = datetime.datetime.fromtimestamp(tact['timestamp'],
            dateutil.tz.tzutc())
        if ttime > end:
            break
        if tact['authorPHID']:
            if ttime > start and ttime < end:
                taskstate['actorset'].add(tact['authorPHID'])
        if tact['transactionType'] == 'projectcolumn':
            taskstate['column'] = tact['newValue']
        elif tact['transactionType'] == 'status':
            taskstate['status'] = tact['newValue']
        elif tact['transactionType'] == 'reassign':
            taskstate['assignee'] = tact['newValue']
            if tact['newValue']:
                taskstate['actorset'].add(tact['newValue'])
    return taskstate

def render_transaction(tact, phidquery):
    time = datetime.datetime.fromtimestamp(
        tact['timestamp']).strftime("%Y-%m-%d %H:%M UTC")
    author = phidquery[tact['authorPHID']]['name']
    if tact['transactionType'] == 'projectcolumn':
        if tact['oldValue']:
            oldcolumn = phidquery[tact['oldValue']]['name']
        else:
            oldcolumn = '(none)'
        newcolumn = phidquery[tact['newValue']]['name']
        retval = "  {0} {1} Column: '{2}' '{3}'".format(
            time, author, oldcolumn, newcolumn)
    elif tact['transactionType'] == 'status':
        oldstatus = str(tact['oldValue'])
        newstatus = tact['newValue']
        retval = "  {0} {1} Status: '{2}' '{3}'".format(
            time, author, oldstatus, newstatus)
    elif tact['transactionType'] == 'reassign':
        if tact['oldValue']:
            oldassignee = phidquery[tact['oldValue']]['name']
        else:
            oldassignee = '(unassigned)'
        if tact['newValue']:
            newassignee = phidquery[tact['newValue']]['name']
        else:
            newassignee = '(unassigned)'
        retval = "  {0} {1} Assignee: '{2}' '{3}'".format(
            time, author, oldassignee, newassignee)
    return retval

def main():
    phab = phabricator.Phabricator()
    cachedir = WORKBOARD_PICKLE_CACHE
    start = dateutil.parser.parse("2014-12-22T0:00PST")
    end = dateutil.parser.parse("2014-12-23T0:00PST")
    old_workboard = parse_workboard_html(start)
    new_workboard = parse_workboard_html(end)
    diff = get_workboard_diff(old_workboard, new_workboard)
    allkeys = list(set(old_workboard.keys()).union(new_workboard.keys()))
    activity = get_activity_for_tasks(phab, cachedir, allkeys)

    transactions = {}
    phids = set()
    for tasknum, taskfeed in activity.iteritems():
        tacts, newphids = get_filtered_transactions_for_task(taskfeed)
        transactions[tasknum] = tacts
        phids.update(newphids)

    actortasks = {}
    taskstate = {}
    for task in transactions.keys():
        taskstate[task] = process_transactions(transactions[task], start, end)
        for actor in taskstate[task]['actorset']:
            assert actor
            if actortasks.get(actor):
                actortasks[actor].append(task)
            else:
                actortasks[actor] = [task]

    phidapifunc = lambda: phab.phid.query(phids=list(phids))
    phidquery = call_phab_via_cache(cachedir, "phidquery", phidapifunc)
    for actor, tasklist in actortasks.iteritems():
        print "Actor: " + phidquery[actor]['name']
        for task in tasklist:
            print "  Task T{0}".format(task)
            for tact in transactions[task]:
                ttime = datetime.datetime.fromtimestamp(tact['timestamp'],
                    dateutil.tz.tzutc())
                if ttime > start and ttime < end:
                    print "  " + render_transaction(tact, phidquery)

if __name__ == "__main__":
    main()
<commit_msg>PEP8 cleanup<commit_after>#!/usr/bin/env python

from bs4 import BeautifulSoup
import phabricator
import json
import datetime
import dateutil.parser
import os
import string
import pickle

MWCORETEAM_PHID = "PHID-PROJ-oft3zinwvih7bgdhpfgj"
WORKBOARD_HTML_CACHE = '/home/robla/2014/phabworkboard-data/html'
WORKBOARD_PICKLE_CACHE = '/home/robla/2014/phabworkboard-data/pickles'

# Scrape the HTML for a Phabricator workboard, and return a simple dict
# that represents the workboard.  The HTML is pre-retrieved via cron job
# that snarfs the HTML as much as hourly.  This function accesses the
# cache via timestamp, which is rounded to the nearest hour in the file
# name.
def parse_workboard_html(wbtime):
    # File name will look something like workboard-2014-12-22T00.html,
    # which corresponds to midnight on 2014-12-22
    filename = 'workboard-{:%Y-%m-%dT%H%Z}.html'.format(wbtime)
    htmlhandle = open(os.path.join(WORKBOARD_HTML_CACHE, filename))
    soup = BeautifulSoup(htmlhandle)
    columns = soup.find_all(class_="phui-workpanel-view")
    retval = {}
    for col in columns:
        state = col.find(class_="phui-action-header-title").strings.next()
        objnames = col.find_all(class_="phui-object-item-objname")
        for objname in objnames:
            retval[objname.string] = state
    return retval

# Take data structures representing the task states in two workboards,
# and return a dict that contains the tasks for which the state changed
# between the two workboards.  The contents of each item in the dict
# should be a tuple with the old state and the new state.
def get_workboard_diff(old_workboard, new_workboard):
    allkeys = list(set(old_workboard.keys()).union(new_workboard.keys()))
    diff = {}
    for key in allkeys:
        oldvalue = old_workboard.get(key)
        newvalue = new_workboard.get(key)
        if oldvalue != newvalue:
            diff[key] = (oldvalue, newvalue)
    return diff

def get_activity_for_tasks(phab, cachedir, tasks):
    tasknums = [int(string.lstrip(x, "T")) for x in tasks]
    activityquery = lambda: phab.maniphest.gettasktransactions(ids=tasknums)
    activity = call_phab_via_cache(
        cachedir, "gettasktransactions", activityquery)
    return activity

# Really lame overzealous caching implementation that's only currently
# useful as a developer convenience.  It stores queries to Phabricator
# based on named token.  Purging is entirely manual, even if the query
# changes (no signature checking).
def call_phab_via_cache(cachedir, key, apicall):
    picklefile = os.path.join(cachedir, key + ".pickle")
    try:
        result = pickle.load(open(picklefile))
    except IOError:
        result = apicall()
        pickle.dump(result, open(picklefile, "wb"))
    return result

# Return an item if it's relevant to our current search, or {} if it isn't.
# Also return any PHIDs that need to be resolved.

def get_filtered_transactions_for_task(taskfeed):
    transactions = []
    phids = set()
    for tact in taskfeed:
        item = {}
        item['transactionType'] = tact["transactionType"]
        item['timestamp'] = int(tact['dateCreated'])
        item['authorPHID'] = tact['authorPHID']
        phids.add(item['authorPHID'])
        if (tact["transactionType"] == "status"):
            item['oldValue'] = tact['oldValue']
            item['newValue'] = tact['newValue']
        elif (tact["transactionType"] == "reassign"):
            item['oldValue'] = tact['oldValue']
            if tact['oldValue']:
                phids.add(item['oldValue'])
            item['newValue'] = tact['newValue']
            if tact['newValue']:
                phids.add(item['newValue'])
        elif (tact["transactionType"] == "projectcolumn" and
                tact["oldValue"]["projectPHID"] == MWCORETEAM_PHID):
            oldvalphids = tact['oldValue']['columnPHIDs']
            if isinstance(oldvalphids, dict):
                item['oldValue'] = oldvalphids.values()[0]
                phids.add(item['oldValue'])
            else:
                item['oldValue'] = None
            item['newValue'] = tact['newValue']['columnPHIDs'][0]
            phids.add(item['newValue'])
        else:
            item = {}
        if item:
            transactions.append(item)
    return transactions, phids

def process_transactions(transactions, start, end):
    taskstate = {}
    taskstate['actorset'] = set()
    for tact in transactions:
        ttime = datetime.datetime.fromtimestamp(tact['timestamp'],
                                                dateutil.tz.tzutc())
        if ttime > end:
            break
        if tact['authorPHID']:
            if ttime > start and ttime < end:
                taskstate['actorset'].add(tact['authorPHID'])
        if tact['transactionType'] == 'projectcolumn':
            taskstate['column'] = tact['newValue']
        elif tact['transactionType'] == 'status':
            taskstate['status'] = tact['newValue']
        elif tact['transactionType'] == 'reassign':
            taskstate['assignee'] = tact['newValue']
            if tact['newValue']:
                taskstate['actorset'].add(tact['newValue'])
    return taskstate

def render_transaction(tact, phidquery):
    time = datetime.datetime.fromtimestamp(
        tact['timestamp']).strftime("%Y-%m-%d %H:%M UTC")
    author = phidquery[tact['authorPHID']]['name']
    if tact['transactionType'] == 'projectcolumn':
        if tact['oldValue']:
            oldcolumn = phidquery[tact['oldValue']]['name']
        else:
            oldcolumn = '(none)'
        newcolumn = phidquery[tact['newValue']]['name']
        retval = "  {0} {1} Column: '{2}' '{3}'".format(
            time, author, oldcolumn, newcolumn)
    elif tact['transactionType'] == 'status':
        oldstatus = str(tact['oldValue'])
        newstatus = tact['newValue']
        retval = "  {0} {1} Status: '{2}' '{3}'".format(
            time, author, oldstatus, newstatus)
    elif tact['transactionType'] == 'reassign':
        if tact['oldValue']:
            oldassignee = phidquery[tact['oldValue']]['name']
        else:
            oldassignee = '(unassigned)'
        if tact['newValue']:
            newassignee = phidquery[tact['newValue']]['name']
        else:
            newassignee = '(unassigned)'
        retval = "  {0} {1} Assignee: '{2}' '{3}'".format(
            time, author, oldassignee, newassignee)
    return retval

def main():
    phab = phabricator.Phabricator()
    cachedir = WORKBOARD_PICKLE_CACHE
    start = dateutil.parser.parse("2014-12-22T0:00PST")
    end = dateutil.parser.parse("2014-12-23T0:00PST")
    old_workboard = parse_workboard_html(start)
    new_workboard = parse_workboard_html(end)
    diff = get_workboard_diff(old_workboard, new_workboard)
    allkeys = list(set(old_workboard.keys()).union(new_workboard.keys()))
    activity = get_activity_for_tasks(phab, cachedir, allkeys)

    transactions = {}
    phids = set()
    for tasknum, taskfeed in activity.iteritems():
        tacts, newphids = get_filtered_transactions_for_task(taskfeed)
        transactions[tasknum] = tacts
        phids.update(newphids)

    actortasks = {}
    taskstate = {}
    for task in transactions.keys():
        taskstate[task] = process_transactions(transactions[task], start, end)
        for actor in taskstate[task]['actorset']:
            assert actor
            if actortasks.get(actor):
                actortasks[actor].append(task)
            else:
                actortasks[actor] = [task]

    phidapifunc = lambda: phab.phid.query(phids=list(phids))
    phidquery = call_phab_via_cache(cachedir, "phidquery", phidapifunc)
    for actor, tasklist in actortasks.iteritems():
        print "Actor: " + phidquery[actor]['name']
        for task in tasklist:
            print "  Task T{0}".format(task)
            for tact in transactions[task]:
                ttime = datetime.datetime.fromtimestamp(tact['timestamp'],
                                                        dateutil.tz.tzutc())
                if ttime > start and ttime < end:
                    print "  " + render_transaction(tact, phidquery)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python
#===============================================================================
# weatherx.py
#
# Get weather forecast from NOAA and turn it into "icons"
#   * Kind of tailored to work with 12-hour format
#   * NOAA's doc: http://graphical.weather.gov/xml/rest.php
#   * Set location with LAT and LON (negative LON for west)
#   * Somewhat generalized for any number of days
#
#===============================================================================
import sys
import httplib
import time
from xml.dom.minidom import parseString
from Adafruit_LED_Backpack import Matrix8x8
from led8x8icons import LED8x8_ICONS

matrix = []
matrix.append(Matrix8x8.Matrix8x8(address=0x70))
matrix.append(Matrix8x8.Matrix8x8(address=0x71))
matrix.append(Matrix8x8.Matrix8x8(address=0x72))
matrix.append(Matrix8x8.Matrix8x8(address=0x73))

#------------------------------------
# do this if anything bad happens
#------------------------------------
def giveup():
  for i in xrange(4):
    set_led(LED8x8_ICONS['UNKNOWN'],i)
  print "something happened...giving up...."
  sys.exit(1)

#------------------------------------
# set up
#------------------------------------
NOAA_URL    = "graphical.weather.gov"
REQ_BASE    = r"/xml/sample_products/browser_interface/ndfdBrowserClientByDay.php?"
LAT         = 47.54583
LON         = -122.31361
TIME_FORMAT = "12+hourly"
NUM_DAYS    = 4
REQUEST = REQ_BASE + "lat={0}&".format(LAT)+\
                     "lon={0}&".format(LON)+\
                     "format={0}&".format(TIME_FORMAT)+\
                     "numDays={0}".format(NUM_DAYS)

#------------------------------------
# set 8x8 matrix based on bitmap
#------------------------------------
def set_led(bitmap,i):
  for x in xrange(8):
    for y in xrange(8):
      matrix[i].set_pixel(x, y, bitmap[y][x])
  matrix[i].write_display()

#------------------------------------
# set 8x8 matrix based on 64 bit value
#------------------------------------
def set_raw64(value, i):
  matrix[i].clear()
  for y in [0, 1, 2, 3, 4, 5, 6, 7]:
    row_byte = value>>(8*y)
    for x in [0, 1, 2, 3, 4, 5, 6, 7]:
      pixel_bit = row_byte>>x&1
      matrix[i].set_pixel(x,y,pixel_bit)
  matrix[i].write_display()

#------------------------------------
# make the request
#------------------------------------
try:
  conn = httplib.HTTPConnection(NOAA_URL)
  conn.request("GET", REQUEST)
  resp = conn.getresponse()
  data = resp.read()
except:
  giveup()

#filename = time.strftime('%y%m%d_%H%M')+"_weather.log"
#with open(filename,"w") as FILE:
#  FILE.write("-"*80+"\n")
#  FILE.write("URL = " + NOAA_URL + "\n")
#  FILE.write("REQUEST = " + REQUEST + "\n")
#  FILE.write("-"*80+"\n")
#  FILE.write(data)

#------------------------------------
# parse the XML response into the DOM
#------------------------------------
dom = parseString(data)

#------------------------------------
# get the weather summay elements
#------------------------------------
vals = dom.getElementsByTagName("weather-conditions")
if len(vals)<2*NUM_DAYS:
    print "Wrong stuff: len = %d   NUM_DAYS = %d" % (len(vals), NUM_DAYS)
    giveup()

#------------------------------------
# The first entry may be for the current day, or current night
# I'll try and fix that using local time.
# Then I want every other entry for the daytime forecast
#------------------------------------
offset = 0
if time.localtime().tm_hour > 12:
   offset = 1
forecast = [e.getAttribute("weather-summary") for e in vals[offset::2]]
#print forecast

#------------------------------------
# print results
#------------------------------------
print '-'*20
print time.strftime('%Y/%m/%d %H:%M:%S')
print '-'*20
for day in forecast:
    print day

#------------------------------------
# set matrix icons based on forecast text
#------------------------------------
for i in xrange(4):
  if "SUNNY" in forecast[i].encode('ascii','ignore').upper():
    set_raw64(LED8x8_ICONS['SUNNY'], i)
  elif "RAIN" in forecast[i].encode('ascii','ignore').upper():
    set_raw64(LED8x8_ICONS['RAIN'], i)
  elif "CLOUD" in forecast[i].encode('ascii','ignore').upper():
    set_raw64(LED8x8_ICONS['CLOUD'], i)
  elif "SHOWERS" in forecast[i].encode('ascii','ignore').upper():
    set_raw64(LED8x8_ICONS['SHOWERS'], i)
  else:
    set_raw64(LED8x8_ICONS['UNKNOWN'], i)
<commit_msg>tweaks and fixes dealing with device tree<commit_after>#!/usr/bin/python
#===============================================================================
# weatherx.py
#
# Get weather forecast from NOAA and turn it into "icons"
#   * Kind of tailored to work with 12-hour format
#   * NOAA's doc: http://graphical.weather.gov/xml/rest.php
#   * Set location with LAT and LON (negative LON for west)
#   * Somewhat generalized for any number of days
#
#===============================================================================
import sys
import httplib
import time
from xml.dom.minidom import parseString
from Adafruit_LED_Backpack import Matrix8x8
from led8x8icons import LED8x8_ICONS

matrix = []
matrix.append(Matrix8x8.Matrix8x8(address=0x70, busnum=1))
matrix.append(Matrix8x8.Matrix8x8(address=0x71, busnum=1))
matrix.append(Matrix8x8.Matrix8x8(address=0x72, busnum=1))
matrix.append(Matrix8x8.Matrix8x8(address=0x73, busnum=1))
for m in matrix:
  m.begin()

#------------------------------------
# do this if anything bad happens
#------------------------------------
def giveup():
  for i in xrange(4):
    set_led(LED8x8_ICONS['UNKNOWN'],i)
  print "something happened...giving up...."
  sys.exit(1)

#------------------------------------
# set up
#------------------------------------
NOAA_URL    = "graphical.weather.gov"
REQ_BASE    = r"/xml/sample_products/browser_interface/ndfdBrowserClientByDay.php?"
LAT         = 47.54583
LON         = -122.31361
TIME_FORMAT = "12+hourly"
NUM_DAYS    = 4
REQUEST = REQ_BASE + "lat={0}&".format(LAT)+\
                     "lon={0}&".format(LON)+\
                     "format={0}&".format(TIME_FORMAT)+\
                     "numDays={0}".format(NUM_DAYS)

#------------------------------------
# set 8x8 matrix based on bitmap
#------------------------------------
def set_led(bitmap,i):
  for x in xrange(8):
    for y in xrange(8):
      matrix[i].set_pixel(x, y, bitmap[y][x])
  matrix[i].write_display()

#------------------------------------
# set 8x8 matrix based on 64 bit value
#------------------------------------
def set_raw64(value, i):
  matrix[i].clear()
  for y in [0, 1, 2, 3, 4, 5, 6, 7]:
    row_byte = value>>(8*y)
    for x in [0, 1, 2, 3, 4, 5, 6, 7]:
      pixel_bit = row_byte>>x&1
      matrix[i].set_pixel(x,y,pixel_bit)
  matrix[i].write_display()

#------------------------------------
# make the request
#------------------------------------
try:
  conn = httplib.HTTPConnection(NOAA_URL)
  conn.request("GET", REQUEST)
  resp = conn.getresponse()
  data = resp.read()
except:
  giveup()

#filename = time.strftime('%y%m%d_%H%M')+"_weather.log"
#with open(filename,"w") as FILE:
#  FILE.write("-"*80+"\n")
#  FILE.write("URL = " + NOAA_URL + "\n")
#  FILE.write("REQUEST = " + REQUEST + "\n")
#  FILE.write("-"*80+"\n")
#  FILE.write(data)

#------------------------------------
# parse the XML response into the DOM
#------------------------------------
dom = parseString(data)

#------------------------------------
# get the weather summay elements
#------------------------------------
vals = dom.getElementsByTagName("weather-conditions")
if len(vals)<2*NUM_DAYS:
    print "Wrong stuff: len = %d   NUM_DAYS = %d" % (len(vals), NUM_DAYS)
    giveup()

#------------------------------------
# The first entry may be for the current day, or current night
# I'll try and fix that using local time.
# Then I want every other entry for the daytime forecast
#------------------------------------
offset = 0
if time.localtime().tm_hour > 12:
   offset = 1
forecast = [e.getAttribute("weather-summary") for e in vals[offset::2]]
#print forecast

#------------------------------------
# print results
#------------------------------------
print '-'*20
print time.strftime('%Y/%m/%d %H:%M:%S')
print '-'*20
for day in forecast:
    print day

#------------------------------------
# set matrix icons based on forecast text
#------------------------------------
for i in xrange(4):
  if "SUNNY" in forecast[i].encode('ascii','ignore').upper():
    set_raw64(LED8x8_ICONS['SUNNY'], i)
  elif "RAIN" in forecast[i].encode('ascii','ignore').upper():
    set_raw64(LED8x8_ICONS['RAIN'], i)
  elif "CLOUD" in forecast[i].encode('ascii','ignore').upper():
    set_raw64(LED8x8_ICONS['CLOUD'], i)
  elif "SHOWERS" in forecast[i].encode('ascii','ignore').upper():
    set_raw64(LED8x8_ICONS['SHOWERS'], i)
  else:
    set_raw64(LED8x8_ICONS['UNKNOWN'], i)
<|endoftext|>


<commit_before>#################################################################################
#
# The MIT License (MIT)
#
# Copyright (c) 2015 Dmitry Sovetov
#
# https://github.com/dmsovetov
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
#################################################################################

import os, fnmatch, collections, json

# File item data type
Item = collections.namedtuple('Item', ['path', 'info'])

# Scans the directory recursively and returns the list of outdated files
def find_outdated(path):
    print 'Scanning', path

    result = {}

    for root, dirs, files in os.walk(path):
        for file_name in files:
            name = os.path.splitext(file_name)[0]
            path = os.path.join(root, file_name)

            if name in result:
                raise Exception('Duplicate file name "{0}"\n\t{1}\n\t{2}'.format(name, path, result[name]))

            result[name] = Item(path=path, info=dict(identifier=name))

    return result

# Generates the manifest from files
def generate_manifest(files):
    assets = [item.info for k, item in files.items()]
    return json.dumps(assets)

# Builds each file to an output folder
def build(tasks, files, path, rules):
    for file_name, item in files.items():
        rule = None
        file_path = item.path
        info = item.info

        for k, v in rules.items():
            if fnmatch.fnmatch(file_path, k):
                rule = v
                break

        if rule:
            tasks.push(rule(info, file_path, os.path.join(path, file_name)))<commit_msg>Modified identifier generation, now all in lower case<commit_after>#################################################################################
#
# The MIT License (MIT)
#
# Copyright (c) 2015 Dmitry Sovetov
#
# https://github.com/dmsovetov
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
#################################################################################

import os, fnmatch, collections, json

# File item data type
Item = collections.namedtuple('Item', ['path', 'info'])

# Scans the directory recursively and returns the list of outdated files
def find_outdated(path):
    print 'Scanning', path

    result = {}

    for root, dirs, files in os.walk(path):
        for file_name in files:
            name = os.path.splitext(file_name)[0].lower()
            path = os.path.join(root, file_name)

            if name in result:
                raise Exception('Duplicate file name "{0}"\n\t{1}\n\t{2}'.format(name, path, result[name]))

            result[name] = Item(path=path, info=dict(identifier=name))

    return result

# Generates the manifest from files
def generate_manifest(files):
    assets = [item.info for k, item in files.items()]
    return json.dumps(assets)

# Builds each file to an output folder
def build(tasks, files, path, rules):
    for file_name, item in files.items():
        rule = None
        file_path = item.path
        info = item.info

        for k, v in rules.items():
            if fnmatch.fnmatch(file_path, k):
                rule = v
                break

        if rule:
            tasks.push(rule(info, file_path, os.path.join(path, file_name)))<|endoftext|>


<commit_before>class Solution:
    # @param a ListNode
    # @return a ListNode
    def swapPairs(self, head):
        if head == None or head.next == None:
            return head
        dummy = ListNode(0); dummy.next = head
        p = dummy
        while p.next and p.next.next:
            tmp = p.next.next
            p.next.next = tmp.next
            tmp.next = p.next
            p.next = tmp
            p = p.next.next
        return dummy.next
<commit_msg>Update SwapPairNodes_001.py<commit_after>#@author: cchen

class Solution:
    # @param a ListNode
    # @return a ListNode
    def swapPairs(self, head):
        if head == None or head.next == None:
            return head
        dummy = ListNode(0); dummy.next = head
        p = dummy
        while p.next and p.next.next:
            tmp = p.next.next
            p.next.next = tmp.next
            tmp.next = p.next
            p.next = tmp
            p = p.next.next
        return dummy.next
<|endoftext|>