<commit_before>import sublime
import sublime_plugin
import os
import re
import functools
import subprocess
from subprocess import check_output as shell

PHANTOM_KEY_ALL = 'git-blame-all'

stylesheet_one = '''

'''

template_one = '''

        {stylesheet}

                Git Blame ({user})
                {date} {time} |
                {sha}
                [Copy]
                [Show]
                ''' + chr(0x00D7) + '''

'''

stylesheet_all = '''

'''

template_all = '''

        {stylesheet}

                {sha} ({user} {date} {time})
                ''' + chr(0x00D7) + '''

'''

# Sometimes this fails on other OS, just error silently
try:
    si = subprocess.STARTUPINFO()
    si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
except:
    si = None

class BlameCommand(sublime_plugin.TextCommand):

    def __init__(self, view):
        self.view = view
        self.phantom_set = sublime.PhantomSet(view, 'git-blame')

    @functools.lru_cache(128, False)
    def get_blame(self, line, path):
        try:
            return shell(["git", "blame", "--minimal", "-w",
                "-L {0},{0}".format(line), path],
                cwd=os.path.dirname(os.path.realpath(path)),
                startupinfo=si,
                stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            print("Git blame: git error {}:\n{}".format(e.returncode, e.output.decode("UTF-8")))
        except Exception as e:
            print("Git blame: Unexpected error:", e)

    def parse_blame(self, blame):
        sha, file_path, user, date, time, tz_offset, *_ = blame.decode('utf-8').split()

        # Was part of the inital commit so no updates
        if file_path[0] == '(':
            user, date, time, tz_offset = file_path, user, date, time
            file_path = None

        # Fix an issue where the username has a space
        # Im going to need to do something better though if people
        # start to have multiple spaces in their names.
        if not date[0].isdigit():
            user = "{0} {1}".format(user, date)
            date, time = time, tz_offset

        return(sha, user[1:], date, time)

    def get_commit(self, sha, path):
        try:
            return shell(["git", "show", sha],
                cwd=os.path.dirname(os.path.realpath(path)),
                startupinfo=si)
        except Exception as e:
            return

    def on_phantom_close(self, href):
        href_parts = href.split('-')

        if len(href_parts) > 1:
            intent = href_parts[0]
            sha = href_parts[1]
            # The SHA output by git-blame may have a leading caret to indicate
            # that it is a "boundary commit". That useful information has
            # already been shown in the phantom, so strip it before going on to
            # use the SHA programmatically.
            sha = sha.strip('^')

            if intent == "copy":
                sublime.set_clipboard(sha)
                sublime.status_message('Git SHA copied to clipboard')
            elif intent == "show":
                desc = self.get_commit(sha, self.view.file_name()).decode('utf-8')
                buf = self.view.window().new_file()
                buf.run_command('insert_commit_description', {'desc': desc})
            else:
                self.view.erase_phantoms('git-blame')
        else:
            self.view.erase_phantoms('git-blame')

    def run(self, edit):
        phantoms = []
        self.view.erase_phantoms('git-blame')

        for region in self.view.sel():
            line = self.view.line(region)
            (row, col) = self.view.rowcol(region.begin())
            full_path = self.view.file_name()
            result = self.get_blame(int(row) + 1, full_path)
            if not result:
                # Unable to get blame
                return

            sha, user, date, time = self.parse_blame(result)

            body = template_one.format(sha=sha, user=user, date=date, time=time, stylesheet=stylesheet_one)

            phantom = sublime.Phantom(line, body, sublime.LAYOUT_BLOCK, self.on_phantom_close)
            phantoms.append(phantom)
        self.phantom_set.update(phantoms)

class BlameShowAllCommand(sublime_plugin.TextCommand):

    # The fixed length for author names
    NAME_LENGTH = 10

    def __init__(self, view):
        super().__init__(view)
        self.phantom_set = sublime.PhantomSet(self.view, PHANTOM_KEY_ALL)
        self.pattern = None

    def run(self, edit):
        if self.view.is_dirty():
            sublime.status_message("The file needs to be saved for git blame.")
            return

        self.view.erase_phantoms(PHANTOM_KEY_ALL)

        blame_lines = self.get_blame_lines(self.view.file_name())

        if not blame_lines:
            return

        phantoms = []
        for l in blame_lines:
            parsed = self.parse_blame(l)
            if not parsed:
                continue

            sha, author, date, time, line_number = parsed

            body = template_all.format(sha=sha,
                                       user=self.format_name(author),
                                       date=date,
                                       time=time,
                                       stylesheet=stylesheet_all)

            line_point = self.get_line_point(line_number - 1)
            phantom = sublime.Phantom(line_point,
                                      body,
                                      sublime.LAYOUT_INLINE,
                                      self.on_phantom_close)
            phantoms.append(phantom)

        self.phantom_set.update(phantoms)

    def get_blame_lines(self, path):
        '''Run `git blame` and get the output lines.
        '''
        try:
            output = shell(["git", "blame", "--minimal", "-w", path],
                cwd=os.path.dirname(os.path.realpath(path)),
                startupinfo=si,
                stderr=subprocess.STDOUT)
            return output.decode("UTF-8").splitlines()
        except subprocess.CalledProcessError as e:
            print("Git blame: git error {}:\n{}".format(e.returncode, e.output.decode("UTF-8")))
        except Exception as e:
            print("Git blame: Unexpected error:", e)

    def parse_blame(self, blame):
        '''Parses git blame output.
        '''
        if not self.pattern:
            self.prepare_pattern()

        m = self.pattern.match(blame)
        if m:
            sha = m.group('sha')
            author = m.group('author')
            date = m.group('date')
            time = m.group('time')
            line_number = int(m.group('line_number'))
            return sha, author, date, time, line_number
        else:
            return None

    def prepare_pattern(self):
        '''Prepares the regex pattern to parse git blame output.
        '''
        # The SHA output by git-blame may have a leading caret to indicate
        # that it is a "boundary commit".
        p_sha = r'(?P<sha>\^?\w+)'
        p_author = r'(?P<author>.+?)'
        p_date = r'(?P<date>\d{4}-\d{2}-\d{2})'
        p_time = r'(?P<time>\d{2}:\d{2}:\d{2})'
        p_timezone = r'(?P<timezone>[\+-]\d+)'
        p_line = r'(?P<line_number>\d+)'
        s = r'\s+'

        self.pattern = re.compile(r'^' + p_sha + s + r'\(' + p_author +
                                  s + p_date + s + p_time + s +
                                  p_timezone + s + p_line + r'\) ')

    def format_name(self, name):
        '''Formats author names so that widths of phantoms become equal.
        '''
        ellipsis = '...'
        if len(name) > self.NAME_LENGTH:
            return name[:self.NAME_LENGTH] + ellipsis
        else:
            return name + '.' * (self.NAME_LENGTH - len(name)) + ellipsis

    def get_line_point(self, line):
        '''Get the point of specified line in a view.
        '''
        return self.view.line(self.view.text_point(line, 0))

    def on_phantom_close(self, href):
        '''Closes opened phantoms.
        '''
        if href == 'close':
            self.view.run_command('blame_erase_all')

class BlameEraseAllCommand(sublime_plugin.TextCommand):

    def run(self, edit):
        '''Erases the blame results.
        '''
        sublime.status_message("The git blame result is cleared.")
        self.view.erase_phantoms(PHANTOM_KEY_ALL)

class BlameEraseAllListener(sublime_plugin.ViewEventListener):

    def on_modified(self):
        '''Automatically erases the blame results to prevent mismatches.
        '''
        self.view.run_command('blame_erase_all')

class InsertCommitDescriptionCommand(sublime_plugin.TextCommand):

    def run(self, edit, desc):
        view = self.view
        view.set_scratch(True)
        view.set_syntax_file('Packages/Diff/Diff.sublime-syntax')
        view.insert(edit, 0, desc)
<commit_msg>Issue #15: Add support for blame results with file names in "blame_show_all" command.<commit_after>import sublime
import sublime_plugin
import os
import re
import functools
import subprocess
from subprocess import check_output as shell

PHANTOM_KEY_ALL = 'git-blame-all'

stylesheet_one = '''

'''

template_one = '''

        {stylesheet}

                Git Blame ({user})
                {date} {time} |
                {sha}
                [Copy]
                [Show]
                ''' + chr(0x00D7) + '''

'''

stylesheet_all = '''

'''

template_all = '''

        {stylesheet}

                {sha} ({user} {date} {time})
                ''' + chr(0x00D7) + '''

'''

# Sometimes this fails on other OS, just error silently
try:
    si = subprocess.STARTUPINFO()
    si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
except:
    si = None

class BlameCommand(sublime_plugin.TextCommand):

    def __init__(self, view):
        self.view = view
        self.phantom_set = sublime.PhantomSet(view, 'git-blame')

    @functools.lru_cache(128, False)
    def get_blame(self, line, path):
        try:
            return shell(["git", "blame", "--minimal", "-w",
                "-L {0},{0}".format(line), path],
                cwd=os.path.dirname(os.path.realpath(path)),
                startupinfo=si,
                stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            print("Git blame: git error {}:\n{}".format(e.returncode, e.output.decode("UTF-8")))
        except Exception as e:
            print("Git blame: Unexpected error:", e)

    def parse_blame(self, blame):
        sha, file_path, user, date, time, tz_offset, *_ = blame.decode('utf-8').split()

        # Was part of the inital commit so no updates
        if file_path[0] == '(':
            user, date, time, tz_offset = file_path, user, date, time
            file_path = None

        # Fix an issue where the username has a space
        # Im going to need to do something better though if people
        # start to have multiple spaces in their names.
        if not date[0].isdigit():
            user = "{0} {1}".format(user, date)
            date, time = time, tz_offset

        return(sha, user[1:], date, time)

    def get_commit(self, sha, path):
        try:
            return shell(["git", "show", sha],
                cwd=os.path.dirname(os.path.realpath(path)),
                startupinfo=si)
        except Exception as e:
            return

    def on_phantom_close(self, href):
        href_parts = href.split('-')

        if len(href_parts) > 1:
            intent = href_parts[0]
            sha = href_parts[1]
            # The SHA output by git-blame may have a leading caret to indicate
            # that it is a "boundary commit". That useful information has
            # already been shown in the phantom, so strip it before going on to
            # use the SHA programmatically.
            sha = sha.strip('^')

            if intent == "copy":
                sublime.set_clipboard(sha)
                sublime.status_message('Git SHA copied to clipboard')
            elif intent == "show":
                desc = self.get_commit(sha, self.view.file_name()).decode('utf-8')
                buf = self.view.window().new_file()
                buf.run_command('insert_commit_description', {'desc': desc})
            else:
                self.view.erase_phantoms('git-blame')
        else:
            self.view.erase_phantoms('git-blame')

    def run(self, edit):
        phantoms = []
        self.view.erase_phantoms('git-blame')

        for region in self.view.sel():
            line = self.view.line(region)
            (row, col) = self.view.rowcol(region.begin())
            full_path = self.view.file_name()
            result = self.get_blame(int(row) + 1, full_path)
            if not result:
                # Unable to get blame
                return

            sha, user, date, time = self.parse_blame(result)

            body = template_one.format(sha=sha, user=user, date=date, time=time, stylesheet=stylesheet_one)

            phantom = sublime.Phantom(line, body, sublime.LAYOUT_BLOCK, self.on_phantom_close)
            phantoms.append(phantom)
        self.phantom_set.update(phantoms)

class BlameShowAllCommand(sublime_plugin.TextCommand):

    # The fixed length for author names
    NAME_LENGTH = 10

    def __init__(self, view):
        super().__init__(view)
        self.phantom_set = sublime.PhantomSet(self.view, PHANTOM_KEY_ALL)
        self.pattern = None

    def run(self, edit):
        if self.view.is_dirty():
            sublime.status_message("The file needs to be saved for git blame.")
            return

        self.view.erase_phantoms(PHANTOM_KEY_ALL)

        blame_lines = self.get_blame_lines(self.view.file_name())

        if not blame_lines:
            return

        phantoms = []
        for l in blame_lines:
            parsed = self.parse_blame(l)
            if not parsed:
                continue

            sha, author, date, time, line_number = parsed

            body = template_all.format(sha=sha,
                                       user=self.format_name(author),
                                       date=date,
                                       time=time,
                                       stylesheet=stylesheet_all)

            line_point = self.get_line_point(line_number - 1)
            phantom = sublime.Phantom(line_point,
                                      body,
                                      sublime.LAYOUT_INLINE,
                                      self.on_phantom_close)
            phantoms.append(phantom)

        self.phantom_set.update(phantoms)

    def get_blame_lines(self, path):
        '''Run `git blame` and get the output lines.
        '''
        try:
            # The option --show-name is necessary to force file name display.
            command = ["git", "blame", "--show-name", "--minimal", "-w", path]
            output = shell(command,
                cwd=os.path.dirname(os.path.realpath(path)),
                startupinfo=si,
                stderr=subprocess.STDOUT)
            return output.decode("UTF-8").splitlines()
        except subprocess.CalledProcessError as e:
            print("Git blame: git error {}:\n{}".format(e.returncode, e.output.decode("UTF-8")))
        except Exception as e:
            print("Git blame: Unexpected error:", e)

    def parse_blame(self, blame):
        '''Parses git blame output.
        '''
        if not self.pattern:
            self.prepare_pattern()

        m = self.pattern.match(blame)
        if m:
            sha = m.group('sha')
            # Currently file is not used.
            # file = m.group('file')
            author = m.group('author')
            date = m.group('date')
            time = m.group('time')
            line_number = int(m.group('line_number'))
            return sha, author, date, time, line_number
        else:
            return None

    def prepare_pattern(self):
        '''Prepares the regex pattern to parse git blame output.
        '''
        # The SHA output by git-blame may have a leading caret to indicate
        # that it is a "boundary commit".
        p_sha = r'(?P<sha>\^?\w+)'
        p_file = r'((?P<file>[\S ]+)\s+)'
        p_author = r'(?P<author>.+?)'
        p_date = r'(?P<date>\d{4}-\d{2}-\d{2})'
        p_time = r'(?P<time>\d{2}:\d{2}:\d{2})'
        p_timezone = r'(?P<timezone>[\+-]\d+)'
        p_line = r'(?P<line_number>\d+)'
        s = r'\s+'

        self.pattern = re.compile(r'^' + p_sha + s + p_file + r'\(' +
                                  p_author + s + p_date + s + p_time + s +
                                  p_timezone + s + p_line + r'\) ')

    def format_name(self, name):
        '''Formats author names so that widths of phantoms become equal.
        '''
        ellipsis = '...'
        if len(name) > self.NAME_LENGTH:
            return name[:self.NAME_LENGTH] + ellipsis
        else:
            return name + '.' * (self.NAME_LENGTH - len(name)) + ellipsis

    def get_line_point(self, line):
        '''Get the point of specified line in a view.
        '''
        return self.view.line(self.view.text_point(line, 0))

    def on_phantom_close(self, href):
        '''Closes opened phantoms.
        '''
        if href == 'close':
            self.view.run_command('blame_erase_all')

class BlameEraseAllCommand(sublime_plugin.TextCommand):

    def run(self, edit):
        '''Erases the blame results.
        '''
        sublime.status_message("The git blame result is cleared.")
        self.view.erase_phantoms(PHANTOM_KEY_ALL)

class BlameEraseAllListener(sublime_plugin.ViewEventListener):

    def on_modified(self):
        '''Automatically erases the blame results to prevent mismatches.
        '''
        self.view.run_command('blame_erase_all')

class InsertCommitDescriptionCommand(sublime_plugin.TextCommand):

    def run(self, edit, desc):
        view = self.view
        view.set_scratch(True)
        view.set_syntax_file('Packages/Diff/Diff.sublime-syntax')
        view.insert(edit, 0, desc)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# rfwsutils - utilities for manipulating RooFit workspaces from the command line
#
# Copyright 2013 University of California, San Diego
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os, re, sys
import ConfigParser

#----------------------------------------------------------------------

def rootListTolist(rootList):
    """ converts a ROOT.list object to a list

        see also http://root.cern.ch/phpBB3/viewtopic.php?f=14&t=11376
    """

    if not rootListTolist.initialized:
        # make sure we load the corresponding class dictionary
        # TODO: this is not thread safe...

        import ROOT
        import tempfile

        linkDefFile = tempfile.NamedTemporaryFile(suffix = ".h")

        print >> linkDefFile, '#include "RooAbsData.h"'
        print >> linkDefFile, '#include <list>'
        print >> linkDefFile, '#ifdef __CINT__'
        print >> linkDefFile, '#pragma link C++ class std::list<RooAbsData*>::iterator;'
        print >> linkDefFile, '#endif'

        linkDefFile.flush()

        # workaround for CMSSW
        if os.environ.has_key("ROOFITSYS"):
            ROOT.gInterpreter.AddIncludePath(os.path.join(os.environ["ROOFITSYS"], "include"))

        # load this generated macro
        ROOT.gROOT.LoadMacro(linkDefFile.name + "+")

        rootListTolist.initialized = True

    #----------

    retval = []

    it = rootList.begin()
    end = rootList.end()

    while it != end:
        retval.append(it.__deref__())
        it.__postinc__(1)

    return retval

# static variable
rootListTolist.initialized = False

#----------------------------------------------------------------------
def addCommonOptions(parser,
                     addSetVars = False,
                     ):
    """ adds common options to the command line arguments parser """

    conf = ConfigParser.SafeConfigParser()
    conf.read(os.path.expanduser("~/.rfwsutilsrc"))

    try:
        defaultLibs = conf.get('options','lib')
        defaultLibs = re.split(',\s*',defaultLibs)

    except ConfigParser.Error:
        defaultLibs = []

    parser.add_option("--lib",
                  dest="lib",
                  default = defaultLibs,
                  type="str",
                  action="append", # append to list
                  help="library to load before opening the files. Can be specified multiple times.",
                  metavar="LIB")

    parser.add_option("--nolib",
                      # note the inverted logic here
                      dest="loadProfileLibs",
                      default = True,
                      action="store_false",
                      help="do NOT load the libraries specified in the startup profile",
                      )

    parser.add_option("-w",
                      dest="workspaceName",
                      default = None,
                      type = str,
                      help="specify the workspace name explicitly (in case more than one is in top level directory in the file)",
                      metavar = "WSNAME",
                      )
    if addSetVars:
        parser.add_option("--set",
                          dest="setVars",
                          default = "",
                          type = str,
                          help="comma separated list of expressions var1=value1,var2=value2,... will set variables to the given values after loading the workspace continuing",
                          metavar = "EXPRS",
                          )

#----------------------------------------------------------------------

def checkCommonOptions(options):
    """ perform some common checks on command line options """

    # TODO: is there a way to see whether the libraries were specified
    #       in --lib or read from the profile ?

    # replace environment variables in the variable expressions
    options.lib = [ os.path.expandvars(x) for x in options.lib ]

    if options.loadProfileLibs:
        for lib in options.lib:
            if not os.path.exists(lib):
                print >> sys.stderr,"library " + lib + " does not exist, exiting"
                sys.exit(1)

    if hasattr(options, "setVars"):
        # parse this

        tmp = {}

        if options.setVars != "":
            for spec in options.setVars.split(","):
                varname, value = spec.split("=",1)
                tmp[varname] = float(value)

        options.setVars = tmp

#----------------------------------------------------------------------

def findWorkspaces(topdir, options):
    import ROOT

    # topdir can be a TFile or more generally a TDirectory
    retval = []

    if options.workspaceName != None:
        # a workspace was specified explicitly
        ws = topdir.Get(options.workspaceName)
        if ws == None:
            print >> sys.stderr,"no workspace named",options.workspaceName,"found"
            sys.exit(1)

        retval = [ ws ]

    else:
        # find all workspaces in the top level directory in the file

        for key in topdir.GetListOfKeys():

            # TODO: support subdirectories
            obj = topdir.Get(key.GetName())
            if isinstance(obj,ROOT.RooWorkspace):
                retval.append(obj)

    return retval

#----------------------------------------------------------------------

def findSingleWorkspace(topdir, options):
    """like findWorkspaces(..) but insists that exactly one workspace was selected/found"""
    import ROOT

    workspaces = findWorkspaces(topdir, options)

    if len(workspaces) > 1:
        print >> sys.stderr,"more than one workspace found"
        sys.exit(1)

    if len(workspaces) < 1:
        print >> sys.stderr,"no workspace found"
        sys.exit(1)

    return workspaces[0]

#----------------------------------------------------------------------

def getObj(ws, name):
    """ tries to find the given object in the workspace or prints
    an error message an exits if there is no such object """

    obj = ws.obj(name)

    if obj == None:
        print >> sys.stderr,"object '" + name + "' not found in workspace " + ws.GetName()
        sys.exit(1)

    return obj

#----------------------------------------------------------------------

def loadLibraries(options):
    """ loads the libraries specified in the options """

    if not options.loadProfileLibs:
        return

    import ROOT

    for lib in options.lib:
        ROOT.gSystem.Load(lib)

#----------------------------------------------------------------------

def getAllMembers(ws):
    """ @return a list of all components of the given workspace """

    components = ws.components()
    it = components.iterator()

    allMembers = []

    while True:

        obj = it.Next()
        if obj == None:
            break

        allMembers.append(obj)

    # add datasets (which seem NOT to be included in the above ?)
    allMembers += rootListTolist(ws.allData())

    return allMembers

#----------------------------------------------------------------------

def applySetVars(workspace, setVarsDict):
    for name, value in setVarsDict.items():
        var = getObj(workspace, name)

        import ROOT

        if not isinstance(var, ROOT.RooRealVar):
            raise Exception("object %s in workspace %s is not of type RooRealVar" % (name, workspace.GetName()))

        # TODO: should we print a warning if the value is outside the range ?

        # adapt maximum or minimum if necessary
        if value < var.getMin():
            var.setMin(value)
        elif value > var.getMax():
            var.setMax(value)

        var.setVal(value)

#----------------------------------------------------------------------

def rooArgSetToList(argset):
    """creates a python list with the contents of argset (which should be a RooArgSet)"""
    it = argset.createIterator()

    retval = []
    while True:
        obj = it.Next()

        if obj == None:
            break

        retval.append(obj)

    return retval

#----------------------------------------------------------------------

def filterAndSortObjects(objs, sortKeyFunc = None, filterFunc = None):

    # potentially filter
    if filterFunc != None:
        objs = filter(filterFunc, objs)
    else:
        # make a copy in any case
        objs = list(objs)

    # potentially sort
    if sortKeyFunc != None:
        objs.sort(key = sortKeyFunc)

    return objs

#----------------------------------------------------------------------

def printVars(vars, sortKeyFunc = None, filterFunc = None):
    # vars is a RooArgSet
    # convert this to a python list
    vars = rooArgSetToList(vars)

    vars = filterAndSortObjects(vars, sortKeyFunc, filterFunc)

    # print them
    for var in vars:
        var.Print()

#----------------------------------------------------------------------

def printVarsCSV(vars):
    # vars is a RooArgSet

    it = vars.iterator()

    print ",".join([
        "name",
        "value",
        "min",
        "max",
        "constant"])

    while True:
        var = it.Next()
        if var == None:
            break

        parts = [
            var.GetName(),
            var.getVal(),
            var.getMin(),
            var.getMax(),
            var.isConstant(),
            ]

        print ",".join([ str(p) for p in parts ])

#----------------------------------------------------------------------

<commit_msg>added arguments for sorting and filtering to function printVarsCSV(..) in wsutils.py<commit_after>#!/usr/bin/env python

# rfwsutils - utilities for manipulating RooFit workspaces from the command line
#
# Copyright 2013 University of California, San Diego
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os, re, sys
import ConfigParser

#----------------------------------------------------------------------

def rootListTolist(rootList):
    """ converts a ROOT.list object to a list

        see also http://root.cern.ch/phpBB3/viewtopic.php?f=14&t=11376
    """

    if not rootListTolist.initialized:
        # make sure we load the corresponding class dictionary
        # TODO: this is not thread safe...

        import ROOT
        import tempfile

        linkDefFile = tempfile.NamedTemporaryFile(suffix = ".h")

        print >> linkDefFile, '#include "RooAbsData.h"'
        print >> linkDefFile, '#include <list>'
        print >> linkDefFile, '#ifdef __CINT__'
        print >> linkDefFile, '#pragma link C++ class std::list<RooAbsData*>::iterator;'
        print >> linkDefFile, '#endif'

        linkDefFile.flush()

        # workaround for CMSSW
        if os.environ.has_key("ROOFITSYS"):
            ROOT.gInterpreter.AddIncludePath(os.path.join(os.environ["ROOFITSYS"], "include"))

        # load this generated macro
        ROOT.gROOT.LoadMacro(linkDefFile.name + "+")

        rootListTolist.initialized = True

    #----------

    retval = []

    it = rootList.begin()
    end = rootList.end()

    while it != end:
        retval.append(it.__deref__())
        it.__postinc__(1)

    return retval

# static variable
rootListTolist.initialized = False

#----------------------------------------------------------------------
def addCommonOptions(parser,
                     addSetVars = False,
                     ):
    """ adds common options to the command line arguments parser """

    conf = ConfigParser.SafeConfigParser()
    conf.read(os.path.expanduser("~/.rfwsutilsrc"))

    try:
        defaultLibs = conf.get('options','lib')
        defaultLibs = re.split(',\s*',defaultLibs)

    except ConfigParser.Error:
        defaultLibs = []

    parser.add_option("--lib",
                  dest="lib",
                  default = defaultLibs,
                  type="str",
                  action="append", # append to list
                  help="library to load before opening the files. Can be specified multiple times.",
                  metavar="LIB")

    parser.add_option("--nolib",
                      # note the inverted logic here
                      dest="loadProfileLibs",
                      default = True,
                      action="store_false",
                      help="do NOT load the libraries specified in the startup profile",
                      )

    parser.add_option("-w",
                      dest="workspaceName",
                      default = None,
                      type = str,
                      help="specify the workspace name explicitly (in case more than one is in top level directory in the file)",
                      metavar = "WSNAME",
                      )
    if addSetVars:
        parser.add_option("--set",
                          dest="setVars",
                          default = "",
                          type = str,
                          help="comma separated list of expressions var1=value1,var2=value2,... will set variables to the given values after loading the workspace continuing",
                          metavar = "EXPRS",
                          )

#----------------------------------------------------------------------

def checkCommonOptions(options):
    """ perform some common checks on command line options """

    # TODO: is there a way to see whether the libraries were specified
    #       in --lib or read from the profile ?

    # replace environment variables in the variable expressions
    options.lib = [ os.path.expandvars(x) for x in options.lib ]

    if options.loadProfileLibs:
        for lib in options.lib:
            if not os.path.exists(lib):
                print >> sys.stderr,"library " + lib + " does not exist, exiting"
                sys.exit(1)

    if hasattr(options, "setVars"):
        # parse this

        tmp = {}

        if options.setVars != "":
            for spec in options.setVars.split(","):
                varname, value = spec.split("=",1)
                tmp[varname] = float(value)

        options.setVars = tmp

#----------------------------------------------------------------------

def findWorkspaces(topdir, options):
    import ROOT

    # topdir can be a TFile or more generally a TDirectory
    retval = []

    if options.workspaceName != None:
        # a workspace was specified explicitly
        ws = topdir.Get(options.workspaceName)
        if ws == None:
            print >> sys.stderr,"no workspace named",options.workspaceName,"found"
            sys.exit(1)

        retval = [ ws ]

    else:
        # find all workspaces in the top level directory in the file

        for key in topdir.GetListOfKeys():

            # TODO: support subdirectories
            obj = topdir.Get(key.GetName())
            if isinstance(obj,ROOT.RooWorkspace):
                retval.append(obj)

    return retval

#----------------------------------------------------------------------

def findSingleWorkspace(topdir, options):
    """like findWorkspaces(..) but insists that exactly one workspace was selected/found"""
    import ROOT

    workspaces = findWorkspaces(topdir, options)

    if len(workspaces) > 1:
        print >> sys.stderr,"more than one workspace found"
        sys.exit(1)

    if len(workspaces) < 1:
        print >> sys.stderr,"no workspace found"
        sys.exit(1)

    return workspaces[0]

#----------------------------------------------------------------------

def getObj(ws, name):
    """ tries to find the given object in the workspace or prints
    an error message an exits if there is no such object """

    obj = ws.obj(name)

    if obj == None:
        print >> sys.stderr,"object '" + name + "' not found in workspace " + ws.GetName()
        sys.exit(1)

    return obj

#----------------------------------------------------------------------

def loadLibraries(options):
    """ loads the libraries specified in the options """

    if not options.loadProfileLibs:
        return

    import ROOT

    for lib in options.lib:
        ROOT.gSystem.Load(lib)

#----------------------------------------------------------------------

def getAllMembers(ws):
    """ @return a list of all components of the given workspace """

    components = ws.components()
    it = components.iterator()

    allMembers = []

    while True:

        obj = it.Next()
        if obj == None:
            break

        allMembers.append(obj)

    # add datasets (which seem NOT to be included in the above ?)
    allMembers += rootListTolist(ws.allData())

    return allMembers

#----------------------------------------------------------------------

def applySetVars(workspace, setVarsDict):
    for name, value in setVarsDict.items():
        var = getObj(workspace, name)

        import ROOT

        if not isinstance(var, ROOT.RooRealVar):
            raise Exception("object %s in workspace %s is not of type RooRealVar" % (name, workspace.GetName()))

        # TODO: should we print a warning if the value is outside the range ?

        # adapt maximum or minimum if necessary
        if value < var.getMin():
            var.setMin(value)
        elif value > var.getMax():
            var.setMax(value)

        var.setVal(value)

#----------------------------------------------------------------------

def rooArgSetToList(argset):
    """creates a python list with the contents of argset (which should be a RooArgSet)"""
    it = argset.createIterator()

    retval = []
    while True:
        obj = it.Next()

        if obj == None:
            break

        retval.append(obj)

    return retval

#----------------------------------------------------------------------

def filterAndSortObjects(objs, sortKeyFunc = None, filterFunc = None):

    # potentially filter
    if filterFunc != None:
        objs = filter(filterFunc, objs)
    else:
        # make a copy in any case
        objs = list(objs)

    # potentially sort
    if sortKeyFunc != None:
        objs.sort(key = sortKeyFunc)

    return objs

#----------------------------------------------------------------------

def printVars(vars, sortKeyFunc = None, filterFunc = None):
    # vars is a RooArgSet
    # convert this to a python list
    vars = rooArgSetToList(vars)

    vars = filterAndSortObjects(vars, sortKeyFunc, filterFunc)

    # print them
    for var in vars:
        var.Print()

#----------------------------------------------------------------------

def printVarsCSV(vars, sortKeyFunc = None, filterFunc = None):
    # vars is a RooArgSet
    # convert this to a python list
    vars = rooArgSetToList(vars)

    vars = filterAndSortObjects(vars, sortKeyFunc, filterFunc)

    print ",".join([
        "name",
        "value",
        "min",
        "max",
        "constant"])

    for var in vars:

        parts = [
            var.GetName(),
            var.getVal(),
            var.getMin(),
            var.getMax(),
            var.isConstant(),
            ]

        print ",".join([ str(p) for p in parts ])

#----------------------------------------------------------------------

<|endoftext|>


<commit_before>import inspect
import string

class WrapperMeta(type):
    """
    Meta class for scikit-learn wrapper classes.

    This is used for docstring generation/templating upon
    class definition.
    The client (class using this meta class) should define a class
    attribute `__wraps__` which contains the wrapped class.

    For instance::

        >>> class Foo(metaclass=WrappedMeta):
        ...     __wrapped__ = Bar
        ...

        >>> print(Foo.__doc__)
        A wrapper for `Bar`

        .. seealso: Bar

    The client can also define a template for the docstring

        >>> class Foo(metaclass=WrappedMeta):
        ...    '''
        ...    Here is what ${sklname} says about itself
        ...    ${skldoc}
        ...    '''
        ...     __wrapped__ = Bar
        ...

        >>> print(Bar.__doc__)
        I am a Bar

        >>> print(Foo.__doc__)
        Here is what Bar says about itself
        I am a Bar

    """

    class DocTemplate(string.Template):
        pattern = r"""
            \$(?:
              (?P<escaped>\$)         |  # escape (double $$)
              (?P<named>\A(?!x)x)     |  # never match unbraced identifiers
              {(?P<braced>[_a-z]\w*)} |  # braced identifier
              (?P<invalid>)           |  # invalid anything else
            )
        """

    def __new__(cls, name, bases, dict_):
        cls = type.__new__(cls, name, bases, dict_)
        docstring = getattr(cls, "__doc__", None)
        skl_wrapped = getattr(cls, "__wraps__", None)

        parent_docs = [parent.__doc__ for parent in bases if parent.__doc__]
        if docstring is None and skl_wrapped and len(parent_docs) > 0:
            docstring = parent_docs[0]

        if docstring is not None and skl_wrapped is not None:
            docstring = WrapperMeta.format_docstring(docstring, skl_wrapped)
            cls.__doc__ = docstring

        return cls

    @staticmethod
    def format_docstring(doc, sklclass):
        module = inspect.getmodule(sklclass)
        # TODO: prettify the name (pull the class up if it is imported at
        # a higher level and included in __all__, like ipython's help)
        doc_prefix = """
    A wrapper for `${sklname}`. The following is the documentation
    from `scikit-learn <http://scikit-learn.org>`_.
    """
        doc = doc_prefix + doc
        sklname = "{}.{}".format(module.__name__, sklclass.__name__)
        mapping = {"sklname": sklname}
        skldoc = inspect.getdoc(sklclass)
        if "Attributes\n---------" in skldoc:
            skldoc = skldoc[:skldoc.index('Attributes\n---------')]
            mapping["skldoc"] = skldoc
        if "Parameters\n---------" in skldoc:
            skldoc = skldoc[:skldoc.index('Parameters\n---------')]
            mapping["sklpar"] = skldoc

        doc = inspect.cleandoc(doc)
        template = WrapperMeta.DocTemplate(doc)
        return template.safe_substitute(mapping)
<commit_msg>Fix WrapperMeta for SklLearners without attributes<commit_after>import inspect
import string

class WrapperMeta(type):
    """
    Meta class for scikit-learn wrapper classes.

    This is used for docstring generation/templating upon
    class definition.
    The client (class using this meta class) should define a class
    attribute `__wraps__` which contains the wrapped class.

    For instance::

        >>> class Foo(metaclass=WrappedMeta):
        ...     __wrapped__ = Bar
        ...

        >>> print(Foo.__doc__)
        A wrapper for `Bar`

        .. seealso: Bar

    The client can also define a template for the docstring

        >>> class Foo(metaclass=WrappedMeta):
        ...    '''
        ...    Here is what ${sklname} says about itself
        ...    ${skldoc}
        ...    '''
        ...     __wrapped__ = Bar
        ...

        >>> print(Bar.__doc__)
        I am a Bar

        >>> print(Foo.__doc__)
        Here is what Bar says about itself
        I am a Bar

    """

    class DocTemplate(string.Template):
        pattern = r"""
            \$(?:
              (?P<escaped>\$)         |  # escape (double $$)
              (?P<named>\A(?!x)x)     |  # never match unbraced identifiers
              {(?P<braced>[_a-z]\w*)} |  # braced identifier
              (?P<invalid>)           |  # invalid anything else
            )
        """

    def __new__(cls, name, bases, dict_):
        cls = type.__new__(cls, name, bases, dict_)
        docstring = getattr(cls, "__doc__", None)
        skl_wrapped = getattr(cls, "__wraps__", None)

        parent_docs = [parent.__doc__ for parent in bases if parent.__doc__]
        if docstring is None and skl_wrapped and len(parent_docs) > 0:
            docstring = parent_docs[0]

        if docstring is not None and skl_wrapped is not None:
            docstring = WrapperMeta.format_docstring(docstring, skl_wrapped)
            cls.__doc__ = docstring

        return cls

    @staticmethod
    def format_docstring(doc, sklclass):
        module = inspect.getmodule(sklclass)
        # TODO: prettify the name (pull the class up if it is imported at
        # a higher level and included in __all__, like ipython's help)
        doc_prefix = """
    A wrapper for `${sklname}`. The following is the documentation
    from `scikit-learn <http://scikit-learn.org>`_.
    """
        doc = doc_prefix + doc
        sklname = "{}.{}".format(module.__name__, sklclass.__name__)
        mapping = {"sklname": sklname}
        skldoc = inspect.getdoc(sklclass)
        if "Attributes\n---------" in skldoc:
            skldoc = skldoc[:skldoc.index('Attributes\n---------')]
            mapping["skldoc"] = skldoc
        if "Examples\n--------" in skldoc:
            skldoc = skldoc[:skldoc.index('Examples\n--------')]
            mapping["skldoc"] = skldoc
        if "Parameters\n---------" in skldoc:
            skldoc = skldoc[:skldoc.index('Parameters\n---------')]
            mapping["sklpar"] = skldoc

        doc = inspect.cleandoc(doc)
        template = WrapperMeta.DocTemplate(doc)
        return template.safe_substitute(mapping)
<|endoftext|>


<commit_before>#!/usr/bin/env python
""" Make box plots from the output of the get_results command"""

# Input:  stdin stream of lines with, hopefully, trend data lines included
#  ex. of a valid line is:
#    Test4.2x32(10 1024) jid(109959) 2014-09-04T08:43:22-0600 avg_iterTime 0.002910 sec
# Output: Should be a messaged indicating where output plot files reside

import sys
import re

print "Making box plots with data from:"
for line in sys.stdin:
    searchObj = re.search(r'jid\(', line, re.M | re.I)
    if searchObj:
        # Should do work here, see correct output above
        print line,  # this line for debug purposes

if __name__ == '__main__':
    # pass entire command line to main except for the command name
    sys.exit()
<commit_msg>initial real code from Phil<commit_after>#!/usr/bin/env python
""" Make box plots from the output of the get_results command"""

# Input:  stdin stream of lines with, hopefully, trend data lines included
#  ex. of a valid line is:
#    Test4.2x32(10 1024) jid(109959) 2014-09-04T08:43:22-0600 avg_iterTime 0.002910 sec
# Output: Should be a messaged indicating where output plot files reside

import sys
import re
import os
import datetime
from numpy import loadtxt
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

import logging

def get_subdirectories( dir):
    return [name for name in os.listdir( dir )
        if os.path.isdir(os.path.join(dir, name))]

def makesingleboxplot(thisdirname, subdirname, thisfilename):

    # print "thisdir ", thisdirname
    maxnodeint = 1
    firsttime = 0
    firstdigit = 1
    segname=[]
    ylabelunits = []
    idecpoint = -1
    ivallen = 0
    iseform = -1
    decdigits = 2

    tablefmtstring = "%d %d "

    thisdatafile = thisdirname + "/" + thisfilename
    datafile = open( thisdatafile )
    for line in datafile:
        thisnode=[]
        del thisnode[:]
        tud,jud,tid,mname,pname2,nodename,trest = line.split(' ',6)
        nfile = thisdirname + "/" + ''.join(nodename)
        if firsttime < 1:

            firstdigit = re.search("\d", ''.join(nodename))
            firstdigit = firstdigit.start()
            # print firstdigit, ''.join(nodename)
            firsttime = 1
            for i in xrange( 0, firstdigit  ):
                # segname[i] = nodename[i]
                segname.append(  nodename[i] )
            segname.append( '%' )

            segname.append( '0' )
            nlen = len( nodename ) - firstdigit
            segname.append( "%d" % nlen )
            # segname.append( len( nodename ) - firstdigit + 1 )
            segname.append( 'd' )
            # print trest
            trestlen = len( trest )
            spdigit = -1
            spdigit = trest.find(' ')
            valstring = []
            if spdigit > -1:
                for j in xrange( 0, spdigit - 1):
                    valstring.append( trest[j] )
            else:
                valstring = trest

            # print "valstring is ", valstring
            if spdigit > 0:
                print "spdigit is not 0, can make units string"
                for j in xrange( spdigit + 1, trestlen - 1 ):
                    ylabelunits.append( trest[j] )

            idecpoint = valstring.find( '.')
            ivallen = len( valstring )
            iseform = valstring.find( 'e' )
            decdigits = 2
            if idecpoint > -1:
                decdigits = decdigits + ivallen - idecpoint - 2

            if iseform > -1:
                vfmtstring = "%%.%de %%.%de %%.%de %%.%de " % (decdigits, decdigits, decdigits - 2, decdigits - 2)
            else:
                vfmtstring = "%%.%df %%.%df %%.%df %%.%df\n" % (decdigits, decdigits, decdigits - 2, decdigits - 2)
            tablefmtstring = tablefmtstring + vfmtstring
            # print tablefmtstring
            # print  segname
        tnode = int( nodename[firstdigit:] )
        if tnode > maxnodeint:
            maxnodeint = tnode

        nodefile = open( nfile, "a" )
        print >> nodefile, trest,
        nodefile.close()

	# Begin processing a list of files in the current directory, complicated list of lists
    chtodir = "cd " + thisdirname
    os.system( chtodir )
    # print "issued command ", chtodir

    fstring = ''.join(segname) # "wf%03d"
    maxnodenum = maxnodeint + 1 #  620
    # print maxnodeint
    if len( ylabelunits) < 1:
        ytitle = "Performance (MB/sec)"
    else:
        ytitle = "Performance " + ylabelunits

    ctitle = subdirname +  " Performance Boxplot"

    fname={}
    dlist=[]
    hpldata=[]
    tmarks=[]
    tname=[]
    nnum=[]
    nticks=0

    for x in range(1,maxnodenum):
        fname[x] = fstring % (x)
        # print "x ", x
        # print fname[x]

        # if  os.path.isfile(fname[x]) :
        if  os.path.isfile( thisdirname + "/" + fname[x]) :
            nticks += 1
            dlist.append( loadtxt( thisdirname + "/" + fname[x]) )
            nnum.append( x )
            if ( x % 2 ) == 0 :
                tmarks.append( "%d" % (x) )
            else:
                tmarks.append('')
    # print "Dlistlength is ",  len(dlist )
    # print dlist

    dmeans=[]
    for j in dlist:
        dmeans.append( j.mean() )

    dmins=[]
    for j in dlist:
        dmins.append( j.min() )

    dmaxs=[]
    for j in dlist:
        dmaxs.append( j.max() )

    dsamps=[]
    for j in dlist:
        dsamps.append( j.size )

    dstds=[]
    for j in dlist:
        dstds.append( j.std() )

    nodetable=[]
    nodetable = zip( nnum, dsamps, dmeans, dstds, dmins, dmaxs )
    # print nodetable
    # for i in nodetable:
        # print i

    tablename = thisdirname + "/FinalDataTable.txt"
    tablefile = open( tablename, "w" )
    for i in nodetable:
        # print >> tablefile, i
        # print >> tablefile, '%d %d %24f %.4f %.2f %.2f', i
        # print >> tablefile, '%d %d %.4f %.4f %.2f %.2f', i[0], i[1], i[2], i[3], i[4], i[5]
        # tablefile.write( '%d %d %.4f %.4f %.2f %.2f\n' % ( i[0], i[1], i[2], i[3], i[4], i[5] ))
        tablefile.write( tablefmtstring % ( i[0], i[1], i[2], i[3], i[4], i[5] ))
    tablefile.close()

    nodemeans=[]
    nodemeans = zip( nnum, dmeans )

    # print nodemeans
    nodemeans.sort(key=lambda x: x[1])
    xtickmarks=[]

    # try to add a null tick mark at beginning
    xtickmarks.append( '' )
    # end try

    for i, v in nodemeans:
        #	xname = "mu%04d" % (i)
        xname = fstring % (i)
        xtickmarks.append( xname )

    dlist.sort(key=lambda a: a.mean() )
    plt.figure()
    # boxplot(multidata)
    # print dlist
    if len( dlist ) < 1:
        return 0

    # break
    plt.boxplot(dlist)
    # boxplot(hpldata2)
    plt.xlabel('Node')
    # ylabel('Performance (GFlops)')
    plt.ylabel(ytitle)
    # title("Mustang Quarantined HPL with Proper Affinities Set Performance Variation")
    plt.title(ctitle)

    nnodes = len( dlist )

    adjustedwidth = nnodes / 100.0 * 10.0 ;
    if adjustedwidth < 10.0:
        adjustedwidth = 10.0

    # print 'Nodes collected for ', nnodes, ' adjusted width is ', adjustedwidth

    ticktable=[]
    for k in xtickmarks:
        ticktable.append( k )

    plt.xticks( range( len( xtickmarks)), rotation=90, fontsize=6 )
    plt.xticks( range( len( ticktable)), xtickmarks,  rotation=90, fontsize=6 )

    fig = matplotlib.pyplot.gcf()
    # fig.set_size_inches(10.0,7.0)
    fig.set_size_inches(adjustedwidth,7.0)

    # show()
    plotname = thisdirname + "/" + subdirname + "Boxplot.png"
    # plt.savefig('BoxplotGraph.png', dpi=150 )
    plt.savefig(plotname, dpi=150 )

    return nnodes

def main():
    sub_dir = "Boxplots-" + datetime.datetime.now().strftime('%m-%d-%YT%H:%M:%S:%f')

    # if "HOME" in self.output_dir_root:
    # root = os.environ['HOME']
    # else:
    # root = self.output_dir_roo
    root = "/usr/projects/splunk/results/prr/wf"
    output_dir = root + "/Boxplots/" + sub_dir

    # self.logger.info(self.lh + " Make Boxplots directory: " + output_dir)
    try:
        os.umask(0o002)
        os.makedirs(output_dir, 0o755)
    except OSError:
        print " Error creating Boxplots directory : \n\t" + output_dir
        #    self.logger.info(self.lh + " Error creating Boxplots directory : \n\t" + output_dir)
        output_dir = ''
        pass

    print "Created Boxplots dir: " + output_dir
    AllDataFile = output_dir + "/AllDataFile.txt"

    prelimout = open( AllDataFile, "w" )
    # print "Making box plots with data from:"
    for line in sys.stdin:
        searchObj = re.search(r'jid\(', line, re.M|re.I)
        isreportable = line.find( "+" )
        if searchObj and isreportable  > 1:
            # if searchObj:
            # Should do work here, see correct output above
            # print line,  # this line for debug purpose
            tname= []
            restline= []
            newline= []

            jidx = line.find( " jid(" )
            for i in xrange(0,jidx ):
                if line[i] == ' ':
                    tname.append(  "-" )
                else:
                    tname.append( line[i])

            for i in xrange(jidx, len( line ) - 1 ):
                if line[i] == '+':
                    restline.append( " " )
                else:
                    restline.append(  line[i])

            print >> prelimout, ''.join(tname), ''.join(restline)
    prelimout.close()

    alltests = open( AllDataFile , "r" )
    for line in alltests:
        thisname=[]
        del thisname[:]

        jidx = line.find( " jid(" )
        for i in xrange(1,jidx - 1 ):
            if line[i] == '.':
                break
            else:
                thisname.append( line[i])

        tud,jud,tid,pname,pname2,trest = line.split(' ', 5)
        tdirectory = output_dir + "/" + ''.join(thisname) + "-" + ''.join(pname2)
        if not os.path.isdir(tdirectory):
            mkdircmd = "mkdir " + tdirectory
            os.system( mkdircmd )
        thisfname = tdirectory + "/TableRes.txt"
        thisfile = open( thisfname, "a" )
        print >> thisfile, line,
        thisfile.close()

    # [subdirList[0] for subdirList in os.walk(output_dir)]
    # subdirList = os.listdir( output_dir )
    subdirList = get_subdirectories( output_dir)
    ndone = 0
    ntodo = len( subdirList )
    print "Start Processing Directory List ", subdirList
    for dres in subdirList:
        subdirname = ''.join( dres )
        procthisdir = output_dir + "/" + ''.join( dres)
        # procthisdir = ''.join( dres)
        # print "procdir is ", procthisdir
        # if os.path.isdir( procthisdir ):
        nodesfound = makesingleboxplot(procthisdir,subdirname, "TableRes.txt" )
        print "Finished Processing Directory ", ndone, " Percent done ", (float(ndone + 1) / float(ntodo) ) * 100.0, \
        " UniqueNodesFound ", nodesfound
        ndone += 1

# def makesingleboxplot():
        # return "made boxplot"

if __name__ == '__main__':
    # pass entire command line to main except for the command name
    main()
    sys.exit()

<|endoftext|>


<commit_before>"""
Commonly used messaging components.
"""
import os
import uuid
import socket
import humanhash

import zipline.util as qutil
from zipline.protocol import CONTROL_PROTOCOL, COMPONENT_STATE

class Component(object):

    def __init__(self):
        """
        :addresses: a dict of name_string -> zmq port address strings. Must have the following entries::

            - sync_address: socket address used for synchronizing the start of all workers, heartbeating, and exit notification
                            will be used in REP/REQ sockets. Bind is always on the REP side.
            - data_address: socket address used for data sources to stream their records.
                            will be used in PUSH/PULL sockets between data sources and a ParallelBuffer (aka the Feed). Bind
                            will always be on the PULL side (we always have N producers and 1 consumer)
            - feed_address: socket address used to publish consolidated feed from serialization of data sources
                            will be used in PUB/SUB sockets between Feed and Transforms. Bind is always on the PUB side.
            - merge_address: socket address used to publish transformed values.
                            will be used in PUSH/PULL from many transforms to one MergedParallelBuffer (aka the Merge). Bind
                            will always be on the PULL side (we always have N producers and 1 consumer)
            - result_address: socket address used to publish merged data source feed and transforms to clients
                            will be used in PUB/SUB from one Merge to one or many clients. Bind is always on the PUB side.

        Bind/Connect methods will return the correct socket type for each address. Any sockets on which recv is expected to be called
        will also return a Poller.

        """
        self.zmq               = None
        self.context           = None
        self.addresses         = None
        self.out_socket        = None
        self.gevent_needed     = False
        self.killed            = False
        self.heartbeat_timeout = 2000
        self.state_flag        = COMPONENT_STATE.OK # OK | DONE | EXCEPTION

        # Humanhashes make this way easier to debug because they
        # stick in your mind unlike a 32 byte string of random hex.
        self.guid = uuid.uuid4()
        self.huid = humanhash.humanize(self.guid.hex)

    # ------------
    # Core Methods
    # ------------

    def open(self):
        raise NotImplementedError

    def destroy(self):
        """
        Tear down after normal operation.
        """
        raise NotImplementedError

    def kill(self):
        """
        Tear down ( fast ) as a mode of failure in the
        simulation.
        """
        raise NotImplementedError

    def do_work(self):
        raise NotImplementedError

    def _run(self):
        self.done       = False # TODO: use state flag
        self.sockets    = []

        if self.gevent_needed:
            qutil.LOGGER.info("Loading gevent specific zmq for {id}".format(id=self.get_id))
            import gevent_zeromq
            self.zmq = gevent_zeromq.zmq
        else:
            import zmq
            self.zmq = zmq

        self.context = self.zmq.Context()
        self.setup_poller()

        self.open()
        self.setup_sync()
        self.setup_control()
        self.loop()

        #close all the sockets
        for sock in self.sockets:
            sock.close()

    def run(self, catch_exceptions=False):
        """
        Run the component.

        Optionally takes an argument to catch and log all exceptions raised
        during execution ues this with care since it makes it very hard to
        debug since it mucks up your stacktraces.
        """

        fail = None

        if catch_exceptions:
            try:
                self._run()
            except Exception as exc:
                # TODO, we want to do this grab the stack
                # frame so we can preserve stacktraces when we
                # reraise the exception.
                self.signal_exception(exc)
                fail = exc
            finally:
                if self.context:
                    self.context.destroy()
                if fail:
                    raise fail
        else:
            self._run()
            if(self.context != None):
                self.context.destroy()

    def loop(self):
        """
        Loop to do work while we still have work to do.
        """
        while not self.done: # TODO: use state flag
            self.confirm()
            self.do_work()

    def confirm(self):
        """
        Send a synchronization request to the host.
        """

        # TODO: proper framing
        self.sync_socket.send(self.get_id + ":RUN")

        self.receive_sync_ack() # blocking

    # ----------------------
    #  Internal Maintenance
    # ----------------------

    def signal_exception(self, exc=None):
        self.state_flag = COMPONENT_STATE.EXCEPTION
        qutil.LOGGER.exception("Unexpected error in run for {id}.".format(id=self.get_id))

    def signal_done(self):
        """
        Notify down stream components that we're done.
        """

        self.state_flag = COMPONENT_STATE.DONE

        if self.out_socket:
            self.out_socket.send(str(CONTROL_PROTOCOL.DONE))

        #notify host we're done
        # TODO: proper framing
        self.sync_socket.send(self.get_id + ":" + str(CONTROL_PROTOCOL.DONE))

        self.receive_sync_ack()
        #notify internal work look that we're done
        self.done = True # TODO: use state flag

    # -----------
    #  Messaging
    # -----------

    def setup_poller(self):
        """
        Setup the poller used for multiplexing the incoming data
        handling sockets.
        """

        self.poll = self.zmq.Poller()

    def receive_sync_ack(self):
        """
        Wait for synchronization reply from the host.
        """

        socks = dict(self.sync_poller.poll(self.heartbeat_timeout))
        if self.sync_socket in socks and socks[self.sync_socket] == self.zmq.POLLIN:
            message = self.sync_socket.recv()
        else:
            raise Exception("Sync ack timed out on response for {id}".format(id=self.get_id))

    def bind_data(self):
        return self.bind_pull_socket(self.addresses['data_address'])

    def connect_data(self):
        return self.connect_push_socket(self.addresses['data_address'])

    def bind_feed(self):
        return self.bind_pub_socket(self.addresses['feed_address'])

    def connect_feed(self):
        return self.connect_sub_socket(self.addresses['feed_address'])

    def bind_merge(self):
        return self.bind_pull_socket(self.addresses['merge_address'])

    def connect_merge(self):
        return self.connect_push_socket(self.addresses['merge_address'])

    def bind_result(self):
        return self.bind_pub_socket(self.addresses['result_address'])

    def connect_result(self):
        return self.connect_sub_socket(self.addresses['result_address'])

    def bind_pull_socket(self, addr):
        pull_socket = self.context.socket(self.zmq.PULL)
        pull_socket.bind(addr)
        self.poll.register(pull_socket, self.zmq.POLLIN)

        self.sockets.append(pull_socket)

        return pull_socket

    def connect_push_socket(self, addr):
        push_socket = self.context.socket(self.zmq.PUSH)
        push_socket.connect(addr)
        #push_socket.setsockopt(self.zmq.LINGER,0)
        self.sockets.append(push_socket)
        self.out_socket = push_socket

        return push_socket

    def bind_pub_socket(self, addr):
        pub_socket = self.context.socket(self.zmq.PUB)
        pub_socket.bind(addr)
        #pub_socket.setsockopt(self.zmq.LINGER,0)
        self.out_socket = pub_socket

        return pub_socket

    def connect_sub_socket(self, addr):
        sub_socket = self.context.socket(self.zmq.SUB)
        sub_socket.connect(addr)
        sub_socket.setsockopt(self.zmq.SUBSCRIBE,'')
        self.sockets.append(sub_socket)

        self.poll.register(sub_socket, self.zmq.POLLIN)

        return sub_socket

    def setup_control(self):
        """
        Set up the control socket. Used to monitor the
        overall status of the simulation and to forcefully tear
        down the simulation in case of a failure.
        """
        assert self.controller

        self.control_out = self.controller.message_sender()
        self.control_in = self.controller.message_listener()

        self.poll.register(self.control_in, self.zmq.POLLIN)
        self.sockets.extend([self.control_in, self.control_out])

    def setup_sync(self):
        """
        Setup the sync socket and poller.
        """

        qutil.LOGGER.debug("Connecting sync client for {id}".format(id=self.get_id))

        self.sync_socket = self.context.socket(self.zmq.REQ)
        self.sync_socket.connect(self.addresses['sync_address'])
        #self.sync_socket.setsockopt(self.zmq.LINGER,0)

        # Explictly, a different poller for obvious reasons.
        # I'm not fond of having this poller init'd as a side
        # effect of a method call. Still thinking about where to
        # put it at the moment though...
        self.sync_poller = self.zmq.Poller()
        self.sync_poller.register(self.sync_socket, self.zmq.POLLIN)

        self.sockets.append(self.sync_socket)

    # ---------------------
    # Description and Debug
    # ---------------------

    @property
    def get_id(self):
        return 'UNKNOWN COMPONENT'

    def debug(self):
        """
        Debug information about the component.
        """
        return (
            self.get_id          ,
            self.huid            ,
            socket.gethostname() ,
            os.getpid()          ,
            hex(id(self))        ,
            self.sockets         ,
        )

    def __repr__(self):
        """
        Return a usefull string representation of the component
        to indicate its type, unique identifier, and computational
        context identifier name.
        """

        return "<{name} {uuid} at {host} {pid} {pointer}>".format(
            name    = self.get_id          ,
            uuid    = self.huid            ,
            host    = socket.gethostname() ,
            pid     = os.getpid()          ,
            pointer = hex(id(self))        ,
        )
<commit_msg>Removed sockets attribute, does not exist ( yet )<commit_after>"""
Commonly used messaging components.
"""
import os
import uuid
import socket
import humanhash

import zipline.util as qutil
from zipline.protocol import CONTROL_PROTOCOL, COMPONENT_STATE

class Component(object):

    def __init__(self):
        """
        :addresses: a dict of name_string -> zmq port address strings. Must have the following entries::

            - sync_address: socket address used for synchronizing the start of all workers, heartbeating, and exit notification
                            will be used in REP/REQ sockets. Bind is always on the REP side.
            - data_address: socket address used for data sources to stream their records.
                            will be used in PUSH/PULL sockets between data sources and a ParallelBuffer (aka the Feed). Bind
                            will always be on the PULL side (we always have N producers and 1 consumer)
            - feed_address: socket address used to publish consolidated feed from serialization of data sources
                            will be used in PUB/SUB sockets between Feed and Transforms. Bind is always on the PUB side.
            - merge_address: socket address used to publish transformed values.
                            will be used in PUSH/PULL from many transforms to one MergedParallelBuffer (aka the Merge). Bind
                            will always be on the PULL side (we always have N producers and 1 consumer)
            - result_address: socket address used to publish merged data source feed and transforms to clients
                            will be used in PUB/SUB from one Merge to one or many clients. Bind is always on the PUB side.

        Bind/Connect methods will return the correct socket type for each address. Any sockets on which recv is expected to be called
        will also return a Poller.

        """
        self.zmq               = None
        self.context           = None
        self.addresses         = None
        self.out_socket        = None
        self.gevent_needed     = False
        self.killed            = False
        self.heartbeat_timeout = 2000
        self.state_flag        = COMPONENT_STATE.OK # OK | DONE | EXCEPTION

        # Humanhashes make this way easier to debug because they
        # stick in your mind unlike a 32 byte string of random hex.
        self.guid = uuid.uuid4()
        self.huid = humanhash.humanize(self.guid.hex)

    # ------------
    # Core Methods
    # ------------

    def open(self):
        raise NotImplementedError

    def destroy(self):
        """
        Tear down after normal operation.
        """
        raise NotImplementedError

    def kill(self):
        """
        Tear down ( fast ) as a mode of failure in the
        simulation.
        """
        raise NotImplementedError

    def do_work(self):
        raise NotImplementedError

    def _run(self):
        self.done       = False # TODO: use state flag
        self.sockets    = []

        if self.gevent_needed:
            qutil.LOGGER.info("Loading gevent specific zmq for {id}".format(id=self.get_id))
            import gevent_zeromq
            self.zmq = gevent_zeromq.zmq
        else:
            import zmq
            self.zmq = zmq

        self.context = self.zmq.Context()
        self.setup_poller()

        self.open()
        self.setup_sync()
        self.setup_control()
        self.loop()

        #close all the sockets
        for sock in self.sockets:
            sock.close()

    def run(self, catch_exceptions=False):
        """
        Run the component.

        Optionally takes an argument to catch and log all exceptions raised
        during execution ues this with care since it makes it very hard to
        debug since it mucks up your stacktraces.
        """

        fail = None

        if catch_exceptions:
            try:
                self._run()
            except Exception as exc:
                # TODO, we want to do this grab the stack
                # frame so we can preserve stacktraces when we
                # reraise the exception.
                self.signal_exception(exc)
                fail = exc
            finally:
                if self.context:
                    self.context.destroy()
                if fail:
                    raise fail
        else:
            self._run()
            if(self.context != None):
                self.context.destroy()

    def loop(self):
        """
        Loop to do work while we still have work to do.
        """
        while not self.done: # TODO: use state flag
            self.confirm()
            self.do_work()

    def confirm(self):
        """
        Send a synchronization request to the host.
        """

        # TODO: proper framing
        self.sync_socket.send(self.get_id + ":RUN")

        self.receive_sync_ack() # blocking

    # ----------------------
    #  Internal Maintenance
    # ----------------------

    def signal_exception(self, exc=None):
        self.state_flag = COMPONENT_STATE.EXCEPTION
        qutil.LOGGER.exception("Unexpected error in run for {id}.".format(id=self.get_id))

    def signal_done(self):
        """
        Notify down stream components that we're done.
        """

        self.state_flag = COMPONENT_STATE.DONE

        if self.out_socket:
            self.out_socket.send(str(CONTROL_PROTOCOL.DONE))

        #notify host we're done
        # TODO: proper framing
        self.sync_socket.send(self.get_id + ":" + str(CONTROL_PROTOCOL.DONE))

        self.receive_sync_ack()
        #notify internal work look that we're done
        self.done = True # TODO: use state flag

    # -----------
    #  Messaging
    # -----------

    def setup_poller(self):
        """
        Setup the poller used for multiplexing the incoming data
        handling sockets.
        """

        self.poll = self.zmq.Poller()

    def receive_sync_ack(self):
        """
        Wait for synchronization reply from the host.
        """

        socks = dict(self.sync_poller.poll(self.heartbeat_timeout))
        if self.sync_socket in socks and socks[self.sync_socket] == self.zmq.POLLIN:
            message = self.sync_socket.recv()
        else:
            raise Exception("Sync ack timed out on response for {id}".format(id=self.get_id))

    def bind_data(self):
        return self.bind_pull_socket(self.addresses['data_address'])

    def connect_data(self):
        return self.connect_push_socket(self.addresses['data_address'])

    def bind_feed(self):
        return self.bind_pub_socket(self.addresses['feed_address'])

    def connect_feed(self):
        return self.connect_sub_socket(self.addresses['feed_address'])

    def bind_merge(self):
        return self.bind_pull_socket(self.addresses['merge_address'])

    def connect_merge(self):
        return self.connect_push_socket(self.addresses['merge_address'])

    def bind_result(self):
        return self.bind_pub_socket(self.addresses['result_address'])

    def connect_result(self):
        return self.connect_sub_socket(self.addresses['result_address'])

    def bind_pull_socket(self, addr):
        pull_socket = self.context.socket(self.zmq.PULL)
        pull_socket.bind(addr)
        self.poll.register(pull_socket, self.zmq.POLLIN)

        self.sockets.append(pull_socket)

        return pull_socket

    def connect_push_socket(self, addr):
        push_socket = self.context.socket(self.zmq.PUSH)
        push_socket.connect(addr)
        #push_socket.setsockopt(self.zmq.LINGER,0)
        self.sockets.append(push_socket)
        self.out_socket = push_socket

        return push_socket

    def bind_pub_socket(self, addr):
        pub_socket = self.context.socket(self.zmq.PUB)
        pub_socket.bind(addr)
        #pub_socket.setsockopt(self.zmq.LINGER,0)
        self.out_socket = pub_socket

        return pub_socket

    def connect_sub_socket(self, addr):
        sub_socket = self.context.socket(self.zmq.SUB)
        sub_socket.connect(addr)
        sub_socket.setsockopt(self.zmq.SUBSCRIBE,'')
        self.sockets.append(sub_socket)

        self.poll.register(sub_socket, self.zmq.POLLIN)

        return sub_socket

    def setup_control(self):
        """
        Set up the control socket. Used to monitor the
        overall status of the simulation and to forcefully tear
        down the simulation in case of a failure.
        """
        assert self.controller

        self.control_out = self.controller.message_sender()
        self.control_in = self.controller.message_listener()

        self.poll.register(self.control_in, self.zmq.POLLIN)
        self.sockets.extend([self.control_in, self.control_out])

    def setup_sync(self):
        """
        Setup the sync socket and poller.
        """

        qutil.LOGGER.debug("Connecting sync client for {id}".format(id=self.get_id))

        self.sync_socket = self.context.socket(self.zmq.REQ)
        self.sync_socket.connect(self.addresses['sync_address'])
        #self.sync_socket.setsockopt(self.zmq.LINGER,0)

        # Explictly, a different poller for obvious reasons.
        # I'm not fond of having this poller init'd as a side
        # effect of a method call. Still thinking about where to
        # put it at the moment though...
        self.sync_poller = self.zmq.Poller()
        self.sync_poller.register(self.sync_socket, self.zmq.POLLIN)

        self.sockets.append(self.sync_socket)

    # ---------------------
    # Description and Debug
    # ---------------------

    @property
    def get_id(self):
        return 'UNKNOWN COMPONENT'

    def debug(self):
        """
        Debug information about the component.
        """
        return (
            self.get_id          ,
            self.huid            ,
            socket.gethostname() ,
            os.getpid()          ,
            hex(id(self))        ,
        )

    def __repr__(self):
        """
        Return a usefull string representation of the component
        to indicate its type, unique identifier, and computational
        context identifier name.
        """

        return "<{name} {uuid} at {host} {pid} {pointer}>".format(
            name    = self.get_id          ,
            uuid    = self.huid            ,
            host    = socket.gethostname() ,
            pid     = os.getpid()          ,
            pointer = hex(id(self))        ,
        )
<|endoftext|>


<commit_before>"""
Django settings for PYNWapp project.

Generated by 'django-admin startproject' using Django 1.11.1.

For more information on this file, see
https://docs.djangoproject.com/en/1.11/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.11/ref/settings/
"""

import os

import settings_secret

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'PYNWapp.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'PYNWapp.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.11/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Password validation
# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.11/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.11/howto/static-files/

STATIC_URL = '/static/'
<commit_msg>update secret key import<commit_after>"""
Django settings for PYNWapp project.

Generated by 'django-admin startproject' using Django 1.11.1.

For more information on this file, see
https://docs.djangoproject.com/en/1.11/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.11/ref/settings/
"""

import os

from settings_secret import *

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'PYNWapp.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'PYNWapp.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.11/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Password validation
# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.11/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.11/howto/static-files/

STATIC_URL = '/static/'
<|endoftext|>


<commit_before>import os
import json

from scrapi import settings
from scrapi.util import make_dir
from scrapi.util import doc_id_to_path

class BaseStorage(object):
    METHOD = None

    # :: Str -> Str -> Nothing
    def _store(string, path):
        raise NotImplementedError('No store method')

    # :: Str -> Bool -> [RawDocument]
    def iter_raws(source, include_normalized=False):
        raise NotImplementedError('No iter raws method')

    # :: Str -> Str
    def get_as_string(self, path):
        raise NotImplementedError('No get as string method')

    # :: Str -> Dict
    def get_as_json(self, path):
        return json.loads(self.get_as_string(path))

    # :: Str -> Str
    def _build_path(self, raw_doc):
        path = [
            settings.ARCHIVE_DIRECTORY,
            raw_doc.get('source'),
            doc_id_to_path(raw_doc.get('doc_id')),
            raw_doc.get('timestamp')
        ]

        path = os.path.join(*path)
        make_dir(path)

        return path

    # :: NormalizedDocument -> Nothing
    def store_normalized(self, raw_doc, document):
        path = self._build_path(raw_doc)
        manifest = settings.MANIFESTS[document.get('source')]
        manifest_update = {
            'normalizeVersion': manifest['version']
        }

        self.update_manifest(path, manifest_update)

        path = os.path.join(path, 'normalized.json')

        self._store(json.dumps(document.attributes), path)

    # :: RawDocument -> Nothing
    def store_raw(self, document):
        manifest = settings.MANIFESTS[document.get('source')]
        doc_name = 'raw.{}'.format(manifest['file_format'])
        path = self._build_path(document)
        manifest = {
            'timestamp': document.get('timestamp'),
            'source': document.get('source'),
            'consumeVersion': manifest['version']
        }

        self.update_manifest(path, manifest)

        path = os.path.join(path, doc_name)

        self._store(document.get('doc'), path)

    # :: RawDocument -> Dict -> Nothing
    def update_manifest(self, path, fields):
        path = os.path.join(path, 'manifest.json')
        try:
            manifest = self.get_as_json(path)
        except Exception:  # TODO Make this more specific
            manifest = {}

        manifest.update(fields)
        self._store(json.dumps(manifest), path, overwrite=True)
<commit_msg>Conform to the latest schema<commit_after>import os
import json

from scrapi import settings
from scrapi.util import make_dir
from scrapi.util import doc_id_to_path

class BaseStorage(object):
    METHOD = None

    # :: Str -> Str -> Nothing
    def _store(string, path):
        raise NotImplementedError('No store method')

    # :: Str -> Bool -> [RawDocument]
    def iter_raws(source, include_normalized=False):
        raise NotImplementedError('No iter raws method')

    # :: Str -> Str
    def get_as_string(self, path):
        raise NotImplementedError('No get as string method')

    # :: Str -> Dict
    def get_as_json(self, path):
        return json.loads(self.get_as_string(path))

    # :: Str -> Str
    def _build_path(self, raw_doc):
        path = [
            settings.ARCHIVE_DIRECTORY,
            raw_doc.get('source'),
            doc_id_to_path(raw_doc.get('docID')),
            raw_doc.get('timestamp')
        ]

        path = os.path.join(*path)
        make_dir(path)

        return path

    # :: NormalizedDocument -> Nothing
    def store_normalized(self, raw_doc, document):
        path = self._build_path(raw_doc)
        manifest = settings.MANIFESTS[document.get('source')]
        manifest_update = {
            'normalizeVersion': manifest['version']
        }

        self.update_manifest(path, manifest_update)

        path = os.path.join(path, 'normalized.json')

        self._store(json.dumps(document.attributes), path)

    # :: RawDocument -> Nothing
    def store_raw(self, document):
        manifest = settings.MANIFESTS[document.get('source')]
        doc_name = 'raw.{}'.format(manifest['fileFormat'])
        path = self._build_path(document)
        manifest = {
            'timestamp': document.get('timestamp'),
            'source': document.get('source'),
            'consumeVersion': manifest['version']
        }

        self.update_manifest(path, manifest)

        path = os.path.join(path, doc_name)

        self._store(document.get('doc'), path)

    # :: RawDocument -> Dict -> Nothing
    def update_manifest(self, path, fields):
        path = os.path.join(path, 'manifest.json')
        try:
            manifest = self.get_as_json(path)
        except Exception:  # TODO Make this more specific
            manifest = {}

        manifest.update(fields)
        self._store(json.dumps(manifest), path, overwrite=True)
<|endoftext|>


<commit_before>"""Garbage collection thread for representing zmq refcount of Python objects
used in zero-copy sends.
"""

#-----------------------------------------------------------------------------
#  Copyright (c) 2013 Brian E. Granger & Min Ragan-Kelley
#
#  This file is part of pyzmq
#
#  Distributed under the terms of the New BSD License.  The full license is in
#  the file COPYING.BSD, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import atexit
import struct

from os import getpid
from collections import namedtuple
from threading import Thread, Event

import zmq

#-----------------------------------------------------------------------------
# Code
#-----------------------------------------------------------------------------

gcref = namedtuple('gcref', ['obj', 'event'])

class GarbageCollectorThread(Thread):
    """Thread in which garbage collection actually happens."""
    def __init__(self, gc):
        super(GarbageCollectorThread, self).__init__()
        self.gc = gc
        self.daemon = True
        self.pid = getpid()
        self.ready = Event()

    def run(self):
        s = self.gc.context.socket(zmq.PULL)
        s.linger = 0
        s.bind(self.gc.url)
        self.ready.set()

        while True:
            # detect fork
            if getpid is None or getpid() != self.pid:
                return
            msg = s.recv()
            if msg == b'DIE':
                break
            fmt = 'L' if len(msg) == 4 else 'Q'
            key = struct.unpack(fmt, msg)[0]
            tup = self.gc.refs.pop(key)
            if tup and tup.event:
                tup.event.set()
            del tup
        s.close()

class GarbageCollector(object):
    """PyZMQ Garbage Collector

    Used for representing the reference held by libzmq during zero-copy sends.
    This object holds a dictionary, keyed by Python id,
    of the Python objects whose memory are currently in use by zeromq.

    When zeromq is done with the memory, it sends a message on an inproc PUSH socket
    containing the packed size_t (32 or 64-bit unsigned int),
    which is the key in the dict.
    When the PULL socket in the gc thread receives that message,
    the reference is popped from the dict,
    and any tracker events that should be signaled fire.
    """

    context = None
    refs = None
    _finished = False
    url = "inproc://pyzmq.gc.01"

    def __init__(self):
        super(GarbageCollector, self).__init__()
        self.refs = {}
        self.pid = getpid()
        self.thread = None
        self.context = None

    def stop(self):
        """stop the garbage-collection thread"""
        if not self.is_alive():
            return
        push = self.context.socket(zmq.PUSH)
        push.connect(self.url)
        push.send(b'DIE')
        push.close()
        self.thread.join()
        self.context.term()
        self.refs.clear()

    def start(self):
        """Start a new garbage collection thread.

        Creates a new zmq Context used for garbage collection.
        Under most circumstances, this will only be called once per process.
        """
        self.context = zmq.Context()
        self.refs = {}
        atexit.register(self.stop)
        self.thread = GarbageCollectorThread(self)
        self.thread.start()
        self.thread.ready.wait()

    def is_alive(self):
        """Is the garbage collection thread currently running?

        Includes checks for process shutdown or fork.
        """
        if (getpid is None or
            getpid() != self.pid or
            self.thread is None or
            not self.thread.is_alive()
            ):
            return False
        return True

    def store(self, obj, event=None):
        """store an object and (optionally) event for zero-copy"""
        if not self.is_alive():
            self.start()
        tup = gcref(obj, event)
        theid = id(tup)
        self.refs[theid] = tup
        return theid

    def __del__(self):
        if not self.is_alive():
            return
        try:
            self.stop()
        except Exception as e:
            raise (e)

gc = GarbageCollector()
<commit_msg>safe pop in garbage collector thread<commit_after>"""Garbage collection thread for representing zmq refcount of Python objects
used in zero-copy sends.
"""

#-----------------------------------------------------------------------------
#  Copyright (c) 2013 Brian E. Granger & Min Ragan-Kelley
#
#  This file is part of pyzmq
#
#  Distributed under the terms of the New BSD License.  The full license is in
#  the file COPYING.BSD, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import atexit
import struct

from os import getpid
from collections import namedtuple
from threading import Thread, Event

import zmq

#-----------------------------------------------------------------------------
# Code
#-----------------------------------------------------------------------------

gcref = namedtuple('gcref', ['obj', 'event'])

class GarbageCollectorThread(Thread):
    """Thread in which garbage collection actually happens."""
    def __init__(self, gc):
        super(GarbageCollectorThread, self).__init__()
        self.gc = gc
        self.daemon = True
        self.pid = getpid()
        self.ready = Event()

    def run(self):
        s = self.gc.context.socket(zmq.PULL)
        s.linger = 0
        s.bind(self.gc.url)
        self.ready.set()

        while True:
            # detect fork
            if getpid is None or getpid() != self.pid:
                return
            msg = s.recv()
            if msg == b'DIE':
                break
            fmt = 'L' if len(msg) == 4 else 'Q'
            key = struct.unpack(fmt, msg)[0]
            tup = self.gc.refs.pop(key, None)
            if tup and tup.event:
                tup.event.set()
            del tup
        s.close()

class GarbageCollector(object):
    """PyZMQ Garbage Collector

    Used for representing the reference held by libzmq during zero-copy sends.
    This object holds a dictionary, keyed by Python id,
    of the Python objects whose memory are currently in use by zeromq.

    When zeromq is done with the memory, it sends a message on an inproc PUSH socket
    containing the packed size_t (32 or 64-bit unsigned int),
    which is the key in the dict.
    When the PULL socket in the gc thread receives that message,
    the reference is popped from the dict,
    and any tracker events that should be signaled fire.
    """

    context = None
    refs = None
    _finished = False
    url = "inproc://pyzmq.gc.01"

    def __init__(self):
        super(GarbageCollector, self).__init__()
        self.refs = {}
        self.pid = getpid()
        self.thread = None
        self.context = None

    def stop(self):
        """stop the garbage-collection thread"""
        if not self.is_alive():
            return
        push = self.context.socket(zmq.PUSH)
        push.connect(self.url)
        push.send(b'DIE')
        push.close()
        self.thread.join()
        self.context.term()
        self.refs.clear()

    def start(self):
        """Start a new garbage collection thread.

        Creates a new zmq Context used for garbage collection.
        Under most circumstances, this will only be called once per process.
        """
        self.context = zmq.Context()
        self.refs = {}
        atexit.register(self.stop)
        self.thread = GarbageCollectorThread(self)
        self.thread.start()
        self.thread.ready.wait()

    def is_alive(self):
        """Is the garbage collection thread currently running?

        Includes checks for process shutdown or fork.
        """
        if (getpid is None or
            getpid() != self.pid or
            self.thread is None or
            not self.thread.is_alive()
            ):
            return False
        return True

    def store(self, obj, event=None):
        """store an object and (optionally) event for zero-copy"""
        if not self.is_alive():
            self.start()
        tup = gcref(obj, event)
        theid = id(tup)
        self.refs[theid] = tup
        return theid

    def __del__(self):
        if not self.is_alive():
            return
        try:
            self.stop()
        except Exception as e:
            raise (e)

gc = GarbageCollector()
<|endoftext|>


<commit_before>"""
This module contains a simple class (Libxml2Document) which provides cache and
garbage collection to libxml2 documents (xmlDoc).
"""

import weakref

from .factories import xmlDoc_from_html

class Libxml2Document(object):

    cache = weakref.WeakKeyDictionary()
    __slots__ = ['xmlDoc', 'xpathContext', '__weakref__']

    def __new__(cls, response, factory=xmlDoc_from_html):
        cache = cls.cache.setdefault(response, {})
        if factory not in cache:
            obj = object.__new__(cls)
            obj.xmlDoc = factory(response)
            obj.xpathContext = obj.xmlDoc.xpathNewContext()
            cache[factory] = obj
        return cache[factory]

    def __del__(self):
        # we must call both cleanup functions, so we try/except all exceptions
        # to make sure one doesn't prevent the other from being called
        # this call sometimes raises a "NoneType is not callable" TypeError
        # also, these calls sometimes raise a "NoneType is not callable"
        # TypeError, so the try/except block silences them
        try:
            self.xmlDoc.freeDoc()
        except:
            pass
        try:
            self.xpathContext.xpathFreeContext()
        except:
            pass

    def __str__(self):
        return "<Libxml2Document %s>" % self.xmlDoc.name
<commit_msg>trackref Libxml2Document objects and do not silence xml exceptions on __del__<commit_after>"""
This module contains a simple class (Libxml2Document) which provides cache and
garbage collection to libxml2 documents (xmlDoc).
"""

import weakref

from scrapy.utils.trackref import object_ref
from .factories import xmlDoc_from_html

class Libxml2Document(object_ref):

    cache = weakref.WeakKeyDictionary()
    __slots__ = ['xmlDoc', 'xpathContext', '__weakref__']

    def __new__(cls, response, factory=xmlDoc_from_html):
        cache = cls.cache.setdefault(response, {})
        if factory not in cache:
            obj = object_ref.__new__(cls)
            obj.xmlDoc = factory(response)
            obj.xpathContext = obj.xmlDoc.xpathNewContext()
            cache[factory] = obj
        return cache[factory]

    def __del__(self):
        # we must call both cleanup functions, so we try/except all exceptions
        # to make sure one doesn't prevent the other from being called
        # this call sometimes raises a "NoneType is not callable" TypeError
        # also, these calls sometimes raise a "NoneType is not callable"
        # TypeError, so the try/except block silences them
        try:
            self.xmlDoc.freeDoc()
        finally:
            self.xpathContext.xpathFreeContext()

    def __str__(self):
        return "<Libxml2Document %s>" % self.xmlDoc.name
<|endoftext|>