<commit_before># -*- coding: utf-8 -*-
"""
repeater/repeater.py
~~~~~~~~~~~~~~~~~~~~

This class defines the underlying TCP<->websocket repeater that powers the
pyrc server. This maps a websocket connection directly to a single TCP
connection. Data is repeated from one end to the other in a beautiful display
of asynchronous joy.
"""
import socket
from tornado.websocket import WebSocketHandler
from tornado.iostream import IOStream

class Repeater(WebSocketHandler):
    """
    The underlying TCP<->websocket repeater that powers the pyrc server. Right
    now this class has a few hard-coded limitations: it only connects to one
    server location, on one port, without SSL. All of these limitations will
    be removed at one time or another.
    """

    def __init__(self, *args, **kwargs):
        super(Repeater, self).__init__(*args, **kwargs)

        self.tcp = None
        self.target_host = '127.0.0.1'
        self.target_port = 6667
        self.half_closed = False

    def open(self):
        """
        When a websocket connection is opened, we open a TCP connection to the
        remote server. Wait to start hearing traffic from it.
        """
        sck = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.tcp = IOStream(sck)
        self.tcp.set_close_callback(self.async_callback(self.on_irc_close))
        self.tcp.connect((self.target_host, self.target_port))
        self.tcp.read_until(b"\n", self.async_callback(self.on_irc_message))

    def on_message(self, message):
        """
        Called whenever a websocket message is received. Echoes the message,
        without adjustment, on the TCP connection.
        """
        self.tcp.write(message.encode('utf-8'))

    def on_irc_message(self, data):
        """
        Called whenever a message is received on the TCP connection. Echoes the
        message, without qualification, on the websocket connection.
        """
        self.write_message(data)
        self.tcp.read_until(b"\n", self.async_callback(self.on_irc_message))

    def on_close(self):
        """
        When a websocket connection is closed, we close the TCP connection as
        well.
        """
        if not self.half_closed:
            self.half_closed = True
            self.tcp.close()

    def on_irc_close(self):
        """
        When a TCP connection is closed, we close the websocket connection as
        well.
        """
        if not self.half_closed:
            self.half_closed = True
            self.close()
<commit_msg>Make it easier to change the urlspec.<commit_after># -*- coding: utf-8 -*-
"""
repeater/repeater.py
~~~~~~~~~~~~~~~~~~~~

This class defines the underlying TCP<->websocket repeater that powers the
pyrc server. This maps a websocket connection directly to a single TCP
connection. Data is repeated from one end to the other in a beautiful display
of asynchronous joy.
"""
import socket
from tornado.websocket import WebSocketHandler
from tornado.iostream import IOStream

class Repeater(WebSocketHandler):
    """
    The underlying TCP<->websocket repeater that powers the pyrc server. Right
    now this class has a few hard-coded limitations: it only connects to one
    server location, on one port, without SSL. All of these limitations will
    be removed at one time or another.
    """

    def __init__(self, *args, **kwargs):
        super(Repeater, self).__init__(*args, **kwargs)

        self.tcp = None
        self.target_host = '127.0.0.1'
        self.target_port = 6667
        self.half_closed = False

    def open(self, *args, **kwargs):
        """
        When a websocket connection is opened, we open a TCP connection to the
        remote server. Wait to start hearing traffic from it.
        """
        sck = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.tcp = IOStream(sck)
        self.tcp.set_close_callback(self.async_callback(self.on_irc_close))
        self.tcp.connect((self.target_host, self.target_port))
        self.tcp.read_until(b"\n", self.async_callback(self.on_irc_message))

    def on_message(self, message):
        """
        Called whenever a websocket message is received. Echoes the message,
        without adjustment, on the TCP connection.
        """
        self.tcp.write(message.encode('utf-8'))

    def on_irc_message(self, data):
        """
        Called whenever a message is received on the TCP connection. Echoes the
        message, without qualification, on the websocket connection.
        """
        self.write_message(data)
        self.tcp.read_until(b"\n", self.async_callback(self.on_irc_message))

    def on_close(self):
        """
        When a websocket connection is closed, we close the TCP connection as
        well.
        """
        if not self.half_closed:
            self.half_closed = True
            self.tcp.close()

    def on_irc_close(self):
        """
        When a TCP connection is closed, we close the websocket connection as
        well.
        """
        if not self.half_closed:
            self.half_closed = True
            self.close()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    tipfyext.jinja2
    ~~~~~~~~~~~~~~~

    Jinja2 template support for Tipfy.

    Learn more about Jinja2 at http://jinja.pocoo.org/2/

    :copyright: 2010 by tipfy.org.
    :license: BSD, see LICENSE.txt for more details.
"""
from jinja2 import Environment, FileSystemLoader, ModuleLoader

from werkzeug import cached_property

from tipfy import current_handler, APPENGINE, DEV_APPSERVER
from tipfy.utils import url_for

#: Default configuration values for this module. Keys are:
#:
#: templates_dir
#:     Directory for templates. Default is `templates`.
#:
#: templates_compiled_target
#:     Target for compiled templates. If set, uses the loader for compiled
#:     templates when deployed. If it ends with a '.zip' it will be treated
#:     as a zip file. Default is None.
#:
#: force_use_compiled
#:     Forces the use of compiled templates even in the development server.
#:
#: environment_args
#:     Keyword arguments used to instantiate the Jinja2 environment. By
#:     default autoescaping is enabled and two extensions are set:
#:     'jinja2.ext.autoescape' and 'jinja2.ext.with_'.
default_config = {
    'templates_dir': 'templates',
    'templates_compiled_target': None,
    'force_use_compiled': False,
    'environment_args': dict(
        autoescape=True,
        extensions=['jinja2.ext.autoescape', 'jinja2.ext.with_'],
    ),
}

class Jinja2Mixin(object):
    """Mixin that adds ``render_template`` and ``render_response`` methods
    to a :class:`tipfy.RequestHandler`. It will use the request context to
    render templates.
    """
    @cached_property
    def jinja2(self):
        return Jinja2.factory(self.app, 'jinja2')

    def render_template(self, _filename, **context):
        return self.jinja2.render_template(self, _filename, **context)

    def render_response(self, _filename, **context):
        return self.jinja2.render_response(self, _filename, **context)

class Jinja2(object):
    def __init__(self, app, _globals=None, filters=None):
        self.app = app
        config = app.config[__name__]
        kwargs = config['environment_args'].copy()
        enable_i18n = 'jinja2.ext.i18n' in kwargs.get('extensions', [])

        if not kwargs.get('loader'):
            templates_compiled_target = config['templates_compiled_target']
            use_compiled = not app.debug or config['force_use_compiled']

            if templates_compiled_target and use_compiled:
                # Use precompiled templates loaded from a module or zip.
                kwargs['loader'] = ModuleLoader(templates_compiled_target)
            else:
                # Parse templates for every new environment instances.
                kwargs['loader'] = FileSystemLoader(config['templates_dir'])

        #if APPENGINE and not DEV_APPSERVER:
        #    # We don't need to check if templates changed in production.
        #    kwargs['auto_reload'] = False

        # Initialize the environment.
        env = Environment(**kwargs)

        if _globals:
            env.globals.update(_globals)

        if filters:
            env.filters.update(filters)

        if enable_i18n:
            # Install i18n.
            from tipfy import i18n
            env.install_gettext_callables(
                lambda s: i18n.gettext(s),
                lambda s, p, n: i18n.ngettext(s, p, n),
                newstyle=True)
            env.filters.update({
                'format_date':     i18n.format_date,
                'format_time':     i18n.format_time,
                'format_datetime': i18n.format_datetime,
            })

        env.globals['url_for'] = url_for
        self.environment = env

    def render(self, _filename, **context):
        """Renders a template and returns a response object.

        :param _filename:
            The template filename, related to the templates directory.
        :param context:
            Keyword arguments used as variables in the rendered template.
            These will override values set in the request context.
       :returns:
            A rendered template.
        """
        return self.environment.get_template(_filename).render(**context)

    def render_template(self, _handler, _filename, **context):
        """Renders a template and returns a response object.

        :param _filename:
            The template filename, related to the templates directory.
        :param context:
            Keyword arguments used as variables in the rendered template.
            These will override values set in the request context.
       :returns:
            A rendered template.
        """
        ctx = _handler.context.copy()
        ctx.update(context)
        return self.render(_filename, **ctx)

    def render_response(self, _handler, _filename, **context):
        """Returns a response object with a rendered template.

        :param _filename:
            The template filename, related to the templates directory.
        :param context:
            Keyword arguments used as variables in the rendered template.
            These will override values set in the request context.
        """
        res = self.render_template(_handler, _filename, **context)
        return self.app.response_class(res)

    def get_template_attribute(self, filename, attribute):
        """Loads a macro (or variable) a template exports.  This can be used to
        invoke a macro from within Python code.  If you for example have a
        template named `_foo.html` with the following contents:

        .. sourcecode:: html+jinja

           {% macro hello(name) %}Hello {{ name }}!{% endmacro %}

        You can access this from Python code like this::

            hello = get_template_attribute('_foo.html', 'hello')
            return hello('World')

        This function is borrowed from `Flask`.

        :param filename:
            The template filename.
        :param attribute:
            The name of the variable of macro to acccess.
        """
        template = self.environment.get_template(filename)
        return getattr(template.module, attribute)

    @classmethod
    def factory(cls, _app, _name, **kwargs):
        if _name not in _app.registry:
            _app.registry[_name] = cls(_app, **kwargs)

        return _app.registry[_name]
<commit_msg>Removed unnecessary lambda.<commit_after># -*- coding: utf-8 -*-
"""
    tipfyext.jinja2
    ~~~~~~~~~~~~~~~

    Jinja2 template support for Tipfy.

    Learn more about Jinja2 at http://jinja.pocoo.org/2/

    :copyright: 2010 by tipfy.org.
    :license: BSD, see LICENSE.txt for more details.
"""
from jinja2 import Environment, FileSystemLoader, ModuleLoader

from werkzeug import cached_property

from tipfy import current_handler, APPENGINE, DEV_APPSERVER
from tipfy.utils import url_for

#: Default configuration values for this module. Keys are:
#:
#: templates_dir
#:     Directory for templates. Default is `templates`.
#:
#: templates_compiled_target
#:     Target for compiled templates. If set, uses the loader for compiled
#:     templates when deployed. If it ends with a '.zip' it will be treated
#:     as a zip file. Default is None.
#:
#: force_use_compiled
#:     Forces the use of compiled templates even in the development server.
#:
#: environment_args
#:     Keyword arguments used to instantiate the Jinja2 environment. By
#:     default autoescaping is enabled and two extensions are set:
#:     'jinja2.ext.autoescape' and 'jinja2.ext.with_'.
default_config = {
    'templates_dir': 'templates',
    'templates_compiled_target': None,
    'force_use_compiled': False,
    'environment_args': dict(
        autoescape=True,
        extensions=['jinja2.ext.autoescape', 'jinja2.ext.with_'],
    ),
}

class Jinja2Mixin(object):
    """Mixin that adds ``render_template`` and ``render_response`` methods
    to a :class:`tipfy.RequestHandler`. It will use the request context to
    render templates.
    """
    @cached_property
    def jinja2(self):
        return Jinja2.factory(self.app, 'jinja2')

    def render_template(self, _filename, **context):
        return self.jinja2.render_template(self, _filename, **context)

    def render_response(self, _filename, **context):
        return self.jinja2.render_response(self, _filename, **context)

class Jinja2(object):
    def __init__(self, app, _globals=None, filters=None):
        self.app = app
        config = app.config[__name__]
        kwargs = config['environment_args'].copy()
        enable_i18n = 'jinja2.ext.i18n' in kwargs.get('extensions', [])

        if not kwargs.get('loader'):
            templates_compiled_target = config['templates_compiled_target']
            use_compiled = not app.debug or config['force_use_compiled']

            if templates_compiled_target and use_compiled:
                # Use precompiled templates loaded from a module or zip.
                kwargs['loader'] = ModuleLoader(templates_compiled_target)
            else:
                # Parse templates for every new environment instances.
                kwargs['loader'] = FileSystemLoader(config['templates_dir'])

        #if APPENGINE and not DEV_APPSERVER:
        #    # We don't need to check if templates changed in production.
        #    kwargs['auto_reload'] = False

        # Initialize the environment.
        env = Environment(**kwargs)

        if _globals:
            env.globals.update(_globals)

        if filters:
            env.filters.update(filters)

        if enable_i18n:
            # Install i18n.
            from tipfy import i18n
            env.install_gettext_callables(i18n.gettext, i18n.ngettext,
                newstyle=True)
            env.filters.update({
                'format_date':      i18n.format_date,
                'format_time':      i18n.format_time,
                'format_datetime':  i18n.format_datetime,
                'format_timedelta': i18n.format_timedelta,
            })

        env.globals['url_for'] = url_for
        self.environment = env

    def render(self, _filename, **context):
        """Renders a template and returns a response object.

        :param _filename:
            The template filename, related to the templates directory.
        :param context:
            Keyword arguments used as variables in the rendered template.
            These will override values set in the request context.
       :returns:
            A rendered template.
        """
        return self.environment.get_template(_filename).render(**context)

    def render_template(self, _handler, _filename, **context):
        """Renders a template and returns a response object.

        :param _filename:
            The template filename, related to the templates directory.
        :param context:
            Keyword arguments used as variables in the rendered template.
            These will override values set in the request context.
       :returns:
            A rendered template.
        """
        ctx = _handler.context.copy()
        ctx.update(context)
        return self.render(_filename, **ctx)

    def render_response(self, _handler, _filename, **context):
        """Returns a response object with a rendered template.

        :param _filename:
            The template filename, related to the templates directory.
        :param context:
            Keyword arguments used as variables in the rendered template.
            These will override values set in the request context.
        """
        res = self.render_template(_handler, _filename, **context)
        return self.app.response_class(res)

    def get_template_attribute(self, filename, attribute):
        """Loads a macro (or variable) a template exports.  This can be used to
        invoke a macro from within Python code.  If you for example have a
        template named `_foo.html` with the following contents:

        .. sourcecode:: html+jinja

           {% macro hello(name) %}Hello {{ name }}!{% endmacro %}

        You can access this from Python code like this::

            hello = get_template_attribute('_foo.html', 'hello')
            return hello('World')

        This function is borrowed from `Flask`.

        :param filename:
            The template filename.
        :param attribute:
            The name of the variable of macro to acccess.
        """
        template = self.environment.get_template(filename)
        return getattr(template.module, attribute)

    @classmethod
    def factory(cls, _app, _name, **kwargs):
        if _name not in _app.registry:
            _app.registry[_name] = cls(_app, **kwargs)

        return _app.registry[_name]
<|endoftext|>


<commit_before>from django.contrib import admin
from .models import Product,ProductVariation, ProductImage

class ProductAdmin(admin.ModelAdmin):
    model = Product
    list_display = ['title', 'active', ]

class ProductVariationAdmin(admin.ModelAdmin):
    model = ProductVariation
    list_display = ['title', 'active','stock','prices','product' ]

class PhotoAdmin(admin.ModelAdmin):
    model = ProductImage
    list_display = ['image','product']

    def image_img(self):
        if self.image:
            return u'' % self.image.url
        else:
            return '(No image found)'
        image_img.short_description = 'Thumb'
        image_img.allow_tags = True

admin.site.register(Product,ProductAdmin)
admin.site.register(ProductVariation,ProductVariationAdmin)
admin.site.register(ProductImage,PhotoAdmin)
<commit_msg>add dispaly image django admin<commit_after>from django.contrib import admin
from .models import Product,ProductVariation, ProductImage

class ProductAdmin(admin.ModelAdmin):
    model = Product
    list_display = ['title', 'active', ]

class ProductVariationAdmin(admin.ModelAdmin):
    model = ProductVariation
    list_display = ['title', 'active','stock','prices','product' ]

class PhotoAdmin(admin.ModelAdmin):
    model = ProductImage
    list_display = ['image','product']
    readonly_fields = ('image',)

    def image_img(self):
        if self.image:
            return u'' % self.image.url
        else:
            return '(No image found)'
        image_img.short_description = 'Thumb'
        image_img.allow_tags = True

admin.site.register(Product,ProductAdmin)
admin.site.register(ProductVariation,ProductVariationAdmin)
admin.site.register(ProductImage,PhotoAdmin)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import json
import requests

from libmproxy.script import concurrent

# Arguments:
# 1) webservice receiving all requests

def start(context, argv):
    if len(argv) != 2:
        raise ValueError('Usage: -s "sendToServer.py webservice_api"')
    # You may want to use Python's argparse for more sophisticated argument
    # parsing.
    context.api = argv[1]

#For more info on requests: https://mitmproxy.org/doc/scripting/inlinescripts.html
# method must be inside ['GET', 'POST']

# @concurrent
def request(context, flow):
    if flow.request.host in ['il.srgssr.ch', 'b.scorecardresearch.com', 'sb.scorecardresearch.com']:

        headers = []
        user_agent = None
        for header in flow.request.headers:
            json_header = {
                "key": header[0],
                "value": header[1]
            }
            headers.append(json_header)
            if header[0] == 'User-Agent':
                user_agent = header[1]

        payload = {
            "method": flow.request.method.upper(),
            "url": flow.request.url.strip(),
            "headers": headers,
            "origin": flow.client_conn.address.host
        }

        if user_agent is not None:
            payload['user_agent'] = user_agent

#        cookies = flow.request.get_cookies()
#        if cookies.get("UID", None) is not None and len(cookies.get("UID")) > 0:
#            payload['sessionid'] = cookies.get("UID")[0]

        r = requests.post(context.api, json=payload)
        print 10*'-', r.status_code, 80*"-"
<commit_msg>Enabling concurrency to avoid lags.<commit_after>#!/usr/bin/env python

import json
import requests

from libmproxy.script import concurrent

# Arguments:
# 1) webservice receiving all requests

def start(context, argv):
    if len(argv) != 2:
        raise ValueError('Usage: -s "sendToServer.py webservice_api"')
    # You may want to use Python's argparse for more sophisticated argument
    # parsing.
    context.api = argv[1]

#For more info on requests: https://mitmproxy.org/doc/scripting/inlinescripts.html
# method must be inside ['GET', 'POST']

@concurrent
def request(context, flow):
    if flow.request.host in ['il.srgssr.ch', 'b.scorecardresearch.com', 'sb.scorecardresearch.com']:

        headers = []
        user_agent = None
        for header in flow.request.headers:
            json_header = {
                "key": header[0],
                "value": header[1]
            }
            headers.append(json_header)
            if header[0] == 'User-Agent':
                user_agent = header[1]

        payload = {
            "method": flow.request.method.upper(),
            "url": flow.request.url.strip(),
            "headers": headers,
            "origin": flow.client_conn.address.host
        }

        if user_agent is not None:
            payload['user_agent'] = user_agent

#        cookies = flow.request.get_cookies()
#        if cookies.get("UID", None) is not None and len(cookies.get("UID")) > 0:
#            payload['sessionid'] = cookies.get("UID")[0]

        r = requests.post(context.api, json=payload)
        print 10*'-', r.status_code, 80*"-"
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os, sys, re, collections, gnuplot, getopt
import cpistack_data, cpistack_items, cpistack_results

try:
  collections.defaultdict()
except AttributeError, e:
  print sys.argv[0], "Error: This script requires Python version 2.5 or greater"
  sys.exit()

# New-style function returning CPI stack results
def cpistack_compute(jobid = 0, resultsdir = '.', data = None, partial = None,
                     cores_list = None, core_mincomp = 0., aggregate = False,
                     items = None, groups = None, use_simple = False, use_simple_mem = True,
                     no_collapse = False):

  # Get the data
  cpidata = cpistack_data.CpiData(jobid = jobid, resultsdir = resultsdir, data = data, partial = partial)
  cpidata.filter(cores_list = cores_list, core_mincomp = core_mincomp)
  if aggregate:
    cpidata.aggregate()

  # Build the structure descriptor
  cpiitems = cpistack_items.CpiItems(items = items, groups = groups, use_simple = use_simple, use_simple_mem = use_simple_mem)

  # Group data according to descriptor
  results = cpistack_results.CpiResults(cpidata, cpiitems, no_collapse = no_collapse)

  return results

def output_cpistack_text(results):
  print '                        CPI       Time'
  cpi = results.get_data('cpi')
  time = results.get_data('time')

  for core in results.cores:
    if len(results.cores) > 1:
      print 'Core', core
    total = { 'cpi': 0, 'time': 0 }
    for label in results.labels:
      print '  %-15s    %6.2f    %6.2f%%' % (label, cpi[core][label], 100 * time[core][label])
      total['cpi'] += cpi[core][label]
      total['time'] += time[core][label]
    print
    print '  %-15s    %6.2f    %6.2f%%' % ('total', total['cpi'], 100 * total['time'])

def output_cpistack_gnuplot(results, metric = 'time', outputfile = 'cpi-stack', outputdir = '.', title = '', size = (640, 480)):
  # Use Gnuplot to make stacked bargraphs of these cpi-stacks
  plot_data = results.get_data(metric)
  colors = results.get_colors()
  plot_labels_with_color = [ (label, 'rgb "#%02x%02x%02x"' % colors[label]) for label in results.labels ]
  gnuplot.make_stacked_bargraph(os.path.join(outputdir, outputfile), plot_labels_with_color, plot_data, size = size, title = title,
    ylabel = metric == 'cpi' and 'Cycles per instruction' or (metric == 'abstime' and 'Time (seconds)' or 'Fraction of time'))

if __name__ == '__main__':
  def usage():
    print 'Usage:', sys.argv[0], '[-h|--help (help)] [-j <jobid> | -d <resultsdir (default: .)>] [-o <output-filename (cpi-stack)>] [--title=""] [--without-roi] [--simplified] [--no-collapse] [--no-simple-mem] [--time|--cpi|--abstime (default: time)] [--aggregate]'

  jobid = 0
  resultsdir = '.'
  partial = None
  outputfile = 'cpi-stack'
  title = ''
  metric = 'time'
  use_simple = False
  use_simple_mem = True
  no_collapse = False
  aggregate = False

  try:
    opts, args = getopt.getopt(sys.argv[1:], "hj:d:o:", [ "help", "title=", "no-roi", "simplified", "no-collapse", "no-simple-mem", "cpi", "time", "abstime", "aggregate", "partial=" ])
  except getopt.GetoptError, e:
    print e
    usage()
    sys.exit()
  for o, a in opts:
    if o == '-h' or o == '--help':
      usage()
      sys.exit()
    if o == '-d':
      resultsdir = a
    if o == '-j':
      jobid = long(a)
    if o == '-o':
      outputfile = a
    if o == '--title':
      title = a
    if o == '--no-roi':
      partial = ('start', 'stop')
    if o == '--simplified':
      use_simple = True
    if o == '--no-collapse':
      no_collapse = True
    if o == '--no-simple-mem':
      use_simple_mem = False
    if o == '--time':
      pass
    if o == '--cpi':
      metric = 'cpi'
    if o == '--abstime':
      metric = 'abstime'
    if o == '--aggregate':
      aggregate = True
    if o == '--partial':
      if ':' not in a:
        sys.stderr.write('--partial=<from>:<to>\n')
        usage()
      partial = a.split(':')

  if args:
    usage()
    sys.exit(-1)

  results = cpistack_compute(jobid = jobid, resultsdir = resultsdir, partial = partial, aggregate = aggregate,
                             groups = use_simple and cpistack_items.build_grouplist(legacy = True) or None,
                             use_simple = use_simple, use_simple_mem = use_simple_mem, no_collapse = no_collapse)

  output_cpistack_text(results)
  output_cpistack_gnuplot(results = results, metric = metric, outputfile = outputfile, title = title)
<commit_msg>[PATCH 095/123] [cpistack] Add compatibility layer for legacy cpistack.cpistack() to avoid having to change all our papers scripts<commit_after>#!/usr/bin/env python

import os, sys, re, collections, gnuplot, getopt
import cpistack_data, cpistack_items, cpistack_results

try:
  collections.defaultdict()
except AttributeError, e:
  print sys.argv[0], "Error: This script requires Python version 2.5 or greater"
  sys.exit()

# New-style function returning CPI stack results
def cpistack_compute(jobid = 0, resultsdir = '.', data = None, partial = None,
                     cores_list = None, core_mincomp = 0., aggregate = False,
                     items = None, groups = None, use_simple = False, use_simple_mem = True,
                     no_collapse = False):

  # Get the data
  cpidata = cpistack_data.CpiData(jobid = jobid, resultsdir = resultsdir, data = data, partial = partial)
  cpidata.filter(cores_list = cores_list, core_mincomp = core_mincomp)
  if aggregate:
    cpidata.aggregate()

  # Build the structure descriptor
  cpiitems = cpistack_items.CpiItems(items = items, groups = groups, use_simple = use_simple, use_simple_mem = use_simple_mem)

  # Group data according to descriptor
  results = cpistack_results.CpiResults(cpidata, cpiitems, no_collapse = no_collapse)

  return results

def output_cpistack_text(results):
  print '                        CPI       Time'
  cpi = results.get_data('cpi')
  time = results.get_data('time')

  for core in results.cores:
    if len(results.cores) > 1:
      print 'Core', core
    total = { 'cpi': 0, 'time': 0 }
    for label in results.labels:
      print '  %-15s    %6.2f    %6.2f%%' % (label, cpi[core][label], 100 * time[core][label])
      total['cpi'] += cpi[core][label]
      total['time'] += time[core][label]
    print
    print '  %-15s    %6.2f    %6.2f%%' % ('total', total['cpi'], 100 * total['time'])

def output_cpistack_gnuplot(results, metric = 'time', outputfile = 'cpi-stack', outputdir = '.', title = '', size = (640, 480)):
  # Use Gnuplot to make stacked bargraphs of these cpi-stacks
  plot_data = results.get_data(metric)
  colors = results.get_colors()
  plot_labels_with_color = [ (label, 'rgb "#%02x%02x%02x"' % colors[label]) for label in results.labels ]
  gnuplot.make_stacked_bargraph(os.path.join(outputdir, outputfile), plot_labels_with_color, plot_data, size = size, title = title,
    ylabel = metric == 'cpi' and 'Cycles per instruction' or (metric == 'abstime' and 'Time (seconds)' or 'Fraction of time'))

# Legacy cpistack.cpistack() function, does most of the old one as not to break most of the scripts in papers/
def cpistack(jobid = 0, resultsdir = '.', data = None, partial = None, outputfile = 'cpi-stack', outputdir = '.',
             use_cpi = False, use_abstime = False, use_roi = True,
             use_simple = False, use_simple_mem = True, no_collapse = False,
             gen_text_stack = True, gen_plot_stack = True, gen_csv_stack = False, csv_print_header = False,
             job_name = '', title = '', threads = None, threads_mincomp = .5, return_data = False, aggregate = False,
             size = (640, 480)):

  if gen_csv_stack or csv_print_header or return_data:
    # Not all functionality has been forward-ported, preferably use cpistack_compute instead
    raise NotImplementedError

  results = cpistack_compute(jobid = jobid, resultsdir = resultsdir, partial = partial, aggregate = aggregate,
                             cores_list = threads, core_mincomp = threads_mincomp,
                             groups = use_simple and cpistack_items.build_grouplist(legacy = True) or None,
                             use_simple = use_simple, use_simple_mem = use_simple_mem, no_collapse = no_collapse)

  if gen_text_stack:
    output_cpistack_text(results)
  if gen_plot_stack:
    metric = 'cpi' if use_cpi else ('abstime' if use_abstime else 'time')
    output_cpistack_gnuplot(results = results, metric = metric, outputfile = os.path.join(outputdir, outputfile), title = title, size = size)

if __name__ == '__main__':
  def usage():
    print 'Usage:', sys.argv[0], '[-h|--help (help)] [-j <jobid> | -d <resultsdir (default: .)>] [-o <output-filename (cpi-stack)>] [--title=""] [--without-roi] [--simplified] [--no-collapse] [--no-simple-mem] [--time|--cpi|--abstime (default: time)] [--aggregate]'

  jobid = 0
  resultsdir = '.'
  partial = None
  outputfile = 'cpi-stack'
  title = ''
  metric = 'time'
  use_simple = False
  use_simple_mem = True
  no_collapse = False
  aggregate = False

  try:
    opts, args = getopt.getopt(sys.argv[1:], "hj:d:o:", [ "help", "title=", "no-roi", "simplified", "no-collapse", "no-simple-mem", "cpi", "time", "abstime", "aggregate", "partial=" ])
  except getopt.GetoptError, e:
    print e
    usage()
    sys.exit()
  for o, a in opts:
    if o == '-h' or o == '--help':
      usage()
      sys.exit()
    if o == '-d':
      resultsdir = a
    if o == '-j':
      jobid = long(a)
    if o == '-o':
      outputfile = a
    if o == '--title':
      title = a
    if o == '--no-roi':
      partial = ('start', 'stop')
    if o == '--simplified':
      use_simple = True
    if o == '--no-collapse':
      no_collapse = True
    if o == '--no-simple-mem':
      use_simple_mem = False
    if o == '--time':
      pass
    if o == '--cpi':
      metric = 'cpi'
    if o == '--abstime':
      metric = 'abstime'
    if o == '--aggregate':
      aggregate = True
    if o == '--partial':
      if ':' not in a:
        sys.stderr.write('--partial=<from>:<to>\n')
        usage()
      partial = a.split(':')

  if args:
    usage()
    sys.exit(-1)

  results = cpistack_compute(jobid = jobid, resultsdir = resultsdir, partial = partial, aggregate = aggregate,
                             groups = use_simple and cpistack_items.build_grouplist(legacy = True) or None,
                             use_simple = use_simple, use_simple_mem = use_simple_mem, no_collapse = no_collapse)

  output_cpistack_text(results)
  output_cpistack_gnuplot(results = results, metric = metric, outputfile = outputfile, title = title)
<|endoftext|>


<commit_before>#! /usr/bin/python3

import sys
import argparse
import xml_utils as u
import datetime
from collections import defaultdict

##------------------------------------------------------------
##  can be called with:
##    tsv_gen_embeddings *.xml dirs
##------------------------------------------------------------
def main (argv) :
	parser = argparse.ArgumentParser(description='Generate tsv of embeddings from xml.',
		formatter_class=lambda prog: argparse.HelpFormatter(prog,max_help_position=50))
	parser.add_argument ('files', nargs='+')
	parser.add_argument ('-v', '--verbosity', type=int, default=1,
		choices=[0, 1, 2, 3], help='')
		# help="increase output verbosity"
	u.set_argv (argv)
	args = parser.parse_args()
	u.set_verbosity (args.verbosity)
	u.set_argv (argv)
	u.set_filetype ('embeddings')
	verbose = 0
	if verbose > 0:
		print("files: ", args.files)

	xml_files = u.generate_xml_file_list (args.files)
	u.plot_embeddings (xml_files)

if __name__ == "__main__":
	main (sys.argv)

<commit_msg>Write out tsv embedding files instead of plotting them.<commit_after>#! /usr/bin/python3

import sys
import argparse
import xml_utils as u
import datetime
from collections import defaultdict

##------------------------------------------------------------
##  can be called with:
##    tsv_gen_embeddings *.xml dirs
##------------------------------------------------------------
def main (argv) :
	parser = argparse.ArgumentParser(description='Generate tsv of embeddings from xml.',
		formatter_class=lambda prog: argparse.HelpFormatter(prog,max_help_position=50))
	parser.add_argument ('files', nargs='+')
	parser.add_argument ('-v', '--verbosity', type=int, default=1,
		choices=[0, 1, 2, 3], help='')
		# help="increase output verbosity"
	u.set_argv (argv)
	args = parser.parse_args()
	u.set_verbosity (args.verbosity)
	u.set_argv (argv)
	u.set_filetype ('embeddings')
	verbose = 0
	if verbose > 0:
		print("files: ", args.files)

	xml_files = u.generate_xml_file_list (args.files)
	u.write_embed_tsv (xml_files)

if __name__ == "__main__":
	main (sys.argv)

<|endoftext|>


<commit_before>
import time
import cStringIO
import struct

from twisted.internet import reactor

from logs import lg

from lib import udp

#------------------------------------------------------------------------------

"""
Datagram Format

[Data] packet

bytes:
  0        software version number
  1        command identifier, see ``lib.udp`` module
  2-5      stream_id
  6-9      total data size to be transferred, peer must know when to stop receiving
  10-13    block_id, outgoing blocks are counted from 1
  from 14  payload data

"""

UDP_DATAGRAM_SIZE = 508
BLOCK_SIZE = UDP_DATAGRAM_SIZE - 14

BLOCKS_PER_ACK = 16

MAX_BUFFER_SIZE = 64*1024
BUFFER_SIZE = BLOCK_SIZE * 8 # int(float(BLOCKS_PER_ACK)*0.8) - 20% extra space in ack packet

RTT_MIN_LIMIT = 0.001
RTT_MAX_LIMIT = 0.1

#------------------------------------------------------------------------------

class UDPStream():
    def __init__(self, stream_id, consumer, producer):
        self.stream_id = stream_id
        self.consumer = consumer
        self.producer = producer
        self.consumer.stream = self
        # self.send_data_packet_func = send_data_packet_func
        # self.send_ack_packet_func = send_ack_packet_func
        # self.received_raw_data_callback = received_raw_data_callback
        # self.sent_raw_data_callback = sent_raw_data_callback
        self.output_buffer_size = 0
        self.output_blocks = {}
        self.output_blocks_acks = []
        self.output_block_id = 0
        self.input_blocks = {}
        self.input_block_id = 0
        self.blocks_to_ack = set()
        self.bytes_in = 0
        self.bytes_sent = 0
        self.bytes_acked = 0
        self.resend_bytes = 0
        self.last_ack_moment = 0
        self.last_ack_rtt = RTT_MIN_LIMIT
        self.resend_task = None
        self.resend_inactivity_counter = 0
        self.creation_time = time.time()
        lg.out(18, 'udp_stream.__init__ %d' % self.stream_id)

    def __del__(self):
        lg.out(18, 'udp_stream.__del__ %d' % self.stream_id)

    def close(self):
        print 'udp_stream.close %d in:%d out:%d acked:%d resend:%d' % (
            self.stream_id, self.bytes_in, self.bytes_sent, self.bytes_acked, self.resend_bytes)
        self.stop_resending()
        self.consumer.stream = None
        self.consumer = None
        self.producer = None
        # self.send_data_packet_func = None
        # self.send_ack_packet_func = None
        # self.received_raw_data_callback = None
        # self.sent_raw_data_callback = None

    def block_received(self, inpt):
        if self.consumer:
            block_id = struct.unpack('i', inpt.read(4))[0]
            data = inpt.read()
            self.input_blocks[block_id] = data
            self.bytes_in += len(data)
            self.blocks_to_ack.add(block_id)
            eof_state = False
            # print 'block', block_id, self.bytes_in, block_id % BLOCKS_PER_ACK
            if block_id == self.input_block_id + 1:
                newdata = []
                # print 'newdata',
                while True:
                    next_block_id = self.input_block_id + 1
                    try:
                        newdata.append(self.input_blocks.pop(next_block_id))
                    except:
                        break
                    self.input_block_id = next_block_id
                    # print next_block_id,
                num_blocks = len(newdata)
                newdata = ''.join(newdata)
                eof_state = self.consumer.on_received_raw_data(newdata)
                # print 'received %d bytes in %d blocks, eof=%r' % (len(newdata), num_blocks, eof_state)
            # want to send the first ack asap
            if time.time() - self.last_ack_moment > RTT_MAX_LIMIT \
                or block_id % BLOCKS_PER_ACK == 1 \
                or eof_state:
                    # self.send_ack()
                    self.resend()

    def ack_received(self, inpt):
        if self.consumer:
            acks = []
            eof = False
            raw_bytes = ''
            while True:
                raw_bytes = inpt.read(4)
                if not raw_bytes:
                    break
                block_id = struct.unpack('i', raw_bytes)[0]
                acks.append(block_id)
                try:
                    outblock = self.output_blocks.pop(block_id)
                except KeyError:
                    # lg.out(10, 'udp_stream.ack_received WARNING block %d not found' % (block_id))
                    continue
                block_size = len(outblock[0])
                self.output_buffer_size -= block_size
                self.bytes_acked += block_size
                relative_time = time.time() - self.creation_time
                self.last_ack_rtt = relative_time - outblock[1]
                self.consumer.on_sent_raw_data(block_size)
                eof = self.consumer and self.consumer.size == self.bytes_acked
            if len(acks) > 0:
                print 'ack', len(acks), 'blocks,     more:', len(self.output_blocks.keys()),
                print 'rtt:', self.last_ack_rtt, 'eof:', eof,
                print 'acked:', self.bytes_acked, 'sent:', self.bytes_sent
                self.resend()
                return
            print 'STOP IT NOW!!!!, ZERO ACK!!!! SEEMS FINE.!!!'
            self.stop_resending()
            sum_not_acked_blocks = sum(map(lambda block: len(block[0]),
                                           self.output_blocks.values()))
            self.output_blocks.clear()
            self.output_buffer_size = 0
            self.bytes_acked += sum_not_acked_blocks
            relative_time = time.time() - self.creation_time
            self.consumer.on_zero_ack(sum_not_acked_blocks)
            return

    def write(self, data):
        if self.output_buffer_size + len(data) > MAX_BUFFER_SIZE:
            raise BufferOverflow('buffer size is %d' % self.output_buffer_size)
        # print 'write', len(data)
        outp = cStringIO.StringIO(data)
        while True:
            piece = outp.read(BLOCK_SIZE)
            if not piece:
                break
            self.output_block_id += 1
            self.output_blocks[self.output_block_id] = (piece, -1)
            self.output_buffer_size += len(piece)
        outp.close()
        self.resend()

    def send_blocks(self):
        if self.consumer:
            relative_time = time.time() - self.creation_time
            for block_id in self.output_blocks.keys():
                piece, time_sent = self.output_blocks[block_id]
                data_size = len(piece)
                if time_sent >= 0:
                    dt = relative_time - time_sent
                    if dt > RTT_MAX_LIMIT:
                        self.resend_bytes += data_size
                        # print 're -',
                    else:
                        # print 'skip', block_id, dt, self.last_ack_rtt
                        continue
                else:
                    pass
                    # print 'go', block_id
                time_sent = relative_time
                self.output_blocks[block_id] = (piece, time_sent)
                output = ''.join((
                    struct.pack('i', block_id),
                    piece))
                # DEBUG
                # import random
                # if random.randint(0, 9) >= 1:
                    # 10 % percent lost
                    # self.producer.do_send_data(self.stream_id, self.consumer, output)
                self.producer.do_send_data(self.stream_id, self.consumer, output)
                self.bytes_sent += data_size
                # print 'send block', block_id, self.bytes_sent, self.bytes_acked, self.resend_bytes

    def send_ack(self):
        if self.consumer:
            ack_data = ''.join(map(lambda bid: struct.pack('i', bid), self.blocks_to_ack))
            self.producer.do_send_ack(self.stream_id, self.consumer, ack_data)
            self.output_blocks_acks += list(self.blocks_to_ack)
            print 'send ack', len(self.output_blocks_acks), len(self.blocks_to_ack), self.bytes_in
            self.blocks_to_ack.clear()
            self.last_ack_moment = time.time()

    def resend(self):
        activitiy = len(self.output_blocks.keys()) + len(self.blocks_to_ack)
        if activitiy > 0:
            # print 'resend out:%s acks:%s' % (len(self.output_blocks.keys()), len(self.blocks_to_ack))
            self.resend_inactivity_counter = 0
        else:
            self.resend_inactivity_counter += 1
        next_resend = max(min(self.last_ack_rtt, RTT_MAX_LIMIT), RTT_MIN_LIMIT)
        if self.resend_inactivity_counter > 10:
            print 'drop resend out:%s acks:%s' % (len(self.output_blocks.keys()), len(self.blocks_to_ack))
            next_resend = RTT_MAX_LIMIT * 2.0
        if len(self.blocks_to_ack) > 0:
            if time.time() - self.last_ack_moment > RTT_MAX_LIMIT:
                self.send_ack()
        if len(self.output_blocks):
            self.send_blocks()
        if self.resend_task is None:
            self.resend_task = reactor.callLater(next_resend, self.resend)
            return
        if self.resend_task.called:
            self.resend_task = reactor.callLater(next_resend, self.resend)
            return
        if self.resend_task.cancelled:
            self.resend_task = None

    def stop_resending(self):
        if self.resend_task:
            if self.resend_task.active():
                self.resend_task.cancel()
                self.resend_task = None

#------------------------------------------------------------------------------

class BufferOverflow(Exception):
    pass

<commit_msg>27.07.2014 at 19:47:13<commit_after>
import time
import cStringIO
import struct

from twisted.internet import reactor

from logs import lg

from lib import udp

#------------------------------------------------------------------------------

"""
Datagram Format

[Data] packet

bytes:
  0        software version number
  1        command identifier, see ``lib.udp`` module
  2-5      stream_id
  6-9      total data size to be transferred, peer must know when to stop receiving
  10-13    block_id, outgoing blocks are counted from 1
  from 14  payload data

"""

UDP_DATAGRAM_SIZE = 508
BLOCK_SIZE = UDP_DATAGRAM_SIZE - 14

BLOCKS_PER_ACK = 16

MAX_BUFFER_SIZE = 64*1024
BUFFER_SIZE = BLOCK_SIZE * 8 # int(float(BLOCKS_PER_ACK)*0.8) - 20% extra space in ack packet

RTT_MIN_LIMIT = 0.001
RTT_MAX_LIMIT = 0.1

#------------------------------------------------------------------------------

class UDPStream():
    def __init__(self, stream_id, consumer, producer):
        self.stream_id = stream_id
        self.consumer = consumer
        self.producer = producer
        self.consumer.stream = self
        # self.send_data_packet_func = send_data_packet_func
        # self.send_ack_packet_func = send_ack_packet_func
        # self.received_raw_data_callback = received_raw_data_callback
        # self.sent_raw_data_callback = sent_raw_data_callback
        self.output_buffer_size = 0
        self.output_blocks = {}
        self.output_blocks_acks = []
        self.output_block_id = 0
        self.input_blocks = {}
        self.input_block_id = 0
        self.blocks_to_ack = set()
        self.bytes_in = 0
        self.bytes_sent = 0
        self.bytes_acked = 0
        self.resend_bytes = 0
        self.last_ack_moment = 0
        self.last_ack_rtt = RTT_MIN_LIMIT
        self.resend_task = None
        self.resend_inactivity_counter = 0
        self.creation_time = time.time()
        lg.out(18, 'udp_stream.__init__ %d' % self.stream_id)

    def __del__(self):
        lg.out(18, 'udp_stream.__del__ %d' % self.stream_id)

    def close(self):
        print 'udp_stream.close %d in:%d out:%d acked:%d resend:%d' % (
            self.stream_id, self.bytes_in, self.bytes_sent, self.bytes_acked, self.resend_bytes)
        self.stop_resending()
        self.consumer.stream = None
        self.consumer = None
        self.producer = None
        # self.send_data_packet_func = None
        # self.send_ack_packet_func = None
        # self.received_raw_data_callback = None
        # self.sent_raw_data_callback = None

    def block_received(self, inpt):
        if self.consumer:
            block_id = struct.unpack('i', inpt.read(4))[0]
            data = inpt.read()
            self.input_blocks[block_id] = data
            self.bytes_in += len(data)
            self.blocks_to_ack.add(block_id)
            eof_state = False
            # print 'block', block_id, self.bytes_in, block_id % BLOCKS_PER_ACK
            if block_id == self.input_block_id + 1:
                newdata = []
                # print 'newdata',
                while True:
                    next_block_id = self.input_block_id + 1
                    try:
                        newdata.append(self.input_blocks.pop(next_block_id))
                    except:
                        break
                    self.input_block_id = next_block_id
                    # print next_block_id,
                num_blocks = len(newdata)
                newdata = ''.join(newdata)
                eof_state = self.consumer.on_received_raw_data(newdata)
                # print 'received %d bytes in %d blocks, eof=%r' % (len(newdata), num_blocks, eof_state)
            # want to send the first ack asap
            if time.time() - self.last_ack_moment > RTT_MAX_LIMIT \
                or block_id % BLOCKS_PER_ACK == 1 \
                or eof_state:
                    # self.send_ack()
                    self.resend()

    def ack_received(self, inpt):
        if self.consumer:
            acks = []
            eof = False
            raw_bytes = ''
            while True:
                raw_bytes = inpt.read(4)
                if not raw_bytes:
                    break
                block_id = struct.unpack('i', raw_bytes)[0]
                acks.append(block_id)
                try:
                    outblock = self.output_blocks.pop(block_id)
                except KeyError:
                    # lg.out(10, 'udp_stream.ack_received WARNING block %d not found' % (block_id))
                    continue
                block_size = len(outblock[0])
                self.output_buffer_size -= block_size
                self.bytes_acked += block_size
                relative_time = time.time() - self.creation_time
                self.last_ack_rtt = relative_time - outblock[1]
                self.consumer.on_sent_raw_data(block_size)
                eof = self.consumer and self.consumer.size == self.bytes_acked
            if len(acks) > 0:
                print 'ack', len(acks), 'blocks,     more:', len(self.output_blocks.keys()),
                print 'rtt:', self.last_ack_rtt, 'eof:', eof,
                print 'acked:', self.bytes_acked, 'sent:', self.bytes_sent
                self.resend()
                return
            print 'STOP IT NOW!!!!, ZERO ACK!!!! SEEMS FINE.!!!'
            self.stop_resending()
            sum_not_acked_blocks = sum(map(lambda block: len(block[0]),
                                           self.output_blocks.values()))
            self.output_blocks.clear()
            self.output_buffer_size = 0
            self.bytes_acked += sum_not_acked_blocks
            relative_time = time.time() - self.creation_time
            self.consumer.on_zero_ack(sum_not_acked_blocks)
            return

    def write(self, data):
        if self.output_buffer_size + len(data) > MAX_BUFFER_SIZE:
            raise BufferOverflow('buffer size is %d' % self.output_buffer_size)
        # print 'write', len(data)
        outp = cStringIO.StringIO(data)
        while True:
            piece = outp.read(BLOCK_SIZE)
            if not piece:
                break
            self.output_block_id += 1
            self.output_blocks[self.output_block_id] = (piece, -1)
            self.output_buffer_size += len(piece)
        outp.close()
        self.resend()

    def send_blocks(self):
        if self.consumer:
            relative_time = time.time() - self.creation_time
            for block_id in self.output_blocks.keys():
                piece, time_sent = self.output_blocks[block_id]
                data_size = len(piece)
                if time_sent >= 0:
                    dt = relative_time - time_sent
                    if dt > self.last_ack_rtt * 2:
                        self.resend_bytes += data_size
                        # print 're -',
                    else:
                        # print 'skip', block_id, dt, self.last_ack_rtt
                        continue
                else:
                    pass
                    # print 'go', block_id
                time_sent = relative_time
                self.output_blocks[block_id] = (piece, time_sent)
                output = ''.join((
                    struct.pack('i', block_id),
                    piece))
                # DEBUG
                # import random
                # if random.randint(0, 9) >= 1:
                    # 10 % percent lost
                    # self.producer.do_send_data(self.stream_id, self.consumer, output)
                self.producer.do_send_data(self.stream_id, self.consumer, output)
                self.bytes_sent += data_size
                # print 'send block', block_id, self.bytes_sent, self.bytes_acked, self.resend_bytes

    def send_ack(self):
        if self.consumer:
            ack_data = ''.join(map(lambda bid: struct.pack('i', bid), self.blocks_to_ack))
            self.producer.do_send_ack(self.stream_id, self.consumer, ack_data)
            self.output_blocks_acks += list(self.blocks_to_ack)
            print 'send ack', len(self.output_blocks_acks), len(self.blocks_to_ack), self.bytes_in
            self.blocks_to_ack.clear()
            self.last_ack_moment = time.time()

    def resend(self):
        activitiy = len(self.output_blocks.keys()) + len(self.blocks_to_ack)
        if activitiy > 0:
            # print 'resend out:%s acks:%s' % (len(self.output_blocks.keys()), len(self.blocks_to_ack))
            self.resend_inactivity_counter = 0
        else:
            self.resend_inactivity_counter += 1
        next_resend = max(min(self.last_ack_rtt, RTT_MAX_LIMIT), RTT_MIN_LIMIT)
        if self.resend_inactivity_counter > 10:
            if self.resend_inactivity_counter % 10 == 10:
                print 'drop resend out:%s acks:%s' % (len(self.output_blocks.keys()), len(self.blocks_to_ack))
            next_resend = RTT_MAX_LIMIT * 4.0
        if len(self.blocks_to_ack) > 0:
            if time.time() - self.last_ack_moment > next_resend:
                self.send_ack()
        if len(self.output_blocks):
            self.send_blocks()
        if self.resend_task is None:
            self.resend_task = reactor.callLater(next_resend, self.resend)
            return
        if self.resend_task.called:
            self.resend_task = reactor.callLater(next_resend, self.resend)
            return
        if self.resend_task.cancelled:
            self.resend_task = None

    def stop_resending(self):
        if self.resend_task:
            if self.resend_task.active():
                self.resend_task.cancel()
                self.resend_task = None

#------------------------------------------------------------------------------

class BufferOverflow(Exception):
    pass

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

#   __
#  /__)  _  _     _   _ _/   _
# / (   (- (/ (/ (- _)  /  _)
#          /

"""
Requests HTTP library
~~~~~~~~~~~~~~~~~~~~~

Requests is an HTTP library, written in Python, for human beings. Basic GET
usage:

   >>> import requests
   >>> r = requests.get('https://www.python.org')
   >>> r.status_code
   200
   >>> 'Python is a programming language' in r.content
   True

... or POST:

   >>> payload = dict(key1='value1', key2='value2')
   >>> r = requests.post('http://httpbin.org/post', data=payload)
   >>> print(r.text)
   {
     ...
     "form": {
       "key2": "value2",
       "key1": "value1"
     },
     ...
   }

The other HTTP methods are supported - see `requests.api`. Full documentation
is at <http://python-requests.org>.

:copyright: (c) 2016 by Kenneth Reitz.
:license: Apache 2.0, see LICENSE for more details.
"""

__title__ = 'requests'
__version__ = '2.12.4'
__build__ = 0x021204
__author__ = 'Kenneth Reitz'
__license__ = 'Apache 2.0'
__copyright__ = 'Copyright 2016 Kenneth Reitz'

# Attempt to enable urllib3's SNI support, if possible
try:
    from .packages.urllib3.contrib import pyopenssl
    pyopenssl.inject_into_urllib3()
except ImportError:
    pass

import warnings

# urllib3's DependencyWarnings should be silenced.
from .packages.urllib3.exceptions import DependencyWarning
warnings.simplefilter('ignore', DependencyWarning)

from . import utils
from .models import Request, Response, PreparedRequest
from .api import request, get, head, post, patch, put, delete, options
from .sessions import session, Session
from .status_codes import codes
from .exceptions import (
    RequestException, Timeout, URLRequired,
    TooManyRedirects, HTTPError, ConnectionError,
    FileModeWarning, ConnectTimeout, ReadTimeout
)

# Set default logging handler to avoid "No handler found" warnings.
import logging
try:  # Python 2.7+
    from logging import NullHandler
except ImportError:
    class NullHandler(logging.Handler):
        def emit(self, record):
            pass

logging.getLogger(__name__).addHandler(NullHandler())

# FileModeWarnings go off per the default.
warnings.simplefilter('default', FileModeWarning, append=True)
<commit_msg>v2.12.5<commit_after># -*- coding: utf-8 -*-

#   __
#  /__)  _  _     _   _ _/   _
# / (   (- (/ (/ (- _)  /  _)
#          /

"""
Requests HTTP library
~~~~~~~~~~~~~~~~~~~~~

Requests is an HTTP library, written in Python, for human beings. Basic GET
usage:

   >>> import requests
   >>> r = requests.get('https://www.python.org')
   >>> r.status_code
   200
   >>> 'Python is a programming language' in r.content
   True

... or POST:

   >>> payload = dict(key1='value1', key2='value2')
   >>> r = requests.post('http://httpbin.org/post', data=payload)
   >>> print(r.text)
   {
     ...
     "form": {
       "key2": "value2",
       "key1": "value1"
     },
     ...
   }

The other HTTP methods are supported - see `requests.api`. Full documentation
is at <http://python-requests.org>.

:copyright: (c) 2016 by Kenneth Reitz.
:license: Apache 2.0, see LICENSE for more details.
"""

__title__ = 'requests'
__version__ = '2.12.5'
__build__ = 0x021205
__author__ = 'Kenneth Reitz'
__license__ = 'Apache 2.0'
__copyright__ = 'Copyright 2016 Kenneth Reitz'

# Attempt to enable urllib3's SNI support, if possible
try:
    from .packages.urllib3.contrib import pyopenssl
    pyopenssl.inject_into_urllib3()
except ImportError:
    pass

import warnings

# urllib3's DependencyWarnings should be silenced.
from .packages.urllib3.exceptions import DependencyWarning
warnings.simplefilter('ignore', DependencyWarning)

from . import utils
from .models import Request, Response, PreparedRequest
from .api import request, get, head, post, patch, put, delete, options
from .sessions import session, Session
from .status_codes import codes
from .exceptions import (
    RequestException, Timeout, URLRequired,
    TooManyRedirects, HTTPError, ConnectionError,
    FileModeWarning, ConnectTimeout, ReadTimeout
)

# Set default logging handler to avoid "No handler found" warnings.
import logging
try:  # Python 2.7+
    from logging import NullHandler
except ImportError:
    class NullHandler(logging.Handler):
        def emit(self, record):
            pass

logging.getLogger(__name__).addHandler(NullHandler())

# FileModeWarnings go off per the default.
warnings.simplefilter('default', FileModeWarning, append=True)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from ..formatters.output_formatter import OutputFormatter

class OutputError(Exception):
    pass

class Output(object):

    VERBOSITY_QUIET = 0
    VERBOSITY_NORMAL = 1
    VERBOSITY_VERBOSE = 2

    OUTPUT_NORMAL = 0
    OUTPUT_RAW = 1
    OUTPUT_PLAIN = 2

    def __init__(self, verbosity=VERBOSITY_NORMAL, decorated=None, formatter=None):
        self.verbosity = self.VERBOSITY_NORMAL if verbosity is None else verbosity
        self.formatter = formatter or OutputFormatter()
        self.formatter.set_decorated(bool(decorated))

    def set_formatter(self, formatter):
        self.formatter = formatter

    def get_formatter(self):
        return self.formatter

    def set_decorated(self, decorated):
        self.formatter.set_decorated(bool(decorated))

    def is_decorated(self):
        return self.formatter.is_decorated()

    def set_verbosity(self, level):
        self.verbosity = int(level)

    def get_verbosity(self):
        return self.verbosity

    def is_quiet(self):
        return self.VERBOSITY_QUIET == self.verbosity

    def is_verbose(self):
        return self.VERBOSITY_VERBOSE == self.verbosity

    def write(self, messages, newline=False, output_type=OUTPUT_NORMAL):
        if self.verbosity == self.VERBOSITY_QUIET:
            return

        if not isinstance(messages, (list, tuple)):
            messages = [messages]

        for message in messages:
            if output_type == self.OUTPUT_NORMAL:
                message = self.formatter.format(message)
            elif output_type == self.OUTPUT_RAW:
                pass
            elif output_type == self.OUTPUT_PLAIN:
                message = self.formatter.format(message)
            else:
                raise OutputError('Unknown output type given (%s)' % output_type)

            self.do_write(message, newline)

    def writeln(self, messages, output_type=OUTPUT_NORMAL):
        self.write(messages, True, output_type)

    def do_write(self, message, newline):
        raise NotImplementedError()
<commit_msg>Adding more levels of verbosity<commit_after># -*- coding: utf-8 -*-

from ..formatters.output_formatter import OutputFormatter

class OutputError(Exception):
    pass

class Output(object):

    VERBOSITY_QUIET = 16
    VERBOSITY_NORMAL = 32
    VERBOSITY_VERBOSE = 64
    VERBOSITY_VERY_VERBOSE = 128
    VERBOSITY_DEBUG = 256

    OUTPUT_NORMAL = 0
    OUTPUT_RAW = 1
    OUTPUT_PLAIN = 2

    def __init__(self, verbosity=VERBOSITY_NORMAL, decorated=None, formatter=None):
        self.verbosity = self.VERBOSITY_NORMAL if verbosity is None else verbosity
        self.formatter = formatter or OutputFormatter()
        self.formatter.set_decorated(bool(decorated))

    def set_formatter(self, formatter):
        self.formatter = formatter

    def get_formatter(self):
        return self.formatter

    def set_decorated(self, decorated):
        self.formatter.set_decorated(bool(decorated))

    def is_decorated(self):
        return self.formatter.is_decorated()

    def set_verbosity(self, level):
        self.verbosity = int(level)

    def get_verbosity(self):
        return self.verbosity

    def is_quiet(self):
        return self.VERBOSITY_QUIET == self.verbosity

    def is_verbose(self):
        return self.VERBOSITY_VERBOSE == self.verbosity

    def is_very_verbose(self):
        return self.VERBOSITY_VERY_VERBOSE == self.verbosity

    def is_debug(self):
        return self.VERBOSITY_DEBUG == self.verbosity

    def write(self, messages, newline=False, output_type=OUTPUT_NORMAL):
        if self.verbosity == self.VERBOSITY_QUIET:
            return

        if not isinstance(messages, (list, tuple)):
            messages = [messages]

        for message in messages:
            if output_type == self.OUTPUT_NORMAL:
                message = self.formatter.format(message)
            elif output_type == self.OUTPUT_RAW:
                pass
            elif output_type == self.OUTPUT_PLAIN:
                message = self.formatter.format(message)
            else:
                raise OutputError('Unknown output type given (%s)' % output_type)

            self.do_write(message, newline)

    def writeln(self, messages, output_type=OUTPUT_NORMAL):
        self.write(messages, True, output_type)

    def do_write(self, message, newline):
        raise NotImplementedError()
<|endoftext|>


<commit_before>from django.db import models
from vocab.managers import ItemIndexManager

class AbstractItem(models.Model):
    "The foreign key or many-to-many field to parent items must be defined."
    terminal = models.BooleanField(default=False)

    class Meta(object):
        abstract = True

    def ancestors(self, include_self=False):
        "Returns a ``QuerySet`` containing all ancestors of this item."
        ids = self.item_indexes.exclude(parent=None).values_list('parent', flat=True)
        if include_self:
            ids = list(ids) + [self.pk]
        return self.__class__.objects.filter(pk__in=ids)

    def descendents(self, include_self=False):
        "Returns a ``QuerySet`` containing all descendents of this item."
        ids = self.parent_indexes.values_list('item', flat=True)
        if include_self:
            ids = [self.pk] + list(ids)
        return self.__class__.objects.filter(pk__in=ids)

class AbstractItemIndex(models.Model):
    "Foreign keys to the item and it's must parent must be defined."
    depth = models.IntegerField(default=0)

    objects = ItemIndexManager()

    def __unicode__(self):
        return u'%s via %s' % (self.item, self.parent)

    class Meta(object):
        abstract = True
        ordering = ('depth',)
<commit_msg>Altered `terminal` to be nullable A value for `terminal` may not exist since it is tied to the index<commit_after>from django.db import models
from vocab.managers import ItemIndexManager

class AbstractItem(models.Model):
    "The foreign key or many-to-many field to parent items must be defined."
    terminal = models.NullBooleanField()

    class Meta(object):
        abstract = True

    def ancestors(self, include_self=False):
        "Returns a ``QuerySet`` containing all ancestors of this item."
        ids = self.item_indexes.exclude(parent=None).values_list('parent', flat=True)
        if include_self:
            ids = list(ids) + [self.pk]
        return self.__class__.objects.filter(pk__in=ids)

    def descendents(self, include_self=False):
        "Returns a ``QuerySet`` containing all descendents of this item."
        ids = self.parent_indexes.values_list('item', flat=True)
        if include_self:
            ids = [self.pk] + list(ids)
        return self.__class__.objects.filter(pk__in=ids)

class AbstractItemIndex(models.Model):
    "Foreign keys to the item and it's must parent must be defined."
    depth = models.IntegerField(default=0)

    objects = ItemIndexManager()

    def __unicode__(self):
        return u'%s via %s' % (self.item, self.parent)

    class Meta(object):
        abstract = True
        ordering = ('depth',)
<|endoftext|>


<commit_before>"""Functions invoked in a single execution session."""
# (c) 2012-2022 Wibowo Arindrarto <contact@arindrarto.dev>

import os
import time
from contextlib import suppress
from pathlib import Path
from typing import Optional

import click
import pendulum
import structlog

from . import constants, exceptions as excs
from .config import Config
from .server import Rebuilder, make_server
from .site import Site
from .utils import (
    get_fuzzy_match,
    infer_author,
    infer_front_matter,
    infer_lang,
)

log = structlog.get_logger(__name__)

def new(
    dirname: Optional[str],
    invoc_dir: Path,
    project_dir: Path,
    name: str,
    url: str,
    author: Optional[str],
    description: str,
    language: Optional[str],
    force: bool,
    config_fname: str = constants.CONFIG_FNAME,
) -> Path:
    """Create a new project.

    This function may overwrite any preexisting files and or directories
    in the target working directory.

    :param dirname: Name of the directory in which the project is created.
    :param invoc_dir: Path to the invocation directory.
    :param project_dir: Path to the parent directory in which ``dirname`` is created.
    :param name: Name of the static site, to be put inside the generated config file.
    :param url: URL of the static site, to be put inside the generated config file.
    :param description: Description of the site, to be put inside the generated
        config file.
    :param description: Language of the site, to be put inside the generated
        config file. If set to ``None``, the value will be inferred from the system
        locale.
    :param force: Whether to force project creation in nonempty directories or not.
    :param config_name: Name of the config file to generate.

    :raises ~volt.exceptions.VoltCliError:
        * when the given project directory is not empty and force is False.
        * when any directory creation fails.

    """
    project_dir = (
        project_dir.resolve() / (dirname or ".")
        if dirname is not None and not os.path.isabs(dirname)
        else Path(dirname or ".")
    )
    if not name and dirname is not None:
        name = project_dir.name
    try:
        project_dir.mkdir(parents=True, exist_ok=True)
    except OSError as e:
        raise excs.VoltCliError(e.strerror) from e

    if not force and any(True for _ in project_dir.iterdir()):
        raise excs.VoltCliError(
            f"project directory {project_dir} contains files -- use the `-f` flag to"
            " force creation in nonempty directories"
        )

    # Bootstrap directories.
    config = Config(invoc_dir=invoc_dir, project_dir=project_dir)
    for dp in (
        config.sources_dir,
        config.static_dir,
        config.themes_dir,
    ):
        try:
            dp.mkdir(parents=True, exist_ok=True)
        except OSError as e:
            raise excs.VoltCliError(e.strerror) from e

    # Create initial YAML config file.
    new_conf = f"""---
# Volt configuration file.
name: "{name}"
url: "{url}"
description: "{description}"
author: "{author or (infer_author() or '')}"
language: "{language or (infer_lang() or '')}"
"""

    # Create initial YAML config file.
    (project_dir / config_fname).write_text(new_conf)

    return project_dir

def build(
    config: Config,
    clean: bool = True,
    with_drafts: bool = False,
) -> Site:
    """Build the site.

    This function may overwrite and/or remove any preexisting files
    and or directories.

    :param config: Site configuration.
    :param clean: Whether to remove the entire site output directory prior
        to building, or not.

    """
    config._with_drafts = with_drafts

    start_time = time.monotonic()
    config["build_time"] = pendulum.now()

    site = Site(config=config)
    site.build(clean=clean)

    log.info("build completed", duration=f"{(time.monotonic() - start_time):.2f}s")

    return site

def edit(
    config: Config,
    query: str,
    create: Optional[str] = None,
    title: Optional[str] = None,
    lookup_drafts: bool = False,
) -> None:
    """Open a draft file in an editor."""

    if create is not None:
        fn = Path(config.drafts_dirname) / create / query
        new_fp = fn.with_suffix(constants.MARKDOWN_EXT)
        new_fp.parent.mkdir(parents=True, exist_ok=True)
        if new_fp.exists():
            click.edit(filename=f"{new_fp}")
            return None

        contents = click.edit(
            text=infer_front_matter(query, title),
            extension=constants.MARKDOWN_EXT,
            require_save=False,
        )
        if contents:
            log.info(
                f"created new draft at {str(new_fp.relative_to(config.invoc_dir))!r}"
            )
            new_fp.write_text(contents)

        return None

    match_fp = get_fuzzy_match(
        query=query,
        ext=constants.MARKDOWN_EXT,
        start_dir=config.sources_dir,
        ignore_dirname=None if lookup_drafts else config.drafts_dirname,
    )
    if match_fp is not None:
        click.edit(filename=f"{match_fp}")
        return None

    raise excs.VoltResourceError(f"found no matching file for {query!r}")

def serve(
    config: Config,
    host: Optional[str],
    port: int,
    do_build: bool,
    build_with_drafts: bool,
    build_clean: bool,
) -> None:

    eff_host = "127.0.0.1"
    if host is not None:
        eff_host = host
    elif config.in_docker:
        eff_host = "0.0.0.0"

    serve = make_server(config, eff_host, port)

    if do_build:

        def builder() -> None:
            nonlocal config
            try:
                # TODO: Only reload config post-init, on config file change.
                config = config.reload()
                build(config, build_clean, build_with_drafts)
            except Exception as e:
                msg = "build failed"
                build_exists = False
                target_dir = config.target_dir
                if target_dir.exists() and any(True for _ in target_dir.iterdir()):
                    build_exists = True
                if build_exists:
                    msg += " -- keeping current build"
                log.error(msg)
                log.exception(e)
                return None
            return None

        with Rebuilder(config, builder):
            log.info("starting dev server")
            builder()
            serve()
    else:
        serve()
<commit_msg>feat: Ensure server exception handling code does not raise exceptions<commit_after>"""Functions invoked in a single execution session."""
# (c) 2012-2022 Wibowo Arindrarto <contact@arindrarto.dev>

import os
import time
from contextlib import suppress
from pathlib import Path
from typing import Optional

import click
import pendulum
import structlog

from . import constants, exceptions as excs
from .config import Config
from .server import Rebuilder, make_server
from .site import Site
from .utils import (
    get_fuzzy_match,
    infer_author,
    infer_front_matter,
    infer_lang,
)

log = structlog.get_logger(__name__)

def new(
    dirname: Optional[str],
    invoc_dir: Path,
    project_dir: Path,
    name: str,
    url: str,
    author: Optional[str],
    description: str,
    language: Optional[str],
    force: bool,
    config_fname: str = constants.CONFIG_FNAME,
) -> Path:
    """Create a new project.

    This function may overwrite any preexisting files and or directories
    in the target working directory.

    :param dirname: Name of the directory in which the project is created.
    :param invoc_dir: Path to the invocation directory.
    :param project_dir: Path to the parent directory in which ``dirname`` is created.
    :param name: Name of the static site, to be put inside the generated config file.
    :param url: URL of the static site, to be put inside the generated config file.
    :param description: Description of the site, to be put inside the generated
        config file.
    :param description: Language of the site, to be put inside the generated
        config file. If set to ``None``, the value will be inferred from the system
        locale.
    :param force: Whether to force project creation in nonempty directories or not.
    :param config_name: Name of the config file to generate.

    :raises ~volt.exceptions.VoltCliError:
        * when the given project directory is not empty and force is False.
        * when any directory creation fails.

    """
    project_dir = (
        project_dir.resolve() / (dirname or ".")
        if dirname is not None and not os.path.isabs(dirname)
        else Path(dirname or ".")
    )
    if not name and dirname is not None:
        name = project_dir.name
    try:
        project_dir.mkdir(parents=True, exist_ok=True)
    except OSError as e:
        raise excs.VoltCliError(e.strerror) from e

    if not force and any(True for _ in project_dir.iterdir()):
        raise excs.VoltCliError(
            f"project directory {project_dir} contains files -- use the `-f` flag to"
            " force creation in nonempty directories"
        )

    # Bootstrap directories.
    config = Config(invoc_dir=invoc_dir, project_dir=project_dir)
    for dp in (
        config.sources_dir,
        config.static_dir,
        config.themes_dir,
    ):
        try:
            dp.mkdir(parents=True, exist_ok=True)
        except OSError as e:
            raise excs.VoltCliError(e.strerror) from e

    # Create initial YAML config file.
    new_conf = f"""---
# Volt configuration file.
name: "{name}"
url: "{url}"
description: "{description}"
author: "{author or (infer_author() or '')}"
language: "{language or (infer_lang() or '')}"
"""

    # Create initial YAML config file.
    (project_dir / config_fname).write_text(new_conf)

    return project_dir

def build(
    config: Config,
    clean: bool = True,
    with_drafts: bool = False,
) -> Site:
    """Build the site.

    This function may overwrite and/or remove any preexisting files
    and or directories.

    :param config: Site configuration.
    :param clean: Whether to remove the entire site output directory prior
        to building, or not.

    """
    config._with_drafts = with_drafts

    start_time = time.monotonic()
    config["build_time"] = pendulum.now()

    site = Site(config=config)
    site.build(clean=clean)

    log.info("build completed", duration=f"{(time.monotonic() - start_time):.2f}s")

    return site

def edit(
    config: Config,
    query: str,
    create: Optional[str] = None,
    title: Optional[str] = None,
    lookup_drafts: bool = False,
) -> None:
    """Open a draft file in an editor."""

    if create is not None:
        fn = Path(config.drafts_dirname) / create / query
        new_fp = fn.with_suffix(constants.MARKDOWN_EXT)
        new_fp.parent.mkdir(parents=True, exist_ok=True)
        if new_fp.exists():
            click.edit(filename=f"{new_fp}")
            return None

        contents = click.edit(
            text=infer_front_matter(query, title),
            extension=constants.MARKDOWN_EXT,
            require_save=False,
        )
        if contents:
            log.info(
                f"created new draft at {str(new_fp.relative_to(config.invoc_dir))!r}"
            )
            new_fp.write_text(contents)

        return None

    match_fp = get_fuzzy_match(
        query=query,
        ext=constants.MARKDOWN_EXT,
        start_dir=config.sources_dir,
        ignore_dirname=None if lookup_drafts else config.drafts_dirname,
    )
    if match_fp is not None:
        click.edit(filename=f"{match_fp}")
        return None

    raise excs.VoltResourceError(f"found no matching file for {query!r}")

def serve(
    config: Config,
    host: Optional[str],
    port: int,
    do_build: bool,
    build_with_drafts: bool,
    build_clean: bool,
) -> None:

    eff_host = "127.0.0.1"
    if host is not None:
        eff_host = host
    elif config.in_docker:
        eff_host = "0.0.0.0"

    serve = make_server(config, eff_host, port)

    if do_build:

        def builder() -> None:
            nonlocal config
            try:
                # TODO: Only reload config post-init, on config file change.
                config = config.reload()
                build(config, build_clean, build_with_drafts)
            except Exception as e:
                msg = "build failed"
                build_exists = False
                target_dir = config.target_dir
                with suppress(Exception):
                    if target_dir.exists() and any(True for _ in target_dir.iterdir()):
                        build_exists = True
                if build_exists:
                    msg += " -- keeping current build"
                log.error(msg)
                log.exception(e)
                return None
            return None

        with Rebuilder(config, builder):
            log.info("starting dev server")
            builder()
            serve()
    else:
        serve()
<|endoftext|>