<commit_before>"""
General-purpose tags for working with various aspects of Snippets --
whether a user has bookmarked/rated a given Snippet, etc.

"""

from django import template
from cab.models import Bookmark, Rating

register = template.Library()

class IfBookmarkedNode(template.Node):
    def __init__(self, user_id, snippet_id, nodelist_true, nodelist_false):
        self.user_id = user_id
        self.snippet_id = snippet_id
        self.nodelist_true, self.nodelist_false = nodelist_true, nodelist_false

    def render(self, context):
        try:
            self.user_id = template.resolve_variable(self.user_id, context)
            self.snippet_id = template.resolve_variable(self.snippet_id, context)
        except template.VariableDoesNotExist:
            return ''
        if Bookmark.objects.already_bookmarked(self.user_id, self.snippet_id):
            return self.nodelist_true.render(context)
        return self.nodelist_false.render(context)

class IfRatedNode(template.Node):
    def __init__(self, user_id, snippet_id, nodelist_true, nodelist_false):
        self.user_id = user_id
        self.snippet_id = snippet_id
        self.nodelist_true, self.nodelist_false = nodelist_true, nodelist_false

    def render(self, context):
        try:
            self.user_id = template.resolve_variable(self.user_id, context)
            self.snippet_id = template.resolve_variable(self.snippet_id, context)
        except template.VariableDoesNotExist:
            return ''
        if Rating.objects.already_rated(self.user_id, self.snippet_id):
            return self.nodelist_true.render(context)
        return self.nodelist_false.render(context)

class RatingForSnippetNode(template.Node):
    def __init__(self, snippet_id, context_var):
        self.snippet_id = snippet_id
        self.context_var = context_var

    def render(self, context):
        try:
            self.snippet_id = template.resolve_variable(self.snippet_id, context)
        except template.VariableDoesNotExist:
            return ''
        context[self.context_var] = Rating.objects.score_for_snippet(self.snippet_id)
        return ''

class RatingByUserNode(template.Node):
    def __init__(self, user_id, snippet_id, context_var):
        self.user_id, self.snippet_id = user_id, snippet_id
        self.context_var = context_var

    def render(self, context):
        try:
            self.user_id = template.resolve_variable(self.user_id, context)
            self.snippet_id = template.resolve_variable(self.snippet_id, context)
            rating = Rating.objects.get(user__pk=self.user_id, snippet__pk=self.snippet_id)
        except template.VariableDoesNotExist:
            return ''
        except Rating.DoesNotExist:
            return ''
        context[self.context_var] = rating
        return ''

def do_if_bookmarked(parser, token):
    """
    Returns either of two blocks of content depending on whether a
    given User has bookmarked a given Snippet.

    Example::
        {% if_already_bookmarked user.id object.id %}

This Snippet is in your bookmarks.

        {% else %}

You haven't bookmarked this Snippet.

        {% endif_already_bookmarked %}

    """
    bits = token.contents.split()
    if len(bits) != 3:
        raise template.TemplateSyntaxError("'%s' tag takes exactly two arguments" % bits[0])
    nodelist_true = parser.parse(('else', 'endif_bookmarked'))
    token = parser.next_token()
    if token.contents == 'else':
        nodelist_false = parser.parse(('endif_bookmarked',))
        parser.delete_first_token()
    else:
        nodelist_false = template.NodeList()
    return IfBookmarkedNode(bits[1], bits[2], nodelist_true, nodelist_false)

def do_if_rated(parser, token):
    """
    Returns either of two blocks of content depending on whether a
    given User has rated a given Snippet.

    Example::
        {% if_rated user.id object.id %}

You have already rated this Snippet.

        {% else %}

You haven't rated this Snippet yet.

        {% endif_rated %}

    """
    bits = token.contents.split()
    if len(bits) != 3:
        raise template.TemplateSyntaxError("'%s' tag takes exactly two arguments" % bits[0])
    nodelist_true = parser.parse(('else', 'endif_rated'))
    token = parser.next_token()
    if token.contents == 'else':
        nodelist_false = parser.parse(('endif_rated',))
        parser.delete_first_token()
    else:
        nodelist_false = template.NodeList()
    return IfRatedNode(bits[1], bits[2], nodelist_true, nodelist_false)

def do_rating_for_snippet(parser, token):
    """
    Retrieves a list containing the total score for a Snippet and the
    number of Ratings it's received, and stores them in a specified
    context variable.

    Example usage::
        {% get_rating_for_snippet object.id as score %}

    """
    bits = token.contents.split()
    if len(bits) != 4:
        raise template.TemplateSyntaxError("'%s' tag takes exactly three arguments" % bits[0])
    if bits[2] != 'as':
        raise template.TemplateSyntaxError("second argument to '%s' tag must be 'as'" % bits[0])
    return RatingForSnippetNode(bits[1], bits[3])

def do_rating_by_user(parser, token):
    """
    Returns a User's Rating of a Snippet, if any.

    Example::
        {% get_rating_by_user user.id object.id as rating %}

    """
    bits = token.contents.split()
    if len(bits) != 5:
        raise template.TemplateSyntaxError("'%s' tag takes exactly four arguments" % bits[0])
    if bits[3] != 'as':
        raise template.TemplateSyntaxError("third argument to '%s' tag must be 'as'" % bits[0])
    return RatingByUserNode(bits[1], bits[2], bits[4])

register.tag('get_rating_for_snippet', do_rating_for_snippet)
register.tag('if_bookmarked', do_if_bookmarked)
register.tag('if_rated', do_if_rated)
register.tag('get_rating_by_user', do_rating_by_user)
<commit_msg>Add 'get_latest_snippets' template tag<commit_after>"""
General-purpose tags for working with various aspects of Snippets --
whether a user has bookmarked/rated a given Snippet, etc.

"""

from django import template
from cab.models import Bookmark, Rating

register = template.Library()

class IfBookmarkedNode(template.Node):
    def __init__(self, user_id, snippet_id, nodelist_true, nodelist_false):
        self.user_id = user_id
        self.snippet_id = snippet_id
        self.nodelist_true, self.nodelist_false = nodelist_true, nodelist_false

    def render(self, context):
        try:
            self.user_id = template.resolve_variable(self.user_id, context)
            self.snippet_id = template.resolve_variable(self.snippet_id, context)
        except template.VariableDoesNotExist:
            return ''
        if Bookmark.objects.already_bookmarked(self.user_id, self.snippet_id):
            return self.nodelist_true.render(context)
        return self.nodelist_false.render(context)

class IfRatedNode(template.Node):
    def __init__(self, user_id, snippet_id, nodelist_true, nodelist_false):
        self.user_id = user_id
        self.snippet_id = snippet_id
        self.nodelist_true, self.nodelist_false = nodelist_true, nodelist_false

    def render(self, context):
        try:
            self.user_id = template.resolve_variable(self.user_id, context)
            self.snippet_id = template.resolve_variable(self.snippet_id, context)
        except template.VariableDoesNotExist:
            return ''
        if Rating.objects.already_rated(self.user_id, self.snippet_id):
            return self.nodelist_true.render(context)
        return self.nodelist_false.render(context)

class LatestSnippetsnode(template.Node):
    def __init__(self, num, varname):
        self.num, self.varname = num, varname

    def render(self, context):
        context[self.varname] = Snippet.objects.all().select_related()[:self.num]
        return ''

class RatingForSnippetNode(template.Node):
    def __init__(self, snippet_id, context_var):
        self.snippet_id = snippet_id
        self.context_var = context_var

    def render(self, context):
        try:
            self.snippet_id = template.resolve_variable(self.snippet_id, context)
        except template.VariableDoesNotExist:
            return ''
        context[self.context_var] = Rating.objects.score_for_snippet(self.snippet_id)
        return ''

class RatingByUserNode(template.Node):
    def __init__(self, user_id, snippet_id, context_var):
        self.user_id, self.snippet_id = user_id, snippet_id
        self.context_var = context_var

    def render(self, context):
        try:
            self.user_id = template.resolve_variable(self.user_id, context)
            self.snippet_id = template.resolve_variable(self.snippet_id, context)
            rating = Rating.objects.get(user__pk=self.user_id, snippet__pk=self.snippet_id)
        except template.VariableDoesNotExist:
            return ''
        except Rating.DoesNotExist:
            return ''
        context[self.context_var] = rating
        return ''

def do_if_bookmarked(parser, token):
    """
    Returns either of two blocks of content depending on whether a
    given User has bookmarked a given Snippet.

    Example::
        {% if_already_bookmarked user.id object.id %}

This Snippet is in your bookmarks.

        {% else %}

You haven't bookmarked this Snippet.

        {% endif_already_bookmarked %}

    """
    bits = token.contents.split()
    if len(bits) != 3:
        raise template.TemplateSyntaxError("'%s' tag takes exactly two arguments" % bits[0])
    nodelist_true = parser.parse(('else', 'endif_bookmarked'))
    token = parser.next_token()
    if token.contents == 'else':
        nodelist_false = parser.parse(('endif_bookmarked',))
        parser.delete_first_token()
    else:
        nodelist_false = template.NodeList()
    return IfBookmarkedNode(bits[1], bits[2], nodelist_true, nodelist_false)

def do_if_rated(parser, token):
    """
    Returns either of two blocks of content depending on whether a
    given User has rated a given Snippet.

    Example::
        {% if_rated user.id object.id %}

You have already rated this Snippet.

        {% else %}

You haven't rated this Snippet yet.

        {% endif_rated %}

    """
    bits = token.contents.split()
    if len(bits) != 3:
        raise template.TemplateSyntaxError("'%s' tag takes exactly two arguments" % bits[0])
    nodelist_true = parser.parse(('else', 'endif_rated'))
    token = parser.next_token()
    if token.contents == 'else':
        nodelist_false = parser.parse(('endif_rated',))
        parser.delete_first_token()
    else:
        nodelist_false = template.NodeList()
    return IfRatedNode(bits[1], bits[2], nodelist_true, nodelist_false)

def do_latest_snippets(parser, token):
    """
    Returns the latest ``num`` Snippets.

    Example::
        {% get_latest_snippets 5 as latest_snippets %}

    """
    bits = token.contents.split()
    if len(bits) != 4:
        raise template.TemplateSyntaxError("'%s' tag takes exactly three arguments" % bits[0])
    return LatestSnippetsNode(bits[1], bits[3])

def do_rating_for_snippet(parser, token):
    """
    Retrieves a list containing the total score for a Snippet and the
    number of Ratings it's received, and stores them in a specified
    context variable.

    Example usage::
        {% get_rating_for_snippet object.id as score %}

    """
    bits = token.contents.split()
    if len(bits) != 4:
        raise template.TemplateSyntaxError("'%s' tag takes exactly three arguments" % bits[0])
    if bits[2] != 'as':
        raise template.TemplateSyntaxError("second argument to '%s' tag must be 'as'" % bits[0])
    return RatingForSnippetNode(bits[1], bits[3])

def do_rating_by_user(parser, token):
    """
    Returns a User's Rating of a Snippet, if any.

    Example::
        {% get_rating_by_user user.id object.id as rating %}

    """
    bits = token.contents.split()
    if len(bits) != 5:
        raise template.TemplateSyntaxError("'%s' tag takes exactly four arguments" % bits[0])
    if bits[3] != 'as':
        raise template.TemplateSyntaxError("third argument to '%s' tag must be 'as'" % bits[0])
    return RatingByUserNode(bits[1], bits[2], bits[4])

register.tag('get_latest_snippets', do_latest_snippets)
register.tag('get_rating_for_snippet', do_rating_for_snippet)
register.tag('get_rating_by_user', do_rating_by_user)
register.tag('if_bookmarked', do_if_bookmarked)
register.tag('if_rated', do_if_rated)
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
#
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Computes the diff between two qps runs and outputs significant results """

import shutil
import subprocess
import os
import multiprocessing
import json
import sys
import tabulate
import argparse

sys.path.append(
  os.path.join(
    os.path.dirname(sys.argv[0]), '..', '..', '..', 'tools', 'profiling', 'microbenchmarks', 'bm_diff'))
import bm_speedup

sys.path.append(
  os.path.join(
    os.path.dirname(sys.argv[0]), '..', '..', '..', 'tools', 'run_tests', 'python_utils'))
import comment_on_pr

_SCENARIOS = {
  'large-message-throughput': '{"scenarios":[{"name":"large-message-throughput", "spawn_local_worker_count": -2, "warmup_seconds": 30, "benchmark_seconds": 270, "num_servers": 1, "server_config": {"async_server_threads": 1, "security_params": null, "server_type": "ASYNC_SERVER"}, "num_clients": 1, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "payload_config": {"simple_params": {"resp_size": 1048576, "req_size": 1048576}}, "client_channels": 1, "async_client_threads": 1, "outstanding_rpcs_per_channel": 1, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}}}]}',
  'multi-channel-64-KiB': '{"scenarios":[{"name":"multi-channel-64-KiB", "spawn_local_worker_count": -3, "warmup_seconds": 30, "benchmark_seconds": 270, "num_servers": 1, "server_config": {"async_server_threads": 31, "security_params": null, "server_type": "ASYNC_SERVER"}, "num_clients": 2, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "payload_config": {"simple_params": {"resp_size": 65536, "req_size": 65536}}, "client_channels": 32, "async_client_threads": 31, "outstanding_rpcs_per_channel": 100, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}}}]}'
}

def _args():
  argp = argparse.ArgumentParser(
    description='Perform diff on QPS Driver')
  argp.add_argument(
    '-d',
    '--diff_base',
    type=str,
    help='Commit or branch to compare the current one to')
  argp.add_argument(
    '-l',
    '--loops',
    type=int,
    default=6,
    help='Number of times to loops the benchmarks. More loops cuts down on noise'
  )
  argp.add_argument(
    '-j',
    '--jobs',
    type=int,
    default=multiprocessing.cpu_count(),
    help='Number of CPUs to use')
  args = argp.parse_args()
  assert args.diff_base, "diff_base must be set"
  return args

def _make_cmd(jobs):
  return ['make', '-j', '%d' % jobs, 'qps_json_driver', 'qps_worker',]

def build(name, jobs):
  shutil.rmtree('qps_diff_%s' % name, ignore_errors=True)
  subprocess.check_call(['git', 'submodule', 'update'])
  try:
    subprocess.check_call(_make_cmd(jobs))
  except subprocess.CalledProcessError, e:
    subprocess.check_call(['make', 'clean'])
    subprocess.check_call(_make_cmd(jobs))
  os.rename('bins', 'qps_diff_%s' % name)

def _run_cmd(name, scenario, fname):
  return ['qps_diff_%s/opt/qps_json_driver' % name, '--scenarios_json', scenario, '--json_file_out', fname]

def run(name, scenarios, loops):
  for sn in scenarios:
    for i in range(0, loops):
      fname = "%s.%s.%d.json" % (sn, name, i)
      subprocess.check_call(_run_cmd(name, scenarios[sn], fname))

def _load_qps(fname):
  try:
    with open(fname) as f:
      return json.loads(f.read())['qps']
  except IOError, e:
    print("IOError occurred reading file: %s" % fname)
    return None
  except ValueError, e:
    print("ValueError occurred reading file: %s" % fname)
    return None

def _median(ary):
  assert (len(ary))
  ary = sorted(ary)
  n = len(ary)
  if n % 2 == 0:
    return (ary[(n - 1) / 2] + ary[(n - 1) / 2 + 1]) / 2.0
  else:
    return ary[n / 2]

def diff(scenarios, loops, old, new):
  old_data = {}
  new_data = {}

  # collect data
  for sn in scenarios:
    old_data[sn] = []
    new_data[sn] = []
    for i in range(0, loops):
      old_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, old, i)))
      new_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, new, i)))

  # crunch data
  headers = ['Benchmark', 'qps']
  rows = []
  for sn in scenarios:
    mdn_diff = abs(_median(new_data[sn]) - _median(old_data[sn]))
    print('%s: %s=%r %s=%r mdn_diff=%r' % (sn, new, new_data[sn], old, old_data[sn], mdn_diff))
    s = bm_speedup.speedup(new_data[sn], old_data[sn])
    if abs(s) > 3 and mdn_diff > 0.5:
      rows.append([sn, '%+d%%' % s])

  if rows:
    return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f')
  else:
    return None

def main(args):

  build('new', args.jobs)

  if args.diff_base:
    where_am_i = subprocess.check_output(
      ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
    subprocess.check_call(['git', 'checkout', args.diff_base])
    try:
      build('old', args.jobs)
    finally:
      subprocess.check_call(['git', 'checkout', where_am_i])
      subprocess.check_call(['git', 'submodule', 'update'])

  run('new', _SCENARIOS, args.loops)
  run('old', _SCENARIOS, args.loops)

  diff = diff(_SCENARIOS, args.loops, 'old', 'new')

  if diff:
    text = '[qps] Performance differences noted:\n%s' % diff
  else:
    text = '[qps] No significant performance differences'
  print('%s' % text)
  comment_on_pr.comment_on_pr('```\n%s\n```' % text)

if __name__ == '__main__':
  args = _args()
  main(args);
<commit_msg>relax QPS diff threshold<commit_after>#!/usr/bin/env python2.7
#
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Computes the diff between two qps runs and outputs significant results """

import shutil
import subprocess
import os
import multiprocessing
import json
import sys
import tabulate
import argparse

sys.path.append(
  os.path.join(
    os.path.dirname(sys.argv[0]), '..', '..', '..', 'tools', 'profiling', 'microbenchmarks', 'bm_diff'))
import bm_speedup

sys.path.append(
  os.path.join(
    os.path.dirname(sys.argv[0]), '..', '..', '..', 'tools', 'run_tests', 'python_utils'))
import comment_on_pr

_SCENARIOS = {
  'large-message-throughput': '{"scenarios":[{"name":"large-message-throughput", "spawn_local_worker_count": -2, "warmup_seconds": 30, "benchmark_seconds": 270, "num_servers": 1, "server_config": {"async_server_threads": 1, "security_params": null, "server_type": "ASYNC_SERVER"}, "num_clients": 1, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "payload_config": {"simple_params": {"resp_size": 1048576, "req_size": 1048576}}, "client_channels": 1, "async_client_threads": 1, "outstanding_rpcs_per_channel": 1, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}}}]}',
  'multi-channel-64-KiB': '{"scenarios":[{"name":"multi-channel-64-KiB", "spawn_local_worker_count": -3, "warmup_seconds": 30, "benchmark_seconds": 270, "num_servers": 1, "server_config": {"async_server_threads": 31, "security_params": null, "server_type": "ASYNC_SERVER"}, "num_clients": 2, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "payload_config": {"simple_params": {"resp_size": 65536, "req_size": 65536}}, "client_channels": 32, "async_client_threads": 31, "outstanding_rpcs_per_channel": 100, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}}}]}'
}

def _args():
  argp = argparse.ArgumentParser(
    description='Perform diff on QPS Driver')
  argp.add_argument(
    '-d',
    '--diff_base',
    type=str,
    help='Commit or branch to compare the current one to')
  argp.add_argument(
    '-l',
    '--loops',
    type=int,
    default=6,
    help='Number of times to loops the benchmarks. More loops cuts down on noise'
  )
  argp.add_argument(
    '-j',
    '--jobs',
    type=int,
    default=multiprocessing.cpu_count(),
    help='Number of CPUs to use')
  args = argp.parse_args()
  assert args.diff_base, "diff_base must be set"
  return args

def _make_cmd(jobs):
  return ['make', '-j', '%d' % jobs, 'qps_json_driver', 'qps_worker',]

def build(name, jobs):
  shutil.rmtree('qps_diff_%s' % name, ignore_errors=True)
  subprocess.check_call(['git', 'submodule', 'update'])
  try:
    subprocess.check_call(_make_cmd(jobs))
  except subprocess.CalledProcessError, e:
    subprocess.check_call(['make', 'clean'])
    subprocess.check_call(_make_cmd(jobs))
  os.rename('bins', 'qps_diff_%s' % name)

def _run_cmd(name, scenario, fname):
  return ['qps_diff_%s/opt/qps_json_driver' % name, '--scenarios_json', scenario, '--json_file_out', fname]

def run(name, scenarios, loops):
  for sn in scenarios:
    for i in range(0, loops):
      fname = "%s.%s.%d.json" % (sn, name, i)
      subprocess.check_call(_run_cmd(name, scenarios[sn], fname))

def _load_qps(fname):
  try:
    with open(fname) as f:
      return json.loads(f.read())['qps']
  except IOError, e:
    print("IOError occurred reading file: %s" % fname)
    return None
  except ValueError, e:
    print("ValueError occurred reading file: %s" % fname)
    return None

def _median(ary):
  assert (len(ary))
  ary = sorted(ary)
  n = len(ary)
  if n % 2 == 0:
    return (ary[(n - 1) / 2] + ary[(n - 1) / 2 + 1]) / 2.0
  else:
    return ary[n / 2]

def diff(scenarios, loops, old, new):
  old_data = {}
  new_data = {}

  # collect data
  for sn in scenarios:
    old_data[sn] = []
    new_data[sn] = []
    for i in range(0, loops):
      old_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, old, i)))
      new_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, new, i)))

  # crunch data
  headers = ['Benchmark', 'qps']
  rows = []
  for sn in scenarios:
    mdn_diff = abs(_median(new_data[sn]) - _median(old_data[sn]))
    print('%s: %s=%r %s=%r mdn_diff=%r' % (sn, new, new_data[sn], old, old_data[sn], mdn_diff))
    s = bm_speedup.speedup(new_data[sn], old_data[sn], 10e-5)
    if abs(s) > 3 and mdn_diff > 0.5:
      rows.append([sn, '%+d%%' % s])

  if rows:
    return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f')
  else:
    return None

def main(args):

  build('new', args.jobs)

  if args.diff_base:
    where_am_i = subprocess.check_output(
      ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
    subprocess.check_call(['git', 'checkout', args.diff_base])
    try:
      build('old', args.jobs)
    finally:
      subprocess.check_call(['git', 'checkout', where_am_i])
      subprocess.check_call(['git', 'submodule', 'update'])

  run('new', _SCENARIOS, args.loops)
  run('old', _SCENARIOS, args.loops)

  diff = diff(_SCENARIOS, args.loops, 'old', 'new')

  if diff:
    text = '[qps] Performance differences noted:\n%s' % diff
  else:
    text = '[qps] No significant performance differences'
  print('%s' % text)
  comment_on_pr.comment_on_pr('```\n%s\n```' % text)

if __name__ == '__main__':
  args = _args()
  main(args);
<|endoftext|>


<commit_before># Copyright 2013-2014 Sebastian Kreft
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import io
import os
import shutil
import subprocess
import tempfile
import unittest

import gitlint

# pylint: disable=too-many-public-methods

class TestGitE2E(unittest.TestCase):
    @staticmethod
    def init_repo():
        """Initializes a git repo."""
        subprocess.check_output(['git', 'init'], stderr=subprocess.STDOUT)

    @staticmethod
    def commit(message):
        """Commit a changeset to the repo.

        The option --no-verify is used as a pre-commit check could be globally
        installed.
        """
        subprocess.check_output(
            ['git', 'commit', '-m', message, '--no-verify'],
            stderr=subprocess.STDOUT)

    @staticmethod
    def add(filename):
        """Add a file to the repo."""
        subprocess.check_output(['git', 'add', filename],
                                stderr=subprocess.STDOUT)

    @staticmethod
    def lint():
        """Returns the response and ouput of git-lint."""
        out = io.StringIO()
        response = gitlint.main([], stdout=out, stderr=out)

        return response, out.getvalue()

    @classmethod
    def setUpClass(cls):
        cls.original_cwd = os.getcwd()
        cls.temp_directory = tempfile.mkdtemp(prefix='gitlint')
        os.chdir(cls.temp_directory)
        cls.init_repo()

    @classmethod
    def tearDownClass(cls):
        shutil.rmtree(cls.temp_directory, True)
        os.chdir(cls.original_cwd)

    def setUp(self):
        self.out = io.StringIO()

    @classmethod
    def add_linter_e2echeck(cls, linter_name, extension):
        """Adds a test for the given linter and extension."""
        def test_linter(self):
            self.assert_linter_works(linter_name, extension)
        test_linter.__name__ = 'test_linter_%s_with_%s' % (linter_name,
                                                           extension[1:])
        setattr(cls, test_linter.__name__, test_linter)

    def test_extension_not_defined(self):
        extension = '.areallyfakeextension'
        filename = os.path.join(self.temp_directory, 'filename' + extension)
        with open(filename, 'w') as f:
            f.write('Foo')
        self.add(filename)
        response, output = self.lint()
        if response != 0:
            self.fail(output)

        # Python3 does not like mixing bytes and strings. So we need to convert
        # the first element to unicode first.
        self.assertIn(os.path.relpath(filename).encode('utf-8'), output)
        self.assertIn('SKIPPED'.encode('utf-8'), output)
        self.assertIn(extension.encode('utf-8'), output)

    # TODO(skreft): check that the first file has more than 1 error, check that
    # the second file has 1 new error, check also the lines that changed.
    def assert_linter_works(self, linter_name, extension):
        """Checks that the given linter works well for all the extensions.

        It requires that 3 files are defined:
        - <linter>/original.<extension>: A file with errors
        - <linter>/error.<extension>: New errors are introduced.
        - <linter>/nonewerror.<extension>: A line was modified/added from the
          last file, but no new errors are introduced.
        """
        data_dirname = os.path.join(
            os.path.dirname(os.path.realpath(__file__)), 'data')
        filename_repo = os.path.join(
            self.temp_directory, '%s%s' % (linter_name, extension))
        filename_original = os.path.join(
            data_dirname, linter_name, 'original%s' % extension)
        filename_error = os.path.join(
            data_dirname, linter_name, 'error%s' % extension)
        filename_nonewerror = os.path.join(
            data_dirname, linter_name, 'nonewerror%s' % extension)

        self.assertTrue(
            os.path.exists(filename_original),
            'You must define file "%s"' % filename_original)
        self.assertTrue(
            os.path.exists(filename_error),
            'You must define file "%s"' % filename_error)
        self.assertTrue(os.path.exists(
            filename_nonewerror),
            'You must define file "%s"' % filename_nonewerror)

        # Add file 1 (original) to repo
        shutil.copy(filename_original, filename_repo)
        self.add(filename_repo)
        self.commit('Commit 1')

        # Add file 2 (error) to repo
        shutil.copy(filename_error, filename_repo)
        response, output = self.lint()
        if response == 0:
            self.fail(('Git lint for file %s should have failed.\n' +
                       'Output:\n%s') % (filename_error, output))
        self.add(filename_repo)
        self.commit('Commit 2')

        # Add file 3 (nonewerror) to repo
        shutil.copy(filename_nonewerror, filename_repo)
        response, output = self.lint()
        if response != 0:
            self.fail(('Git lint for file %s should have not failed. \n' +
                       'Output:\n%s') % (filename_nonewerror, output))
        self.add(filename_repo)
        self.commit('Commit 3')

def populate_linter_checks():
    """Add a test for each defined linter and extension."""
    for extension, linter_list in gitlint.get_config(None).items():
        for linter in linter_list:
            TestGitE2E.add_linter_e2echeck(linter.args[0], extension)

populate_linter_checks()

class TestHgE2E(TestGitE2E):
    @staticmethod
    def init_repo():
        """Initializes a mercurial repo."""
        subprocess.check_output(['hg', 'init'], stderr=subprocess.STDOUT)

    @staticmethod
    def commit(message):
        """Commit a changeset to the repo.

        The environment variable NO_VERIFY=1 is required as a git-lint could be
        installed as pre-commit hook.
        """
        # NO_VERIFY=1 is required as a pre-commit hook could be installed.
        environ = dict(os.environ)
        environ['NO_VERIFY'] = '1'
        subprocess.check_output(
            ['hg', 'commit', '-m', message],
            stderr=subprocess.STDOUT,
            env=environ)

    @staticmethod
    def add(filename):
        """Add a file to the repo."""
        subprocess.check_output(['hg', 'add', filename],
                                stderr=subprocess.STDOUT)
<commit_msg>Replaced fail() with assert*()<commit_after># Copyright 2013-2014 Sebastian Kreft
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import io
import os
import shutil
import subprocess
import tempfile
import unittest

import gitlint

# pylint: disable=too-many-public-methods

class TestGitE2E(unittest.TestCase):
    @staticmethod
    def init_repo():
        """Initializes a git repo."""
        subprocess.check_output(['git', 'init'], stderr=subprocess.STDOUT)

    @staticmethod
    def commit(message):
        """Commit a changeset to the repo.

        The option --no-verify is used as a pre-commit check could be globally
        installed.
        """
        subprocess.check_output(
            ['git', 'commit', '-m', message, '--no-verify'],
            stderr=subprocess.STDOUT)

    @staticmethod
    def add(filename):
        """Add a file to the repo."""
        subprocess.check_output(['git', 'add', filename],
                                stderr=subprocess.STDOUT)

    @staticmethod
    def lint():
        """Returns the response and ouput of git-lint."""
        out = io.StringIO()
        response = gitlint.main([], stdout=out, stderr=out)

        return response, out.getvalue()

    @classmethod
    def setUpClass(cls):
        cls.original_cwd = os.getcwd()
        cls.temp_directory = tempfile.mkdtemp(prefix='gitlint')
        os.chdir(cls.temp_directory)
        cls.init_repo()

    @classmethod
    def tearDownClass(cls):
        shutil.rmtree(cls.temp_directory, True)
        os.chdir(cls.original_cwd)

    def setUp(self):
        self.out = io.StringIO()

    @classmethod
    def add_linter_e2echeck(cls, linter_name, extension):
        """Adds a test for the given linter and extension."""
        def test_linter(self):
            self.assert_linter_works(linter_name, extension)
        test_linter.__name__ = 'test_linter_%s_with_%s' % (linter_name,
                                                           extension[1:])
        setattr(cls, test_linter.__name__, test_linter)

    def test_extension_not_defined(self):
        extension = '.areallyfakeextension'
        filename = os.path.join(self.temp_directory, 'filename' + extension)
        with open(filename, 'w') as f:
            f.write('Foo')
        self.add(filename)
        response, output = self.lint()
        self.assertEquals(
            0, response, 'Response %s != 0.\nOutput:\n%s' % (response, output))

        # Python3 does not like mixing bytes and strings. So we need to convert
        # the first element to unicode first.
        self.assertIn(os.path.relpath(filename).encode('utf-8'), output)
        self.assertIn('SKIPPED'.encode('utf-8'), output)
        self.assertIn(extension.encode('utf-8'), output)

    # TODO(skreft): check that the first file has more than 1 error, check that
    # the second file has 1 new error, check also the lines that changed.
    def assert_linter_works(self, linter_name, extension):
        """Checks that the given linter works well for all the extensions.

        It requires that 3 files are defined:
        - <linter>/original.<extension>: A file with errors
        - <linter>/error.<extension>: New errors are introduced.
        - <linter>/nonewerror.<extension>: A line was modified/added from the
          last file, but no new errors are introduced.
        """
        data_dirname = os.path.join(
            os.path.dirname(os.path.realpath(__file__)), 'data')
        filename_repo = os.path.join(
            self.temp_directory, '%s%s' % (linter_name, extension))
        filename_original = os.path.join(
            data_dirname, linter_name, 'original%s' % extension)
        filename_error = os.path.join(
            data_dirname, linter_name, 'error%s' % extension)
        filename_nonewerror = os.path.join(
            data_dirname, linter_name, 'nonewerror%s' % extension)

        self.assertTrue(
            os.path.exists(filename_original),
            'You must define file "%s"' % filename_original)
        self.assertTrue(
            os.path.exists(filename_error),
            'You must define file "%s"' % filename_error)
        self.assertTrue(os.path.exists(
            filename_nonewerror),
            'You must define file "%s"' % filename_nonewerror)

        # Add file 1 (original) to repo
        shutil.copy(filename_original, filename_repo)
        self.add(filename_repo)
        self.commit('Commit 1')

        # Add file 2 (error) to repo
        shutil.copy(filename_error, filename_repo)
        response, output = self.lint()
        self.assertNotEquals(
            0, response,
            ('Git lint for file %s should have failed.\n Output:\n%s') %
            (filename_error, output))
        self.add(filename_repo)
        self.commit('Commit 2')

        # Add file 3 (nonewerror) to repo
        shutil.copy(filename_nonewerror, filename_repo)
        response, output = self.lint()
        self.assertEquals(
            0, response,
            ('Git lint for file %s should have not failed. \nOutput:\n%s') %
            (filename_nonewerror, output))
        self.add(filename_repo)
        self.commit('Commit 3')

def populate_linter_checks():
    """Add a test for each defined linter and extension."""
    for extension, linter_list in gitlint.get_config(None).items():
        for linter in linter_list:
            TestGitE2E.add_linter_e2echeck(linter.args[0], extension)

populate_linter_checks()

class TestHgE2E(TestGitE2E):
    @staticmethod
    def init_repo():
        """Initializes a mercurial repo."""
        subprocess.check_output(['hg', 'init'], stderr=subprocess.STDOUT)

    @staticmethod
    def commit(message):
        """Commit a changeset to the repo.

        The environment variable NO_VERIFY=1 is required as a git-lint could be
        installed as pre-commit hook.
        """
        # NO_VERIFY=1 is required as a pre-commit hook could be installed.
        environ = dict(os.environ)
        environ['NO_VERIFY'] = '1'
        subprocess.check_output(
            ['hg', 'commit', '-m', message],
            stderr=subprocess.STDOUT,
            env=environ)

    @staticmethod
    def add(filename):
        """Add a file to the repo."""
        subprocess.check_output(['hg', 'add', filename],
                                stderr=subprocess.STDOUT)
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# Copyright (c) 2012 Dave Pifke.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#

"""SHA256 (FIPS 180-3) implementation for experimentation."""

import binascii
import collections
import struct

class SHA256(object):
    """
    SHA256 (FIPS 180-3) implementation for experimentation.

    This is an implementation of the hash function designed not for
    efficiency, but for clarity and ability to experiment.  The details
    of the algorithm are abstracted out with subclassing in mind.

    """

    # Container for the state registers between rounds:
    State = collections.namedtuple('State', 'a b c d e f g h')

    # From FIPS 180-3 section 5.3.3 (page 15):
    INITIAL_STATE = State(
        0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a,
        0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19
    )

    # From FIPS 180-3 section 4.2.2 (page 11):
    K = (
        0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
        0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
        0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
        0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
        0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
        0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
        0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
        0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
        0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
        0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
        0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
        0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
        0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
        0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
        0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
        0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
    )

    # Abstract bitwise operations, which can be overridden to provide tracing
    # or alternate implementations:
    @staticmethod
    def _sum_mod32(*args):
        return sum(args) & 0xffffffff
    @classmethod
    def _xor(cls, *args):
        if len(args) == 2:
            return args[0] ^ args[1]
        else:
            return args[0] ^ cls._xor(*args[1:])
    _and = staticmethod(lambda x, y: x & y)
    _invert = staticmethod(lambda x: ~x)

    # Operations defined by FIPS 180-3 section 3.2 (page 8):
    _rrot = staticmethod(lambda x, n: ((x & 0xffffffff) >> n) | (x << (32 - n)) & 0xffffffff)
    _shr = staticmethod(lambda x, n: (x & 0xffffffff) >> n)

    # Operations defined by FIPS 180-3 section 4.1.2 (page 10):
    _ch = classmethod(lambda cls, x, y, z: cls._xor(cls._and(x, y), cls._and(cls._invert(x), z)))
    _maj = classmethod(lambda cls, x, y, z: cls._xor(cls._and(x, y), cls._and(x, z), cls._and(y, z)))
    _S0 = classmethod(lambda cls, x: cls._xor(cls._rrot(x, 2), cls._rrot(x, 13), cls._rrot(x, 22)))
    _S1 = classmethod(lambda cls, x: cls._xor(cls._rrot(x, 6), cls._rrot(x, 11), cls._rrot(x, 25)))
    _s0 = classmethod(lambda cls, x: cls._xor(cls._rrot(x, 7), cls._rrot(x, 18), cls._shr(x, 3)))
    _s1 = classmethod(lambda cls, x: cls._xor(cls._rrot(x, 17), cls._rrot(x, 19), cls._shr(x, 10)))

    # Operations defined by FIPS 180-3 section 6.2.2 (page 22):
    _T1 = classmethod(lambda cls, prev, w, k: cls._sum_mod32(cls._S1(prev.e), cls._ch(prev.e, prev.f, prev.g), prev.h, w, k))
    _T2 = classmethod(lambda cls, prev: cls._sum_mod32(cls._S0(prev.a), cls._maj(prev.a, prev.b, prev.c)))

    @classmethod
    def _round(cls, number, w, prev=INITIAL_STATE):
        """
        Performs one round of SHA256 message transformation, returning the new
        message state.  See FIPS 180-3 section 6.2.2 (page 21).

        :param number:
            The round number.

        :param w:
            The expanded word of the input for this round.

        :param prev:
            Named tuple containing the working state from the previous round.

        """

        t1 = cls._T1(prev, w, cls.K[number % 64])
        return cls.State(
            a=cls._sum_mod32(t1, cls._T2(prev)),
            b=prev.a,
            c=prev.b,
            d=prev.c,
            e=cls._sum_mod32(prev.d, t1),
            f=prev.e,
            g=prev.f,
            h=prev.g
        )

    @classmethod
    def _finalize(cls, state):
        """
        Returns the new state after the final internal round is complete.
        """

        return cls.State(
            a=cls._sum_mod32(state.a, cls.INITIAL_STATE.a),
            b=cls._sum_mod32(state.b, cls.INITIAL_STATE.b),
            c=cls._sum_mod32(state.c, cls.INITIAL_STATE.c),
            d=cls._sum_mod32(state.d, cls.INITIAL_STATE.d),
            e=cls._sum_mod32(state.e, cls.INITIAL_STATE.e),
            f=cls._sum_mod32(state.f, cls.INITIAL_STATE.f),
            g=cls._sum_mod32(state.g, cls.INITIAL_STATE.g),
            h=cls._sum_mod32(state.h, cls.INITIAL_STATE.h)
        )

    @classmethod
    def _expand_message(cls, message):
        """
        Returns a list of 64 32-bit words based upon 16 32-bit words from the
        message block being hashed.

        :param message:
            Array of 16 32-bit values (512 bits total).

        """

        assert len(message) == 16, '_expand_message() got %d words, expected 16' % len(message)

        w = list(message)
        for i in range(16, 64):
            w.append(cls._sum_mod32(w[i - 16], cls._s0(w[i - 15]), w[i - 7], cls._s1(w[i - 2])))
        return w

    @classmethod
    def _process_block(cls, message, state=INITIAL_STATE):
        """
        Processes a block of message data, returning the new digest state.

        :param message:
            Byte string of length 64 containing the block data to hash.
        """

        assert len(message) == 64, '_process_block() got %d bytes, expected 64' % len(message)

        w = cls._expand_message(struct.unpack('>LLLLLLLLLLLLLLLL', message))
        for i in range(64):
            state = cls._round(i, w[i], state)
        return cls._finalize(state)

    @classmethod
    def _pad_message(cls, message, length):
        """
        Returns a list containing the final 1 or 2 message blocks, which
        include the message padding per FIPS 180-3 section 5.1.1 (page 13).

        :param message:
            Byte string containing the final block data to hash.  Should be
            less than a full block's worth (63 bytes or less).

        :param length:
            Length of the message, in bits.

        """

        assert len(message) < 64, 'Input to _pad_message() must be less than 512 bits'

        if len(message) <= 55:
            # Append trailing 1 bit, then padding, then length
            return [b''.join((
                message,
                b'\x80',
                b'\x00' * (55 - len(message)),
                struct.pack('>LL', length >> 32, length & 0xffffffff),
            ))]
        else:
            # Not enough room to append length, return two blocks:
            return [
                # First is trailing 1 bit, then padding
                b''.join((
                    message,
                    b'\x80',
                    b'\x00' * (63 - len(message)),
                )),
                # Next is more padding, then length
                ''.join((
                    b'\x00' * 56,
                    struct.pack('>LL', length >> 32, length & 0xffffffff),
                )),
            ]

    def __init__(self, message=b''):
        """
        Constructor.  Optional argument is the initial message to be hashed.
        """

        self.state = self.INITIAL_STATE
        self.length = 0
        self.buffer = b''

        self.update(message)

    def update(self, message):
        """
        Updates the hash with the contents of *message*.

        Hashing uses 512-bit blocks, so the message is buffered until there's
        enough data to process a complete block.  When digest() is called,
        any remaining data in the buffer will be padded and digested.
        """

        message = bytes(message)
        self.length += len(message) * 8
        self.buffer = b''.join((self.buffer, message))

        while len(self.buffer) >= 64:
            self.state = self._process_block(self.buffer[:64], self.state)
            self.buffer[:64] = []

    def digest(self):
        """
        Returns the SHA256 digest of the message.

        The hash is based on all data passed thus far via the constructor and
        update().  Any buffered data will be processed (along with the
        terminating length), however the internal state is not modified.  This
        means that update() can safely be used again after digest().
        """

        final_state = self.state
        for block in self._pad_message(self.buffer, self.length):
            final_state = self._process_block(block, final_state)

        return struct.pack('>LLLLLLLL', *final_state)

    def hexdigest(self):
        """
        Like digest(), but returns a hexadecimal string.
        """

        return binascii.hexlify(self.digest())

if __name__ == '__main__':
    # Test routine.  Compares our output to that of the stdlib.

    import hashlib
    import os

    message = b''
    for i in range(2048):
        mine = SHA256(message).hexdigest()
        stdlib = hashlib.sha256(message).hexdigest()
        assert mine == stdlib, '%r (mine) != %r (stdlib) calculating SHA256(%r) (length %d)' % (mine, stdlib, message, len(message))
        message = ''.join((message, os.urandom(1)))

# eof
<commit_msg>Debug and clean up SHA256 code.<commit_after>#!/usr/bin/python
#
# Copyright (c) 2012 Dave Pifke.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#

"""SHA256 (FIPS 180-3) implementation for experimentation."""

import binascii
import codecs
import collections
import struct
import sys

if sys.version > '3':
    long = int

class SHA256(object):
    """
    SHA256 (FIPS 180-3) implementation for experimentation.

    This is an implementation of the hash function designed not for
    efficiency, but for clarity and ability to experiment.  The details
    of the algorithm are abstracted out with subclassing in mind.

    """

    # Container for the state registers between rounds:
    State = collections.namedtuple('State', 'a b c d e f g h')

    # From FIPS 180-3 section 5.3.3 (page 15):
    INITIAL_STATE = State(
        0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a,
        0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19
    )

    # From FIPS 180-3 section 4.2.2 (page 11):
    K = (
        0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
        0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
        0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
        0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
        0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
        0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
        0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
        0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
        0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
        0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
        0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
        0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
        0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
        0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
        0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
        0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
    )

    # Abstract bitwise operations, which can be overridden to provide tracing
    # or alternate implementations:
    @staticmethod
    def _sum_mod32(*args):
        return sum(args) & 0xffffffff
    @classmethod
    def _xor(cls, *args):
        if len(args) == 2:
            return args[0] ^ args[1]
        else:
            return args[0] ^ cls._xor(*args[1:])
    _and = staticmethod(lambda x, y: x & y)
    _invert = staticmethod(lambda x: ~x)

    # Operations defined by FIPS 180-3 section 3.2 (page 8):
    _rrot = staticmethod(lambda x, n: ((x & 0xffffffff) >> n) | (x << (32 - n)) & 0xffffffff)
    _shr = staticmethod(lambda x, n: (x & 0xffffffff) >> n)

    # Operations defined by FIPS 180-3 section 4.1.2 (page 10):
    _ch = classmethod(lambda cls, x, y, z: cls._xor(cls._and(x, y), cls._and(cls._invert(x), z)))
    _maj = classmethod(lambda cls, x, y, z: cls._xor(cls._and(x, y), cls._and(x, z), cls._and(y, z)))
    _S0 = classmethod(lambda cls, x: cls._xor(cls._rrot(x, 2), cls._rrot(x, 13), cls._rrot(x, 22)))
    _S1 = classmethod(lambda cls, x: cls._xor(cls._rrot(x, 6), cls._rrot(x, 11), cls._rrot(x, 25)))
    _s0 = classmethod(lambda cls, x: cls._xor(cls._rrot(x, 7), cls._rrot(x, 18), cls._shr(x, 3)))
    _s1 = classmethod(lambda cls, x: cls._xor(cls._rrot(x, 17), cls._rrot(x, 19), cls._shr(x, 10)))

    # Operations defined by FIPS 180-3 section 6.2.2 (page 22):
    _T1 = classmethod(lambda cls, prev, w, k: cls._sum_mod32(cls._S1(prev.e), cls._ch(prev.e, prev.f, prev.g), prev.h, w, k))
    _T2 = classmethod(lambda cls, prev: cls._sum_mod32(cls._S0(prev.a), cls._maj(prev.a, prev.b, prev.c)))

    @classmethod
    def _round(cls, number, w, prev=INITIAL_STATE):
        """
        Performs one round of SHA256 message transformation, returning the new
        message state.  See FIPS 180-3 section 6.2.2 step 3 (pages 21-22).

        :param number:
            The round number.

        :param w:
            The expanded word of the input for this round.

        :param prev:
            Named tuple containing the working state from the previous round.

        """

        t1 = cls._T1(prev, w, cls.K[number % 64])
        return cls.State(
            a=cls._sum_mod32(t1, cls._T2(prev)),
            b=prev.a,
            c=prev.b,
            d=prev.c,
            e=cls._sum_mod32(prev.d, t1),
            f=prev.e,
            g=prev.f,
            h=prev.g
        )

    @classmethod
    def _finalize(cls, state, initial_state=INITIAL_STATE):
        """
        Returns the intermediate state after the final round for a given block
        is complete.  See FIPS 180-3 section 6.2.2 step 4 (page 22).

        :param state:
            The digest state after the final round.

        :param initial_state:
            The digest state from before the first round.

        """

        return cls.State(
            a=cls._sum_mod32(state.a, initial_state.a),
            b=cls._sum_mod32(state.b, initial_state.b),
            c=cls._sum_mod32(state.c, initial_state.c),
            d=cls._sum_mod32(state.d, initial_state.d),
            e=cls._sum_mod32(state.e, initial_state.e),
            f=cls._sum_mod32(state.f, initial_state.f),
            g=cls._sum_mod32(state.g, initial_state.g),
            h=cls._sum_mod32(state.h, initial_state.h)
        )

    @classmethod
    def _expand_message(cls, message):
        """
        Returns a list of 64 32-bit words based upon 16 32-bit words from the
        message block being hashed.  See FIPS 180-3 section 6.2.2 step 1
        (page 21).

        :param message:
            Array of 16 32-bit values (512 bits total).

        """

        assert len(message) == 16, '_expand_message() got %d words, expected 16' % len(message)

        w = list(message)
        for i in range(16, 64):
            w.append(cls._sum_mod32(w[i - 16], cls._s0(w[i - 15]), w[i - 7], cls._s1(w[i - 2])))

        return w

    @classmethod
    def _process_block(cls, message, state=INITIAL_STATE):
        """
        Processes a block of message data, returning the new digest state
        (the intermediate hash value).  See FIPS 180-3 section 6.2.2 (pages
        21 and 22).

        :param message:
            Byte string of length 64 containing the block data to hash.

        """

        assert len(message) == 64, '_process_block() got %d bytes, expected 64' % len(message)

        w = cls._expand_message(struct.unpack('>LLLLLLLLLLLLLLLL', message))

        midstate = state
        for i in range(64):
            midstate = cls._round(i, w[i], midstate)

        return cls._finalize(midstate, state)

    @classmethod
    def _pad_message(cls, message, length):
        """
        Returns a list containing the final 1 or 2 message blocks, which
        include the message padding per FIPS 180-3 section 5.1.1 (page 13).

        :param message:
            Byte string containing the final block data to hash.  Should be
            less than a full block's worth (63 bytes or less).

        :param length:
            Length of the message, in bits.

        """

        assert len(message) < 64, 'Input to _pad_message() must be less than 512 bits'

        if len(message) <= 55:
            # Append trailing 1 bit, then padding, then length
            return [b''.join((
                message,
                b'\x80',
                b'\x00' * (55 - len(message)),
                struct.pack('>LL', length >> 32, length & 0xffffffff),
            ))]

        else:
            # Not enough room to append length, return two blocks:
            return [
                # First is trailing 1 bit, then padding
                b''.join((
                    message,
                    b'\x80',
                    b'\x00' * (63 - len(message)),
                )),
                # Next is more padding, then length
                b''.join((
                    b'\x00' * 56,
                    struct.pack('>LL', length >> 32, length & 0xffffffff),
                )),
            ]

    def __init__(self, message=b''):
        """
        Constructor.  Optional argument is the initial message to be hashed.
        """

        self.state = self.INITIAL_STATE
        self.length = long(0)
        self.buffer = b''

        self.update(message)

    def update(self, message):
        """
        Updates the hash with the contents of *message*.

        Hashing uses 512-bit blocks, so the message is buffered until there's
        enough data to process a complete block.  When digest() is called,
        any remaining data in the buffer will be padded and digested.

        :param message:
            A byte string to digest.

        """

        message = bytes(message)
        self.length += len(message) * 8
        self.buffer = b''.join((self.buffer, message))

        while len(self.buffer) >= 64:
            self.state = self._process_block(self.buffer[:64], self.state)
            self.buffer = self.buffer[64:]

    def digest(self):
        """
        Returns the SHA256 digest of the message.

        The hash is based on all data passed thus far via the constructor and
        update().  Any buffered data will be processed (along with the
        terminating length), however the internal state is not modified.  This
        means that update() can safely be used again after digest().

        """

        final_state = self.state
        for block in self._pad_message(self.buffer, self.length):
            final_state = self._process_block(block, final_state)

        return struct.pack('>LLLLLLLL', *final_state)

    def hexdigest(self):
        """Like digest(), but returns a hexadecimal string."""

        return binascii.hexlify(self.digest())

if __name__ == '__main__':
    # Test routine.  Compares our output to that of the stdlib.  We also
    # print some timings, although keep in mind we're not built for speed so
    # the performance comparison is of dubious utility.

    import hashlib
    import os
    import sys
    import time

    try:
        count = int(sys.argv[1])
    except (ValueError, IndexError):
        count = 1000 # default

    mine = []
    stdlib = []
    message = os.urandom(count)

    try:
        consumed = 0
        start = time.time()
        for i in range(count):
            mine.append(SHA256(message[:i]).hexdigest())
            consumed += i
    finally:
        elapsed = time.time() - start
        print ('Mine: %d hashes (%d bytes) in %0.2f secs (%0.2f H/s %d B/s)' % (i+1, consumed, elapsed, float(i+1) / elapsed, float(consumed) / elapsed))

    try:
        consumed = 0
        start = time.time()
        for i in range(count):
            # The encoding before is to make sure we're comparing the same
            # types (bytes to bytes).  This is a Python 3 issue.
            stdlib.append(codecs.latin_1_encode(hashlib.sha256(message[:i]).hexdigest())[0])
            consumed += i
    finally:
        elapsed = time.time() - start
        print ('stdlib: %d hashes (%d bytes) in %0.2f secs (%0.2f H/s %d B/s)' % (i+1, consumed, elapsed, float(i+1) / elapsed, float(consumed) / elapsed))

    for a, b, i in zip(mine, stdlib, range(count)):
        assert a == b, '%r (mine) != %r (stdlib) calculating SHA256(%r) of length %d' % (a, b, message[:i], i)

# eof
<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
import os
import os.path
import pygments
import pygments.formatter
import pygments.lexers
import sys
import urllib.parse
import urllib.request
import validators

import xorcise

# constants

__version__ = "0.0.3"

DESCRIPTION = "{} is a tool to learn about something just copying it " \
              "by hand. Type Esc or ^[ to exit while running it." \
              .format(os.path.basename(__file__))

TTY_DEVICE_FILE = "/dev/tty" # POSIX compliant
ENCODING = "UTF-8"
CURSOR_WIDTH = 1

QUIT_CHARS = xorcise.ESCAPE_CHARS
DELETE_CHARS = xorcise.DELETE_CHARS | xorcise.BACKSPACE_CHARS
CLEAR_CHAR = xorcise.ctrl('u')
SKIP_CHAR = xorcise.ctrl('n')

CAN_SKIP = False

SUPPORTED_SCHEMES = {"http", "https", "ftp"}

# the order of bugs

def message(*text):
  print("{}:".format(os.path.basename(__file__)), *text, file=sys.stderr)

def error(*text):
  message("error:", *text)

def fail(*text):
  error(*text)
  exit(1)

# classes

class Shakyo:
  ATTR_CORRECT = xorcise.RenditionAttribute.normal
  ATTR_WRONG = xorcise.RenditionAttribute.reverse

  def __init__(self, console, example_text):
    self.__console = console
    self.__geometry = Geometry(self.__console)
    self.__input_line = xorcise.Line()
    self.__example_lines \
        = format(example_text, max_width=(self.__console.screen_width - 1))
    if len(self.__example_lines) == 0:
      raise Exception("No line can be read from example source.")

  def do(self):
    self.__print_all_example_lines()

    while len(self.__example_lines) != 0:
      self.__update_input_line()
      char = self.__console.get_char()

      if char in QUIT_CHARS:
        break
      elif char == CLEAR_CHAR:
        self.__input_line = xorcise.Line()
      elif char in DELETE_CHARS:
        self.__input_line = self.__input_line[:-1]
      elif (char == '\n' and self.__input_line.normalized
                             == self.__example_lines[0].normalized) \
           or (char == SKIP_CHAR and CAN_SKIP):
        if len(self.__example_lines) == 1: break
        self.__scroll()
      elif xorcise.is_printable_char(char) \
           and (self.__input_line + xorcise.Character(char)).width \
               + CURSOR_WIDTH <= self.__console.screen_width:
        self.__input_line += xorcise.Character(char, self.__next_attr(char))

  def __update_input_line(self):
    self.__console.print_line(self.__geometry.y_input, self.__example_lines[0])
    self.__console.print_line(self.__geometry.y_input,
                              self.__input_line,
                              clear=False)

  def __scroll(self):
    del self.__example_lines[0]
    self.__input_line = xorcise.Line()

    bottom_line_index = self.__geometry.y_bottom - self.__geometry.y_input
    if bottom_line_index < len(self.__example_lines):
      self.__console.scroll(self.__example_lines[bottom_line_index])
    else:
      self.__console.scroll()

  def __print_all_example_lines(self):
    for index in range(self.__geometry.y_bottom - self.__geometry.y_input + 1):
      if index >= len(self.__example_lines): break
      self.__console.scroll(self.__example_lines[index])

  def __next_attr(self, char):
    normalized_input_line = self.__input_line.normalized
    normalized_example_line = self.__example_lines[0].normalized
    if len(normalized_input_line) >= len(normalized_example_line):
      return self.ATTR_WRONG
    return (self.ATTR_CORRECT if self.__is_correct_char(char)
            else self.ATTR_WRONG) \
           | normalized_example_line[min(len(normalized_input_line),
                                         len(normalized_example_line) - 1)] \
                                         .attr

  def __is_correct_char(self, char):
    next_input_line = self.__input_line + xorcise.Character(char)
    for index in range(len(self.__input_line.normalized),
                       len(next_input_line.normalized)):
      if index >= len(self.__example_lines[0].normalized) \
         or next_input_line.normalized[index].value \
            != self.__example_lines[0].normalized[index].value:
        return False
    return True

class Geometry:
  def __init__(self, console):
    self.y_input = (console.screen_height - 1) // 2
    self.y_bottom = console.screen_height - 1

class CuiFormatter(pygments.formatter.Formatter):
  def __init__(self, **options):
    super().__init__(**options)

    self.__attrs = {}
    for token_type, style in self.style:
      attr = xorcise.RenditionAttribute.normal
      if style["color"]:
        attr |= xorcise.ColorAttribute.red
      if style["bold"]:
        attr |= xorcise.RenditionAttribute.bold
      if style["underline"]:
        attr |= xorcise.RenditionAttribute.underline
      self.__attrs[token_type] = attr

  def format(self, tokens):
    lines = []
    line = xorcise.Line()
    for token_type, value in tokens:
      while token_type not in self.__attrs:
        token_type = token_type.parent

      for char in value:
        if char == '\n':
          lines.append(line)
          line = xorcise.Line()
        elif xorcise.is_printable_char(char):
          line += xorcise.Character(char, self.__attrs[token_type])
    return lines

# functions

def format(text, max_width=79):
  lexer_options = {"stripall" : True}
  try:
    lexer = pygments.lexers.guess_lexer(text, **lexer_options)
  except pygments.util.ClassNotFound:
    lexer = pygments.lexers.special.TextLexer(**lexer_options)
  tokens = lexer.get_tokens(text)
  lines = CuiFormatter().format(tokens)

  new_lines = []
  for line in lines:
    while line.width > max_width:
      new_line, line = split_line(line, max_width)
      new_lines.append(new_line)
    new_lines.append(line)
  return new_lines

def split_line(line, max_width):
  assert max_width >= 2 # for double-width characters
  assert line.width > max_width

  # binary search for max index to construct a line of max width
  min_index = 0
  max_index = len(line)
  while min_index != max_index:
    middle_index = (max_index - min_index + 1) // 2 + min_index
    if line[:middle_index].width <= max_width:
      min_index = middle_index
    else:
      max_index = middle_index - 1

  assert line[:min_index].width <= max_width
  return line[:min_index], line[min_index:]

def parse_args():
  arg_parser = argparse.ArgumentParser(description=DESCRIPTION)
  arg_parser.add_argument("example_path", nargs='?', default=None,
                          help="file path or URI to example")
  arg_parser.add_argument("-a", "--asciize",
                          dest="asciize", action="store_true",
                          help="enable asciization")
  arg_parser.add_argument("-s", "--skip",
                          dest="can_skip", action="store_true",
                          help="enable the skip key, {}"
                               .format(xorcise.unctrl(SKIP_CHAR)))
  arg_parser.add_argument("-t", "--spaces-per-tab",
                          dest="spaces_per_tab", type=int, default=4,
                          help="set number of spaces per tab")
  arg_parser.add_argument("-v", "--version",
                          dest="show_version", action="store_true",
                          help="show version information")

  args = arg_parser.parse_args()

  if args.show_version:
    print("version:", __version__)
    exit()

  global CAN_SKIP
  CAN_SKIP = args.can_skip

  return args

def read_from_stdin():
  text = ""
  for line in sys.stdin:
    text += line

  sys.stdin.close()
  sys.stdin = open(TTY_DEVICE_FILE)

  return text

def read_local_file(path):
  with open(path, "rb") as f:
    return f.read().decode(ENCODING, "replace")

def read_remote_file(uri):
  if urllib.parse.urlparse(uri).scheme not in SUPPORTED_SCHEMES:
    fail("Invalid scheme of URI is detected. "
         "(supported schemes: {})".format(", ".join(SUPPORTED_SCHEMES)))

  message("Loading a page...")
  with urllib.request.urlopen(uri) as response:
    return response.read().decode(ENCODING, "replace")

# main routine

def main():
  if not sys.stdout.isatty(): fail("stdout is not a tty.")

  args = parse_args()

  if args.example_path == None:
    example_text = read_from_stdin()
  elif validators.url(args.example_path):
    example_text = read_remote_file(args.example_path)
  else:
    example_text = read_local_file(args.example_path)

  try:
    # CAUTION:
    # You need to raise some Exception instead of calling exit() here
    # to prevent curses from messing up your terminal.

    console = xorcise.turn_on_console(asciize=args.asciize,
                                      spaces_per_tab=args.spaces_per_tab)

    shakyo = Shakyo(console, example_text)
    shakyo.do()
  finally:
    xorcise.turn_off_console()

if __name__ == "__main__":
  main()
<commit_msg>Fix a bug of Shakyo.__print_all_example_lines()<commit_after>#!/usr/bin/env python

import argparse
import os
import os.path
import pygments
import pygments.formatter
import pygments.lexers
import sys
import urllib.parse
import urllib.request
import validators

import xorcise

# constants

__version__ = "0.0.3"

DESCRIPTION = "{} is a tool to learn about something just copying it " \
              "by hand. Type Esc or ^[ to exit while running it." \
              .format(os.path.basename(__file__))

TTY_DEVICE_FILE = "/dev/tty" # POSIX compliant
ENCODING = "UTF-8"
CURSOR_WIDTH = 1

QUIT_CHARS = xorcise.ESCAPE_CHARS
DELETE_CHARS = xorcise.DELETE_CHARS | xorcise.BACKSPACE_CHARS
CLEAR_CHAR = xorcise.ctrl('u')
SKIP_CHAR = xorcise.ctrl('n')

CAN_SKIP = False

SUPPORTED_SCHEMES = {"http", "https", "ftp"}

# the order of bugs

def message(*text):
  print("{}:".format(os.path.basename(__file__)), *text, file=sys.stderr)

def error(*text):
  message("error:", *text)

def fail(*text):
  error(*text)
  exit(1)

# classes

class Shakyo:
  ATTR_CORRECT = xorcise.RenditionAttribute.normal
  ATTR_WRONG = xorcise.RenditionAttribute.reverse

  def __init__(self, console, example_text):
    self.__console = console
    self.__geometry = Geometry(self.__console)
    self.__input_line = xorcise.Line()
    self.__example_lines \
        = format(example_text, max_width=(self.__console.screen_width - 1))
    if len(self.__example_lines) == 0:
      raise Exception("No line can be read from example source.")

  def do(self):
    self.__print_all_example_lines()

    while len(self.__example_lines) != 0:
      self.__update_input_line()
      char = self.__console.get_char()

      if char in QUIT_CHARS:
        break
      elif char == CLEAR_CHAR:
        self.__input_line = xorcise.Line()
      elif char in DELETE_CHARS:
        self.__input_line = self.__input_line[:-1]
      elif (char == '\n' and self.__input_line.normalized
                             == self.__example_lines[0].normalized) \
           or (char == SKIP_CHAR and CAN_SKIP):
        if len(self.__example_lines) == 1: break
        self.__scroll()
      elif xorcise.is_printable_char(char) \
           and (self.__input_line + xorcise.Character(char)).width \
               + CURSOR_WIDTH <= self.__console.screen_width:
        self.__input_line += xorcise.Character(char, self.__next_attr(char))

  def __update_input_line(self):
    self.__console.print_line(self.__geometry.y_input, self.__example_lines[0])
    self.__console.print_line(self.__geometry.y_input,
                              self.__input_line,
                              clear=False)

  def __scroll(self):
    del self.__example_lines[0]
    self.__input_line = xorcise.Line()

    bottom_line_index = self.__geometry.y_bottom - self.__geometry.y_input
    if bottom_line_index < len(self.__example_lines):
      self.__console.scroll(self.__example_lines[bottom_line_index])
    else:
      self.__console.scroll()

  def __print_all_example_lines(self):
    for index in range(self.__geometry.y_bottom - self.__geometry.y_input + 1):
      if index >= len(self.__example_lines): break
      self.__console.print_line(self.__geometry.y_input + index,
                                self.__example_lines[index])

  def __next_attr(self, char):
    normalized_input_line = self.__input_line.normalized
    normalized_example_line = self.__example_lines[0].normalized
    if len(normalized_input_line) >= len(normalized_example_line):
      return self.ATTR_WRONG
    return (self.ATTR_CORRECT if self.__is_correct_char(char)
            else self.ATTR_WRONG) \
           | normalized_example_line[min(len(normalized_input_line),
                                         len(normalized_example_line) - 1)] \
                                         .attr

  def __is_correct_char(self, char):
    next_input_line = self.__input_line + xorcise.Character(char)
    for index in range(len(self.__input_line.normalized),
                       len(next_input_line.normalized)):
      if index >= len(self.__example_lines[0].normalized) \
         or next_input_line.normalized[index].value \
            != self.__example_lines[0].normalized[index].value:
        return False
    return True

class Geometry:
  def __init__(self, console):
    self.y_input = (console.screen_height - 1) // 2
    self.y_bottom = console.screen_height - 1

class CuiFormatter(pygments.formatter.Formatter):
  def __init__(self, **options):
    super().__init__(**options)

    self.__attrs = {}
    for token_type, style in self.style:
      attr = xorcise.RenditionAttribute.normal
      if style["color"]:
        attr |= xorcise.ColorAttribute.red
      if style["bold"]:
        attr |= xorcise.RenditionAttribute.bold
      if style["underline"]:
        attr |= xorcise.RenditionAttribute.underline
      self.__attrs[token_type] = attr

  def format(self, tokens):
    lines = []
    line = xorcise.Line()
    for token_type, value in tokens:
      while token_type not in self.__attrs:
        token_type = token_type.parent

      for char in value:
        if char == '\n':
          lines.append(line)
          line = xorcise.Line()
        elif xorcise.is_printable_char(char):
          line += xorcise.Character(char, self.__attrs[token_type])
    return lines

# functions

def format(text, max_width=79):
  lexer_options = {"stripall" : True}
  try:
    lexer = pygments.lexers.guess_lexer(text, **lexer_options)
  except pygments.util.ClassNotFound:
    lexer = pygments.lexers.special.TextLexer(**lexer_options)
  tokens = lexer.get_tokens(text)
  lines = CuiFormatter().format(tokens)

  new_lines = []
  for line in lines:
    while line.width > max_width:
      new_line, line = split_line(line, max_width)
      new_lines.append(new_line)
    new_lines.append(line)
  return new_lines

def split_line(line, max_width):
  assert max_width >= 2 # for double-width characters
  assert line.width > max_width

  # binary search for max index to construct a line of max width
  min_index = 0
  max_index = len(line)
  while min_index != max_index:
    middle_index = (max_index - min_index + 1) // 2 + min_index
    if line[:middle_index].width <= max_width:
      min_index = middle_index
    else:
      max_index = middle_index - 1

  assert line[:min_index].width <= max_width
  return line[:min_index], line[min_index:]

def parse_args():
  arg_parser = argparse.ArgumentParser(description=DESCRIPTION)
  arg_parser.add_argument("example_path", nargs='?', default=None,
                          help="file path or URI to example")
  arg_parser.add_argument("-a", "--asciize",
                          dest="asciize", action="store_true",
                          help="enable asciization")
  arg_parser.add_argument("-s", "--skip",
                          dest="can_skip", action="store_true",
                          help="enable the skip key, {}"
                               .format(xorcise.unctrl(SKIP_CHAR)))
  arg_parser.add_argument("-t", "--spaces-per-tab",
                          dest="spaces_per_tab", type=int, default=4,
                          help="set number of spaces per tab")
  arg_parser.add_argument("-v", "--version",
                          dest="show_version", action="store_true",
                          help="show version information")

  args = arg_parser.parse_args()

  if args.show_version:
    print("version:", __version__)
    exit()

  global CAN_SKIP
  CAN_SKIP = args.can_skip

  return args

def read_from_stdin():
  text = ""
  for line in sys.stdin:
    text += line

  sys.stdin.close()
  sys.stdin = open(TTY_DEVICE_FILE)

  return text

def read_local_file(path):
  with open(path, "rb") as f:
    return f.read().decode(ENCODING, "replace")

def read_remote_file(uri):
  if urllib.parse.urlparse(uri).scheme not in SUPPORTED_SCHEMES:
    fail("Invalid scheme of URI is detected. "
         "(supported schemes: {})".format(", ".join(SUPPORTED_SCHEMES)))

  message("Loading a page...")
  with urllib.request.urlopen(uri) as response:
    return response.read().decode(ENCODING, "replace")

# main routine

def main():
  if not sys.stdout.isatty(): fail("stdout is not a tty.")

  args = parse_args()

  if args.example_path == None:
    example_text = read_from_stdin()
  elif validators.url(args.example_path):
    example_text = read_remote_file(args.example_path)
  else:
    example_text = read_local_file(args.example_path)

  try:
    # CAUTION:
    # You need to raise some Exception instead of calling exit() here
    # to prevent curses from messing up your terminal.

    console = xorcise.turn_on_console(asciize=args.asciize,
                                      spaces_per_tab=args.spaces_per_tab)

    shakyo = Shakyo(console, example_text)
    shakyo.do()
  finally:
    xorcise.turn_off_console()

if __name__ == "__main__":
  main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3.5
# coding: utf-8
import sys
from mikado import Goals
from os import path
from subprocess import run
from system import save, load, dot_export
from PyQt5.QtCore import pyqtSignal, Qt
from PyQt5.QtGui import QImage, QPixmap
from PyQt5.QtWidgets import (
    QApplication,
    QDockWidget,
    QLabel,
    QLineEdit,
    QMainWindow,
    QScrollArea,
    QSizePolicy
)

class SiebenApp(QMainWindow):
    refresh = pyqtSignal()
    quit_app = pyqtSignal()

    def __init__(self):
        super().__init__()
        self.scr = QScrollArea()
        self.l = QLabel()
        self.dock = QDockWidget('Edit area')
        self.input = QLineEdit()
        self.refresh.connect(self.reload_image)
        self.quit_app.connect(QApplication.instance().quit)
        if path.exists('sieben.db'):
            self.goals = load()
        else:
            self.goals = Goals('Rename me')

    def setup(self):
        self.setWindowTitle('Sieben 7')
        self.resize(700, 500)
        self.scr.setWidget(self.l)
        self.setCentralWidget(self.scr)
        self.l.setScaledContents(True)
        self.dock.setFeatures(QDockWidget.NoDockWidgetFeatures)
        self.dock.setWidget(self.input)
        self.addDockWidget(Qt.BottomDockWidgetArea, self.dock)
        self.l.setVisible(True)
        self.scr.setVisible(True)
        self.dock.setVisible(True)
        self.input.setVisible(True)
        self.input.setEnabled(False)
        self.refresh.emit()

    def reload_image(self):
        save(self.goals)
        with open('work.dot', 'w') as f:
            f.write(dot_export(self.goals))
        run(['dot', '-Tpng', '-o', 'work.png', 'work.dot'])
        img = QImage('work.png')
        self.l.setPixmap(QPixmap.fromImage(img))
        self.l.resize(img.size().width(), img.size().height())

    def keyPressEvent(self, event):
        key_handlers = {
            Qt.Key_1: lambda: self.select_number(1),
            Qt.Key_2: lambda: self.select_number(2),
            Qt.Key_3: lambda: self.select_number(3),
            Qt.Key_4: lambda: self.select_number(4),
            Qt.Key_5: lambda: self.select_number(5),
            Qt.Key_6: lambda: self.select_number(6),
            Qt.Key_7: lambda: self.select_number(7),
            Qt.Key_8: lambda: self.select_number(8),
            Qt.Key_9: lambda: self.select_number(9),
            Qt.Key_A: lambda: self.start_adding_goal(),
            Qt.Key_C: lambda: self.toggle_close_goal(),
            Qt.Key_D: lambda: self.delete_goal(),
            Qt.Key_I: lambda: self.start_inserting_goal(),
            Qt.Key_L: lambda: self.toggle_link_goals(),
            Qt.Key_Q: lambda: self.quit_app.emit(),
            Qt.Key_R: lambda: self.start_renaming_goal(),
            Qt.Key_Space: lambda: self.hold_current_selection(),
        }
        if event.key() in key_handlers:
            key_handlers[event.key()]()
        else:
            super().keyPressEvent(event)

    def start_adding_goal(self):
        self.input.setEnabled(True)
        self.input.setFocus(True)
        self.input.editingFinished.connect(self.end_adding_goal)

    def end_adding_goal(self):
        self.input.editingFinished.disconnect()
        goal_name = self.input.text()
        self.goals.add(goal_name)
        self.input.setEnabled(False)
        self.input.setText('')
        self.refresh.emit()

    def start_renaming_goal(self):
        self.input.setEnabled(True)
        self.input.setFocus(True)
        self.input.editingFinished.connect(self.end_renaming_goal)

    def end_renaming_goal(self):
        self.input.editingFinished.disconnect()
        new_name = self.input.text()
        self.goals.rename(new_name)
        self.input.setEnabled(False)
        self.input.setText('')
        self.refresh.emit()

    def start_inserting_goal(self):
        self.input.setEnabled(True)
        self.input.setFocus(True)
        self.input.editingFinished.connect(self.end_inserting_goal)

    def end_inserting_goal(self):
        self.input.editingFinished.disconnect()
        goal_name = self.input.text()
        self.goals.insert(goal_name)
        self.input.setEnabled(False)
        self.input.setText('')
        self.refresh.emit()

    def select_number(self, num):
        self.goals.select(num)
        self.refresh.emit()

    def toggle_close_goal(self):
        self.goals.toggle_close()
        self.refresh.emit()

    def delete_goal(self):
        self.goals.delete()
        self.refresh.emit()

    def toggle_link_goals(self):
        self.goals.toggle_link()
        self.refresh.emit()

    def hold_current_selection(self):
        self.goals.hold_select()
        self.refresh.emit()

if __name__ == '__main__':
    app = QApplication(sys.argv)
    w = SiebenApp()
    w.setup()
    w.show()

    sys.exit(app.exec_())
<commit_msg>Remove unneeded lambdas<commit_after>#!/usr/bin/env python3.5
# coding: utf-8
import sys
from mikado import Goals
from os import path
from subprocess import run
from system import save, load, dot_export
from PyQt5.QtCore import pyqtSignal, Qt
from PyQt5.QtGui import QImage, QPixmap
from PyQt5.QtWidgets import (
    QApplication,
    QDockWidget,
    QLabel,
    QLineEdit,
    QMainWindow,
    QScrollArea,
    QSizePolicy
)

class SiebenApp(QMainWindow):
    refresh = pyqtSignal()
    quit_app = pyqtSignal()

    def __init__(self):
        super().__init__()
        self.scr = QScrollArea()
        self.l = QLabel()
        self.dock = QDockWidget('Edit area')
        self.input = QLineEdit()
        self.refresh.connect(self.reload_image)
        self.quit_app.connect(QApplication.instance().quit)
        if path.exists('sieben.db'):
            self.goals = load()
        else:
            self.goals = Goals('Rename me')

    def setup(self):
        self.setWindowTitle('Sieben 7')
        self.resize(700, 500)
        self.scr.setWidget(self.l)
        self.setCentralWidget(self.scr)
        self.l.setScaledContents(True)
        self.dock.setFeatures(QDockWidget.NoDockWidgetFeatures)
        self.dock.setWidget(self.input)
        self.addDockWidget(Qt.BottomDockWidgetArea, self.dock)
        self.l.setVisible(True)
        self.scr.setVisible(True)
        self.dock.setVisible(True)
        self.input.setVisible(True)
        self.input.setEnabled(False)
        self.refresh.emit()

    def reload_image(self):
        save(self.goals)
        with open('work.dot', 'w') as f:
            f.write(dot_export(self.goals))
        run(['dot', '-Tpng', '-o', 'work.png', 'work.dot'])
        img = QImage('work.png')
        self.l.setPixmap(QPixmap.fromImage(img))
        self.l.resize(img.size().width(), img.size().height())

    def keyPressEvent(self, event):
        key_handlers = {
            Qt.Key_1: lambda: self.select_number(1),
            Qt.Key_2: lambda: self.select_number(2),
            Qt.Key_3: lambda: self.select_number(3),
            Qt.Key_4: lambda: self.select_number(4),
            Qt.Key_5: lambda: self.select_number(5),
            Qt.Key_6: lambda: self.select_number(6),
            Qt.Key_7: lambda: self.select_number(7),
            Qt.Key_8: lambda: self.select_number(8),
            Qt.Key_9: lambda: self.select_number(9),
            Qt.Key_A: self.start_adding_goal,
            Qt.Key_C: self.toggle_close_goal,
            Qt.Key_D: self.delete_goal,
            Qt.Key_I: self.start_inserting_goal,
            Qt.Key_L: self.toggle_link_goals,
            Qt.Key_Q: self.quit_app.emit,
            Qt.Key_R: self.start_renaming_goal,
            Qt.Key_Space: self.hold_current_selection,
        }
        if event.key() in key_handlers:
            key_handlers[event.key()]()
        else:
            super().keyPressEvent(event)

    def start_adding_goal(self):
        self.input.setEnabled(True)
        self.input.setFocus(True)
        self.input.editingFinished.connect(self.end_adding_goal)

    def end_adding_goal(self):
        self.input.editingFinished.disconnect()
        goal_name = self.input.text()
        self.goals.add(goal_name)
        self.input.setEnabled(False)
        self.input.setText('')
        self.refresh.emit()

    def start_renaming_goal(self):
        self.input.setEnabled(True)
        self.input.setFocus(True)
        self.input.editingFinished.connect(self.end_renaming_goal)

    def end_renaming_goal(self):
        self.input.editingFinished.disconnect()
        new_name = self.input.text()
        self.goals.rename(new_name)
        self.input.setEnabled(False)
        self.input.setText('')
        self.refresh.emit()

    def start_inserting_goal(self):
        self.input.setEnabled(True)
        self.input.setFocus(True)
        self.input.editingFinished.connect(self.end_inserting_goal)

    def end_inserting_goal(self):
        self.input.editingFinished.disconnect()
        goal_name = self.input.text()
        self.goals.insert(goal_name)
        self.input.setEnabled(False)
        self.input.setText('')
        self.refresh.emit()

    def select_number(self, num):
        self.goals.select(num)
        self.refresh.emit()

    def toggle_close_goal(self):
        self.goals.toggle_close()
        self.refresh.emit()

    def delete_goal(self):
        self.goals.delete()
        self.refresh.emit()

    def toggle_link_goals(self):
        self.goals.toggle_link()
        self.refresh.emit()

    def hold_current_selection(self):
        self.goals.hold_select()
        self.refresh.emit()

if __name__ == '__main__':
    app = QApplication(sys.argv)
    w = SiebenApp()
    w.setup()
    w.show()

    sys.exit(app.exec_())
<|endoftext|>