<commit_before>import wave, struct, random, math
import os

def frequency_from_key(key):
	return 440 * 2 ** ((key - 49) / 12.0)

def frequency_from_note(note):
	return frequency_from_key(key_from_note(note))

def key_from_note(note):
	indices = { 'C':0, 'D':2, 'E':4, 'F':5, 'G':7, 'A':9, 'B':11 }

	octave = 4

	if note[-1] in '012345678':
		octave = int(note[-1])

	tone = indices[note[0].upper()]
	key = 12 * octave + tone

	if len(note) > 1 and note[1] == '#':
		key += 1
	elif len(note) > 1 and note[1] == 'b':
		key -= 1

	return key - 8;

def note_from_key(key):
	notes = ['a','a#','b','c','c#','d','d#','e','f','f#','g','g#']
	octave = (key+8) / 12
	note = notes[(key-1) % 12]

	return note.upper() + str(octave)

class Melopy:
	def __init__(self, title='sound', volume=50):
		if title == '':
			raise Exception('Title must be non-null.')

		self.melopy_writer = wave.open(title + '.wav', 'w')
		self.melopy_writer.setparams((2, 2, 44100, 0, 'NONE', 'not compressed'))
		self.rate = 44100
		self.volume = volume / 100.0 * 32767
		self.data = []

	def add_wave(self, wave_type, frequency, length, location='END'):
		if location == 'END':
			location = len(self.data)
		elif location < 0:
			location = 0
		elif location * 44100 > len(self.data):
			location = len(self.data) / 44100.0

		# location is a time, so let's adjust
		location = int(location * 44100)

		for n in range(0, int(44100 * length)):
			period = 44100.0 / frequency

			if wave_type == 'square':
				val = (n % int(period) >= (int(period)/2)) * self.volume
			elif wave_type == 'triangle':
				val = (n % int(period)) / period * self.volume

			if location + n >= len(self.data):
				self.data.append(val)
			else:
				self.data[location + n] += val
				# check that we're not above the volume
				if self.data[location + n] > self.volume:
					self.data[location + n] = self.volume

	def add_note(self, note, length, wave_type='square', location='END'):
		if wave_type == 'square':
			self.add_square_wave(frequency_from_note(note), length, location)
		elif wave_type == 'triangle':
			self.add_triangle_wave(frequency_from_note(note), length, location)

	def add_square_wave(self, frequency, length, location='END'):
		self.add_wave('square', frequency, length, location)

	def add_triangle_wave(self, frequency, length, location='END'):
		self.add_wave('triangle', frequency, length, location)

	def iterate(self, start, pattern):
		start_key = key_from_note(start)
		ret = [start_key]
		for step in pattern:
			ret.append(ret[-1] + step)

		ret = map(note_from_key, ret)
		return ret

	def generate_major_scale(self, start):
		major_steps = [2,2,1,2,2,2,1]
		return self.iterate(start, major_steps)

	def generate_minor_scale(self, start):
		minor_steps = [2,1,2,2,2,1,2]
		return self.iterate(start, minor_steps)

	def generate_major_triad(self, start):
		major_triad = [4, 3]
		return self.iterate(start, major_triad)

	def generate_minor_triad(self, start):
		minor_triad = [3, 4]
		return self.iterate(start, minor_triad)

	def add_melody(self, melody, length, wave_form='square'):
		for note in melody:
			self.add_wave(wave_form, frequency_from_note(note), length)

	def render(self):
		for item in self.data:
			packed_val = struct.pack('h', int(item))
			self.melopy_writer.writeframes(packed_val)
			self.melopy_writer.writeframes(packed_val)

		self.melopy_writer.close()

			<commit_msg>moved generate and iterate to outside of the class. added 'add_rest()' class method<commit_after>import wave, struct, random, math
import os

def frequency_from_key(key):
	return 440 * 2 ** ((key - 49) / 12.0)

def frequency_from_note(note):
	return frequency_from_key(key_from_note(note))

def key_from_note(note):
	indices = { 'C':0, 'D':2, 'E':4, 'F':5, 'G':7, 'A':9, 'B':11 }

	octave = 4

	if note[-1] in '012345678':
		octave = int(note[-1])

	tone = indices[note[0].upper()]
	key = 12 * octave + tone

	if len(note) > 1 and note[1] == '#':
		key += 1
	elif len(note) > 1 and note[1] == 'b':
		key -= 1

	return key - 8;

def note_from_key(key):
	notes = ['a','a#','b','c','c#','d','d#','e','f','f#','g','g#']
	octave = (key+8) / 12
	note = notes[(key-1) % 12]

	return note.upper() + str(octave)

def iterate(start, pattern):
	start_key = key_from_note(start)
	ret = [start_key]
	for step in pattern:
		ret.append(ret[-1] + step)

	ret = map(note_from_key, ret)
	return ret

def generate_major_scale(start):
	major_steps = [2,2,1,2,2,2,1]
	return iterate(start, major_steps)

def generate_minor_scale(start):
	minor_steps = [2,1,2,2,2,1,2]
	return iterate(start, minor_steps)

def generate_major_triad(start):
	major_triad = [4, 3]
	return iterate(start, major_triad)

def generate_minor_triad(start):
	minor_triad = [3, 4]
	return iterate(start, minor_triad)

class Melopy:
	def __init__(self, title='sound', volume=50):
		if title == '':
			raise Exception('Title must be non-null.')

		self.melopy_writer = wave.open(title + '.wav', 'w')
		self.melopy_writer.setparams((2, 2, 44100, 0, 'NONE', 'not compressed'))
		self.rate = 44100
		self.volume = volume / 100.0 * 32767
		self.data = []

	def add_wave(self, wave_type, frequency, length, location='END'):
		if location == 'END':
			location = len(self.data)
		elif location < 0:
			location = 0
		elif location * 44100 > len(self.data):
			location = len(self.data) / 44100.0

		# location is a time, so let's adjust
		location = int(location * 44100)

		for n in range(0, int(44100 * length)):
			period = 44100.0 / frequency

			if wave_type == 'square':
				val = (n % int(period) >= (int(period)/2)) * self.volume
			elif wave_type == 'triangle':
				val = (n % int(period)) / period * self.volume

			if location + n >= len(self.data):
				self.data.append(val)
			else:
				self.data[location + n] += val
				# check that we're not above the volume
				if self.data[location + n] > self.volume:
					self.data[location + n] = self.volume

	def add_note(self, note, length, wave_type='square', location='END'):
		if wave_type == 'square':
			self.add_square_wave(frequency_from_note(note), length, location)
		elif wave_type == 'triangle':
			self.add_triangle_wave(frequency_from_note(note), length, location)

	def add_square_wave(self, frequency, length, location='END'):
		self.add_wave('square', frequency, length, location)

	def add_triangle_wave(self, frequency, length, location='END'):
		self.add_wave('triangle', frequency, length, location)

	def add_melody(self, melody, length, wave_form='square'):
		for note in melody:
			self.add_wave(wave_form, frequency_from_note(note), length)

	def add_rest(length):
		for i in range(self.rate * length):
			self.data.append(0)

	def render(self):
		for item in self.data:
			packed_val = struct.pack('h', int(item))
			self.melopy_writer.writeframes(packed_val)
			self.melopy_writer.writeframes(packed_val)

		self.melopy_writer.close()

			<|endoftext|>


<commit_before>#
# Copyright (C) 2011 - 2021 Satoru SATOH <satoru.satoh @ gmail.com>
# SPDX-License-Identifier: MIT
#
"""Misc simple parsers."""
import re
import typing
import warnings

INT_PATTERN: typing.Pattern = re.compile(r"^(\d|([1-9]\d+))$")
FLOAT_PATTERN: typing.Pattern = re.compile(r"^\d+[\.]\d+$")
BOOL_PATTERN: typing.Pattern = re.compile(r"^(true|false)$", re.I)
STR_PATTERN: typing.Pattern = re.compile(r"^['\"](.*)['\"]$")

PrimitiveT = typing.Union[str, int, float, bool]
PrimitivesT = typing.List[PrimitiveT]

def parse_single(str_: typing.Optional[str]) -> PrimitiveT:
    """Very simple parser to parse an expression represents a primitive value.
    """
    if str_ is None:
        return ''

    str_ = str_.strip()

    if not str_:
        return ''

    if BOOL_PATTERN.match(str_) is not None:
        return bool(str_)

    if INT_PATTERN.match(str_) is not None:
        return int(str_)

    if FLOAT_PATTERN.match(str_) is not None:
        return float(str_)

    if STR_PATTERN.match(str_) is not None:
        return str_[1:-1]

    return str_

def parse_list(str_: str, sep: str = ',') -> PrimitivesT:
    """
    Simple parser to parse an expression ``str_`` contains a list of str-es
    separated with ``sep``, represents a list of primitive values.

    >>> parse_list('')
    []
    >>> parse_list('1')
    [1]
    >>> parse_list('a,b')
    ['a', 'b']
    >>> parse_list('1,2')
    [1, 2]
    >>> parse_list('a,b,')
    ['a', 'b']
    """
    return [parse_single(x) for x in str_.split(sep) if x]

AttrValsT = typing.Tuple[str, typing.Union[PrimitivesT, PrimitiveT]]

def attr_val_itr(str_: str, avs_sep: str = ':', vs_sep: str = ',',
                 as_sep: str = ';') -> typing.Iterator[AttrValsT]:
    """
    Atrribute and value pair parser.

    :param str_: String represents a list of pairs of attribute and value
    :param avs_sep: char to separate attribute and values
    :param vs_sep: char to separate values
    :param as_sep: char to separate attributes
    """
    for rel in parse_list(str_, as_sep):
        rel = typing.cast(str, rel)
        if avs_sep not in rel or rel.endswith(avs_sep):
            continue

        (_attr, _values, *_rest) = parse_list(rel, avs_sep)

        if _rest:
            warnings.warn(f'Extra strings {_rest!s} in {rel!s}'
                          f'It should be in the form of attr{avs_sep}value.')

        _attr = typing.cast(str, _attr)

        if vs_sep in str(_values):
            yield (_attr, parse_list(typing.cast(str, _values), vs_sep))
        elif _values:
            yield (_attr, typing.cast(PrimitiveT, _values))

def parse_attrlist_0(str_: str, avs_sep: str = ':', vs_sep: str = ',',
                     as_sep: str = ';') -> typing.List[AttrValsT]:
    """
    Simple parser to parse expressions in the form of
    [ATTR1:VAL0,VAL1,...;ATTR2:VAL0,VAL2,..].

    :param str_: input string
    :param avs_sep:  char to separate attribute and values
    :param vs_sep:  char to separate values
    :param as_sep:  char to separate attributes

    :return:
        a list of tuples of (key, value | [value])
            where key = (Int | String | ...),
            value = (Int | Bool | String | ...) | [Int | Bool | String | ...]

    >>> parse_attrlist_0('a:1')
    [('a', 1)]
    >>> parse_attrlist_0('a:1;b:xyz')
    [('a', 1), ('b', 'xyz')]
    >>> parse_attrlist_0('requires:bash,zsh')
    [('requires', ['bash', 'zsh'])]
    >>> parse_attrlist_0('obsoletes:sysdata;conflicts:sysdata-old')
    [('obsoletes', 'sysdata'), ('conflicts', 'sysdata-old')]
    """
    return list(attr_val_itr(str_, avs_sep, vs_sep, as_sep))

AttrValsDictT = typing.Dict[str, typing.Union[PrimitivesT, PrimitiveT]]

def parse_attrlist(str_: str, avs_sep: str = ':', vs_sep: str = ',',
                   as_sep: str = ';') -> AttrValsDictT:
    """
    Simple parser to parse expressions in the form of
    [ATTR1:VAL0,VAL1,...;ATTR2:VAL0,VAL2,..].

    :param str_: input string
    :param avs_sep:  char to separate attribute and values
    :param vs_sep:  char to separate values
    :param as_sep:  char to separate attributes

    >>> parse_attrlist('requires:bash,zsh')
    {'requires': ['bash', 'zsh']}
    """
    return dict(parse_attrlist_0(str_, avs_sep, vs_sep, as_sep))

ResultsT = typing.Union[
    PrimitiveT,
    PrimitivesT,
    AttrValsDictT
]

def parse(str_: typing.Optional[str],
          lsep: str = ',', avsep: str = ':', vssep: str = ',',
          avssep: str = ';') -> ResultsT:
    """Generic parser"""
    if str_ is None or not str_:
        return parse_single(str_)

    if avsep in str_:
        return parse_attrlist(str_, avsep, vssep, avssep)
    if lsep in str_:
        return parse_list(str_, lsep)

    return parse_single(str_)

# vim:sw=4:ts=4:et:
<commit_msg>fix: [doc] pydocstyle errors in .parser<commit_after>#
# Copyright (C) 2011 - 2021 Satoru SATOH <satoru.satoh @ gmail.com>
# SPDX-License-Identifier: MIT
#
"""Misc simple parsers."""
import re
import typing
import warnings

INT_PATTERN: typing.Pattern = re.compile(r"^(\d|([1-9]\d+))$")
FLOAT_PATTERN: typing.Pattern = re.compile(r"^\d+[\.]\d+$")
BOOL_PATTERN: typing.Pattern = re.compile(r"^(true|false)$", re.I)
STR_PATTERN: typing.Pattern = re.compile(r"^['\"](.*)['\"]$")

PrimitiveT = typing.Union[str, int, float, bool]
PrimitivesT = typing.List[PrimitiveT]

def parse_single(str_: typing.Optional[str]) -> PrimitiveT:
    """Parse an expression gives a primitive value."""
    if str_ is None:
        return ''

    str_ = str_.strip()

    if not str_:
        return ''

    if BOOL_PATTERN.match(str_) is not None:
        return bool(str_)

    if INT_PATTERN.match(str_) is not None:
        return int(str_)

    if FLOAT_PATTERN.match(str_) is not None:
        return float(str_)

    if STR_PATTERN.match(str_) is not None:
        return str_[1:-1]

    return str_

def parse_list(str_: str, sep: str = ',') -> PrimitivesT:
    """Parse an expression gives a list of values.

    An expression ``str_`` might contain a list of str-es separated with
    ``sep``, represents a list of primitive values.
    """
    return [parse_single(x) for x in str_.split(sep) if x]

AttrValsT = typing.Tuple[str, typing.Union[PrimitivesT, PrimitiveT]]

def attr_val_itr(str_: str, avs_sep: str = ':', vs_sep: str = ',',
                 as_sep: str = ';') -> typing.Iterator[AttrValsT]:
    """Parse a list of atrribute and value pairs.

    This is a helper function for parse_attrlist_0.

    :param str_: String represents a list of pairs of attribute and value
    :param avs_sep: char to separate attribute and values
    :param vs_sep: char to separate values
    :param as_sep: char to separate attributes
    """
    for rel in parse_list(str_, as_sep):
        rel = typing.cast(str, rel)
        if avs_sep not in rel or rel.endswith(avs_sep):
            continue

        (_attr, _values, *_rest) = parse_list(rel, avs_sep)

        if _rest:
            warnings.warn(f'Extra strings {_rest!s} in {rel!s}'
                          f'It should be in the form of attr{avs_sep}value.')

        _attr = typing.cast(str, _attr)

        if vs_sep in str(_values):
            yield (_attr, parse_list(typing.cast(str, _values), vs_sep))
        elif _values:
            yield (_attr, typing.cast(PrimitiveT, _values))

def parse_attrlist_0(str_: str, avs_sep: str = ':', vs_sep: str = ',',
                     as_sep: str = ';') -> typing.List[AttrValsT]:
    """Parse a list of atrribute and value pairs.

    This is a helper function for parse_attrlist.

    The expressions to parse should be in the form of
    [ATTR1:VAL0,VAL1,...;ATTR2:VAL0,VAL2,..].

    :param str_: input string
    :param avs_sep:  char to separate attribute and values
    :param vs_sep:  char to separate values
    :param as_sep:  char to separate attributes

    :return:
        a list of tuples of (key, value | [value])
            where key = (Int | String | ...),
            value = (Int | Bool | String | ...) | [Int | Bool | String | ...]
    """
    return list(attr_val_itr(str_, avs_sep, vs_sep, as_sep))

AttrValsDictT = typing.Dict[str, typing.Union[PrimitivesT, PrimitiveT]]

def parse_attrlist(str_: str, avs_sep: str = ':', vs_sep: str = ',',
                   as_sep: str = ';') -> AttrValsDictT:
    """Parse a list of atrribute and value pairs.

    The expressions to parse should be in the form of
    [ATTR1:VAL0,VAL1,...;ATTR2:VAL0,VAL2,..].

    :param str_: input string
    :param avs_sep:  char to separate attribute and values
    :param vs_sep:  char to separate values
    :param as_sep:  char to separate attributes
    """
    return dict(parse_attrlist_0(str_, avs_sep, vs_sep, as_sep))

ResultsT = typing.Union[
    PrimitiveT,
    PrimitivesT,
    AttrValsDictT
]

def parse(str_: typing.Optional[str],
          lsep: str = ',', avsep: str = ':', vssep: str = ',',
          avssep: str = ';') -> ResultsT:
    """Very simple generic parser."""
    if str_ is None or not str_:
        return parse_single(str_)

    if avsep in str_:
        return parse_attrlist(str_, avsep, vssep, avssep)
    if lsep in str_:
        return parse_list(str_, lsep)

    return parse_single(str_)

# vim:sw=4:ts=4:et:
<|endoftext|>


<commit_before># Copyright 2013 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

import fileinput
import re
import subprocess
import sys
import os
DEVNULL = open(os.devnull, 'wb')

def mutate_line(file_name, line_number):
    for line in fileinput.input(file_name, inplace=True):
        if fileinput.lineno() == line_number:
            line = re.sub(r'\s&&\s', ' || ', line)
        print line.rstrip()

def mutation_test(file_name, tests):
    line_numbers = []
    for line in fileinput.input(file_name):
        if re.search(r'\s&&\s', line):
            line_numbers.append(fileinput.lineno())

    for line_to_mutate in line_numbers:
        print "Mutating {0} at line {1}".format(file_name, line_to_mutate)
        mutate_line(file_name, line_to_mutate)
        print "compling mutant {0}:{1}".format(file_name, line_to_mutate)
        sys.stdout.flush()
        subprocess.call('python mach build --release', shell=True, stdout=DEVNULL)
        print "running tests for mutant {0}:{1}".format(file_name, line_to_mutate)
        sys.stdout.flush()
        for test in tests:
            test_command = "python mach test-wpt {0} --release".format(test.encode('utf-8'))
            test_status = subprocess.call(test_command, shell=True, stdout=DEVNULL)
            if test_status != 0:
                print("Failed in while running `{0}`".format(test_command))
                print "mutated file {0} diff".format(file_name)
                sys.stdout.flush()
                subprocess.call('git --no-pager diff {0}'.format(file_name), shell=True)
        print "reverting mutant {0}:{1}".format(file_name, line_to_mutate)
        sys.stdout.flush()
        subprocess.call('git checkout {0}'.format(file_name), shell=True)
<commit_msg>Loging success message on mutation test success<commit_after># Copyright 2013 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

import fileinput
import re
import subprocess
import sys
import os
DEVNULL = open(os.devnull, 'wb')

def mutate_line(file_name, line_number):
    for line in fileinput.input(file_name, inplace=True):
        if fileinput.lineno() == line_number:
            line = re.sub(r'\s&&\s', ' || ', line)
        print line.rstrip()

def mutation_test(file_name, tests):
    line_numbers = []
    for line in fileinput.input(file_name):
        if re.search(r'\s&&\s', line):
            line_numbers.append(fileinput.lineno())

    for line_to_mutate in line_numbers:
        print "Mutating {0} at line {1}".format(file_name, line_to_mutate)
        mutate_line(file_name, line_to_mutate)
        print "compling mutant {0}:{1}".format(file_name, line_to_mutate)
        sys.stdout.flush()
        subprocess.call('python mach build --release', shell=True, stdout=DEVNULL)
        print "running tests for mutant {0}:{1}".format(file_name, line_to_mutate)
        sys.stdout.flush()
        for test in tests:
            test_command = "python mach test-wpt {0} --release".format(test.encode('utf-8'))
            test_status = subprocess.call(test_command, shell=True, stdout=DEVNULL)
            if test_status != 0:
                print("Failed: while running `{0}`".format(test_command))
                print "mutated file {0} diff".format(file_name)
                sys.stdout.flush()
                subprocess.call('git --no-pager diff {0}'.format(file_name), shell=True)
            else:
                print("Success: Mutation killed by {0}".format(test.encode('utf-8')))
                break
        print "reverting mutant {0}:{1}".format(file_name, line_to_mutate)
        sys.stdout.flush()
        subprocess.call('git checkout {0}'.format(file_name), shell=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Licker - Github License Checker for Organisational Repositories
# Copyright 2015 Jan Brennenstuhl
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from collections import namedtuple

import urllib.request
import json

Repositories = namedtuple('Repositories', ['with_license', 'without_license'])

class Licker(object):

    def __init__(self, organisation, token):
        self.api = 'https://api.github.com'
        self.repository_url = '%s/orgs/%s/repos?type=all' % (self.api, organisation)
        self.license_url = '%s/repos/%s/' % (self.api, organisation)
        self.page_parameter = '&page='

        self.request_headers = {
            'Authorization': 'token %s' % token,
            'Accept': 'application/vnd.github.drax-preview+json'}

    def _fetch_repositories(self):
        page_number = 1
        repositories = []
        json_response = []

        print('start connecting to github...')
        while json_response or page_number == 1:
            request = urllib.request.Request(
                url='%s%s%s' % (self.repository_url, self.page_parameter, page_number),
                headers=self.request_headers)

            if page_number == 1:
                print('start collecting repositories...')

            response = urllib.request.urlopen(request).read()
            json_response = json.loads(response.decode('utf-8'))
            repositories.extend(json_response)

            page_number += 1

        return repositories

    def _fetch_license_information(self, repositories):
        repos_with_license = []
        repos_without_license = []

        print('start fetching license information...')
        for repository in repositories:
            if not repository['fork']:
                request = urllib.request.Request(
                    url=self.license_url + repository['name'],
                    headers=self.request_headers)

                response = urllib.request.urlopen(request).read()
                json_response = json.loads(response.decode('utf-8'))

                if json_response['license']:
                    repos_with_license.append({
                        'name': repository['name'],
                        'license': json_response['license']['name']})
                else:
                    repos_without_license.append(repository['name'])

        return Repositories(repos_with_license, repos_without_license)

    def run_license_analysis(self):
        repositories = self._fetch_repositories()
        repos = self._fetch_license_information(repositories)

        if repos.with_license:
            print('\nfound %s repositories with license:\n' % len(repos.with_license))
            for repository in repos.with_license:
                print('\t%s -- %s' % (repository['name'], repository['license']))

        if repos.without_license:
            print('\nfound %s repositories without license:\n' % len(repos.without_license))
            for repository in repos.without_license:
                print('\t%s' % repository)
<commit_msg>adds developer preview comment.<commit_after>#!/usr/bin/env python
# Licker - Github License Checker for Organisational Repositories
# Copyright 2015 Jan Brennenstuhl
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from collections import namedtuple

import urllib.request
import json

Repositories = namedtuple('Repositories', ['with_license', 'without_license'])

class Licker(object):

    def __init__(self, organisation, token):
        self.api = 'https://api.github.com'
        self.repository_url = '%s/orgs/%s/repos?type=all' % (self.api, organisation)
        self.license_url = '%s/repos/%s/' % (self.api, organisation)
        self.page_parameter = '&page='

        self.request_headers = {
            'Authorization': 'token %s' % token,
            'Accept': 'application/vnd.github.drax-preview+json'  # Required for developer preview
        }

    def _fetch_repositories(self):
        page_number = 1
        repositories = []
        json_response = []

        print('start connecting to github...')
        while json_response or page_number == 1:
            request = urllib.request.Request(
                url='%s%s%s' % (self.repository_url, self.page_parameter, page_number),
                headers=self.request_headers)

            if page_number == 1:
                print('start collecting repositories...')

            response = urllib.request.urlopen(request).read()
            json_response = json.loads(response.decode('utf-8'))
            repositories.extend(json_response)

            page_number += 1

        return repositories

    def _fetch_license_information(self, repositories):
        repos_with_license = []
        repos_without_license = []

        print('start fetching license information...')
        for repository in repositories:
            if not repository['fork']:
                request = urllib.request.Request(
                    url=self.license_url + repository['name'],
                    headers=self.request_headers)

                response = urllib.request.urlopen(request).read()
                json_response = json.loads(response.decode('utf-8'))

                if json_response['license']:
                    repos_with_license.append({
                        'name': repository['name'],
                        'license': json_response['license']['name']})
                else:
                    repos_without_license.append(repository['name'])

        return Repositories(repos_with_license, repos_without_license)

    def run_license_analysis(self):
        repositories = self._fetch_repositories()
        repos = self._fetch_license_information(repositories)

        if repos.with_license:
            print('\nfound %s repositories with license:\n' % len(repos.with_license))
            for repository in repos.with_license:
                print('\t%s -- %s' % (repository['name'], repository['license']))

        if repos.without_license:
            print('\nfound %s repositories without license:\n' % len(repos.without_license))
            for repository in repos.without_license:
                print('\t%s' % repository)
<|endoftext|>


<commit_before>"""

Process Pools.

"""
import os
import time
import errno
import multiprocessing

from multiprocessing.pool import Pool, worker
from celery.datastructures import ExceptionInfo
from celery.utils import gen_unique_id
from functools import partial as curry

def pid_is_dead(pid):
    try:
        return kill(pid, 0)
    except OSError, err:
        if err.errno == errno.ESRCH:
            return True # No such process.
        elif err.errno == errno.EPERM:
            return False # Operation not permitted.
        else:
            raise

def reap_process(pid):
    if pid_is_dead(pid):
        return True

    try:
        is_dead, _ = os.waitpid(pid, os.WNOHANG)
    except OSError, err:
        if err.errno == errno.ECHILD:
            return False # No child processes.
        raise
    return is_dead

class DynamicPool(Pool):
    """Version of :class:`multiprocessing.Pool` that can dynamically grow
    in size."""

    def __init__(self, processes=None, initializer=None, initargs=()):
        super(DynamicPool, self).__init__(processes=processes,
                                          initializer=initializer,
                                          initargs=initargs)
        self._initializer = initializer
        self._initargs = initargs

    def add_worker(self):
        """Add another worker to the pool."""
        w = self.Process(target=worker,
                         args=(self._inqueue, self._outqueue,
                               self._initializer, self._initargs))
        self._pool.append(w)
        w.name = w.name.replace("Process", "PoolWorker")
        w.daemon = True
        w.start()

    def grow(self, size=1):
        """Add ``size`` new workers to the pool."""
        map(self._add_worker, range(size))

    def is_dead(self, process):
        # First try to see if the process is actually running,
        # and reap zombie proceses while we're at it.
        if reap_process(process.pid):
            return True

        # Then try to ping the process using its pipe.
        return not process.is_alive()

    def replace_dead_workers(self):
        dead_processes = filter(self.is_dead, self._pool)

        if dead_processes:
            dead_pids = [process.pid for process in dead_processes]
            self._pool = [process for process in self._pool
                            if process not in dead_pids]
            self.grow(len(dead_processes))

        return dead_processes

class TaskPool(object):
    """Process Pool for processing tasks in parallel.

    :param limit: see :attr:`limit` attribute.
    :param logger: see :attr:`logger` attribute.

    .. attribute:: limit

        The number of processes that can run simultaneously.

    .. attribute:: logger

        The logger used for debugging.

    """

    def __init__(self, limit, logger=None):
        self.limit = limit
        self.logger = logger or multiprocessing.get_logger()
        self._pool = None

    def start(self):
        """Run the task pool.

        Will pre-fork all workers so they're ready to accept tasks.

        """
        self._pool = DynamicPool(processes=self.limit)

    def stop(self):
        """Terminate the pool."""
        self._pool.terminate()
        self._pool = None

    def replace_dead_workers(self):
        self.logger.debug("TaskPool: Finding dead pool processes...")
        dead = self._pool.replace_dead_workers()
        if dead:
            self.logger.info(
                "TaskPool: Replaced %s dead pool workers..." % (
                    len(dead)))

    def apply_async(self, target, args=None, kwargs=None, callbacks=None,
            errbacks=None, on_ack=None, meta=None):
        """Equivalent of the :func:``apply`` built-in function.

        All ``callbacks`` and ``errbacks`` should complete immediately since
        otherwise the thread which handles the result will get blocked.

        """
        args = args or []
        kwargs = kwargs or {}
        callbacks = callbacks or []
        errbacks = errbacks or []
        meta = meta or {}

        on_return = curry(self.on_return, callbacks, errbacks,
                          on_ack, meta)

        self.logger.debug("TaskPool: Apply %s (args:%s kwargs:%s)" % (
            target, args, kwargs))

        self.replace_dead_workers()

        return self._pool.apply_async(target, args, kwargs,
                                        callback=on_return)

    def on_return(self, callbacks, errbacks, on_ack, meta, ret_value):
        """What to do when the process returns."""

        # Acknowledge the task as being processed.
        if on_ack:
            on_ack()

        self.on_ready(callbacks, errbacks, meta, ret_value)

    def on_ready(self, callbacks, errbacks, meta, ret_value):
        """What to do when a worker task is ready and its return value has
        been collected."""

        if isinstance(ret_value, ExceptionInfo):
            if isinstance(ret_value.exception, (
                    SystemExit, KeyboardInterrupt)):
                raise ret_value.exception
            for errback in errbacks:
                errback(ret_value, meta)
        else:
            for callback in callbacks:
                callback(ret_value, meta)
<commit_msg>Fix typo kill( -> os.kill(<commit_after>"""

Process Pools.

"""
import os
import time
import errno
import multiprocessing

from multiprocessing.pool import Pool, worker
from celery.datastructures import ExceptionInfo
from celery.utils import gen_unique_id
from functools import partial as curry

def pid_is_dead(pid):
    try:
        return os.kill(pid, 0)
    except OSError, err:
        if err.errno == errno.ESRCH:
            return True # No such process.
        elif err.errno == errno.EPERM:
            return False # Operation not permitted.
        else:
            raise

def reap_process(pid):
    if pid_is_dead(pid):
        return True

    try:
        is_dead, _ = os.waitpid(pid, os.WNOHANG)
    except OSError, err:
        if err.errno == errno.ECHILD:
            return False # No child processes.
        raise
    return is_dead

class DynamicPool(Pool):
    """Version of :class:`multiprocessing.Pool` that can dynamically grow
    in size."""

    def __init__(self, processes=None, initializer=None, initargs=()):
        super(DynamicPool, self).__init__(processes=processes,
                                          initializer=initializer,
                                          initargs=initargs)
        self._initializer = initializer
        self._initargs = initargs

    def add_worker(self):
        """Add another worker to the pool."""
        w = self.Process(target=worker,
                         args=(self._inqueue, self._outqueue,
                               self._initializer, self._initargs))
        self._pool.append(w)
        w.name = w.name.replace("Process", "PoolWorker")
        w.daemon = True
        w.start()

    def grow(self, size=1):
        """Add ``size`` new workers to the pool."""
        map(self._add_worker, range(size))

    def is_dead(self, process):
        # First try to see if the process is actually running,
        # and reap zombie proceses while we're at it.
        if reap_process(process.pid):
            return True

        # Then try to ping the process using its pipe.
        try:
            proc_is_alive = process.is_alive()
        except OSError:
            return True
        else:
            return not proc_is_alive

    def replace_dead_workers(self):
        dead_processes = filter(self.is_dead, self._pool)

        if dead_processes:
            dead_pids = [process.pid for process in dead_processes]
            self._pool = [process for process in self._pool
                            if process not in dead_pids]
            self.grow(len(dead_processes))

        return dead_processes

class TaskPool(object):
    """Process Pool for processing tasks in parallel.

    :param limit: see :attr:`limit` attribute.
    :param logger: see :attr:`logger` attribute.

    .. attribute:: limit

        The number of processes that can run simultaneously.

    .. attribute:: logger

        The logger used for debugging.

    """

    def __init__(self, limit, logger=None):
        self.limit = limit
        self.logger = logger or multiprocessing.get_logger()
        self._pool = None

    def start(self):
        """Run the task pool.

        Will pre-fork all workers so they're ready to accept tasks.

        """
        self._pool = DynamicPool(processes=self.limit)

    def stop(self):
        """Terminate the pool."""
        self._pool.terminate()
        self._pool = None

    def replace_dead_workers(self):
        self.logger.debug("TaskPool: Finding dead pool processes...")
        dead = self._pool.replace_dead_workers()
        if dead:
            self.logger.info(
                "TaskPool: Replaced %s dead pool workers..." % (
                    len(dead)))

    def apply_async(self, target, args=None, kwargs=None, callbacks=None,
            errbacks=None, on_ack=None, meta=None):
        """Equivalent of the :func:``apply`` built-in function.

        All ``callbacks`` and ``errbacks`` should complete immediately since
        otherwise the thread which handles the result will get blocked.

        """
        args = args or []
        kwargs = kwargs or {}
        callbacks = callbacks or []
        errbacks = errbacks or []
        meta = meta or {}

        on_return = curry(self.on_return, callbacks, errbacks,
                          on_ack, meta)

        self.logger.debug("TaskPool: Apply %s (args:%s kwargs:%s)" % (
            target, args, kwargs))

        self.replace_dead_workers()

        return self._pool.apply_async(target, args, kwargs,
                                        callback=on_return)

    def on_return(self, callbacks, errbacks, on_ack, meta, ret_value):
        """What to do when the process returns."""

        # Acknowledge the task as being processed.
        if on_ack:
            on_ack()

        self.on_ready(callbacks, errbacks, meta, ret_value)

    def on_ready(self, callbacks, errbacks, meta, ret_value):
        """What to do when a worker task is ready and its return value has
        been collected."""

        if isinstance(ret_value, ExceptionInfo):
            if isinstance(ret_value.exception, (
                    SystemExit, KeyboardInterrupt)):
                raise ret_value.exception
            for errback in errbacks:
                errback(ret_value, meta)
        else:
            for callback in callbacks:
                callback(ret_value, meta)
<|endoftext|>


<commit_before>""" mediacollectiondb database define the media collection datbase
and provides tools to manage the database."""
import sqlite3

class MediaCollection(object):

    """ Persistent media collection, i.e. collection saved into
    SQLite database.

    Attributes:
        connection: sqlite connection.
        cursor: sqlite cursor.
    """

    COLLECTIONS_TABLE = "media_collections"

    def __init__(self, filename):
        self.connection = sqlite3.connect(filename)
        self.cursor = self.connection.cursor()
        table_creation = """CREATE TABLE IF NOT EXISTS %s
                                (date, name, donation, comment)
                        """ % MediaCollection.COLLECTIONS_TABLE
        self.cursor.execute(table_creation)
        self.cursor.commit()

    def add(self, media):
        """ Add media to the persistent media collection.

        Args:
            media: media object to save to the into the persistent collection.
        """
        pass

    def reset(self):
        """ Delete all entry in the database. """
        reset_query = "DELETE * FROM %s" % MediaCollection.COLLECTIONS_TABLE
        self.cursor.execute(reset_query)
<commit_msg>fixing table creation with appropriate database fields<commit_after>""" mediacollectiondb database define the media collection datbase
and provides tools to manage the database."""
import sqlite3

class MediaCollection(object):

    """ Persistent media collection, i.e. collection saved into
    SQLite database.

    Attributes:
        connection: sqlite connection.
        cursor: sqlite cursor.
    """

    COLLECTIONS_TABLE = "media_collections"

    def __init__(self, filename):
        self.connection = sqlite3.connect(filename)
        self.cursor = self.connection.cursor()
        table_creation = """CREATE TABLE IF NOT EXISTS %s
                                (pageid,
                                filename,
                                uploader,
                                width,
                                height,
                                size,
                                quality_image,
                                featured_pictured,
                                valued_image,
                                timestamp)
                        """ % MediaCollection.COLLECTIONS_TABLE
        self.cursor.execute(table_creation)
        self.cursor.commit()

    def add(self, media):
        """ Add media to the persistent media collection.

        Args:
            media: media object to save to the into the persistent collection.
        """
        pass

    def reset(self):
        """ Delete all entry in the database. """
        reset_query = "DELETE * FROM %s" % MediaCollection.COLLECTIONS_TABLE
        self.cursor.execute(reset_query)
<|endoftext|>


<commit_before>#!/usr/bin/env python

#------------------------------------------------------------------------------
# Copyright: 2013
# Author:    Dewey Garrett <dgarrett@panix.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#------------------------------------------------------------------------------

"""
# popup keyboard for use with touchscreen applications
# used by pyngcgui.py
# based on work of John Thornton's Buglump

Optional __init__() args:
  glade_file         (default = 'popupkeyboard.ui')
  dialog_name        (default = 'dialog')     Main window
  num_entry_name     (default = 'num_entry')  Entry for display
  coord_buttons_name (default = 'coords')     Box for coord buttons
  use_coord_buttons  (default = True)         Enable coord buttons

Required objects for glade_file:
  gtk.Window (Main window)
  gtk.Entry  (Entry for display)

Optional objects for galde_file:
  gtk.*Box:  (Box for coord buttons)

All buttons use a single handler named 'on_click'.
The PopupKeyboard class recognizes buttons by their LABEL.

Required button LABELS (case insensitive):
  0,1,2,3,4,5,6,7,8,9,. (numbers and decimal point)
  +/-                   (toggle sign)
  clear                 (clear)
  bs                    (back space)
  save                  (save)
  cancel                (cancel)
Optional button LABELS (case insensitive):
  x,y,z,a,b,c,u,v,w,d   (9 axes plus d for diameter)

"""

import linuxcnc
import sys
import os
import pango

# rip or package, expect __file__ at: lib/python/
mdir = os.path.abspath(os.path.join(
                       os.path.dirname(
                       os.path.realpath(__file__))))
BASE = os.path.abspath(os.path.join(mdir,"../.."))
if (BASE.split('/')[-1] == 'lib'):
    BASE = os.path.abspath(os.path.join(mdir,"../../.."))

g_ui_dir = os.path.join(BASE, "share", "linuxcnc")

try:
    import pygtk
    pygtk.require('2.0')
except:
    pass

try:
    import gtk
except ImportError,msg:
    print('GTK not available: %s' % msg)
    sys.exit(1)

class PopupKeyboard:
    def __init__(self
                ,glade_file=None
                ,dialog_name='dialog'
                ,num_entry_name='num_entry'
                ,coord_buttons_name='coords'
                ,use_coord_buttons=True
                ,theme_name=None
                ):

        if (glade_file is None):
            glade_file = os.path.join(g_ui_dir,'popupkeyboard.ui')

        fontname ='sans 12 bold'
        self.use_coord_buttons = use_coord_buttons

        try:
            import gtk.glade
        except ImportError,detail:
            print 'ImportError:',detail
        except Exception,msg:
            print 'Exception:',Exception
            print sys.exc_info()
            sys.exit(1)

        self.builder = gtk.Builder()
        self.builder.add_from_file(glade_file)
        self.builder.connect_signals(self)

        self.dialog    = self.builder.get_object(dialog_name)
        self.set_theme(theme_name)
        self.num_entry = self.builder.get_object(num_entry_name)
        try:
            self.coord_buttons = self.builder.get_object(coord_buttons_name)
        except:
            self.coord_buttons = None

        self.result = None
        self.location = None

        self.top = self.dialog.get_toplevel()
        self.top.set_title(glade_file)
        self.top.set_keep_above(True)

        if (not self.use_coord_buttons):
            if self.coord_buttons:
                self.coord_buttons.hide()

        # prevent closing of dialog by window manager, escape key , etc
        # http://faq.pygtk.org/index.py?file=faq10.013.htp&req=show
        self.top.connect("response",    self.on_response) #reqd
        self.top.connect("delete-event",self.on_delete)   #reqd
        self.top.connect("close",       self.on_close)    #probably not reqd

        # find buttons with labels XYZABCUVW or D
        # and show iff corresponding axis is in axis_mask
        label_to_btn = {}
        for btn in self.builder.get_objects():
            if type(btn) is not gtk.Button:
                continue
            label_to_btn[btn.get_label().upper()] = btn

            if isinstance(btn.child, gtk.Label):
                lbl = btn.child
                lbl.modify_font(pango.FontDescription(fontname))

        try:
            self.stat = linuxcnc.stat()
            self.stat.poll()
            has_x = False
            for axno in range(0,9):
                axname = 'XYZABCUVW'[axno]
                if label_to_btn.has_key(axname):
                    b = label_to_btn[axname]
                    if bool(self.stat.axis_mask & (1 << axno)):
                        b.show()
                        if axno == 0: has_x = True
                    else:
                        b.hide()
            bdiam = None
            if label_to_btn.has_key('D'):
                bdiam = label_to_btn['D']
            if bdiam and has_x:
                bdiam.show()
            elif bdiam:
                bdiam.hide()
        except linuxcnc.error,msg:
            self.stat = None
            if self.coord_buttons is not None:
                self.coord_buttons.hide()
                print "linuxcnc must be running to use axis keys"
            # continue without buttons for testing when linuxnc not running
        except Exception, err:
            print 'Exception:',Exception
            print sys.exc_info()
            sys.exit(1)
        # making it insensitive clears the inital selection region
        self.num_entry.set_state(gtk.STATE_INSENSITIVE)
        self.num_entry.modify_text(gtk.STATE_INSENSITIVE
                      ,gtk.gdk.color_parse('black'))
        self.num_entry.modify_font(pango.FontDescription(fontname))

    def set_theme(self,tname=None):
        if tname is None:
            return
        screen   = self.dialog.get_screen()
        settings = gtk.settings_get_for_screen(screen)
        settings.set_string_property('gtk-theme-name',tname,"")
        theme    = settings.get_property('gtk-theme-name')

    def on_response(self,widget,response):
        if response < 0:
            widget.emit_stop_by_name('response')
        return True

    def on_delete(self,widget,event): return True

    def on_close(self,widget,event): return True

    def run(self,initial_value='',title=None):
        if title is not None:
            self.top.set_title(title)
        self.num_entry.set_text(str(initial_value))
        if self.location:
            self.dialog.parse_geometry('+%d+%d'
                                      % (self.location[0],self.location[1]))
        self.num_entry.set_position(0)
        self.dialog.run()
        if self.result is None:
            return False #user canceled
        else:
            return True

    def on_click(self, widget, data=None):
        l = widget.get_label()
        e = self.num_entry
        self.label_to_method(l)(e,l)

    def label_to_method(self,l):
        ll = l.lower()
        if ll.find('clear') >= 0:  return self.do_clear
        if ll.find('save')  >= 0:  return self.do_save
        if ll.find('cancel')>= 0:  return self.do_cancel
        if ll.find('+/-')   >= 0:  return self.do_sign
        if ll.find('bs')    >= 0:  return self.do_backspace
        if ll in  ('.0123456789'): return self.do_number
        if ll in  ('xyzabcuvwd'):  return self.do_axis_letter
        return self.do_unknown_label

    def do_unknown_label(self,e,l):
        print 'PopupKeyboard:do_unknown_label: <%s>' % l

    def do_number(self,e,l):
        # not needed if INSENSITIVE:
        # if e.get_selection_bounds(): e.delete_selection()
        e.set_text(e.get_text() + l)

    def do_backspace(self,e,l):
        e.set_text(e.get_text()[:-1])

    def do_sign(self,e,l):
        current = e.get_text()
        if current == '':
            current = '-'
        elif current[0] == '-':
            current = current[1:]
        else:
            current = '-' + current
        e.set_text(current)

    def do_save(self,e,l):
        self.result = e.get_text()
        e.set_text('')
        self.location = self.dialog.get_position()
        self.dialog.hide() # tells dialog.run() to finish

    def do_cancel(self,e,l):
        self.result = None # None means canceled
        self.dialog.hide() # tells dialog.run() to finish

    def do_clear(self,e,l):
        e.set_text('')

    def do_axis_letter(self,e,l):
        if self.stat:
            self.stat.poll()
            e.set_text("%.6g" % self.coord_value(l))
        else:
            print "linuxcnc must be running to use <%s> key" % l

    def get_result(self):
        return(self.result)

    def coord_value(self,char):
        # offset calc copied from emc_interface.py
        # char = 'x' | 'y' | ...
        # 'd' is for diameter
        s = self.stat
        s.poll()
        p = s.position # tuple=(xvalue, yvalue, ...
        return {
                'x': p[0] - s.g5x_offset[0] - s.tool_offset[0],
                'y': p[1] - s.g5x_offset[1] - s.tool_offset[1],
                'z': p[2] - s.g5x_offset[2] - s.tool_offset[2],
                'a': p[3] - s.g5x_offset[3] - s.tool_offset[3],
                'b': p[4] - s.g5x_offset[4] - s.tool_offset[4],
                'c': p[5] - s.g5x_offset[5] - s.tool_offset[5],
                'u': p[6] - s.g5x_offset[6] - s.tool_offset[6],
                'v': p[7] - s.g5x_offset[7] - s.tool_offset[7],
                'w': p[8] - s.g5x_offset[8] - s.tool_offset[8],
                'd':(p[0] - s.g5x_offset[0] - s.tool_offset[0])* 2,#2*R
               }[char.lower()]

if __name__ == "__main__":
    m = PopupKeyboard()
    print "\nClear and Save to end test\n"
    ct = 100
    while True:
        m.run(initial_value=''
             ,title=str(ct)
             )
        result = m.get_result()
        print 'result = <%s>' % result
        if result=='':
            sys.exit(0)
        ct += 1
    gtk.main()
# vim: sts=4 sw=4 et
<commit_msg>popupkeyboard.py: take advantage of linuxcnc.SHARE<commit_after>#!/usr/bin/env python

#------------------------------------------------------------------------------
# Copyright: 2013
# Author:    Dewey Garrett <dgarrett@panix.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#------------------------------------------------------------------------------

"""
# popup keyboard for use with touchscreen applications
# used by pyngcgui.py
# based on work of John Thornton's Buglump

Optional __init__() args:
  glade_file         (default = 'popupkeyboard.ui')
  dialog_name        (default = 'dialog')     Main window
  num_entry_name     (default = 'num_entry')  Entry for display
  coord_buttons_name (default = 'coords')     Box for coord buttons
  use_coord_buttons  (default = True)         Enable coord buttons

Required objects for glade_file:
  gtk.Window (Main window)
  gtk.Entry  (Entry for display)

Optional objects for galde_file:
  gtk.*Box:  (Box for coord buttons)

All buttons use a single handler named 'on_click'.
The PopupKeyboard class recognizes buttons by their LABEL.

Required button LABELS (case insensitive):
  0,1,2,3,4,5,6,7,8,9,. (numbers and decimal point)
  +/-                   (toggle sign)
  clear                 (clear)
  bs                    (back space)
  save                  (save)
  cancel                (cancel)
Optional button LABELS (case insensitive):
  x,y,z,a,b,c,u,v,w,d   (9 axes plus d for diameter)

"""

import linuxcnc
import sys
import os
import pango

g_ui_dir = linuxcnc.SHARE + "/linuxcnc"

try:
    import pygtk
    pygtk.require('2.0')
except:
    pass

try:
    import gtk
except ImportError,msg:
    print('GTK not available: %s' % msg)
    sys.exit(1)

class PopupKeyboard:
    def __init__(self
                ,glade_file=None
                ,dialog_name='dialog'
                ,num_entry_name='num_entry'
                ,coord_buttons_name='coords'
                ,use_coord_buttons=True
                ,theme_name=None
                ):

        if (glade_file is None):
            glade_file = os.path.join(g_ui_dir,'popupkeyboard.ui')

        fontname ='sans 12 bold'
        self.use_coord_buttons = use_coord_buttons

        try:
            import gtk.glade
        except ImportError,detail:
            print 'ImportError:',detail
        except Exception,msg:
            print 'Exception:',Exception
            print sys.exc_info()
            sys.exit(1)

        self.builder = gtk.Builder()
        self.builder.add_from_file(glade_file)
        self.builder.connect_signals(self)

        self.dialog    = self.builder.get_object(dialog_name)
        self.set_theme(theme_name)
        self.num_entry = self.builder.get_object(num_entry_name)
        try:
            self.coord_buttons = self.builder.get_object(coord_buttons_name)
        except:
            self.coord_buttons = None

        self.result = None
        self.location = None

        self.top = self.dialog.get_toplevel()
        self.top.set_title(glade_file)
        self.top.set_keep_above(True)

        if (not self.use_coord_buttons):
            if self.coord_buttons:
                self.coord_buttons.hide()

        # prevent closing of dialog by window manager, escape key , etc
        # http://faq.pygtk.org/index.py?file=faq10.013.htp&req=show
        self.top.connect("response",    self.on_response) #reqd
        self.top.connect("delete-event",self.on_delete)   #reqd
        self.top.connect("close",       self.on_close)    #probably not reqd

        # find buttons with labels XYZABCUVW or D
        # and show iff corresponding axis is in axis_mask
        label_to_btn = {}
        for btn in self.builder.get_objects():
            if type(btn) is not gtk.Button:
                continue
            label_to_btn[btn.get_label().upper()] = btn

            if isinstance(btn.child, gtk.Label):
                lbl = btn.child
                lbl.modify_font(pango.FontDescription(fontname))

        try:
            self.stat = linuxcnc.stat()
            self.stat.poll()
            has_x = False
            for axno in range(0,9):
                axname = 'XYZABCUVW'[axno]
                if label_to_btn.has_key(axname):
                    b = label_to_btn[axname]
                    if bool(self.stat.axis_mask & (1 << axno)):
                        b.show()
                        if axno == 0: has_x = True
                    else:
                        b.hide()
            bdiam = None
            if label_to_btn.has_key('D'):
                bdiam = label_to_btn['D']
            if bdiam and has_x:
                bdiam.show()
            elif bdiam:
                bdiam.hide()
        except linuxcnc.error,msg:
            self.stat = None
            if self.coord_buttons is not None:
                self.coord_buttons.hide()
                print "linuxcnc must be running to use axis keys"
            # continue without buttons for testing when linuxnc not running
        except Exception, err:
            print 'Exception:',Exception
            print sys.exc_info()
            sys.exit(1)
        # making it insensitive clears the inital selection region
        self.num_entry.set_state(gtk.STATE_INSENSITIVE)
        self.num_entry.modify_text(gtk.STATE_INSENSITIVE
                      ,gtk.gdk.color_parse('black'))
        self.num_entry.modify_font(pango.FontDescription(fontname))

    def set_theme(self,tname=None):
        if tname is None:
            return
        screen   = self.dialog.get_screen()
        settings = gtk.settings_get_for_screen(screen)
        settings.set_string_property('gtk-theme-name',tname,"")
        theme    = settings.get_property('gtk-theme-name')

    def on_response(self,widget,response):
        if response < 0:
            widget.emit_stop_by_name('response')
        return True

    def on_delete(self,widget,event): return True

    def on_close(self,widget,event): return True

    def run(self,initial_value='',title=None):
        if title is not None:
            self.top.set_title(title)
        self.num_entry.set_text(str(initial_value))
        if self.location:
            self.dialog.parse_geometry('+%d+%d'
                                      % (self.location[0],self.location[1]))
        self.num_entry.set_position(0)
        self.dialog.run()
        if self.result is None:
            return False #user canceled
        else:
            return True

    def on_click(self, widget, data=None):
        l = widget.get_label()
        e = self.num_entry
        self.label_to_method(l)(e,l)

    def label_to_method(self,l):
        ll = l.lower()
        if ll.find('clear') >= 0:  return self.do_clear
        if ll.find('save')  >= 0:  return self.do_save
        if ll.find('cancel')>= 0:  return self.do_cancel
        if ll.find('+/-')   >= 0:  return self.do_sign
        if ll.find('bs')    >= 0:  return self.do_backspace
        if ll in  ('.0123456789'): return self.do_number
        if ll in  ('xyzabcuvwd'):  return self.do_axis_letter
        return self.do_unknown_label

    def do_unknown_label(self,e,l):
        print 'PopupKeyboard:do_unknown_label: <%s>' % l

    def do_number(self,e,l):
        # not needed if INSENSITIVE:
        # if e.get_selection_bounds(): e.delete_selection()
        e.set_text(e.get_text() + l)

    def do_backspace(self,e,l):
        e.set_text(e.get_text()[:-1])

    def do_sign(self,e,l):
        current = e.get_text()
        if current == '':
            current = '-'
        elif current[0] == '-':
            current = current[1:]
        else:
            current = '-' + current
        e.set_text(current)

    def do_save(self,e,l):
        self.result = e.get_text()
        e.set_text('')
        self.location = self.dialog.get_position()
        self.dialog.hide() # tells dialog.run() to finish

    def do_cancel(self,e,l):
        self.result = None # None means canceled
        self.dialog.hide() # tells dialog.run() to finish

    def do_clear(self,e,l):
        e.set_text('')

    def do_axis_letter(self,e,l):
        if self.stat:
            self.stat.poll()
            e.set_text("%.6g" % self.coord_value(l))
        else:
            print "linuxcnc must be running to use <%s> key" % l

    def get_result(self):
        return(self.result)

    def coord_value(self,char):
        # offset calc copied from emc_interface.py
        # char = 'x' | 'y' | ...
        # 'd' is for diameter
        s = self.stat
        s.poll()
        p = s.position # tuple=(xvalue, yvalue, ...
        return {
                'x': p[0] - s.g5x_offset[0] - s.tool_offset[0],
                'y': p[1] - s.g5x_offset[1] - s.tool_offset[1],
                'z': p[2] - s.g5x_offset[2] - s.tool_offset[2],
                'a': p[3] - s.g5x_offset[3] - s.tool_offset[3],
                'b': p[4] - s.g5x_offset[4] - s.tool_offset[4],
                'c': p[5] - s.g5x_offset[5] - s.tool_offset[5],
                'u': p[6] - s.g5x_offset[6] - s.tool_offset[6],
                'v': p[7] - s.g5x_offset[7] - s.tool_offset[7],
                'w': p[8] - s.g5x_offset[8] - s.tool_offset[8],
                'd':(p[0] - s.g5x_offset[0] - s.tool_offset[0])* 2,#2*R
               }[char.lower()]

if __name__ == "__main__":
    m = PopupKeyboard()
    print "\nClear and Save to end test\n"
    ct = 100
    while True:
        m.run(initial_value=''
             ,title=str(ct)
             )
        result = m.get_result()
        print 'result = <%s>' % result
        if result=='':
            sys.exit(0)
        ct += 1
    gtk.main()
# vim: sts=4 sw=4 et
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import math

__all__ = ["sieve_of_eratosthenes", "prime_factors", "all_factors"]

def sieve_of_eratosthenes(N):
    """
    Return an ascending ordered list of prime numbers up to the given value,
    generated via the sieve of eratosthenes method.

    TODO: Add starting offset so as to be able to get primes within a range
            / add dynamic programming concepts.
    """
    # TODO: Optimization - all primes after 2 are odd, so special case when
    #           N == 2, else loop over and deal only with odd numbers.

    # in case we get N as a string or something
    N = float(N)

    # maps non-prime number to an increment value defining the next value after
    # the key that is not a prime.
    not_primes = {}
    primes = []

    c = 2
    while c <= N:
        if c in not_primes:
            i = not_primes[c]
            # next non-prime number. Need to skip entries already in not_primes
            # so we don't interfere with recorded key-interval pairings
            nnp = c + i
            while (nnp in not_primes) and (nnp < N):
                nnp += i
            # npp may become >N, meaning results of this interval have already
            # been fully covered, and we can safely assign it out of the range
            # of N as we know we won't be looking past N at the most in the map.
            not_primes[nnp] = i
            # Won't be looking at c again, so we can remove it from the map to
            # save memory
            del not_primes[c]
        else:
            primes.append(c)
            # The lowest value that will
            c_squared = c*c
            if c_squared <= N:
                not_primes[c_squared] = c
        c += 1

    return primes

def prime_factors(N):
    """
    Returns ordered ascending prime factors of N.

    TODO: Could add caching of answers for quicker successive responses.
          Could make this recursive and leverage dynamic programming concepts
            -> find a prime factor, call prime_factors on remaining divisor
    """
    sqrt_N = math.sqrt(N)
    # sqrt_N = math.sqrt(N) + math.log10(N)
    primes = sieve_of_eratosthenes(sqrt_N)

    # prime factorize N
    p_factors = []
    remaining = N
    for p in primes:
        # Short-cut loop exit when we know we cant factorize any more
        if remaining == 1:
            break
        while remaining > 1 and ((remaining % p) == 0):
            p_factors.append(p)
            remaining /= p

    if remaining != 1:
        p_factors.append(remaining)

    return p_factors

def factors(N):
    """
    Return divisors/factors of N.
    """
    pf = prime_factors(N)
    # noinspection PySetFunctionToLiteral
    ftors = set([1])

    # Compute product of all primes
    for p in pf:
        new_factors = set()
        # multiply the current prime factor with all current factors in order to
        # iteratively build up component factors of N
        for e in ftors:
            new_factors.add(e * p)
        ftors.update(new_factors)

    # using sorted as we want to return a list, and directly converting a set
    # to a list causes things to be out of sequential order.
    return sorted(ftors)

def factor_pairs(N):
    """
    Return a list of factor pairs of N, whose individual product produce N.
    """
    ftors = factors(N)
    # since what comes out of factors is sorted, step indices from the edges
    # towards the center, pairing the edges to produce factor pairs.
    i = 0
    j = len(ftors) - 1
    pairs = []
    while i <= j:
        pairs.append((ftors[i], ftors[j]))
        i += 1
        j -= 1
    return pairs<commit_msg>fix factors module __all__ value<commit_after># -*- coding: utf-8 -*-

import math

__all__ = ["sieve_of_eratosthenes", "prime_factors", "factors", "factor_pairs"]

def sieve_of_eratosthenes(N):
    """
    Return an ascending ordered list of prime numbers up to the given value,
    generated via the sieve of eratosthenes method.

    TODO: Add starting offset so as to be able to get primes within a range
            / add dynamic programming concepts.
    """
    # TODO: Optimization - all primes after 2 are odd, so special case when
    #           N == 2, else loop over and deal only with odd numbers.

    # in case we get N as a string or something
    N = float(N)

    # maps non-prime number to an increment value defining the next value after
    # the key that is not a prime.
    not_primes = {}
    primes = []

    c = 2
    while c <= N:
        if c in not_primes:
            i = not_primes[c]
            # next non-prime number. Need to skip entries already in not_primes
            # so we don't interfere with recorded key-interval pairings
            nnp = c + i
            while (nnp in not_primes) and (nnp < N):
                nnp += i
            # npp may become >N, meaning results of this interval have already
            # been fully covered, and we can safely assign it out of the range
            # of N as we know we won't be looking past N at the most in the map.
            not_primes[nnp] = i
            # Won't be looking at c again, so we can remove it from the map to
            # save memory
            del not_primes[c]
        else:
            primes.append(c)
            # The lowest value that will
            c_squared = c*c
            if c_squared <= N:
                not_primes[c_squared] = c
        c += 1

    return primes

def prime_factors(N):
    """
    Returns ordered ascending prime factors of N.

    TODO: Could add caching of answers for quicker successive responses.
          Could make this recursive and leverage dynamic programming concepts
            -> find a prime factor, call prime_factors on remaining divisor
    """
    sqrt_N = math.sqrt(N)
    # sqrt_N = math.sqrt(N) + math.log10(N)
    primes = sieve_of_eratosthenes(sqrt_N)

    # prime factorize N
    p_factors = []
    remaining = N
    for p in primes:
        # Short-cut loop exit when we know we cant factorize any more
        if remaining == 1:
            break
        # factory out
        while remaining > 1 and ((remaining % p) == 0):
            p_factors.append(p)
            remaining /= p

    if remaining != 1:
        p_factors.append(remaining)

    return p_factors

def factors(N):
    """
    Return divisors/factors of N.
    """
    pf = prime_factors(N)
    # noinspection PySetFunctionToLiteral
    ftors = set([1])

    # Compute product of all primes
    for p in pf:
        new_factors = set()
        # multiply the current prime factor with all current factors in order to
        # iteratively build up component factors of N
        for e in ftors:
            new_factors.add(e * p)
        ftors.update(new_factors)

    # using sorted as we want to return a list, and directly converting a set
    # to a list causes things to be out of sequential order.
    return sorted(ftors)

def factor_pairs(N):
    """
    Return a list of factor pairs of N, whose individual product produce N.
    """
    ftors = factors(N)
    # since what comes out of factors is sorted, step indices from the edges
    # towards the center, pairing the edges to produce factor pairs.
    i = 0
    j = len(ftors) - 1
    pairs = []
    while i <= j:
        pairs.append((ftors[i], ftors[j]))
        i += 1
        j -= 1
    return pairs<|endoftext|>


<commit_before>import errno
import io
import json
import logging
import os
import sys
import time
from datetime import datetime
from hashlib import md5
from optparse import OptionParser

from memento_damage.damage_analysis import MementoDamageAnalysis

base_dir = os.path.join(os.path.dirname(__file__))
base_dir = os.path.abspath(base_dir)
sys.path.insert(0, base_dir)
from memento_damage.tools import Command, rmdir_recursive

class MementoDamage(object):
    # Define all output files
    APP_LOG_FILE_NAME = 'app.log'
    HTML_FILE_NAME = 'source.html'
    NETWORK_LOG_FILE_NAME = 'network.log'
    IMAGE_LOG_FILE_NAME = 'image.log'
    CSS_LOG_FILE_NAME = 'css.log'
    VIDEO_LOG_FILE_NAME = 'video.log'
    SCREENSHOT_FILE_NAME = 'screenshot.png'
    JSON_RESULT_FILE_NAME = 'result.json'

    background_color = 'FFFFFF'

    _crawljs_script = os.path.join(base_dir, 'phantomjs', 'crawl.js')
    _debug = False
    _info = False
    _mode = 'simple'
    _follow_redirection = False
    _clean_cache = True

    _result = None

    def __init__(self, uri, output_dir, options={}):
        self.uri = str(uri)
        self.output_dir = output_dir

        # Initialize variable
        self.app_log_file = os.path.join(self.output_dir, self.APP_LOG_FILE_NAME)
        self.html_file = os.path.join(self.output_dir, self.HTML_FILE_NAME)
        self.network_log_file = os.path.join(self.output_dir, self.NETWORK_LOG_FILE_NAME)
        self.image_log_file = os.path.join(self.output_dir, self.IMAGE_LOG_FILE_NAME)
        self.css_log_file = os.path.join(self.output_dir, self.CSS_LOG_FILE_NAME)
        self.video_log_file = os.path.join(self.output_dir, self.VIDEO_LOG_FILE_NAME)
        self.screenshot_file = os.path.join(self.output_dir, self.SCREENSHOT_FILE_NAME)
        self.json_result_file = os.path.join(self.output_dir, self.JSON_RESULT_FILE_NAME)

        # options
        if 'debug' in options: self._debug = options['debug']
        if 'info' in options: self._info = options['info']
        if 'mode' in options: self._mode = options['mode']
        if 'redirect' in options: self._follow_redirection = options['redirect']
        if 'clean_cache' in options: self._clean_cache = options['clean_cache']

        # Setup logger --> to show debug verbosity
        log_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')

        # To stdout
        log_stdout_handler = logging.StreamHandler(sys.stdout)
        log_stdout_handler.setFormatter(log_formatter)

        # To file
        log_file_handler = logging.FileHandler(self.app_log_file)
        log_file_handler.setFormatter(log_formatter)

        # Configure logger
        self.logger = logging.getLogger(uri)
        self.logger.addHandler(log_file_handler)
        self.logger.addHandler(log_stdout_handler)

        self.setup_logger()

    def setup_logger(self):
        if self._info:
            self.logger.setLevel(logging.INFO)
        elif self._debug:
            self.logger.setLevel(logging.DEBUG)
        else:
            self.logger.setLevel(logging.ERROR)

    def log_stdout(self, out, write_fn):
        if out and hasattr(out, 'readline'):
            for line in iter(out.readline, b''):
                line = line.strip()
                write_fn(line)
        else:
            write_fn(out)

    def log_output(self, msg):
        if self.logger.level == logging.DEBUG: self.logger.debug(msg)
        elif self.logger.level == logging.INFO: self.logger.info(msg)

        if 'background_color' in msg:
            msg = json.loads(msg)
            if msg['background_color']:
                self.background_color = msg['background_color']

        if 'crawl_result' in msg:
            msg = json.loads(msg)
            crawl_result = msg['crawl_result']
            self._crawl_result = crawl_result

            if crawl_result['error']:
                self.response_time = datetime.now()

                if self._mode == 'simple':
                    self.logger.error(crawl_result['message'])
                elif self._mode == 'json':
                    self.logger.error(json.dumps(crawl_result, indent=4))
                else:
                    self.logger.error('Choose mode "simple" or "json"')

    def log_stderr(self, out, write_fn):
        if out and hasattr(out, 'readline'):
            for line in iter(out.readline, b''):
                line = line.strip()
                write_fn(line)
        else:
            write_fn(out)

    def log_error(self, msg):
        self.logger.error(msg)

    def run(self):
        self.request_time = datetime.now()

        # Crawl page with phantomjs crawl.js via arguments
        # Equivalent with console:
        phantomjs = os.getenv('PHANTOMJS', 'phantomjs')

        pjs_cmd = [phantomjs, '--ssl-protocol=any', self._crawljs_script, self.uri, self.output_dir,
                   str(self._follow_redirection), str(self.logger.level)]
        cmd = Command(pjs_cmd, pipe_stdout_callback=self.log_stdout, pipe_stderr_callback=self.log_stderr)
        err_code = cmd.run(10 * 60,
                           stdout_callback_args=(self.log_output, ),
                           stderr_callback_args=(self.log_error, ))

        if err_code != 0:
            self.log_error('Application closed unexpectedly')
            self._do_clean_cache()
            return

        # get result of damage analysis
        self._result = self._do_analysis()
        self.response_time = datetime.now()

        self._result['message'] = 'Calculation is finished in {} seconds'.format(
            (self.response_time - self.request_time).seconds)
        self._result['timer'] = {
            'request_time': (self.request_time - datetime(1970, 1, 1)).total_seconds(),
            'response_time': (self.response_time - datetime(1970, 1, 1)).total_seconds()
        }
        self._result['calculation_time'] = (self.response_time - self.request_time).seconds

        # Save output
        io.open(self.json_result_file, 'wb').write(json.dumps(self._result))

        self._do_clean_cache()
        return self._result

    def get_result(self):
        return self._result

    def print_result(self):
        # Print total damage
        if self._result:
            if self._mode == 'simple':
                print('Total damage of {} is {}'.format(self.uri, str(self._result['total_damage'])))
            elif self._mode == 'json':
                print(json.dumps(self._result, indent=4))
            else:
                self.logger.error('Choose mode "simple" or "json"')

    def _do_analysis(self):
        # Calculate damage
        analysis = MementoDamageAnalysis(self)
        analysis.run()

        return analysis.get_result()

    def _do_clean_cache(self):
        # Remove cache directory
        if self._clean_cache:
            time.sleep(3)
            rmdir_recursive(self.output_dir)

    def set_show_debug_message(self):
        self._debug = True
        self.setup_logger()

    def set_show_info_message(self):
        self._info = True
        self.setup_logger()

    def set_output_mode_json(self):
        self._mode = 'json'

    def set_follow_redirection(self):
        self._follow_redirection = True

    def set_dont_clean_cache_on_finish(self):
        self._clean_cache = False

def main():
    parser = OptionParser()
    parser.set_usage(parser.get_usage().replace('\n', '') + ' <URI> <output_dir>')
    parser.add_option("-m", "--mode",
                      dest="mode", default="simple",
                      help="output mode: simple or json [default: %default]")
    parser.add_option("-d", "--debug",
                      action="store_true", dest="debug", default=False,
                      help="print debug messages")
    parser.add_option("-i", "--info",
                      action="store_true", dest="info", default=False,
                      help="print info messages")
    parser.add_option("-L", "--redirect",
                      action="store_true", dest="redirect", default=False,
                      help="follow url redirection")
    parser.add_option("-k", "--keep-cache",
                      action="store_false", dest="clean_cache", default=True,
                      help="dont clean cache after process finished")

    (options, args) = parser.parse_args()
    options = vars(options)

    if len(args) < 2:
        parser.print_help()
        exit()

    uri = args[0]
    hashed_url = md5(uri).hexdigest()
    output_dir = args[1]

    if not os.path.isabs(output_dir):
        output_dir = os.path.join(os.getcwd(), output_dir)
        output_dir = os.path.abspath(output_dir)

    output_dir = os.path.join(output_dir, hashed_url)

    try:
        os.makedirs(output_dir)
    except OSError, e:
        if e.errno != errno.EEXIST:
            raise

    damage = MementoDamage(uri, output_dir, options)
    damage.run()
    damage.print_result()

if __name__ == "__main__":
    main()<commit_msg>Remove option -k, add option -O<commit_after>import errno
import io
import json
import logging
import os
import sys
import tempfile
import time
from datetime import datetime
from hashlib import md5
from optparse import OptionParser

from memento_damage.damage_analysis import MementoDamageAnalysis

base_dir = os.path.join(os.path.dirname(__file__))
base_dir = os.path.abspath(base_dir)
sys.path.insert(0, base_dir)
from memento_damage.tools import Command, rmdir_recursive

class MementoDamage(object):
    # Define all output files
    APP_LOG_FILE_NAME = 'app.log'
    HTML_FILE_NAME = 'source.html'
    NETWORK_LOG_FILE_NAME = 'network.log'
    IMAGE_LOG_FILE_NAME = 'image.log'
    CSS_LOG_FILE_NAME = 'css.log'
    VIDEO_LOG_FILE_NAME = 'video.log'
    SCREENSHOT_FILE_NAME = 'screenshot.png'
    JSON_RESULT_FILE_NAME = 'result.json'

    background_color = 'FFFFFF'

    _crawljs_script = os.path.join(base_dir, 'phantomjs', 'crawl.js')
    _debug = False
    _info = False
    _mode = 'simple'
    _follow_redirection = False
    _clean_cache = True

    _result = None

    def __init__(self, uri, output_dir, options={}):
        self.uri = str(uri)
        self.output_dir = output_dir

        # Initialize variable
        self.app_log_file = os.path.join(self.output_dir, self.APP_LOG_FILE_NAME)
        self.html_file = os.path.join(self.output_dir, self.HTML_FILE_NAME)
        self.network_log_file = os.path.join(self.output_dir, self.NETWORK_LOG_FILE_NAME)
        self.image_log_file = os.path.join(self.output_dir, self.IMAGE_LOG_FILE_NAME)
        self.css_log_file = os.path.join(self.output_dir, self.CSS_LOG_FILE_NAME)
        self.video_log_file = os.path.join(self.output_dir, self.VIDEO_LOG_FILE_NAME)
        self.screenshot_file = os.path.join(self.output_dir, self.SCREENSHOT_FILE_NAME)
        self.json_result_file = os.path.join(self.output_dir, self.JSON_RESULT_FILE_NAME)

        # options
        if 'debug' in options: self._debug = options['debug']
        if 'info' in options: self._info = options['info']
        if 'mode' in options: self._mode = options['mode']
        if 'redirect' in options: self._follow_redirection = options['redirect']

        # Setup logger --> to show debug verbosity
        log_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')

        # To stdout
        log_stdout_handler = logging.StreamHandler(sys.stdout)
        log_stdout_handler.setFormatter(log_formatter)

        # To file
        log_file_handler = logging.FileHandler(self.app_log_file)
        log_file_handler.setFormatter(log_formatter)

        # Configure logger
        self.logger = logging.getLogger(uri)
        self.logger.addHandler(log_file_handler)
        self.logger.addHandler(log_stdout_handler)

        self.setup_logger()

    def setup_logger(self):
        if self._info:
            self.logger.setLevel(logging.INFO)
        elif self._debug:
            self.logger.setLevel(logging.DEBUG)
        else:
            self.logger.setLevel(logging.ERROR)

    def log_stdout(self, out, write_fn):
        if out and hasattr(out, 'readline'):
            for line in iter(out.readline, b''):
                line = line.strip()
                write_fn(line)
        else:
            write_fn(out)

    def log_output(self, msg):
        if self.logger.level == logging.DEBUG: self.logger.debug(msg)
        elif self.logger.level == logging.INFO: self.logger.info(msg)

        if 'background_color' in msg:
            msg = json.loads(msg)
            if msg['background_color']:
                self.background_color = msg['background_color']

        if 'crawl_result' in msg:
            msg = json.loads(msg)
            crawl_result = msg['crawl_result']
            self._crawl_result = crawl_result

            if crawl_result['error']:
                self.response_time = datetime.now()

                if self._mode == 'simple':
                    self.logger.error(crawl_result['message'])
                elif self._mode == 'json':
                    self.logger.error(json.dumps(crawl_result, indent=4))
                else:
                    self.logger.error('Choose mode "simple" or "json"')

    def log_stderr(self, out, write_fn):
        if out and hasattr(out, 'readline'):
            for line in iter(out.readline, b''):
                line = line.strip()
                write_fn(line)
        else:
            write_fn(out)

    def log_error(self, msg):
        self.logger.error(msg)

    def run(self):
        self.request_time = datetime.now()

        # Crawl page with phantomjs crawl.js via arguments
        # Equivalent with console:
        phantomjs = os.getenv('PHANTOMJS', 'phantomjs')

        pjs_cmd = [phantomjs, '--ssl-protocol=any', self._crawljs_script, self.uri, self.output_dir,
                   str(self._follow_redirection), str(self.logger.level)]
        cmd = Command(pjs_cmd, pipe_stdout_callback=self.log_stdout, pipe_stderr_callback=self.log_stderr)
        err_code = cmd.run(10 * 60,
                           stdout_callback_args=(self.log_output, ),
                           stderr_callback_args=(self.log_error, ))

        if err_code != 0:
            self.log_error('Application closed unexpectedly')
            self._do_clean_cache()
            return

        # get result of damage analysis
        self._result = self._do_analysis()
        self.response_time = datetime.now()

        self._result['message'] = 'Calculation is finished in {} seconds'.format(
            (self.response_time - self.request_time).seconds)
        self._result['timer'] = {
            'request_time': (self.request_time - datetime(1970, 1, 1)).total_seconds(),
            'response_time': (self.response_time - datetime(1970, 1, 1)).total_seconds()
        }
        self._result['calculation_time'] = (self.response_time - self.request_time).seconds

        # Save output
        io.open(self.json_result_file, 'wb').write(json.dumps(self._result))

        self._do_clean_cache()
        return self._result

    def get_result(self):
        return self._result

    def print_result(self):
        # Print total damage
        if self._result:
            if self._mode == 'simple':
                print('Total damage of {} is {}'.format(self.uri, str(self._result['total_damage'])))
            elif self._mode == 'json':
                print(json.dumps(self._result, indent=4))
            else:
                self.logger.error('Choose mode "simple" or "json"')

    def _do_analysis(self):
        # Calculate damage
        analysis = MementoDamageAnalysis(self)
        analysis.run()

        return analysis.get_result()

    def _do_clean_cache(self):
        # Remove cache directory
        if self._clean_cache:
            time.sleep(3)
            rmdir_recursive(self.output_dir)

    def set_show_debug_message(self):
        self._debug = True
        self.setup_logger()

    def set_show_info_message(self):
        self._info = True
        self.setup_logger()

    def set_output_mode_json(self):
        self._mode = 'json'

    def set_follow_redirection(self):
        self._follow_redirection = True

    def set_dont_clean_cache_on_finish(self):
        self._clean_cache = False

def main():
    parser = OptionParser()
    parser.set_usage(parser.get_usage().replace('\n', '') + ' <URI>')
    parser.add_option("-m", "--mode",
                      dest="mode", default="simple",
                      help="output mode: simple or json [default: %default]")
    parser.add_option("-d", "--debug",
                      action="store_true", dest="debug", default=False,
                      help="print debug messages")
    parser.add_option("-i", "--info",
                      action="store_true", dest="info", default=False,
                      help="print info messages")
    parser.add_option("-L", "--redirect",
                      action="store_true", dest="redirect", default=False,
                      help="follow url redirection")
    parser.add_option("-O", "--output-dir",
                      dest="output_dir", default=None,
                      help="output directory (optional)")

    (options, args) = parser.parse_args()
    options = vars(options)

    if len(args) < 1:
        parser.print_help()
        exit()

    uri = args[0]

    is_use_tempdir = False
    # If option -O is provided, use it
    if options['output_dir']:
        output_dir = options['output_dir']
        if not os.path.isabs(output_dir):
            output_dir = os.path.join(os.getcwd(), output_dir)
            output_dir = os.path.abspath(output_dir)
    # Otherwise make temp dir
    else:
        output_dir = tempfile.mkdtemp()
        is_use_tempdir = True

    hashed_url = md5(uri).hexdigest()
    output_dir = os.path.join(output_dir, hashed_url)

    # Make output_dir recursive
    try:
        os.makedirs(output_dir)
    except OSError, e:
        if e.errno != errno.EEXIST: raise

    # Instantiate and run
    damage = MementoDamage(uri, output_dir, options)
    if not is_use_tempdir:
        damage.set_dont_clean_cache_on_finish()
    damage.run()
    damage.print_result()

if __name__ == "__main__":
    main()<|endoftext|>


<commit_before>#!/usr/bin/env python3

from collections import defaultdict, OrderedDict
from typing import Iterable, Set

class DataToken:
    def __init__(self, offset, word, lowercase, pos_tag, mwe_tag, parent_offset, strength, supersense, sentence_id):
        self.offset = int(offset)
        self.word = word
        self.lowercase_lemma = lowercase
        self.pos_tag = pos_tag
        self.mwe_tag = mwe_tag
        self.parent_offset = None if parent_offset == '0' else int(parent_offset)
        self.strength = strength
        self.supersense = supersense
        self.sentence_id = sentence_id

class DataSentence:
    def __init__(self):
        self._tokens: OrderedDict = OrderedDict()
        self.sentence_id = None

    def __iter__(self) -> Iterable[DataToken]:
        return iter(self._tokens.values())

    def __getitem__(self, item) -> DataToken:
        return self._tokens[item]

    def __bool__(self) -> bool:
        return bool(self._tokens)

    def append(self, token: DataToken):
        if not self.sentence_id:
            self.sentence_id = token.sentence_id
        else:
            if token.sentence_id != self.sentence_id:
                raise ValueError("sentence IDs do not match: {} (sentence) != {} (token)".format(self.sentence_id,
                                                                                token.sentence_id))
        self._tokens[int(token.offset)] = token

    @property
    def sentence(self) -> str:
        return ' '.join(map(lambda t: t.word, self._tokens.values()))

class EmissionProbabilities:
    def __init__(self, features: Set[str]):
        self.counts = {feature: 0 for feature in features}
        self.total_count = 0

    def __getitem__(self, feature: str):
        return self.get_probability(feature)

    def add_count(self, feature: str, count=1):
        self.counts[feature] += count
        self.total_count += 1

    def get_probability(self, feature: str):
        return self.counts[feature] / self.total_count

    def __str__(self):
        lines = []
        for feature in self.counts:
            lines.append(f'{round(self.get_probability(feature), 4):>8} : {feature}')
        return '\n'.join(lines)

class HMM:
    def __init__(self, features: Set[str]):
        self.features = features
        self.emissions = {}

    def get_features_for_token(self, token: DataToken) -> Iterable[str]:
        features = []
        mwe_feature = f'{token.mwe_tag}_mwe'
        features.append(mwe_feature)
        return features

    def train(self, sentences: Iterable[DataSentence]):
        for sentence in sentences:
            for token in sentence:
                for feature in self.get_features_for_token(token):
                    ep = self.emissions.get(token.lowercase_lemma)
                    if ep is None:
                        ep = EmissionProbabilities(self.features)
                    ep.add_count(feature)

    def test(self, sentences: Iterable[DataSentence]):
        ...

def generate_features(sentences: DataSentence):
    features = set()
    features.add('O_mwe')
    features.add('o_mwe')
    features.add('B_mwe')
    features.add('b_mwe')
    features.add('I_mwe')
    features.add('i_mwe')
    return features

def read_data_file(datafile: str) -> Iterable[DataSentence]:
    data_sentences = defaultdict(DataSentence)
    with open(datafile) as df:
        for line in df:
            if not line.isspace():
                # Strip extraneous whitespace from the line, then split it on the tabs. There should be exactly enough
                # fields to fill the DataToken.
                token = DataToken(*(line.strip(' \n').split('\t')))
                data_sentences[token.sentence_id].append(token)
    return data_sentences.values()

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('training-data', help='the datafile to train with')
    parser.add_argument('testing-data', help='the datafile to test with')
    args = parser.parse_args()

    # Read the data.
    training_sentences = read_data_file(args.training_data)
    testing_sentences = read_data_file(args.testing_data)
<commit_msg>Handle MWE and SS emissions separately<commit_after>#!/usr/bin/env python3

from collections import defaultdict, OrderedDict
from typing import Iterable, Set

class DataToken:
    def __init__(self, offset, word, lowercase, pos_tag, mwe_tag, parent_offset, strength, supersense, sentence_id):
        self.offset = int(offset)
        self.word = word
        self.lowercase_lemma = lowercase
        self.pos_tag = pos_tag
        self.mwe_tag = mwe_tag
        self.parent_offset = None if parent_offset == '0' else int(parent_offset)
        self.strength = strength
        self.supersense = supersense
        self.sentence_id = sentence_id

class DataSentence:
    def __init__(self):
        self._tokens: OrderedDict = OrderedDict()
        self.sentence_id = None

    def __iter__(self) -> Iterable[DataToken]:
        return iter(self._tokens.values())

    def __getitem__(self, item) -> DataToken:
        return self._tokens[item]

    def __bool__(self) -> bool:
        return bool(self._tokens)

    def append(self, token: DataToken):
        if not self.sentence_id:
            self.sentence_id = token.sentence_id
        else:
            if token.sentence_id != self.sentence_id:
                raise ValueError("sentence IDs do not match: {} (sentence) != {} (token)".format(self.sentence_id,
                                                                                token.sentence_id))
        self._tokens[int(token.offset)] = token

    @property
    def sentence(self) -> str:
        return ' '.join(map(lambda t: t.word, self._tokens.values()))

class EmissionProbabilities:
    def __init__(self, features: Set[str]):
        self.counts = {feature: 0 for feature in features}
        self.total_count = 0

    def __getitem__(self, feature: str):
        return self.get_probability(feature)

    def add_count(self, feature: str, count=1):
        self.counts[feature] += count
        self.total_count += 1

    def get_probability(self, feature: str):
        return self.counts[feature] / self.total_count

    def __str__(self):
        lines = []
        for feature in self.counts:
            lines.append(f'{round(self.get_probability(feature), 4):>8} : {feature}')
        return '\n'.join(lines)

class HMM:
    def __init__(self, mwe_features: Set[str], ss_features: Set[str]):
        self.mwe_features = mwe_features
        self.ss_features = ss_features
        self.mwe_emissions = {}
        self.ss_emissions = {}

    def train(self, sentences: Iterable[DataSentence]):
        for sentence in sentences:
            for token in sentence:
                # Count emission probability for MWE.
                mwe_feature = token.mwe_tag
                mwe_ep = self.mwe_emissions.get(token.lowercase_lemma)
                if mwe_ep is None:
                    mwe_ep = EmissionProbabilities(self.mwe_features)
                    self.mwe_emissions[token.lowercase_lemma] = mwe_ep
                mwe_ep.add_count(mwe_feature)
                # Count emission probability for SS.
                ss_feature = token.supersense
                ss_ep = self.ss_emissions.get(token.lowercase_lemma)
                if ss_ep is None:
                    ss_ep = EmissionProbabilities(self.ss_features)
                    self.ss_emissions[token.lowercase_lemma] = ss_ep
                ss_ep.add_count(ss_feature)

    def test(self, sentences: Iterable[DataSentence]):
        pass

MWE_FEATURES = {'O', 'o', 'B', 'b', 'I', 'i'}

def generate_ss_features(sentences: Iterable[DataSentence]) -> Set[str]:
    ss_features = set()
    for sentence in sentences:
        for token in sentence:
            ss_features.add(token.supersense)
    return ss_features

def read_data_file(datafile: str) -> Iterable[DataSentence]:
    data_sentences = defaultdict(DataSentence)
    with open(datafile) as df:
        for line in df:
            if not line.isspace():
                # Strip extraneous whitespace from the line, then split it on the tabs. There should be exactly enough
                # fields to fill the DataToken.
                token = DataToken(*(line.strip(' \n').split('\t')))
                data_sentences[token.sentence_id].append(token)
    return data_sentences.values()

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('training-data', help='the datafile to train with')
    parser.add_argument('testing-data', help='the datafile to test with')
    args = parser.parse_args()

    # Read the data.
    training_sentences = read_data_file(args.training_data)
    testing_sentences = read_data_file(args.testing_data)

    # Generate Features
    user_mwe_features = MWE_FEATURES
    user_ss_features = generate_ss_features(training_sentences)

    # Build model.
    hmm = HMM(user_mwe_features, user_ss_features)
    hmm.train(training_sentences)

    # Test model.
    hmm.test(testing_sentences)
<|endoftext|>


<commit_before>import bpy

class Scene:
    """Scene object"""
    def __init__(self, filepath, render_engine='CYCLES'):
        self.filepath = filepath
        self.render_engine = 'CYCLES'

    def setup(self):
        self._cleanup()

        bpy.context.scene.render.filepath = self.filepath
        bpy.context.scene.render.engine = self.render_engine
        bpy.context.scene.frame_start = 1
        bpy.context.scene.frame_end = 1

    def render(self, samples=50):
        bpy.context.scene.cycles.samples = samples
        bpy.ops.render.render(animation=True)

    def _cleanup(self):
        """Delete everything"""
        bpy.ops.object.delete(use_global=False)
<commit_msg>Allow to setup scene render resolution<commit_after>import bpy

class Scene:
    """Scene object"""
    def __init__(self, resolution_x, resolution_y, filepath, render_engine='CYCLES'):
        self.resolution_x = resolution_x
        self.resolution_y = resolution_y
        self.filepath = filepath
        self.render_engine = 'CYCLES'

    def setup(self):
        self._cleanup()

        bpy.context.scene.render.filepath = self.filepath
        bpy.context.scene.render.engine = self.render_engine
        bpy.context.scene.frame_start = 1
        bpy.context.scene.frame_end = 1

        bpy.context.scene.render.resolution_x = self.resolution_x
        bpy.context.scene.render.resolution_y = self.resolution_y

    def render(self, samples=50):
        bpy.context.scene.cycles.samples = samples
        bpy.ops.render.render(animation=True)

    def _cleanup(self):
        """Delete everything"""
        bpy.ops.object.delete(use_global=False)
<|endoftext|>