<commit_before>"""Modifiable runtime configuration parameters."""

from __future__ import absolute_import, division, print_function
import argparse
import logging

from dwi.files import Path
import dwi.util

# TODO: Read from file.

# Default runtime configuration parameters. Somewhat similar to matplotlib.
rcParamsDefault = {
    'texture.methods': [
        'stats',
        # 'haralick',
        # 'moment',
        # 'haralick_mbb',
        'glcm',
        # 'glcm_mbb',
        'lbp',
        'hog',
        'gabor',
        'haar',
        'hu',
        'zernike',
        'sobel',
        # 'stats_all',
        ],
    'texture.winsizes.large': (3, 36, 4),  # T2, T2w.
    'texture.winsizes.small': (3, 16, 2),  # DWI.
    'texture.avg': False,  # Boolean: average result texture map?
    'texture.path': None,  # Write result directly to disk, if string.
    'texture.dtype': 'float32',  # Output texture map type.
    'texture.glcm.names': ('contrast', 'dissimilarity', 'homogeneity',
                           'energy', 'correlation', 'ASM'),
    'texture.glcm.distances': (1, 2, 3, 4),  # GLCM pixel distances.
    'texture.gabor.orientations': 4,  # Number of orientations.
    # 'texture.gabor.orientations': 6,  # Number of orientations.
    'texture.gabor.sigmas': (1, 2, 3),
    # 'texture.gabor.sigmas': (None,),
    'texture.gabor.freqs': (0.1, 0.2, 0.3, 0.4, 0.5),
    'texture.lbp.neighbours': 8,  # Number of neighbours.
    'texture.zernike.degree': 8,  # Maximum degree.
    'texture.haar.levels': 4,  # Numer of levels.
    'texture.hog.orientations': 1,  # Numer of orientations.
    }
rcParams = dict(rcParamsDefault)

def rcdefaults():
    """Restore default rc params."""
    rcParams.update(rcParamsDefault)

def get_config_paths():
    """Return existing default configuration files."""
    dirnames = ['/etc/dwilib', '~/.config/dwilib', '.']
    filename = 'dwilib.cfg'
    paths = [Path(x).expanduser() / filename for x in dirnames]
    paths = [x for x in paths if x.exists()]
    return paths

def parse_config(parser):
    """Parse configuration files."""
    prefix = parser.fromfile_prefix_chars[0]
    args = ['{}{}'.format(prefix, x) for x in get_config_paths()]
    namespace, _ = parser.parse_known_args(args)
    return namespace

def convert_arg_line_to_args(line):
    """A better replacement for ArgumentParser."""
    COMMENT_PREFIX = '#'
    line = line.split(COMMENT_PREFIX, 1)[0].strip()
    return line.split()

def get_parser(**kwargs):
    """Get an argument parser with the usual standard arguments ready. Function
    'add' is added for convenience.
    """
    p = argparse.ArgumentParser(fromfile_prefix_chars='@', **kwargs)
    p.convert_arg_line_to_args = convert_arg_line_to_args
    p.add = p.add_argument
    p.add('-v', '--verbose', action='count', default=0,
          help='increase verbosity')
    p.add('--logfile', help='log file')
    p.add('--loglevel', default='WARNING', help='log level name')
    return p

def init_logging(args):
    """Initialize logging."""
    logging.basicConfig(filename=args.logfile,
                        level=dwi.util.get_loglevel(args.loglevel))

def parse_args(parser):
    """Parse args and configuration as well."""
    namespace = parse_config(parser)
    args = parser.parse_args(namespace=namespace)
    dwi.conf.init_logging(args)
    return args
<commit_msg>support py3.4 (no pathlib.Path.expanduser())<commit_after>"""Modifiable runtime configuration parameters."""

from __future__ import absolute_import, division, print_function
import argparse
import logging
import os.path

from dwi.files import Path
import dwi.util

# TODO: Read from file.

# Default runtime configuration parameters. Somewhat similar to matplotlib.
rcParamsDefault = {
    'texture.methods': [
        'stats',
        # 'haralick',
        # 'moment',
        # 'haralick_mbb',
        'glcm',
        # 'glcm_mbb',
        'lbp',
        'hog',
        'gabor',
        'haar',
        'hu',
        'zernike',
        'sobel',
        # 'stats_all',
        ],
    'texture.winsizes.large': (3, 36, 4),  # T2, T2w.
    'texture.winsizes.small': (3, 16, 2),  # DWI.
    'texture.avg': False,  # Boolean: average result texture map?
    'texture.path': None,  # Write result directly to disk, if string.
    'texture.dtype': 'float32',  # Output texture map type.
    'texture.glcm.names': ('contrast', 'dissimilarity', 'homogeneity',
                           'energy', 'correlation', 'ASM'),
    'texture.glcm.distances': (1, 2, 3, 4),  # GLCM pixel distances.
    'texture.gabor.orientations': 4,  # Number of orientations.
    # 'texture.gabor.orientations': 6,  # Number of orientations.
    'texture.gabor.sigmas': (1, 2, 3),
    # 'texture.gabor.sigmas': (None,),
    'texture.gabor.freqs': (0.1, 0.2, 0.3, 0.4, 0.5),
    'texture.lbp.neighbours': 8,  # Number of neighbours.
    'texture.zernike.degree': 8,  # Maximum degree.
    'texture.haar.levels': 4,  # Numer of levels.
    'texture.hog.orientations': 1,  # Numer of orientations.
    }
rcParams = dict(rcParamsDefault)

def rcdefaults():
    """Restore default rc params."""
    rcParams.update(rcParamsDefault)

def get_config_paths():
    """Return existing default configuration files."""
    dirnames = ['/etc/dwilib', '~/.config/dwilib', '.']
    # Py3.4 pathlib has no expanduser().
    dirnames = [os.path.expanduser(x) for x in dirnames]
    filename = 'dwilib.cfg'
    # paths = [Path(x).expanduser() / filename for x in dirnames]
    paths = [Path(x) / filename for x in dirnames]
    paths = [x for x in paths if x.exists()]
    return paths

def parse_config(parser):
    """Parse configuration files."""
    prefix = parser.fromfile_prefix_chars[0]
    args = ['{}{}'.format(prefix, x) for x in get_config_paths()]
    namespace, _ = parser.parse_known_args(args)
    return namespace

def convert_arg_line_to_args(line):
    """A better replacement for ArgumentParser."""
    COMMENT_PREFIX = '#'
    line = line.split(COMMENT_PREFIX, 1)[0].strip()
    return line.split()

def get_parser(**kwargs):
    """Get an argument parser with the usual standard arguments ready. Function
    'add' is added for convenience.
    """
    p = argparse.ArgumentParser(fromfile_prefix_chars='@', **kwargs)
    p.convert_arg_line_to_args = convert_arg_line_to_args
    p.add = p.add_argument
    p.add('-v', '--verbose', action='count', default=0,
          help='increase verbosity')
    p.add('--logfile', help='log file')
    p.add('--loglevel', default='WARNING', help='log level name')
    return p

def init_logging(args):
    """Initialize logging."""
    logging.basicConfig(filename=args.logfile,
                        level=dwi.util.get_loglevel(args.loglevel))

def parse_args(parser):
    """Parse args and configuration as well."""
    namespace = parse_config(parser)
    args = parser.parse_args(namespace=namespace)
    dwi.conf.init_logging(args)
    return args
<|endoftext|>


<commit_before>import hashlib
import ssl
import json
import urllib
import pkg_resources
from email.utils import formatdate
from datetime import datetime
from time import mktime
from base64 import encodebytes

import requests
from requests.auth import HTTPDigestAuth, HTTPBasicAuth, AuthBase
from requests.cookies import extract_cookies_to_jar, RequestsCookieJar
from requests_toolbelt import SSLAdapter
from requests_toolbelt.user_agent import user_agent
from django.conf import settings
from django.utils.encoding import force_bytes
from httpsig.requests_auth import HTTPSignatureAuth

from keybar.api.auth import ALGORITHM, REQUIRED_HEADERS
from keybar.utils.http import is_secure_transport, InsecureTransport

class Client(requests.Session):

    content_type = 'application/json'

    def __init__(self, device_id, secret):
        super(Client, self).__init__()

        keybar_url = 'https://{0}'.format(settings.KEYBAR_HOST)
        self.mount(keybar_url, SSLAdapter(ssl.PROTOCOL_TLSv1_2))
        self.device_id = device_id

        # TODO: find a way to avoid holding this in-memory for too long.
        self.secret = secret

    def request(self, method, url, *args, **kwargs):
        if not is_secure_transport(url):
            raise InsecureTransport('Please make sure to use HTTPS')

        data = kwargs.pop('data', {})

        now = datetime.utcnow()
        stamp = mktime(now.timetuple())

        raw_data = force_bytes(json.dumps(data))
        content_md5 = encodebytes(hashlib.md5(raw_data).digest()).strip()

        parse_result = urllib.parse.urlparse(url)

        dist = pkg_resources.get_distribution('keybar')

        headers = {
            'User-Agent': user_agent('keybar', dist.version),
            'Host': parse_result.netloc,
            'Method': method,
            'Path': parse_result.path,
            'Accept': self.content_type,
            'X-Device-Id': self.device_id,
            'Content-MD5': content_md5,
            'Date': formatdate(timeval=stamp, localtime=False, usegmt=True)
        }

        headers.update(kwargs.pop('headers', {}))

        auth = HTTPSignatureAuth(
            key_id=self.device_id,
            secret=self.secret,
            headers=REQUIRED_HEADERS,
            algorithm=ALGORITHM)

        kwargs.update({
            'auth': auth,
            'headers': headers,
            'verify': settings.KEYBAR_CA_BUNDLE
        })

        return super(Client, self).request(method, url, *args, **kwargs)
<commit_msg>Remove unused imports<commit_after>import hashlib
import ssl
import json
import urllib
import pkg_resources
from email.utils import formatdate
from datetime import datetime
from time import mktime
from base64 import encodebytes

import requests
from requests_toolbelt import SSLAdapter
from requests_toolbelt.user_agent import user_agent
from django.conf import settings
from django.utils.encoding import force_bytes
from httpsig.requests_auth import HTTPSignatureAuth

from keybar.api.auth import ALGORITHM, REQUIRED_HEADERS
from keybar.utils.http import is_secure_transport, InsecureTransport

class Client(requests.Session):

    content_type = 'application/json'

    def __init__(self, device_id, secret):
        super(Client, self).__init__()

        keybar_url = 'https://{0}'.format(settings.KEYBAR_HOST)
        self.mount(keybar_url, SSLAdapter(ssl.PROTOCOL_TLSv1_2))
        self.device_id = device_id

        # TODO: find a way to avoid holding this in-memory for too long.
        self.secret = secret

    def request(self, method, url, *args, **kwargs):
        if not is_secure_transport(url):
            raise InsecureTransport('Please make sure to use HTTPS')

        data = kwargs.pop('data', {})

        now = datetime.utcnow()
        stamp = mktime(now.timetuple())

        raw_data = force_bytes(json.dumps(data))
        content_md5 = encodebytes(hashlib.md5(raw_data).digest()).strip()

        parse_result = urllib.parse.urlparse(url)

        dist = pkg_resources.get_distribution('keybar')

        headers = {
            'User-Agent': user_agent('keybar', dist.version),
            'Host': parse_result.netloc,
            'Method': method,
            'Path': parse_result.path,
            'Accept': self.content_type,
            'X-Device-Id': self.device_id,
            'Content-MD5': content_md5,
            'Date': formatdate(timeval=stamp, localtime=False, usegmt=True)
        }

        headers.update(kwargs.pop('headers', {}))

        auth = HTTPSignatureAuth(
            key_id=self.device_id,
            secret=self.secret,
            headers=REQUIRED_HEADERS,
            algorithm=ALGORITHM)

        kwargs.update({
            'auth': auth,
            'headers': headers,
            'verify': settings.KEYBAR_CA_BUNDLE
        })

        return super(Client, self).request(method, url, *args, **kwargs)
<|endoftext|>


<commit_before>from __future__ import division
import sys
import os
import pyglet
from pyglet.gl import *
from pyglet.window import key
import mode
import gui

class MenuMode(mode.Mode):
    name = "menu_mode"

    def connect(self, controller):
        super(MenuMode, self).connect(controller)
        self.init_opengl()
        self.init_menu()

    @property
    def full_screen_label(self):
        return 'Exit full screen' if self.window.fullscreen else 'Fullscreen'

    def init_menu(self):
        font = gui.MainMenuFont()
        font2 = gui.GameMenuFont()
        self.b_fullscreen = gui.Button(self.full_screen_label, font2, self.on_toggle_full_screen)

        buttons = []
        if len(sys.argv) > 1:
            buttons.append(gui.Button('Play', font, self.on_play_pressed))
            buttons.append(gui.Button('Edit', font, self.on_edit_pressed))
        else:
            buttons.append(gui.Button('New game', font, self.on_play_pressed))
            #buttons.append(gui.Button('Continue', font, self.on_play_pressed))
        buttons.append(self.b_fullscreen)
        buttons.append(gui.Button('Exit', font2, self.on_exit_pressed))
        self.gui.replace(buttons)

    def on_toggle_full_screen(self, manager, args):
        self.toggle_full_screen()

    def toggle_full_screen(self):
        self.window.set_fullscreen(not self.window.fullscreen)
        self.window.set_minimum_size(800, 600)
        self.b_fullscreen.label.text = self.full_screen_label

    def on_play_pressed(self, manager, args):
        self.control.switch_handler("game_mode", False)

    def on_edit_pressed(self, manager, args):
        self.control.switch_handler("game_mode", True)

    def on_exit_pressed(self, manager, args):
        sys.exit(0)

    def on_draw(self):
        self.window.clear()
        self.gui.draw()

    def init_opengl(self):
        glMatrixMode(GL_PROJECTION)
        glLoadIdentity()
        gluOrtho2D(0, self.window.width, 0, self.window.height, -1, 1)
        glMatrixMode(GL_MODELVIEW)
        glLoadIdentity()
<commit_msg>Show background in the menu.<commit_after>from __future__ import division
import sys
import os
import pyglet
from pyglet.gl import *
from pyglet.window import key
import mode
import gui

class MenuMode(mode.Mode):
    name = "menu_mode"

    def connect(self, controller):
        super(MenuMode, self).connect(controller)
        self.init_opengl()
        self.init_menu()
        self.bg = pyglet.sprite.Sprite(pyglet.resource.image('menu-bg.png'))
        glClearColor(20/255, 20/255, 20/255, 1)

    def disconnect(self):
        self.bg.delete()

    @property
    def full_screen_label(self):
        return 'Exit full screen' if self.window.fullscreen else 'Fullscreen'

    def init_menu(self):
        font = gui.MainMenuFont()
        font2 = gui.GameMenuFont()
        self.b_fullscreen = gui.Button(self.full_screen_label, font2, self.on_toggle_full_screen)

        buttons = []
        if len(sys.argv) > 1:
            buttons.append(gui.Button('Play', font, self.on_play_pressed))
            buttons.append(gui.Button('Edit', font, self.on_edit_pressed))
        else:
            buttons.append(gui.Button('New game', font, self.on_play_pressed))
            #buttons.append(gui.Button('Continue', font, self.on_play_pressed))
        buttons.append(self.b_fullscreen)
        buttons.append(gui.Button('Exit', font2, self.on_exit_pressed))
        self.gui.replace(buttons)

    def on_toggle_full_screen(self, manager, args):
        self.toggle_full_screen()

    def toggle_full_screen(self):
        self.window.set_fullscreen(not self.window.fullscreen)
        self.window.set_minimum_size(800, 600)
        self.b_fullscreen.label.text = self.full_screen_label

    def on_play_pressed(self, manager, args):
        self.control.switch_handler("game_mode", False)

    def on_edit_pressed(self, manager, args):
        self.control.switch_handler("game_mode", True)

    def on_exit_pressed(self, manager, args):
        sys.exit(0)

    def on_draw(self):
        self.window.clear()
        self.bg.x = self.window.width - self.bg.width
        self.bg.draw()
        self.gui.draw()

    def init_opengl(self):
        glMatrixMode(GL_PROJECTION)
        glLoadIdentity()
        gluOrtho2D(0, self.window.width, 0, self.window.height, -1, 1)
        glMatrixMode(GL_MODELVIEW)
        glLoadIdentity()
<|endoftext|>


<commit_before># Copyright 2016-2017 Red Hat Inc & Xena Networks.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Contributors:
#   Christian Trautman, Red Hat Inc.
#

"""
Xena 2544 execution from command line with final verification step

Instructions:
1.  For linux Install Mono ->
    rpm --import "http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF"
    yum-config-manager --add-repo http://download.mono-project.com/repo/centos/
    yum -y install mono-complete
2. If python 3 not installed, install python 3. For RHEL instructions are below->
    # install SCL for python33 by adding a repo to find its location to install it
    [rhscl-python33-el7]
    name=Copr repo for python33-el7 owned by rhscl
    baseurl=https://copr-be.cloud.fedoraproject.org/results/rhscl/python33-el7/epel-7-$basearch/
    type=rpm-md
    skip_if_unavailable=True
    gpgcheck=1
    gpgkey=https://copr-be.cloud.fedoraproject.org/results/rhscl/python33-el7/pubkey.gpg
    repo_gpgcheck=0
    enabled=1
    enabled_metadata=1

    # install python33 package
    yum -y install python33 python33-python-tkinter
    # cleanup python 33 repo file
    rm -f /etc/yum.repos.d/python33.repo

3. Enable python33 -> scl enable python33 bash

4. Make sure Xena2544.exe is present in the current folder

5. Arguments to run this script
    -f <path to config file> saved from Xena2544.exe GUI with your config.
    -s enable smart search, if verify fails will resume the search at the half
       way point between the last verify attempt and the minimum search value.
       Otherwise it will just resume at the last verify attempt value minus
       the value threshhold.
    -l <verify length in seconds> Default of 2 hours.
    -r <retry attempts> Maximum number of verify attempts for giving up
    -d Enable debug mode
    -p Output PDF file. By default output of PDF report is disabled. Will cause
       a crash on linux usually as a pdf renderer is not installed.
    -w Enable windows mode. By default it will use the mono package to run the
       exe file. If running on windows this is not necessary.
    -t <search trial duration in seconds> Modify original config to use the
       duration specified.

"""

import argparse
import json
import locale
import logging
import os
import subprocess
import sys
from time import sleep
import xml.etree.ElementTree as ET

_LOGGER = logging.getLogger(__name__)
_LOCALE = locale.getlocale()[1]
_XENA_USER = 'TestUser'
_PYTHON_2 = sys.version_info[0] < 3

class XenaJSON(object):
    """
    Class to modify and read Xena JSON configuration files.
    """
    def __init__(self, json_path):
        """
        Constructor
        :param json_path: path to JSON file to read. Expected files must have
         two module ports with each port having its own stream config profile.
        :return: XenaJSON object
        """
        self.json_data = read_json_file(json_path)
        self.min_tput = self.json_data['TestOptions']['TestTypeOptionMap'][
            'Throughput']['RateIterationOptions']['MinimumValue']
        self.init_tput = self.json_data['TestOptions']['TestTypeOptionMap'][
            'Throughput']['RateIterationOptions']['InitialValue']
        self.max_tput = self.json_data['TestOptions']['TestTypeOptionMap'][
            'Throughput']['RateIterationOptions']['MaximumValue']
        self.value_thresh = self.json_data['TestOptions']['TestTypeOptionMap'][
            'Throughput']['RateIterationOptions']['ValueResolution']
        self.duration = self.json_data['TestOptions']['TestTypeOptionMap'][
            'Throughput']['Duration']

    # pylint: disable=too-many-arguments
    def modify_2544_tput_options(self, initial_value, minimum_value,
                                 maximum_value):
        """
        modify_2544_tput_options
        """
        self.json_data['TestOptions']['TestTypeOptionMap']['Throughput'][
            'RateIterationOptions']['InitialValue'] = initial_value
        self.json_data['TestOptions']['TestTypeOptionMap']['Throughput'][
            'RateIterationOptions']['MinimumValue'] = minimum_value
        self.json_data['TestOptions']['TestTypeOptionMap']['Throughput'][
            'RateIterationOptions']['MaximumValue'] = maximum_value

    def modify_duration(self, duration):
        """
        Modify test duration
        :param duration: test time duration in seconds as int
        :return: None
        """
        self.json_data['TestOptions']['TestTypeOptionMap']['Throughput'][
            'Duration'] = duration

    def modify_reporting(self, pdf_enable=True, csv_enable=False,
                         xml_enable=True, html_enable=False,
                         timestamp_enable=False):
        """
        Modify the reporting options
        :param pdf_enable: Enable pdf output, should disable for linux
        :param csv_enable: Enable csv output
        :param xml_enable: Enable xml output
        :param html_enable: Enable html output
        :param timestamp_enable: Enable timestamp to report
        :return: None
        """
        self.json_data['ReportConfig'][
            'GeneratePdf'] = 'true' if pdf_enable else 'false'
        self.json_data['ReportConfig'][
            'GenerateCsv'] = 'true' if csv_enable else 'false'
        self.json_data['ReportConfig'][
            'GenerateXml'] = 'true' if xml_enable else 'false'
        self.json_data['ReportConfig'][
            'GenerateHtml'] = 'true' if html_enable else 'false'
        self.json_data['ReportConfig'][
            'AppendTimestamp'] = 'true' if timestamp_enable else 'false'

    def write_config(self, path='./2bUsed.x2544'):
        """
        Write the config to out as file
        :param path: Output file to export the json data to
        :return: None
        """
        if not write_json_file(self.json_data, path):
            raise RuntimeError("Could not write out file, please check config")

def main(args):
    _LOGGER.setLevel(logging.DEBUG if args.debug else logging.INFO)
    stream_logger = logging.StreamHandler(sys.stdout)
    stream_logger.setFormatter(logging.Formatter(
        '[%(levelname)-5s]  %(asctime)s : (%(name)s) - %(message)s'))
    _LOGGER.addHandler(stream_logger)
    # get the current json config into an object
    xena_current = XenaJSON(args.config_file)
    # Modify to output xml always as its needed to parse, turn off PDF output
    # unless user specifies it. Usually not supported on Linux. Also need to
    # disable the timestamp
    xena_current.modify_reporting(True if args.pdf_output else False,
                                  True, True, False, False)
    if args.search_trial_duration:
        xena_current.modify_duration(args.search_trial_duration)
    xena_current.write_config('./2bUsed.x2544')

    result = run_xena('./2bUsed.x2544', args.windows_mode)

    # now run the verification step by creating a new config with the desired
    # params
    for _ in range(1, args.retry_attempts +1):
        if result[0] != 'PASS':
            _LOGGER.error('Xena2544.exe Test failed. Please check test config.')
            break
        _LOGGER.info('Verify attempt {}'.format(_))
        old_min = xena_current.min_tput # need this if verify fails
        old_duration = xena_current.duration
        xena_current.modify_2544_tput_options(result[1], result[1],
                                              result[1])
        xena_current.modify_duration(args.verify_duration)
        xena_current.write_config('./verify.x2544')
        # run verify step
        _LOGGER.info('Running verify for {} seconds'.format(
            args.verify_duration))
        verify_result = run_xena('./verify.x2544', args.windows_mode)
        if verify_result[0] == 'PASS':
            _LOGGER.info('Verify passed. Packets lost = {} Exiting'.format(
                verify_result[3]))
            _LOGGER.info('Pass result transmit rate = {}'.format(
                verify_result[1]))
            _LOGGER.info('Pass result transmit fps = {}'.format(
                verify_result[2]))
            break
        else:
            _LOGGER.warn('Verify failed. Packets lost = {}'.format(
                verify_result[3]))
            _LOGGER.info('Restarting Xena2544.exe with new values')
            if args.smart_search:
                new_init = (verify_result[1] - old_min) / 2
            else:
                new_init = result[1] - xena_current.value_thresh
            xena_current.modify_2544_tput_options(
                new_init, old_min, result[1] - xena_current.value_thresh)
            xena_current.modify_duration(
                args.search_trial_duration if args.search_trial_duration else
                old_duration)
            _LOGGER.info('New minimum value: {}'.format(old_min))
            _LOGGER.info('New maximum value: {}'.format(
                result[1] - xena_current.value_thresh))
            _LOGGER.info('New initial rate: {}'.format(new_init))
            xena_current.write_config('./verify.x2544')
            result = run_xena('./verify.x2544', args.windows_mode)
    else:
        _LOGGER.error('Maximum number of verify retries attempted. Exiting...')

def read_json_file(json_file):
    """
    Read the json file path and return a dictionary of the data
    :param json_file: path to json file
    :return: dictionary of json data
    """
    try:
        if _PYTHON_2:
            with open(json_file, 'r') as data_file:
                file_data = json.loads(data_file.read())
        else:
            with open(json_file, 'r', encoding=_LOCALE) as data_file:
                file_data = json.loads(data_file.read())
    except ValueError as exc:
        # general json exception, Python 3.5 adds new exception type
        _LOGGER.exception("Exception with json read: %s", exc)
        raise
    except IOError as exc:
        _LOGGER.exception(
            'Exception during file open: %s file=%s', exc, json_file)
        raise
    return file_data

def run_xena(config_file, windows_mode=False):
    """
    Run Xena2544.exe with the config file specified.
    :param config_file: config file to use
    :param windows_mode: enable windows mode which bypasses the usage of mono
    :return: Tuple of pass or fail result as str, and current transmit rate as
    float, transmit fps, and packets lost
    """
    user_home = os.path.expanduser('~')
    log_path = '{}/Xena/Xena2544-2G/Logs/xena2544.log'.format(user_home)
    # make the folder and log file if they doesn't exist
    if not os.path.exists(log_path):
        os.makedirs(os.path.dirname(log_path))

    # empty the file contents
    open(log_path, 'w').close()

    # setup the xena command line
    args = ["mono" if not windows_mode else "",
            "Xena2544.exe", "-c", config_file, "-e", "-r", "./", "-u",
            _XENA_USER]

    # Sometimes Xena2544.exe completes, but mono holds the process without
    # releasing it, this can cause a deadlock of the main thread. Use the
    # xena log file as a way to detect this.
    log_handle = open(log_path, 'r')
    # read the contents of the log before we start so the next read in the
    # wait method are only looking at the text from this test instance
    log_handle.read()
    mono_pipe = subprocess.Popen(args, stdout=sys.stdout)
    data = ''
    if _PYTHON_2:
        _LOGGER.error('Not supported yet for python 2...')
    else:
        while True:
            try:
                mono_pipe.wait(60)
                log_handle.close()
                break
            except subprocess.TimeoutExpired:
                # check the log to see if Xena2544 has completed and mono is
                # deadlocked.
                data += log_handle.read()
                if 'TestCompletedSuccessfully' in data:
                    log_handle.close()
                    mono_pipe.terminate()
                    break

    # parse the result file and return the needed data
    root = ET.parse(r'./xena2544-report.xml').getroot()
    return (root[0][1][0].get('TestState'),
            float(root[0][1][0].get('TotalTxRatePcnt')),
            int(root[0][1][0].get('TotalTxRateFps')),
            root[0][1][0].get('TotalLossFrames'))

def write_json_file(json_data, output_path):
    """
    Write out the dictionary of data to a json file
    :param json_data: dictionary of json data
    :param output_path: file path to write output
    :return: Boolean if success
    """
    try:
        if _PYTHON_2:
            with open(output_path, 'w') as fileh:
                json.dump(json_data, fileh, indent=2, sort_keys=True,
                          ensure_ascii=True)
        else:
            with open(output_path, 'w', encoding=_LOCALE) as fileh:
                json.dump(json_data, fileh, indent=2, sort_keys=True,
                          ensure_ascii=True)
        return True
    except ValueError as exc:
        # general json exception, Python 3.5 adds new exception type
        _LOGGER.exception(
            "Exception with json write: %s", exc)
        return False
    except IOError as exc:
        _LOGGER.exception(
            'Exception during file write: %s file=%s', exc, output_path)
        return False

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-f', '--config_file', type=str, required=True,
                        help='Xena 2544 json config file name')
    parser.add_argument('-d', '--debug', action='store_true', required=False,
                        help='Enable debug logging')
    parser.add_argument('-w', '--windows_mode', required=False,
                        action='store_true', help='Use windows mode, no mono')
    parser.add_argument('-l', '--verify_duration', required=False,
                        type=int, default=7200,
                        help='Verification duration in seconds')
    parser.add_argument('-r', '--retry_attempts', type=int, default=10,
                        required=False, help='Maximum verify attempts')
    parser.add_argument('-s', '--smart_search', action='store_true',
                        required=False, help='Enable smart search',
                        default=False)
    parser.add_argument('-p', '--pdf_output', action='store_true',
                        required=False,
                        help='Generate PDF report, do not use on Linux!',
                        default=False)
    parser.add_argument('-t', '--search_trial_duration', required=False,
                        help='Search trial duration in seconds', type=int,
                        default=0)
    args = parser.parse_args()
    if args.debug:
        print("DEBUG ENABLED!!!")
    main(args)

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

<commit_msg>Add more to instructions<commit_after># Copyright 2016-2017 Red Hat Inc & Xena Networks.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Contributors:
#   Christian Trautman, Red Hat Inc.
#

"""
Xena 2544 execution from command line with final verification step

Instructions:
1.  For linux Install Mono ->
    rpm --import "http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF"
    yum-config-manager --add-repo http://download.mono-project.com/repo/centos/
    yum -y install mono-complete
2. If python 3 not installed, install python 3. For RHEL instructions are below->
   Add this repo as python33.repo for your repo file locations. /etc/yum.repos.d
    [rhscl-python33-el7]
    name=Copr repo for python33-el7 owned by rhscl
    baseurl=https://copr-be.cloud.fedoraproject.org/results/rhscl/python33-el7/epel-7-$basearch/
    type=rpm-md
    skip_if_unavailable=True
    gpgcheck=1
    gpgkey=https://copr-be.cloud.fedoraproject.org/results/rhscl/python33-el7/pubkey.gpg
    repo_gpgcheck=0
    enabled=1
    enabled_metadata=1

    # install python33 package
    yum -y install python33 python33-python-tkinter
    # cleanup python 33 repo file
    rm -f /etc/yum.repos.d/python33.repo

3. Enable python33 -> scl enable python33 bash

4. Make sure Xena2544.exe is present in the current folder

5. Copy your x2544 config file to the script folder

6. Arguments to run this script
    -f <path to config file> saved from Xena2544.exe GUI with your config.
    -s enable smart search, if verify fails will resume the search at the half
       way point between the last verify attempt and the minimum search value.
       Otherwise it will just resume at the last verify attempt value minus
       the value threshhold.
    -l <verify length in seconds> Default of 2 hours.
    -r <retry attempts> Maximum number of verify attempts for giving up
    -d Enable debug mode
    -p Output PDF file. By default output of PDF report is disabled. Will cause
       a crash on linux usually as a pdf renderer is not installed.
    -w Enable windows mode. By default it will use the mono package to run the
       exe file. If running on windows this is not necessary.
    -t <search trial duration in seconds> Modify original config to use the
       duration specified.

7. Sample execution ->
   Runs a 60 second trial with a 600 second verify using the myconfig.x2544
   configuration file.
   python XenaVerify.py -f myconfig.x2544 -s -l 600 -t 60

Improvements to be done
- Add debug logging
- Add more customized options for modifying the running config
- Add python 2.7 support

"""

import argparse
import json
import locale
import logging
import os
import subprocess
import sys
from time import sleep
import xml.etree.ElementTree as ET

_LOGGER = logging.getLogger(__name__)
_LOCALE = locale.getlocale()[1]
_XENA_USER = 'TestUser'
_PYTHON_2 = sys.version_info[0] < 3

class XenaJSON(object):
    """
    Class to modify and read Xena JSON configuration files.
    """
    def __init__(self, json_path):
        """
        Constructor
        :param json_path: path to JSON file to read. Expected files must have
         two module ports with each port having its own stream config profile.
        :return: XenaJSON object
        """
        self.json_data = read_json_file(json_path)
        self.min_tput = self.json_data['TestOptions']['TestTypeOptionMap'][
            'Throughput']['RateIterationOptions']['MinimumValue']
        self.init_tput = self.json_data['TestOptions']['TestTypeOptionMap'][
            'Throughput']['RateIterationOptions']['InitialValue']
        self.max_tput = self.json_data['TestOptions']['TestTypeOptionMap'][
            'Throughput']['RateIterationOptions']['MaximumValue']
        self.value_thresh = self.json_data['TestOptions']['TestTypeOptionMap'][
            'Throughput']['RateIterationOptions']['ValueResolution']
        self.duration = self.json_data['TestOptions']['TestTypeOptionMap'][
            'Throughput']['Duration']

    # pylint: disable=too-many-arguments
    def modify_2544_tput_options(self, initial_value, minimum_value,
                                 maximum_value):
        """
        modify_2544_tput_options
        """
        self.json_data['TestOptions']['TestTypeOptionMap']['Throughput'][
            'RateIterationOptions']['InitialValue'] = initial_value
        self.json_data['TestOptions']['TestTypeOptionMap']['Throughput'][
            'RateIterationOptions']['MinimumValue'] = minimum_value
        self.json_data['TestOptions']['TestTypeOptionMap']['Throughput'][
            'RateIterationOptions']['MaximumValue'] = maximum_value

    def modify_duration(self, duration):
        """
        Modify test duration
        :param duration: test time duration in seconds as int
        :return: None
        """
        self.json_data['TestOptions']['TestTypeOptionMap']['Throughput'][
            'Duration'] = duration

    def modify_reporting(self, pdf_enable=True, csv_enable=False,
                         xml_enable=True, html_enable=False,
                         timestamp_enable=False):
        """
        Modify the reporting options
        :param pdf_enable: Enable pdf output, should disable for linux
        :param csv_enable: Enable csv output
        :param xml_enable: Enable xml output
        :param html_enable: Enable html output
        :param timestamp_enable: Enable timestamp to report
        :return: None
        """
        self.json_data['ReportConfig'][
            'GeneratePdf'] = 'true' if pdf_enable else 'false'
        self.json_data['ReportConfig'][
            'GenerateCsv'] = 'true' if csv_enable else 'false'
        self.json_data['ReportConfig'][
            'GenerateXml'] = 'true' if xml_enable else 'false'
        self.json_data['ReportConfig'][
            'GenerateHtml'] = 'true' if html_enable else 'false'
        self.json_data['ReportConfig'][
            'AppendTimestamp'] = 'true' if timestamp_enable else 'false'

    def write_config(self, path='./2bUsed.x2544'):
        """
        Write the config to out as file
        :param path: Output file to export the json data to
        :return: None
        """
        if not write_json_file(self.json_data, path):
            raise RuntimeError("Could not write out file, please check config")

def main(args):
    _LOGGER.setLevel(logging.DEBUG if args.debug else logging.INFO)
    stream_logger = logging.StreamHandler(sys.stdout)
    stream_logger.setFormatter(logging.Formatter(
        '[%(levelname)-5s]  %(asctime)s : (%(name)s) - %(message)s'))
    _LOGGER.addHandler(stream_logger)
    # get the current json config into an object
    xena_current = XenaJSON(args.config_file)
    # Modify to output xml always as its needed to parse, turn off PDF output
    # unless user specifies it. Usually not supported on Linux. Also need to
    # disable the timestamp
    xena_current.modify_reporting(True if args.pdf_output else False,
                                  True, True, False, False)
    if args.search_trial_duration:
        xena_current.modify_duration(args.search_trial_duration)
    xena_current.write_config('./2bUsed.x2544')

    result = run_xena('./2bUsed.x2544', args.windows_mode)

    # now run the verification step by creating a new config with the desired
    # params
    for _ in range(1, args.retry_attempts +1):
        if result[0] != 'PASS':
            _LOGGER.error('Xena2544.exe Test failed. Please check test config.')
            break
        _LOGGER.info('Verify attempt {}'.format(_))
        old_min = xena_current.min_tput # need this if verify fails
        old_duration = xena_current.duration
        xena_current.modify_2544_tput_options(result[1], result[1],
                                              result[1])
        xena_current.modify_duration(args.verify_duration)
        xena_current.write_config('./verify.x2544')
        # run verify step
        _LOGGER.info('Running verify for {} seconds'.format(
            args.verify_duration))
        verify_result = run_xena('./verify.x2544', args.windows_mode)
        if verify_result[0] == 'PASS':
            _LOGGER.info('Verify passed. Packets lost = {} Exiting'.format(
                verify_result[3]))
            _LOGGER.info('Pass result transmit rate = {}'.format(
                verify_result[1]))
            _LOGGER.info('Pass result transmit fps = {}'.format(
                verify_result[2]))
            break
        else:
            _LOGGER.warn('Verify failed. Packets lost = {}'.format(
                verify_result[3]))
            _LOGGER.info('Restarting Xena2544.exe with new values')
            if args.smart_search:
                new_init = (verify_result[1] - old_min) / 2
            else:
                new_init = result[1] - xena_current.value_thresh
            xena_current.modify_2544_tput_options(
                new_init, old_min, result[1] - xena_current.value_thresh)
            xena_current.modify_duration(
                args.search_trial_duration if args.search_trial_duration else
                old_duration)
            _LOGGER.info('New minimum value: {}'.format(old_min))
            _LOGGER.info('New maximum value: {}'.format(
                result[1] - xena_current.value_thresh))
            _LOGGER.info('New initial rate: {}'.format(new_init))
            xena_current.write_config('./verify.x2544')
            result = run_xena('./verify.x2544', args.windows_mode)
    else:
        _LOGGER.error('Maximum number of verify retries attempted. Exiting...')

def read_json_file(json_file):
    """
    Read the json file path and return a dictionary of the data
    :param json_file: path to json file
    :return: dictionary of json data
    """
    try:
        if _PYTHON_2:
            with open(json_file, 'r') as data_file:
                file_data = json.loads(data_file.read())
        else:
            with open(json_file, 'r', encoding=_LOCALE) as data_file:
                file_data = json.loads(data_file.read())
    except ValueError as exc:
        # general json exception, Python 3.5 adds new exception type
        _LOGGER.exception("Exception with json read: %s", exc)
        raise
    except IOError as exc:
        _LOGGER.exception(
            'Exception during file open: %s file=%s', exc, json_file)
        raise
    return file_data

def run_xena(config_file, windows_mode=False):
    """
    Run Xena2544.exe with the config file specified.
    :param config_file: config file to use
    :param windows_mode: enable windows mode which bypasses the usage of mono
    :return: Tuple of pass or fail result as str, and current transmit rate as
    float, transmit fps, and packets lost
    """
    user_home = os.path.expanduser('~')
    log_path = '{}/Xena/Xena2544-2G/Logs/xena2544.log'.format(user_home)
    # make the folder and log file if they doesn't exist
    if not os.path.exists(log_path):
        os.makedirs(os.path.dirname(log_path))

    # empty the file contents
    open(log_path, 'w').close()

    # setup the xena command line
    args = ["mono" if not windows_mode else "",
            "Xena2544.exe", "-c", config_file, "-e", "-r", "./", "-u",
            _XENA_USER]

    # Sometimes Xena2544.exe completes, but mono holds the process without
    # releasing it, this can cause a deadlock of the main thread. Use the
    # xena log file as a way to detect this.
    log_handle = open(log_path, 'r')
    # read the contents of the log before we start so the next read in the
    # wait method are only looking at the text from this test instance
    log_handle.read()
    mono_pipe = subprocess.Popen(args, stdout=sys.stdout)
    data = ''
    if _PYTHON_2:
        _LOGGER.error('Not supported yet for python 2...')
    else:
        while True:
            try:
                mono_pipe.wait(60)
                log_handle.close()
                break
            except subprocess.TimeoutExpired:
                # check the log to see if Xena2544 has completed and mono is
                # deadlocked.
                data += log_handle.read()
                if 'TestCompletedSuccessfully' in data:
                    log_handle.close()
                    mono_pipe.terminate()
                    break

    # parse the result file and return the needed data
    root = ET.parse(r'./xena2544-report.xml').getroot()
    return (root[0][1][0].get('TestState'),
            float(root[0][1][0].get('TotalTxRatePcnt')),
            int(root[0][1][0].get('TotalTxRateFps')),
            root[0][1][0].get('TotalLossFrames'))

def write_json_file(json_data, output_path):
    """
    Write out the dictionary of data to a json file
    :param json_data: dictionary of json data
    :param output_path: file path to write output
    :return: Boolean if success
    """
    try:
        if _PYTHON_2:
            with open(output_path, 'w') as fileh:
                json.dump(json_data, fileh, indent=2, sort_keys=True,
                          ensure_ascii=True)
        else:
            with open(output_path, 'w', encoding=_LOCALE) as fileh:
                json.dump(json_data, fileh, indent=2, sort_keys=True,
                          ensure_ascii=True)
        return True
    except ValueError as exc:
        # general json exception, Python 3.5 adds new exception type
        _LOGGER.exception(
            "Exception with json write: %s", exc)
        return False
    except IOError as exc:
        _LOGGER.exception(
            'Exception during file write: %s file=%s', exc, output_path)
        return False

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-f', '--config_file', type=str, required=True,
                        help='Xena 2544 json config file name')
    parser.add_argument('-d', '--debug', action='store_true', required=False,
                        help='Enable debug logging')
    parser.add_argument('-w', '--windows_mode', required=False,
                        action='store_true', help='Use windows mode, no mono')
    parser.add_argument('-l', '--verify_duration', required=False,
                        type=int, default=7200,
                        help='Verification duration in seconds')
    parser.add_argument('-r', '--retry_attempts', type=int, default=10,
                        required=False, help='Maximum verify attempts')
    parser.add_argument('-s', '--smart_search', action='store_true',
                        required=False, help='Enable smart search',
                        default=False)
    parser.add_argument('-p', '--pdf_output', action='store_true',
                        required=False,
                        help='Generate PDF report, do not use on Linux!',
                        default=False)
    parser.add_argument('-t', '--search_trial_duration', required=False,
                        help='Search trial duration in seconds', type=int,
                        default=0)
    args = parser.parse_args()
    if args.debug:
        print("DEBUG ENABLED!!!")
    main(args)

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

<|endoftext|>


<commit_before>from compiler.ast import *
import copy
import cPickle
import logging
import os
logger = logging.getLogger('ExprManip.Differentiation')

import AST
from AST import strip_parse
import Simplify
import Substitution

_ZERO = Const(0)
_ONE = Const(1)

__deriv_saved = {}
def load_derivs(filename):
    """
    Load up a pickled dictionary of saved derivatives.
    """
    global __deriv_saved
    try:
        f = file(filename, 'rb')
    except IOError:
        # This failure probably indicates that the file doesn't exist.
        return
    try:
        __deriv_saved = cPickle.load(f)
        return
    except:
        # For some reason, pulling the data from the file failed.
        logger.warn('Failed to load saved derivative file %s. Trying to delete it to '
                    'avoid future problems.' % filename)
        try:
            # Let's try to remove it...
            f.close()
            os.remove(filename)
        except:
            logger.warn('File removal failed. Please delete %s manually.' % filename)

def save_derivs(filename):
    f = file(filename, 'wb')
    cPickle.dump(__deriv_saved, f, 2)
    f.close()

def diff_expr(expr, wrt):
    """
    Return the derivative of the expression with respect to a given variable.
    """
    logger.debug('Taking derivative of %s wrt %s' % (expr, wrt))
    key = '%s__derivWRT__%s' % (expr, wrt)
    if __deriv_saved.has_key(key):
        deriv = __deriv_saved[key]
        logger.debug('Found saved result %s.' % deriv)
        return deriv

    ast = AST.strip_parse(expr)
    deriv = _diff_ast(ast, wrt)
    deriv = Simplify._simplify_ast(deriv)
    deriv  = AST.ast2str(deriv)
    __deriv_saved[key] = deriv
    logger.debug('Computed result %s.' % deriv)
    return deriv

# This dictionary stores how to differentiate various functions. The keys are
#  (function name, # of arguments). The values are tuples of strings which
#  give the partial derivive wrt each argument. It is important to use arg# to
#  denote the arguments.
_KNOWN_FUNCS = {('acos', 1): ('1/sqrt(1-arg0**2)',),
                ('asin', 1): ('1/sqrt(1-arg0**2)',),
                ('atan', 1): ('1/(1+arg0**2)',),
                ('cos', 1): ('-sin(arg0)',),
                ('cosh', 1): ('sinh(arg0)',),
                ('exp', 1): ('exp(arg0)',),
                ('log', 1): ('1/arg0',),
                ('log10', 1): ('1/(log(10)*arg0)',),
                ('sin', 1): ('cos(arg0)',),
                ('sinh', 1): ('cosh(arg0)',),
                ('arcsinh', 1): ('1/sqrt(1+arg0**2)',),
                ('arccosh', 1): ('1/sqrt(arg0**2 - 1.)',),
                ('arctanh', 1): ('1/(1.-arg0**2)',),
                ('sqrt', 1): ('1/(2*sqrt(arg0))',),
                ('tan', 1): ('1/cos(arg0)**2',),
                ('tanh', 1): ('1/cosh(arg0)**2',),
                ('pow', 2): ('arg1 * arg0**(arg1-1)',
                             'log(arg0) * arg0**arg1')
                }
for key, terms in _KNOWN_FUNCS.items():
    _KNOWN_FUNCS[key] = [strip_parse(term) for term in terms]

def _diff_ast(ast, wrt):
    """
    Return an AST that is the derivative of ast with respect the variable with
    name 'wrt'.
    """

    # For now, the strategy is to return the most general forms, and let
    #  the simplifier take care of the special cases.
    if isinstance(ast, Name):
        if ast.name == wrt:
            return _ONE
        else:
            return _ZERO
    elif isinstance(ast, Const):
        return _ZERO
    elif isinstance(ast, Add) or isinstance(ast, Sub):
        # Just take the derivative of the arguments. The call to ast.__class__
        #  lets us use the same code from Add and Sub.
        return ast.__class__((_diff_ast(ast.left, wrt),
                              _diff_ast(ast.right, wrt)))
    elif isinstance(ast, Mul) or isinstance(ast, Div):
        # Collect all the numerators and denominators together
        nums, denoms = [], []
        AST._collect_num_denom(ast, nums, denoms)

        # Collect the numerator terms into a single AST
        num = AST._make_product(nums)
        # Take the derivative of the numerator terms as a product
        num_d = _product_deriv(nums, wrt)
        if not denoms:
            # If there is no denominator
            return num_d

        denom = AST._make_product(denoms)
        denom_d = _product_deriv(denoms, wrt)

        # Derivative of x/y is x'/y + -x*y'/y**2
        term1 = Div((num_d, denom))
        term2 = Div((Mul((UnarySub(num), denom_d)), Power((denom, Const(2)))))
        return Add((term1, term2))

    elif isinstance(ast, Power):
        # Use the derivative of the 'pow' function
        ast =  CallFunc(Name('pow'), [ast.left, ast.right])
        return _diff_ast(ast, wrt)

    elif isinstance(ast, CallFunc):
        func_name = AST.ast2str(ast.node)
        args = ast.args
        args_d = [_diff_ast(arg, wrt) for arg in args]

        if _KNOWN_FUNCS.has_key((func_name, len(args))):
            form = copy.deepcopy(_KNOWN_FUNCS[(func_name, len(args))])
        else:
            # If this isn't a known function, our form is
            #  (f_0(args), f_1(args), ...)
            args_expr = [Name('arg%i' % ii) for ii in range(len(args))]
            form = [CallFunc(Name('%s_%i' % (func_name, ii)), args_expr) for
                    ii in range(len(args))]

        # We build up the terms in our derivative
        #  f_0(x,y)*x' + f_1(x,y)*y', etc.
        outs = []
        for arg_d, arg_form_d in zip(args_d, form):
            # We skip arguments with 0 derivative
            if arg_d == _ZERO:
                continue
            for ii, arg in enumerate(args):
                Substitution._sub_subtrees_for_vars(arg_form_d,
                                                    {'arg%i'%ii:arg})
            outs.append(Mul((arg_form_d, arg_d)))

        # If all arguments had zero deriviative
        if not outs:
            return _ZERO
        else:
            # We add up all our terms
            ret = outs[0]
            for term in outs[1:]:
                ret = Add((ret, term))
            return ret

    elif isinstance(ast, UnarySub):
        return UnarySub(_diff_ast(ast.expr, wrt))

    elif isinstance(ast, UnaryAdd):
        return UnaryAdd(_diff_ast(ast.expr, wrt))

def _product_deriv(terms, wrt):
    """
    Return an AST expressing the derivative of the product of all the terms.
    """
    if len(terms) == 1:
        return _diff_ast(terms[0], wrt)
    deriv_terms = []
    for ii, term in enumerate(terms):
        term_d = _diff_ast(term, wrt)
        other_terms = terms[:ii] + terms[ii+1:]
        deriv_terms.append(AST._make_product(other_terms + [term_d]))
    sum = deriv_terms[0]
    for term in deriv_terms[1:]:
        sum = Add((term, sum))

    return sum
<commit_msg>Differentiation.py: Avoid load or saving an outdated derivatives cache file. (Outdated because Differentiation.py has been changed.)<commit_after>from compiler.ast import *
import copy
import cPickle
import logging
import os
logger = logging.getLogger('ExprManip.Differentiation')

import AST
from AST import strip_parse
import Simplify
import Substitution

_ZERO = Const(0)
_ONE = Const(1)

# Record the version of the Differentiation.py file we loaded.
__version_loaded = os.path.getmtime(__file__)

__deriv_saved = {}
def load_derivs(filename):
    """
    Load up a pickled dictionary of saved derivatives.
    """
    global __deriv_saved
    # If the cache file is older than the this Derivatives.py file, we don't
    # want to load it, because it may contain incorrect results.
    if os.path.getmtime(filename) < __version_loaded:
        logger.warn('Derivative cache file %s appears outdated. Trying to '
                    'delete it to avoid future problems.' % filename)
        try:
            # Let's try to remove it...
            os.remove(filename)
        except:
            logger.warn('File removal failed. Please delete %s manually.'
                        % filename)
        return

    try:
        f = file(filename, 'rb')
    except IOError:
        # This failure probably indicates that the file doesn't exist.
        return
    try:
        __deriv_saved = cPickle.load(f)
        return
    except:
        # For some reason, pulling the data from the file failed.
        logger.warn('Failed to load saved derivative file %s. Trying to delete it to '
                    'avoid future problems.' % filename)
        try:
            # Let's try to remove it...
            f.close()
            os.remove(filename)
        except:
            logger.warn('File removal failed. Please delete %s manually.' % filename)

def save_derivs(filename):
    if os.path.getmtime(__file__) > __version_loaded:
        logger.warn('Differentiation.py appears to have been modified during '
                    'this Python session (and not reloaded). Not saving '
                    'current, potentially, outdated derivatives cache.')
        return
    f = file(filename, 'wb')
    cPickle.dump(__deriv_saved, f, 2)
    f.close()

def diff_expr(expr, wrt):
    """
    Return the derivative of the expression with respect to a given variable.
    """
    logger.debug('Taking derivative of %s wrt %s' % (expr, wrt))
    key = '%s__derivWRT__%s' % (expr, wrt)
    if __deriv_saved.has_key(key):
        deriv = __deriv_saved[key]
        logger.debug('Found saved result %s.' % deriv)
        return deriv

    ast = AST.strip_parse(expr)
    deriv = _diff_ast(ast, wrt)
    deriv = Simplify._simplify_ast(deriv)
    deriv  = AST.ast2str(deriv)
    __deriv_saved[key] = deriv
    logger.debug('Computed result %s.' % deriv)
    return deriv

# This dictionary stores how to differentiate various functions. The keys are
#  (function name, # of arguments). The values are tuples of strings which
#  give the partial derivive wrt each argument. It is important to use arg# to
#  denote the arguments.
_KNOWN_FUNCS = {('acos', 1): ('1/sqrt(1-arg0**2)',),
                ('asin', 1): ('1/sqrt(1-arg0**2)',),
                ('atan', 1): ('1/(1+arg0**2)',),
                ('cos', 1): ('-sin(arg0)',),
                ('cosh', 1): ('sinh(arg0)',),
                ('exp', 1): ('exp(arg0)',),
                ('log', 1): ('1/arg0',),
                ('log10', 1): ('1/(log(10)*arg0)',),
                ('sin', 1): ('cos(arg0)',),
                ('sinh', 1): ('cosh(arg0)',),
                ('arcsinh', 1): ('1/sqrt(1+arg0**2)',),
                ('arccosh', 1): ('1/sqrt(arg0**2 - 1.)',),
                ('arctanh', 1): ('1/(1.-arg0**2)',),
                ('sqrt', 1): ('1/(2*sqrt(arg0))',),
                ('tan', 1): ('1/cos(arg0)**2',),
                ('tanh', 1): ('1/cosh(arg0)**2',),
                ('pow', 2): ('arg1 * arg0**(arg1-1)',
                             'log(arg0) * arg0**arg1')
                }
for key, terms in _KNOWN_FUNCS.items():
    _KNOWN_FUNCS[key] = [strip_parse(term) for term in terms]

def _diff_ast(ast, wrt):
    """
    Return an AST that is the derivative of ast with respect the variable with
    name 'wrt'.
    """

    # For now, the strategy is to return the most general forms, and let
    #  the simplifier take care of the special cases.
    if isinstance(ast, Name):
        if ast.name == wrt:
            return _ONE
        else:
            return _ZERO
    elif isinstance(ast, Const):
        return _ZERO
    elif isinstance(ast, Add) or isinstance(ast, Sub):
        # Just take the derivative of the arguments. The call to ast.__class__
        #  lets us use the same code from Add and Sub.
        return ast.__class__((_diff_ast(ast.left, wrt),
                              _diff_ast(ast.right, wrt)))
    elif isinstance(ast, Mul) or isinstance(ast, Div):
        # Collect all the numerators and denominators together
        nums, denoms = [], []
        AST._collect_num_denom(ast, nums, denoms)

        # Collect the numerator terms into a single AST
        num = AST._make_product(nums)
        # Take the derivative of the numerator terms as a product
        num_d = _product_deriv(nums, wrt)
        if not denoms:
            # If there is no denominator
            return num_d

        denom = AST._make_product(denoms)
        denom_d = _product_deriv(denoms, wrt)

        # Derivative of x/y is x'/y + -x*y'/y**2
        term1 = Div((num_d, denom))
        term2 = Div((Mul((UnarySub(num), denom_d)), Power((denom, Const(2)))))
        return Add((term1, term2))

    elif isinstance(ast, Power):
        # Use the derivative of the 'pow' function
        ast =  CallFunc(Name('pow'), [ast.left, ast.right])
        return _diff_ast(ast, wrt)

    elif isinstance(ast, CallFunc):
        func_name = AST.ast2str(ast.node)
        args = ast.args
        args_d = [_diff_ast(arg, wrt) for arg in args]

        if _KNOWN_FUNCS.has_key((func_name, len(args))):
            form = copy.deepcopy(_KNOWN_FUNCS[(func_name, len(args))])
        else:
            # If this isn't a known function, our form is
            #  (f_0(args), f_1(args), ...)
            args_expr = [Name('arg%i' % ii) for ii in range(len(args))]
            form = [CallFunc(Name('%s_%i' % (func_name, ii)), args_expr) for
                    ii in range(len(args))]

        # We build up the terms in our derivative
        #  f_0(x,y)*x' + f_1(x,y)*y', etc.
        outs = []
        for arg_d, arg_form_d in zip(args_d, form):
            # We skip arguments with 0 derivative
            if arg_d == _ZERO:
                continue
            for ii, arg in enumerate(args):
                Substitution._sub_subtrees_for_vars(arg_form_d,
                                                    {'arg%i'%ii:arg})
            outs.append(Mul((arg_form_d, arg_d)))

        # If all arguments had zero deriviative
        if not outs:
            return _ZERO
        else:
            # We add up all our terms
            ret = outs[0]
            for term in outs[1:]:
                ret = Add((ret, term))
            return ret

    elif isinstance(ast, UnarySub):
        return UnarySub(_diff_ast(ast.expr, wrt))

    elif isinstance(ast, UnaryAdd):
        return UnaryAdd(_diff_ast(ast.expr, wrt))

def _product_deriv(terms, wrt):
    """
    Return an AST expressing the derivative of the product of all the terms.
    """
    if len(terms) == 1:
        return _diff_ast(terms[0], wrt)
    deriv_terms = []
    for ii, term in enumerate(terms):
        term_d = _diff_ast(term, wrt)
        other_terms = terms[:ii] + terms[ii+1:]
        deriv_terms.append(AST._make_product(other_terms + [term_d]))
    sum = deriv_terms[0]
    for term in deriv_terms[1:]:
        sum = Add((term, sum))

    return sum
<|endoftext|>


<commit_before>"""Support for HDF5 files."""

from __future__ import absolute_import, division, print_function
from collections import OrderedDict

import numpy as np
import h5py

import dwi.util

DEFAULT_DSETNAME = 'default'

def write_hdf5(filename, array, attrs, dsetname=DEFAULT_DSETNAME):
    """Write an array with attributes into a newly created, compressed HDF5
    file.
    """
    f = h5py.File(filename, 'w')
    dset = f.create_dataset(dsetname, data=array, compression='gzip',
                            shuffle=True, fletcher32=True)
    for k, v in attrs.iteritems():
        # HDF5 doesn't understand None objects, so replace any with nan values.
        if dwi.util.iterable(v) and not isinstance(v, str) and None in v:
            v = type(v)([(np.nan if x is None else x) for x in v])
        dset.attrs[k] = v
    f.close()

def read_hdf5(filename, dsetname=DEFAULT_DSETNAME):
    """Read an array with attributes from an HDF5 file."""
    try:
        f = h5py.File(filename, 'r')
    except IOError, e:
        if e.filename is None:
            e.filename = filename
        raise
    if dsetname not in f:
        if len(f.keys()) != 1:
            raise ValueError('Ambiguous content: {}'.format(filename))
        dsetname = f.keys()[0]
    dset = f[dsetname]
    array = np.array(dset)
    attrs = OrderedDict(dset.attrs)
    f.close()
    return array, attrs
<commit_msg>removed a direct comparison to None<commit_after>"""Support for HDF5 files."""

from __future__ import absolute_import, division, print_function
from collections import OrderedDict

import numpy as np
import h5py

import dwi.util

DEFAULT_DSETNAME = 'default'

def write_hdf5(filename, array, attrs, dsetname=DEFAULT_DSETNAME):
    """Write an array with attributes into a newly created, compressed HDF5
    file.
    """
    f = h5py.File(filename, 'w')
    dset = f.create_dataset(dsetname, data=array, compression='gzip',
                            shuffle=True, fletcher32=True)
    for k, v in attrs.iteritems():
        # HDF5 doesn't understand None objects, so replace any with nan values.
        if dwi.util.iterable(v) and not isinstance(v, str):
            v = type(v)([(np.nan if x is None else x) for x in v])
        dset.attrs[k] = v
    f.close()

def read_hdf5(filename, dsetname=DEFAULT_DSETNAME):
    """Read an array with attributes from an HDF5 file."""
    try:
        f = h5py.File(filename, 'r')
    except IOError, e:
        if e.filename is None:
            e.filename = filename
        raise
    if dsetname not in f:
        if len(f.keys()) != 1:
            raise ValueError('Ambiguous content: {}'.format(filename))
        dsetname = f.keys()[0]
    dset = f[dsetname]
    array = np.array(dset)
    attrs = OrderedDict(dset.attrs)
    f.close()
    return array, attrs
<|endoftext|>


<commit_before>#
# Copyright (C) 2010-2017 Samuel Abels
# The MIT License (MIT)
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files
# (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge,
# publish, distribute, sublicense, and/or sell copies of the Software,
# and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
The otp module implements the RFC2289 (previously RFC1938) standard
for One-Time Passwords.

For details of RFC2289 adherence, module limitations, and a to-do list,
please see __init__.py

<insert Python license here>
"""
from __future__ import print_function
from __future__ import absolute_import

__version__ = '$Revision: 1.4 $'

import string, random
from Crypto.Hash import MD4
try:
    from hashlib import sha1 as SHA
    from hashlib import md5 as MD5
except ImportError:
    from Crypto.Hash import SHA, MD5
from .AppendixB import DefaultDictionary
from .keywrangling import keyformat, convertkey

_VALIDSEEDCHARACTERS = string.letters + string.digits

_HASHMODULE = { 'md4': MD4, 'md5' : MD5, 'sha' : SHA }

def _fold_md5(digest):
    result = ''
    if len(digest)<16:
        print(digest)
        raise AssertionError
    for i in range(0,8):
        result = result + chr(ord(digest[i]) ^ ord(digest[i+8]))
    return result

def _fold_sha(hash):
    # BROKEN
    ordhash = map(ord, hash)
    result = [0, 0, 0, 0, 0, 0, 0, 0]

    result[3] = ordhash[0] ^ ordhash[8] ^  ordhash[16]
    result[2] = ordhash[1] ^ ordhash[9] ^  ordhash[17]
    result[1] = ordhash[2] ^ ordhash[10] ^ ordhash[18]
    result[0] = ordhash[3] ^ ordhash[11] ^ ordhash[19]
    result[7] = ordhash[4] ^ ordhash[12]
    result[6] = ordhash[5] ^ ordhash[13]
    result[5] = ordhash[6] ^ ordhash[14]
    result[4] = ordhash[7] ^ ordhash[15]

    return ''.join(map(chr, result))

_FOLDFUNCTION = { 'md4': _fold_md5, 'md5' : _fold_md5, 'sha' : _fold_sha }

def generate(passphrase, seed,
             startkey=0, numkeys=499, hashfunction='md5',
             keyformat = 'hex'):
    """Generate a sequence of OTP keys from a pass phrase and seed
    using the specified hash function. Results are returned as a
    list of long integers suitable for conversion to six-word or
    hexadecimal one-time passwords.

    passphrase   -- the shared secret pass phrase as supplied by the
                    user over a secure connection and stored by the
                    OTP server.
    seed         -- the seed phrase used both to initialize keys and
                    sent in the clear by OTP servers as part of the
                    OTP challenge.
    startkey     -- the number of iterations to run before the first
                    key in the result list is taken.
    numkeys      -- the number of keys to return in the result list.
    hashfunction -- the hash function to use when generating keys.
    keyformat    -- the key format to generate
    """

    # check arguments for validity and standards compliance
    if hashfunction not in _HASHMODULE.keys():
        raise Exception('hashfunction')
    if len(passphrase) not in range(4,64):
        raise Exception('passphrase length')
    if len(seed) not in range(1,17):
        raise Exception('seed length')
    for x in seed:
        if not x in _VALIDSEEDCHARACTERS:
            raise Exception('seed composition')
    if startkey < 0:
        raise Exception('startkey')
    if numkeys < 1:
        raise Exception('numkeys')
    # not checked: argument types, startkey and numkeys out of range

    hashmodule = _HASHMODULE[hashfunction]
    folder = _FOLDFUNCTION[hashfunction]

    hash = folder(hashmodule.new(seed + passphrase).digest())

    # discard the first <startkey> keys
    for iterations in range(0, startkey):
        hash = folder(hashmodule.new(hash).digest())

    # generate the results
    keylist = []

    for keys in range(0, numkeys):
        keylist.append(hash)
        hash = folder(hashmodule.new(hash).digest())

    return convertkey(keyformat,keylist)

def generateseed(length=5):
    """Generate a random, valid seed of a given length."""
    # check standards compliance of arguments
    if length not in range(1,11):
        raise Exception('length')
    seed = ''
    vsclen = len(_VALIDSEEDCHARACTERS)
    bignum = 2**32 - 1
    for i in range(0, length):
        index = long(random.random() * bignum) % vsclen
        seed = seed + _VALIDSEEDCHARACTERS[index]
    return seed
<commit_msg>According to RFC2289 ascii_letters is correct<commit_after>#
# Copyright (C) 2010-2017 Samuel Abels
# The MIT License (MIT)
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files
# (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge,
# publish, distribute, sublicense, and/or sell copies of the Software,
# and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
The otp module implements the RFC2289 (previously RFC1938) standard
for One-Time Passwords.

For details of RFC2289 adherence, module limitations, and a to-do list,
please see __init__.py

<insert Python license here>
"""
from __future__ import print_function
from __future__ import absolute_import

__version__ = '$Revision: 1.4 $'

import string, random
from Crypto.Hash import MD4
try:
    from hashlib import sha1 as SHA
    from hashlib import md5 as MD5
except ImportError:
    from Crypto.Hash import SHA, MD5
from .AppendixB import DefaultDictionary
from .keywrangling import keyformat, convertkey

_VALIDSEEDCHARACTERS = string.ascii_letters + string.digits

_HASHMODULE = { 'md4': MD4, 'md5' : MD5, 'sha' : SHA }

def _fold_md5(digest):
    result = ''
    if len(digest)<16:
        print(digest)
        raise AssertionError
    for i in range(0,8):
        result = result + chr(ord(digest[i]) ^ ord(digest[i+8]))
    return result

def _fold_sha(hash):
    # BROKEN
    ordhash = map(ord, hash)
    result = [0, 0, 0, 0, 0, 0, 0, 0]

    result[3] = ordhash[0] ^ ordhash[8] ^  ordhash[16]
    result[2] = ordhash[1] ^ ordhash[9] ^  ordhash[17]
    result[1] = ordhash[2] ^ ordhash[10] ^ ordhash[18]
    result[0] = ordhash[3] ^ ordhash[11] ^ ordhash[19]
    result[7] = ordhash[4] ^ ordhash[12]
    result[6] = ordhash[5] ^ ordhash[13]
    result[5] = ordhash[6] ^ ordhash[14]
    result[4] = ordhash[7] ^ ordhash[15]

    return ''.join(map(chr, result))

_FOLDFUNCTION = { 'md4': _fold_md5, 'md5' : _fold_md5, 'sha' : _fold_sha }

def generate(passphrase, seed,
             startkey=0, numkeys=499, hashfunction='md5',
             keyformat = 'hex'):
    """Generate a sequence of OTP keys from a pass phrase and seed
    using the specified hash function. Results are returned as a
    list of long integers suitable for conversion to six-word or
    hexadecimal one-time passwords.

    passphrase   -- the shared secret pass phrase as supplied by the
                    user over a secure connection and stored by the
                    OTP server.
    seed         -- the seed phrase used both to initialize keys and
                    sent in the clear by OTP servers as part of the
                    OTP challenge.
    startkey     -- the number of iterations to run before the first
                    key in the result list is taken.
    numkeys      -- the number of keys to return in the result list.
    hashfunction -- the hash function to use when generating keys.
    keyformat    -- the key format to generate
    """

    # check arguments for validity and standards compliance
    if hashfunction not in _HASHMODULE.keys():
        raise Exception('hashfunction')
    if len(passphrase) not in range(4,64):
        raise Exception('passphrase length')
    if len(seed) not in range(1,17):
        raise Exception('seed length')
    for x in seed:
        if not x in _VALIDSEEDCHARACTERS:
            raise Exception('seed composition')
    if startkey < 0:
        raise Exception('startkey')
    if numkeys < 1:
        raise Exception('numkeys')
    # not checked: argument types, startkey and numkeys out of range

    hashmodule = _HASHMODULE[hashfunction]
    folder = _FOLDFUNCTION[hashfunction]

    hash = folder(hashmodule.new(seed + passphrase).digest())

    # discard the first <startkey> keys
    for iterations in range(0, startkey):
        hash = folder(hashmodule.new(hash).digest())

    # generate the results
    keylist = []

    for keys in range(0, numkeys):
        keylist.append(hash)
        hash = folder(hashmodule.new(hash).digest())

    return convertkey(keyformat,keylist)

def generateseed(length=5):
    """Generate a random, valid seed of a given length."""
    # check standards compliance of arguments
    if length not in range(1,11):
        raise Exception('length')
    seed = ''
    vsclen = len(_VALIDSEEDCHARACTERS)
    bignum = 2**32 - 1
    for i in range(0, length):
        index = long(random.random() * bignum) % vsclen
        seed = seed + _VALIDSEEDCHARACTERS[index]
    return seed
<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# written by Benedikt Waldvogel

from flask import Flask, render_template, flash, redirect, request, url_for
from flask_bootstrap import Bootstrap
from flask_login import login_user, current_user, login_required, logout_user
from model import db, Move, Sample
from datetime import timedelta, date, datetime
from sqlalchemy.sql import func
import os
from flask_bcrypt import Bcrypt
import old_xml_import
import sml_import
import gpx_export
import dateutil.parser
import gzip
from flask.helpers import make_response
from flask_script import Manager, Server
from flask_migrate import Migrate, MigrateCommand
from commands import AddUser
from filters import register_filters
from login import login_manager, load_user, LoginForm

app = Flask('openmoves')

app_bcrypt = Bcrypt()
migrate = Migrate(app, db)

register_filters(app)

def initialize_config(f):

    random_bytes = os.urandom(32)

    if isinstance(random_bytes[0], str):
        random_bytes = [ord(c) for c in random_bytes]

    data = "SECRET_KEY = '%s'\n" % "".join("{:02x}".format(c) for c in random_bytes)
    f.write(data)

def init(configFile):
    app.config.from_pyfile('openmoves.cfg.default', silent=False)
    if configFile:
        if not os.path.exists(configFile):
            with open(configFile, 'w') as f:
                initialize_config(f)
            print("created %s" % configFile)

        app.config.from_pyfile(configFile, silent=False)
        assert app.config["SECRET_KEY"]

    db.init_app(app)

    with app.app_context():
        if db.engine.name == 'sqlite':
            db.create_all()

    Bootstrap(app)
    app_bcrypt.init_app(app)

    login_manager.init_app(app)
    login_manager.login_view = "login"

    return app

def command_app_context():
    app.config.update(SQLALCHEMY_ECHO=False)
    return app.app_context()

manager = Manager(init)

manager.add_option('-c', '--config', dest='configFile', default='openmoves.cfg', required=False)

manager.add_command("runserver", Server(use_debugger=True))
manager.add_command('db', MigrateCommand)
manager.add_command('add-user', AddUser(command_app_context, app_bcrypt))

@app.errorhandler(404)
def error404(error):
    return render_template('_404.html'), 404

@app.route('/import', methods=['GET', 'POST'])
@login_required
def moveImport():
    xmlfiles = request.files.getlist('files')
    importedMoves = []

    for xmlfile in xmlfiles:
        move = None
        filename = xmlfile.filename
        if filename.endswith('.gz'):
            xmlfile = gzip.GzipFile(fileobj=xmlfile, mode='rb', filename=filename)
            filename = filename[:-len('.gz')]

        if filename.endswith('.xml'):
            move = old_xml_import.oldXmlImport(xmlfile)
        elif filename.endswith('.sml'):
            move = sml_import.smlImport(xmlfile)
        else:
            flash("unknown fileformat: '%s'" % xmlfile.filename, 'error')

        move.temperatureAvg = db.session.query(func.avg(Sample.temperature)).filter(Sample.move == move, Sample.temperature > 0).one()
        db.session.commit()

        if move:
            importedMoves.append(move)

    if importedMoves:
        if len(importedMoves) == 1:
            move = importedMoves[0]
            flash("imported '%s': move %d" % (xmlfile.filename, move.id))
            return redirect(url_for('move', id=move.id))
        else:
            flash("imported %d moves" % len(importedMoves))
            return redirect(url_for('moves'))
    else:
        return render_template('import.html')

@app.route('/login', methods=['GET', 'POST'])
def login():
    form = LoginForm()

    if form.validate_on_submit():
        user = load_user(username=form.username.data)
        if not user:
            flash("no such user", 'error')
            return render_template('login.html', form=form)

        if app_bcrypt.check_password_hash(user.password, form.password.data):
            login_user(user)

            return redirect(request.args.get('next') or url_for('moves'))
        else:
            flash("login failed", 'error')
            return render_template('login.html', form=form)

    return render_template('login.html', form=form)

@app.route("/logout")
@login_required
def logout():
    logout_user()
    return redirect(url_for('index'))

@app.route("/")
def index():
    nrOfMoves = Move.query.count()

    return render_template('index.html', nrOfMoves=nrOfMoves)

@app.route("/dashboard")
@login_required
def dashboard():
    now = datetime.utcnow()
    # Determine dashboad startDate
    if 'startDate' in request.args:
        startDate = dateutil.parser.parse(request.args.get('startDate'))
    else:
        startDate = date(now.year, now.month, now.day) - timedelta(days=7)
    # Determine dashboad endDate
    if 'endDate' in request.args:
        endDate = dateutil.parser.parse(request.args.get('startDate'))
    else:
        endDate = date(now.year, now.month, now.day)

    moves = Move.query.filter_by(user=current_user).filter(Move.dateTime >= startDate).filter(Move.dateTime <= endDate).order_by(Move.dateTime.asc()).all()

    totalDistance = 0
    totalDuration = timedelta(0)
    totalAscent = 0
    totalDescent = 0
    for move in moves:
        totalDistance += move.distance
        totalDuration += move.duration
        if move.ascent:
            totalAscent += move.ascent
        if move.descent:
            totalDescent += move.descent

    nrOfMoves = len(moves)

    return render_template('dashboard.html', moves=moves,
                           startDate=startDate, endDate=endDate,
                           nrOfMoves=nrOfMoves,
                           totalDistance=totalDistance, totalDuration=totalDuration,
                           totalAscent=totalAscent, totalDescent=totalDescent)

@app.route("/moves")
@login_required
def moves():
    moves = moves = Move.query.filter_by(user=current_user)
    totalMovesCount = moves.count()
    sort = request.args.get('sort')
    sortOrder = request.args.get('sortOrder')
    sortDefault = 'dateTime'
    if not sort:
        sort = sortDefault
        sortOrder = 'desc'
    if not sortOrder:
        sortOrder = 'asc'

    if not hasattr(Move, sort):
        flash("illegal sort field: %s" % sort, 'error')
        sort = sortDefault

    sortAttr = getattr(Move, sort)
    if not sortOrder or sortOrder == 'asc':
        sortAttr = sortAttr.asc()
    else:
        sortAttr = sortAttr.desc()

    if db.engine.name == "postgresql":
        sortAttr = sortAttr.nullslast()

    moves = moves.order_by(sortAttr)
    return render_template('moves.html', moves=moves, totalMovesCount=totalMovesCount, sort=sort, sortOrder=sortOrder)

@app.route("/moves/<int:id>/delete")
@login_required
def deleteMove(id):
    move = Move.query.filter_by(user=current_user, id=id).first_or_404()
    Sample.query.filter_by(move=move).delete(synchronize_session=False)
    db.session.delete(move)
    db.session.commit()
    flash("move %d deleted" % id, 'success')

    return redirect(url_for('moves'))

@app.route("/moves/<int:id>/export")
@login_required
def exportMove(id):
    move = Move.query.filter_by(user=current_user, id=id).first_or_404()

    if "format" in request.args:
        format = request.args.get("format").lower()
    else:
        format = "gpx"  # default

    formatHandlers = {}
    formatHandlers['gpx'] = gpx_export.gpxExport
    # formatHandlers['tcx'] = exportTcx
    if format not in formatHandlers:
        flash("Export format %s not supported" % format, 'error')
        return redirect(url_for('move', id=id))

    exportFile = formatHandlers[format](move)
    if not exportFile:
        return redirect(url_for('move', id=id))
    response = make_response(exportFile)
    dateTime = move.dateTime.strftime("%Y-%m-%dT%H:%M:%S")
    response.headers["Content-Disposition"] = "attachment; filename= %s_%s_%s.%s" % (dateTime, move.activity, move.id, format)
    return response

@app.route("/moves/<int:id>")
@login_required
def move(id):
    move = Move.query.filter_by(user=current_user, id=id).first_or_404()

    samples = move.samples.order_by('time asc').all()
    events = [sample for sample in samples if sample.events]

    filteredEvents = []
    pauses = []
    laps = []
    pauseBegin = None
    for sample in events:
        assert len(sample.events.keys()) == 1
        if 'pause' in sample.events:
            state = sample.events['pause']['state'].lower() == 'true'
            if state:
                pauseBegin = sample
            elif not state and pauseBegin:
                pauses.append([pauseBegin, sample])
        elif 'lap' in sample.events:
            laps.append(sample)
        else:
            filteredEvents.append(sample)

    gpsSamples = [sample for sample in samples if sample.sampleType and sample.sampleType.startswith('gps-')]

    # gpsSamples = [sample for sample in gpsSamples if not sample.ehpe or sample.ehpe < 30]

    activity = "".join([a[0].upper() + a[1:] for a in move.activity.split(" ")])
    activity = activity[0].lower() + activity[1:]
    return render_template("move/%s.html" % activity, move=move, samples=samples, events=filteredEvents, pauses=pauses, laps=laps, gpsSamples=gpsSamples)

if __name__ == '__main__':
    manager.run()
<commit_msg>small refactoring<commit_after>#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# written by Benedikt Waldvogel

from flask import Flask, render_template, flash, redirect, request, url_for
from flask_bootstrap import Bootstrap
from flask_login import login_user, current_user, login_required, logout_user
from model import db, Move, Sample
from datetime import timedelta, date, datetime
from sqlalchemy.sql import func
import os
from flask_bcrypt import Bcrypt
import old_xml_import
import sml_import
import gpx_export
import dateutil.parser
import gzip
from flask.helpers import make_response
from flask_script import Manager, Server
from flask_migrate import Migrate, MigrateCommand
from commands import AddUser
from filters import register_filters
from login import login_manager, load_user, LoginForm

app = Flask('openmoves')

app_bcrypt = Bcrypt()
migrate = Migrate(app, db)

register_filters(app)

def initialize_config(f):

    random_bytes = os.urandom(32)

    if isinstance(random_bytes[0], str):
        random_bytes = [ord(c) for c in random_bytes]

    data = "SECRET_KEY = '%s'\n" % "".join("{:02x}".format(c) for c in random_bytes)
    f.write(data)

def init(configFile):
    app.config.from_pyfile('openmoves.cfg.default', silent=False)
    if configFile:
        if not os.path.exists(configFile):
            with open(configFile, 'w') as f:
                initialize_config(f)
            print("created %s" % configFile)

        app.config.from_pyfile(configFile, silent=False)
        assert app.config["SECRET_KEY"]

    db.init_app(app)

    with app.app_context():
        if db.engine.name == 'sqlite':
            db.create_all()

    Bootstrap(app)
    app_bcrypt.init_app(app)

    login_manager.init_app(app)
    login_manager.login_view = "login"

    return app

def command_app_context():
    app.config.update(SQLALCHEMY_ECHO=False)
    return app.app_context()

manager = Manager(init)

manager.add_option('-c', '--config', dest='configFile', default='openmoves.cfg', required=False)

manager.add_command("runserver", Server(use_debugger=True))
manager.add_command('db', MigrateCommand)
manager.add_command('add-user', AddUser(command_app_context, app_bcrypt))

@app.errorhandler(404)
def error404(error):
    return render_template('_404.html'), 404

@app.route('/import', methods=['GET', 'POST'])
@login_required
def moveImport():
    xmlfiles = request.files.getlist('files')
    importedMoves = []

    for xmlfile in xmlfiles:
        move = None
        filename = xmlfile.filename
        if filename.endswith('.gz'):
            xmlfile = gzip.GzipFile(fileobj=xmlfile, mode='rb', filename=filename)
            filename = filename[:-len('.gz')]

        if filename.endswith('.xml'):
            move = old_xml_import.oldXmlImport(xmlfile)
        elif filename.endswith('.sml'):
            move = sml_import.smlImport(xmlfile)
        else:
            flash("unknown fileformat: '%s'" % xmlfile.filename, 'error')

        move.temperatureAvg = db.session.query(func.avg(Sample.temperature)).filter(Sample.move == move, Sample.temperature > 0).one()
        db.session.commit()

        if move:
            importedMoves.append(move)

    if importedMoves:
        if len(importedMoves) == 1:
            move = importedMoves[0]
            flash("imported '%s': move %d" % (xmlfile.filename, move.id))
            return redirect(url_for('move', id=move.id))
        else:
            flash("imported %d moves" % len(importedMoves))
            return redirect(url_for('moves'))
    else:
        return render_template('import.html')

@app.route('/login', methods=['GET', 'POST'])
def login():
    form = LoginForm()

    if form.validate_on_submit():
        user = load_user(username=form.username.data)
        if not user:
            flash("no such user", 'error')
            return render_template('login.html', form=form)

        if app_bcrypt.check_password_hash(user.password, form.password.data):
            login_user(user)

            return redirect(request.args.get('next') or url_for('moves'))
        else:
            flash("login failed", 'error')
            return render_template('login.html', form=form)

    return render_template('login.html', form=form)

@app.route("/logout")
@login_required
def logout():
    logout_user()
    return redirect(url_for('index'))

@app.route("/")
def index():
    nrOfMoves = Move.query.count()

    return render_template('index.html', nrOfMoves=nrOfMoves)

@app.route("/dashboard")
@login_required
def dashboard():
    now = datetime.utcnow()
    # Determine dashboad startDate
    if 'startDate' in request.args:
        startDate = dateutil.parser.parse(request.args.get('startDate'))
    else:
        startDate = date(now.year, now.month, now.day) - timedelta(days=7)
    # Determine dashboad endDate
    if 'endDate' in request.args:
        endDate = dateutil.parser.parse(request.args.get('startDate'))
    else:
        endDate = date(now.year, now.month, now.day)

    moves = Move.query.filter_by(user=current_user).filter(Move.dateTime >= startDate).filter(Move.dateTime <= endDate).order_by(Move.dateTime.asc()).all()

    totalDistance = 0
    totalDuration = timedelta(0)
    totalAscent = 0
    totalDescent = 0
    for move in moves:
        totalDistance += move.distance
        totalDuration += move.duration
        if move.ascent:
            totalAscent += move.ascent
        if move.descent:
            totalDescent += move.descent

    nrOfMoves = len(moves)

    return render_template('dashboard.html', moves=moves,
                           startDate=startDate, endDate=endDate,
                           nrOfMoves=nrOfMoves,
                           totalDistance=totalDistance, totalDuration=totalDuration,
                           totalAscent=totalAscent, totalDescent=totalDescent)

@app.route("/moves")
@login_required
def moves():
    moves = moves = Move.query.filter_by(user=current_user)
    totalMovesCount = moves.count()
    sort = request.args.get('sort')
    sortOrder = request.args.get('sortOrder')
    sortDefault = 'dateTime'
    if not sort:
        sort = sortDefault
        sortOrder = 'desc'
    if not sortOrder:
        sortOrder = 'asc'

    if not hasattr(Move, sort):
        flash("illegal sort field: %s" % sort, 'error')
        sort = sortDefault

    sortAttr = getattr(Move, sort)
    if not sortOrder or sortOrder == 'asc':
        sortAttr = sortAttr.asc()
    else:
        sortAttr = sortAttr.desc()

    if db.engine.name == "postgresql":
        sortAttr = sortAttr.nullslast()

    moves = moves.order_by(sortAttr)
    return render_template('moves.html', moves=moves, totalMovesCount=totalMovesCount, sort=sort, sortOrder=sortOrder)

@app.route("/moves/<int:id>/delete")
@login_required
def deleteMove(id):
    move = Move.query.filter_by(user=current_user, id=id).first_or_404()
    Sample.query.filter_by(move=move).delete(synchronize_session=False)
    db.session.delete(move)
    db.session.commit()
    flash("move %d deleted" % id, 'success')

    return redirect(url_for('moves'))

@app.route("/moves/<int:id>/export")
@login_required
def exportMove(id):
    move = Move.query.filter_by(user=current_user, id=id).first_or_404()

    if "format" in request.args:
        format = request.args.get("format").lower()
    else:
        format = "gpx"  # default

    formatHandlers = {}
    formatHandlers['gpx'] = gpx_export.gpxExport
    # formatHandlers['tcx'] = exportTcx
    if format not in formatHandlers:
        flash("Export format %s not supported" % format, 'error')
        return redirect(url_for('move', id=id))

    exportFile = formatHandlers[format](move)
    if not exportFile:
        return redirect(url_for('move', id=id))
    response = make_response(exportFile)
    dateTime = move.dateTime.strftime("%Y-%m-%dT%H:%M:%S")
    response.headers["Content-Disposition"] = "attachment; filename= %s_%s_%s.%s" % (dateTime, move.activity, move.id, format)
    return response

@app.route("/moves/<int:id>")
@login_required
def move(id):
    move = Move.query.filter_by(user=current_user, id=id).first_or_404()

    samples = move.samples.order_by('time asc').all()
    events = [sample for sample in samples if sample.events]

    filteredEvents = []
    pauses = []
    laps = []
    pauseBegin = None
    for sample in events:
        assert len(sample.events.keys()) == 1
        if 'pause' in sample.events:
            state = sample.events['pause']['state'].lower() == 'true'
            if state:
                pauseBegin = sample
            elif not state and pauseBegin:
                pauses.append([pauseBegin, sample])
        elif 'lap' in sample.events:
            laps.append(sample)
        else:
            filteredEvents.append(sample)

    model = {}
    model['move'] = move
    model['samples'] = samples
    model['events'] = filteredEvents
    model['pauses'] = pauses
    model['laps'] = laps
    model['gpsSamples'] = [sample for sample in samples if sample.sampleType and sample.sampleType.startswith('gps-')]

    # eg. 'Pool swimming' → 'poolSwimming'
    activityName = "".join([a[0].upper() + a[1:] for a in move.activity.split(" ")])
    activityName = activityName[0].lower() + activityName[1:]
    return render_template("move/%s.html" % activityName, **model)

if __name__ == '__main__':
    manager.run()
<|endoftext|>


<commit_before>

import re
import random
from geotext.geotext import GeoText

class text_processing(object):
	'''

	This class has methods to process the text.
	Methods like uppercase, lowercase, tokenize and sentence parser

	'''
	def __init__(self,text):
		self.text = text

	def getUpperCaps(self):
		return self.text.upper()

	def getLowerCaps(self):
		return self.text.lower()

	def Word_Tokenize(self):
		'''
		Splits the sentence into lower case tokens

		'''
		return self.text.lower().split()

class SearchKeyWords(object):

	def __init__(self,string=None):
		self.string = string

	def match_words(self,string):

		'''
		Takes a string as an argument.

		This method matches any keywords like weather, temperature, etcetera.
		if keywords are found, then it returns the keyword

		example :

				>> Whats the weather and temperature in London
				.. ['weather ', 'temperature']

		But when using just use the first element.
			Ex : list[0]

		Since multiple reading functionality is not added yet

		'''

		# regular exxpression to find or match with the string
		self.regex_match = r"[A-Za-z]eath[a-z]*r |[Tt][Ee][A-Za-z]*p[a-z]*"
		# searches for the regex_match in the string
		self.Matched_keyword = re.findall(self.regex_match,string)
		return self.Matched_keyword

	def KeyWords(self):
		'''

		returns the first keyword from the list

		'''
		return self.match_words(self.string)[0]

import Wex_data

class WexWeather(object):

	def __init__(self,string,SendRequest=True,Api_Key):

		'''

		WexWeather class takes 2 arguments, first one is a string.
		And second argument is a flag variable.

		string : Input anything you want, if there is or there are city
				 names in that string, you'll recieve the city names

		SendRequest: This flag is by default set to true. That means it
					 can send requests to get the weather data. Keep it true
					 if you want to send requests. In anycase you want to
					 set it to false, do so. It just won't send any request.
					 This is useful if you just want to identify the city names
					 from a string. Setting it to true will also do the same, but
					 sending a request and recieving the data takes some time.

					 Do this only if you want to extract the city names. But if you want
					 the weather data, set it to true or leave it without passing the
					 argument.

					 example: with SendRequest=True

							 >> Whats the weather in London and New York and San Jose and San Francisco

								# some seconds later

							.. ['London', 'New York', 'San Jose', 'San Francisco']

					example: with SendRequest=False
							 >> Whats the weather in London and New York and San Jose and San Francisco

								# 0.00000001s later. This is fast!

							.. ['London', 'New York', 'San Jose', 'San Francisco']

			Api_Key : Get your api key from openweather.com and use it.

		'''

		self.Keywords = SearchKeyWords(string)
		# uses GeoText() module to Extract the cities
		self.CityName = GeoText(string).cities
		if SendRequest is True:
			self.WeatherObject = Wex_data.GetWeatherConditions(self.CityName[0],Api_Key)
		else:
			pass

	def WexGetWeather(self):
		self.CityList = self.Keywords.KeyWords()
		self.CurrentWeather = self.WeatherObject.GetCurrentWeather()
		self.CurrentTemperature = self.WeatherObject.GetCurrentTemperature()
		return [self.CurrentWeather,self.CurrentTemperature]

	def GetCityNames(self):
		'''
			Returns the city names in a list.

			Useful if you have multiple city names in a string
			and you want to use anyone of the names.

			example:
				>> Whats the weather in London and New York and San Jose and San Francisco

				.. ['London', 'New York', 'San Jose', 'San Francisco']

		'''
		return self.CityName

class ShowWeatherConditions(object):

	def __init__(self,data,place):
		# The data argument can be any list.
		self.data = data
		self.place = place

	def FormatWeatherConditions(self):
		# user friendly responses
		self.WeatherResp = ["It's {2}C and {0} in {1}"]
		return self.WeatherResp[0].format(self.data[0],self.place,self.data[1])

if __name__== '__main__':

	w = WexWeather("whats the weather in London","[API-KEY]")
	print(w.WexGetWeather())<commit_msg>Delete Extract_Weather.py<commit_after><|endoftext|>