<commit_before>#
#   Copyright 2013 Geodelic
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
#

from functools import wraps
from pprint import pformat
from collections import Iterable
import logging
logger = logging.getLogger(__name__)

import boto
import boto.ec2

DEFAULT_TTL = 120
DEFAULT_TYPE = 'CNAME'

route53 = boto.connect_route53()

def memoize(fctn):
    memory = {}
    @wraps(fctn)
    def memo(*args):
        haxh = tuple(args)

        if haxh not in memory:
            memory[haxh] = fctn(*args)

        return memory[haxh]
    return memo

def trampoline(*list_of_lists):
    stack = [list_of_lists]
    iteree = iter(stack)
    while 1:
        try:
            item = iteree.next()
        except StopIteration:
            try:
                iteree = iter(stack.pop())
                continue
            except IndexError:
                break

        if isinstance(item, Iterable) and not isinstance(item, basestring):
            stack.append(iteree)
            iteree = iter(item)
            continue
        yield item

@memoize
def get_records(name):
    zones = route53.get_all_hosted_zones()['ListHostedZonesResponse']['HostedZones']
    name = name.strip('.')
    zones = filter(lambda x: x['Name'].strip('.') == name, zones)
    if zones:
        zone = zones[0]
    else:
        response = route53.create_hosted_zone(name)
        zone = response['CreateHostedZoneResponse']['HostedZone']

    zid = zone['Id'].replace('/hostedzone/', '')
    results = rrsets = route53.get_all_rrsets(zid)

    while rrsets.is_truncated:
        rrsets = route53.get_all_rrsets(zid, name=rrsets.next_record_name, type=rrsets.next_record_type)
        results.extend(rrsets)

    return results

def delete_record(rrset, record):
    c = rrset.add_change('DELETE', record.name, record.type, record.ttl)
    for v in record.resource_records:
        c.add_value(v)
    return rrset

def find_record(rrset, name):
    name = name.strip('.')
    return filter(lambda x: x.name.strip('.') == name, rrset)[0]

def process_record(rrset, name, value):
    name = name.strip('.')
    try:
        record = find_record(rrset, name)
    except IndexError:
        logger.info("New: %s" % name)
        pass
    else:
        if record.resource_records[0] == value:
            logger.debug("Unchanged: %s" % name)
            return rrset

        logger.info("Updating: %s" % name)
        logger.debug("Name: %s, Old values: %r, New value: %r" % (name, record.resource_records, value))
        rrset = delete_record(rrset, record)

    c = rrset.add_change('CREATE', name, 'CNAME', DEFAULT_TTL)
    c.add_value(value)

    return rrset

def gatherinstances():
    regions = (region.connect() for region in boto.ec2.regions())
    reservations = (region.get_all_instances() for region in regions)
    instances = (reservation.instances for reservation in trampoline(reservations))

    return trampoline(instances)

def process_all(instances):
    instances = filter(lambda x: x.tags and x.dns_name and all((y in x.tags for y in ('domain_base', 'fqdn', 'deployment'))), instances)

    domain_bases = set(x.tags['domain_base'] for x in instances)

    unseen = {}
    for d in domain_bases:
        unseen[d] = set(r.name.strip('.') for r in get_records(d) if r.type == 'CNAME')

    for instance in instances:
        unseen[instance.tags['domain_base']].discard(instance.tags['fqdn'])
        rrset = get_records(instance.tags['domain_base'])
        process_record(rrset, instance.tags['fqdn'], instance.dns_name)

    arrays = filter(lambda x: x.tags['fqdn'].startswith('array-'), instances)
    array_types = {}

    for array_instance in arrays:
        deployment = array_instance.tags['deployment']

        fqdn = array_instance.tags['fqdn']
        fqdn_base = fqdn[fqdn.index('.'):]

        if 'type' in array_instance.tags:
            array_type = array_instance.tags['type']
        else:
            array_type = fqdn[:fqdn.index('.')][:-3]
            array_type = array_type.replace('array-','')

        fqdn_alias = array_type + 'XX' + fqdn_base

        atypedeploy = array_types.setdefault(deployment, {})
        atypes = atypedeploy.setdefault(array_type, [])
        atypes.append((fqdn_alias, array_instance))

    for deploy,atypesdict in array_types.iteritems():
        for atype,v in atypesdict.iteritems():
            v = sorted(v, key=lambda x: x[1].launch_time)
            for e,(fqdn_alias,array_instance) in enumerate(v):
                e = str(e+1).zfill(2)
                fqdn = fqdn_alias.replace('XX', e)

                unseen[array_instance.tags['domain_base']].discard(fqdn)
                rrset = get_records(array_instance.tags['domain_base'])
                process_record(rrset, fqdn, array_instance.dns_name)

    for d,records in unseen.iteritems():
        rrset = get_records(d)
        for rname in records:
            logger.info("Deleting: %s" % rname)
            record = find_record(rrset, rname)
            delete_record(rrset, record)

    for d in domain_bases:
        rrset = get_records(d)
        if rrset.changes:
            logger.info("committing changes for domain base: %s" % d)
            logger.debug(pformat(rrset.changes))
            rrset.commit()

def do_update():
    process_all(gatherinstances())

if __name__ == '__main__':
    from time import time
    logger.addHandler(logging.StreamHandler())
    logger.setLevel(logging.DEBUG)
    t = time()
    do_update()
    print('took %f seconds to run' % (time() - t))

<commit_msg>Refactor instance filter to be more leinient<commit_after>#
#   Copyright 2013 Geodelic
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
#

from functools import wraps
from pprint import pformat
from collections import Iterable
import logging
logger = logging.getLogger(__name__)

import boto
import boto.ec2

DEFAULT_TTL = 120
DEFAULT_TYPE = 'CNAME'

route53 = boto.connect_route53()

def memoize(fctn):
    memory = {}
    @wraps(fctn)
    def memo(*args):
        haxh = tuple(args)

        if haxh not in memory:
            memory[haxh] = fctn(*args)

        return memory[haxh]
    return memo

def trampoline(*list_of_lists):
    stack = [list_of_lists]
    iteree = iter(stack)
    while 1:
        try:
            item = iteree.next()
        except StopIteration:
            try:
                iteree = iter(stack.pop())
                continue
            except IndexError:
                break

        if isinstance(item, Iterable) and not isinstance(item, basestring):
            stack.append(iteree)
            iteree = iter(item)
            continue
        yield item

@memoize
def get_records(name):
    zones = route53.get_all_hosted_zones()['ListHostedZonesResponse']['HostedZones']
    name = name.strip('.')
    zones = filter(lambda x: x['Name'].strip('.') == name, zones)
    if zones:
        zone = zones[0]
    else:
        response = route53.create_hosted_zone(name)
        zone = response['CreateHostedZoneResponse']['HostedZone']

    zid = zone['Id'].replace('/hostedzone/', '')
    results = rrsets = route53.get_all_rrsets(zid)

    while rrsets.is_truncated:
        rrsets = route53.get_all_rrsets(zid, name=rrsets.next_record_name, type=rrsets.next_record_type)
        results.extend(rrsets)

    return results

def delete_record(rrset, record):
    c = rrset.add_change('DELETE', record.name, record.type, record.ttl)
    for v in record.resource_records:
        c.add_value(v)
    return rrset

def find_record(rrset, name):
    name = name.strip('.')
    return filter(lambda x: x.name.strip('.') == name, rrset)[0]

def process_record(rrset, name, value):
    name = name.strip('.')
    try:
        record = find_record(rrset, name)
    except IndexError:
        logger.info("New: %s" % name)
        pass
    else:
        if record.resource_records[0] == value:
            logger.debug("Unchanged: %s" % name)
            return rrset

        logger.info("Updating: %s" % name)
        logger.debug("Name: %s, Old values: %r, New value: %r" % (name, record.resource_records, value))
        rrset = delete_record(rrset, record)

    c = rrset.add_change('CREATE', name, 'CNAME', DEFAULT_TTL)
    c.add_value(value)

    return rrset

def gatherinstances():
    regions = (region.connect() for region in boto.ec2.regions())
    reservations = (region.get_all_instances() for region in regions)
    instances = (reservation.instances for reservation in trampoline(reservations))

    return trampoline(instances)

def process_all(instances):
    instances = filter(lambda x: x.tags and x.dns_name and all((y in x.tags for y in ('domain_base', 'fqdn'))), instances)

    domain_bases = set(x.tags['domain_base'] for x in instances)

    unseen = {}
    for d in domain_bases:
        unseen[d] = set(r.name.strip('.') for r in get_records(d) if r.type == 'CNAME')

    for instance in instances:
        unseen[instance.tags['domain_base']].discard(instance.tags['fqdn'])
        rrset = get_records(instance.tags['domain_base'])
        process_record(rrset, instance.tags['fqdn'], instance.dns_name)

    arrays = filter(lambda x: x.tags['fqdn'].startswith('array-') and \
                    all((_ in x.tags for _ in ('deployment', 'type'))), instances)
    array_types = {}

    for array_instance in arrays:
        deployment = array_instance.tags['deployment']

        fqdn = array_instance.tags['fqdn']
        fqdn_base = fqdn[fqdn.index('.'):]

        array_type = array_instance.tags['type']

        fqdn_alias = array_type + 'XX' + fqdn_base

        atypedeploy = array_types.setdefault(deployment, {})
        atypes = atypedeploy.setdefault(array_type, [])
        atypes.append((fqdn_alias, array_instance))

    for deploy,atypesdict in array_types.iteritems():
        for atype,v in atypesdict.iteritems():
            v = sorted(v, key=lambda x: x[1].launch_time)
            for e,(fqdn_alias,array_instance) in enumerate(v):
                e = str(e+1).zfill(2)
                fqdn = fqdn_alias.replace('XX', e)

                unseen[array_instance.tags['domain_base']].discard(fqdn)
                rrset = get_records(array_instance.tags['domain_base'])
                process_record(rrset, fqdn, array_instance.dns_name)

    for d,records in unseen.iteritems():
        rrset = get_records(d)
        for rname in records:
            logger.info("Deleting: %s" % rname)
            record = find_record(rrset, rname)
            delete_record(rrset, record)

    for d in domain_bases:
        rrset = get_records(d)
        if rrset.changes:
            logger.info("committing changes for domain base: %s" % d)
            logger.debug(pformat(rrset.changes))
            rrset.commit()

def do_update():
    process_all(gatherinstances())

if __name__ == '__main__':
    from time import time
    logger.addHandler(logging.StreamHandler())
    logger.setLevel(logging.DEBUG)
    t = time()
    do_update()
    print('took %f seconds to run' % (time() - t))

<|endoftext|>


<commit_before>from .webhdfs import WebHDFS
from .oozie import Oozie,Job
__all__ = [ 'WebHDFS','Oozie','Job']
__version__ = '0.2.0'
<commit_msg>updated version<commit_after>from .webhdfs import WebHDFS
from .oozie import Oozie,Job
__all__ = [ 'WebHDFS','Oozie','Job']
__version__ = '0.3.0'
<|endoftext|>


<commit_before># Copyright 2011,2012 James McCauley
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pox.lib.revent.revent import *
from pox.core import core as core
from pox.messenger import *

log = core.getLogger()

from pox.lib.recoco.recoco import *

class TCPConnection (Connection, Task):
  def __init__ (self, transport, socket):
    self._socket = socket
    # Note: we cache name of the socket because socket.getpeername()
    # is unavailable after the socket was closed!
    self._socket_name = self._get_socket_name(socket)
    Connection.__init__(self, transport)
    Task.__init__(self)

    #self.start()
    self._send_welcome()

  def _close (self):
    super(TCPConnection, self)._close()
    try:
      self._socket.shutdown(socket.SHUT_RDWR)
    except:
      pass

  def send_raw (self, data):
    try:
      l = self._socket.send(data)
      if l == len(data): return
    except:
      pass
    #TODO: do something more graceful!
    self._close()

  def run (self):
    log.debug("%s started" % (self,))
    while self.is_connected:
      d = yield Recv(self._socket)
      if d is None or len(d) == 0:
        break
      #print "RECV", d
      self._rx_raw(d)
    self._close()
    log.debug("%s stopped" % (self,))

  def __str__ (self):
    s = "" + self.__class__.__name__ + " " + self._socket_name
    return s

  @staticmethod
  def _get_socket_name(socket):
    s = "%s:%i" % socket.getsockname()
    s += "/%s:%i" % socket.getpeername()
    return s

class TCPTransport (Task, Transport):
  def __init__ (self, address = "0.0.0.0", port = 7790, nexus = None):
    port = int(port)
    Task.__init__(self)
    Transport.__init__(self, nexus)
    self._addr = (address,port)
    self._connections = set()

  def _forget (self, connection):
    """ Forget about a connection (because it has closed) """
    if connection in self._connections:
      #print "forget about",connection
      self._connections.remove(connection)

  def run (self):
    listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    listener.bind(self._addr)
    listener.listen(0)

    log.debug("Listening on %s:%i" % (self._addr))

    con = None
    while core.running:
      try:
        rlist, wlist, elist = yield Select([listener])
        if len(rlist) == 0:
          # Must have been interrupted
          break

        rc = TCPConnection(self, listener.accept()[0])
        self._connections.add(rc)
        rc.start()
      except:
        traceback.print_exc()
        break

    try:
      listener.close()
    except:
      pass
    log.debug("No longer listening for connections")

import pox.core

def launch (tcp_address = "0.0.0.0", tcp_port = 7790):
  def start ():
    t = TCPTransport(tcp_address, tcp_port)
    t.start()
  core.call_when_ready(start, "MessengerNexus", __name__)
<commit_msg>messenger.tcp_transport: Clean up imports<commit_after># Copyright 2011,2012 James McCauley
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pox.lib.revent import *
from pox.lib.recoco import *
from pox.core import core
from pox.messenger import *

log = core.getLogger()

class TCPConnection (Connection, Task):
  def __init__ (self, transport, socket):
    self._socket = socket
    # Note: we cache name of the socket because socket.getpeername()
    # is unavailable after the socket was closed!
    self._socket_name = self._get_socket_name(socket)
    Connection.__init__(self, transport)
    Task.__init__(self)

    #self.start()
    self._send_welcome()

  def _close (self):
    super(TCPConnection, self)._close()
    try:
      self._socket.shutdown(socket.SHUT_RDWR)
    except:
      pass

  def send_raw (self, data):
    try:
      l = self._socket.send(data)
      if l == len(data): return
    except:
      pass
    #TODO: do something more graceful!
    self._close()

  def run (self):
    log.debug("%s started" % (self,))
    while self.is_connected:
      d = yield Recv(self._socket)
      if d is None or len(d) == 0:
        break
      #print "RECV", d
      self._rx_raw(d)
    self._close()
    log.debug("%s stopped" % (self,))

  def __str__ (self):
    s = "" + self.__class__.__name__ + " " + self._socket_name
    return s

  @staticmethod
  def _get_socket_name(socket):
    s = "%s:%i" % socket.getsockname()
    s += "/%s:%i" % socket.getpeername()
    return s

class TCPTransport (Task, Transport):
  def __init__ (self, address = "0.0.0.0", port = 7790, nexus = None):
    port = int(port)
    Task.__init__(self)
    Transport.__init__(self, nexus)
    self._addr = (address,port)
    self._connections = set()

  def _forget (self, connection):
    """ Forget about a connection (because it has closed) """
    if connection in self._connections:
      #print "forget about",connection
      self._connections.remove(connection)

  def run (self):
    listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    listener.bind(self._addr)
    listener.listen(0)

    log.debug("Listening on %s:%i" % (self._addr))

    con = None
    while core.running:
      try:
        rlist, wlist, elist = yield Select([listener])
        if len(rlist) == 0:
          # Must have been interrupted
          break

        rc = TCPConnection(self, listener.accept()[0])
        self._connections.add(rc)
        rc.start()
      except:
        traceback.print_exc()
        break

    try:
      listener.close()
    except:
      pass
    log.debug("No longer listening for connections")

def launch (tcp_address = "0.0.0.0", tcp_port = 7790):
  def start ():
    t = TCPTransport(tcp_address, tcp_port)
    t.start()
  core.call_when_ready(start, "MessengerNexus", __name__)
<|endoftext|>


<commit_before>from .base import *

DEBUG = True
TEMPLATE_DEBUG = DEBUG

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = '/var/www/chembiohub_ws/deployment/static'

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
# Put strings here, like "/home/html/static" or "C:/www/django/static".
# Always use forward slashes, even on Windows.
# Don't forget to use absolute paths, not relative paths.
'/var/www/chembiohub_ws/src/ng-chem',
)

# List of finder classes that know how to find static files in
# various locations.

from .stagingsecret import *<commit_msg>settings<commit_after>from .base import *

DEBUG = True
TEMPLATE_DEBUG = DEBUG

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = '/var/www/chembiohub_ws/deployment/static'

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/reg/'

# Additional locations of static files
STATICFILES_DIRS = (
# Put strings here, like "/home/html/static" or "C:/www/django/static".
# Always use forward slashes, even on Windows.
# Don't forget to use absolute paths, not relative paths.
'/var/www/chembiohub_ws/src/ng-chem',
)

# List of finder classes that know how to find static files in
# various locations.

from .stagingsecret import *<|endoftext|>


<commit_before>#!/usr/bin/env python3

#
# Copyright (c) 2015 SUSE Linux GmbH
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
#

"""SDSC Module that handles CLI-related tasks
"""

from . import __description__, __programname__, __version__

import argparse
import sys

# TODO: Get rid of the entire "positional arguments" thing that argparse adds
# (self-explanatory anyway). Get rid of "optional arguments" header. Make sure
# least important options (--help, --version) are listed last. Also, I really
# liked being able to use sentences in the parameter descriptions.

def parseargs(cliargs=None):
    """Parse command line arguments

    :param list cliargs: Arguments to parse or None (=use sys.argv)
    :return: parsed arguments
    :rtype: argparse.Namespace
    """
    parser = argparse.ArgumentParser(
        usage = "%(prog)s [options] inputfile [outputfile]",
        description = __description__ )
    fileorbookmark = parser.add_mutually_exclusive_group(required=True)
    parser.add_argument('-v', '--version',
        action = 'version',
        version = __programname__ + " " + __version__,
        help = "show version number and exit")
    fileorbookmark.add_argument( '-b', '--bookmarklet',
        action = 'store_true',
        default = False,
        help = """open Web page that lets you install a bookmarklet to manage
            style checker results""" )
    parser.add_argument( '-s', '--show',
        action = 'store_true',
        default = False,
        help = """show final report in $BROWSER, or default browser if unset; not
            all browsers open report files correctly and for some users, a text
            editor will open; in such cases, set the BROWSER variable with:
            export BROWSER=/MY/BROWSER ; Chromium or Firefox will both
            do the right thing""" )
    parser.add_argument( '--module',
        action = 'store_true',
        default = False,
        help = "writes name of current check module to stdout" )
    parser.add_argument( '--performance',
        action = 'store_true',
        default = False,
        help = "write performance measurements to stdout" )
    parser.add_argument( '--checkpatterns',
        action = 'store_true',
        default = False,
        help = """check formal validity of built-in regular expression
            patterns""" )
    fileorbookmark.add_argument( 'inputfile', type=argparse.FileType('r'),
        nargs = "?" )
    parser.add_argument( 'outputfile', nargs = "?" )

    return parser.parse_args(args=cliargs)

def printcolor( message, messagetype = None ):
    """Print a colored message

    :param str message: Message
    :param str messagetype: Type (none => green, 'error' => red)
    """
    if sys.stdout.isatty():
        if type == 'error':
            print( '\033[0;31m' + message + '\033[0m' )
        else:
            print( '\033[0;32m' + message + '\033[0m' )
    else:
        print( message )
<commit_msg>Correct for attribute name change + add debug message type<commit_after>#!/usr/bin/env python3

#
# Copyright (c) 2015 SUSE Linux GmbH
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
#

"""SDSC Module that handles CLI-related tasks
"""

from . import __description__, __programname__, __version__

import argparse
import sys

# TODO: Get rid of the entire "positional arguments" thing that argparse adds
# (self-explanatory anyway). Get rid of "optional arguments" header. Make sure
# least important options (--help, --version) are listed last. Also, I really
# liked being able to use sentences in the parameter descriptions.

def parseargs(cliargs=None):
    """Parse command line arguments

    :param list cliargs: Arguments to parse or None (=use sys.argv)
    :return: parsed arguments
    :rtype: argparse.Namespace
    """
    parser = argparse.ArgumentParser(
        usage = "%(prog)s [options] inputfile [outputfile]",
        description = __description__ )
    fileorbookmark = parser.add_mutually_exclusive_group(required=True)
    parser.add_argument('-v', '--version',
        action = 'version',
        version = __programname__ + " " + __version__,
        help = "show version number and exit")
    fileorbookmark.add_argument( '-b', '--bookmarklet',
        action = 'store_true',
        default = False,
        help = """open Web page that lets you install a bookmarklet to manage
            style checker results""" )
    parser.add_argument( '-s', '--show',
        action = 'store_true',
        default = False,
        help = """show final report in $BROWSER, or default browser if unset; not
            all browsers open report files correctly and for some users, a text
            editor will open; in such cases, set the BROWSER variable with:
            export BROWSER=/MY/BROWSER ; Chromium or Firefox will both
            do the right thing""" )
    parser.add_argument( '--module',
        action = 'store_true',
        default = False,
        help = "writes name of current check module to stdout" )
    parser.add_argument( '--performance',
        action = 'store_true',
        default = False,
        help = "write performance measurements to stdout" )
    parser.add_argument( '--checkpatterns',
        action = 'store_true',
        default = False,
        help = """check formal validity of built-in regular expression
            patterns""" )
    fileorbookmark.add_argument( 'inputfile', type=argparse.FileType('r'),
        nargs = "?" )
    parser.add_argument( 'outputfile', nargs = "?" )

    return parser.parse_args(args=cliargs)

def printcolor( message, messagetype = None ):
    """Print a colored message

    :param str message: Message
    :param str messagetype: Type (none => green, 'error' => red, 'debug' => blue)
    """
    if sys.stdout.isatty():
        if messagetype == 'error':
            print( '\033[0;31m' + message + '\033[0m' )
        elif messagetype == 'debug':
            print( '\033[0;36m' + message + '\033[0m' )
        else:
            print( '\033[0;32m' + message + '\033[0m' )
    else:
        print( message )
<|endoftext|>


<commit_before>#! /usr/bin/env python3

#
#  Copyright 2016 - 2021 gary@drinkingtea.net
#
#  This Source Code Form is subject to the terms of the Mozilla Public
#  License, v. 2.0. If a copy of the MPL was not distributed with this
#  file, You can obtain one at http://mozilla.org/MPL/2.0/.
#

# "Python Busy Box" - adds cross platform equivalents to Unix commands that
#                     don't translate well to that other operating system

import os
import shutil
import subprocess
import sys

def mkdir(path):
    if not os.path.exists(path) and os.path.isdir(path):
        os.mkdir(path)

# this exists because Windows is utterly incapable of providing a proper rm -rf
def rm(path):
    if (os.path.exists(path) or os.path.islink(path)) and not os.path.isdir(path):
        os.remove(path)
    elif os.path.isdir(path):
        shutil.rmtree(path)

def cmake_build(base_path, target):
    if not os.path.isdir(base_path):
        # nothing to build
        return 0
    for d in os.listdir(base_path):
        args = ['cmake', '--build', os.path.join(base_path, d), '--target']
        if target is not None:
            args.append(target)
        err = subprocess.run(args).returncode
        if err != 0:
            return err

def main():
    if sys.argv[1] == 'mkdir':
        mkdir(sys.argv[2])
    elif sys.argv[1] == 'rm':
        for i in range(2, len(sys.argv)):
            rm(sys.argv[i])
    elif sys.argv[1] == 'cmake-build':
        err = cmake_build(sys.argv[2], sys.argv[3] if len(sys.argv) > 3 else None)
        sys.exit(err)

if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        sys.exit(1)

<commit_msg>[buildcore] Remove whole --target option from cmake --build command (for CMake 3.20)<commit_after>#! /usr/bin/env python3

#
#  Copyright 2016 - 2021 gary@drinkingtea.net
#
#  This Source Code Form is subject to the terms of the Mozilla Public
#  License, v. 2.0. If a copy of the MPL was not distributed with this
#  file, You can obtain one at http://mozilla.org/MPL/2.0/.
#

# "Python Busy Box" - adds cross platform equivalents to Unix commands that
#                     don't translate well to that other operating system

import os
import shutil
import subprocess
import sys

def mkdir(path):
    if not os.path.exists(path) and os.path.isdir(path):
        os.mkdir(path)

# this exists because Windows is utterly incapable of providing a proper rm -rf
def rm(path):
    if (os.path.exists(path) or os.path.islink(path)) and not os.path.isdir(path):
        os.remove(path)
    elif os.path.isdir(path):
        shutil.rmtree(path)

def cmake_build(base_path, target):
    if not os.path.isdir(base_path):
        # nothing to build
        return 0
    for d in os.listdir(base_path):
        args = ['cmake', '--build', os.path.join(base_path, d)]
        if target is not None:
            args.extend(['--target', target])
        err = subprocess.run(args).returncode
        if err != 0:
            return err

def main():
    if sys.argv[1] == 'mkdir':
        mkdir(sys.argv[2])
    elif sys.argv[1] == 'rm':
        for i in range(2, len(sys.argv)):
            rm(sys.argv[i])
    elif sys.argv[1] == 'cmake-build':
        err = cmake_build(sys.argv[2], sys.argv[3] if len(sys.argv) > 3 else None)
        sys.exit(err)

if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        sys.exit(1)

<|endoftext|>


<commit_before>from model import *

# testSet = dataset('results/')
with tf.Session() as sess:
    loadModel(sess, model_save_path[1])

    # batch = testSet.test_batch(batch_size)
    batch = prepareImages(['ball_dataset/8.png']*batch_size)
    newBatch = sess.run([view, vox], feed_dict={
        # view: batch[0]
        view: batch
    })

    saveResults(newBatch, fileName='vox.pkl', saveImages=False)<commit_msg>changed this file to use the model and test it<commit_after>from model import *

# testSet = dataset('results/')
ballDataset = dataset('ball_dataset/')
with tf.Session() as sess:
    loadModel(sess, model_save_path[1])

    batch = ballDataset.test_batch(batch_size*12)
    batch = ballDataset.test_batch(batch_size)
    # imgs = prepareImages(batch[0])
    newBatch = sess.run([view, vox], feed_dict={
        view: batch[0]
    })

    saveResults(newBatch, fileName='vox_2.pkl', saveImages=False)

    for i in range(batch_size):
        img = toImage(batch[0][i:i+1])
        img.save('results_ball/%s.png'%(i+5))<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from kivy.clock import Clock

import connection
import model

import sqlite3
import time
import os
import random
import threading
import hashlib

log_levels = ["TRACE","DEBUG","INFO","WARNING","WARN","ERROR","SEVERE"]

class LogManager():
    def __init__(self):
        try:
            os.remove(os.path.join(model.gstore.data_dir,"log.db"))
        except FileNotFoundError:
            print("no database, skipping deleting")

        self.lock = threading.Lock()

        self.db = sqlite3.connect(os.path.join(model.gstore.data_dir,"log.db"))
        c = self.db.cursor()
        c.execute("CREATE TABLE logs (ID integer, logID integer, line text, type integer, parent integer)")

        self.logs = {}

    def openLog(self,path,connection):
        key = hash(path+connection.addr)
        if key in self.logs.keys():
            return self.logs[key]

        self.logs[key] = LogFile(self,connection,key,path)

        return self.logs[key]

    def getCursor(self):
        self.lock.acquire()
        return self.db.cursor()

    def closeCursor(self,cur):
        cur.close()
        self.db.commit()
        self.lock.release()

class LogFile():
    def __init__(self,db,connection,logID,path):
        self.db = db
        self.connection = connection
        self.logID = logID
        self.path = path

        self.row_count = 0

        self.populate()

    def query(self,query):
        c = self.db.getCursor()

        c.execute(query)

        val = c.fetchall()

        self.db.closeCursor(c)

        return val

    def populate(self):
        self.file = self.connection.openFile(self.path)

        val = self.file.uread().split('\n')

        self.last = val[-1]

        val = val[:-1]

        c = self.db.getCursor()

        tracer = 0
        for i in val:
            #this was to elegant to totally remove. I'll leave it for at least one commit.
            #but i'm afraid of how much time it will take when it has to be run across several
            #thousand rows.
            #if True in [j in i[:10] for j in log_levels]:

            tip = i[:i.find(' ')]
            try:
                log_type = log_levels.index(tip)
            except ValueError:
                log_type = -1
            if log_type != -1:
                tracer = self.row_count

            c.execute("INSERT INTO logs values (?,?,?,?,?)",
                      (self.row_count,self.logID,i,log_type,tracer))
            self.row_count += 1

        self.db.closeCursor(c)

        Clock.schedule_interval(self.update,1)

    def update(self, *args):
        val = (self.last+self.file.uread()).split('\n')
        self.last = val[-1]
        val = val[:-1]
        c = self.db.getCursor()

        tracer = 0
        for i in val:
            tip = i[:i.find(' ')]
            try:
                log_type = log_levels.index(tip)
            except ValueError:
                log_type = -1
            if log_type != -1:
                tracer = self.row_count
            c.execute("INSERT INTO logs values (?,?,?,?,?)",
                      (self.row_count,self.logID,i,log_type,tracer))
            self.row_count += 1

        #print(self.row_count)
        self.db.closeCursor(c)
<commit_msg>Fixed an problem for python 2.7.<commit_after># -*- coding: utf-8 -*-

from kivy.clock import Clock

import connection
import model

import sqlite3
import time
import os
import random
import threading
import hashlib

log_levels = ["TRACE","DEBUG","INFO","WARNING","WARN","ERROR","SEVERE"]

class LogManager():
    def __init__(self):
        try:
            os.remove(os.path.join(model.gstore.data_dir,"log.db"))
        except Exception as e:
            print('Error: {0}'.format(e))
            #print("no database, skipping deleting")

        self.lock = threading.Lock()

        self.db = sqlite3.connect(os.path.join(model.gstore.data_dir,"log.db"))
        c = self.db.cursor()
        c.execute("CREATE TABLE logs (ID integer, logID integer, line text, type integer, parent integer)")

        self.logs = {}

    def openLog(self,path,connection):
        key = hash(path+connection.addr)
        if key in self.logs.keys():
            return self.logs[key]

        self.logs[key] = LogFile(self,connection,key,path)

        return self.logs[key]

    def getCursor(self):
        self.lock.acquire()
        return self.db.cursor()

    def closeCursor(self,cur):
        cur.close()
        self.db.commit()
        self.lock.release()

class LogFile():
    def __init__(self,db,connection,logID,path):
        self.db = db
        self.connection = connection
        self.logID = logID
        self.path = path

        self.row_count = 0

        self.populate()

    def query(self,query):
        c = self.db.getCursor()

        c.execute(query)

        val = c.fetchall()

        self.db.closeCursor(c)

        return val

    def populate(self):
        self.file = self.connection.openFile(self.path)

        val = self.file.uread().split('\n')

        self.last = val[-1]

        val = val[:-1]

        c = self.db.getCursor()

        tracer = 0
        for i in val:
            #this was to elegant to totally remove. I'll leave it for at least one commit.
            #but i'm afraid of how much time it will take when it has to be run across several
            #thousand rows.
            #if True in [j in i[:10] for j in log_levels]:

            tip = i[:i.find(' ')]
            try:
                log_type = log_levels.index(tip)
            except ValueError:
                log_type = -1
            if log_type != -1:
                tracer = self.row_count

            c.execute("INSERT INTO logs values (?,?,?,?,?)",
                      (self.row_count,self.logID,i,log_type,tracer))
            self.row_count += 1

        self.db.closeCursor(c)

        Clock.schedule_interval(self.update,1)

    def update(self, *args):
        val = (self.last+self.file.uread()).split('\n')
        self.last = val[-1]
        val = val[:-1]
        c = self.db.getCursor()

        tracer = 0
        for i in val:
            tip = i[:i.find(' ')]
            try:
                log_type = log_levels.index(tip)
            except ValueError:
                log_type = -1
            if log_type != -1:
                tracer = self.row_count
            c.execute("INSERT INTO logs values (?,?,?,?,?)",
                      (self.row_count,self.logID,i,log_type,tracer))
            self.row_count += 1

        #print(self.row_count)
        self.db.closeCursor(c)
<|endoftext|>


<commit_before>"""Contains the classes representing a request to and a response of a module."""

import json

from .abstractnode import register, AbstractNode

class Request:
    """Represents a request.
    https://github.com/ProjetPP/Documentation/blob/master/module-communication.md#request
    """
    __slots__ = ('language', 'sentence', 'tree')

    def __init__(self, language, tree=None, sentence=None):
        assert tree or sentence
        self.sentence = sentence
        if isinstance(tree, dict) or isinstance(tree, str):
            tree = AbstractNode.from_json(tree)
        self.tree = tree
        self.language = language

    def __repr__(self):
        return '<PPP request language=%r, tree=%r>' % \
                (self.language, self.tree)

    def __eq__(self, other):
        if isinstance(other, dict) or isinstance(other, str):
            other = Request.from_json(other)
        return self.language == other.language and \
                self.tree == other.tree

    @staticmethod
    def from_json(data):
        if isinstance(data, str):
            data = json.loads(data)
        return Request(data['language'], data['tree'])

    def as_dict(self):
        return {'language': self.language,
                'tree': self.tree.as_dict()}
    def as_json(self):
        return json.dumps(self.as_dict())

class Response:
    """Represents a response.
    https://github.com/ProjetPP/Documentation/blob/master/module-communication.md#response
    """
    __slots__ = ('language', 'pertinence', 'tree')

    def __init__(self, language, pertinence, tree):
        if isinstance(tree, dict) or isinstance(tree, str):
            tree = AbstractNode.from_json(tree)
        self.language = language
        self.pertinence = pertinence
        self.tree = tree

    def __repr__(self):
        return '<PPP response language=%r, pertinence=%r, tree=%r>' % \
                (self.language, self.pertinence, self.tree)

    def __eq__(self, other):
        if isinstance(other, dict) or isinstance(other, str):
            other = Response.from_json(other)
        return self.language == other.language and \
                self.pertinence == other.pertinence and \
                self.tree == other.tree

    @staticmethod
    def from_json(data):
        if isinstance(data, str):
            data = json.loads(data)
        return Response(data['language'], data['pertinence'], data['tree'])

    def as_dict(self):
        return {'language': self.language,
                'pertinence': self.pertinence,
                'tree': self.tree.as_dict()}
    def as_json(self):
        return json.dumps(self.as_dict())
<commit_msg>Add full support for sentences.<commit_after>"""Contains the classes representing a request to and a response of a module."""

import json

from .abstractnode import register, AbstractNode

class Request:
    """Represents a request.
    https://github.com/ProjetPP/Documentation/blob/master/module-communication.md#request
    """
    __slots__ = ('language', 'sentence', 'tree')

    def __init__(self, language, tree=None, sentence=None):
        assert tree or sentence
        self.sentence = sentence
        if isinstance(tree, dict) or isinstance(tree, str):
            tree = AbstractNode.from_json(tree)
        self.tree = tree
        self.language = language

    def __repr__(self):
        return '<PPP request language=%r, tree=%r>' % \
                (self.language, self.tree)

    def __eq__(self, other):
        if isinstance(other, dict) or isinstance(other, str):
            other = Request.from_json(other)
        return self.language == other.language and \
                self.tree == other.tree and \
                self.sentence == other.sentence

    @staticmethod
    def from_json(data):
        if isinstance(data, str):
            data = json.loads(data)
        if 'tree' not in data and 'sentence' not in data:
            raise KeyError("'tree' or 'sentence'")
        return Request(data['language'],
                       data.get('tree', None),
                       data.get('sentence', None))

    def as_dict(self):
        d = {'language': self.language}
        if self.tree:
            d['tree'] = self.tree.as_dict()
        if self.sentence:
            d['sentence'] = self.sentence.as_dict()
        return d
    def as_json(self):
        return json.dumps(self.as_dict())

class Response:
    """Represents a response.
    https://github.com/ProjetPP/Documentation/blob/master/module-communication.md#response
    """
    __slots__ = ('language', 'pertinence', 'tree')

    def __init__(self, language, pertinence, tree):
        if isinstance(tree, dict) or isinstance(tree, str):
            tree = AbstractNode.from_json(tree)
        self.language = language
        self.pertinence = pertinence
        self.tree = tree

    def __repr__(self):
        return '<PPP response language=%r, pertinence=%r, tree=%r>' % \
                (self.language, self.pertinence, self.tree)

    def __eq__(self, other):
        if isinstance(other, dict) or isinstance(other, str):
            other = Response.from_json(other)
        return self.language == other.language and \
                self.pertinence == other.pertinence and \
                self.tree == other.tree

    @staticmethod
    def from_json(data):
        if isinstance(data, str):
            data = json.loads(data)
        return Response(data['language'], data['pertinence'], data['tree'])

    def as_dict(self):
        return {'language': self.language,
                'pertinence': self.pertinence,
                'tree': self.tree.as_dict()}
    def as_json(self):
        return json.dumps(self.as_dict())
<|endoftext|>


<commit_before>#! /usr/bin/python
"""
This script is based on https://github.com/akamai-open/api-kickstart
"""
import ConfigParser,sys,requests, json
from akamai.edgegrid import EdgeGridAuth
from os	import path
from urlparse import urljoin
from pprint import pprint

home = path.expanduser("~")
config_file = "%s/.edgerc" % home

class BlankDict(dict):
        def __missing__(self, key):
            return ''

class AKAMAI:
	def __init__(self,block,config_file,contract=""):
		filename = config_file
		if not path.exists(filename):
			sure = ""
			while sure !="y" and sure !="n" and sure !="no" and sure !="yes":
				sure = raw_input("This Config file don\'t exist. Do you want create it (y/n)?").lower().strip()
			if sure == "no" or sure == "n":
				print "Exit. Please review filename for exporting users"
				exit(4)
			else:
				print "Creating config file"
				self.install(block,filename)
		if path.isfile(config_file):
			try:
				config = ConfigParser.ConfigParser()
				config.readfp(open(config_file))
				config.items(block)
			except(ConfigParser.NoSectionError):
				sure = ""
				while sure !="y" and sure !="n" and sure !="no" and sure !="yes":
					sure = raw_input("Section don\'t exists. Do you want create it (y/n)?").lower().strip()
				if sure == "no" or sure == "n":
					print "Exit. Please review config file in %s" % config_file
					exit(5)
				else:
					print "Creating section"
					self.install(block,filename)
					config = ConfigParser.ConfigParser()
					config.readfp(open(config_file))
			self.session = ""
			for key, value in config.items(block):
				# ConfigParser lowercases magically
				if key == "example":
					print "This is an EXAMPLE!!! Create your on config file and change this attribute"
					exit()
				elif key == "client_secret":
					self.client_secret = value
				elif key == "contract":
					self.contract = value
				elif key == "access_token":
					self.access_token = value
				elif key == "client_token":
					self.client_token = value
				elif key == "host":
					self.host = value
				else:
					print "I have no mapping for %s" % key
		else:
			print "Missing configuration file."
			exit()
	def install(self,block,filename):
		Config = ConfigParser.ConfigParser()
		config_items = ["host","access_token","client_secret","client_token","contract"]
		print "Take yours API information from luna control center"
		print "You will need host, access_token, client_secret, client_token, contract"
		# First, if we have a 'default' section protect it here
		if not path.exists(filename):
			myfile = open (filename, "w")
			myfile.write("\n")
			myfile.close()
		with open (filename, "r") as myfile:
 			data=myfile.read().replace('default','----DEFAULT----')
			myfile.close()
		with open (filename, "w") as myfile:
			myfile.write(data)
			myfile.close()

		Config.read(filename)
		config = open(filename,'w')

		if block in Config.sections():
			print "\n\nReplacing section: %s" % block
			Config.remove_section(block)
		else:
			print "\n\nCreating section: %s" % block

		Config.add_section(block)

		for i in config_items:
			var = ""
			while not var:
				message = "Insert your %s please:" % i
				var = raw_input(message).strip()
				if " " in var:
					print "Error"
					var = ""
				else:
					if i == "client_secret":
						Config.set(block,'client_secret',var)
					elif i == "host":
						Config.set(block,'host',var)
					elif i == "access_token":
						Config.set(block,'access_token',var)
					elif i == "client_token":
						Config.set(block,'client_token',var)
					elif i == "contract":
						Config.set(block,'contract',var)
		Config.write(config)
		config.close ()
		with open (filename, "r") as myfile:
 			data=myfile.read().replace('----DEFAULT----','default')
			myfile.close()
		with open (filename, "w") as myfile:
			myfile.write(data)
			myfile.close()

	def connection(self,block="----DEFAULT----"):
		self.baseurl = '%s://%s/' % ('https', self.host)
		self.session = requests.Session()
		self.session.auth = EdgeGridAuth(self.client_token,self.client_secret,self.access_token)

	def users(self):
		if not self.session:
			self.connection()
	 	user_url = '/user-admin/v1/accounts/'+self.contract+'/users'
		endpoint_result = self.session.get(urljoin(self.baseurl,user_url))
	 	user_result = endpoint_result.json()
	 	user_dump = json.loads(json.dumps(user_result), object_hook=BlankDict)
	 	return user_dump

	def export_userscsv (self,filename=""):
		if not filename:
			print "You need to specify a file to export users"
			exit (2)
		if path.exists(filename):
			sure = ""
			while sure !="y" and sure !="n" and sure !="no" and sure !="yes":
				sure = raw_input("This file ("+filename+") exists. Do you want to overwrite it(y/n)? ").lower().strip()
			if sure == "no" or sure == "n":
				print "Exit. Please review filename for exporting users"
				exit(3)
			else:
				print "Exporting users..."
				exportados = 0
		result = self.users()
		try:
			if str(result["httpStatus"]) == "403":
				print "Permisions Error"
				raise SystemExit
			if str(result["status"]) == "401":
				print "Permisions Error"
				raise SystemExit
	 	except (TypeError):
			pass
		with open (filename, "w") as myfile:
			cabecera = " USERS FOR CONTRACT "
	 		myfile.write(cabecera.center(50,"=")+"\n")
	 		myfile.write("Username;FirstName;LastName;Phone;Role;Group;Email;userType;2faEnabled;2faConfigured;lastLogin\n")
	 		while result:
				exportados +=1
				user = result.pop()
				username= str(user["username"])
				firstname = str(user["firstName"])
				lastname = str(user["lastName"])
				phone = str(user["phone"])
				contactId = str(user["contactId"])
				userType = str(user["userType"])
				twofaEnabled = str(user["tfaEnabled"])
				twofaConfigured = str(user["tfaConfigured"])
				if user["lastLoginDate"]:
					lastlogin = str(user["lastLoginDate"])
				else:
					lastlogin = str("")
				email = str(user["email"])
				roles = user["roleAssignments"]
				while roles:
					j = roles.pop()
					role = str(j["roleName"])
					group = str(j["groupName"])
				myfile.write(username+";"+firstname+";"+lastname+";"+phone+";"+role+";"+group+";"+email+";"+userType+";"+twofaEnabled+";"+twofaConfigured+";"+lastlogin+"\n")
		print "%i Users exported in %s" % (exportados,filename)

## Codigo por defecto
if len(sys.argv)>1:
	command = str(sys.argv[1]).lower()

	if command == "test":
		print "Test"
	elif command == "iniciate":
		if len(sys.argv)==3:
			block = sys.argv[2]
		elif len(sys.argv)==2:
			block = "user"
		else:
			print "error"
			exit(1)
		AKAMAI_instance = AKAMAI (block,config_file)
	elif command == "export_users":
		block = "user"
		if len(sys.argv)==3:
			AKAMAI_instance = AKAMAI(block,config_file).export_userscsv(sys.argv[2])
		elif len(sys.argv)==4:
			contract = sys.argv[3]
			AKAMAI_instance = AKAMAI(block,config_file,contract).export_userscsv(sys.argv[2])
		elif len(sys.argv)==5:
			contract = sys.argv[3]
			config_file = sys.argv[4]
			AKAMAI_instance = AKAMAI(block,config_file,contract).export_userscsv(sys.argv[2])
		else:
			print "%s %s export_file [contract [config_file]]]" % (sys.argv[0], sys.argv[1])
			exit(1)

	elif command == "config":
		if len(sys.argv)==3:
			block = sys.argv[2]
		elif len(sys.argv)==2:
			block = "user"
		else:
			print "error"
			exit(1)
		install(config_file,block)
else:
	print "Usage: %s [argvs]" % str(sys.argv[0])
	exit(99)

#AKAMAI_instance = AKAMAI ("user",config_file)
#AKAMAI_instance.connection()

<commit_msg>New commands for groups and roles<commit_after>#! /usr/bin/python
"""
This script is based on https://github.com/akamai-open/api-kickstart
"""
import ConfigParser,sys,requests, json
from akamai.edgegrid import EdgeGridAuth
from os	import path
from urlparse import urljoin
from pprint import pprint
import unicodedata

home = path.expanduser("~")
config_file = "%s/.edgerc" % home

class BlankDict(dict):
        def __missing__(self, key):
            return ''

class AKAMAI:
	def __init__(self,block,config_file,contract=""):
		filename = config_file
		if not path.exists(filename):
			sure = ""
			while sure !="y" and sure !="n" and sure !="no" and sure !="yes":
				sure = raw_input("This Config file don\'t exist. Do you want create it (y/n)?").lower().strip()
			if sure == "no" or sure == "n":
				print "Exit. Please review filename for exporting users"
				exit(4)
			else:
				print "Creating config file"
				self.install(block,filename)
		if path.isfile(config_file):
			try:
				config = ConfigParser.ConfigParser()
				config.readfp(open(config_file))
				config.items(block)
			except(ConfigParser.NoSectionError):
				sure = ""
				while sure !="y" and sure !="n" and sure !="no" and sure !="yes":
					sure = raw_input("Section don\'t exists. Do you want create it (y/n)?").lower().strip()
				if sure == "no" or sure == "n":
					print "Exit. Please review config file in %s" % config_file
					exit(5)
				else:
					print "Creating section"
					self.install(block,filename)
					config = ConfigParser.ConfigParser()
					config.readfp(open(config_file))
			self.session = ""
			for key, value in config.items(block):
				# ConfigParser lowercases magically
				if key == "example":
					print "This is an EXAMPLE!!! Create your on config file and change this attribute"
					exit()
				elif key == "client_secret":
					self.client_secret = value
				elif key == "contract":
					self.contract = value
				elif key == "access_token":
					self.access_token = value
				elif key == "client_token":
					self.client_token = value
				elif key == "host":
					self.host = value
				else:
					print "I have no mapping for %s" % key
		else:
			print "Missing configuration file."
			exit()
	def install(self,block,filename):
		Config = ConfigParser.ConfigParser()
		config_items = ["host","access_token","client_secret","client_token","contract"]
		print "Take yours API information from luna control center"
		print "You will need host, access_token, client_secret, client_token, contract"
		# First, if we have a 'default' section protect it here
		if not path.exists(filename):
			myfile = open (filename, "w")
			myfile.write("\n")
			myfile.close()
		with open (filename, "r") as myfile:
 			data=myfile.read().replace('default','----DEFAULT----')
			myfile.close()
		with open (filename, "w") as myfile:
			myfile.write(data)
			myfile.close()

		Config.read(filename)
		config = open(filename,'w')

		if block in Config.sections():
			print "\n\nReplacing section: %s" % block
			Config.remove_section(block)
		else:
			print "\n\nCreating section: %s" % block

		Config.add_section(block)

		for i in config_items:
			var = ""
			while not var:
				message = "Insert your %s please:" % i
				var = raw_input(message).strip()
				if " " in var:
					print "Error"
					var = ""
				else:
					if i == "client_secret":
						Config.set(block,'client_secret',var)
					elif i == "host":
						Config.set(block,'host',var)
					elif i == "access_token":
						Config.set(block,'access_token',var)
					elif i == "client_token":
						Config.set(block,'client_token',var)
					elif i == "contract":
						Config.set(block,'contract',var)
		Config.write(config)
		config.close ()
		with open (filename, "r") as myfile:
 			data=myfile.read().replace('----DEFAULT----','default')
			myfile.close()
		with open (filename, "w") as myfile:
			myfile.write(data)
			myfile.close()

	def connection(self,block="----DEFAULT----"):
		self.baseurl = '%s://%s/' % ('https', self.host)
		self.session = requests.Session()
		self.session.auth = EdgeGridAuth(self.client_token,self.client_secret,self.access_token)

	def users(self):
		if not self.session:
			self.connection()
	 	user_url = '/user-admin/v1/accounts/'+self.contract+'/users'
		endpoint_result = self.session.get(urljoin(self.baseurl,user_url))
	 	user_result = endpoint_result.json()
	 	user_dump = json.loads(json.dumps(user_result), object_hook=BlankDict)
	 	return user_dump

	def groups(self):
		if not self.session:
			self.connection()
	 	group_url = '/user-admin/v1/accounts/'+self.contract+'/groups'
		endpoint_result = self.session.get(urljoin(self.baseurl,group_url))
	 	group_result = endpoint_result.json()
	 	group_dump = json.loads(json.dumps(group_result), object_hook=BlankDict)
	 	return group_dump

	def roles(self):
		if not self.session:
			self.connection()
	 	roles_url = '/user-admin/v1/accounts/'+self.contract+'/roles'
		endpoint_result = self.session.get(urljoin(self.baseurl,roles_url))
	 	roles_result = endpoint_result.json()
	 	roles_dump = json.loads(json.dumps(roles_result), object_hook=BlankDict)
	 	return roles_dump

	def export_userscsv (self,filename=""):
		if not filename:
			print "You need to specify a file to export users"
			exit (2)
		if path.exists(filename):
			sure = ""
			while sure !="y" and sure !="n" and sure !="no" and sure !="yes":
				sure = raw_input("This file ("+filename+") exists. Do you want to overwrite it(y/n)? ").lower().strip()
			if sure == "no" or sure == "n":
				print "Exit. Please review filename for exporting users"
				exit(3)
			else:
				print "Exporting users..."
				exportados = 0
		result = self.users()
		try:
			if str(result["httpStatus"]) == "403":
				print "Permisions Error"
				raise SystemExit
			if str(result["status"]) == "401":
				print "Permisions Error"
				raise SystemExit
	 	except (TypeError):
			pass
		with open (filename, "w") as myfile:
			cabecera = " USERS FOR CONTRACT "
	 		myfile.write(cabecera.center(50,"=")+"\n")
	 		myfile.write("Username;FirstName;LastName;Phone;Role;Group;Email;userType;2faEnabled;2faConfigured;lastLogin\n")
	 		while result:
				exportados +=1
				user = result.pop()
				username= str(user["username"])
				firstname = str(user["firstName"])
				lastname = str(user["lastName"])
				phone = str(user["phone"])
				contactId = str(user["contactId"])
				userType = str(user["userType"])
				twofaEnabled = str(user["tfaEnabled"])
				twofaConfigured = str(user["tfaConfigured"])
				if user["lastLoginDate"]:
					lastlogin = str(user["lastLoginDate"])
				else:
					lastlogin = str("")
				email = str(user["email"])
				roles = user["roleAssignments"]
				while roles:
					j = roles.pop()
					role = str(j["roleName"])
					group = str(j["groupName"])
				myfile.write(username+";"+firstname+";"+lastname+";"+phone+";"+role+";"+group+";"+email+";"+userType+";"+twofaEnabled+";"+twofaConfigured+";"+lastlogin+"\n")
		print "%i Users exported in %s" % (exportados,filename)

	def export_groupscsv (self,filename=""):
		if not filename:
			print "You need to specify a file to export users"
			exit (2)
		if path.exists(filename):
			sure = ""
			while sure !="y" and sure !="n" and sure !="no" and sure !="yes":
				sure = raw_input("This file ("+filename+") exists. Do you want to overwrite it(y/n)? ").lower().strip()
			if sure == "no" or sure == "n":
				print "Exit. Please review filename for exporting Groups"
				exit(3)
			else:
				print "Exporting groups..."
				exportados = 0
		result = self.groups()
		try:
			if str(result["httpStatus"]) == "403":
				print "Permisions Error"
				raise SystemExit
			if str(result["status"]) == "401":
				print "Permisions Error"
				raise SystemExit
	 	except (TypeError):
			pass
		with open (filename, "w") as myfile:
			cabecera = " GROUPS FOR CONTRACT "
	 		myfile.write(cabecera.center(50,"=")+"\n")
	 		myfile.write("GroupID;accountId;groupName;topLevelGroup;parentGroupId;createdBy;createdDate\n")
			while result:
				exportados +=1
				group = result.pop()
				createdBy = str(group["createdBy"])
				topLevelGroup = str(group["topLevelGroup"])
				groupName = str(group["groupName"])
				parentGroupId = str(group["parentGroupId"])
				accountId = str(group["accountId"])
				groupId = str(group["groupId"])
				createdDate = str(group["createdDate"])
				myfile.write(groupId+";"+accountId+";"+groupName+";"+topLevelGroup+";"+parentGroupId+";"+createdBy+";"+createdDate+"\n")
			print "%i Groups exported in %s" % (exportados,filename)

	def export_rolescsv (self,filename=""):
		if not filename:
			print "You need to specify a file to export users"
			exit (2)
		if path.exists(filename):
			sure = ""
			while sure !="y" and sure !="n" and sure !="no" and sure !="yes":
				sure = raw_input("This file ("+filename+") exists. Do you want to overwrite it(y/n)? ").lower().strip()
			if sure == "no" or sure == "n":
				print "Exit. Please review filename for exporting Roles"
				exit(3)
			else:
				print "Exporting roles..."
				exportados = 0
		result = self.roles()
		try:
			if str(result["httpStatus"]) == "403":
				print "Permisions Error"
				raise SystemExit
			if str(result["status"]) == "401":
				print "Permisions Error"
				raise SystemExit
	 	except (TypeError):
			pass
		with open (filename, "w") as myfile:
			cabecera = " ROLES FOR CONTRACT "
	 		myfile.write(cabecera.center(50,"=")+"\n")
	 		myfile.write("roles_dump;roleName;roleDescription;contractTypeId;numUsers;type;modifiedBy;modifiedDate\n")
			while result:
				exportados += 1
				role = result.pop()
				roleId = str(role["roleId"])
				contractTypeId = str(role["contractTypeId"])
				roleDescription = role["roleDescription"]
				for char in "'":
					roleDescription = roleDescription.replace(char,'')
				## FIX por a error that I cant understand
				roleDescription = roleDescription.replace(u'\x9d',"")
				modifiedBy = str(role["modifiedBy"])
				modifiedDate = str(role["modifiedDate"])
				numUsers = str(role["numUsers"])
				roleName = str(role["roleName"])
				roletype = str(role["type"])
				myfile.write(roleId+";"+roleName+";"+roleDescription+";"+contractTypeId+";"+numUsers+";"+roletype+";"+modifiedBy+";"+modifiedDate+"\n")
			print "%i Roles exported in %s" % (exportados,filename)

	def export_allcsv (self,filename=""):
		filename_users=filename+"_users"
		self.export_userscsv(filename_users)
		filename_groups=filename+"_groups"
		self.export_groupscsv(filename_groups)
		filename_roles=filename+"_roles"
		self.export_rolescsv(filename_roles)

## Codigo por defecto
if len(sys.argv)>1:
	command = str(sys.argv[1]).lower()

	if command == "test":
		print "Test"
	elif command == "iniciate":
		if len(sys.argv)==3:
			block = sys.argv[2]
		elif len(sys.argv)==2:
			block = "user"
		else:
			print "error"
			exit(1)
		AKAMAI_instance = AKAMAI (block,config_file)
	elif command == "export_users":
		block = "user"
		if len(sys.argv)==3:
			AKAMAI_instance = AKAMAI(block,config_file).export_userscsv(sys.argv[2])
		elif len(sys.argv)==4:
			contract = sys.argv[3]
			AKAMAI_instance = AKAMAI(block,config_file,contract).export_userscsv(sys.argv[2])
		elif len(sys.argv)==5:
			contract = sys.argv[3]
			config_file = sys.argv[4]
			AKAMAI_instance = AKAMAI(block,config_file,contract).export_userscsv(sys.argv[2])
		else:
			print "%s %s export_file [contract [config_file]]]" % (sys.argv[0], sys.argv[1])
			exit(1)
	elif command == "export_groups":
		block = "user"
		if len(sys.argv)==3:
			AKAMAI_instance = AKAMAI(block,config_file).export_groupsscsv(sys.argv[2])
		elif len(sys.argv)==4:
			contract = sys.argv[3]
			AKAMAI_instance = AKAMAI(block,config_file,contract).export_groupscsv(sys.argv[2])
		elif len(sys.argv)==5:
			contract = sys.argv[3]
			config_file = sys.argv[4]
			AKAMAI_instance = AKAMAI(block,config_file,contract).export_groupscsv(sys.argv[2])
		else:
			print "%s %s export_file [contract [config_file]]]" % (sys.argv[0], sys.argv[1])
			exit(1)
	elif command == "export_roles":
		block = "user"
		if len(sys.argv)==3:
			AKAMAI_instance = AKAMAI(block,config_file).export_rolescsv(sys.argv[2])
		elif len(sys.argv)==4:
			contract = sys.argv[3]
			AKAMAI_instance = AKAMAI(block,config_file,contract).export_rolescsv(sys.argv[2])
		elif len(sys.argv)==5:
			contract = sys.argv[3]
			config_file = sys.argv[4]
			AKAMAI_instance = AKAMAI(block,config_file,contract).export_rolesscsv(sys.argv[2])
		else:
			print "%s %s export_file [contract [config_file]]]" % (sys.argv[0], sys.argv[1])
			exit(1)
	elif command == "export_all":
		block = "user"
		if len(sys.argv)==3:
			AKAMAI_instance = AKAMAI(block,config_file).export_allcsv(sys.argv[2])
		elif len(sys.argv)==4:
			contract = sys.argv[3]
			AKAMAI_instance = AKAMAI(block,config_file,contract).export_allcsv(sys.argv[2])
		elif len(sys.argv)==5:
			contract = sys.argv[3]
			config_file = sys.argv[4]
			AKAMAI_instance = AKAMAI(block,config_file,contract).export_allcsv(sys.argv[2])
		else:
			print "%s %s export_file [contract [config_file]]]" % (sys.argv[0], sys.argv[1])
			exit(1)
	elif command == "config":
		if len(sys.argv)==3:
			block = sys.argv[2]
		elif len(sys.argv)==2:
			block = "user"
		else:
			print "error"
			exit(1)
		install(config_file,block)
else:
	print "Usage: %s [argvs]" % str(sys.argv[0])
	exit(99)

<|endoftext|>


<commit_before>import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
import numpy as np
from textwrap import dedent
from scipy import signal

class IIRfilter():
    def __init__(self, Q, fc, fs, filter_type, G=0.0):
        self.Q  = float(Q)
        self.fc = float(fc)
        self.fs = float(fs)
        self.filter_type = filter_type
        self.G  = float(G)
        self.valid_types = {'high_shelf' : 'High Shelf Filter', 'high_pass' : 'High Pass Filter'}
        if self.filter_type not in self.valid_types:
            raise ValueError("Invalid filter type. Valid types: {valid_types}".format(valid_types=self.valid_types.keys()))
        self.b, self.a = self.generate_filter_coefficients()

    def __repr__(self):
        info = ("""
        {type}
        ----------------------
        Gain         = {G} dB
        Q factor     = {Q}
        Center freq. = {fc} Hz
        Sample rate  = {fs} Hz
        ----------------------
        b0 = {_b0}
        b1 = {_b1}
        b2 = {_b2}
        a0 = {_a0}
        a1 = {_a1}
        a2 = {_a2}
        ----------------------
        """).format(type=self.valid_types[self.filter_type],
        G=self.G, Q=self.Q, fc=self.fc, fs=self.fs,
        _b0=self.b[0], _b1=self.b[1], _b2=self.b[2],
        _a0=self.a[0], _a1=self.a[1], _a2=self.a[2])

        return dedent(info)

    def generate_filter_coefficients(self):
        A  = np.sqrt(10**(self.G/20.0))
        w0 = 2.0 * np.pi * (self.fc / self.fs)
        alpha = np.sin(w0) / (2.0 * self.Q)

        if self.filter_type == 'high_shelf':
            b0 =      A * ( (A+1) + (A-1) * np.cos(w0) + 2 * np.sqrt(A) * alpha )
            b1 = -2 * A * ( (A-1) + (A+1) * np.cos(w0)                          )
            b2 =      A * ( (A+1) + (A-1) * np.cos(w0) - 2 * np.sqrt(A) * alpha )
            a0 =            (A+1) - (A-1) * np.cos(w0) + 2 * np.sqrt(A) * alpha
            a1 =      2 * ( (A-1) - (A+1) * np.cos(w0)                          )
            a2 =            (A+1) - (A-1) * np.cos(w0) - 2 * np.sqrt(A) * alpha
        else: # high pass filter
            b0 =  (1 + np.cos(w0))/2
            b1 = -(1 + np.cos(w0))
            b2 =  (1 + np.cos(w0))/2
            a0 =   1 + alpha
            a1 =  -2 * np.cos(w0)
            a2 =   1 - alpha

        return np.array([b0, b1, b2])/a0, np.array([a0, a1, a2])/a0

    def plot_magnitude(self):
        fig = plt.figure(figsize=(9,9))
        w, h = signal.freqz(self.b, self.a, worN=8000)
        plt.semilogx((self.fs * 0.5 / np.pi) * w, 20 * np.log10(abs(h)))
        plt.xlabel('Frequency [Hz]')
        plt.ylabel('Gain [dB]')
        if self.filter_type == 'high_shelf':
            plt.title('Pre-filter 1: High Shelving Filter')
            plt.xlim([20, 20000])
            plt.ylim([-10,10])
            plt.grid(True, which='both')
            ax = plt.axes()
            ax.yaxis.set_major_locator(ticker.MultipleLocator(2))
        else:
            plt.title('Pre-filter 2: Highpass Filter')
            plt.xlim([10, 20000])
            plt.ylim([-30,5])
            plt.grid(True, which='both')
            ax = plt.axes()
            ax.yaxis.set_major_locator(ticker.MultipleLocator(5))
        plt.show()

    def apply_filter(signal):
        return signal.lfilter(self.b, self.a, signal)

def generate_high_shelf_filter(fs, G=4.0, fc=1680.0, plot=False):
    """ Generates filter coeffcients for first stage of 2-stage pre-filtering

    This method generates a high shelf filter as described in ITU-R 1770-4 (pg. 3-4)
    This filter forms the first stage of the 2-stage pre-filtering process. The high
    shelf filter accounts for the acoustic effects of the head, where the head is
    modelled as a rigid spehere.

    Parameters
    ----------
    fs : int
        Sampling rate in samples per second.
    G : float, optional
        Gain of the shelf in dB.
    fc : float, optional
        Center frequency of the shelf in Hertz.
    plot : bool, optional
        Prints a plot of the filter magnatiude response.

    Returns
    -------
    b : list of floats
        Numerator filter coefficients stored as [b0, b1, b2]
    a : list of floats
        Denomenator filter coefficients stored as [a0, a1, a2]

    Examples
    --------
    >>> from pyloudnorm import normalize
    """

    Q = 1.0 / np.sqrt(2.0)
    A  = np.sqrt(10**(G/20.0))
    w0 = 2.0 * np.pi * (fc / fs)
    alpha = np.sin(w0) / (2 * Q)

    b0 =      A * ( (A+1) + (A-1) * np.cos(w0) + 2 * np.sqrt(A) * alpha )
    b1 = -2 * A * ( (A-1) + (A+1) * np.cos(w0)                          )
    b2 =      A * ( (A+1) + (A-1) * np.cos(w0) - 2 * np.sqrt(A) * alpha )
    a0 =            (A+1) - (A-1) * np.cos(w0) + 2 * np.sqrt(A) * alpha
    a1 =      2 * ( (A-1) - (A+1) * np.cos(w0)                          )
    a2 =            (A+1) - (A-1) * np.cos(w0) - 2 * np.sqrt(A) * alpha

    b = [b0/a0, b1/a0, b2/a0]
    a = [a0/a0, a1/a0, a2/a0]

    if plot:
        fig = plt.figure(figsize=(5,5))
        w, h = signal.freqz(b, a, worN=8000)
        plt.semilogx((fs * 0.5 / np.pi) * w, 20 * np.log10(abs(h)))
        plt.title('Pre-filter 1: High Shelving Filter')
        plt.xlabel('Frequency [Hz]')
        plt.ylabel('Gain [dB]')
        plt.xlim([20, 20000])
        plt.ylim([-10,10])
        plt.grid(True, which='both')
        ax = plt.axes()
        ax.yaxis.set_major_locator(ticker.MultipleLocator(2))
        plt.show()

    return b, a

def generate_high_pass_filter(fc, fs, plot=False):

    Q  = 0.5
    w0 = 2.0 * np.pi * (fc / fs)
    alpha = np.sin(w0) / (2 * Q)

    b0 =  (1 + np.cos(w0))/2
    b1 = -(1 + np.cos(w0))
    b2 =  (1 + np.cos(w0))/2
    a0 =   1 + alpha
    a1 =  -2 * np.cos(w0)
    a2 =   1 - alpha

    b = [b0, b1, b2]
    a = [a0, a1, a2]

    if plot:
        w, h = signal.freqz(b, a, worN=8000)
        fig = plt.figure(figsize=(5,5))
        plt.semilogx((fs * 0.5 / np.pi) * w, 20 * np.log10(abs(h)))
        plt.title('Pre-filter 2: Highpass Filter')
        plt.xlabel('Frequency [Hz]')
        plt.ylabel('Gain [dB]')
        plt.xlim([10, 20000])
        plt.ylim([-30,5])
        plt.grid(True, which='both')
        ax = plt.axes()
        ax.yaxis.set_major_locator(ticker.MultipleLocator(5))
        plt.show()

    return b, a

def apply_K_freq_weighting(audio, fs, stage1_filter, stage2_filter):

    stage1_audio = signal.lfilter(stage1_filter[0], stage1_filter[1], audio)
    stage2_audio = signal.lfilter(stage2_filter[0], stage2_filter[1], stage1_audio)

    return stage2_audio

def measure_loudness(audio, fs):
    """ Measure the loudness for a signal."""

    if len(audio.shape) == 1: # for mono input standardize shape
        audio = audio.reshape((audio.shape[0],1))

    numSamples  = audio.shape[0] # length of input in samples
    numChannels = audio.shape[1] # number of input channels

    # generate the two stages of filters
    stage1_filter = IIRfilter(1/np.sqrt(2), 1680, fs, 'high_shelf', G=4.0)
    stage2_filter = IIRfilter(0.5, 38, fs, 'high_pass')

    # "K" Frequency Weighting - account for the acoustic respose of the head and auditory system
    for ch in range(numChannels):
        audio[:,ch] = stage1_filter.apply_filter(audio[:,ch])
        audio[;,ch] = stage2_filter.apply_filter(audio[:,ch])

    # Gating - ensures sections of silence or ambience do not skew the measurement

    G = [1.0, 1.0, 1.0, 1.41, 1.41] # channel gains
    T_g = 0.4 # 400 ms gating block
    Gamma_a = -70.0 # -70 LKFS = absolute loudness threshold
    overlap = 0.75 # overlap of 75% of the block duration
    step = 1 - overlap

    z = np.ndarray(shape=(numChannels,numSamples)) # instantiate array - trasponse of input
    T = numSamples / fs # length of the input in seconds
    j_range = np.arange(0, int(np.round((T - T_g) / (T_g * step))))

    for i in range(numChannels):
        for j in j_range:
            l = int(np.round(T_g * (j * step    ) * fs)) # lower bound of integration (in samples)
            u = int(np.round(T_g * (j * step + 1) * fs)) # upper bound of integration (in samples)
            z[i,j] = (1 / (T_g * fs)) * np.sum(np.square(audio[l:u,i])) # mean square and integrate

    l = [-0.691 + 10.0 * np.log10(np.sum([G[i] * z[i,j] for i in range(numChannels)])) for j in j_range]

    J_g = [j for j,l_j in enumerate(l) if l_j > Gamma_a]
    z_avg_gated = [np.mean([z[i,j] for j in J_g]) for i in range(numChannels)]
    Gamma_r = -0.691 + 10.0 * np.log10(np.sum([G[i] * z_avg_gated[i] for i in range(numChannels)])) - 10.0

    J_g = [j for j,l_j in enumerate(l) if (l_j > Gamma_a and l_j > Gamma_r)]
    z_avg_gated = [np.mean([z[i,j] for j in J_g]) for i in range(numChannels)]
    L_KG = -0.691 + 10.0 * np.log10(np.sum([G[i] * z_avg_gated[i] for i in range(numChannels)]))

    return L_KG
<commit_msg>REMOVE: took out all of the old methods for generating filtering coefficients. Replaced completely by IIRFilter class.<commit_after>import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
import numpy as np
from textwrap import dedent
from scipy import signal

class IIRfilter():
    """ Generates filter coeffcients for first stage of 2-stage pre-filtering

    This method generates a high shelf filter as described in ITU-R 1770-4 (pg. 3-4)
    This filter forms the first stage of the 2-stage pre-filtering process. The high
    shelf filter accounts for the acoustic effects of the head, where the head is
    modelled as a rigid spehere.

    Parameters
    ----------
    fs : int
        Sampling rate in samples per second.
    G : float, optional
        Gain of the shelf in dB.
    fc : float, optional
        Center frequency of the shelf in Hertz.
    plot : bool, optional
        Prints a plot of the filter magnatiude response.

    Returns
    -------
    b : list of floats
        Numerator filter coefficients stored as [b0, b1, b2]
    a : list of floats
        Denomenator filter coefficients stored as [a0, a1, a2]

    Examples
    --------
    >>> from pyloudnorm import normalize
    """
    def __init__(self, Q, fc, fs, filter_type, G=0.0):
        self.Q  = float(Q)
        self.fc = float(fc)
        self.fs = float(fs)
        self.filter_type = filter_type
        self.G  = float(G)
        self.valid_types = {'high_shelf' : 'High Shelf Filter', 'high_pass' : 'High Pass Filter'}
        if self.filter_type not in self.valid_types:
            raise ValueError("Invalid filter type. Valid types: {valid_types}".format(valid_types=self.valid_types.keys()))
        self.b, self.a = self.generate_filter_coefficients()

    def __repr__(self):
        info = ("""
        {type}
        ----------------------
        Gain         = {G} dB
        Q factor     = {Q}
        Center freq. = {fc} Hz
        Sample rate  = {fs} Hz
        ----------------------
        b0 = {_b0}
        b1 = {_b1}
        b2 = {_b2}
        a0 = {_a0}
        a1 = {_a1}
        a2 = {_a2}
        ----------------------
        """).format(type=self.valid_types[self.filter_type],
        G=self.G, Q=self.Q, fc=self.fc, fs=self.fs,
        _b0=self.b[0], _b1=self.b[1], _b2=self.b[2],
        _a0=self.a[0], _a1=self.a[1], _a2=self.a[2])

        return dedent(info)

    def generate_filter_coefficients(self):
        A  = np.sqrt(10**(self.G/20.0))
        w0 = 2.0 * np.pi * (self.fc / self.fs)
        alpha = np.sin(w0) / (2.0 * self.Q)

        if self.filter_type == 'high_shelf':
            b0 =      A * ( (A+1) + (A-1) * np.cos(w0) + 2 * np.sqrt(A) * alpha )
            b1 = -2 * A * ( (A-1) + (A+1) * np.cos(w0)                          )
            b2 =      A * ( (A+1) + (A-1) * np.cos(w0) - 2 * np.sqrt(A) * alpha )
            a0 =            (A+1) - (A-1) * np.cos(w0) + 2 * np.sqrt(A) * alpha
            a1 =      2 * ( (A-1) - (A+1) * np.cos(w0)                          )
            a2 =            (A+1) - (A-1) * np.cos(w0) - 2 * np.sqrt(A) * alpha
        else: # high pass filter
            b0 =  (1 + np.cos(w0))/2
            b1 = -(1 + np.cos(w0))
            b2 =  (1 + np.cos(w0))/2
            a0 =   1 + alpha
            a1 =  -2 * np.cos(w0)
            a2 =   1 - alpha

        return np.array([b0, b1, b2])/a0, np.array([a0, a1, a2])/a0

    def plot_magnitude(self):
        fig = plt.figure(figsize=(9,9))
        w, h = signal.freqz(self.b, self.a, worN=8000)
        plt.semilogx((self.fs * 0.5 / np.pi) * w, 20 * np.log10(abs(h)))
        plt.xlabel('Frequency [Hz]')
        plt.ylabel('Gain [dB]')
        if self.filter_type == 'high_shelf':
            plt.title('Pre-filter 1: High Shelving Filter')
            plt.xlim([20, 20000])
            plt.ylim([-10,10])
            plt.grid(True, which='both')
            ax = plt.axes()
            ax.yaxis.set_major_locator(ticker.MultipleLocator(2))
        else:
            plt.title('Pre-filter 2: Highpass Filter')
            plt.xlim([10, 20000])
            plt.ylim([-30,5])
            plt.grid(True, which='both')
            ax = plt.axes()
            ax.yaxis.set_major_locator(ticker.MultipleLocator(5))
        plt.show()

    def apply_filter(signal):
        return signal.lfilter(self.b, self.a, signal)

def measure_loudness(audio, fs):
    """ Measure the loudness for a signal."""

    if len(audio.shape) == 1: # for mono input standardize shape
        audio = audio.reshape((audio.shape[0],1))

    numSamples  = audio.shape[0] # length of input in samples
    numChannels = audio.shape[1] # number of input channels

    # generate the two stages of filters
    stage1_filter = IIRfilter(1/np.sqrt(2), 1680, fs, 'high_shelf', G=4.0)
    stage2_filter = IIRfilter(0.5, 38, fs, 'high_pass')

    # "K" Frequency Weighting - account for the acoustic respose of the head and auditory system
    for ch in range(numChannels):
        audio[:,ch] = stage1_filter.apply_filter(audio[:,ch])
        audio[;,ch] = stage2_filter.apply_filter(audio[:,ch])

    # Gating - ensures sections of silence or ambience do not skew the measurement

    G = [1.0, 1.0, 1.0, 1.41, 1.41] # channel gains
    T_g = 0.4 # 400 ms gating block
    Gamma_a = -70.0 # -70 LKFS = absolute loudness threshold
    overlap = 0.75 # overlap of 75% of the block duration
    step = 1 - overlap

    z = np.ndarray(shape=(numChannels,numSamples)) # instantiate array - trasponse of input
    T = numSamples / fs # length of the input in seconds
    j_range = np.arange(0, int(np.round((T - T_g) / (T_g * step))))

    for i in range(numChannels):
        for j in j_range:
            l = int(np.round(T_g * (j * step    ) * fs)) # lower bound of integration (in samples)
            u = int(np.round(T_g * (j * step + 1) * fs)) # upper bound of integration (in samples)
            z[i,j] = (1 / (T_g * fs)) * np.sum(np.square(audio[l:u,i])) # mean square and integrate

    l = [-0.691 + 10.0 * np.log10(np.sum([G[i] * z[i,j] for i in range(numChannels)])) for j in j_range]

    J_g = [j for j,l_j in enumerate(l) if l_j > Gamma_a]
    z_avg_gated = [np.mean([z[i,j] for j in J_g]) for i in range(numChannels)]
    Gamma_r = -0.691 + 10.0 * np.log10(np.sum([G[i] * z_avg_gated[i] for i in range(numChannels)])) - 10.0

    J_g = [j for j,l_j in enumerate(l) if (l_j > Gamma_a and l_j > Gamma_r)]
    z_avg_gated = [np.mean([z[i,j] for j in J_g]) for i in range(numChannels)]
    L_KG = -0.691 + 10.0 * np.log10(np.sum([G[i] * z_avg_gated[i] for i in range(numChannels)]))

    return L_KG
<|endoftext|>


<commit_before>"""Provide the WikiPage class."""
from ...const import API_PATH
from ...util.cache import cachedproperty
from ..listing.generator import ListingGenerator
from .base import RedditBase
from .redditor import Redditor

class WikiPage(RedditBase):
    """An individual WikiPage object.

    **Typical Attributes**

    This table describes attributes that typically belong to objects of this
    class. Since attributes are dynamically provided (see
    :ref:`determine-available-attributes-of-an-object`), there is not a
    guarantee that these attributes will always be present, nor is this list
    necessarily comprehensive.

    ======================= ===================================================
    Attribute               Description
    ======================= ===================================================
    ``content_html``        The contents of the wiki page, as HTML.
    ``content_md``          The contents of the wiki page, as Markdown.
    ``may_revise``          A ``bool`` representing whether or not the
                            authenticated user may edit the wiki page.
    ``name``                The name of the wiki page.
    ``revision_by``         The :class:`.Redditor` who authored this
                            revision of the wiki page.
    ``revision_date``       The time of this revision, in `Unix Time`_.
    ``subreddit``           The :class:`.Subreddit` this wiki page belongs to.
    ======================= ===================================================

    .. _Unix Time: https://en.wikipedia.org/wiki/Unix_time
    """

    __hash__ = RedditBase.__hash__

    @staticmethod
    def _revision_generator(subreddit, url, generator_kwargs):
        for revision in ListingGenerator(
            subreddit._reddit, url, **generator_kwargs
        ):
            if revision["author"] is not None:
                revision["author"] = Redditor(
                    subreddit._reddit, _data=revision["author"]["data"]
                )
            revision["page"] = WikiPage(
                subreddit._reddit, subreddit, revision["page"], revision["id"]
            )
            yield revision

    @cachedproperty
    def mod(self):
        """Provide an instance of :class:`.WikiPageModeration`."""
        return WikiPageModeration(self)

    def __eq__(self, other):
        """Return whether the other instance equals the current."""
        return (
            isinstance(other, self.__class__)
            and str(self).lower() == str(other).lower()
        )

    def __init__(self, reddit, subreddit, name, revision=None, _data=None):
        """Construct an instance of the WikiPage object.

        :param revision: A specific revision ID to fetch. By default, fetches
            the most recent revision.

        """
        self.name = name
        self._revision = revision
        self.subreddit = subreddit
        super().__init__(reddit, _data=_data)

    def __repr__(self):
        """Return an object initialization representation of the instance."""
        return "{}(subreddit={!r}, name={!r})".format(
            self.__class__.__name__, self.subreddit, self.name
        )

    def __str__(self):
        """Return a string representation of the instance."""
        return "{}/{}".format(self.subreddit, self.name)

    def _fetch_info(self):
        return (
            "wiki_page",
            {"subreddit": self.subreddit, "page": self.name},
            {"v": self._revision} if self._revision else None,
        )

    def _fetch_data(self):
        name, fields, params = self._fetch_info()
        path = API_PATH[name].format(**fields)
        return self._reddit.request("GET", path, params)

    def _fetch(self):
        data = self._fetch_data()
        data = data["data"]
        if data["revision_by"] is not None:
            data["revision_by"] = Redditor(
                self._reddit, _data=data["revision_by"]["data"]
            )
        self.__dict__.update(data)
        self._fetched = True

    def edit(self, content, reason=None, **other_settings):
        """Edit this WikiPage's contents.

        :param content: The updated Markdown content of the page.
        :param reason: (Optional) The reason for the revision.
        :param other_settings: Additional keyword arguments to pass.

        """
        other_settings.update(
            {"content": content, "page": self.name, "reason": reason}
        )
        self._reddit.post(
            API_PATH["wiki_edit"].format(subreddit=self.subreddit),
            data=other_settings,
        )

    def revision(self, revision):
        """Return a specific version of this page by revision ID.

        To view revision ``[ID]`` of ``'praw_test'`` in ``'/r/test'``:

        .. code-block:: python

           page = reddit.subreddit('test').wiki['praw_test'].revision('[ID]')

        """
        return WikiPage(
            self.subreddit._reddit, self.subreddit, self.name, revision
        )

    def revisions(self, **generator_kwargs):
        """Return a :class:`.ListingGenerator` for page revisions.

        Additional keyword arguments are passed in the initialization of
        :class:`.ListingGenerator`.

        To view the wiki revisions for ``'praw_test'`` in ``'/r/test'`` try:

        .. code-block:: python

           for item in reddit.subreddit('test').wiki['praw_test'].revisions():
               print(item)

        To get :class:`.WikiPage` objects for each revision:

        .. code-block:: python

           for item in reddit.subreddit('test').wiki['praw_test'].revisions():
               print(item['page'])

        """
        url = API_PATH["wiki_page_revisions"].format(
            subreddit=self.subreddit, page=self.name
        )
        return self._revision_generator(self.subreddit, url, generator_kwargs)

class WikiPageModeration:
    """Provides a set of moderation functions for a WikiPage."""

    def __init__(self, wikipage):
        """Create a WikiPageModeration instance.

        :param wikipage: The wikipage to moderate.

        """
        self.wikipage = wikipage

    def add(self, redditor):
        """Add an editor to this WikiPage.

        :param redditor: A redditor name (e.g., ``'spez'``) or
            :class:`~.Redditor` instance.

        To add ``'spez'`` as an editor on the wikipage ``'praw_test'`` try:

        .. code-block:: python

           reddit.subreddit('test').wiki['praw_test'].mod.add('spez')

        """
        data = {"page": self.wikipage.name, "username": str(redditor)}
        url = API_PATH["wiki_page_editor"].format(
            subreddit=self.wikipage.subreddit, method="add"
        )
        self.wikipage._reddit.post(url, data=data)

    def remove(self, redditor):
        """Remove an editor from this WikiPage.

        :param redditor: A redditor name (e.g., ``'spez'``) or
            :class:`~.Redditor` instance.

        To remove ``'spez'`` as an editor on the wikipage ``'praw_test'`` try:

        .. code-block:: python

           reddit.subreddit('test').wiki['praw_test'].mod.remove('spez')

        """
        data = {"page": self.wikipage.name, "username": str(redditor)}
        url = API_PATH["wiki_page_editor"].format(
            subreddit=self.wikipage.subreddit, method="del"
        )
        self.wikipage._reddit.post(url, data=data)

    def settings(self):
        """Return the settings for this WikiPage."""
        url = API_PATH["wiki_page_settings"].format(
            subreddit=self.wikipage.subreddit, page=self.wikipage.name
        )
        return self.wikipage._reddit.get(url)["data"]

    def update(self, listed, permlevel, **other_settings):
        """Update the settings for this WikiPage.

        :param listed: (boolean) Show this page on page list.
        :param permlevel: (int) Who can edit this page? (0) use subreddit wiki
            permissions, (1) only approved wiki contributors for this page may
            edit (see :meth:`.WikiPageModeration.add`), (2) only mods may edit
            and view
        :param other_settings: Additional keyword arguments to pass.
        :returns: The updated WikiPage settings.

        To set the wikipage ``'praw_test'`` in ``'/r/test'`` to mod only and
          disable it from showing in the page list, try:

        .. code-block:: python

           reddit.subreddit('test').wiki['praw_test'].mod.update(listed=False,
                                                                 permlevel=2)

        """
        other_settings.update({"listed": listed, "permlevel": permlevel})
        url = API_PATH["wiki_page_settings"].format(
            subreddit=self.wikipage.subreddit, page=self.wikipage.name
        )
        return self.wikipage._reddit.post(url, data=other_settings)["data"]
<commit_msg>Type wikipage.py<commit_after>"""Provide the WikiPage class."""
from typing import Any, Dict, Generator, Optional, TypeVar, Union

from ...const import API_PATH
from ...util.cache import cachedproperty
from ..listing.generator import ListingGenerator
from .base import RedditBase
from .redditor import Redditor

_WikiPage = TypeVar("_WikiPage")
Reddit = TypeVar("Reddit")
Subreddit = TypeVar("Subreddit")

class WikiPageModeration:
    """Provides a set of moderation functions for a WikiPage."""

    def __init__(self, wikipage: _WikiPage):
        """Create a WikiPageModeration instance.

        :param wikipage: The wikipage to moderate.

        """
        self.wikipage = wikipage

    def add(self, redditor: Redditor):
        """Add an editor to this WikiPage.

        :param redditor: A redditor name (e.g., ``'spez'``) or
            :class:`~.Redditor` instance.

        To add ``'spez'`` as an editor on the wikipage ``'praw_test'`` try:

        .. code:: python

           reddit.subreddit('test').wiki['praw_test'].mod.add('spez')

        """
        data = {"page": self.wikipage.name, "username": str(redditor)}
        url = API_PATH["wiki_page_editor"].format(
            subreddit=self.wikipage.subreddit, method="add"
        )
        self.wikipage._reddit.post(url, data=data)

    def remove(self, redditor: Redditor):
        """Remove an editor from this WikiPage.

        :param redditor: A redditor name (e.g., ``'spez'``) or
            :class:`~.Redditor` instance.

        To remove ``'spez'`` as an editor on the wikipage ``'praw_test'`` try:

        .. code:: python

           reddit.subreddit('test').wiki['praw_test'].mod.remove('spez')

        """
        data = {"page": self.wikipage.name, "username": str(redditor)}
        url = API_PATH["wiki_page_editor"].format(
            subreddit=self.wikipage.subreddit, method="del"
        )
        self.wikipage._reddit.post(url, data=data)

    def settings(self) -> Dict[str, Any]:
        """Return the settings for this WikiPage."""
        url = API_PATH["wiki_page_settings"].format(
            subreddit=self.wikipage.subreddit, page=self.wikipage.name
        )
        return self.wikipage._reddit.get(url)["data"]

    def update(
        self, listed: bool, permlevel: int, **other_settings: Any
    ) -> Dict[str, Any]:
        """Update the settings for this WikiPage.

        :param listed: (boolean) Show this page on page list.
        :param permlevel: (int) Who can edit this page? (0) use subreddit wiki
            permissions, (1) only approved wiki contributors for this page may
            edit (see :meth:`.WikiPageModeration.add`), (2) only mods may edit
            and view
        :param other_settings: Additional keyword arguments to pass.
        :returns: The updated WikiPage settings.

        To set the wikipage ``'praw_test'`` in ``'/r/test'`` to mod only and
          disable it from showing in the page list, try:

        .. code:: python

           reddit.subreddit('test').wiki['praw_test'].mod.update(listed=False,
                                                                 permlevel=2)

        """
        other_settings.update({"listed": listed, "permlevel": permlevel})
        url = API_PATH["wiki_page_settings"].format(
            subreddit=self.wikipage.subreddit, page=self.wikipage.name
        )
        return self.wikipage._reddit.post(url, data=other_settings)["data"]

class WikiPage(RedditBase):
    """An individual WikiPage object.

    **Typical Attributes**

    This table describes attributes that typically belong to objects of this
    class. Since attributes are dynamically provided (see
    :ref:`determine-available-attributes-of-an-object`), there is not a
    guarantee that these attributes will always be present, nor is this list
    necessarily comprehensive.

    ======================= ===================================================
    Attribute               Description
    ======================= ===================================================
    ``content_html``        The contents of the wiki page, as HTML.
    ``content_md``          The contents of the wiki page, as Markdown.
    ``may_revise``          A ``bool`` representing whether or not the
                            authenticated user may edit the wiki page.
    ``name``                The name of the wiki page.
    ``revision_by``         The :class:`.Redditor` who authored this
                            revision of the wiki page.
    ``revision_date``       The time of this revision, in `Unix Time`_.
    ``subreddit``           The :class:`.Subreddit` this wiki page belongs to.
    ======================= ===================================================

    .. _Unix Time: https://en.wikipedia.org/wiki/Unix_time
    """

    __hash__ = RedditBase.__hash__

    @staticmethod
    def _revision_generator(subreddit, url, generator_kwargs):
        for revision in ListingGenerator(
            subreddit._reddit, url, **generator_kwargs
        ):
            if revision["author"] is not None:
                revision["author"] = Redditor(
                    subreddit._reddit, _data=revision["author"]["data"]
                )
            revision["page"] = WikiPage(
                subreddit._reddit, subreddit, revision["page"], revision["id"]
            )
            yield revision

    @cachedproperty
    def mod(self) -> WikiPageModeration:
        """Provide an instance of :class:`.WikiPageModeration`."""
        return WikiPageModeration(self)

    def __eq__(self, other: _WikiPage):
        """Return whether the other instance equals the current."""
        return (
            isinstance(other, self.__class__)
            and str(self).lower() == str(other).lower()
        )

    def __init__(
        self,
        reddit: Reddit,
        subreddit: Subreddit,
        name: str,
        revision: Optional[str] = None,
        _data: Optional[Dict[str, Any]] = None,
    ):
        """Construct an instance of the WikiPage object.

        :param revision: A specific revision ID to fetch. By default, fetches
            the most recent revision.

        """
        self.name = name
        self._revision = revision
        self.subreddit = subreddit
        super().__init__(reddit, _data=_data)

    def __repr__(self) -> str:
        """Return an object initialization representation of the instance."""
        return "{}(subreddit={!r}, name={!r})".format(
            self.__class__.__name__, self.subreddit, self.name
        )

    def __str__(self) -> str:
        """Return a string representation of the instance."""
        return "{}/{}".format(self.subreddit, self.name)

    def _fetch_info(self):
        return (
            "wiki_page",
            {"subreddit": self.subreddit, "page": self.name},
            {"v": self._revision} if self._revision else None,
        )

    def _fetch_data(self):
        name, fields, params = self._fetch_info()
        path = API_PATH[name].format(**fields)
        return self._reddit.request("GET", path, params)

    def _fetch(self):
        data = self._fetch_data()
        data = data["data"]
        if data["revision_by"] is not None:
            data["revision_by"] = Redditor(
                self._reddit, _data=data["revision_by"]["data"]
            )
        self.__dict__.update(data)
        self._fetched = True

    def edit(
        self, content: str, reason: Optional[str] = None, **other_settings: Any
    ):
        """Edit this WikiPage's contents.

        :param content: The updated Markdown content of the page.
        :param reason: (Optional) The reason for the revision.
        :param other_settings: Additional keyword arguments to pass.

        """
        other_settings.update(
            {"content": content, "page": self.name, "reason": reason}
        )
        self._reddit.post(
            API_PATH["wiki_edit"].format(subreddit=self.subreddit),
            data=other_settings,
        )

    def revision(self, revision: str):
        """Return a specific version of this page by revision ID.

        To view revision ``[ID]`` of ``'praw_test'`` in ``'/r/test'``:

        .. code:: python

           page = reddit.subreddit('test').wiki['praw_test'].revision('[ID]')

        """
        return WikiPage(
            self.subreddit._reddit, self.subreddit, self.name, revision
        )

    def revisions(
        self, **generator_kwargs: Union[str, int, Dict[str, str]]
    ) -> Generator[_WikiPage, None, None]:
        """Return a :class:`.ListingGenerator` for page revisions.

        Additional keyword arguments are passed in the initialization of
        :class:`.ListingGenerator`.

        To view the wiki revisions for ``'praw_test'`` in ``'/r/test'`` try:

        .. code:: python

           for item in reddit.subreddit('test').wiki['praw_test'].revisions():
               print(item)

        To get :class:`.WikiPage` objects for each revision:

        .. code:: python

           for item in reddit.subreddit('test').wiki['praw_test'].revisions():
               print(item['page'])

        """
        url = API_PATH["wiki_page_revisions"].format(
            subreddit=self.subreddit, page=self.name
        )
        return self._revision_generator(self.subreddit, url, generator_kwargs)
<|endoftext|>