<commit_before>"""
PyChromecast: remote control your Chromecast
"""
from __future__ import print_function

import logging
import fnmatch

# pylint: disable=wildcard-import
import threading
from .config import *  # noqa
from .error import *  # noqa
from . import socket_client
from .discovery import discover_chromecasts
from .dial import get_device_status, reboot
from .controllers.media import STREAM_TYPE_BUFFERED  # noqa

IDLE_APP_ID = 'E8C28D3C'
IGNORE_CEC = []

def _get_all_chromecasts(tries=None, retry_wait=None):
    """
    Returns a list of all chromecasts on the network as PyChromecast
    objects.
    """
    hosts = discover_chromecasts()
    cc_list = []
    for ip_address, _ in hosts:
        try:
            cc_list.append(Chromecast(host=ip_address, tries=tries,
                                      retry_wait=retry_wait))
        except ChromecastConnectionError:
            pass
    return cc_list

def get_chromecasts(tries=None, retry_wait=None, **filters):
    """
    Searches the network and returns a list of Chromecast objects.
    Filter is a list of options to filter the chromecasts by.

    ex: get_chromecasts(friendly_name="Living Room")

    May return an empty list if no chromecasts were found matching
    the filter criteria

    Filters include DeviceStatus items:
        friendly_name, model_name, manufacturer, api_version
    Or AppStatus items:
        app_id, description, state, service_url, service_protocols (list)
    Or ip address:
        ip

    Tries is specified if you want to limit the number of times the
    underlying socket associated with your Chromecast objects will
    retry connecting if connection is lost or it fails to connect
    in the first place. The number of seconds spent between each retry
    can be defined by passing the retry_wait parameter, the default is
    to wait 5 seconds.
    """
    logger = logging.getLogger(__name__)

    cc_list = set(_get_all_chromecasts(tries, retry_wait))
    excluded_cc = set()

    if not filters:
        return list(cc_list)

    if 'ip' in filters:
        for chromecast in cc_list:
            if chromecast.host != filters['ip']:
                excluded_cc.add(chromecast)
        filters.pop('ip')

    for key, val in filters.items():
        for chromecast in cc_list:
            for tup in [chromecast.device, chromecast.status]:
                if hasattr(tup, key) and val != getattr(tup, key):
                    excluded_cc.add(chromecast)

    filtered_cc = cc_list - excluded_cc

    for cast in excluded_cc:
        logger.debug("Stopping excluded chromecast %s", cast)
        cast.socket_client.stop.set()

    return list(filtered_cc)

def get_chromecasts_as_dict(tries=None, retry_wait=None, **filters):
    """
    Returns a dictionary of chromecasts with the friendly name as
    the key.  The value is the pychromecast object itself.

    Tries is specified if you want to limit the number of times the
    underlying socket associated with your Chromecast objects will
    retry connecting if connection is lost or it fails to connect
    in the first place. The number of seconds spent between each retry
    can be defined by passing the retry_wait parameter, the default is
    to wait 5 seconds.
    """
    return {cc.device.friendly_name: cc
            for cc in get_chromecasts(tries=tries, retry_wait=retry_wait,
                                      **filters)}

def get_chromecast(strict=False, tries=None, retry_wait=None, **filters):
    """
    Same as get_chromecasts but only if filter matches exactly one
    ChromeCast.

    Returns a Chromecast matching exactly the fitler specified.

    If strict, return one and only one chromecast

    Tries is specified if you want to limit the number of times the
    underlying socket associated with your Chromecast objects will
    retry connecting if connection is lost or it fails to connect
    in the first place. The number of seconds spent between each retry
    can be defined by passing the retry_wait parameter, the default is
    to wait 5 seconds.

    :type retry_wait: float or None
    """

    # If we have filters or are operating in strict mode we have to scan
    # for all Chromecasts to ensure there is only 1 matching chromecast.
    # If no filters given and not strict just use the first dicsovered one.
    if filters or strict:
        results = get_chromecasts(tries=tries, retry_wait=retry_wait,
                                  **filters)
    else:
        results = _get_all_chromecasts(tries, retry_wait)

    if len(results) > 1:
        if strict:
            raise MultipleChromecastsFoundError(
                'More than one Chromecast was found specifying '
                'the filter criteria: {}'.format(filters))
        else:
            return results[0]

    elif not results:
        if strict:
            raise NoChromecastFoundError(
                'No Chromecasts matching filter critera were found:'
                ' {}'.format(filters))
        else:
            return None

    else:
        return results[0]

# pylint: disable=too-many-instance-attributes
class Chromecast(object):
    """
    Class to interface with a ChromeCast.

    :param tries: Number of retries to perform if the connection fails.
                  None for inifinite retries.
    :param retry_wait: A floating point number specifying how many seconds to
                       wait between each retry. None means to use the default
                       which is 5 seconds.
    """

    def __init__(self, host, tries=None, retry_wait=None):
        self.logger = logging.getLogger(__name__)

        # Resolve host to IP address
        self.host = host

        self.logger.info("Querying device status")
        self.device = get_device_status(self.host)

        if not self.device:
            raise ChromecastConnectionError(
                "Could not connect to {}".format(self.host))

        self.status = None
        self.status_event = threading.Event()

        self.socket_client = socket_client.SocketClient(
            host, tries, retry_wait=retry_wait)

        receiver_controller = self.socket_client.receiver_controller
        receiver_controller.register_status_listener(self)

        # Forward these methods
        self.set_volume = receiver_controller.set_volume
        self.set_volume_muted = receiver_controller.set_volume_muted
        self.play_media = self.socket_client.media_controller.play_media
        self.register_handler = self.socket_client.register_handler
        self.register_status_listener = \
            receiver_controller.register_status_listener
        self.register_launch_error_listener = \
            receiver_controller.register_launch_error_listener
        self.register_connection_listener = \
            self.socket_client.register_connection_listener

        self.socket_client.start()

    @property
    def ignore_cec(self):
        """ Returns whether the CEC data should be ignored. """
        return self.device is not None and \
            any([fnmatch.fnmatchcase(self.device.friendly_name, pattern)
                 for pattern in IGNORE_CEC])

    @property
    def is_idle(self):
        """ Returns if there is currently an app running. """
        return (self.status is None or
                self.app_id in (None, IDLE_APP_ID) or
                (not self.status.is_active_input and not self.ignore_cec))

    @property
    def app_id(self):
        """ Returns the current app_id. """
        return self.status.app_id if self.status else None

    @property
    def app_display_name(self):
        """ Returns the name of the current running app. """
        return self.status.display_name if self.status else None

    @property
    def media_controller(self):
        """ Returns the media controller. """
        return self.socket_client.media_controller

    def new_cast_status(self, status):
        """ Called when a new status received from the Chromecast. """
        self.status = status
        if status:
            self.status_event.set()
        else:
            self.status_event.clear()

    def start_app(self, app_id):
        """ Start an app on the Chromecast. """
        self.logger.info("Starting app %s", app_id)

        self.socket_client.receiver_controller.launch_app(app_id)

    def quit_app(self):
        """ Tells the Chromecast to quit current app_id. """
        self.logger.info("Quiting current app")

        self.socket_client.receiver_controller.stop_app()

    def reboot(self):
        """ Reboots the Chromecast. """
        reboot(self.host)

    def volume_up(self):
        """ Increment volume by 0.1 unless it is already maxed.
        Returns the new volume.

        """
        volume = round(self.status.volume_level, 1)
        return self.set_volume(volume + 0.1)

    def volume_down(self):
        """ Decrement the volume by 0.1 unless it is already 0.
        Returns the new volume.
        """
        volume = round(self.status.volume_level, 1)
        return self.set_volume(volume - 0.1)

    def wait(self, timeout=None):
        """
        Waits until the cast device is ready for communication. The device
        is ready as soon a status message has been received.

        If the status has already been received then the method returns
        immediately.

        :param timeout: a floating point number specifying a timeout for the
                        operation in seconds (or fractions thereof). Or None
                        to block forever.
        """
        self.status_event.wait(timeout=timeout)

    def disconnect(self, timeout=None, blocking=True):
        """
        Disconnects the chromecast and waits for it to terminate.

        :param timeout: a floating point number specifying a timeout for the
                        operation in seconds (or fractions thereof). Or None
                        to block forever.
        :param blocking: If True it will block until the disconnection is
                         complete, otherwise it will return immediately.
        """
        self.socket_client.disconnect()
        if blocking:
            self.join(timeout=timeout)

    def join(self, timeout=None):
        """
        Blocks the thread of the caller until the chromecast connection is
        stopped.

        :param timeout: a floating point number specifying a timeout for the
                        operation in seconds (or fractions thereof). Or None
                        to block forever.
        """
        self.socket_client.join(timeout=timeout)

    def __del__(self):
        self.socket_client.stop.set()

    def __repr__(self):
        return "Chromecast({}, {}, {}, {}, api={}.{})".format(
            self.host, self.device.friendly_name, self.device.model_name,
            self.device.manufacturer, self.device.api_version[0],
            self.device.api_version[1])
<commit_msg>Never reset the status_event Event<commit_after>"""
PyChromecast: remote control your Chromecast
"""
from __future__ import print_function

import logging
import fnmatch

# pylint: disable=wildcard-import
import threading
from .config import *  # noqa
from .error import *  # noqa
from . import socket_client
from .discovery import discover_chromecasts
from .dial import get_device_status, reboot
from .controllers.media import STREAM_TYPE_BUFFERED  # noqa

IDLE_APP_ID = 'E8C28D3C'
IGNORE_CEC = []

def _get_all_chromecasts(tries=None, retry_wait=None):
    """
    Returns a list of all chromecasts on the network as PyChromecast
    objects.
    """
    hosts = discover_chromecasts()
    cc_list = []
    for ip_address, _ in hosts:
        try:
            cc_list.append(Chromecast(host=ip_address, tries=tries,
                                      retry_wait=retry_wait))
        except ChromecastConnectionError:
            pass
    return cc_list

def get_chromecasts(tries=None, retry_wait=None, **filters):
    """
    Searches the network and returns a list of Chromecast objects.
    Filter is a list of options to filter the chromecasts by.

    ex: get_chromecasts(friendly_name="Living Room")

    May return an empty list if no chromecasts were found matching
    the filter criteria

    Filters include DeviceStatus items:
        friendly_name, model_name, manufacturer, api_version
    Or AppStatus items:
        app_id, description, state, service_url, service_protocols (list)
    Or ip address:
        ip

    Tries is specified if you want to limit the number of times the
    underlying socket associated with your Chromecast objects will
    retry connecting if connection is lost or it fails to connect
    in the first place. The number of seconds spent between each retry
    can be defined by passing the retry_wait parameter, the default is
    to wait 5 seconds.
    """
    logger = logging.getLogger(__name__)

    cc_list = set(_get_all_chromecasts(tries, retry_wait))
    excluded_cc = set()

    if not filters:
        return list(cc_list)

    if 'ip' in filters:
        for chromecast in cc_list:
            if chromecast.host != filters['ip']:
                excluded_cc.add(chromecast)
        filters.pop('ip')

    for key, val in filters.items():
        for chromecast in cc_list:
            for tup in [chromecast.device, chromecast.status]:
                if hasattr(tup, key) and val != getattr(tup, key):
                    excluded_cc.add(chromecast)

    filtered_cc = cc_list - excluded_cc

    for cast in excluded_cc:
        logger.debug("Stopping excluded chromecast %s", cast)
        cast.socket_client.stop.set()

    return list(filtered_cc)

def get_chromecasts_as_dict(tries=None, retry_wait=None, **filters):
    """
    Returns a dictionary of chromecasts with the friendly name as
    the key.  The value is the pychromecast object itself.

    Tries is specified if you want to limit the number of times the
    underlying socket associated with your Chromecast objects will
    retry connecting if connection is lost or it fails to connect
    in the first place. The number of seconds spent between each retry
    can be defined by passing the retry_wait parameter, the default is
    to wait 5 seconds.
    """
    return {cc.device.friendly_name: cc
            for cc in get_chromecasts(tries=tries, retry_wait=retry_wait,
                                      **filters)}

def get_chromecast(strict=False, tries=None, retry_wait=None, **filters):
    """
    Same as get_chromecasts but only if filter matches exactly one
    ChromeCast.

    Returns a Chromecast matching exactly the fitler specified.

    If strict, return one and only one chromecast

    Tries is specified if you want to limit the number of times the
    underlying socket associated with your Chromecast objects will
    retry connecting if connection is lost or it fails to connect
    in the first place. The number of seconds spent between each retry
    can be defined by passing the retry_wait parameter, the default is
    to wait 5 seconds.

    :type retry_wait: float or None
    """

    # If we have filters or are operating in strict mode we have to scan
    # for all Chromecasts to ensure there is only 1 matching chromecast.
    # If no filters given and not strict just use the first dicsovered one.
    if filters or strict:
        results = get_chromecasts(tries=tries, retry_wait=retry_wait,
                                  **filters)
    else:
        results = _get_all_chromecasts(tries, retry_wait)

    if len(results) > 1:
        if strict:
            raise MultipleChromecastsFoundError(
                'More than one Chromecast was found specifying '
                'the filter criteria: {}'.format(filters))
        else:
            return results[0]

    elif not results:
        if strict:
            raise NoChromecastFoundError(
                'No Chromecasts matching filter critera were found:'
                ' {}'.format(filters))
        else:
            return None

    else:
        return results[0]

# pylint: disable=too-many-instance-attributes
class Chromecast(object):
    """
    Class to interface with a ChromeCast.

    :param tries: Number of retries to perform if the connection fails.
                  None for inifinite retries.
    :param retry_wait: A floating point number specifying how many seconds to
                       wait between each retry. None means to use the default
                       which is 5 seconds.
    """

    def __init__(self, host, tries=None, retry_wait=None):
        self.logger = logging.getLogger(__name__)

        # Resolve host to IP address
        self.host = host

        self.logger.info("Querying device status")
        self.device = get_device_status(self.host)

        if not self.device:
            raise ChromecastConnectionError(
                "Could not connect to {}".format(self.host))

        self.status = None
        self.status_event = threading.Event()

        self.socket_client = socket_client.SocketClient(
            host, tries, retry_wait=retry_wait)

        receiver_controller = self.socket_client.receiver_controller
        receiver_controller.register_status_listener(self)

        # Forward these methods
        self.set_volume = receiver_controller.set_volume
        self.set_volume_muted = receiver_controller.set_volume_muted
        self.play_media = self.socket_client.media_controller.play_media
        self.register_handler = self.socket_client.register_handler
        self.register_status_listener = \
            receiver_controller.register_status_listener
        self.register_launch_error_listener = \
            receiver_controller.register_launch_error_listener
        self.register_connection_listener = \
            self.socket_client.register_connection_listener

        self.socket_client.start()

    @property
    def ignore_cec(self):
        """ Returns whether the CEC data should be ignored. """
        return self.device is not None and \
            any([fnmatch.fnmatchcase(self.device.friendly_name, pattern)
                 for pattern in IGNORE_CEC])

    @property
    def is_idle(self):
        """ Returns if there is currently an app running. """
        return (self.status is None or
                self.app_id in (None, IDLE_APP_ID) or
                (not self.status.is_active_input and not self.ignore_cec))

    @property
    def app_id(self):
        """ Returns the current app_id. """
        return self.status.app_id if self.status else None

    @property
    def app_display_name(self):
        """ Returns the name of the current running app. """
        return self.status.display_name if self.status else None

    @property
    def media_controller(self):
        """ Returns the media controller. """
        return self.socket_client.media_controller

    def new_cast_status(self, status):
        """ Called when a new status received from the Chromecast. """
        self.status = status
        if status:
            self.status_event.set()

    def start_app(self, app_id):
        """ Start an app on the Chromecast. """
        self.logger.info("Starting app %s", app_id)

        self.socket_client.receiver_controller.launch_app(app_id)

    def quit_app(self):
        """ Tells the Chromecast to quit current app_id. """
        self.logger.info("Quiting current app")

        self.socket_client.receiver_controller.stop_app()

    def reboot(self):
        """ Reboots the Chromecast. """
        reboot(self.host)

    def volume_up(self):
        """ Increment volume by 0.1 unless it is already maxed.
        Returns the new volume.

        """
        volume = round(self.status.volume_level, 1)
        return self.set_volume(volume + 0.1)

    def volume_down(self):
        """ Decrement the volume by 0.1 unless it is already 0.
        Returns the new volume.
        """
        volume = round(self.status.volume_level, 1)
        return self.set_volume(volume - 0.1)

    def wait(self, timeout=None):
        """
        Waits until the cast device is ready for communication. The device
        is ready as soon a status message has been received.

        If the status has already been received then the method returns
        immediately.

        :param timeout: a floating point number specifying a timeout for the
                        operation in seconds (or fractions thereof). Or None
                        to block forever.
        """
        self.status_event.wait(timeout=timeout)

    def disconnect(self, timeout=None, blocking=True):
        """
        Disconnects the chromecast and waits for it to terminate.

        :param timeout: a floating point number specifying a timeout for the
                        operation in seconds (or fractions thereof). Or None
                        to block forever.
        :param blocking: If True it will block until the disconnection is
                         complete, otherwise it will return immediately.
        """
        self.socket_client.disconnect()
        if blocking:
            self.join(timeout=timeout)

    def join(self, timeout=None):
        """
        Blocks the thread of the caller until the chromecast connection is
        stopped.

        :param timeout: a floating point number specifying a timeout for the
                        operation in seconds (or fractions thereof). Or None
                        to block forever.
        """
        self.socket_client.join(timeout=timeout)

    def __del__(self):
        self.socket_client.stop.set()

    def __repr__(self):
        return "Chromecast({}, {}, {}, {}, api={}.{})".format(
            self.host, self.device.friendly_name, self.device.model_name,
            self.device.manufacturer, self.device.api_version[0],
            self.device.api_version[1])
<|endoftext|>


<commit_before>"""command line client to fetch statistics via SUSHI"""
from __future__ import print_function

from pycounter import sushi
from pycounter.helpers import convert_date_run

import click

@click.command()
@click.argument('url')
@click.option('--report', '-r', default='JR1',
              help='report name (default JR1)')
@click.option('--release', '-c', default=4,
              help='COUNTER release (default 4)')
@click.option('--start_date', '-s',
              help='Start Date YYYY-MM-DD (default first day of last month)')
@click.option('--end_date', '-e',
              help='End Date YYYY-MM-DD (default last day of last month OR '
                   'last day of start month')
@click.option('--requestor_id', '-i',
              help='Requestor ID')
@click.option('--customer_reference', '-c',
              help='Customer reference')
@click.option('--format', '-f', 'format_', default='tsv',
              help='Output format (default tsv)',
              type=click.Choice(['tsv',]))
def main(url, report, release, start_date, end_date, requestor_id,
         customer_reference, format_):
    # FIXME: doesn't actually default dates yet...
    click.echo("pycounter SUSHI client for URL %s (%s R%s)"
               % (url, report, release))
    report = sushi.get_report(wsdl_url=url,
                              report=report,
                              release=release,
                              requestor_id=requestor_id,
                              customer_reference=customer_reference,
                              start_date=convert_date_run(start_date),
                              end_date=convert_date_run(end_date))
    report.write_tsv('report.tsv')

if __name__ == "__main__":
    main()
<commit_msg>flake8: 2 errors fixed<commit_after>"""command line client to fetch statistics via SUSHI"""
from __future__ import print_function

from pycounter import sushi
from pycounter.helpers import convert_date_run

import click

@click.command()
@click.argument('url')
@click.option('--report', '-r', default='JR1',
              help='report name (default JR1)')
@click.option('--release', '-c', default=4,
              help='COUNTER release (default 4)')
@click.option('--start_date', '-s',
              help='Start Date YYYY-MM-DD (default first day of last month)')
@click.option('--end_date', '-e',
              help='End Date YYYY-MM-DD (default last day of last month OR '
                   'last day of start month')
@click.option('--requestor_id', '-i',
              help='Requestor ID')
@click.option('--customer_reference', '-c',
              help='Customer reference')
@click.option('--format', '-f', 'format_', default='tsv',
              help='Output format (default tsv)',
              type=click.Choice(['tsv']))
def main(url, report, release, start_date, end_date, requestor_id,
         customer_reference, format_):
    # FIXME: doesn't actually default dates yet...
    click.echo("pycounter SUSHI client for URL %s (%s R%s)"
               % (url, report, release))
    report = sushi.get_report(wsdl_url=url,
                              report=report,
                              release=release,
                              requestor_id=requestor_id,
                              customer_reference=customer_reference,
                              start_date=convert_date_run(start_date),
                              end_date=convert_date_run(end_date))
    report.write_tsv('report.tsv')

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

from datetime import datetime, timedelta

from airflow import DAG
from airflow.operators.python import PythonOperator
from airflow.providers.elasticsearch.hooks.elasticsearch import ElasticsearchHook

def show_tables():
    """
    show_tables queries elasticsearch to list available tables
    """
    es = ElasticsearchHook(elasticsearch_conn_id='production-es')

    # Handle ES conn with context manager
    with es.get_conn() as es_conn:
        tables = es_conn.execute('SHOW TABLES')
        for table, *_ in tables:
            print(f"table: {table}")
    return True

# Default settings applied to all tasks
default_args = {
    'owner': 'airflow',
    'depends_on_past': False,
    'email_on_failure': False,
    'email_on_retry': False,
    'retries': 1,
    'retry_delay': timedelta(minutes=5),
}

# Using a DAG context manager, you don't have to specify the dag property of each task
with DAG(
    'elasticsearch_dag',
    start_date=datetime(2021, 8, 30),
    max_active_runs=1,
    schedule_interval=timedelta(days=1),
    default_args=default_args,
    catchup=False,
) as dag:

    es_tables = PythonOperator(task_id='es_print_tables', python_callable=show_tables)
<commit_msg>Updating the Elasticsearch example DAG to use the TaskFlow API (#18565)<commit_after># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

from datetime import datetime, timedelta

from airflow import DAG
from airflow.decorators import task
from airflow.providers.elasticsearch.hooks.elasticsearch import ElasticsearchHook

@task(task_id='es_print_tables')
def show_tables():
    """
    show_tables queries elasticsearch to list available tables
    """
    es = ElasticsearchHook(elasticsearch_conn_id='production-es')

    # Handle ES conn with context manager
    with es.get_conn() as es_conn:
        tables = es_conn.execute('SHOW TABLES')
        for table, *_ in tables:
            print(f"table: {table}")
    return True

# Using a DAG context manager, you don't have to specify the dag property of each task
with DAG(
    'elasticsearch_dag',
    start_date=datetime(2021, 8, 30),
    max_active_runs=1,
    schedule_interval=timedelta(days=1),
    default_args={'retries': 1},  # Default setting applied to all tasks
    catchup=False,
) as dag:

    show_tables()
<|endoftext|>


<commit_before>import os
import pkgutil
import sys

import leapp.libraries.common
import leapp.workflows
from logging import getLogger
from leapp.exceptions import ModuleNameAlreadyExistsError, RepoItemPathDoesNotExistError, UnsupportedDefinitionKindError
from leapp.repository.definition import DefinitionKind
from leapp.repository.actor_definition import ActorDefinition
from leapp.utils.project import get_project_name

class Repository(object):
    """
    The Repository class represents a place where all resources (actors, models, tags, etc.) are defined. The repository directory layout looks like :ref:`Repository Directory Layout <best-practises:repository directory layout>`
    """
    def __init__(self, directory):
        """
        :param directory: Path to the repository folder
        :type directory: str
        """
        self.name = get_project_name(directory)
        self.log = getLogger('leapp.repository').getChild(self.name)
        self._repo_dir = directory
        self._definitions = {}
        self.log.info("New repository '%s' initialized at %s", self.name, directory)

    def lookup_actor(self, name):
        """
        Finds actor in repository

        :param name: Name of the actor
        :type name: str
        :return: None or Actor
        """
        name = name.lower()
        for actor in self.actors:
            actor_name = actor.name.lower()
            actor_class = actor.class_name.lower()
            if name in (actor_name, actor_class):
                return actor
        return None

    @staticmethod
    def lookup_workflow(name):
        """
        Finds workflow in repository

        :param name: Name of the workflow class name, Workflow.name or Workflow.short_name
        :type name: str
        :return: None or Workflow
        """
        name = name.lower()
        for workflow in leapp.workflows.get_workflows():
            workflow_name = workflow.name.lower()
            workflow_class = workflow.class_name.lower()
            workflow_short_name = workflow.short_name
            if name in (workflow_name, workflow_class, workflow_short_name):
                return workflow
        return None

    def add(self, kind, item):
        """
        Adds any supported kind of resource to the repository

        :param kind: specific kind of repository resource
        :type kind: :py:class:`leapp.repository.definition.DefinitionKind`
        :param item: Item that will be added
        :type item: :py:class:`leapp.repository.actor_definition.ActorDefiniton` or str
        """
        self.log.debug('Adding %s - %s', kind.name, item.directory if isinstance(item, ActorDefinition) else item)
        if kind not in DefinitionKind.REPO_WHITELIST:
            raise UnsupportedDefinitionKindError('Repositories do not support {kind}.'.format(kind=kind.name))

        # Except for ActorDefinitions all items added are paths and are supposed to be relative paths to the repository
        if not isinstance(item, ActorDefinition):
            full_path = os.path.join(self._repo_dir, item)
            if not os.path.exists(full_path):
                self.log.error("Attempted to add %s which is not in the repositories path", item)
                raise RepoItemPathDoesNotExistError(kind, item, full_path)
            item = full_path

        self._definitions.setdefault(kind, []).append(item)

    def load(self):
        """
        Loads the repository resources
        """
        self.log.debug("Loading repository %s", self.name)
        self.log.debug("Loading tag modules")
        self._load_modules(self.tags)
        self.log.debug("Loading topic modules")
        self._load_modules(self.topics)
        self.log.debug("Loading model modules")
        self._load_modules(self.models)

        if not leapp.libraries.common.LEAPP_BUILTIN_COMMON_INITIALIZED:
            self.log.debug("Loading builtin common libraries")
            self._load_libraries(path=(os.path.dirname(leapp.libraries.common.__file__) + '/',))
            leapp.libraries.common.LEAPP_BUILTIN_COMMON_INITIALIZED = True

        self.log.debug("Loading repository provided common libraries")
        self._load_libraries()

        self.log.debug("Extending PATH for common snactor paths")
        self._extend_environ_paths('PATH', self.tools)
        self.log.debug("Extending LEAPP_COMMON_FILES for common file paths")
        self._extend_environ_paths('LEAPP_COMMON_FILES', self.files)

        self.log.debug("Running actor discovery")
        for actor in self.actors:
            actor.discover()

        self.log.debug("Loading workflow modules")
        self._load_modules(self.workflows)

    def _load_libraries(self, path=None, mod=None, prefix='leapp.libraries.common'):
        for importer, name, is_pkg in pkgutil.iter_modules(path or self.libraries):
            mod_full_name = prefix + '.' + name
            if mod_full_name in sys.modules:
                self.log.error("Common library module name clash: %s has been already loaded", mod_full_name)
                raise ModuleNameAlreadyExistsError(
                    'Module {name} has been already loaded from somewhere else.\n'
                    'Loaded: {loaded}\nNow: {now}'.format(
                        name=mod_full_name,
                        loaded=sys.modules.get(mod_full_name).__file__,
                        now=importer.find_module(name).file.name))
            loaded = importer.find_module(name).load_module(name)
            if not mod:
                setattr(leapp.libraries.common, name, loaded)
            sys.modules[mod_full_name] = loaded
            if is_pkg:
                self._load_libraries(path=(os.path.dirname(loaded.__file__),), mod=loaded, prefix=mod_full_name)

    def _load_modules(self, modules):
        directories = tuple(os.path.join(self._repo_dir, os.path.dirname(module)) for module in modules)
        for importer, name, is_pkg in pkgutil.iter_modules(directories):
            importer.find_module(name).load_module(name)

    def dump(self):
        """
        :return: Dictionary of all repository resources
        """
        return {
            'repo_dir': self._repo_dir,
            'actors': [a.dump() for a in self.actors],
            'topics': self.relative_paths(self.topics),
            'models': self.relative_paths(self.models),
            'tags': self.relative_paths(self.tags),
            'workflows': self.relative_paths(self.workflows),
            'tools': self.relative_paths(self.tools),
            'files': self.relative_paths(self.files),
            'libraries': self.relative_paths(self.libraries)
        }

    @staticmethod
    def _extend_environ_paths(name, paths):
        os.environ[name] = ':'.join(os.environ.get(name, '').split(':') + list(paths))

    def relative_paths(self, paths):
        """
        :return: Tuple of repository relative paths
        """
        return tuple(map(lambda x: os.path.relpath(x, self._repo_dir), paths))

    @property
    def actors(self):
        """
        :return: Tuple of actors in the repository
        """
        return tuple(self._definitions.get(DefinitionKind.ACTOR, ()))

    @property
    def topics(self):
        """
        :return: Tuple of topics in the repository
        """
        return tuple(self._definitions.get(DefinitionKind.TOPIC, ()))

    @property
    def models(self):
        """
        :return: Tuple of models in the repository
        """
        return tuple(self._definitions.get(DefinitionKind.MODEL, ()))

    @property
    def tags(self):
        """
        :return: Tuple of tags in the repository
        """
        return tuple(self._definitions.get(DefinitionKind.TAG, ()))

    @property
    def workflows(self):
        """
        :return: Tuple of workflows in the repository
        """
        return tuple(self._definitions.get(DefinitionKind.WORKFLOW, ()))

    @property
    def tools(self):
        """
        :return: Tuple of tools in the repository
        """
        return tuple(self._definitions.get(DefinitionKind.TOOLS, ()))

    @property
    def libraries(self):
        """
        :return: Tuple of libraries in the repository
        """
        return tuple(self._definitions.get(DefinitionKind.LIBRARIES, ()))

    @property
    def files(self):
        """
        :return: Tuple of files in the repository
        """
        return tuple(self._definitions.get(DefinitionKind.FILES, ()))
<commit_msg>workflow lookup fix<commit_after>import os
import pkgutil
import sys

import leapp.libraries.common
import leapp.workflows
from logging import getLogger
from leapp.exceptions import ModuleNameAlreadyExistsError, RepoItemPathDoesNotExistError, UnsupportedDefinitionKindError
from leapp.repository.definition import DefinitionKind
from leapp.repository.actor_definition import ActorDefinition
from leapp.utils.project import get_project_name

class Repository(object):
    """
    The Repository class represents a place where all resources (actors, models, tags, etc.) are defined. The repository directory layout looks like :ref:`Repository Directory Layout <best-practises:repository directory layout>`
    """
    def __init__(self, directory):
        """
        :param directory: Path to the repository folder
        :type directory: str
        """
        self.name = get_project_name(directory)
        self.log = getLogger('leapp.repository').getChild(self.name)
        self._repo_dir = directory
        self._definitions = {}
        self.log.info("New repository '%s' initialized at %s", self.name, directory)

    def lookup_actor(self, name):
        """
        Finds actor in repository

        :param name: Name of the actor
        :type name: str
        :return: None or Actor
        """
        name = name.lower()
        for actor in self.actors:
            actor_name = actor.name.lower()
            actor_class = actor.class_name.lower()
            if name in (actor_name, actor_class):
                return actor
        return None

    @staticmethod
    def lookup_workflow(name):
        """
        Finds workflow in repository

        :param name: Name of the workflow class name, Workflow.name or Workflow.short_name
        :type name: str
        :return: None or Workflow
        """
        name = name.lower()
        for workflow in leapp.workflows.get_workflows():
            workflow_name = workflow.name.lower()
            workflow_class = workflow.__name__.lower()
            workflow_short_name = workflow.short_name
            if name in (workflow_name, workflow_class, workflow_short_name):
                return workflow
        return None

    def add(self, kind, item):
        """
        Adds any supported kind of resource to the repository

        :param kind: specific kind of repository resource
        :type kind: :py:class:`leapp.repository.definition.DefinitionKind`
        :param item: Item that will be added
        :type item: :py:class:`leapp.repository.actor_definition.ActorDefiniton` or str
        """
        self.log.debug('Adding %s - %s', kind.name, item.directory if isinstance(item, ActorDefinition) else item)
        if kind not in DefinitionKind.REPO_WHITELIST:
            raise UnsupportedDefinitionKindError('Repositories do not support {kind}.'.format(kind=kind.name))

        # Except for ActorDefinitions all items added are paths and are supposed to be relative paths to the repository
        if not isinstance(item, ActorDefinition):
            full_path = os.path.join(self._repo_dir, item)
            if not os.path.exists(full_path):
                self.log.error("Attempted to add %s which is not in the repositories path", item)
                raise RepoItemPathDoesNotExistError(kind, item, full_path)
            item = full_path

        self._definitions.setdefault(kind, []).append(item)

    def load(self):
        """
        Loads the repository resources
        """
        self.log.debug("Loading repository %s", self.name)
        self.log.debug("Loading tag modules")
        self._load_modules(self.tags)
        self.log.debug("Loading topic modules")
        self._load_modules(self.topics)
        self.log.debug("Loading model modules")
        self._load_modules(self.models)

        if not leapp.libraries.common.LEAPP_BUILTIN_COMMON_INITIALIZED:
            self.log.debug("Loading builtin common libraries")
            self._load_libraries(path=(os.path.dirname(leapp.libraries.common.__file__) + '/',))
            leapp.libraries.common.LEAPP_BUILTIN_COMMON_INITIALIZED = True

        self.log.debug("Loading repository provided common libraries")
        self._load_libraries()

        self.log.debug("Extending PATH for common snactor paths")
        self._extend_environ_paths('PATH', self.tools)
        self.log.debug("Extending LEAPP_COMMON_FILES for common file paths")
        self._extend_environ_paths('LEAPP_COMMON_FILES', self.files)

        self.log.debug("Running actor discovery")
        for actor in self.actors:
            actor.discover()

        self.log.debug("Loading workflow modules")
        self._load_modules(self.workflows)

    def _load_libraries(self, path=None, mod=None, prefix='leapp.libraries.common'):
        for importer, name, is_pkg in pkgutil.iter_modules(path or self.libraries):
            mod_full_name = prefix + '.' + name
            if mod_full_name in sys.modules:
                self.log.error("Common library module name clash: %s has been already loaded", mod_full_name)
                raise ModuleNameAlreadyExistsError(
                    'Module {name} has been already loaded from somewhere else.\n'
                    'Loaded: {loaded}\nNow: {now}'.format(
                        name=mod_full_name,
                        loaded=sys.modules.get(mod_full_name).__file__,
                        now=importer.find_module(name).file.name))
            loaded = importer.find_module(name).load_module(name)
            if not mod:
                setattr(leapp.libraries.common, name, loaded)
            sys.modules[mod_full_name] = loaded
            if is_pkg:
                self._load_libraries(path=(os.path.dirname(loaded.__file__),), mod=loaded, prefix=mod_full_name)

    def _load_modules(self, modules):
        directories = tuple(os.path.join(self._repo_dir, os.path.dirname(module)) for module in modules)
        for importer, name, is_pkg in pkgutil.iter_modules(directories):
            importer.find_module(name).load_module(name)

    def dump(self):
        """
        :return: Dictionary of all repository resources
        """
        return {
            'repo_dir': self._repo_dir,
            'actors': [a.dump() for a in self.actors],
            'topics': self.relative_paths(self.topics),
            'models': self.relative_paths(self.models),
            'tags': self.relative_paths(self.tags),
            'workflows': self.relative_paths(self.workflows),
            'tools': self.relative_paths(self.tools),
            'files': self.relative_paths(self.files),
            'libraries': self.relative_paths(self.libraries)
        }

    @staticmethod
    def _extend_environ_paths(name, paths):
        os.environ[name] = ':'.join(os.environ.get(name, '').split(':') + list(paths))

    def relative_paths(self, paths):
        """
        :return: Tuple of repository relative paths
        """
        return tuple(map(lambda x: os.path.relpath(x, self._repo_dir), paths))

    @property
    def actors(self):
        """
        :return: Tuple of actors in the repository
        """
        return tuple(self._definitions.get(DefinitionKind.ACTOR, ()))

    @property
    def topics(self):
        """
        :return: Tuple of topics in the repository
        """
        return tuple(self._definitions.get(DefinitionKind.TOPIC, ()))

    @property
    def models(self):
        """
        :return: Tuple of models in the repository
        """
        return tuple(self._definitions.get(DefinitionKind.MODEL, ()))

    @property
    def tags(self):
        """
        :return: Tuple of tags in the repository
        """
        return tuple(self._definitions.get(DefinitionKind.TAG, ()))

    @property
    def workflows(self):
        """
        :return: Tuple of workflows in the repository
        """
        return tuple(self._definitions.get(DefinitionKind.WORKFLOW, ()))

    @property
    def tools(self):
        """
        :return: Tuple of tools in the repository
        """
        return tuple(self._definitions.get(DefinitionKind.TOOLS, ()))

    @property
    def libraries(self):
        """
        :return: Tuple of libraries in the repository
        """
        return tuple(self._definitions.get(DefinitionKind.LIBRARIES, ()))

    @property
    def files(self):
        """
        :return: Tuple of files in the repository
        """
        return tuple(self._definitions.get(DefinitionKind.FILES, ()))
<|endoftext|>


<commit_before>#!/bin/python3

"""
https://www.hackerrank.com/challenges/bear-and-workbook?h_r=next-challenge&h_v=zen

Lisa just got a new math workbook. A workbook contains exercise problems, grouped into chapters.

* There are n chapters in Lisa's workbook, numbered from 1 to n.
* The i-th chapter has ti problems, numbered from 1 to ti.
* Each page can hold up to k problems. There are no empty pages or unnecessary spaces, so only the last page of a chapter may contain fewer than k problems.
* Each new chapter starts on a new page, so a page will never contain problems from more than one chapter.
* The page number indexing starts at 1.

Lisa believes a problem to be special if its index (within a chapter) is the same as the page number where it's located. Given the details for Lisa's workbook, can you count its number of special problems?

Note: See the diagram in the Explanation section for more details.

Input Format

The first line contains two integers n and k — the number of chapters and the maximum number of problems per page respectively.
The second line contains n integers t1, t2,...,tn, where ti denotes the number of problems in the i-th chapter.

Contraints

1 <= n,k,ti <= 100

Output Format

Print the number of special problems in Lisa's workbook.

Sample Input

5 3
4 2 6 1 10

Sample Output

4

Explanation

The diagram below depicts Lisa's workbook with n = 5 chapters and a maximum of k = 3 problems per page. Special problems are outlined in red, and page numbers are in yellow squares.

    +-Chap1+--------+ +-Chap1+---+ +-Chap2+---+ +-Chap3+---+ +-Chap3+--------+
    |               | |          | |          | |          | |               |
    | Problem1(RED) | | Problem4 | | Problem1 | | Problem1 | | Problem4      |
    | Problem2      | |          | | Problem2 | | Problem2 | | Problem5(RED) |
    | Problem3      | |          | |          | | Problem3 | | Problem6      |
    |               | |          | |          | |          | |               |
    +---------------+ +----------+ +----------+ +----------+ +---------------+
         P1                P2           P3           P4           P5

    +-Chap4----+ +-Chap5----+ +-Chap5----+ +-Chap5---------+ +-Chap5----------+
    |          | |          | |          | |               | |                |
    | Problem1 | | Problem1 | | Problem4 | | Problem7      | | Problem10(RED) |
    |          | | Problem2 | | Problem5 | | Problem8      | |                |
    |          | | Problem3 | | Problem6 | | Problem9(RED) | |                |
    |          | |          | |          | |               | |                |
    |          | |          | |          | |               | |                |
    +----------+ +----------+ +----------+ +---------------+ +----------------+
         P6           P7           P8           P9                P10

There are 4 special problems and thus we print the number 4 on a new line.
"""

import sys

arr = [int(arr_temp) for arr_temp in input().strip().split(' ')]
n, k = arr[0], arr[1]
t_arr = [int(arr_temp) for arr_temp in input().strip().split(' ')]

page_number = 1
result = 0
for problems_count in t_arr:
    pages_count = (problems_count - 1) // k + 1
    <commit_msg>update problem<commit_after>#!/bin/python3

"""
https://www.hackerrank.com/challenges/bear-and-workbook?h_r=next-challenge&h_v=zen

Lisa just got a new math workbook. A workbook contains exercise problems, grouped into chapters.

* There are n chapters in Lisa's workbook, numbered from 1 to n.
* The i-th chapter has ti problems, numbered from 1 to ti.
* Each page can hold up to k problems. There are no empty pages or unnecessary spaces, so only the last page of a chapter may contain fewer than k problems.
* Each new chapter starts on a new page, so a page will never contain problems from more than one chapter.
* The page number indexing starts at 1.

Lisa believes a problem to be special if its index (within a chapter) is the same as the page number where it's located. Given the details for Lisa's workbook, can you count its number of special problems?

Note: See the diagram in the Explanation section for more details.

Input Format

The first line contains two integers n and k — the number of chapters and the maximum number of problems per page respectively.
The second line contains n integers t1, t2,...,tn, where ti denotes the number of problems in the i-th chapter.

Contraints

1 <= n,k,ti <= 100

Output Format

Print the number of special problems in Lisa's workbook.

Sample Input

5 3
4 2 6 1 10

Sample Output

4

Explanation

The diagram below depicts Lisa's workbook with n = 5 chapters and a maximum of k = 3 problems per page. Special problems are outlined in red, and page numbers are in yellow squares.

    +-Chap1+--------+ +-Chap1+---+ +-Chap2+---+ +-Chap3+---+ +-Chap3+--------+
    |               | |          | |          | |          | |               |
    | Problem1(RED) | | Problem4 | | Problem1 | | Problem1 | | Problem4      |
    | Problem2      | |          | | Problem2 | | Problem2 | | Problem5(RED) |
    | Problem3      | |          | |          | | Problem3 | | Problem6      |
    |               | |          | |          | |          | |               |
    +---------------+ +----------+ +----------+ +----------+ +---------------+
         P1                P2           P3           P4           P5

    +-Chap4----+ +-Chap5----+ +-Chap5----+ +-Chap5---------+ +-Chap5----------+
    |          | |          | |          | |               | |                |
    | Problem1 | | Problem1 | | Problem4 | | Problem7      | | Problem10(RED) |
    |          | | Problem2 | | Problem5 | | Problem8      | |                |
    |          | | Problem3 | | Problem6 | | Problem9(RED) | |                |
    |          | |          | |          | |               | |                |
    |          | |          | |          | |               | |                |
    +----------+ +----------+ +----------+ +---------------+ +----------------+
         P6           P7           P8           P9                P10

There are 4 special problems and thus we print the number 4 on a new line.
"""

import sys

arr = [int(arr_temp) for arr_temp in input().strip().split(' ')]
n, k = arr[0], arr[1]
t_arr = [int(arr_temp) for arr_temp in input().strip().split(' ')]

page_number = 1
result = 0

for problems_count in t_arr:
    pages_count = (problems_count - 1) // k + 1

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Wed Oct 21 12:29:33 2015
Main Class in charge of writing/reading to/from hdf5 file.
@author: Numan Laanait, Suhas Somnath, Chris Smith
"""

import os
import subprocess
import sys
from time import time, sleep
from warnings import warn

import h5py
import numpy as np

from .microdata import MicroDataGroup
from ..__version__ import version

class ioHDF5(object):

    def __init__(self, file_handle, cachemult=1):
        """
        Handles:
            + I/O operation from HDF5 file.
            + Utilities to get data and associated auxiliary.

        Parameters
        ----------
        file_handle : Object - String or Unicode or open hdf5 file
            Absolute path to the h5 file or an open hdf5 file
        cachemult : unsigned int (Optional. default = 1)
            Cache multiplier
        """
        if type(file_handle) in [str, unicode]:
            # file handle is actually a file path
            propfaid = h5py.h5p.create(h5py.h5p.FILE_ACCESS)
            if cachemult != 1:
                settings = list(propfaid.get_cache())
                settings[2] *= cachemult
                propfaid.set_cache(*settings)
            try:
                fid = h5py.h5f.open(file_handle,fapl=propfaid)
                self.file = h5py.File(fid, mode = 'r+')
            except IOError:
                #print('Unable to open file %s. \n Making a new one! \n' %(filename))
                fid = h5py.h5f.create(file_handle,fapl=propfaid)
                self.file = h5py.File(fid, mode = 'w')
            except:
                raise
            self.path = file_handle
        elif type(file_handle) == h5py.File:
            # file handle is actually an open hdf file
            if file_handle.mode == 'r':
                warn('ioHDF5 cannot work with open HDF5 files in read mode. Change to r+ or w')
                return
            self.file = file_handle.file
            self.path = file_handle.filename

    def clear(self):
        """
        Clear h5.file of all contents

        file.clear() only removes the contents, it does not free up previously allocated space.
        To do so, it's necessary to use the h5repack command after clearing.
        Because the file must be closed and reopened, it is best to call this
        function immediately after the creation of the ioHDF5 object.
        """
        self.file.clear()
        self.repack()

    def repack(self):
        """
        Uses the h5repack command to recover cleared space in an hdf5 file.
        h5repack can also be used to change chunking and compression, but these options have
        not yet been implemented here.
        """
        self.close()
        tmpfile = self.path+'.tmp'

        '''
        Repack the opened hdf5 file into a temporary file
        '''
        try:
            repack_line = ' '.join(['h5repack', '"'+self.path+'"', '"'+tmpfile+'"'])
            subprocess.check_output(repack_line,
                                    stderr=subprocess.STDOUT,
                                    shell=True)
            # Check that the file is done being modified
            sleep(0.5)
            while time()-os.stat(tmpfile).st_mtime <= 1:
                sleep(0.5)
        except subprocess.CalledProcessError as err:
            print('Could not repack hdf5 file')
            raise Exception(err.output)
        except:
            raise

        '''
        Delete the original file and move the temporary file to the originals path
        '''
# TODO Find way to get the real OS error that works in and out of Spyder
        try:
            os.remove(self.path)
            os.rename(tmpfile, self.path)
        except:
            print('Could not copy repacked file to original path.')
            print('The original file is located {}'.format(self.path))
            print('The repacked file is located {}'.format(tmpfile))
            raise

        '''
        Open the repacked file
        '''
        self.file = h5py.File(self.path, mode = 'r+')

    def close(self):
        '''Close h5.file'''
        self.file.close()

    def delete(self):
        ''' Delete h5.file'''
        self.close()
        os.remove(self.path)

    def flush(self):
        '''Flush data from memory and commit to file.
        Use this after manually inserting data into the hdf dataset'''
        self.file.flush()

    def writeData(self, data, print_log=False):
        '''
        Writes data into the hdf5 file and assigns data attributes such as region references.
        The tree structure is inferred from the AFMData Object.

        Parameters
        ----------
        data : Instance of MicroData
            Tree structure describing the organization of the data

        Returns
        -------
        refList : List of HDF5dataset or HDF5Datagroup references
            References to the objects written
        '''

        f = self.file

        f.attrs['PySPM version']=version

        # Checking if the data is an MicroDataGroup object
        if not isinstance(data, MicroDataGroup):
            warn('Input of type: {} \n'.format(type(data)))
            sys.exit("Input not of type MicroDataGroup.\n We're done here! \n")

        # Figuring out if the first item in AFMData tree is file or group
        if data.name is '' and data.parent is '/':
            # For file we just write the attributes
            for key, val in data.attrs.iteritems():
                f.attrs[key] = val
            if print_log: print('Wrote attributes of file {} \n'.format(f.name))
            root = f.name
        else:
            # For a group we write it and its attributes
            if data.indexed:
                ''' If the name of the requested group ends in a '_', the user expects
                the suffix index to be appended automatically. Here, we check to
                ensure that the chosen index is new.
                '''
                previous = np.where([data.name in key for key in f[data.parent].keys()])[0]
                if len(previous)==0:
                    index = 0
                else:
                    # assuming that the last element of previous contains the highest index
                    last = f[data.parent].keys()[previous[-1]]
                    index = int(last.split('_')[-1])+1
                data.name += '{:03d}'.format(index)
            try:
                g = f[data.parent].create_group(data.name)
                if print_log: print('Created group {}'.format(g.name))
            except ValueError:
                g = f[data.parent][data.name]
                if print_log: print('Group already exists: {}'.format(g.name))
            except:
                f.flush()
                f.close()
                raise
            for key, val in data.attrs.iteritems():
                if val is None:
                    continue
                g.attrs[key] = val
            if print_log: print('Wrote attributes to group: {} \n'.format(data.name))
            root = g.name

        # Populating the tree structure recursively
        refList = []
        # Recursive function
        def __populate(child, parent):

            if isinstance(child, MicroDataGroup):
                if child.indexed:
                    previous = np.where([child.name in key for key in f[parent].keys()])[0]
                    if len(previous)==0:
                        index = 0
                    else:
                        last = f[parent].keys()[previous[-1]]
                        index = int(last.split('_')[-1])+1
                    child.name+='{:03d}'.format(index)
                try:
                    itm = f[parent].create_group(child.name)
                    if print_log: print('Created Group {}'.format(itm.name))
                except ValueError:
                    itm = f[parent][child.name]
                    print('Found Group already exists {}'.format(itm.name))
                except:
                    f.flush()
                    f.close()
                    raise
                for key, val in child.attrs.iteritems():
                    print key, val
                    if val is None:
                        continue
                    itm.attrs[key] = val
                if print_log: print('Wrote attributes to group {}\n'.format(itm.name))
                # here we do the recursive function call
                for ch in child.children:
                    __populate(ch, parent+'/'+child.name)
            else:
                if not child.resizable:
                    if not bool(child.maxshape):
                        # finite sized dataset and maxshape is not provided
                        # Typically for small / ancilliary datasets
                        try:
                            itm = f[parent].create_dataset(child.name,
                                                        data = child.data,
                                                        compression = child.compression,
                                                        dtype = child.data.dtype,
                                                        chunks= child.chunking)
                        except RuntimeError:
                            itm = f[parent][child.name]
                            warn('Found Dataset already exists {}'.format(itm.name))
                        except:
                            f.flush()
                            f.close()
                            raise
                    else:
                        # In many cases, we DON'T need resizable datasets but we know the max-size
                        # Here, we only allocate the space. The provided data is ignored
                        # print child.name
                        try:
                            itm = f[parent].create_dataset(child.name, child.maxshape,
                                                        compression = child.compression,
                                                        dtype = child.dtype,
                                                        chunks= child.chunking)
                        except RuntimeError:
                            itm = f[parent][child.name]
                            warn('Found Dataset already exists {}'.format(itm.name))
                        except:
                            f.flush()
                            f.close()
                            raise
                else:
                    # Resizable but the written files are significantly larger
                    max_shape = tuple([ None for i in range(len(child.data.shape))])
                    try:
                        itm = f[parent].create_dataset(child.name,
                                                    data = child.data,
                                                    compression = child.compression,
                                                    dtype = child.data.dtype,
                                                    chunks= child.chunking,
                                                    maxshape = max_shape)
                    except RuntimeError:
                        itm = f[parent][child.name]
                        warn('Found Dataset already exists {}'.format(itm.name))
                    except:
                        f.flush()
                        f.close()
                        raise

                if print_log: print('Created Dataset {}'.format(itm.name))
                for key in child.attrs.iterkeys():
                    # print('Found some region references')
                    # writing region reference
                    if key is 'labels':
                        # print('Found some region references')
                        labels = child.attrs[key]# labels here is a dictionary
                        self.write_region_references(itm, labels, print_log=print_log)
                        '''
                        Now make an attribute called 'labels' that is a list of strings
                        First ascertain the dimension of the slicing:
                        '''
                        found_dim = False
                        for dimen, slobj in enumerate(labels[labels.keys()[0]]):
                            # We make the assumption that checking the start is sufficient
                            if slobj.start != None:
                                found_dim = True
                                break
                        if found_dim:
                            headers = [None]*len(labels) # The list that will hold all the names
                            for col_name in labels.keys():
                                headers[labels[col_name][dimen].start] = col_name
                            # Now write the list of col / row names as an attribute:
                            itm.attrs[key] = headers
                        else:
                            warn('Unable to write region labels for %s' %(itm.name.split('/')[-1]))

                        if print_log: print('Wrote Region References of Dataset %s' %(itm.name.split('/')[-1]))
                    else:
                        itm.attrs[key] = child.attrs[key]
                        if print_log: print('Wrote Attributes of Dataset %s \n' %(itm.name.split('/')[-1]))
                        # Make a dictionary of references
            refList.append(itm)
            return refList

        # Recursive function is called at each stage beginning at the root
        for child in data.children:
            __populate(child, root)

        if print_log:
            print('Finished writing to h5 file.\n'+
                  'Right now you got yourself a fancy folder structure. \n'+
                  'Make sure you do some reference linking to take advantage of the full power of HDF5.')
        return refList

    @staticmethod
    def write_region_references(dataset, slices, print_log=False):
        '''
        Creates attributes of a h5.Dataset that refer to regions in the arrays

        Parameters
        ----------
        dataset : h5.Dataset instance
            Dataset to which region references will be added as attributes
        slices : dictionary
            The slicing information must be formatted using tuples of slice objects.
            For example {'region_1':(slice(None, None), slice (0,1))}
        print_log : Boolean (Optional. Default = False)
            Whether or not to print status messages
        '''
        if print_log: print('Starting to write Region References to Dataset', dataset.name, 'of shape:', dataset.shape)
        for sl in slices.iterkeys():
            if print_log: print('About to write region reference:', sl, ':', slices[sl])
            if len(slices[sl]) == len(dataset.shape):
                dataset.attrs[sl] = dataset.regionref[slices[sl]]
                if print_log: print('Wrote Region Reference:%s' % sl)
            else:
                warn('Region reference %s could not be written since the object size was not equal to the dimensions of'
                     ' the dataset' % sl)
                raise ValueError
<commit_msg>Minor docstring formatting and removed old print<commit_after># -*- coding: utf-8 -*-
"""
Created on Wed Oct 21 12:29:33 2015
Main Class in charge of writing/reading to/from hdf5 file.
@author: Numan Laanait, Suhas Somnath, Chris Smith
"""

import os
import subprocess
import sys
from time import time, sleep
from warnings import warn

import h5py
import numpy as np

from .microdata import MicroDataGroup
from ..__version__ import version

class ioHDF5(object):

    def __init__(self, file_handle, cachemult=1):
        """
        Handles:
            + I/O operation from HDF5 file.
            + Utilities to get data and associated auxiliary.

        Parameters
        ----------
        file_handle : Object - String or Unicode or open hdf5 file
            Absolute path to the h5 file or an open hdf5 file
        cachemult : unsigned int (Optional. default = 1)
            Cache multiplier
        """
        if type(file_handle) in [str, unicode]:
            # file handle is actually a file path
            propfaid = h5py.h5p.create(h5py.h5p.FILE_ACCESS)
            if cachemult != 1:
                settings = list(propfaid.get_cache())
                settings[2] *= cachemult
                propfaid.set_cache(*settings)
            try:
                fid = h5py.h5f.open(file_handle,fapl=propfaid)
                self.file = h5py.File(fid, mode = 'r+')
            except IOError:
                #print('Unable to open file %s. \n Making a new one! \n' %(filename))
                fid = h5py.h5f.create(file_handle,fapl=propfaid)
                self.file = h5py.File(fid, mode = 'w')
            except:
                raise
            self.path = file_handle
        elif type(file_handle) == h5py.File:
            # file handle is actually an open hdf file
            if file_handle.mode == 'r':
                warn('ioHDF5 cannot work with open HDF5 files in read mode. Change to r+ or w')
                return
            self.file = file_handle.file
            self.path = file_handle.filename

    def clear(self):
        """
        Clear h5.file of all contents

        file.clear() only removes the contents, it does not free up previously allocated space.
        To do so, it's necessary to use the h5repack command after clearing.
        Because the file must be closed and reopened, it is best to call this
        function immediately after the creation of the ioHDF5 object.
        """
        self.file.clear()
        self.repack()

    def repack(self):
        """
        Uses the h5repack command to recover cleared space in an hdf5 file.
        h5repack can also be used to change chunking and compression, but these options have
        not yet been implemented here.
        """
        self.close()
        tmpfile = self.path+'.tmp'

        '''
        Repack the opened hdf5 file into a temporary file
        '''
        try:
            repack_line = ' '.join(['h5repack', '"'+self.path+'"', '"'+tmpfile+'"'])
            subprocess.check_output(repack_line,
                                    stderr=subprocess.STDOUT,
                                    shell=True)
            # Check that the file is done being modified
            sleep(0.5)
            while time()-os.stat(tmpfile).st_mtime <= 1:
                sleep(0.5)
        except subprocess.CalledProcessError as err:
            print('Could not repack hdf5 file')
            raise Exception(err.output)
        except:
            raise

        '''
        Delete the original file and move the temporary file to the originals path
        '''
# TODO Find way to get the real OS error that works in and out of Spyder
        try:
            os.remove(self.path)
            os.rename(tmpfile, self.path)
        except:
            print('Could not copy repacked file to original path.')
            print('The original file is located {}'.format(self.path))
            print('The repacked file is located {}'.format(tmpfile))
            raise

        '''
        Open the repacked file
        '''
        self.file = h5py.File(self.path, mode = 'r+')

    def close(self):
        """
        Close h5.file
        """
        self.file.close()

    def delete(self):
        """
        Delete h5.file
        """
        self.close()
        os.remove(self.path)

    def flush(self):
        """
        Flush data from memory and commit to file.
        Use this after manually inserting data into the hdf dataset
        """
        self.file.flush()

    def writeData(self, data, print_log=False):
        """
        Writes data into the hdf5 file and assigns data attributes such as region references.
        The tree structure is inferred from the AFMData Object.

        Parameters
        ----------
        data : Instance of MicroData
            Tree structure describing the organization of the data

        Returns
        -------
        refList : List of HDF5dataset or HDF5Datagroup references
            References to the objects written
        """

        f = self.file

        f.attrs['PySPM version']=version

        # Checking if the data is an MicroDataGroup object
        if not isinstance(data, MicroDataGroup):
            warn('Input of type: {} \n'.format(type(data)))
            sys.exit("Input not of type MicroDataGroup.\n We're done here! \n")

        # Figuring out if the first item in AFMData tree is file or group
        if data.name is '' and data.parent is '/':
            # For file we just write the attributes
            for key, val in data.attrs.iteritems():
                f.attrs[key] = val
            if print_log:
                print('Wrote attributes of file {} \n'.format(f.name))
            root = f.name
        else:
            # For a group we write it and its attributes
            if data.indexed:
                ''' If the name of the requested group ends in a '_', the user expects
                the suffix index to be appended automatically. Here, we check to
                ensure that the chosen index is new.
                '''
                previous = np.where([data.name in key for key in f[data.parent].keys()])[0]
                if len(previous) == 0:
                    index = 0
                else:
                    # assuming that the last element of previous contains the highest index
                    last = f[data.parent].keys()[previous[-1]]
                    index = int(last.split('_')[-1])+1
                data.name += '{:03d}'.format(index)
            try:
                g = f[data.parent].create_group(data.name)
                if print_log: print('Created group {}'.format(g.name))
            except ValueError:
                g = f[data.parent][data.name]
                if print_log: print('Group already exists: {}'.format(g.name))
            except:
                f.flush()
                f.close()
                raise
            for key, val in data.attrs.iteritems():
                if val is None:
                    continue
                g.attrs[key] = val
            if print_log: print('Wrote attributes to group: {} \n'.format(data.name))
            root = g.name

        # Populating the tree structure recursively
        refList = []
        # Recursive function
        def __populate(child, parent):

            if isinstance(child, MicroDataGroup):
                if child.indexed:
                    previous = np.where([child.name in key for key in f[parent].keys()])[0]
                    if len(previous)==0:
                        index = 0
                    else:
                        last = f[parent].keys()[previous[-1]]
                        index = int(last.split('_')[-1])+1
                    child.name+='{:03d}'.format(index)
                try:
                    itm = f[parent].create_group(child.name)
                    if print_log: print('Created Group {}'.format(itm.name))
                except ValueError:
                    itm = f[parent][child.name]
                    print('Found Group already exists {}'.format(itm.name))
                except:
                    f.flush()
                    f.close()
                    raise
                for key, val in child.attrs.iteritems():
                    if val is None:
                        continue
                    itm.attrs[key] = val
                if print_log: print('Wrote attributes to group {}\n'.format(itm.name))
                # here we do the recursive function call
                for ch in child.children:
                    __populate(ch, parent+'/'+child.name)
            else:
                if not child.resizable:
                    if not bool(child.maxshape):
                        # finite sized dataset and maxshape is not provided
                        # Typically for small / ancilliary datasets
                        try:
                            itm = f[parent].create_dataset(child.name,
                                                        data = child.data,
                                                        compression = child.compression,
                                                        dtype = child.data.dtype,
                                                        chunks= child.chunking)
                        except RuntimeError:
                            itm = f[parent][child.name]
                            warn('Found Dataset already exists {}'.format(itm.name))
                        except:
                            f.flush()
                            f.close()
                            raise
                    else:
                        # In many cases, we DON'T need resizable datasets but we know the max-size
                        # Here, we only allocate the space. The provided data is ignored
                        # print child.name
                        try:
                            itm = f[parent].create_dataset(child.name, child.maxshape,
                                                        compression = child.compression,
                                                        dtype = child.dtype,
                                                        chunks= child.chunking)
                        except RuntimeError:
                            itm = f[parent][child.name]
                            warn('Found Dataset already exists {}'.format(itm.name))
                        except:
                            f.flush()
                            f.close()
                            raise
                else:
                    # Resizable but the written files are significantly larger
                    max_shape = tuple([ None for i in range(len(child.data.shape))])
                    try:
                        itm = f[parent].create_dataset(child.name,
                                                    data = child.data,
                                                    compression = child.compression,
                                                    dtype = child.data.dtype,
                                                    chunks= child.chunking,
                                                    maxshape = max_shape)
                    except RuntimeError:
                        itm = f[parent][child.name]
                        warn('Found Dataset already exists {}'.format(itm.name))
                    except:
                        f.flush()
                        f.close()
                        raise

                if print_log: print('Created Dataset {}'.format(itm.name))
                for key in child.attrs.iterkeys():
                    # print('Found some region references')
                    # writing region reference
                    if key is 'labels':
                        # print('Found some region references')
                        labels = child.attrs[key]# labels here is a dictionary
                        self.write_region_references(itm, labels, print_log=print_log)
                        '''
                        Now make an attribute called 'labels' that is a list of strings
                        First ascertain the dimension of the slicing:
                        '''
                        found_dim = False
                        for dimen, slobj in enumerate(labels[labels.keys()[0]]):
                            # We make the assumption that checking the start is sufficient
                            if slobj.start != None:
                                found_dim = True
                                break
                        if found_dim:
                            headers = [None]*len(labels) # The list that will hold all the names
                            for col_name in labels.keys():
                                headers[labels[col_name][dimen].start] = col_name
                            # Now write the list of col / row names as an attribute:
                            itm.attrs[key] = headers
                        else:
                            warn('Unable to write region labels for %s' %(itm.name.split('/')[-1]))

                        if print_log: print('Wrote Region References of Dataset %s' %(itm.name.split('/')[-1]))
                    else:
                        itm.attrs[key] = child.attrs[key]
                        if print_log: print('Wrote Attributes of Dataset %s \n' %(itm.name.split('/')[-1]))
                        # Make a dictionary of references
            refList.append(itm)
            return refList

        # Recursive function is called at each stage beginning at the root
        for child in data.children:
            __populate(child, root)

        if print_log:
            print('Finished writing to h5 file.\n'+
                  'Right now you got yourself a fancy folder structure. \n'+
                  'Make sure you do some reference linking to take advantage of the full power of HDF5.')
        return refList

    @staticmethod
    def write_region_references(dataset, slices, print_log=False):
        """
        Creates attributes of a h5.Dataset that refer to regions in the arrays

        Parameters
        ----------
        dataset : h5.Dataset instance
            Dataset to which region references will be added as attributes
        slices : dictionary
            The slicing information must be formatted using tuples of slice objects.
            For example {'region_1':(slice(None, None), slice (0,1))}
        print_log : Boolean (Optional. Default = False)
            Whether or not to print status messages
        """
        if print_log: print('Starting to write Region References to Dataset', dataset.name, 'of shape:', dataset.shape)
        for sl in slices.iterkeys():
            if print_log:
                print('About to write region reference:', sl, ':', slices[sl])
            if len(slices[sl]) == len(dataset.shape):
                dataset.attrs[sl] = dataset.regionref[slices[sl]]
                if print_log:
                    print('Wrote Region Reference:%s' % sl)
            else:
                warn('Region reference %s could not be written since the object size was not equal to the dimensions of'
                     ' the dataset' % sl)
                raise ValueError
<|endoftext|>


<commit_before>'''
lawmakerStats.py
A python script to store a json file containing various statistical calculations for lawmakers

usage:
export DJANGO_SETTINGS_MODULE=leg_tracker.settings
python lawmakerStats.py

By Tyler Dukes, WRAL
'''

import json, os, urllib, re

import django
django.setup()

#load your models
from billcatcher.models import Lawmaker, Party

lawmaker_url = 'http://52.22.90.29/lawmakers/?format=json'
#for testing
#rollcall_url = 'http://52.22.90.29/rollcalls/?format=json&bill_identifier=3540'
rollcall_url = 'http://52.22.90.29/rollcalls/?format=json'

#global variables to be used in other calculations
dem_missed = []
gop_missed = []
dem_loyalty = []
gop_loyalty = []

def calc_votes():
	#load lawmaker data
	lawmaker_response = urllib.urlopen(lawmaker_url)
	lawmaker_data = json.loads(lawmaker_response.read())
	# Filter python objects with list comprehensions
	output_dict = [x for x in lawmaker_data if x['active'] == True]
	# Transform python object back into json
	lawmaker_data = json.dumps(output_dict)
	dem_list = []
	gop_list = []
	dem_record = {}
	gop_record = {}

	#create array of dem lawmakers and gop lawmakers
	for lawmaker in lawmaker_data:
		if lawmaker['party'] == "D":
			dem_list.append(lawmaker['url'])
		elif lawmaker['party'] == "R":
			gop_list.append(lawmaker['url'])

	print "...lawmakers loaded and filed"

	#load all rollcall votes
	rollcall_response = urllib.urlopen(rollcall_url)
	rollcall_data = json.loads(rollcall_response.read())

	print "...rollcall data loaded"

	#for each vote:
	#calculate majority vote of dems
	#calculate majority vote of gop
	for vote in rollcall_data:
		#blank array tracks number of Yea, Nay, NV, Absent
		#codes correspond to 1, 2, 3, 4 defined in the vote file
		dem_votes = [0,0,0,0]
		gop_votes = [0,0,0,0]
		for v in vote['votes']:
			if v['member'] in dem_list:
				dem_votes[v['vote_code']-1] += 1
			elif v['member'] in gop_list:
				gop_votes[v['vote_code']-1] += 1
		#calculate majority vote and add to dem_record
		if dem_votes[0] > dem_votes[1]:
			dem_record[vote['url']] = 1
		elif dem_votes[1] > dem_votes[0]:
			dem_record[vote['url']] = 2
		else:
			dem_record[vote['url']] = 0
		#calculate majority vote and add to gop_record
		if gop_votes[0] > gop_votes[1]:
			gop_record[vote['url']] = 1
		elif gop_votes[1] > gop_votes[0]:
			gop_record[vote['url']] = 2
		else:
			gop_record[vote['url']] = 0

	#for each lawmaker
	for lawmaker in lawmaker_data:
		#initialize variables for each lawmaker
		total_votes = 0;
		party_votes = 0;
		missed_votes = 0;
		vote_opps = 0;
		for vote in rollcall_data:
			for v in vote['votes']:
				if lawmaker['url'] == v['member']:
					#calculate total votes, votes w/party, rate
					if v['vote_code'] == 1 or v['vote_code'] == 2:
						vote_opps += 1
						total_votes += 1
						current_vote = v['vote_code']
						if lawmaker['party'] == "D":
							if current_vote == dem_record[vote['url']]:
								party_votes += 1
						elif lawmaker['party'] == "R":
							if current_vote == gop_record[vote['url']]:
								party_votes += 1
					#calculate missed or "not voting"
					elif v['vote_code'] == 3 or v['vote_code'] == 4:
						vote_opps += 1
						missed_votes += 1
		print lawmaker['name'] + " votes: " + str(party_votes) + "/" + str(total_votes)
		print lawmaker['name'] + " missed votes: " + str(missed_votes) + "/" + str(vote_opps)

		#add calculated values to dem/gop array so we can calculate party info later
		#check for zero values in denominator to prevent float errors
		if lawmaker['party'] == "D":
			if vote_opps != 0:
				dem_missed.append(round(missed_votes/float(vote_opps),3))
			else:
				dem_missed.append(0)
			if(total_votes != 0):
				dem_loyalty.append(round(party_votes/float(total_votes),3))
			else:
				dem_loyalty.append(0)
		elif lawmaker['party'] == "R":
			if(vote_opps != 0):
				gop_missed.append(round(missed_votes/float(vote_opps),3))
			else:
				gop_missed.append(0)
			if(total_votes != 0):
				gop_loyalty.append(round(party_votes/float(total_votes),3))
			else:
				gop_loyalty.append(0)

		#provide values to update
		updated_values = {
			'total_votes' : total_votes,
			'party_votes' : party_votes,
			'missed_votes' : missed_votes,
			'vote_opportunities' : vote_opps
		}
		#create a new lawmaker called _ to refer to it later
		#if it exists, update it with the new information
		pk_id = lawmaker['url'].split('/')[-2]
		_, created = Lawmaker.objects.update_or_create(
			legiscan_id = pk_id,
			defaults = updated_values
		)
	dem_values = {
		'loyalty_avg' : calc_average(dem_loyalty),
		'missed_avg' : calc_average(dem_missed)
	}
	gop_values = {
		'loyalty_avg' : calc_average(gop_loyalty),
		'missed_avg' : calc_average(gop_missed)
	}
	_, created = Party.objects.update_or_create(
		party_id = 0,
		defaults = gop_values
	)
	_, created = Party.objects.update_or_create(
		party_id = 1,
		defaults = dem_values
	)

def calc_average(values):
	total = 0
	count = 0
	for value in values:
		total += value
		count += 1
	return total/count

if __name__ == '__main__':
	calc_votes()
	print "...lawmakers stats updated"<commit_msg>try again<commit_after>'''
lawmakerStats.py
A python script to store a json file containing various statistical calculations for lawmakers

usage:
export DJANGO_SETTINGS_MODULE=leg_tracker.settings
python lawmakerStats.py

By Tyler Dukes, WRAL
'''

import json, os, urllib, re

import django
django.setup()

#load your models
from billcatcher.models import Lawmaker, Party

lawmaker_url = 'http://52.22.90.29/lawmakers/?format=json'
#for testing
#rollcall_url = 'http://52.22.90.29/rollcalls/?format=json&bill_identifier=3540'
rollcall_url = 'http://52.22.90.29/rollcalls/?format=json'

#global variables to be used in other calculations
dem_missed = []
gop_missed = []
dem_loyalty = []
gop_loyalty = []

def calc_votes():
	#load lawmaker data
	lawmaker_response = urllib.urlopen(lawmaker_url)
	lawmaker_data = json.loads(lawmaker_response.read())
	lawmaker_clean = [];
	for lawmaker in lawmaker_data:
		if lawmaker['active'] == True:
			lawmaker_clean.append(lawmaker)
	lawmaker_data = lawmaker_clean
	dem_list = []
	gop_list = []
	dem_record = {}
	gop_record = {}

	#create array of dem lawmakers and gop lawmakers
	for lawmaker in lawmaker_data:
		if lawmaker['party'] == "D":
			dem_list.append(lawmaker['url'])
		elif lawmaker['party'] == "R":
			gop_list.append(lawmaker['url'])

	print "...lawmakers loaded and filed"

	#load all rollcall votes
	rollcall_response = urllib.urlopen(rollcall_url)
	rollcall_data = json.loads(rollcall_response.read())

	print "...rollcall data loaded"

	#for each vote:
	#calculate majority vote of dems
	#calculate majority vote of gop
	for vote in rollcall_data:
		#blank array tracks number of Yea, Nay, NV, Absent
		#codes correspond to 1, 2, 3, 4 defined in the vote file
		dem_votes = [0,0,0,0]
		gop_votes = [0,0,0,0]
		for v in vote['votes']:
			if v['member'] in dem_list:
				dem_votes[v['vote_code']-1] += 1
			elif v['member'] in gop_list:
				gop_votes[v['vote_code']-1] += 1
		#calculate majority vote and add to dem_record
		if dem_votes[0] > dem_votes[1]:
			dem_record[vote['url']] = 1
		elif dem_votes[1] > dem_votes[0]:
			dem_record[vote['url']] = 2
		else:
			dem_record[vote['url']] = 0
		#calculate majority vote and add to gop_record
		if gop_votes[0] > gop_votes[1]:
			gop_record[vote['url']] = 1
		elif gop_votes[1] > gop_votes[0]:
			gop_record[vote['url']] = 2
		else:
			gop_record[vote['url']] = 0

	#for each lawmaker
	for lawmaker in lawmaker_data:
		#initialize variables for each lawmaker
		total_votes = 0;
		party_votes = 0;
		missed_votes = 0;
		vote_opps = 0;
		for vote in rollcall_data:
			for v in vote['votes']:
				if lawmaker['url'] == v['member']:
					#calculate total votes, votes w/party, rate
					if v['vote_code'] == 1 or v['vote_code'] == 2:
						vote_opps += 1
						total_votes += 1
						current_vote = v['vote_code']
						if lawmaker['party'] == "D":
							if current_vote == dem_record[vote['url']]:
								party_votes += 1
						elif lawmaker['party'] == "R":
							if current_vote == gop_record[vote['url']]:
								party_votes += 1
					#calculate missed or "not voting"
					elif v['vote_code'] == 3 or v['vote_code'] == 4:
						vote_opps += 1
						missed_votes += 1
		print lawmaker['name'] + " votes: " + str(party_votes) + "/" + str(total_votes)
		print lawmaker['name'] + " missed votes: " + str(missed_votes) + "/" + str(vote_opps)

		#add calculated values to dem/gop array so we can calculate party info later
		#check for zero values in denominator to prevent float errors
		if lawmaker['party'] == "D":
			if vote_opps != 0:
				dem_missed.append(round(missed_votes/float(vote_opps),3))
			else:
				dem_missed.append(0)
			if(total_votes != 0):
				dem_loyalty.append(round(party_votes/float(total_votes),3))
			else:
				dem_loyalty.append(0)
		elif lawmaker['party'] == "R":
			if(vote_opps != 0):
				gop_missed.append(round(missed_votes/float(vote_opps),3))
			else:
				gop_missed.append(0)
			if(total_votes != 0):
				gop_loyalty.append(round(party_votes/float(total_votes),3))
			else:
				gop_loyalty.append(0)

		#provide values to update
		updated_values = {
			'total_votes' : total_votes,
			'party_votes' : party_votes,
			'missed_votes' : missed_votes,
			'vote_opportunities' : vote_opps
		}
		#create a new lawmaker called _ to refer to it later
		#if it exists, update it with the new information
		pk_id = lawmaker['url'].split('/')[-2]
		_, created = Lawmaker.objects.update_or_create(
			legiscan_id = pk_id,
			defaults = updated_values
		)
	dem_values = {
		'loyalty_avg' : calc_average(dem_loyalty),
		'missed_avg' : calc_average(dem_missed)
	}
	gop_values = {
		'loyalty_avg' : calc_average(gop_loyalty),
		'missed_avg' : calc_average(gop_missed)
	}
	_, created = Party.objects.update_or_create(
		party_id = 0,
		defaults = gop_values
	)
	_, created = Party.objects.update_or_create(
		party_id = 1,
		defaults = dem_values
	)

def calc_average(values):
	total = 0
	count = 0
	for value in values:
		total += value
		count += 1
	return total/count

if __name__ == '__main__':
	calc_votes()
	print "...lawmakers stats updated"<|endoftext|>