<commit_before>import pytest
import mock

# AWX
from awx.main.models import (
    Host,
    Inventory,
    InventorySource,
    InventoryUpdate,
)

@pytest.mark.django_db
class TestSCMUpdateFeatures:

    def test_automatic_project_update_on_create(self, inventory, project):
        inv_src = InventorySource(
            source_project=project,
            source_path='inventory_file',
            inventory=inventory,
            update_on_project_update=True,
            source='scm')
        with mock.patch.object(inv_src.source_project, 'update') as mck_update:
            inv_src.save()
            mck_update.assert_called_once_with()

    def test_reset_scm_revision(self, scm_inventory_source):
        starting_rev = scm_inventory_source.scm_last_revision
        assert starting_rev != ''
        scm_inventory_source.source_path = '/newfolder/newfile.ini'
        scm_inventory_source.save()
        assert scm_inventory_source.scm_last_revision == ''

    def test_source_location(self, scm_inventory_source):
        # Combines project directory with the inventory file specified
        inventory_update = InventoryUpdate(
            inventory_source=scm_inventory_source,
            source_path=scm_inventory_source.source_path)
        assert inventory_update.get_actual_source_path().endswith('_92__test_proj/inventory_file')

    def test_no_unwanted_updates(self, scm_inventory_source):
        # Changing the non-sensitive fields should not trigger update
        with mock.patch.object(scm_inventory_source.source_project, 'update') as mck_update:
            scm_inventory_source.name = 'edited_inventory'
            scm_inventory_source.description = "I'm testing this!"
            scm_inventory_source.save()
            assert not mck_update.called

@pytest.fixture
def setup_ec2_gce(organization):
    ec2_inv = Inventory.objects.create(name='test_ec2', organization=organization)

    ec2_source = ec2_inv.inventory_sources.create(name='test_ec2_source', source='ec2')
    for i in range(2):
        ec2_host = ec2_inv.hosts.create(name='test_ec2_{0}'.format(i))
        ec2_host.inventory_sources.add(ec2_source)
    ec2_inv.save()

    gce_inv = Inventory.objects.create(name='test_gce', organization=organization)

    gce_source = gce_inv.inventory_sources.create(name='test_gce_source', source='gce')
    gce_host = gce_inv.hosts.create(name='test_gce_host')
    gce_host.inventory_sources.add(gce_source)
    gce_inv.save()

@pytest.fixture
def setup_inventory_groups(inventory, group_factory):

    groupA = group_factory('test_groupA')
    groupB = group_factory('test_groupB')

    host = Host.objects.create(name='single_host', inventory=inventory)

    groupA.hosts.add(host)
    groupA.save()

    groupB.hosts.add(host)
    groupB.save()

@pytest.mark.django_db
class TestHostManager:
    def test_host_filter_change(self, setup_ec2_gce, organization):
        smart_inventory = Inventory(name='smart',
                                    kind='smart',
                                    organization=organization,
                                    host_filter='inventory_sources__source=ec2')
        smart_inventory.save()
        assert len(smart_inventory.hosts.all()) == 2

        smart_inventory.host_filter = 'inventory_sources__source=gce'
        smart_inventory.save()
        assert len(smart_inventory.hosts.all()) == 1

    def test_host_filter_not_smart(self, setup_ec2_gce, organization):
        smart_inventory = Inventory(name='smart',
                                    organization=organization,
                                    host_filter='inventory_sources__source=ec2')
        assert len(smart_inventory.hosts.all()) == 0

    def test_host_objects_manager(self, setup_ec2_gce, organization):
        smart_inventory = Inventory(kind='smart',
                                    name='smart',
                                    organization=organization,
                                    host_filter='inventory_sources__source=ec2')
        smart_inventory.save()

        hosts = smart_inventory.hosts.all()
        assert len(hosts) == 2
        assert hosts[0].inventory_sources.first().source == 'ec2'
        assert hosts[1].inventory_sources.first().source == 'ec2'

    def test_host_objects_no_dupes(self, setup_inventory_groups, organization):
        smart_inventory = Inventory(name='smart',
                                    kind='smart',
                                    organization=organization,
                                    host_filter='groups__name=test_groupA or groups__name=test_groupB')
        smart_inventory.save()
        assert len(smart_inventory.hosts.all()) == 1
<commit_msg>update SCM test to mock inventory update<commit_after>import pytest
import mock

# AWX
from awx.main.models import (
    Host,
    Inventory,
    InventorySource,
    InventoryUpdate,
)

@pytest.mark.django_db
class TestSCMUpdateFeatures:

    def test_automatic_project_update_on_create(self, inventory, project):
        inv_src = InventorySource(
            source_project=project,
            source_path='inventory_file',
            inventory=inventory,
            update_on_project_update=True,
            source='scm')
        with mock.patch.object(inv_src, 'update') as mck_update:
            inv_src.save()
            mck_update.assert_called_once_with()

    def test_reset_scm_revision(self, scm_inventory_source):
        starting_rev = scm_inventory_source.scm_last_revision
        assert starting_rev != ''
        scm_inventory_source.source_path = '/newfolder/newfile.ini'
        scm_inventory_source.save()
        assert scm_inventory_source.scm_last_revision == ''

    def test_source_location(self, scm_inventory_source):
        # Combines project directory with the inventory file specified
        inventory_update = InventoryUpdate(
            inventory_source=scm_inventory_source,
            source_path=scm_inventory_source.source_path)
        assert inventory_update.get_actual_source_path().endswith('_92__test_proj/inventory_file')

    def test_no_unwanted_updates(self, scm_inventory_source):
        # Changing the non-sensitive fields should not trigger update
        with mock.patch.object(scm_inventory_source.source_project, 'update') as mck_update:
            scm_inventory_source.name = 'edited_inventory'
            scm_inventory_source.description = "I'm testing this!"
            scm_inventory_source.save()
            assert not mck_update.called

@pytest.fixture
def setup_ec2_gce(organization):
    ec2_inv = Inventory.objects.create(name='test_ec2', organization=organization)

    ec2_source = ec2_inv.inventory_sources.create(name='test_ec2_source', source='ec2')
    for i in range(2):
        ec2_host = ec2_inv.hosts.create(name='test_ec2_{0}'.format(i))
        ec2_host.inventory_sources.add(ec2_source)
    ec2_inv.save()

    gce_inv = Inventory.objects.create(name='test_gce', organization=organization)

    gce_source = gce_inv.inventory_sources.create(name='test_gce_source', source='gce')
    gce_host = gce_inv.hosts.create(name='test_gce_host')
    gce_host.inventory_sources.add(gce_source)
    gce_inv.save()

@pytest.fixture
def setup_inventory_groups(inventory, group_factory):

    groupA = group_factory('test_groupA')
    groupB = group_factory('test_groupB')

    host = Host.objects.create(name='single_host', inventory=inventory)

    groupA.hosts.add(host)
    groupA.save()

    groupB.hosts.add(host)
    groupB.save()

@pytest.mark.django_db
class TestHostManager:
    def test_host_filter_change(self, setup_ec2_gce, organization):
        smart_inventory = Inventory(name='smart',
                                    kind='smart',
                                    organization=organization,
                                    host_filter='inventory_sources__source=ec2')
        smart_inventory.save()
        assert len(smart_inventory.hosts.all()) == 2

        smart_inventory.host_filter = 'inventory_sources__source=gce'
        smart_inventory.save()
        assert len(smart_inventory.hosts.all()) == 1

    def test_host_filter_not_smart(self, setup_ec2_gce, organization):
        smart_inventory = Inventory(name='smart',
                                    organization=organization,
                                    host_filter='inventory_sources__source=ec2')
        assert len(smart_inventory.hosts.all()) == 0

    def test_host_objects_manager(self, setup_ec2_gce, organization):
        smart_inventory = Inventory(kind='smart',
                                    name='smart',
                                    organization=organization,
                                    host_filter='inventory_sources__source=ec2')
        smart_inventory.save()

        hosts = smart_inventory.hosts.all()
        assert len(hosts) == 2
        assert hosts[0].inventory_sources.first().source == 'ec2'
        assert hosts[1].inventory_sources.first().source == 'ec2'

    def test_host_objects_no_dupes(self, setup_inventory_groups, organization):
        smart_inventory = Inventory(name='smart',
                                    kind='smart',
                                    organization=organization,
                                    host_filter='groups__name=test_groupA or groups__name=test_groupB')
        smart_inventory.save()
        assert len(smart_inventory.hosts.all()) == 1
<|endoftext|>


<commit_before>"""
homeassistant.components.sensor.swiss_public_transport
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

The Swiss public transport sensor will give you the next two departure times
from a given location to another one. This sensor is limited to Switzerland.

Configuration:

To use the Swiss public transport sensor you will need to add something like
the following to your config/configuration.yaml

sensor:
  platform: swiss_public_transport
  from: STATION_ID
  to: STATION_ID

Variables:

from
*Required
Start station/stop of your trip. To search for the ID of the station, use the
an URL like this: http://transport.opendata.ch/v1/locations?query=Wankdorf
to query for the station. If the score is 100 ("score":"100" in the response),
it is a perfect match.

to
*Required
Destination station/stop of the trip. Same procedure as for the start station.

Details for the API : http://transport.opendata.ch
"""
import logging
from datetime import timedelta

from homeassistant.util import Throttle
import homeassistant.util.dt as dt_util
from homeassistant.helpers.entity import Entity

_LOGGER = logging.getLogger(__name__)
_RESOURCE = 'http://transport.opendata.ch/v1/'

# Return cached results if last scan was less then this time ago
MIN_TIME_BETWEEN_UPDATES = timedelta(seconds=60)

def setup_platform(hass, config, add_devices, discovery_info=None):
    """ Get the Swiss public transport sensor. """

    if None in (hass.config.latitude, hass.config.longitude):
        _LOGGER.error("Latitude or longitude not set in Home Assistant config")
        return False

    try:
        # pylint: disable=unused-variable
        from requests import get

    except ImportError:
        _LOGGER.exception(
            "Unable to import requests. "
            "Did you maybe not install the 'Requests' package?")

        return None

    # journal contains [0] Station ID start, [1] Station ID destination
    # [2] Station name start, and [3] Station name destination
    journey = []
    journey.append(config.get('from', None))
    journey.append(config.get('to', None))
    try:
        for location in [config.get('from', None), config.get('to', None)]:
            # transport.opendata.ch doesn't play nice with requests.Session
            result = get(_RESOURCE + 'locations?query=%s' % location)
            journey.append(result.json()['stations'][0]['name'])
    except KeyError:
        _LOGGER.error(
            "Unable to determine stations. "
            "Check your settings and/or the availability of opendata.ch")

        return None

    dev = []
    data = PublicTransportData(journey)
    dev.append(SwissPublicTransportSensor(data, journey))
    add_devices(dev)

# pylint: disable=too-few-public-methods
class SwissPublicTransportSensor(Entity):
    """ Implements an Swiss public transport sensor. """

    def __init__(self, data, journey):
        self.data = data
        self._name = journey[2] + '-' + journey[3]
        self.update()

    @property
    def name(self):
        """ Returns the name. """
        return self._name

    @property
    def state(self):
        """ Returns the state of the device. """
        return self._state

    # pylint: disable=too-many-branches
    def update(self):
        """ Gets the latest data from opendata.ch and updates the states. """
        times = self.data.update()
        if times is not None:
            self._state = ', '.join(times)

# pylint: disable=too-few-public-methods
class PublicTransportData(object):
    """ Class for handling the data retrieval.  """

    def __init__(self, journey):
        self.times = ['n/a', 'n/a']
        self.start = journey[0]
        self.destination = journey[1]

    @Throttle(MIN_TIME_BETWEEN_UPDATES)
    def update(self):
        """ Gets the latest data from opendata.ch. """

        from requests import get

        response = get(
            _RESOURCE +
            'connections?' +
            'from=' + self.start + '&' +
            'to=' + self.destination + '&' +
            'fields[]=connections/from/departureTimestamp/&' +
            'fields[]=connections/')

        try:
            self.times.insert(0, dt_util.timestamp_to_short_time_str(
                response.json()['connections'][0]['from']
                ['departureTimestamp']))
            self.times.insert(1, dt_util.timestamp_to_short_time_str(
                response.json()['connections'][1]['from']
                ['departureTimestamp']))
            return self.times

        except KeyError:
            return self.times
<commit_msg>remove lat/long check<commit_after>"""
homeassistant.components.sensor.swiss_public_transport
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

The Swiss public transport sensor will give you the next two departure times
from a given location to another one. This sensor is limited to Switzerland.

Configuration:

To use the Swiss public transport sensor you will need to add something like
the following to your config/configuration.yaml

sensor:
  platform: swiss_public_transport
  from: STATION_ID
  to: STATION_ID

Variables:

from
*Required
Start station/stop of your trip. To search for the ID of the station, use the
an URL like this: http://transport.opendata.ch/v1/locations?query=Wankdorf
to query for the station. If the score is 100 ("score":"100" in the response),
it is a perfect match.

to
*Required
Destination station/stop of the trip. Same procedure as for the start station.

Details for the API : http://transport.opendata.ch
"""
import logging
from datetime import timedelta

from homeassistant.util import Throttle
import homeassistant.util.dt as dt_util
from homeassistant.helpers.entity import Entity

_LOGGER = logging.getLogger(__name__)
_RESOURCE = 'http://transport.opendata.ch/v1/'

# Return cached results if last scan was less then this time ago
MIN_TIME_BETWEEN_UPDATES = timedelta(seconds=60)

def setup_platform(hass, config, add_devices, discovery_info=None):
    """ Get the Swiss public transport sensor. """

    try:
        # pylint: disable=unused-variable
        from requests import get

    except ImportError:
        _LOGGER.exception(
            "Unable to import requests. "
            "Did you maybe not install the 'Requests' package?")

        return None

    # journal contains [0] Station ID start, [1] Station ID destination
    # [2] Station name start, and [3] Station name destination
    journey = []
    journey.append(config.get('from', None))
    journey.append(config.get('to', None))
    try:
        for location in [config.get('from', None), config.get('to', None)]:
            # transport.opendata.ch doesn't play nice with requests.Session
            result = get(_RESOURCE + 'locations?query=%s' % location)
            journey.append(result.json()['stations'][0]['name'])
    except KeyError:
        _LOGGER.error(
            "Unable to determine stations. "
            "Check your settings and/or the availability of opendata.ch")

        return None

    dev = []
    data = PublicTransportData(journey)
    dev.append(SwissPublicTransportSensor(data, journey))
    add_devices(dev)

# pylint: disable=too-few-public-methods
class SwissPublicTransportSensor(Entity):
    """ Implements an Swiss public transport sensor. """

    def __init__(self, data, journey):
        self.data = data
        self._name = journey[2] + '-' + journey[3]
        self.update()

    @property
    def name(self):
        """ Returns the name. """
        return self._name

    @property
    def state(self):
        """ Returns the state of the device. """
        return self._state

    # pylint: disable=too-many-branches
    def update(self):
        """ Gets the latest data from opendata.ch and updates the states. """
        times = self.data.update()
        if times is not None:
            self._state = ', '.join(times)

# pylint: disable=too-few-public-methods
class PublicTransportData(object):
    """ Class for handling the data retrieval.  """

    def __init__(self, journey):
        self.times = ['n/a', 'n/a']
        self.start = journey[0]
        self.destination = journey[1]

    @Throttle(MIN_TIME_BETWEEN_UPDATES)
    def update(self):
        """ Gets the latest data from opendata.ch. """

        from requests import get

        response = get(
            _RESOURCE +
            'connections?' +
            'from=' + self.start + '&' +
            'to=' + self.destination + '&' +
            'fields[]=connections/from/departureTimestamp/&' +
            'fields[]=connections/')

        try:
            self.times.insert(0, dt_util.timestamp_to_short_time_str(
                response.json()['connections'][0]['from']
                ['departureTimestamp']))
            self.times.insert(1, dt_util.timestamp_to_short_time_str(
                response.json()['connections'][1]['from']
                ['departureTimestamp']))
            return self.times

        except KeyError:
            return self.times
<|endoftext|>


<commit_before>
import tensorflow as tf
import os
import time
from tqdm import tqdm
from src.utils import get_cli_params, process_cli_params, \
    order_param_settings
from src.val import build_graph, measure_smoothness
from src.train import evaluate_metric_list, update_decays, evaluate_metric
from src import input_data
import numpy as np

def main():

    p = process_cli_params(get_cli_params())

    # -----------------------------
    # Set GPU device to use
    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
    os.environ["CUDA_VISIBLE_DEVICES"] = str(p.which_gpu)

    # Set seeds
    np.random.seed(p.seed)
    tf.set_random_seed(p.seed)

    # Load data
    print("===  Loading Data ===")
    mnist = input_data.read_data_sets("MNIST_data",
                                      n_labeled=p.num_labeled,
                                      validation_size=p.validation,
                                      one_hot=True,
                                      disjoint=False)
    num_examples = mnist.train.num_examples
    p.num_examples = num_examples
    if p.validation > 0:
        mnist.test = mnist.validation
    p.iter_per_epoch = (num_examples // p.batch_size)
    p.num_iter = p.iter_per_epoch * p.end_epoch

    # -----------------------------
    # Build graph
    g, m, trainable_parameters = build_graph(p)

    # Collect losses
    train_losses = [m['loss'], m['cost'], m['uc'], m['vc']]
    test_losses = [m['cost']]

    if p.measure_smoothness:
        s = measure_smoothness(g, p)
    #     print(s.get_shape())
        train_losses.append(tf.reduce_mean(s))

    if p.tb is not False:
        train_merged = tf.summary.merge([
            tf.summary.scalar(x) for x in train_losses
        ] + [tf.summary.scalar(m['acc'])])
        test_merged = tf.summary.merge([
            tf.summary.scalar(x) for x in test_losses
        ] + [tf.summary.scalar(m['acc'])])

        # Set up tensorboard logging
        if not os.path.exists(p.tb):
            os.makedirs(p.tb_dir)

    # Write logs to appropriate directory
    log_dir = p.logdir + p.id
    if not os.path.exists(log_dir):
        os.makedirs(log_dir)

    desc_file = log_dir + "/" + "description"
    with open(desc_file, 'a') as f:
        print(*order_param_settings(p), sep='\n', file=f, flush=True)
        print("Trainable parameters:", trainable_parameters, file=f,
              flush=True)

    log_file = log_dir + "/" + "train_log"

    # -----------------------------
    print("===  Starting Session ===")
    sess = tf.Session()
    i_iter = 0
    # -----------------------------
    # Resume from checkpoint
    ckpt_dir = "checkpoints/" + p.id + "/"
    ckpt = tf.train.get_checkpoint_state(
        ckpt_dir)  # get latest checkpoint (if any)

    if ckpt and ckpt.model_checkpoint_path:
        # if checkpoint exists,
        # restore the parameters
        # and set epoch_n and i_iter
        g['saver'].restore(sess, ckpt.model_checkpoint_path)
        epoch_n = int(ckpt.model_checkpoint_path.split('/')[-1].split('-')[1])
        i_iter = (epoch_n + 1) * (p.num_examples // p.batch_size)
        print("Restored Epoch ", epoch_n)

    else:
        # no checkpoint exists.
        # create checkpoints directory if it does not exist.
        if not os.path.exists(ckpt_dir):
            os.makedirs(ckpt_dir)

        init = tf.global_variables_initializer()
        sess.run(init)

    if p.tb is not False:
        train_writer = tf.summary.FileWriter(p.tb_dir + '/train', sess.graph)
        test_writer = tf.summary.FileWriter(p.tb_dir + '/test', sess.graph)

    # -----------------------------
    print("=== Training ===")
    # -----------------------------

    def eval_metrics(dataset, sess, ops):
        return evaluate_metric_list(dataset, sess, ops, graph=g, params=p)

    def eval_metric(dataset, sess, op):
        return evaluate_metric(dataset, sess, op, graph=g, params=p)

    # Evaluate initial training accuracy and losses
    # init_loss = evaluate_metric(
    # mnist.train.labeled_ds, sess, cost)
    with open(desc_file, 'a') as f:
        print('================================', file=f, flush=True)
        print("Initial Train Accuracy: ",
              eval_metric(mnist.train.labeled_ds, sess, m['acc']),
              "%", file=f, flush=True)

        # -----------------------------
        # Evaluate initial testing accuracy and cross-entropy loss
        print("Initial Test Accuracy: ",
              eval_metric(mnist.test, sess, m['acc']),
              "%", file=f, flush=True)
        # print("Initial Test Losses: ",
        #       *eval_metrics(
        #           mnist.test, sess, test_losses), file=f,
        #       flush=True)

    start = time.time()
    for i in tqdm(range(i_iter, p.num_iter)):

        images, labels = mnist.train.next_batch(p.batch_size, p.ul_batch_size)

        _ = sess.run(
            [g['train_step']],
            feed_dict={g['images']: images,
                       g['labels']: labels,
                       g['train_flag']: True})

        epoch_n = i // (p.num_examples // p.batch_size)
        # ---------------------------------------------
        # Epoch completed?
        if (i > 1) and ((i + 1) % p.iter_per_epoch == 0):
            update_decays(sess, epoch_n, iter=i, graph=g, params=p)

        # ---------------------------------------------
        # Evaluate every test_frequency_in_epochs
        if (i > 1) and ((i + 1) % int(p.test_frequency_in_epochs *
                           p.iter_per_epoch) == 0):

            now = time.time() - start

            if not p.do_not_save:
                g['saver'].save(sess, ckpt_dir + 'model.ckpt', epoch_n)

            # ---------------------------------------------
            # Compute error on testing set (10k examples)
            test_costs = eval_metrics(mnist.test, sess,
                                      test_losses)
            train_costs = sess.run(
                train_losses,
                feed_dict={g['images']: images,
                           g['labels']: labels,
                           g['train_flag']: False})

            # Create log of:
            # time, epoch number, test accuracy, test cross entropy,
            # train accuracy, train loss, train cross entropy,
            # train reconstruction loss, smoothness

            log_i = [now, epoch_n] + sess.run(
                [m['acc']],
                feed_dict={g['images']: mnist.test.images,
                           g['labels']: mnist.test.labels,
                           g['train_flag']: False}
            ) + test_costs + sess.run(
                [m['acc']],
                feed_dict={g['images']:
                               mnist.train.labeled_ds.images,
                           g['labels']: mnist.train.labeled_ds.labels,
                           g['train_flag']: False}
            ) + train_costs

            # if np.isnan(log_i[-1]):

            with open(log_file, 'a') as train_log:
                print(*log_i, sep=',', flush=True, file=train_log)

    with open(desc_file, 'a') as f:
        print("Final Accuracy: ", sess.run(m['acc'], feed_dict={
            g['images']: mnist.test.images, g['labels']:
                mnist.test.labels,
            g['train_flag']: False}),
              "%", file=f, flush=True)

    sess.close()

if __name__ == '__main__':
    main()

<commit_msg>ConvLadder: make accuracy evaluations use minibatches to prevent oom<commit_after>
import tensorflow as tf
import os
import time
from tqdm import tqdm
from src.utils import get_cli_params, process_cli_params, \
    order_param_settings
from src.val import build_graph, measure_smoothness
from src.train import evaluate_metric_list, update_decays, evaluate_metric
from src import input_data
import numpy as np

def main():

    p = process_cli_params(get_cli_params())

    # -----------------------------
    # Set GPU device to use
    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
    os.environ["CUDA_VISIBLE_DEVICES"] = str(p.which_gpu)

    # Set seeds
    np.random.seed(p.seed)
    tf.set_random_seed(p.seed)

    # Load data
    print("===  Loading Data ===")
    mnist = input_data.read_data_sets("MNIST_data",
                                      n_labeled=p.num_labeled,
                                      validation_size=p.validation,
                                      one_hot=True,
                                      disjoint=False)
    num_examples = mnist.train.num_examples
    p.num_examples = num_examples
    if p.validation > 0:
        mnist.test = mnist.validation
    p.iter_per_epoch = (num_examples // p.batch_size)
    p.num_iter = p.iter_per_epoch * p.end_epoch

    # -----------------------------
    # Build graph
    g, m, trainable_parameters = build_graph(p)

    # Collect losses
    train_losses = [m['loss'], m['cost'], m['uc'], m['vc']]
    test_losses = [m['cost']]

    if p.measure_smoothness:
        s = measure_smoothness(g, p)
    #     print(s.get_shape())
        train_losses.append(tf.reduce_mean(s))

    if p.tb is not False:
        train_merged = tf.summary.merge([
            tf.summary.scalar(x) for x in train_losses
        ] + [tf.summary.scalar(m['acc'])])
        test_merged = tf.summary.merge([
            tf.summary.scalar(x) for x in test_losses
        ] + [tf.summary.scalar(m['acc'])])

        # Set up tensorboard logging
        if not os.path.exists(p.tb):
            os.makedirs(p.tb_dir)

    # Write logs to appropriate directory
    log_dir = p.logdir + p.id
    if not os.path.exists(log_dir):
        os.makedirs(log_dir)

    desc_file = log_dir + "/" + "description"
    with open(desc_file, 'a') as f:
        print(*order_param_settings(p), sep='\n', file=f, flush=True)
        print("Trainable parameters:", trainable_parameters, file=f,
              flush=True)

    log_file = log_dir + "/" + "train_log"

    # -----------------------------
    print("===  Starting Session ===")
    sess = tf.Session()
    i_iter = 0
    # -----------------------------
    # Resume from checkpoint
    ckpt_dir = "checkpoints/" + p.id + "/"
    ckpt = tf.train.get_checkpoint_state(
        ckpt_dir)  # get latest checkpoint (if any)

    if ckpt and ckpt.model_checkpoint_path:
        # if checkpoint exists,
        # restore the parameters
        # and set epoch_n and i_iter
        g['saver'].restore(sess, ckpt.model_checkpoint_path)
        epoch_n = int(ckpt.model_checkpoint_path.split('/')[-1].split('-')[1])
        i_iter = (epoch_n + 1) * (p.num_examples // p.batch_size)
        print("Restored Epoch ", epoch_n)

    else:
        # no checkpoint exists.
        # create checkpoints directory if it does not exist.
        if not os.path.exists(ckpt_dir):
            os.makedirs(ckpt_dir)

        init = tf.global_variables_initializer()
        sess.run(init)

    if p.tb is not False:
        train_writer = tf.summary.FileWriter(p.tb_dir + '/train', sess.graph)
        test_writer = tf.summary.FileWriter(p.tb_dir + '/test', sess.graph)

    # -----------------------------
    print("=== Training ===")
    # -----------------------------

    def eval_metrics(dataset, sess, ops):
        return evaluate_metric_list(dataset, sess, ops, graph=g, params=p)

    def eval_metric(dataset, sess, op):
        return evaluate_metric(dataset, sess, op, graph=g, params=p)

    # Evaluate initial training accuracy and losses
    # init_loss = evaluate_metric(
    # mnist.train.labeled_ds, sess, cost)
    with open(desc_file, 'a') as f:
        print('================================', file=f, flush=True)
        print("Initial Train Accuracy: ",
              eval_metric(mnist.train.labeled_ds, sess, m['acc']),
              "%", file=f, flush=True)

        # -----------------------------
        # Evaluate initial testing accuracy and cross-entropy loss
        print("Initial Test Accuracy: ",
              eval_metric(mnist.test, sess, m['acc']),
              "%", file=f, flush=True)
        # print("Initial Test Losses: ",
        #       *eval_metrics(
        #           mnist.test, sess, test_losses), file=f,
        #       flush=True)

    start = time.time()
    for i in tqdm(range(i_iter, p.num_iter)):

        images, labels = mnist.train.next_batch(p.batch_size, p.ul_batch_size)

        _ = sess.run(
            [g['train_step']],
            feed_dict={g['images']: images,
                       g['labels']: labels,
                       g['train_flag']: True})

        epoch_n = i // (p.num_examples // p.batch_size)
        # ---------------------------------------------
        # Epoch completed?
        if (i > 1) and ((i + 1) % p.iter_per_epoch == 0):
            update_decays(sess, epoch_n, iter=i, graph=g, params=p)

        # ---------------------------------------------
        # Evaluate every test_frequency_in_epochs
        if (i > 1) and ((i + 1) % int(p.test_frequency_in_epochs *
                           p.iter_per_epoch) == 0):

            now = time.time() - start

            if not p.do_not_save:
                g['saver'].save(sess, ckpt_dir + 'model.ckpt', epoch_n)

            # ---------------------------------------------
            # Compute error on testing set (10k examples)
            test_acc_and_costs = eval_metrics(mnist.test, sess,
                                      [m['acc']] + test_losses)
            train_acc = eval_metric(mnist.train.labeled_ds, sess, [m['acc']])
            train_costs = sess.run(
                train_losses,
                feed_dict={g['images']: images,
                           g['labels']: labels,
                           g['train_flag']: False})

            # Create log of:
            # time, epoch number, test accuracy, test cross entropy,
            # train accuracy, train loss, train cross entropy,
            # train reconstruction loss, smoothness

            log_i = [now, epoch_n] + test_acc_and_costs + train_acc + \
                    train_costs

            # if np.isnan(log_i[-1]):

            with open(log_file, 'a') as train_log:
                print(*log_i, sep=',', flush=True, file=train_log)

    with open(desc_file, 'a') as f:
        print("Final Accuracy: ", sess.run(m['acc'], feed_dict={
            g['images']: mnist.test.images, g['labels']:
                mnist.test.labels,
            g['train_flag']: False}),
              "%", file=f, flush=True)

    sess.close()

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>from __future__ import print_function, division, absolute_import

import functools
import sys

from future import standard_library
from sqlalchemy.exc import OperationalError, InterfaceError, InternalError, ProgrammingError, ArgumentError

standard_library.install_aliases()
import json
import os
from typing import List
from contextlib import contextmanager

from humanize import naturaltime
from markupsafe import Markup
from sqlalchemy import Sequence, UniqueConstraint, create_engine, Column, Integer, Text, ForeignKey, DateTime, String, \
    Index
from sqlalchemy.ext.declarative import declarative_base, declared_attr
from sqlalchemy.orm import backref, relationship, sessionmaker
from sqlalchemy.dialects.mysql import LONGTEXT
from littleutils import select_attrs, retry
from birdseye.utils import IPYTHON_FILE_PATH, is_ipython_cell
from sqlalchemy.dialects.mysql.base import RESERVED_WORDS

RESERVED_WORDS.add('function')

DB_VERSION = 1

class Database(object):
    def __init__(self, db_uri=None, _skip_version_check=False):
        self.db_uri = db_uri = (
                db_uri
                or os.environ.get('BIRDSEYE_DB')
                or os.path.join(os.path.expanduser('~'),
                                '.birdseye.db'))

        kwargs = dict(
            pool_recycle=280,
            echo=False,  # for convenience when debugging
        )

        try:
            engine = create_engine(db_uri, **kwargs)
        except ArgumentError:
            db_uri = 'sqlite:///' + db_uri
            engine = create_engine(db_uri, **kwargs)

        self.engine = engine

        self.Session = sessionmaker(bind=engine)

        class Base(object):
            @declared_attr
            def __tablename__(cls):
                return cls.__name__.lower()

        Base = declarative_base(cls=Base)  # type: ignore

        class KeyValue(Base):
            key = Column(String(50), primary_key=True)
            value = Column(Text)

        db_self = self

        class KeyValueStore(object):
            def __getitem__(self, item):
                with db_self.session_scope() as session:
                    return (session
                            .query(KeyValue.value)
                            .filter_by(key=item)
                            .scalar())

            def __setitem__(self, key, value):
                with db_self.session_scope() as session:
                    session.query(KeyValue).filter_by(key=key).delete()
                    session.add(KeyValue(key=key, value=str(value)))

            __getattr__ = __getitem__
            __setattr__ = __setitem__

        LongText = LONGTEXT if engine.name == 'mysql' else Text

        class Call(Base):
            id = Column(String(length=32), primary_key=True)
            function_id = Column(Integer, ForeignKey('function.id'), index=True)
            function = relationship('Function', backref=backref('calls', lazy='dynamic'))
            arguments = Column(Text)
            return_value = Column(Text)
            exception = Column(Text)
            traceback = Column(Text)
            data = Column(LongText)
            start_time = Column(DateTime, index=True)

            @property
            def pretty_start_time(self):
                return self._pretty_time(self.start_time)

            @staticmethod
            def _pretty_time(dt):
                if not dt:
                    return ''
                return Markup('%s (%s)' % (
                    dt.strftime('%Y-%m-%d %H:%M:%S'),
                    naturaltime(dt)))

            @property
            def state_icon(self):
                return Markup('' % (
                                  ('ok', 'green') if self.success else
                                  ('remove', 'red')))

            @property
            def success(self):
                if self.exception:
                    assert self.traceback
                    assert self.return_value == 'None'
                    return False
                else:
                    assert not self.traceback
                    return True

            @property
            def result(self):
                if self.success:
                    return str(self.return_value)
                else:
                    return str(self.exception)

            @property
            def arguments_list(self):
                return json.loads(self.arguments)

            @property
            def parsed_data(self):
                return json.loads(self.data)

            @staticmethod
            def basic_dict(call):
                return dict(arguments=call.arguments_list,
                            **select_attrs(call, 'id function_id return_value traceback '
                                                 'exception start_time'))

            basic_columns = (id, function_id, return_value,
                             traceback, exception, start_time, arguments)

        class Function(Base):
            id = Column(Integer, Sequence('function_id_seq'), primary_key=True)
            file = Column(Text)
            name = Column(Text)
            type = Column(Text)  # function or module
            html_body = Column(LongText)
            lineno = Column(Integer)
            data = Column(LongText)
            hash = Column(String(length=64), index=True)
            body_hash = Column(String(length=64), index=True)

            __table_args__ = (
                UniqueConstraint('hash',
                                 name='everything_unique'),
                Index('idx_file', 'file', mysql_length=256),
                Index('idx_name', 'name', mysql_length=32),
            )

            @property
            def parsed_data(self):
                return json.loads(self.data)

            @staticmethod
            def basic_dict(func):
                return select_attrs(func, 'file name lineno hash body_hash type')

            basic_columns = (file, name, lineno, hash, body_hash, type)

        self.Call = Call
        self.Function = Function
        self._KeyValue = KeyValue

        self.key_value_store = kv = KeyValueStore()

        if _skip_version_check:
            return

        if not self.table_exists(Function):
            Base.metadata.create_all(engine)
            kv.version = DB_VERSION
        elif not self.table_exists(KeyValue) or int(kv.version) < DB_VERSION:
            sys.exit('The birdseye database schema is out of date. '
                     'Run "python -m birdseye.clear_db" to delete the existing tables.')

    def table_exists(self, table):
        return self.engine.dialect.has_table(self.engine, table.__name__)

    def all_file_paths(self):
        # type: () -> List[str]
        with self.session_scope() as session:
            paths = [f[0] for f in session.query(self.Function.file).distinct()
                     if not is_ipython_cell(f[0])]
        paths.sort()
        if IPYTHON_FILE_PATH in paths:
            paths.remove(IPYTHON_FILE_PATH)
            paths.insert(0, IPYTHON_FILE_PATH)
        return paths

    def clear(self):
        for model in [self.Call, self.Function, self._KeyValue]:
            if self.table_exists(model):
                model.__table__.drop(self.engine)

    @contextmanager
    def session_scope(self):
        """Provide a transactional scope around a series of operations."""
        session = self.Session()
        try:
            yield session
            session.commit()
        except:
            session.rollback()
            raise
        finally:
            session.close()

    def provide_session(self, func):
        @functools.wraps(func)
        def wrapper(*args, **kwargs):
            with self.session_scope() as session:
                return func(session, *args, **kwargs)

        return retry_db(wrapper)

# Based on https://docs.sqlalchemy.org/en/latest/errors.html#error-dbapi
retry_db = retry(3, (InterfaceError, OperationalError, InternalError, ProgrammingError))
<commit_msg>Fix after sqlalchemy 1.4<commit_after>from __future__ import print_function, division, absolute_import

import functools
import sys

from future import standard_library
from sqlalchemy.exc import OperationalError, InterfaceError, InternalError, ProgrammingError, ArgumentError

standard_library.install_aliases()
import json
import os
from typing import List
from contextlib import contextmanager

from humanize import naturaltime
from markupsafe import Markup
from sqlalchemy import Sequence, UniqueConstraint, create_engine, Column, Integer, Text, ForeignKey, DateTime, String, \
    Index
from sqlalchemy.ext.declarative import declarative_base, declared_attr
from sqlalchemy.orm import backref, relationship, sessionmaker
from sqlalchemy.dialects.mysql import LONGTEXT
from littleutils import select_attrs, retry
from birdseye.utils import IPYTHON_FILE_PATH, is_ipython_cell
from sqlalchemy.dialects.mysql.base import RESERVED_WORDS

RESERVED_WORDS.add('function')

DB_VERSION = 1

class Database(object):
    def __init__(self, db_uri=None, _skip_version_check=False):
        self.db_uri = db_uri = (
                db_uri
                or os.environ.get('BIRDSEYE_DB')
                or os.path.join(os.path.expanduser('~'),
                                '.birdseye.db'))

        kwargs = dict(
            pool_recycle=280,
            echo=False,  # for convenience when debugging
        )

        try:
            engine = create_engine(db_uri, **kwargs)
        except ArgumentError:
            db_uri = 'sqlite:///' + db_uri
            engine = create_engine(db_uri, **kwargs)

        self.engine = engine

        self.Session = sessionmaker(bind=engine)

        class Base(object):
            @declared_attr
            def __tablename__(cls):
                return cls.__name__.lower()

        Base = declarative_base(cls=Base)  # type: ignore

        class KeyValue(Base):
            key = Column(String(50), primary_key=True)
            value = Column(Text)

        db_self = self

        class KeyValueStore(object):
            def __getitem__(self, item):
                with db_self.session_scope() as session:
                    return (session
                            .query(KeyValue.value)
                            .filter_by(key=item)
                            .scalar())

            def __setitem__(self, key, value):
                with db_self.session_scope() as session:
                    session.query(KeyValue).filter_by(key=key).delete()
                    session.add(KeyValue(key=key, value=str(value)))

            __getattr__ = __getitem__
            __setattr__ = __setitem__

        LongText = LONGTEXT if engine.name == 'mysql' else Text

        class Call(Base):
            id = Column(String(length=32), primary_key=True)
            function_id = Column(Integer, ForeignKey('function.id'), index=True)
            function = relationship('Function', backref=backref('calls', lazy='dynamic'))
            arguments = Column(Text)
            return_value = Column(Text)
            exception = Column(Text)
            traceback = Column(Text)
            data = Column(LongText)
            start_time = Column(DateTime, index=True)

            @property
            def pretty_start_time(self):
                return self._pretty_time(self.start_time)

            @staticmethod
            def _pretty_time(dt):
                if not dt:
                    return ''
                return Markup('%s (%s)' % (
                    dt.strftime('%Y-%m-%d %H:%M:%S'),
                    naturaltime(dt)))

            @property
            def state_icon(self):
                return Markup('' % (
                                  ('ok', 'green') if self.success else
                                  ('remove', 'red')))

            @property
            def success(self):
                if self.exception:
                    assert self.traceback
                    assert self.return_value == 'None'
                    return False
                else:
                    assert not self.traceback
                    return True

            @property
            def result(self):
                if self.success:
                    return str(self.return_value)
                else:
                    return str(self.exception)

            @property
            def arguments_list(self):
                return json.loads(self.arguments)

            @property
            def parsed_data(self):
                return json.loads(self.data)

            @staticmethod
            def basic_dict(call):
                return dict(arguments=call.arguments_list,
                            **select_attrs(call, 'id function_id return_value traceback '
                                                 'exception start_time'))

            basic_columns = (id, function_id, return_value,
                             traceback, exception, start_time, arguments)

        class Function(Base):
            id = Column(Integer, Sequence('function_id_seq'), primary_key=True)
            file = Column(Text)
            name = Column(Text)
            type = Column(Text)  # function or module
            html_body = Column(LongText)
            lineno = Column(Integer)
            data = Column(LongText)
            hash = Column(String(length=64), index=True)
            body_hash = Column(String(length=64), index=True)

            __table_args__ = (
                UniqueConstraint('hash',
                                 name='everything_unique'),
                Index('idx_file', 'file', mysql_length=256),
                Index('idx_name', 'name', mysql_length=32),
            )

            @property
            def parsed_data(self):
                return json.loads(self.data)

            @staticmethod
            def basic_dict(func):
                return select_attrs(func, 'file name lineno hash body_hash type')

            basic_columns = (file, name, lineno, hash, body_hash, type)

        self.Call = Call
        self.Function = Function
        self._KeyValue = KeyValue

        self.key_value_store = kv = KeyValueStore()

        if _skip_version_check:
            return

        if not self.table_exists(Function):
            Base.metadata.create_all(engine)
            kv.version = DB_VERSION
        elif not self.table_exists(KeyValue) or int(kv.version) < DB_VERSION:
            sys.exit('The birdseye database schema is out of date. '
                     'Run "python -m birdseye.clear_db" to delete the existing tables.')

    def table_exists(self, table):
        try:
            from sqlalchemy import inspect

            return inspect(self.engine).has_table(table.__name__)
        except (ImportError, AttributeError):
            try:
                return self.engine.has_table(table.__name__)
            except AttributeError:
                return self.engine.dialect.has_table(self.engine, table.__name__)

    def all_file_paths(self):
        # type: () -> List[str]
        with self.session_scope() as session:
            paths = [f[0] for f in session.query(self.Function.file).distinct()
                     if not is_ipython_cell(f[0])]
        paths.sort()
        if IPYTHON_FILE_PATH in paths:
            paths.remove(IPYTHON_FILE_PATH)
            paths.insert(0, IPYTHON_FILE_PATH)
        return paths

    def clear(self):
        for model in [self.Call, self.Function, self._KeyValue]:
            if self.table_exists(model):
                model.__table__.drop(self.engine)

    @contextmanager
    def session_scope(self):
        """Provide a transactional scope around a series of operations."""
        session = self.Session()
        try:
            yield session
            session.commit()
        except:
            session.rollback()
            raise
        finally:
            session.close()

    def provide_session(self, func):
        @functools.wraps(func)
        def wrapper(*args, **kwargs):
            with self.session_scope() as session:
                return func(session, *args, **kwargs)

        return retry_db(wrapper)

# Based on https://docs.sqlalchemy.org/en/latest/errors.html#error-dbapi
retry_db = retry(3, (InterfaceError, OperationalError, InternalError, ProgrammingError))
<|endoftext|>


<commit_before>from random import shuffle
from card import Card
from pubsub import pub

class Board():
    def __init__(self):
        # Values and symbols of the card which will be created
        self.values      = ['1','2','3','4','5','6','7','8','9','10','J','Q','K']
        self.symbols     = ['H', 'S', 'C', 'D']

        # Generate the stock
        self.stock = []
        for symbol in self.symbols:
            for value in self.values:
                self.stock.append(Card(symbol, value))
        shuffle(self.stock)

        # Generate the waste
        self.waste = []

        # generate empty piles for each color
        self.H = []
        self.C = []
        self.S = []
        self.D = []

        # generate empty waste
        self.waste = []

        # generate tableau piles used to play
        self.PlayingStacks = [[], [], [], [], [], [], []]
        for stack in range (0, 7):
            for index in range (-1, stack):
                card = self.stock.pop()
                self.PlayingStacks[stack].append(card)

            self.PlayingStacks[stack][-1].setFaceDown(False)

        # Update GUI
        pub.sendMessage('refreshGUITopic')

    def pickCardFromStock(self):
        # If stock gets empty, recycle the waste
        if (len(self.stock) == 0):
            while (len(self.waste) > 0):
                self.stock.append(self.waste.pop())
                self.stock[-1].setFaceDown(True)
            self.stock[-1].setFaceDown(False)

        card = self.stock.pop()
        self.waste.append(card)
        if (len(self.waste) > 0):
            self.waste[-1].setFaceDown(False)

        # Update GUI
        pub.sendMessage('refreshGUITopic')

    def moveCardFromFoundation(self, choosenFoundation, destinationCard):
        choosenDestination = self.getCardPosition(destinationCard)

        # Get the foundation to treat
        if (choosenFoundation == "H"):
            foundation = self.H
        elif (choosenFoundation == "S"):
            foundation = self.S
        elif (choosenFoundation == "D"):
            foundation = self.D
        elif (choosenFoundation == "C"):
            foundation = self.C
        else:
            return -1

        # check that there is a card to move
        if (len(foundation) < 1):
            # print ("No card to move")
            return -1
        else:
            card = foundation[-1]
            cardValue = self.values.index(card.value)

        # check if we can move the card
        if ( choosenDestination >= 0 and choosenDestination <= 6):
            destination = self.PlayingStacks[choosenDestination]
            if (len(destination) == 0):
                if (card.value != "K"):
                    # print ("You can only move a king here")
                    return -1
            elif (card.color == destination[-1].color or
                  cardValue != self.values.index(destination[-1].value)-1):
                # print("Wrong color or wrong value")
                return -1
        else:
            # print("Incorrect choice of pile")
            return -1

        destination.append(foundation.pop())

        # Update GUI
        pub.sendMessage('refreshGUITopic')
        return 0

    def moveCardFromWaste(self, choice):
        cardValue = self.values.index(self.waste[-1].value)
        # If we try to put the card on the tableau check the color isnt the same
        # and the values are following
        if (choice.isdigit() and int(choice) >= 1 and int(choice) <= 7):
            destination = self.PlayingStacks[int(choice) - 1]
            if (len(destination) == 0):
                if (self.waste[-1].value != "K"):
                    # print ("You can only move a king here")
                    return -1
            elif (self.waste[-1].color == destination[-1].color or
                  cardValue != self.values.index(destination[-1].value)-1):
                # print("Wrong color or wrong value")
                return -1
        # If we try to put the card on the foundations
        else:
            if (choice == "H"):
                destination = self.H
            elif (choice == "S"):
                destination = self.S
            elif (choice == "D"):
                destination = self.D
            elif (choice == "C"):
                destination = self.C
            else:
                return -1

            if (cardValue != len(destination) or
                self.waste[-1].symbol != choice):
                # print("Wrong color or wrong value")
                return -1

        # If all conditions are ok move the card
        destination.append(self.waste.pop())

        # Update GUI
        pub.sendMessage('refreshGUITopic')

    def moveCardFromTableau(self, card, choice):
        card.setFaceDown(False)
        cardValue = self.values.index(card.value)

        # Get the list of cards to move (the list can contain only one card)
        pileIndex = -1
        for s in self.PlayingStacks:
            if (card in s):
                pileIndex = self.PlayingStacks.index(s)
                cardIndex = s.index(card)
                nbOfCards = len(s) - cardIndex
                break
        # Fail if we select a card which isnt in the tableau
        if (pileIndex == -1):
            # print ("This card cant be moved")
            pub.sendMessage('refreshGUITopic')
            return -1

        # Try to move to the tableau
        if (choice.isdigit() and int(choice) >= 1 and int(choice) <= 7):
            destination = self.PlayingStacks[int(choice) - 1]
            # Move only king if the pile is empty
            if (len(destination) == 0):
                if (card.value != "K"):
                    # print("You can only move a king here")
                    pub.sendMessage('refreshGUITopic')
                    return -1
            # is the pile isnt empty check the values and colors
            elif (card.color == destination[-1].color or
                  cardValue != self.values.index(destination[-1].value)-1):
                # print("Wrong color or wrong value")
                pub.sendMessage('refreshGUITopic')
                return -1
        # Try to move in to the foundations
        else:
            if (choice == "H"):
                destination = self.H
            elif (choice == "S"):
                destination = self.S
            elif (choice == "D"):
                destination = self.D
            elif (choice == "C"):
                destination = self.C
            else:
                pub.sendMessage('refreshGUITopic')
                return -1

            # Fail if we try to put several cards at the time in a foundation
            if (nbOfCards != 1):
                # print("You can move more than one card here")
                pub.sendMessage('refreshGUITopic')
                return -1
            # Checks on the values and colors
            if (cardValue != len(destination) or
                card.symbol != choice):
                # print("Wrong color or wrong value")
                pub.sendMessage('refreshGUITopic')
                return -1

        # Actually move the cards
        for c in range(cardIndex, len(s)):
            destination.append(s.pop(cardIndex))

        # Reveal the card which was before the moved one(s)
        if (len(s) > 0):
            s[-1].setFaceDown(False)

        pub.sendMessage('refreshGUITopic')
        return 0

    # Help to determine where is a card of the board
    # Returns:
    # [0-6]: If the card is face up in a tableau pile
    # ["H", "S", "C", "D"]: If the card is the last one in a foundation
    # "W": If the card is the last one in the waste
    # 0: otherwise
    def getCardPosition(self, card):
        if ( card is not None ):
            if ( len(self.waste) and self.waste[-1] == card ):
                return "W"
            elif ( len(self.H)!=0 and self.H[-1] == card ):
                return "H"
            elif ( len(self.S)!=0 and self.S[-1] == card ):
                return "S"
            elif ( len(self.C)!=0 and self.C[-1] == card ):
                return "C"
            elif ( len(self.D)!=0 and self.D[-1] == card ):
                return "D"
            else:
                index = -1
                for pile in self.PlayingStacks:
                    index += 1
                    for c in pile:
                        if (c == card and not c.facedown):
                            return index
        return 0

    def __str__(self):
        str = "stock: "
        card = "  "
        if (len(self.stock) > 0):
            card = self.stock[-1].__str__()
        str = str + card + "\t[" + len(self.stock).__str__() + "]" + '\n'

        str = str + "waste: "
        card = "  "
        if (len(self.waste) > 0):
            card = self.waste[-1].__str__()
        str = str + card + "\t[" + len(self.waste).__str__() + "]" + '\n'

        str = str + "foundations: " + '\n'
        str = str + "H: "
        if (len(self.H) > 0): str = str + self.H[-1].__str__()

        str = str + '\n' + "C: ".__str__()
        if (len(self.C) > 0): str = str + self.C[-1].__str__()

        str = str + '\n' + "S: ".__str__()
        if (len(self.S) > 0): str = str + self.S[-1].__str__()

        str = str + '\n' + "D: ".__str__()
        if (len(self.D) > 0): str = str + self.D[-1].__str__()

        str =  str + '\n' +"stacks: "
        cpt = 0
        for s in self.PlayingStacks:
            cpt += 1
            str = str + "\n" + cpt.__str__() + ": "
            for card in s:
                str = str + card.__str__() + " "

        return str

<commit_msg>Improve getCardPosition to handle empty places<commit_after>from random import shuffle
from card import Card
from pubsub import pub

class Board():
    def __init__(self):
        # Values and symbols of the card which will be created
        self.values      = ['1','2','3','4','5','6','7','8','9','10','J','Q','K']
        self.symbols     = ['H', 'S', 'C', 'D']

        # Generate the stock
        self.stock = []
        for symbol in self.symbols:
            for value in self.values:
                self.stock.append(Card(symbol, value))
        shuffle(self.stock)

        # Generate the waste
        self.waste = []

        # generate empty piles for each color
        self.H = []
        self.C = []
        self.S = []
        self.D = []

        # generate empty waste
        self.waste = []

        # generate tableau piles used to play
        self.PlayingStacks = [[], [], [], [], [], [], []]
        for stack in range (0, 7):
            for index in range (-1, stack):
                card = self.stock.pop()
                self.PlayingStacks[stack].append(card)

            self.PlayingStacks[stack][-1].setFaceDown(False)

        # Update GUI
        pub.sendMessage('refreshGUITopic')

    def pickCardFromStock(self):
        # If stock gets empty, recycle the waste
        if (len(self.stock) == 0):
            while (len(self.waste) > 0):
                self.stock.append(self.waste.pop())
                self.stock[-1].setFaceDown(True)
            self.stock[-1].setFaceDown(False)

        card = self.stock.pop()
        self.waste.append(card)
        if (len(self.waste) > 0):
            self.waste[-1].setFaceDown(False)

        # Update GUI
        pub.sendMessage('refreshGUITopic')

    def moveCardFromFoundation(self, choosenFoundation, destinationCard):
        choosenDestination = self.getCardPosition(destinationCard)

        # Get the foundation to treat
        if (choosenFoundation == "H"):
            foundation = self.H
        elif (choosenFoundation == "S"):
            foundation = self.S
        elif (choosenFoundation == "D"):
            foundation = self.D
        elif (choosenFoundation == "C"):
            foundation = self.C
        else:
            return -1

        # check that there is a card to move
        if (len(foundation) < 1):
            # print ("No card to move")
            return -1
        else:
            card = foundation[-1]
            cardValue = self.values.index(card.value)

        # check if we can move the card
        if ( choosenDestination >= 0 and choosenDestination <= 6):
            destination = self.PlayingStacks[choosenDestination]
            if (len(destination) == 0):
                if (card.value != "K"):
                    # print ("You can only move a king here")
                    return -1
            elif (card.color == destination[-1].color or
                  cardValue != self.values.index(destination[-1].value)-1):
                # print("Wrong color or wrong value")
                return -1
        else:
            # print("Incorrect choice of pile")
            return -1

        destination.append(foundation.pop())

        # Update GUI
        pub.sendMessage('refreshGUITopic')
        return 0

    def moveCardFromWaste(self, choice):
        cardValue = self.values.index(self.waste[-1].value)
        # If we try to put the card on the tableau check the color isnt the same
        # and the values are following
        if (choice.isdigit() and int(choice) >= 1 and int(choice) <= 7):
            destination = self.PlayingStacks[int(choice) - 1]
            if (len(destination) == 0):
                if (self.waste[-1].value != "K"):
                    # print ("You can only move a king here")
                    return -1
            elif (self.waste[-1].color == destination[-1].color or
                  cardValue != self.values.index(destination[-1].value)-1):
                # print("Wrong color or wrong value")
                return -1
        # If we try to put the card on the foundations
        else:
            if (choice == "H"):
                destination = self.H
            elif (choice == "S"):
                destination = self.S
            elif (choice == "D"):
                destination = self.D
            elif (choice == "C"):
                destination = self.C
            else:
                return -1

            if (cardValue != len(destination) or
                self.waste[-1].symbol != choice):
                # print("Wrong color or wrong value")
                return -1

        # If all conditions are ok move the card
        destination.append(self.waste.pop())

        # Update GUI
        pub.sendMessage('refreshGUITopic')

    def moveCardFromTableau(self, card, choice):
        card.setFaceDown(False)
        cardValue = self.values.index(card.value)

        # Get the list of cards to move (the list can contain only one card)
        pileIndex = -1
        for s in self.PlayingStacks:
            if (card in s):
                pileIndex = self.PlayingStacks.index(s)
                cardIndex = s.index(card)
                nbOfCards = len(s) - cardIndex
                break
        # Fail if we select a card which isnt in the tableau
        if (pileIndex == -1):
            # print ("This card cant be moved")
            pub.sendMessage('refreshGUITopic')
            return -1

        # Try to move to the tableau
        if (choice.isdigit() and int(choice) >= 1 and int(choice) <= 7):
            destination = self.PlayingStacks[int(choice) - 1]
            # Move only king if the pile is empty
            if (len(destination) == 0):
                if (card.value != "K"):
                    # print("You can only move a king here")
                    pub.sendMessage('refreshGUITopic')
                    return -1
            # is the pile isnt empty check the values and colors
            elif (card.color == destination[-1].color or
                  cardValue != self.values.index(destination[-1].value)-1):
                # print("Wrong color or wrong value")
                pub.sendMessage('refreshGUITopic')
                return -1
        # Try to move in to the foundations
        else:
            if (choice == "H"):
                destination = self.H
            elif (choice == "S"):
                destination = self.S
            elif (choice == "D"):
                destination = self.D
            elif (choice == "C"):
                destination = self.C
            else:
                pub.sendMessage('refreshGUITopic')
                return -1

            # Fail if we try to put several cards at the time in a foundation
            if (nbOfCards != 1):
                # print("You can move more than one card here")
                pub.sendMessage('refreshGUITopic')
                return -1
            # Checks on the values and colors
            if (cardValue != len(destination) or
                card.symbol != choice):
                # print("Wrong color or wrong value")
                pub.sendMessage('refreshGUITopic')
                return -1

        # Actually move the cards
        for c in range(cardIndex, len(s)):
            destination.append(s.pop(cardIndex))

        # Reveal the card which was before the moved one(s)
        if (len(s) > 0):
            s[-1].setFaceDown(False)

        pub.sendMessage('refreshGUITopic')
        return 0

    # Help to determine where is a card of the board
    # To have a consistent behavior with empty emplacements the function
    # can also take a string as argument representing the empty place
    # Returns:
    # [0-6]: If the card is face up in a tableau pile
    # ["H", "S", "C", "D"]: If the card is the last one in a foundation
    # "W": If the card is the last one in the waste
    # -1: otherwise
    def getCardPosition(self, card):
        if ( card is not None ):
            if card in {"W", "H", "S", "C", "D", "0", "1", "2", "3", "4", "5", "6"}:
                return card
            if ( len(self.waste) and self.waste[-1] == card ):
                return "W"
            elif ( len(self.H)!=0 and self.H[-1] == card ):
                return "H"
            elif ( len(self.S)!=0 and self.S[-1] == card ):
                return "S"
            elif ( len(self.C)!=0 and self.C[-1] == card ):
                return "C"
            elif ( len(self.D)!=0 and self.D[-1] == card ):
                return "D"
            else:
                index = -1
                for pile in self.PlayingStacks:
                    index += 1
                    for c in pile:
                        if (c == card and not c.facedown):
                            return index
        return -1

    def __str__(self):
        str = "stock: "
        card = "  "
        if (len(self.stock) > 0):
            card = self.stock[-1].__str__()
        str = str + card + "\t[" + len(self.stock).__str__() + "]" + '\n'

        str = str + "waste: "
        card = "  "
        if (len(self.waste) > 0):
            card = self.waste[-1].__str__()
        str = str + card + "\t[" + len(self.waste).__str__() + "]" + '\n'

        str = str + "foundations: " + '\n'
        str = str + "H: "
        if (len(self.H) > 0): str = str + self.H[-1].__str__()

        str = str + '\n' + "C: ".__str__()
        if (len(self.C) > 0): str = str + self.C[-1].__str__()

        str = str + '\n' + "S: ".__str__()
        if (len(self.S) > 0): str = str + self.S[-1].__str__()

        str = str + '\n' + "D: ".__str__()
        if (len(self.D) > 0): str = str + self.D[-1].__str__()

        str =  str + '\n' +"stacks: "
        cpt = 0
        for s in self.PlayingStacks:
            cpt += 1
            str = str + "\n" + cpt.__str__() + ": "
            for card in s:
                str = str + card.__str__() + " "

        return str

<|endoftext|>


<commit_before>#    Copyright 2014 Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import os

from setuptools import find_packages
from setuptools import setup

def find_requires():
    prj_root = os.path.dirname(os.path.realpath(__file__))
    requirements = []
    with open(u'{0}/requirements.txt'.format(prj_root), 'r') as reqs:
        requirements = reqs.readlines()
    return requirements

setup(
    name='fuel-plugin-builder',
    version='0.1.0',
    description='Helps to create and build fuel plugins',
    long_description="""Helps to create and build fuel plugins""",
    classifiers=[
        "Programming Language :: Python",
        "Topic :: System :: Software Distribution",
        "License :: OSI Approved :: Apache Software License"],
    author='Mirantis Inc.',
    author_email='product@mirantis.com',
    url='http://mirantis.com',
    keywords='fuel plugins plugin',
    packages=find_packages(),
    zip_safe=False,
    install_requires=find_requires(),
    include_package_data=True,
    entry_points={
        'console_scripts': [
            'fpb = fuel_plugin_builder.cli:main',
            'fuel-plugin-builder = fuel_plugin_builder.cli:main']})
<commit_msg>Add more classifiers for fpb<commit_after>#    Copyright 2014 Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import os

from setuptools import find_packages
from setuptools import setup

def find_requires():
    prj_root = os.path.dirname(os.path.realpath(__file__))
    requirements = []
    with open(u'{0}/requirements.txt'.format(prj_root), 'r') as reqs:
        requirements = reqs.readlines()
    return requirements

setup(
    name='fuel-plugin-builder',
    version='0.1.0',
    description='Helps to create and build fuel plugins',
    long_description="""Helps to create and build fuel plugins""",
    classifiers=[
        "Development Status :: 4 - Beta",
        "License :: OSI Approved :: Apache Software License",
        "Programming Language :: Python",
        "Programming Language :: Python :: 2.6",
        "Programming Language :: Python :: 2.7",
        "Topic :: System :: Software Distribution"],
    author='Mirantis Inc.',
    author_email='product@mirantis.com',
    url='http://mirantis.com',
    keywords='fuel plugins plugin',
    packages=find_packages(),
    zip_safe=False,
    install_requires=find_requires(),
    include_package_data=True,
    entry_points={
        'console_scripts': [
            'fpb = fuel_plugin_builder.cli:main',
            'fuel-plugin-builder = fuel_plugin_builder.cli:main']})
<|endoftext|>


<commit_before>import datetime
import logging
import time
from kombu import Exchange, Queue
from kombu.mixins import ConsumerMixin
import os
import pystache
import re
from taskcluster import slugId, stringDate, fromNow
import yaml
import json

from funsize.utils import properties_to_dict, encrypt_env_var

log = logging.getLogger(__name__)

# TODO: move these to config
BRANCHES = ['mozilla-central', 'mozilla-aurora']
PLATFORMS = ['linux', 'linux64', 'win32', 'win64', 'macosx64']
BUILDERS = [
    r'WINNT \d+\.\d+ (x86-64 )?{branch} nightly',
    r'Linux (x86-64 )?{branch} nightly',
    r'OS X \d+\.\d+ {branch} nightly',
    r'Firefox {branch} (linux|linux64|win32|win64|macosx64) l10n nightly-\d+',
]

class FunsizeWorker(ConsumerMixin):

    def __init__(self, connection, queue_name, exchange, balrog_client,
                 scheduler):
        """Funsize consumer worker
        :type connection: kombu.Connection
        :param queue_name: Full queue name, including queue/<user> prefix
        :type exchange: kombu.Exchange
        :type balrog_client: funsize.balrog.BalrogClient
        :type scheduler: tascluster.Scheduler
        """
        self.connection = connection
        # Using passive mode is important, otherwise pulse returns 403
        self.exchange = Exchange(exchange, type='topic', passive=True)
        self.queue_name = queue_name
        self.balrog_client = balrog_client
        self.scheduler = scheduler

    @property
    def routing_keys(self):
        """Returns an explicit list of routing patterns.

        Instead of using "build.*.*.finished", which generates a lot of noise,
        FunsizeWorker uses explicit list of routing keys to match builders we
        are interested in.
        """
        jobs = [
            # TODO: move to configs
            'build.{branch}-{platform}-nightly.*.finished',
            # TODO: find a better way to specify these
            'build.{branch}-{platform}-l10n-nightly-1.*.finished',
            'build.{branch}-{platform}-l10n-nightly-2.*.finished',
            'build.{branch}-{platform}-l10n-nightly-3.*.finished',
            'build.{branch}-{platform}-l10n-nightly-4.*.finished',
            'build.{branch}-{platform}-l10n-nightly-5.*.finished',
            'build.{branch}-{platform}-l10n-nightly-6.*.finished',
            'build.{branch}-{platform}-l10n-nightly-7.*.finished',
            'build.{branch}-{platform}-l10n-nightly-8.*.finished',
            'build.{branch}-{platform}-l10n-nightly-9.*.finished',
            'build.{branch}-{platform}-l10n-nightly-10.*.finished',
        ]
        return [job.format(branch=branch, platform=platform)
                for job in jobs
                for branch in BRANCHES
                for platform in PLATFORMS]

    @property
    def queues(self):
        """List of queues used by worker.
        Multiple queues are used to track multiple routing keys.
        """
        return [Queue(name=self.queue_name, exchange=self.exchange,
                      routing_key=routing_key, durable=True, exclusive=False,
                      auto_delete=False)
                for routing_key in self.routing_keys]

    def get_consumers(self, Consumer, channel):
        """Implement parent's method called to get the list of consumers"""
        return [Consumer(queues=self.queues, callbacks=[self.process_message])]

    def process_message(self, body, message):
        """Top level callback processing pulse messages.
        The callback tries to handle and log all exceptions

        :type body: kombu.Message.body
        :type message: kombu.Message
        """
        try:
            self.dispatch_message(body)
        except Exception:
            log.exception("Failed to process message")
        finally:
            # TODO: figure out what to do with failed tasks
            message.ack()

    def on_consume_ready(self, connection, channel, consumers, **kwargs):
        """Overrides parent's stub method. Called when ready to consume pulse
         messages.
        """
        log.info('Listening...')

    def dispatch_message(self, body):
        """Dispatches incoming pulse messages.
        If the method detects L10N repacks, it creates multiple Taskcluster
        tasks, otherwise a single en-US task is created.
        :type body: kombu.Message.body
        """
        payload = body["payload"]
        buildername = payload["build"]["builderName"]
        if not interesting_buildername(buildername):
            log.debug("Ignoring %s: not interested", buildername)
            return
        job_result = payload["results"]
        if job_result != 0:
            log.debug("Ignoring %s with result %s", buildername, job_result)
            return
        properties = properties_to_dict(payload["build"]["properties"])
        if "locales" in properties:
            log.debug("L10N repack detected")
            funsize_info = json.loads(properties["funsize_info"])
            locales = json.loads(properties["locales"])
            platform = funsize_info["platform"]
            branch = funsize_info["branch"]
            product = funsize_info["appName"]
            for locale, result in locales.iteritems():
                if result.lower() == "success":
                    self.create_partial(product, branch, platform,
                                        locale)
                else:
                    log.warn("Ignoring %s with result %s", locale, result)
        else:
            log.debug("en-US build detected")
            self.create_partial(properties["appName"],
                                       properties["branch"],
                                       properties["platform"], 'en-US')

    def create_partial(self, product, branch, platform, locale):
        """Calculates "from" and "to" MAR URLs and calls  create_task_graph().
        Currently "from" MAR is 2 releases behind to avoid duplication of
        existing CI partials.

        :param product: capitalized product name, AKA appName, e.g. Firefox
        :param branch: branch name (mozilla-central)
        :param platform: buildbot platform (linux, macosx64)
        :param locale: en-US or locale
        """
        # TODO: move limit to config
        # Get last 3 releases, generate partial from -2 to latest
        last_releases = self.balrog_client.get_releases(product, branch)[:3]
        release_to = last_releases[0]
        release_from = last_releases[-1]
        log.debug("From: %s", release_from)
        log.debug("To: %s", release_to)
        build_from = self.balrog_client.get_build(release_from,
                                                  platform, locale)
        log.debug("Build from: %s", build_from)
        build_to = self.balrog_client.get_build(release_to, platform, locale)
        log.debug("Build to: %s", build_to)
        mar_from = build_from["completes"][0]["fileUrl"]
        mar_to = build_to["completes"][0]["fileUrl"]
        log.info("New Funsize task for %s %s, from %s to %s", platform, locale,
                 mar_from, mar_to)
        self.submit_task_graph(platform, locale, mar_from, mar_to)

    def submit_task_graph(self, platform, locale, from_mar, to_mar):
        graph_id = slugId()
        log.info("Submitting a new graph %s", graph_id)
        task_graph = self.from_template(platform, locale, from_mar, to_mar)
        res = self.scheduler.createTaskGraph(graph_id, task_graph)
        log.info("Result was: %s", res)
        return graph_id

    def from_template(self, platform, locale, from_mar, to_mar):
        """Reads and populates graph template.

        :param platform: buildbot platform (linux, macosx64)
        :param locale: en-US, de, ka, etc.
        :param from_mar: "from" MAR URL
        :param to_mar: "to" MAR URK
        :return: graph definition dictionary
        """
        template_file = os.path.join(os.path.dirname(__file__), "tasks",
                                "funsize.yml")
        template = open(template_file).read()
        now = stringDate(datetime.datetime.utcnow())
        now_ms = int(time.time() * 1000)
        encryption_deadline = now_ms + 24 * 3600 * 1000  # 24 hours
        balrog_task_id = slugId()
        template_vars = {
            "update_generator_task_id": slugId(),
            "signing_task_id": slugId(),
            "balrog_task_id": balrog_task_id,
            "now": now,
            "fromNow": fromNow,  # dynamic function call
            "platform": platform,
            "locale": locale,
            "from_MAR": from_mar,
            "to_MAR": to_mar,
            "BALROG_USERNAME_ENC_MESSAGE": encrypt_env_var(
                balrog_task_id, now_ms, encryption_deadline, 'BALROG_USERNAME',
                self.balrog_client.auth[0]),
            "BALROG_PASSWORD_ENC_MESSAGE": encrypt_env_var(
                balrog_task_id, now_ms, encryption_deadline, 'BALROG_PASSWORD',
                self.balrog_client.auth[1])
        }
        rendered = pystache.render(template, template_vars)
        return yaml.safe_load(rendered)

def interesting_buildername(buildername):
    """Matches related builder names

    :type buildername: str or unicode
    :return: boolean
    """
    interesting_names = [n.format(branch=b) for b in BRANCHES for n in
                         BUILDERS]
    return any(re.match(n, buildername) for n in interesting_names)
<commit_msg>Add exception<commit_after>import datetime
import logging
import time
from kombu import Exchange, Queue
from kombu.mixins import ConsumerMixin
import os
import pystache
import re
from taskcluster import slugId, stringDate, fromNow
import yaml
import json

from funsize.utils import properties_to_dict, encrypt_env_var

log = logging.getLogger(__name__)

# TODO: move these to config
BRANCHES = ['mozilla-central', 'mozilla-aurora']
PLATFORMS = ['linux', 'linux64', 'win32', 'win64', 'macosx64']
BUILDERS = [
    r'WINNT \d+\.\d+ (x86-64 )?{branch} nightly',
    r'Linux (x86-64 )?{branch} nightly',
    r'OS X \d+\.\d+ {branch} nightly',
    r'Firefox {branch} (linux|linux64|win32|win64|macosx64) l10n nightly-\d+',
]

class FunsizeWorker(ConsumerMixin):

    def __init__(self, connection, queue_name, exchange, balrog_client,
                 scheduler):
        """Funsize consumer worker
        :type connection: kombu.Connection
        :param queue_name: Full queue name, including queue/<user> prefix
        :type exchange: kombu.Exchange
        :type balrog_client: funsize.balrog.BalrogClient
        :type scheduler: tascluster.Scheduler
        """
        self.connection = connection
        # Using passive mode is important, otherwise pulse returns 403
        self.exchange = Exchange(exchange, type='topic', passive=True)
        self.queue_name = queue_name
        self.balrog_client = balrog_client
        self.scheduler = scheduler

    @property
    def routing_keys(self):
        """Returns an explicit list of routing patterns.

        Instead of using "build.*.*.finished", which generates a lot of noise,
        FunsizeWorker uses explicit list of routing keys to match builders we
        are interested in.
        """
        jobs = [
            # TODO: move to configs
            'build.{branch}-{platform}-nightly.*.finished',
            # TODO: find a better way to specify these
            'build.{branch}-{platform}-l10n-nightly-1.*.finished',
            'build.{branch}-{platform}-l10n-nightly-2.*.finished',
            'build.{branch}-{platform}-l10n-nightly-3.*.finished',
            'build.{branch}-{platform}-l10n-nightly-4.*.finished',
            'build.{branch}-{platform}-l10n-nightly-5.*.finished',
            'build.{branch}-{platform}-l10n-nightly-6.*.finished',
            'build.{branch}-{platform}-l10n-nightly-7.*.finished',
            'build.{branch}-{platform}-l10n-nightly-8.*.finished',
            'build.{branch}-{platform}-l10n-nightly-9.*.finished',
            'build.{branch}-{platform}-l10n-nightly-10.*.finished',
        ]
        return [job.format(branch=branch, platform=platform)
                for job in jobs
                for branch in BRANCHES
                for platform in PLATFORMS]

    @property
    def queues(self):
        """List of queues used by worker.
        Multiple queues are used to track multiple routing keys.
        """
        return [Queue(name=self.queue_name, exchange=self.exchange,
                      routing_key=routing_key, durable=True, exclusive=False,
                      auto_delete=False)
                for routing_key in self.routing_keys]

    def get_consumers(self, Consumer, channel):
        """Implement parent's method called to get the list of consumers"""
        return [Consumer(queues=self.queues, callbacks=[self.process_message])]

    def process_message(self, body, message):
        """Top level callback processing pulse messages.
        The callback tries to handle and log all exceptions

        :type body: kombu.Message.body
        :type message: kombu.Message
        """
        try:
            self.dispatch_message(body)
        except Exception:
            log.exception("Failed to process message")
        finally:
            # TODO: figure out what to do with failed tasks
            message.ack()

    def on_consume_ready(self, connection, channel, consumers, **kwargs):
        """Overrides parent's stub method. Called when ready to consume pulse
         messages.
        """
        log.info('Listening...')

    def on_connection_error(self, exc, interval):
        log.exception("connection error")
        super(FunsizeWorker, self).on_connection_error(exc, interval)

    def dispatch_message(self, body):
        """Dispatches incoming pulse messages.
        If the method detects L10N repacks, it creates multiple Taskcluster
        tasks, otherwise a single en-US task is created.
        :type body: kombu.Message.body
        """
        payload = body["payload"]
        buildername = payload["build"]["builderName"]
        if not interesting_buildername(buildername):
            log.debug("Ignoring %s: not interested", buildername)
            return
        job_result = payload["results"]
        if job_result != 0:
            log.debug("Ignoring %s with result %s", buildername, job_result)
            return
        properties = properties_to_dict(payload["build"]["properties"])
        if "locales" in properties:
            log.debug("L10N repack detected")
            funsize_info = json.loads(properties["funsize_info"])
            locales = json.loads(properties["locales"])
            platform = funsize_info["platform"]
            branch = funsize_info["branch"]
            product = funsize_info["appName"]
            for locale, result in locales.iteritems():
                if result.lower() == "success":
                    self.create_partial(product, branch, platform,
                                        locale)
                else:
                    log.warn("Ignoring %s with result %s", locale, result)
        else:
            log.debug("en-US build detected")
            self.create_partial(properties["appName"],
                                       properties["branch"],
                                       properties["platform"], 'en-US')

    def create_partial(self, product, branch, platform, locale):
        """Calculates "from" and "to" MAR URLs and calls  create_task_graph().
        Currently "from" MAR is 2 releases behind to avoid duplication of
        existing CI partials.

        :param product: capitalized product name, AKA appName, e.g. Firefox
        :param branch: branch name (mozilla-central)
        :param platform: buildbot platform (linux, macosx64)
        :param locale: en-US or locale
        """
        # TODO: move limit to config
        # Get last 3 releases, generate partial from -2 to latest
        last_releases = self.balrog_client.get_releases(product, branch)[:3]
        release_to = last_releases[0]
        release_from = last_releases[-1]
        log.debug("From: %s", release_from)
        log.debug("To: %s", release_to)
        build_from = self.balrog_client.get_build(release_from,
                                                  platform, locale)
        log.debug("Build from: %s", build_from)
        build_to = self.balrog_client.get_build(release_to, platform, locale)
        log.debug("Build to: %s", build_to)
        mar_from = build_from["completes"][0]["fileUrl"]
        mar_to = build_to["completes"][0]["fileUrl"]
        log.info("New Funsize task for %s %s, from %s to %s", platform, locale,
                 mar_from, mar_to)
        self.submit_task_graph(platform, locale, mar_from, mar_to)

    def submit_task_graph(self, platform, locale, from_mar, to_mar):
        graph_id = slugId()
        log.info("Submitting a new graph %s", graph_id)
        task_graph = self.from_template(platform, locale, from_mar, to_mar)
        res = self.scheduler.createTaskGraph(graph_id, task_graph)
        log.info("Result was: %s", res)
        return graph_id

    def from_template(self, platform, locale, from_mar, to_mar):
        """Reads and populates graph template.

        :param platform: buildbot platform (linux, macosx64)
        :param locale: en-US, de, ka, etc.
        :param from_mar: "from" MAR URL
        :param to_mar: "to" MAR URK
        :return: graph definition dictionary
        """
        template_file = os.path.join(os.path.dirname(__file__), "tasks",
                                "funsize.yml")
        template = open(template_file).read()
        now = stringDate(datetime.datetime.utcnow())
        now_ms = int(time.time() * 1000)
        encryption_deadline = now_ms + 24 * 3600 * 1000  # 24 hours
        balrog_task_id = slugId()
        template_vars = {
            "update_generator_task_id": slugId(),
            "signing_task_id": slugId(),
            "balrog_task_id": balrog_task_id,
            "now": now,
            "fromNow": fromNow,  # dynamic function call
            "platform": platform,
            "locale": locale,
            "from_MAR": from_mar,
            "to_MAR": to_mar,
            "BALROG_USERNAME_ENC_MESSAGE": encrypt_env_var(
                balrog_task_id, now_ms, encryption_deadline, 'BALROG_USERNAME',
                self.balrog_client.auth[0]),
            "BALROG_PASSWORD_ENC_MESSAGE": encrypt_env_var(
                balrog_task_id, now_ms, encryption_deadline, 'BALROG_PASSWORD',
                self.balrog_client.auth[1])
        }
        rendered = pystache.render(template, template_vars)
        return yaml.safe_load(rendered)

def interesting_buildername(buildername):
    """Matches related builder names

    :type buildername: str or unicode
    :return: boolean
    """
    interesting_names = [n.format(branch=b) for b in BRANCHES for n in
                         BUILDERS]
    return any(re.match(n, buildername) for n in interesting_names)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Tue Jan  6 15:36:40 2015

@author: Brian
"""
import numpy as np
import constants as const

#  This routine works but is slow... lots of explicit looping
def convective_adjustment(p, T, c, lapserate=6.5):
    """Convective Adjustment to a specified lapse rate.

    Input argument lapserate gives the lapse rate expressed in degrees K per km
    (positive means temperature increasing downward).

    Default lapse rate is 6.5 K / km.

    Returns the adjusted Column temperature.
    inputs:
    p is pressure in hPa
    T is temperature in K
    c is heat capacity in in J / m**2 / K
    """

    # largely follows notation and algorithm in Akamaev (1991) MWR

    if lapserate is 'DALR':
        lapserate = const.g / const.cp * 1.E3
    try:
        alpha = const.Rd / const.g * lapserate / 1.E3
        # col.conv_lapse_rate = lapserate
    except:
        raise ValueError('Problem with lapse rate')

    # Tcol = np.concatenate(([col.Ts], col.Tatm))
    # pnew = np.concatenate(([const.ps], col.p))
    Tcol = T
    pnew = p
    L = pnew.size
    Pi = (pnew/const.ps)**alpha
    beta = 1./Pi
    theta = Tcol * beta
    # q = Pi * np.concatenate(([col.c_sfc], col.c_atm * np.ones_like(col.p)))
    q = Pi * c

    n_k = np.zeros(L, dtype=np.int8)
    theta_k = np.zeros_like(pnew)
    s_k = np.zeros_like(pnew)
    t_k = np.zeros_like(pnew)

    k = 0
    n_k[0] = 1
    theta_k[0] = beta[0] * Tcol[0]
    for l in range(1, L):
        n = 1
        theta = beta[l] * Tcol[l]
        done = False
        while not done:
            if (theta_k[k] > theta):
                # stratification is unstable
                if n == 1:
                    # current layer is not an earlier-formed neutral layer
                    s = q[l]
                    t = s * theta
                if (n_k[k] < 2):
                    # lower adjacent level is not an earlier-formed neutral layer
                    s_k[k] = q[l-n]
                    t_k[k] = s_k[k] * theta_k[k]
                #  join current and underlying layers
                n += n_k[k]
                s += s_k[k]
                s_k[k] = s
                t += t_k[k]
                t_k[k] = t
                theta = t / s
                if k == 0:
                    # joint neutral layer in the first one, done checking lower layers
                    done = True
                else:
                    k -= 1
                    # go back and check stability of the lower adjacent layer
            else:
                k += 1  # statification is stable
                done = True
        # if l < L-1:
        n_k[k] = n
        theta_k[k] = theta
    #  finished looping through to check stability

    # update the temperatures
    newtheta = np.zeros(L)
    count = 0
    for i in range(L):
        newtheta[count+np.arange(n_k[i])] = theta_k[i]
        count += n_k[i]

    Tcol = newtheta * Pi

    return Tcol
<commit_msg>Cleanup of convadj.py<commit_after># -*- coding: utf-8 -*-
"""
Created on Tue Jan  6 15:36:40 2015

@author: Brian
"""
import numpy as np
import constants as const

#  This routine works but is slow... lots of explicit looping
def convective_adjustment(p, T, c, lapserate=6.5):
    """Convective Adjustment to a specified lapse rate.

    Input argument lapserate gives the lapse rate expressed in degrees K per km
    (positive means temperature increasing downward).

    Default lapse rate is 6.5 K / km.

    Returns the adjusted Column temperature.
    inputs:
    p is pressure in hPa
    T is temperature in K
    c is heat capacity in in J / m**2 / K
    """

    # largely follows notation and algorithm in Akamaev (1991) MWR

    if lapserate is 'DALR':
        lapserate = const.g / const.cp * 1.E3
    try:
        alpha = const.Rd / const.g * lapserate / 1.E3
    except:
        raise ValueError('Problem with lapse rate')

    Tcol = T
    pnew = p
    L = pnew.size
    Pi = (pnew/const.ps)**alpha
    beta = 1./Pi
    theta = Tcol * beta
    q = Pi * c

    n_k = np.zeros(L, dtype=np.int8)
    theta_k = np.zeros_like(pnew)
    s_k = np.zeros_like(pnew)
    t_k = np.zeros_like(pnew)

    k = 0
    n_k[0] = 1
    theta_k[0] = beta[0] * Tcol[0]
    for l in range(1, L):
        n = 1
        theta = beta[l] * Tcol[l]
        done = False
        while not done:
            if (theta_k[k] > theta):
                # stratification is unstable
                if n == 1:
                    # current layer is not an earlier-formed neutral layer
                    s = q[l]
                    t = s * theta
                if (n_k[k] < 2):
                    # lower adjacent level is not an earlier-formed neutral layer
                    s_k[k] = q[l-n]
                    t_k[k] = s_k[k] * theta_k[k]
                #  join current and underlying layers
                n += n_k[k]
                s += s_k[k]
                s_k[k] = s
                t += t_k[k]
                t_k[k] = t
                theta = t / s
                if k == 0:
                    # joint neutral layer in the first one, done checking lower layers
                    done = True
                else:
                    k -= 1
                    # go back and check stability of the lower adjacent layer
            else:
                k += 1  # statification is stable
                done = True
        # if l < L-1:
        n_k[k] = n
        theta_k[k] = theta
    #  finished looping through to check stability

    # update the temperatures
    newtheta = np.zeros(L)
    count = 0
    for i in range(L):
        newtheta[count+np.arange(n_k[i])] = theta_k[i]
        count += n_k[i]
    Tcol = newtheta * Pi
    return Tcol
<|endoftext|>