<commit_before>import pytest
import numpy as np
from astropy.constants import M_jup, M_sun

from calculations import flux_mag_ratio, calculate_flux_ratio, calculate_companion_magnitude
from table_search import mass_table_search, magnitude_table_search, age_table
from BDmass_to_flux_ratio import main as mass_main
from flux_ratio_to_BDmass import main as ratio_main
from db_queries import get_sweet_cat_temp

# Test two main functions.
@pytest.mark.xfail  # Query fails when offline
def test_main(raises=Exception):
    """Check it returns 0 (Runs normally).

    If there is no internet then an Exception is raised.
        Exception: Query failed: HTTPConnectionPool(host='simbad.u-strasbg.fr', port=80): Max retries exceeded with url: /simbad/sim-script (Caused by NewConnectionError('<requests.packages.urllib3.connection.HTTPConnection object at 0x7fe4e509b438>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution',)).
    """
    assert mass_main("HD30501", 90, 5) is 0

@pytest.mark.xfail  # Query fails when offline
def test_ratio_main(raises=Exception):
    """Check it returns 0 (Runs normally)
    If there is no internet then an Exception is raised.
        Exception: Query failed: HTTPConnectionPool(host='simbad.u-strasbg.fr', port=80): Max retries exceeded with url: /simbad/sim-script (Caused by NewConnectionError('<requests.packages.urllib3.connection.HTTPConnection object at 0x7fe4e509b438>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution',)).
    """
    assert ratio_main("HD30501", 0.01, 5) is 0

def test_flux_mag_ratio():
    """Test flux-magnitude ratio.

    A magnitude difference of 5 should be 100 (by definition).

    """
    # positive 5 magnitude difference
    assert np.allclose(flux_mag_ratio(1, 6), 100)

    # negative 5 magnitude difference
    assert np.allclose(flux_mag_ratio(7, 2), 1. / 100)

def test_calculate_flux_ratio():
    """Returns a dict with the 3 overlapping values."""
    star_params = {"FLUX_J": 1, "FLUX_H": 1, "FLUX_K": 2}   # Names from simbad
    companion_params = {"Mj": 6, "Mh": 11, "Mk": 7}      # Names from Baraffe

    flux_ratios = calculate_flux_ratio(star_params, companion_params, bands=["J", "H", "K"])
    assert isinstance(flux_ratios, dict)

    assert np.allclose(flux_ratios["J"], 100)
    assert np.allclose(flux_ratios["H"], 10000)
    assert np.allclose(flux_ratios["K"], 100)

#@pytest.mark.xfail
def test_get_sweet_cat_temp():
    """Test getting from sweet-cat."""
    # hd number in SweetCat
    a = get_sweet_cat_temp("HD107383")
    assert isinstance(a, float)
    assert a == 4830
    # hd number not in sweet
    b = get_sweet_cat_temp("HD1")
    assert b is False
    # non hd id
    with pytest.raises(NotImplementedError):
        get_sweet_cat_temp("GJ 422")  # in sweetcat but not an hd number

# Test Flux ratio to Mass
@pytest.mark.parametrize("band", ["H", "J", "K"])
def test_mag_conversions(band):
    """ Test converting from flux ratio to magnitude back to flux ratio etc.

    Tests show the conversion goes back and forward.
    """
    comp_vals = {"Mj": 4, "Mk": 6, "Mh": 12}
    star_vals = {"FLUX_J": 3, "FLUX_K": 11, "FLUX_H": 5}

    ratios = calculate_flux_ratio(star_vals, comp_vals, band)
    print("Ratios from function", ratios)
    magnitudes = calculate_companion_magnitude(star_vals, 1. / ratios[band], band)
    print("magnitudes from ratios", magnitudes)
    magnitudes["M{}".format(band.lower())] = magnitudes[band]
    print("magnitudes from ratios", magnitudes)
    new_ratios = calculate_flux_ratio(star_vals, magnitudes, band)
    print("new_ratios from mags", new_ratios)

    assert np.allclose(new_ratios[band], ratios[band])

# @pytest.mark.xfail()
@pytest.mark.parametrize("mass_model", [(50, "2003"), (150, "2015")])
@pytest.mark.parametrize("age", [1, 5, 10])
@pytest.mark.parametrize("band", ["H", "J", "K"])
# @pytest.mark.parametrize("model", ["2003", "2015"])
def test_table_searches(mass_model, age, band):
    """That a given mass calculates a magnitude and that mag finds a correct mass."""
    starting_mass = mass_model[0]  # M_jup
    model = mass_model[1]
    start_sol_mass = starting_mass * (M_jup / M_sun).value
    print("Starting_mass = {0} (Mjup), {1} (Msun)".format(starting_mass, start_sol_mass))

    comp_params = mass_table_search(start_sol_mass, age=age, model=model)
    found_mag = comp_params["M{0!s}".format(band.lower())]
    print("Band", band, "Found magnitude", found_mag)
    magnitude = {band: found_mag}
    print("magnitude", magnitude)

    mag_params = magnitude_table_search(magnitude, age=age, band=band, model=model)
    print("mag_params", mag_params)

    found_mass = mag_params["M/Ms"]
    print("found_mass", found_mass, "start_mass", start_sol_mass)

    assert np.allclose(found_mass, start_sol_mass)
    assert np.allclose(found_mass * M_sun / M_jup, starting_mass)  # back int M_jup

@pytest.mark.parametrize("age", [0.1, 1, 10])
@pytest.mark.parametrize("model", ["2003", "2015", pytest.mark.xfail("2016")])
def test_age_table(age, model):
    """Selects a baraffe table of certian age and model."""
    model_03, cols_03 = age_table(5, "2003")

    model_15, cols_15 = age_table(5, "2015")

    assert isinstance(model_03, dict)
    assert isinstance(cols_03, list)
    assert isinstance(model_15, dict)
    assert isinstance(cols_15, list)
    for key in model_03:
        assert isinstance(model_03[key], np.ndarray)
    for key in model_15:
        assert isinstance(model_15[key], np.ndarray)

    # Assert actually different
    assert np.all(cols_15 != cols_03)
    assert np.all(model_15 != model_03)
    assert len(cols_15) != len(cols_03)
    assert len(model_15) != len(model_03)

    # check common columns are present
    common_cols = ["M/Ms", "Teff", "L/Ls", "g", "Mv",
                   "Mr", "Mi", "Mj", "Mh", "Mk", "Mll", "Mm"]
    for header in common_cols:
        assert header in cols_15
        assert header in cols_03

@pytest.mark.parametrize("model", ["2016", "", 2003, 15, "word"])
def test_bad_model_age_table(model):
    """Call model with wrong values."""
    with pytest.raises(ValueError):
        age_table(5, model)

def test_calculate_comp_magnitude():
    """Test t returns values fr all bands given."""
    bands = ["H", "J", "K"]
    star_vals = {"FLUX_J": 1, "FLUX_H": 1, "FLUX_K": 2}   # Names from simbad
    magnitudes = calculate_companion_magnitude(star_vals, 0.001, bands)

    assert isinstance(magnitudes, dict)
    assert len(magnitudes) == 3
    for band in bands:
        assert band in magnitudes.keys()

def test_mag_table_search_band():
    """One one band value is allowed."""
    age = 5
    magnitudes = {"H": 1, "J": 4, "K": 5}

    with pytest.raises(ValueError):
        magnitude_table_search(magnitudes, age, band=["H", "J", "K"], model="2003")

def test_mass_table_search_03():
    """That a value from the table returns the correct row."""
    mass = 0.09
    comp_params = mass_table_search(mass, 5, model="2003")
    print(comp_params)
    assert comp_params["M/Ms"] == 0.09
    assert comp_params["Teff"] == 2622
    assert comp_params["R"] == 0.113
    assert comp_params["Mk"] == 10.04

def test_mass_table_search_15():
    mass = 0.09
    comp_params_15 = mass_table_search(mass, 5, model="2015")
    print(comp_params_15)
    assert comp_params_15["M/Ms"] == 0.09
    assert comp_params_15["Teff"] == 2644
    assert comp_params_15["R/Rs"] == 0.113
    assert comp_params_15["Mk"] == 9.91

def test_magnitude_table_search_03():
    """That a value from the table returns the correct row."""
    mag_params = magnitude_table_search({"K": 10.04}, 5, band="K", model="2003")
    print("mag_params", mag_params)
    assert mag_params["M/Ms"] == 0.09
    assert mag_params["Teff"] == 2622
    assert mag_params["R"] == 0.113
    assert mag_params["Mk"] == 10.04

def test_magnitude_table_search_15():
    print("Input value = magnitude_table_search({""K"": 9.91}, 5, band=""K"", model=""2015"")")
    mag_params_15 = magnitude_table_search({"K": 9.91}, 5, band="K", model="2015")
    print("mag_params returned", mag_params_15)
    assert mag_params_15["M/Ms"] == 0.09
    assert mag_params_15["Teff"] == 2644
    assert mag_params_15["R/Rs"] == 0.113
    assert mag_params_15["Mk"] == 9.91

def test_magnitude_table_search_errors():
    """Test only takes one band as a str."""
    # Check single band as list is ok.
    with pytest.raises(ValueError):
        # More than one band not allowed
        magnitude_table_search({"K": 9.91, "H": 9.91}, 5, band=["H", "K"], model="2015")

    with pytest.raises(ValueError):
        # More than one band not allowed
        magnitude_table_search({"K": 9.91, "H": 9.91}, 5, band=("K",), model="2015")

    with pytest.raises(ValueError):
        # More then one band not allowed
        magnitude_table_search({"K": 9.91, "H": 9.91}, 5, band=["K"], model="2015")

    with pytest.raises(ValueError):
        # Bad band (Not present on cols)
        magnitude_table_search({"K": 9.91, "H": 9.91}, 5, band="Z", model="2015")

    with pytest.raises(ValueError):
        # Band not in magnitudes given
        magnitude_table_search({"K": 9.91, "H": 9.91}, 5, band="J", model="2015")
<commit_msg>test a bad band in calculation.<commit_after>import pytest
import numpy as np
from astropy.constants import M_jup, M_sun

from calculations import flux_mag_ratio, calculate_flux_ratio, calculate_companion_magnitude
from table_search import mass_table_search, magnitude_table_search, age_table
from BDmass_to_flux_ratio import main as mass_main
from flux_ratio_to_BDmass import main as ratio_main
from db_queries import get_sweet_cat_temp

# Test two main functions.
@pytest.mark.xfail  # Query fails when offline
def test_main(raises=Exception):
    """Check it returns 0 (Runs normally).

    If there is no internet then an Exception is raised.
        Exception: Query failed: HTTPConnectionPool(host='simbad.u-strasbg.fr', port=80): Max retries exceeded with url: /simbad/sim-script (Caused by NewConnectionError('<requests.packages.urllib3.connection.HTTPConnection object at 0x7fe4e509b438>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution',)).
    """
    assert mass_main("HD30501", 90, 5) is 0

@pytest.mark.xfail  # Query fails when offline
def test_ratio_main(raises=Exception):
    """Check it returns 0 (Runs normally)
    If there is no internet then an Exception is raised.
        Exception: Query failed: HTTPConnectionPool(host='simbad.u-strasbg.fr', port=80): Max retries exceeded with url: /simbad/sim-script (Caused by NewConnectionError('<requests.packages.urllib3.connection.HTTPConnection object at 0x7fe4e509b438>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution',)).
    """
    assert ratio_main("HD30501", 0.01, 5) is 0

def test_flux_mag_ratio():
    """Test flux-magnitude ratio.

    A magnitude difference of 5 should be 100 (by definition).

    """
    # positive 5 magnitude difference
    assert np.allclose(flux_mag_ratio(1, 6), 100)

    # negative 5 magnitude difference
    assert np.allclose(flux_mag_ratio(7, 2), 1. / 100)

def test_calculate_flux_ratio():
    """Returns a dict with the 3 overlapping values."""
    star_params = {"FLUX_J": 1, "FLUX_H": 1, "FLUX_K": 2}   # Names from simbad
    companion_params = {"Mj": 6, "Mh": 11, "Mk": 7}      # Names from Baraffe

    flux_ratios = calculate_flux_ratio(star_params, companion_params, bands=["J", "H", "K"])
    assert isinstance(flux_ratios, dict)

    assert np.allclose(flux_ratios["J"], 100)
    assert np.allclose(flux_ratios["H"], 10000)
    assert np.allclose(flux_ratios["K"], 100)

#@pytest.mark.xfail
def test_get_sweet_cat_temp():
    """Test getting from sweet-cat."""
    # hd number in SweetCat
    a = get_sweet_cat_temp("HD107383")
    assert isinstance(a, float)
    assert a == 4830
    # hd number not in sweet
    b = get_sweet_cat_temp("HD1")
    assert b is False
    # non hd id
    with pytest.raises(NotImplementedError):
        get_sweet_cat_temp("GJ 422")  # in sweetcat but not an hd number

# Test Flux ratio to Mass
@pytest.mark.parametrize("band", ["H", "J", "K"])
def test_mag_conversions(band):
    """ Test converting from flux ratio to magnitude back to flux ratio etc.

    Tests show the conversion goes back and forward.
    """
    comp_vals = {"Mj": 4, "Mk": 6, "Mh": 12}
    star_vals = {"FLUX_J": 3, "FLUX_K": 11, "FLUX_H": 5}

    ratios = calculate_flux_ratio(star_vals, comp_vals, band)
    print("Ratios from function", ratios)
    magnitudes = calculate_companion_magnitude(star_vals, 1. / ratios[band], band)
    print("magnitudes from ratios", magnitudes)
    magnitudes["M{}".format(band.lower())] = magnitudes[band]
    print("magnitudes from ratios", magnitudes)
    new_ratios = calculate_flux_ratio(star_vals, magnitudes, band)
    print("new_ratios from mags", new_ratios)

    assert np.allclose(new_ratios[band], ratios[band])

# @pytest.mark.xfail()
@pytest.mark.parametrize("mass_model", [(50, "2003"), (150, "2015")])
@pytest.mark.parametrize("age", [1, 5, 10])
@pytest.mark.parametrize("band", ["H", "J", "K"])
# @pytest.mark.parametrize("model", ["2003", "2015"])
def test_table_searches(mass_model, age, band):
    """That a given mass calculates a magnitude and that mag finds a correct mass."""
    starting_mass = mass_model[0]  # M_jup
    model = mass_model[1]
    start_sol_mass = starting_mass * (M_jup / M_sun).value
    print("Starting_mass = {0} (Mjup), {1} (Msun)".format(starting_mass, start_sol_mass))

    comp_params = mass_table_search(start_sol_mass, age=age, model=model)
    found_mag = comp_params["M{0!s}".format(band.lower())]
    print("Band", band, "Found magnitude", found_mag)
    magnitude = {band: found_mag}
    print("magnitude", magnitude)

    mag_params = magnitude_table_search(magnitude, age=age, band=band, model=model)
    print("mag_params", mag_params)

    found_mass = mag_params["M/Ms"]
    print("found_mass", found_mass, "start_mass", start_sol_mass)

    assert np.allclose(found_mass, start_sol_mass)
    assert np.allclose(found_mass * M_sun / M_jup, starting_mass)  # back int M_jup

@pytest.mark.parametrize("age", [0.1, 1, 10])
@pytest.mark.parametrize("model", ["2003", "2015", pytest.mark.xfail("2016")])
def test_age_table(age, model):
    """Selects a baraffe table of certian age and model."""
    model_03, cols_03 = age_table(5, "2003")

    model_15, cols_15 = age_table(5, "2015")

    assert isinstance(model_03, dict)
    assert isinstance(cols_03, list)
    assert isinstance(model_15, dict)
    assert isinstance(cols_15, list)
    for key in model_03:
        assert isinstance(model_03[key], np.ndarray)
    for key in model_15:
        assert isinstance(model_15[key], np.ndarray)

    # Assert actually different
    assert np.all(cols_15 != cols_03)
    assert np.all(model_15 != model_03)
    assert len(cols_15) != len(cols_03)
    assert len(model_15) != len(model_03)

    # check common columns are present
    common_cols = ["M/Ms", "Teff", "L/Ls", "g", "Mv",
                   "Mr", "Mi", "Mj", "Mh", "Mk", "Mll", "Mm"]
    for header in common_cols:
        assert header in cols_15
        assert header in cols_03

@pytest.mark.parametrize("model", ["2016", "", 2003, 15, "word"])
def test_bad_model_age_table(model):
    """Call model with wrong values."""
    with pytest.raises(ValueError):
        age_table(5, model)

def test_calculate_comp_magnitude():
    """Test t returns values fr all bands given."""
    bands = ["H", "J", "K"]
    star_vals = {"FLUX_J": 1, "FLUX_H": 1, "FLUX_K": 2}   # Names from simbad
    magnitudes = calculate_companion_magnitude(star_vals, 0.001, bands)

    assert isinstance(magnitudes, dict)
    assert len(magnitudes) == 3
    for band in bands:
        assert band in magnitudes.keys()

    # Test error
    bands = ["H", "P"]
    with pytest.raises(ValueError):
        calculate_companion_magnitude(star_vals, 0.001, bands)

def test_mag_table_search_band():
    """One one band value is allowed."""
    age = 5
    magnitudes = {"H": 1, "J": 4, "K": 5}

    with pytest.raises(ValueError):
        magnitude_table_search(magnitudes, age, band=["H", "J", "K"], model="2003")

def test_mass_table_search_03():
    """That a value from the table returns the correct row."""
    mass = 0.09
    comp_params = mass_table_search(mass, 5, model="2003")
    print(comp_params)
    assert comp_params["M/Ms"] == 0.09
    assert comp_params["Teff"] == 2622
    assert comp_params["R"] == 0.113
    assert comp_params["Mk"] == 10.04

def test_mass_table_search_15():
    mass = 0.09
    comp_params_15 = mass_table_search(mass, 5, model="2015")
    print(comp_params_15)
    assert comp_params_15["M/Ms"] == 0.09
    assert comp_params_15["Teff"] == 2644
    assert comp_params_15["R/Rs"] == 0.113
    assert comp_params_15["Mk"] == 9.91

def test_magnitude_table_search_03():
    """That a value from the table returns the correct row."""
    mag_params = magnitude_table_search({"K": 10.04}, 5, band="K", model="2003")
    print("mag_params", mag_params)
    assert mag_params["M/Ms"] == 0.09
    assert mag_params["Teff"] == 2622
    assert mag_params["R"] == 0.113
    assert mag_params["Mk"] == 10.04

def test_magnitude_table_search_15():
    print("Input value = magnitude_table_search({""K"": 9.91}, 5, band=""K"", model=""2015"")")
    mag_params_15 = magnitude_table_search({"K": 9.91}, 5, band="K", model="2015")
    print("mag_params returned", mag_params_15)
    assert mag_params_15["M/Ms"] == 0.09
    assert mag_params_15["Teff"] == 2644
    assert mag_params_15["R/Rs"] == 0.113
    assert mag_params_15["Mk"] == 9.91

def test_magnitude_table_search_errors():
    """Test only takes one band as a str."""
    # Check single band as list is ok.
    with pytest.raises(ValueError):
        # More than one band not allowed
        magnitude_table_search({"K": 9.91, "H": 9.91}, 5, band=["H", "K"], model="2015")

    with pytest.raises(ValueError):
        # More than one band not allowed
        magnitude_table_search({"K": 9.91, "H": 9.91}, 5, band=("K",), model="2015")

    with pytest.raises(ValueError):
        # More then one band not allowed
        magnitude_table_search({"K": 9.91, "H": 9.91}, 5, band=["K"], model="2015")

    with pytest.raises(ValueError):
        # Bad band (Not present on cols)
        magnitude_table_search({"K": 9.91, "H": 9.91}, 5, band="Z", model="2015")

    with pytest.raises(ValueError):
        # Band not in magnitudes given
        magnitude_table_search({"K": 9.91, "H": 9.91}, 5, band="J", model="2015")
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# pylint: disable=invalid-name
"""Logsnarf application.

Application code to connect the various modules to do something useful..
"""
import sys
import logging
import re

from twisted.python import usage
from twisted.internet import reactor
from oauth2client.client import SignedJwtAssertionCredentials
import simplejson as json

from . import config
from . import schema
from . import service
from . import snarf
from . import uploader
from . import state
from . import errors

class Options(usage.Options):
    optParameters = [
        ['resource_name', 'n', 'logsnarf', 'Resource name'],
        ['config_file', 'f', None, 'Config file'],
    ]

def install_custom_verifiers(sch, default_domain):
    # noinspection PyUnusedLocal
    def add_default_domain(unused_arg, value):
        if len(value.split('.')) == 1:
            return u'{}.{}'.format(value, default_domain)
        return unicode(value)

    def fix_pid_mess(root_obj, value):
        if value is None:
            return 0
        if isinstance(value, basestring) and '[' in value:
            pidparts = value.split('[')
            pid = int(pidparts[-1])
            pname = pidparts[0]
            root_obj['pname'] = pname
            return pid
        else:
            value = int(value)
            return value

    sch.setFieldValidator('host', add_default_domain)
    sch.setFieldValidator('src.host', add_default_domain)
    sch.setFieldValidator('dst.host', add_default_domain)
    sch.setFieldValidator('pid', fix_pid_mess)
    return sch

def install_schema_load_hook(sch):
    def loadHook(obj):
        """object_hook for JSONDecoder.

        :param obj: non-literal json decoded object
        :type obj: dict
        :return: updated object
        :rvalue: dict
        """
        fieldpath_re = re.compile(r'[.!]')
        for k in obj.copy():
            if k == 'pid' and obj[k] == '-':
                obj[k] = 0
            kparts = fieldpath_re.split(k)
            if len(kparts) > 1:
                value = obj.pop(k)
                kparts.reverse()
                entry = obj
                while len(kparts) > 1:
                    name = kparts.pop()
                    entry = entry.setdefault(name, {})
                name = kparts.pop()
                entry[name] = value
        return obj

    sch.setObjectLoadHook(loadHook)

class App(object):
    """Logsnarf application class.

    Currently this is just a convenient way to encapsulate the setup of the
    various components. It minimizes the use of the configuration object to
    make for cleaner implementation classes.

    Currently a logsnarf.App contains three main components.

    logsnarf.snarf.Logsnarf - Monitors directories for log updates, feeds them
            linewise to the next in the chain.

    logsnarf.uploader.BigQueryUploader - Receieves lines from Logsnarf,
        pushes them through the logsnarf.schema.Schema object for
        munging/verification, adds an insertId to every row, batches them into
        groups, which get a transaction id (internal use only), and passes
        them on to the BigQueryService. Partial failures (only some rows) are
        handled here. Full request failures are generally handled by the
        BigQueryService. Currently this is where flush-on-exit is handled.

    logsnarf.service.BigQueryService - Encapsulates a standard googleapi
            BigQueryService along with our project/dataset information and
            credentials.
    """

    def __init__(self, cfg, section_name):
        """

        :param cfg: config object
        :type cfg: logsnarf.config.Config
        :param section_name: section name from the config that tells us what
          to do
        :type section_name: str
        """
        self.cfg = cfg
        self.section = section = cfg[section_name]
        with cfg.openConfigFile(section['keyfile'], 'rb') as f:
            creds = SignedJwtAssertionCredentials(
                section['service_email'],
                f.read(),
                scope='https://www.googleapis.com/auth/bigquery')
        svc = service.BigQueryService(
            section['project_id'],
            section['dataset'],
            creds, debug=True)
        default_tz = section.get('default_tz', 'UTC')
        default_domain = section.get('default_domain', None)
        schema_file = cfg.openConfigFile(section['schema_file'], 'rb')
        schema_obj = schema.Schema(schema_file, default_tz)
        install_schema_load_hook(schema_obj)
        install_custom_verifiers(schema_obj, default_domain)

        table_name_fmt = section.get('table_name_fmt', None)
        upl = uploader.BigQueryUploader(schema_obj, svc, table_name_fmt)
        if 'batchsize' in section:
            upl.setBatchSize(section['batchsize'])
        if 'max_buffer' in section:
            upl.setMaxBuffer(section['max_buffer'])
        if 'flush_interval' in section:
            upl.setFlushInterval(section['flush_interval'])
        upl.setDefaultTZ(default_tz)

        state_path = cfg.saveConfigPath(section['state_file'])
        state_object = state.State(state_path)

        dirs = section.get('directories', None)
        if dirs is None:
            logging.fatal('No directories configured in section %s',
                          section_name)
            sys.exit(1)
        else:
            dirs = json.loads(dirs)
        pattern = section.get('pattern', None)
        if pattern is not None:
            pattern = re.compile(pattern)
        recursive = section.get('recursive', True)
        snarfer = snarf.LogSnarf(state_object, upl)
        if pattern:
            logging.info(
                'Setting up Snarfer to watch directories %s '
                'pattern: %s recursive: %s', dirs, pattern.pattern, recursive)
        else:
            logging.info(
                'Setting up Snarfer to watch directories %s recursive: %s',
                dirs, recursive)
        for d in dirs:
            snarfer.watch(d, pattern, recursive)
        self.snarfer = snarfer
        self.schema = schema_obj
        self.service = svc
        self.uploader = upl

    def start(self):
        self.snarfer.start()

def main():
    opts = Options()
    try:
        opts.parseOptions()
    except usage.UsageError, e:
        print "%s: %s" % (sys.argv[0], e)
        print "%s: Try --help for usage details." % (sys.argv[0])
        sys.exit(1)
    cfg = config.Config(resource_name=opts['resource_name'],
                        config_file=opts['config_file'])
    cfg.load()
    sections = cfg.get('logsnarf', None)
    if sections is None:
        raise errors.ConfigError('No logsnarf section configured.')
    else:
        try:
            config_apps = json.loads(sections['apps'])
        except json.JSONDecodeError, e:
            raise errors.ConfigError('No valid apps section in configuration')
    apps = []
    for s in config_apps:
        if s not in cfg.keys():
            raise errors.ConfigError('Application section %s referenced but '
                                     'does not exist. %s', s, sections)
        else:
            apps.append(App(cfg, s))
    for app in apps:
        app.start()

    # noinspection PyUnresolvedReferences
    reactor.run()

if __name__ == '__main__':
    main()
<commit_msg>Update logsnarf to use newer version of google oauth2client that's moved the service account credential handling to client.service_account<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# pylint: disable=invalid-name
"""Logsnarf application.

Application code to connect the various modules to do something useful..
"""
import os
import sys
import logging
import re

from twisted.python import usage
from twisted.internet import reactor
from oauth2client.service_account import ServiceAccountCredentials
import simplejson as json

from . import config
from . import schema
from . import service
from . import snarf
from . import uploader
from . import state
from . import errors

class Options(usage.Options):
    optParameters = [
        ['resource_name', 'n', 'logsnarf', 'Resource name'],
        ['config_file', 'f', None, 'Config file'],
    ]

def install_custom_verifiers(sch, default_domain):
    # noinspection PyUnusedLocal
    def add_default_domain(unused_arg, value):
        if len(value.split('.')) == 1:
            return u'{}.{}'.format(value, default_domain)
        return unicode(value)

    def fix_pid_mess(root_obj, value):
        if value is None:
            return 0
        if isinstance(value, basestring) and '[' in value:
            pidparts = value.split('[')
            pid = int(pidparts[-1])
            pname = pidparts[0]
            root_obj['pname'] = pname
            return pid
        else:
            value = int(value)
            return value

    sch.setFieldValidator('host', add_default_domain)
    sch.setFieldValidator('src.host', add_default_domain)
    sch.setFieldValidator('dst.host', add_default_domain)
    sch.setFieldValidator('pid', fix_pid_mess)
    return sch

def install_schema_load_hook(sch):
    def loadHook(obj):
        """object_hook for JSONDecoder.

        :param obj: non-literal json decoded object
        :type obj: dict
        :return: updated object
        :rvalue: dict
        """
        fieldpath_re = re.compile(r'[.!]')
        for k in obj.copy():
            if k == 'pid' and obj[k] == '-':
                obj[k] = 0
            kparts = fieldpath_re.split(k)
            if len(kparts) > 1:
                value = obj.pop(k)
                kparts.reverse()
                entry = obj
                while len(kparts) > 1:
                    name = kparts.pop()
                    entry = entry.setdefault(name, {})
                name = kparts.pop()
                entry[name] = value
        return obj

    sch.setObjectLoadHook(loadHook)

class App(object):
    """Logsnarf application class.

    Currently this is just a convenient way to encapsulate the setup of the
    various components. It minimizes the use of the configuration object to
    make for cleaner implementation classes.

    Currently a logsnarf.App contains three main components.

    logsnarf.snarf.Logsnarf - Monitors directories for log updates, feeds them
            linewise to the next in the chain.

    logsnarf.uploader.BigQueryUploader - Receieves lines from Logsnarf,
        pushes them through the logsnarf.schema.Schema object for
        munging/verification, adds an insertId to every row, batches them into
        groups, which get a transaction id (internal use only), and passes
        them on to the BigQueryService. Partial failures (only some rows) are
        handled here. Full request failures are generally handled by the
        BigQueryService. Currently this is where flush-on-exit is handled.

    logsnarf.service.BigQueryService - Encapsulates a standard googleapi
            BigQueryService along with our project/dataset information and
            credentials.
    """

    def __init__(self, cfg, section_name):
        """

        :param cfg: config object
        :type cfg: logsnarf.config.Config
        :param section_name: section name from the config that tells us what
          to do
        :type section_name: str
        """
        self.cfg = cfg
        self.section = section = cfg[section_name]
        creds = ServiceAccountCredentials.from_p12_keyfile(
            section['service_email'],
            os.path.join(cfg.saveConfigPath(), section['keyfile']),
            scopes='https://www.googleapis.com/auth/bigquery')
        svc = service.BigQueryService(
            section['project_id'],
            section['dataset'],
            creds, debug=True)
        default_tz = section.get('default_tz', 'UTC')
        default_domain = section.get('default_domain', None)
        schema_file = cfg.openConfigFile(section['schema_file'], 'rb')
        schema_obj = schema.Schema(schema_file, default_tz)
        install_schema_load_hook(schema_obj)
        install_custom_verifiers(schema_obj, default_domain)

        table_name_fmt = section.get('table_name_fmt', None)
        upl = uploader.BigQueryUploader(schema_obj, svc, table_name_fmt)
        if 'batchsize' in section:
            upl.setBatchSize(section['batchsize'])
        if 'max_buffer' in section:
            upl.setMaxBuffer(section['max_buffer'])
        if 'flush_interval' in section:
            upl.setFlushInterval(section['flush_interval'])
        upl.setDefaultTZ(default_tz)

        state_path = cfg.saveConfigPath(section['state_file'])
        state_object = state.State(state_path)

        dirs = section.get('directories', None)
        if dirs is None:
            logging.fatal('No directories configured in section %s',
                          section_name)
            sys.exit(1)
        else:
            dirs = json.loads(dirs)
        pattern = section.get('pattern', None)
        if pattern is not None:
            pattern = re.compile(pattern)
        recursive = section.get('recursive', True)
        snarfer = snarf.LogSnarf(state_object, upl)
        if pattern:
            logging.info(
                'Setting up Snarfer to watch directories %s '
                'pattern: %s recursive: %s', dirs, pattern.pattern, recursive)
        else:
            logging.info(
                'Setting up Snarfer to watch directories %s recursive: %s',
                dirs, recursive)
        for d in dirs:
            snarfer.watch(d, pattern, recursive)
        self.snarfer = snarfer
        self.schema = schema_obj
        self.service = svc
        self.uploader = upl

    def start(self):
        self.snarfer.start()

def main():
    opts = Options()
    try:
        opts.parseOptions()
    except usage.UsageError, e:
        print "%s: %s" % (sys.argv[0], e)
        print "%s: Try --help for usage details." % (sys.argv[0])
        sys.exit(1)
    cfg = config.Config(resource_name=opts['resource_name'],
                        config_file=opts['config_file'])
    cfg.load()
    sections = cfg.get('logsnarf', None)
    if sections is None:
        raise errors.ConfigError('No logsnarf section configured.')
    else:
        try:
            config_apps = json.loads(sections['apps'])
        except json.JSONDecodeError, e:
            raise errors.ConfigError('No valid apps section in configuration')
    apps = []
    for s in config_apps:
        if s not in cfg.keys():
            raise errors.ConfigError('Application section %s referenced but '
                                     'does not exist. %s', s, sections)
        else:
            apps.append(App(cfg, s))
    for app in apps:
        app.start()

    # noinspection PyUnresolvedReferences
    reactor.run()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>TABLES = {
    'pluto_16v2': {'pluto_16v2': 800000},
    'pluto_17v1': {'pluto_17v1': 800000},
    'dobjobs': {'dobjobs': 1000000},
    'hpd_violations': {'hpd_violations': 4000000},
    'hpd_registrations': {
        'hpd_registrations': 150000,
        'hpd_contacts': 600000,
        'hpd_corporate_owners': 30000,
        'hpd_business_addresses': 130000,
        'hpd_registrations_grouped_by_bbl': 130000,
        'hpd_registrations_grouped_by_bbl_with_contacts': 130000
    },
    'dof_sales': {'dof_sales': 75000},
    'rentstab': {'rentstab': 45000},
    'dob_complaints': {'dob_complaints': 1000000},
    'hpd_complaints': {
        'hpd_complaints': 1000000,
        'hpd_complaint_problems': 2000000
    },
    'acris': {
        'real_property_remarks': 1000000,
        'real_property_legals': 1000000,
        'real_property_master': 1000000,
        'real_property_parties': 1000000,
        'real_property_references': 1000000,
        'personal_property_legals': 1000000,
        'personal_property_master': 1000000,
        'personal_property_parties': 1000000,
        'personal_property_references': 1000000,
        'personal_property_remarks': 400000,
        'acris_country_codes': 250,
        'acris_document_control_codes': 123,
        'acris_property_type_codes': 46,
        'acris_ucc_collateral_codes': 8
    }
}

class colors:
    BLUE = '\033[94m'
    GREEN = '\033[92m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'

def check_dataset(db, dataset):
    """ input: nycdb.Database, str """

    for table_name, min_row_count in TABLES[dataset].items():
        if db.table_exists(table_name):
            cnt = db.row_count(table_name)
            if cnt >= min_row_count:
                print(colors.GREEN + table_name + ' has ' + format(cnt, ',') + ' rows' + colors.ENDC)
            else:
                if cnt == 0:
                    print(colors.FAIL + table_name + ' has no rows!' + colors.ENDC)
                else:
                    has_rows = colors.FAIL + table_name + ' has ' + format(cnt, ',') + ' rows.' + colors.ENDC
                    expecting = colors.FAIL + 'Expecting at least ' + colors.BLUE + format(min_row_count, ',') + colors.ENDC + colors.FAIL + ' rows' + colors.ENDC
                    print(has_rows + expecting)
        else:
            print(colors.FAIL + table_name + ' is missing!' + colors.ENDC)

# 'acris': [
#         'country_codes',
#         'document_control_codes',
#         'ucc_collateral_codes',
#         'personal_property_legals',
#         'personal_property_master',
#         'personal_property_parties',
# #       'personal_property_references',
# #       'personal_property_remarks',
#         'real_property_legals',
#         'real_property_master',
#         'real_property_parties',
#         'real_property_references',
#         'real_property_remarks'
# ],
<commit_msg>add pluto_18v1 to verify<commit_after>TABLES = {
    'pluto_16v2': {'pluto_16v2': 800000},
    'pluto_17v1': {'pluto_17v1': 800000},
    'pluto_18v1': {'pluto_18v1': 800000},
    'dobjobs': {'dobjobs': 1000000},
    'hpd_violations': {'hpd_violations': 4000000},
    'hpd_registrations': {
        'hpd_registrations': 150000,
        'hpd_contacts': 600000,
        'hpd_corporate_owners': 30000,
        'hpd_business_addresses': 130000,
        'hpd_registrations_grouped_by_bbl': 130000,
        'hpd_registrations_grouped_by_bbl_with_contacts': 130000
    },
    'dof_sales': {'dof_sales': 75000},
    'rentstab': {'rentstab': 45000},
    'dob_complaints': {'dob_complaints': 1000000},
    'hpd_complaints': {
        'hpd_complaints': 1000000,
        'hpd_complaint_problems': 2000000
    },
    'acris': {
        'real_property_remarks': 1000000,
        'real_property_legals': 1000000,
        'real_property_master': 1000000,
        'real_property_parties': 1000000,
        'real_property_references': 1000000,
        'personal_property_legals': 1000000,
        'personal_property_master': 1000000,
        'personal_property_parties': 1000000,
        'personal_property_references': 1000000,
        'personal_property_remarks': 400000,
        'acris_country_codes': 250,
        'acris_document_control_codes': 123,
        'acris_property_type_codes': 46,
        'acris_ucc_collateral_codes': 8
    }
}

class colors:
    BLUE = '\033[94m'
    GREEN = '\033[92m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'

def check_dataset(db, dataset):
    """ input: nycdb.Database, str """

    for table_name, min_row_count in TABLES[dataset].items():
        if db.table_exists(table_name):
            cnt = db.row_count(table_name)
            if cnt >= min_row_count:
                print(colors.GREEN + table_name + ' has ' + format(cnt, ',') + ' rows' + colors.ENDC)
            else:
                if cnt == 0:
                    print(colors.FAIL + table_name + ' has no rows!' + colors.ENDC)
                else:
                    has_rows = colors.FAIL + table_name + ' has ' + format(cnt, ',') + ' rows.' + colors.ENDC
                    expecting = colors.FAIL + 'Expecting at least ' + colors.BLUE + format(min_row_count, ',') + colors.ENDC + colors.FAIL + ' rows' + colors.ENDC
                    print(has_rows + expecting)
        else:
            print(colors.FAIL + table_name + ' is missing!' + colors.ENDC)

# 'acris': [
#         'country_codes',
#         'document_control_codes',
#         'ucc_collateral_codes',
#         'personal_property_legals',
#         'personal_property_master',
#         'personal_property_parties',
# #       'personal_property_references',
# #       'personal_property_remarks',
#         'real_property_legals',
#         'real_property_master',
#         'real_property_parties',
#         'real_property_references',
#         'real_property_remarks'
# ],
<|endoftext|>


<commit_before>"""The CLI module for OCSPdash."""
import datetime
import json
import logging
import os
import urllib.parse
from collections import OrderedDict

import click

from .server_query import ServerQuery, check_ocsp_response
from .web.app import create_application
from .web.manager import Manager

@click.group()
def main():
    """Run OCSP Dashboard"""

@main.command()
@click.option('-n', default=2, type=int, help='Number of top authorities')
@click.option('-o', is_flag=True, help='Output as JSON')
@click.option('-v', is_flag=True, help='Verbose output')
def run(n, o, v):
    if v:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)

    server_query = ServerQuery(os.environ.get('UID'), os.environ.get('SECRET'))

    issuers = server_query.get_top_authorities(n)  # TODO: cache this result for 24 hours

    ocsp_reports = OrderedDict(  # TODO: cache this result for 24 hours
        (issuer, server_query.get_ocsp_urls_for_issuer(issuer))
        for issuer in issuers.keys()
    )

    test_results = OrderedDict(
        (
            issuer,
            OrderedDict(
                (url, {'current': None, 'ping': None, 'ocsp_response': None})
                for url in urls
            )
        )
        for issuer, urls in ocsp_reports.items()
    )

    for issuer, urls in test_results.items():
        for url, results in urls.items():
            results['timestamp'] = datetime.datetime.utcnow().timestamp()
            # check if current
            results['current'] = server_query.is_ocsp_url_current_for_issuer(issuer, url)
            # run ping test
            parse_result = urllib.parse.urlparse(url)
            results['ping'] = server_query.ping(parse_result.netloc)

            # run OCSP response test
            # TODO: cache this for the validity time of subject_cert or 7 days, whichever is smaller
            certs = server_query.get_certs_for_issuer_and_url(issuer, url)

            if certs is None:
                results['ocsp_response'] = False
            else:
                subject_cert, issuer_cert = certs
                results['ocsp_response'] = check_ocsp_response(subject_cert, issuer_cert, url)

    if o:
        print(json.dumps(test_results, indent=2))
    else:
        for issuer, urls in test_results.items():
            print(issuer)
            for url, results in urls.items():
                print(
                    f'>>> {url}: {"." if results["current"] else "X"}{"." if results["ping"] else "X"}{"." if results["ocsp_response"] else "X"}')

@main.command()
def web():
    create_application().run()

@main.command()
@click.option('-n', default=2, type=int, help='Number of top authorities')
@click.option('--connection')
@click.option('-v', is_flag=True, help='Verbose output')
def update(n, connection, v):
    if v:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)
    m = Manager(connection=connection)
    user = m.get_or_create_user('test')
    m.update(user, n=n)

if __name__ == '__main__':
    main()
<commit_msg>Update CLI<commit_after>"""The CLI module for OCSPdash."""
import datetime
import json
import logging
import os
import urllib.parse
from collections import OrderedDict

import click

from .server_query import ServerQuery, check_ocsp_response
from .web.app import create_application
from .web.manager import Manager

@click.group()
def main():
    """Run OCSP Dashboard"""

@main.command()
@click.option('-n', default=2, type=int, help='Number of top authorities')
@click.option('-o', is_flag=True, help='Output as JSON')
@click.option('-v', is_flag=True, help='Verbose output')
def run(n, o, v):
    if v:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)

    server_query = ServerQuery(os.environ.get('UID'), os.environ.get('SECRET'))

    issuers = server_query.get_top_authorities(n)  # TODO: cache this result for 24 hours

    ocsp_reports = OrderedDict(  # TODO: cache this result for 24 hours
        (issuer, server_query.get_ocsp_urls_for_issuer(issuer))
        for issuer in issuers.keys()
    )

    test_results = OrderedDict(
        (
            issuer,
            OrderedDict(
                (url, {'current': None, 'ping': None, 'ocsp_response': None})
                for url in urls
            )
        )
        for issuer, urls in ocsp_reports.items()
    )

    for issuer, urls in test_results.items():
        for url, results in urls.items():
            results['timestamp'] = datetime.datetime.utcnow().timestamp()
            # check if current
            results['current'] = server_query.is_ocsp_url_current_for_issuer(issuer, url)
            # run ping test
            parse_result = urllib.parse.urlparse(url)
            results['ping'] = server_query.ping(parse_result.netloc)

            # run OCSP response test
            # TODO: cache this for the validity time of subject_cert or 7 days, whichever is smaller
            certs = server_query.get_certs_for_issuer_and_url(issuer, url)

            if certs is None:
                results['ocsp_response'] = False
            else:
                subject_cert, issuer_cert = certs
                results['ocsp_response'] = check_ocsp_response(subject_cert, issuer_cert, url)

    if o:
        print(json.dumps(test_results, indent=2))
    else:
        for issuer, urls in test_results.items():
            print(issuer)
            for url, results in urls.items():
                print(
                    f'>>> {url}: {"." if results["current"] else "X"}{"." if results["ping"] else "X"}{"." if results["ocsp_response"] else "X"}')

@main.command()
@click.option('--host', default='0.0.0.0', help='Flask host. Defaults to localhost')
@click.option('--port', type=int, help='Flask port. Defaults to 5000')
@click.option('--flask-debug', is_flag=True)
@click.option('-v', '--verbose', is_flag=True, help='Verbose output')
def web(host, port, flask_debug, verbose):
    logging.basicConfig(level=(logging.DEBUG if verbose else logging.INFO))

    create_application().run(host=host, port=port, debug=flask_debug)

@main.command()
@click.option('-n', default=2, type=int, help='Number of top authorities')
@click.option('--connection')
@click.option('-v', '--verbose', is_flag=True, help='Verbose output')
def update(n, connection, verbose):
    logging.basicConfig(level=(logging.DEBUG if verbose else logging.INFO))

    m = Manager(connection=connection)
    user = m.get_or_create_user('test')
    m.update(user, n=n)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

#import remote_machine
import datetime
import socketserver
import struct
import threading
import signal
import pickle
import sys
import os
import socket
import Global

Address = Global.Address
VERSION = Global.VERSION
MAX_BLOCK_LEN = Global.MAX_BLOCK_LEN
PICKLE_PROTOCOL = Global.PICKLE_PROTOCOL
WAIT_TIME = Global.WAIT_TIME
SET_STRUCT_PARAM = Global.SET_STRUCT_PARAM
STATUS = Global.STATUS
PROPERTIES = Global.PROPERTIES
CONFIGFOLDER  = os.path.join(os.path.split(
                             os.path.split(Global.__file__)[0])[0],
                             '.Configs')
IDGEN_FILENAME   = 'last_id'
QUEUE_FILENAME   = 'Queue'
CONFIGS_FILENAME = 'clients_configs'
SUBMITTED_FILENAME = 'submitted_jobs.txt'

class ManageJobsServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
    pass

class Finish(Exception): pass

class RequestHandler(socketserver.StreamRequestHandler):

    ConfigsLock = threading.Lock()
    QueueLock   = threading.Lock()
    CallLock    = threading.Lock()
    IdGenLock   = threading.Lock()

    Configs = None
    IdGen   = None
    Queue   = None
    Call = dict(
        GIME_JOBS=(
            lambda self, *args: self.send_job_from_queue(*args)),
        GIME_RESULTS=(
            lambda self, *args: self.send_results_to_host(*args)),
        NEW_JOB=(
            lambda self, *args: self.add_new_job_to_queue(*args)),
        UPDATE_JOBS=(
            lambda self, *args: self.update_jobs_in_queue(*args)),
        CHANGE_JOBS_REQUEST=(
            lambda self, *args: self.change_jobs_request(*args)),
        SIGNAL_JOBS=(
            lambda self, *args: self.send_signal_jobs(*args)),
        STATUS_QUEUE=(
            lambda self, *args: self.print_queue_status(*args)),
        GIME_CONFIGS=(
            lambda self, *args: self.send_configs_to_client(*args)),
        SET_CONFIGS=(
            lambda self, *args: self.set_configs_of_clients(*args)),
        GET_CONFIGS=(
            lambda self, *args: self.get_configs_of_clients(*args)),
        SHUTDOWN=(
            lambda self, *args: self.shutdown(*args)),
        GOODBYE=(
            lambda self, *args: self.client_shutdown(*args))
        )

    def handle(self):
        InfoStruct = struct.Struct(SET_STRUCT_PARAM)
        info = self.rfile.read(InfoStruct.size)
        size, version = InfoStruct.unpack(info)
        data = pickle.loads(self.rfile.read(size))
        if version != VERSION:
            reply = (False, 'client is incompatible')
        else:
            try:
                with self.CallLock:
                    function = self.Call[data[0]]
                reply = function(self, *data[1:])
            except Finish:
                return
        data = pickle.dumps(reply, PICKLE_PROTOCOL)
        self.wfile.write(InfoStruct.pack(len(data), VERSION))
        self.wfile.write(data)

    def add_new_job_to_queue(self, job):
        clientName = get_client_name(self)
        cls = self.__class__
        with cls.IdGenLock, cls.QueueLock:
            jobid = cls.IdGen
            cls.IdGen += 1
            job.creation_date = datetime.datetime.now()
            job.hostname = clientName
            cls.Queue.update({jobid : job})
        save_id()
        return (True, jobid)

    def send_results_to_host(self):
        clientName = get_client_name(self)
        with self.QueueLock:
            ResQueue = self.Queue.SelAttrVal(attr='status_key',
                                             value={'e','t'})
        EnvQueue = ResQueue.SelAttrVal(attr='hostname',value={clientName})
        if EnvQueue:
            with self.QueueLock:
                for k in EnvQueue.keys():
                    self.Queue.pop(k)
            return (True, EnvQueue)
        return (False, None)

    def send_job_from_queue(self,jobs2send):
        QueuedJobs = Global.JobQueue()
        Jobs2Send = Global.JobQueue()
        clientName = get_client_name(self)
        with self.QueueLock:
            QueuedJobs.update(self.Queue.SelAttrVal(attr='status_key',
                                                    value={'q'}))
            if not len(QueuedJobs): return (False, None)
            for k,v in QueuedJobs.items():
                if v.possiblehosts == 'all' or clientName in v.possiblehosts:
                    v.status_key = 's'
                    v.runninghost = get_client_name(self)
                    Jobs2Send.update({k:v})
                    self.Queue.update({k:v})
                    if len(Jobs2Send) >= jobs2send: break

        #Write file with job information:
        ordem = ['creation_date','runninghost','description']
        data = ''
        for k,v in Jobs2Send.items():
            data += '{:^7d}'.format(k)
            for at in ordem:
                data += PROPERTIES[at][0].format(PROPERTIES[at][2](getattr(v,at)))
            data += '\n'
        try:
            name = os.path.join(CONFIGFOLDER,SUBMITTED_FILENAME)
            with open(name,mode='a') as fh:
                fh.write(data)
        except (TypeError, IOError, OSError) as err:
            print('Problem with file {0}:\n'.format(name),err)

        #Return the Queue
        return (True, Jobs2Send)

    def update_jobs_in_queue(self, ItsQueue):
        keys2remove = []

        with self.QueueLock:
            for k, v in ItsQueue.items():
                if (not isinstance(v,Global.JobView)) and v.status_key in {'e','t','q'}:
                    keys2remove.append(k)
                    self.Queue.update({k:v}) # not a jobview
                elif isinstance(v,Global.JobView):
                    if k in self.Queue:
                        self.Queue[k].update(v)# jobview
                else:
                    self.Queue.update({k:v})# not a jobview

        return (True, keys2remove)

    def change_jobs_request(self,ChanQueue):
        keyschanged = set()
        with self.QueueLock:
            ChanQueueNoRun = ChanQueue.SelAttrVal(attr='runninghost',
                                                  value={None})
            for k,v in ChanQueueNoRun.items():
                vl = self.Queue.get(k)
                if not vl or vl.runninghost:
                    continue
                keyschanged.add(k)
                self.Queue[k].update(v) # v is a jobview!
                if v.status_key == 'tu':
                    self.Queue.pop(k)

            keys2change = set(ChanQueue.keys()) - set(ChanQueueNoRun.keys())
            keyssched2change = set()
            for k in keys2change:
                if self.Queue[k].status_key in {'t','e','tu', 'qu','q'}:
                    continue
                self.Queue[k].update(ChanQueue[k]) # jobview!!!
                keyssched2change.add(k)
            return (True, keyschanged, keyssched2change)

    def print_queue_status(self, onlymine = False):
        if onlymine:
            Queue2Send = self.Queue.SelAttrVal(attr='runninghost',
                                               value={get_client_name(self)})
        else:
            Queue2Send = Global.JobQueue(self.Queue)

        for k, v in Queue2Send.items():
            Queue2Send.update({k:Global.JobView(v)})

        return (True, Queue2Send)

    def send_configs_to_client(self, ItsConfigs):
        clientName = get_client_name(self)
        with self.ConfigsLock:
            if clientName in self.Configs.keys():
                self.Configs[clientName].numcpus = ItsConfigs.numcpus
                self.Configs[clientName].totalJobs = ItsConfigs.totalJobs
                self.Configs[clientName].running = ItsConfigs.running
                self.Configs[clientName].active = 'on'
                self.Configs[clientName].last_contact = datetime.datetime.now()
                return (True, self.Configs[clientName])

            self.Configs.update({clientName:ItsConfigs})
            self.Configs[clientName].active = 'on'
            self.Configs[clientName].last_contact = datetime.datetime.now()
            return (False, True)

    def get_configs_of_clients(self, clients):
        if clients == 'all':
            clients = tuple(self.Configs.keys())
        elif clients == 'this':
            clients = (get_client_name(self),)

        Configs2Send = {}
        with self.ConfigsLock:
            for clientName in clients:
                if clientName in self.Configs.keys():
                    if (self.Configs[clientName].active == 'on' and
                        3*WAIT_TIME < datetime.datetime.now().timestamp() -
                        self.Configs[clientName].last_contact.timestamp()):
                        self.Configs[clientName]. active = 'dead'
                    ClientConfigs = self.Configs[clientName]
                    Configs2Send.update({clientName: ClientConfigs})
            return (True, Configs2Send)

    def set_configs_of_clients(self, NewConfigs, RmClie):
        clients = tuple(NewConfigs.keys() - self.Configs.keys())
        if clients:
            return (False, clients)
        with self.ConfigsLock:
            self.Configs.update(NewConfigs)
            for client in RmClie:
                self.Configs.pop(client)
        return (True, None)

    def shutdown(self):
        with self.ConfigsLock:
            for k in self.Configs:
                self.Configs[k].active= 'off'
            self.server.shutdown()
        raise Finish()

    def client_shutdown(self,down):
        clientName = get_client_name(self)
        with self.ConfigsLock:
            if down:
                self.Configs[clientName].active = 'off'
                self.Configs[clientName].shutdown = False
                self.Configs[clientName].MoreJobs = True
            else:
                self.Configs[clientName].active = 'paused'
        return True

def get_client_name(client):
    clientName = socket.gethostbyaddr(client.client_address[0])[0]
    if clientName == 'localhost':
        clientName = socket.gethostname()
    return clientName

def load_existing_Queue():
    name   = os.path.join(CONFIGFOLDER,QUEUE_FILENAME)
    data = Global.load_file(name=name, ignore = True)
    if data and data[1]: return eval(data[1])
    return

def load_last_id():
    name   = os.path.join(CONFIGFOLDER,IDGEN_FILENAME)
    data = Global.load_file(name=name, ignore = True)
    if data and data[1]: return eval(data[1])

def load_existing_Configs():
    name   = os.path.join(CONFIGFOLDER,CONFIGS_FILENAME)
    data = Global.load_file(name=name, ignore = True)
    if data and data[1]: return eval(data[1])

def save_Configs():
    if not os.path.isdir(CONFIGFOLDER):
        os.mkdir(path=CONFIGFOLDER)
    configname = os.path.join(CONFIGFOLDER,CONFIGS_FILENAME)
    Global.createfile(name=configname, data=repr(RequestHandler.Configs))

def save_Queue():
    if not os.path.isdir(CONFIGFOLDER):
        os.mkdir(path=CONFIGFOLDER)
    queuename   = os.path.join(CONFIGFOLDER,QUEUE_FILENAME)
    Global.createfile(name=queuename, data=repr(RequestHandler.Queue))

def save_id():
    if not os.path.isdir(CONFIGFOLDER):
        os.mkdir(path=CONFIGFOLDER)
    idgenname   = os.path.join(CONFIGFOLDER,IDGEN_FILENAME)
    Global.createfile(name=idgenname, data=repr(RequestHandler.IdGen))

def save_all():
    if not os.path.isdir(CONFIGFOLDER):
        os.mkdir(path=CONFIGFOLDER)
    idgenname   = os.path.join(CONFIGFOLDER,IDGEN_FILENAME)
    queuename   = os.path.join(CONFIGFOLDER,QUEUE_FILENAME)
    configname = os.path.join(CONFIGFOLDER,CONFIGS_FILENAME)
    Global.createfile(name=idgenname, data=repr(RequestHandler.IdGen))
    Global.createfile(name=queuename, data=repr(RequestHandler.Queue))
    Global.createfile(name=configname, data=repr(RequestHandler.Configs))

def signal_handler(signal, frame):
    save_all()
    sys.exit()

def main():
    RequestHandler.Queue = load_existing_Queue() or Global.JobQueue()
    RequestHandler.IdGen = load_last_id() or int(1)
    RequestHandler.Configs = load_existing_Configs() or dict()
    server = None
    try:
        server = ManageJobsServer(("", Address[1]), RequestHandler)
        server.serve_forever()
    except Exception as err:
        print("ERROR", err)
    finally:
        if server is not None:
            server.shutdown()
        save_all()

if __name__ == '__main__':
    signal.signal(signal.SIGTERM, signal_handler)
    main()
<commit_msg>bug correction.<commit_after>#!/usr/bin/env python3

#import remote_machine
import datetime
import socketserver
import struct
import threading
import signal
import pickle
import sys
import os
import socket
import Global

Address = Global.Address
VERSION = Global.VERSION
MAX_BLOCK_LEN = Global.MAX_BLOCK_LEN
PICKLE_PROTOCOL = Global.PICKLE_PROTOCOL
WAIT_TIME = Global.WAIT_TIME
SET_STRUCT_PARAM = Global.SET_STRUCT_PARAM
STATUS = Global.STATUS
PROPERTIES = Global.PROPERTIES
CONFIGFOLDER  = os.path.join(os.path.split(
                             os.path.split(Global.__file__)[0])[0],
                             '.Configs')
IDGEN_FILENAME   = 'last_id'
QUEUE_FILENAME   = 'Queue'
CONFIGS_FILENAME = 'clients_configs'
SUBMITTED_FILENAME = 'submitted_jobs.txt'

class ManageJobsServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
    pass

class Finish(Exception): pass

class RequestHandler(socketserver.StreamRequestHandler):

    ConfigsLock = threading.Lock()
    QueueLock   = threading.Lock()
    CallLock    = threading.Lock()
    IdGenLock   = threading.Lock()

    Configs = None
    IdGen   = None
    Queue   = None
    Call = dict(
        GIME_JOBS=(
            lambda self, *args: self.send_job_from_queue(*args)),
        GIME_RESULTS=(
            lambda self, *args: self.send_results_to_host(*args)),
        NEW_JOB=(
            lambda self, *args: self.add_new_job_to_queue(*args)),
        UPDATE_JOBS=(
            lambda self, *args: self.update_jobs_in_queue(*args)),
        CHANGE_JOBS_REQUEST=(
            lambda self, *args: self.change_jobs_request(*args)),
        SIGNAL_JOBS=(
            lambda self, *args: self.send_signal_jobs(*args)),
        STATUS_QUEUE=(
            lambda self, *args: self.print_queue_status(*args)),
        GIME_CONFIGS=(
            lambda self, *args: self.send_configs_to_client(*args)),
        SET_CONFIGS=(
            lambda self, *args: self.set_configs_of_clients(*args)),
        GET_CONFIGS=(
            lambda self, *args: self.get_configs_of_clients(*args)),
        SHUTDOWN=(
            lambda self, *args: self.shutdown(*args)),
        GOODBYE=(
            lambda self, *args: self.client_shutdown(*args))
        )

    def handle(self):
        InfoStruct = struct.Struct(SET_STRUCT_PARAM)
        info = self.rfile.read(InfoStruct.size)
        size, version = InfoStruct.unpack(info)
        data = pickle.loads(self.rfile.read(size))
        if version != VERSION:
            reply = (False, 'client is incompatible')
        else:
            try:
                with self.CallLock:
                    function = self.Call[data[0]]
                reply = function(self, *data[1:])
            except Finish:
                return
        data = pickle.dumps(reply, PICKLE_PROTOCOL)
        self.wfile.write(InfoStruct.pack(len(data), VERSION))
        self.wfile.write(data)

    def add_new_job_to_queue(self, job):
        clientName = get_client_name(self)
        cls = self.__class__
        with cls.IdGenLock, cls.QueueLock:
            jobid = cls.IdGen
            cls.IdGen += 1
            job.creation_date = datetime.datetime.now()
            job.hostname = clientName
            cls.Queue.update({jobid : job})
        save_id()
        return (True, jobid)

    def send_results_to_host(self):
        clientName = get_client_name(self)
        with self.QueueLock:
            ResQueue = self.Queue.SelAttrVal(attr='status_key',
                                             value={'e','t'})
        EnvQueue = ResQueue.SelAttrVal(attr='hostname',value={clientName})
        if EnvQueue:
            with self.QueueLock:
                for k in EnvQueue.keys():
                    self.Queue.pop(k)
            return (True, EnvQueue)
        return (False, None)

    def send_job_from_queue(self,jobs2send):
        QueuedJobs = Global.JobQueue()
        Jobs2Send = Global.JobQueue()
        clientName = get_client_name(self)
        with self.QueueLock:
            QueuedJobs.update(self.Queue.SelAttrVal(attr='status_key',
                                                    value={'q'}))
            if not len(QueuedJobs): return (False, None)
            for k,v in QueuedJobs.items():
                if v.possiblehosts == 'all' or clientName in v.possiblehosts:
                    v.status_key = 's'
                    v.runninghost = get_client_name(self)
                    Jobs2Send.update({k:v})
                    self.Queue.update({k:v})
                    if len(Jobs2Send) >= jobs2send: break

        #Write file with job information:
        ordem = ['creation_date','runninghost','description']
        data = ''
        for k,v in Jobs2Send.items():
            data += '{:^7d}'.format(k)
            for at in ordem:
                data += PROPERTIES[at][0].format(PROPERTIES[at][2](getattr(v,at)))
            data += '\n'
        try:
            name = os.path.join(CONFIGFOLDER,SUBMITTED_FILENAME)
            with open(name,mode='a') as fh:
                fh.write(data)
        except (TypeError, IOError, OSError) as err:
            print('Problem with file {0}:\n'.format(name),err)

        #Return the Queue
        return (True, Jobs2Send)

    def update_jobs_in_queue(self, ItsQueue):
        keys2remove = []

        with self.QueueLock:
            for k, v in ItsQueue.items():
                if (not isinstance(v,Global.JobView)) and v.status_key in {'e','t','q'}:
                    keys2remove.append(k)
                    self.Queue.update({k:v}) # not a jobview
                elif isinstance(v,Global.JobView):
                    if k in self.Queue:
                        self.Queue[k].update(v)# jobview
                    else:
                        jo = Global.Jobs() # turn JobView in a Jobs object
                        jo.update(v)
                        self.Queue.update({k:jo})
                else:
                    self.Queue.update({k:v})# not a jobview

        return (True, keys2remove)

    def change_jobs_request(self,ChanQueue):
        keyschanged = set()
        with self.QueueLock:
            ChanQueueNoRun = ChanQueue.SelAttrVal(attr='runninghost',
                                                  value={None})
            for k,v in ChanQueueNoRun.items():
                vl = self.Queue.get(k)
                if not vl or vl.runninghost:
                    continue
                keyschanged.add(k)
                self.Queue[k].update(v) # v is a jobview!
                if v.status_key == 'tu':
                    self.Queue.pop(k)

            keys2change = set(ChanQueue.keys()) - set(ChanQueueNoRun.keys())
            keyssched2change = set()
            for k in keys2change:
                if self.Queue[k].status_key in {'t','e','tu', 'qu','q'}:
                    continue
                self.Queue[k].update(ChanQueue[k]) # jobview!!!
                keyssched2change.add(k)
            return (True, keyschanged, keyssched2change)

    def print_queue_status(self, onlymine = False):
        if onlymine:
            Queue2Send = self.Queue.SelAttrVal(attr='runninghost',
                                               value={get_client_name(self)})
        else:
            Queue2Send = Global.JobQueue(self.Queue)

        for k, v in Queue2Send.items():
            Queue2Send.update({k:Global.JobView(v)})

        return (True, Queue2Send)

    def send_configs_to_client(self, ItsConfigs):
        clientName = get_client_name(self)
        with self.ConfigsLock:
            if clientName in self.Configs.keys():
                self.Configs[clientName].numcpus = ItsConfigs.numcpus
                self.Configs[clientName].totalJobs = ItsConfigs.totalJobs
                self.Configs[clientName].running = ItsConfigs.running
                self.Configs[clientName].active = 'on'
                self.Configs[clientName].last_contact = datetime.datetime.now()
                return (True, self.Configs[clientName])

            self.Configs.update({clientName:ItsConfigs})
            self.Configs[clientName].active = 'on'
            self.Configs[clientName].last_contact = datetime.datetime.now()
            return (False, True)

    def get_configs_of_clients(self, clients):
        if clients == 'all':
            clients = tuple(self.Configs.keys())
        elif clients == 'this':
            clients = (get_client_name(self),)

        Configs2Send = {}
        with self.ConfigsLock:
            for clientName in clients:
                if clientName in self.Configs.keys():
                    if (self.Configs[clientName].active == 'on' and
                        3*WAIT_TIME < datetime.datetime.now().timestamp() -
                        self.Configs[clientName].last_contact.timestamp()):
                        self.Configs[clientName]. active = 'dead'
                    ClientConfigs = self.Configs[clientName]
                    Configs2Send.update({clientName: ClientConfigs})
            return (True, Configs2Send)

    def set_configs_of_clients(self, NewConfigs, RmClie):
        clients = tuple(NewConfigs.keys() - self.Configs.keys())
        if clients:
            return (False, clients)
        with self.ConfigsLock:
            self.Configs.update(NewConfigs)
            for client in RmClie:
                self.Configs.pop(client)
        return (True, None)

    def shutdown(self):
        with self.ConfigsLock:
            for k in self.Configs:
                self.Configs[k].active= 'off'
            self.server.shutdown()
        raise Finish()

    def client_shutdown(self,down):
        clientName = get_client_name(self)
        with self.ConfigsLock:
            if down:
                self.Configs[clientName].active = 'off'
                self.Configs[clientName].shutdown = False
                self.Configs[clientName].MoreJobs = True
            else:
                self.Configs[clientName].active = 'paused'
        return True

def get_client_name(client):
    clientName = socket.gethostbyaddr(client.client_address[0])[0]
    if clientName == 'localhost':
        clientName = socket.gethostname()
    return clientName

def load_existing_Queue():
    name   = os.path.join(CONFIGFOLDER,QUEUE_FILENAME)
    data = Global.load_file(name=name, ignore = True)
    if data and data[1]: return eval(data[1])
    return

def load_last_id():
    name   = os.path.join(CONFIGFOLDER,IDGEN_FILENAME)
    data = Global.load_file(name=name, ignore = True)
    if data and data[1]: return eval(data[1])

def load_existing_Configs():
    name   = os.path.join(CONFIGFOLDER,CONFIGS_FILENAME)
    data = Global.load_file(name=name, ignore = True)
    if data and data[1]: return eval(data[1])

def save_Configs():
    if not os.path.isdir(CONFIGFOLDER):
        os.mkdir(path=CONFIGFOLDER)
    configname = os.path.join(CONFIGFOLDER,CONFIGS_FILENAME)
    Global.createfile(name=configname, data=repr(RequestHandler.Configs))

def save_Queue():
    if not os.path.isdir(CONFIGFOLDER):
        os.mkdir(path=CONFIGFOLDER)
    queuename   = os.path.join(CONFIGFOLDER,QUEUE_FILENAME)
    Global.createfile(name=queuename, data=repr(RequestHandler.Queue))

def save_id():
    if not os.path.isdir(CONFIGFOLDER):
        os.mkdir(path=CONFIGFOLDER)
    idgenname   = os.path.join(CONFIGFOLDER,IDGEN_FILENAME)
    Global.createfile(name=idgenname, data=repr(RequestHandler.IdGen))

def save_all():
    if not os.path.isdir(CONFIGFOLDER):
        os.mkdir(path=CONFIGFOLDER)
    idgenname   = os.path.join(CONFIGFOLDER,IDGEN_FILENAME)
    queuename   = os.path.join(CONFIGFOLDER,QUEUE_FILENAME)
    configname = os.path.join(CONFIGFOLDER,CONFIGS_FILENAME)
    Global.createfile(name=idgenname, data=repr(RequestHandler.IdGen))
    Global.createfile(name=queuename, data=repr(RequestHandler.Queue))
    Global.createfile(name=configname, data=repr(RequestHandler.Configs))

def signal_handler(signal, frame):
    save_all()
    sys.exit()

def main():
    RequestHandler.Queue = load_existing_Queue() or Global.JobQueue()
    RequestHandler.IdGen = load_last_id() or int(1)
    RequestHandler.Configs = load_existing_Configs() or dict()
    server = None
    try:
        server = ManageJobsServer(("", Address[1]), RequestHandler)
        server.serve_forever()
    except Exception as err:
        print("ERROR", err)
    finally:
        if server is not None:
            server.shutdown()
        save_all()

if __name__ == '__main__':
    signal.signal(signal.SIGTERM, signal_handler)
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# libavg - Media Playback Engine.
# Copyright (C) 2003-2011 Ulrich von Zadow
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
# Current versions can be found at www.libavg.de
# Original author of this file is Sebastian Maulbeck
# <sm (at) archimedes-solutions (dot) de>

"""
this class provides a test to emulate one or two TOUCH/TRACK events.
by pressing "ctrl left/right" the TOUCH events will be switched into TRACK events and the
other way around.
by pressing "shift left/right" a second event is created whenever the mousebutton (left)
is clicked.

Note: remove any mouse event handling from your application to avoid emulation issues

For example:

1) avg.MOUSE | avg.TOUCH filters can be safely reduced to avg.TOUCH
2) the current libavg button module breaks the emulation
3) the current libavg Grabbable class breaks the emulation, unless is created with
source = avg.TOUCH parameter
"""

from libavg import avg, Point2D

g_Player = avg.Player.get()

class MTemu(object):

    mouseState = 'Up'
    cursorID = 0
    lastCursorPos = None
    dualTouch = False
    secondTouch = False
    source = avg.TOUCH

    def __init__(self):
        self.__rootNode = g_Player.getRootNode()
        self.__rootNode.connectEventHandler(avg.CURSORUP, avg.MOUSE,
                                            self, self.__onMouseUp)
        self.__rootNode.connectEventHandler(avg.CURSORDOWN, avg.MOUSE,
                                            self, self.__onMouseDown)
        self.__rootNode.connectEventHandler(avg.CURSORMOTION, avg.MOUSE,
                                            self, self.__onMouseMotion)
        posX = self.__rootNode.size.x * 2/5
        posY = self.__rootNode.size.y-40

        self.__layer = avg.WordsNode(text='Multitouch emulation active',
                                     pos=(posX, posY),
                                     color='DDDDDD',
                                     sensitive=False,
                                     fontsize=20,
                                     parent=self.__rootNode)

    def deinit(self):
        self.__rootNode.disconnectEventHandler(self)
        self.__rootNode = None
        self.__layer.unlink()
        if self.mouseState == 'Down':
            self._releaseTouch(self.cursorID)
            if self.secondTouch:
                self.releaseTouch(self.cursorID+1)

    def __onMouseDown(self, event):
        if self.mouseState == 'Up' and event.button == 1:
            self._sendFakeTouch(self.cursorID, event.pos, event.type)
            if self.dualTouch and not self.secondTouch:
                self._sendFakeTouch(self.cursorID+1, event.pos, event.type,
                                    True)
                self.secondTouch = True
            self.mouseState = 'Down'
            self.lastCursorPos = event.pos

    def __onMouseMotion(self, event):
        if self.mouseState == 'Down':
            self._sendFakeTouch(self.cursorID, event.pos, event.type)
            if self.dualTouch and self.secondTouch:
                self._sendFakeTouch(self.cursorID+1, event.pos,
                                    event.type, True)
            self.lastCursorPos = event.pos

    def __onMouseUp(self, event):
        if self.mouseState == 'Down' and event.button == 1:
            self._sendFakeTouch(self.cursorID, event.pos, event.type)
            if self.dualTouch and self.secondTouch:
                self._sendFakeTouch(self.cursorID+1, event.pos,
                                    event.type, True)
                self.secondTouch = False
            self.mouseState = 'Up'
            self.cursorID += 2 #Even for left uneven for right touch
            self.lastCursorPos = None

    def toggleSource(self):
        """
        Switch between avg.TOUC and avg.TRACK - source
        """
        if self.mouseState == 'Down':
            self._releaseTouch(self.cursorID)
            if self.secondTouch:
                self._releaseTouch(self.cursorID+1)
            self.mouseState = 'Up'
            self.secondTouch = False
        self.source = avg.TOUCH if self.source == avg.TRACK else avg.TRACK

    def toggleDualTouch(self):
        self.dualTouch = not(self.dualTouch)
        if self.mouseState == 'Down':
            if self.secondTouch:
                self._releaseTouch(self.cursorID+1)
            else:
                self._sendFakeTouch(self.cursorID+1, Point2D(0,0),
                                   avg.CURSORDOWN, mirror=True)
            self.secondTouch = not(self.secondTouch)

    def _sendFakeTouch(self, cursorID, pos, touchType, mirror=False):
        if mirror:
            pos = pos + Point2D(-20,-20)
            pos = self.__rootNode.size - Point2D(20,20) - pos
            pos = self.__clampPos(pos)
        if not self.lastCursorPos:
            self.lastCursorPos = self.__clampPos(pos)
        g_Player.getTestHelper().fakeTouchEvent(cursorID,
                                    touchType, self.source, pos,
                                    Point2D(0,0))
    def _releaseTouch(self, cursorID):
       self._sendFakeTouch(cursorID, Point2D(0,0),
                                   avg.CURSORUP)

    def __clampPos(self, pos):
        if pos[0] < 0:
            pos[0] = 0
        if pos[1] < 0:
            pos[1] = 0
        if pos[0] >= g_Player.getRootNode().size[0]:
            pos[0] = g_Player.getRootNode().size[0]-1
        if pos[1] >= g_Player.getRootNode().size[1]:
            pos[1] = g_Player.getRootNode().size[1]-1
        return pos

<commit_msg>mtemu.py: Coding convention adherence<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# libavg - Media Playback Engine.
# Copyright (C) 2003-2011 Ulrich von Zadow
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
# Current versions can be found at www.libavg.de
# Original author of this file is Sebastian Maulbeck
# <sm (at) archimedes-solutions (dot) de>

'''
this class provides a test to emulate one or two TOUCH/TRACK events.
by pressing "ctrl left/right" the TOUCH events will be switched into TRACK events and the
other way around.
by pressing "shift left/right" a second event is created whenever the mousebutton (left)
is clicked.

Note: remove any mouse event handling from your application to avoid emulation issues

For example:

1) avg.MOUSE | avg.TOUCH filters can be safely reduced to avg.TOUCH
2) the current libavg button module breaks the emulation
3) the current libavg Grabbable class breaks the emulation, unless is created with
source = avg.TOUCH parameter
'''

from libavg import avg, Point2D

g_Player = avg.Player.get()

class MTemu(object):

    mouseState = 'Up'
    cursorID = 0
    lastCursorPos = None
    dualTouch = False
    secondTouch = False
    source = avg.TOUCH

    def __init__(self):
        self.__rootNode = g_Player.getRootNode()
        self.__rootNode.connectEventHandler(avg.CURSORUP, avg.MOUSE, self,
                self.__onMouseUp)
        self.__rootNode.connectEventHandler(avg.CURSORDOWN, avg.MOUSE, self,
                self.__onMouseDown)
        self.__rootNode.connectEventHandler(avg.CURSORMOTION, avg.MOUSE, self,
                self.__onMouseMotion)
        posX = self.__rootNode.size.x * 2/5
        posY = self.__rootNode.size.y-40

        self.__layer = avg.WordsNode(text='Multitouch emulation active', pos=(posX, posY),
                color='DDDDDD', sensitive=False, fontsize=20, parent=self.__rootNode)

    def deinit(self):
        self.__rootNode.disconnectEventHandler(self)
        self.__rootNode = None
        self.__layer.unlink()
        if self.mouseState == 'Down':
            self._releaseTouch(self.cursorID)
            if self.secondTouch:
                self.releaseTouch(self.cursorID+1)

    def __onMouseDown(self, event):
        if self.mouseState == 'Up' and event.button == 1:
            self._sendFakeTouch(self.cursorID, event.pos, event.type)
            if self.dualTouch and not self.secondTouch:
                self._sendFakeTouch(self.cursorID+1, event.pos, event.type, True)
                self.secondTouch = True
            self.mouseState = 'Down'
            self.lastCursorPos = event.pos

    def __onMouseMotion(self, event):
        if self.mouseState == 'Down':
            self._sendFakeTouch(self.cursorID, event.pos, event.type)
            if self.dualTouch and self.secondTouch:
                self._sendFakeTouch(self.cursorID+1, event.pos,  event.type, True)
            self.lastCursorPos = event.pos

    def __onMouseUp(self, event):
        if self.mouseState == 'Down' and event.button == 1:
            self._sendFakeTouch(self.cursorID, event.pos, event.type)
            if self.dualTouch and self.secondTouch:
                self._sendFakeTouch(self.cursorID+1, event.pos, event.type, True)
                self.secondTouch = False
            self.mouseState = 'Up'
            self.cursorID += 2 #Even for left uneven for right touch
            self.lastCursorPos = None

    def toggleSource(self):
        '''
        Switch between avg.TOUC and avg.TRACK - source
        '''
        if self.mouseState == 'Down':
            self._releaseTouch(self.cursorID)
            if self.secondTouch:
                self._releaseTouch(self.cursorID+1)
            self.mouseState = 'Up'
            self.secondTouch = False
        self.source = avg.TOUCH if self.source == avg.TRACK else avg.TRACK

    def toggleDualTouch(self):
        self.dualTouch = not(self.dualTouch)
        if self.mouseState == 'Down':
            if self.secondTouch:
                self._releaseTouch(self.cursorID+1)
            else:
                self._sendFakeTouch(self.cursorID+1, Point2D(0,0), avg.CURSORDOWN,
                        mirror=True)
            self.secondTouch = not(self.secondTouch)

    def _sendFakeTouch(self, cursorID, pos, touchType, mirror=False):
        if mirror:
            pos = pos + Point2D(-20,-20)
            pos = self.__rootNode.size - Point2D(20,20) - pos
            pos = self.__clampPos(pos)
        if not self.lastCursorPos:
            self.lastCursorPos = self.__clampPos(pos)
        g_Player.getTestHelper().fakeTouchEvent(cursorID,touchType, self.source, pos,
                Point2D(0,0))

    def _releaseTouch(self, cursorID):
       self._sendFakeTouch(cursorID, Point2D(0,0), avg.CURSORUP)

    def __clampPos(self, pos):
        if pos[0] < 0:
            pos[0] = 0
        if pos[1] < 0:
            pos[1] = 0
        if pos[0] >= g_Player.getRootNode().size[0]:
            pos[0] = g_Player.getRootNode().size[0]-1
        if pos[1] >= g_Player.getRootNode().size[1]:
            pos[1] = g_Player.getRootNode().size[1]-1
        return pos

<|endoftext|>


<commit_before># Always prefer setuptools over distutils
from setuptools import setup, find_packages
# To use a consistent encoding
from codecs import open
from os import path

here = path.abspath(path.dirname(__file__))

# Get the long description from the README file
with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
    long_description = f.read()

print(find_packages(exclude=['contrib', 'docs', 'tests']))
setup(
    name='pipelines',

    # Versions should comply with PEP440.  For a discussion on single-sourcing
    # the version across setup.py and the project code, see
    # https://packaging.python.org/en/latest/single_source_version.html
    version='0.0.1-BETA',

    description='A ',
    long_description=long_description,

    # The project's main homepage.
    url='https://github.com/InformaticsMatters/pipelines/',

    # Author details
    author='Tim Dudgeon',

    # Choose your license
    license='Apache 2.0',

    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
    classifiers=[
        # How mature is this project? Common values are
        #   3 - Alpha
        #   4 - Beta
        #   5 - Production/Stable
        'Development Status :: 4 - Beta',

        # Indicate who your project is intended for
        'Intended Audience :: Developers',
        'Topic :: Software Development :: Build Tools',

        # Pick your license as you wish (should match "license" above)
        'License :: OSI Approved :: MIT License',

        # Specify the Python versions you support here. In particular, ensure
        # that you indicate whether you support Python 2, Python 3 or both.
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.2',
        'Programming Language :: Python :: 3.3',
        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
    ],

    # What does your project relate to?
    keywords='',

    # You can just specify the packages manually here if your project is
    # simple. Or you can use find_packages().
    packages=["pipelines.rdkit","pipelines.utils","pipelines.docking"],

    # Alternatively, if you want to distribute just a my_module.py, uncomment
    # this:
    #   py_modules=["my_module"],

    # List run-time dependencies here.  These will be installed by pip when
    # your project is installed. For an analysis of "install_requires" vs pip's
    # requirements files see:
    # https://packaging.python.org/en/latest/requirements.html
    install_requires=[],

    # List additional groups of dependencies here (e.g. development
    # dependencies). You can install these using the following syntax,
    # for example:
    # $ pip install -e .[dev,test]
    extras_require={
        'dev': ['check-manifest'],
        'test': ['coverage'],
    },

    # If there are data files included in your packages that need to be
    # installed, specify them here.  If using Python 2.6 or less, then these
    # have to be included in MANIFEST.in as well.
#    package_data={
#        'sample': ['package_data.dat'],
#    },

    # Although 'package_data' is the preferred approach, in some case you may
    # need to place data files outside of your packages. See:
    # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # noqa
    # In this case, 'data_file' will be installed into '<sys.prefix>/my_data'
#    data_files=[('my_data', ['data/data_file'])],

    # To provide executable scripts, use entry points in preference to the
    # "scripts" keyword. Entry points provide cross-platform support and allow
    # pip to create the appropriate form of executable for the target platform.
    entry_points={
        'console_scripts': [
            'sample=sample:main',
        ],
    },
)
<commit_msg>Trigger travis build<commit_after># Always prefer setuptools over distutils
from setuptools import setup, find_packages
# To use a consistent encoding
from codecs import open
from os import path

here = path.abspath(path.dirname(__file__))

# Get the long description from the README file
with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
    long_description = f.read()

print(find_packages(exclude=['contrib', 'docs', 'tests']))
setup(
    name='pipelines',

    # Versions should comply with PEP440.  For a discussion on single-sourcing
    # the version across setup.py and the project code, see
    # https://packaging.python.org/en/latest/single_source_version.html
    version='0.0.1-BETA',

    description='A ',
    long_description=long_description,

    # The project's main homepage.
    url='https://github.com/InformaticsMatters/pipelines/',

    # Author details
    author='Tim Dudgeon',

    # Choose your license
    license='Apache 2.0',

    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
    classifiers=[
        # How mature is this project? Common values are
        #   3 - Alpha
        #   4 - Beta
        #   5 - Production/Stable
        'Development Status :: 4 - Beta',

        # Indicate who your project is intended for
        'Intended Audience :: Developers',
        'Topic :: Software Development :: Build Tools',

        # Pick your license as you wish (should match "license" above)
        'License :: OSI Approved :: MIT License',

        # Specify the Python versions you support here. In particular, ensure
        # that you indicate whether you support Python 2, Python 3 or both.
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.2',
        'Programming Language :: Python :: 3.3',
        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
    ],

    # What does your project relate to?
    keywords='',

    # You can just specify the packages manually here if your project is
    # simple. Or you can use find_packages().
    packages=["pipelines.rdkit","pipelines.utils","pipelines.docking"],

    # Alternatively, if you want to distribute just a my_module.py, uncomment
    # this:
    #   py_modules=["my_module"],

    # List run-time dependencies here.  These will be installed by pip when
    # your project is installed. For an analysis of "install_requires" vs pip's
    # requirements files see:
    # https://packaging.python.org/en/latest/requirements.html
    install_requires=[],

    # List additional groups of dependencies here (e.g. development
    # dependencies). You can install these using the following syntax,
    # for example:
    # $ pip install -e .[dev,test]
    extras_require={
        'dev': ['check-manifest'],
        'test': ['coverage'],
    },

    # If there are data files included in your packages that need to be
    # installed, specify them here.  If using Python 2.6 or less, then these
    # have to be included in MANIFEST.in as well.
#    package_data={
#        'sample': ['package_data.dat'],
#    },

    # Although 'package_data' is the preferred approach, in some case you may
    # need to place data files outside of your packages. See:
    # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # noqa
    # In this case, 'data_file' will be installed into '<sys.prefix>/my_data'
#    data_files=[('my_data', ['data/data_file'])],

    # To provide executable scripts, use entry points in preference to the
    # "scripts" keyword. Entry points provide cross-platform support and allow
    # pip to create the appropriate form of executable for the target platform.
    entry_points={
        'console_scripts': [
            'sample=sample:main',
        ],
    },
)
<|endoftext|>


<commit_before>from celery.task.sets import TaskSet
from celery.task import task
from celery.task.sets import subtask
from reader.models import Feed

@task
def fetch_feed(feed_id,callback=None):
    feed = Feed.objects.get(id=feed_id)
    feed.fetch()
    return None

@task
def fetch_feeds(callback=None):
    print 'i am here'
    feeds = Feed.objects.all()
    task_list = []
    for feed in feeds:
        s = fetch_feed.subtask([feed.id])
        task_list.append(s)
    print 'fetching all %s feeds' % (len(task_list))
    fetch_all_tasks = TaskSet(tasks=task_list)
    fetch_all_tasks.apply_async()
    return None<commit_msg>removed debug statement<commit_after>from celery.task.sets import TaskSet
from celery.task import task
from celery.task.sets import subtask
from reader.models import Feed

@task
def fetch_feed(feed_id,callback=None):
    feed = Feed.objects.get(id=feed_id)
    feed.fetch()
    return None

@task
def fetch_feeds(callback=None):
    print 'i am here'
    feeds = Feed.objects.all()
    task_list = []
    for feed in feeds:
        s = fetch_feed.subtask([feed.id])
        task_list.append(s)
    fetch_all_tasks = TaskSet(tasks=task_list)
    fetch_all_tasks.apply_async()
    return None<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
=========================
Base Library (:mod:`ibei`)
=========================

.. currentmodule:: ibei
"""

from uibei import uibei
from sqsolarcell import SQSolarcell
from devossolarcell import DeVosSolarcell

__version__ = "1.0.2"
<commit_msg>Bump patch number<commit_after># -*- coding: utf-8 -*-
"""
=========================
Base Library (:mod:`ibei`)
=========================

.. currentmodule:: ibei
"""

from uibei import uibei
from sqsolarcell import SQSolarcell
from devossolarcell import DeVosSolarcell

__version__ = "1.0.3"
<|endoftext|>


<commit_before>#  Copyright 2008-2015 Nokia Solutions and Networks
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import wx

from robotide.preferences import widgets

class SavingPreferences(widgets.PreferencesPanel):
    location = ('Saving',)
    title = 'Saving Preferences'

    def __init__(self, settings, *args, **kwargs):
        super(SavingPreferences, self).__init__(*args, **kwargs)
        self._settings = settings
        self.SetSizer(wx.FlexGridSizer(rows=7, cols=2))
        for editor in self._create_editors(settings):
            self._add_editor(editor)

    def _create_editors(self, settings):
        return [
            widgets.StringChoiceEditor(settings,
                'default file format',
                'Default file format:',
                ('txt', 'tsv', 'html', 'robot')
            ),
            widgets.StringChoiceEditor(settings,
                'txt format separator',
                'TXT format separator:',
                ('pipe', 'space')
            ),
            widgets.StringChoiceEditor(settings,
                'line separator',
                'Line separator:',
                ('native', 'CRLF', 'LF'),
                'Possible values are native (of current OS) CRLF (Windows) and LF (Unixy)'
            ),
            widgets.IntegerChoiceEditor(settings,
                'txt number of spaces',
                'Separating spaces',
                [str(i) for i in range(2, 11)],
                'Number of spaces between cells when saving in txt format'
            )
        ]

    def _add_editor(self, editor):
        self.Sizer.AddMany([editor.label(self), (editor.chooser(self),)])
        self.Sizer.AddMany([(editor.help(self), 0, wx.BOTTOM, 10),
                            wx.Window(self)])
<commit_msg>preferences: fix usages of FlexGridSizer<commit_after>#  Copyright 2008-2015 Nokia Solutions and Networks
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import wx

from robotide.preferences import widgets

class SavingPreferences(widgets.PreferencesPanel):
    location = ('Saving',)
    title = 'Saving Preferences'

    def __init__(self, settings, *args, **kwargs):
        super(SavingPreferences, self).__init__(*args, **kwargs)
        self._settings = settings
        self.SetSizer(wx.FlexGridSizer(cols=2))
        for editor in self._create_editors(settings):
            self._add_editor(editor)

    def _create_editors(self, settings):
        return [
            widgets.StringChoiceEditor(settings,
                'default file format',
                'Default file format:',
                ('txt', 'tsv', 'html', 'robot')
            ),
            widgets.StringChoiceEditor(settings,
                'txt format separator',
                'TXT format separator:',
                ('pipe', 'space')
            ),
            widgets.StringChoiceEditor(settings,
                'line separator',
                'Line separator:',
                ('native', 'CRLF', 'LF'),
                'Possible values are native (of current OS) CRLF (Windows) and LF (Unixy)'
            ),
            widgets.IntegerChoiceEditor(settings,
                'txt number of spaces',
                'Separating spaces',
                [str(i) for i in range(2, 11)],
                'Number of spaces between cells when saving in txt format'
            )
        ]

    def _add_editor(self, editor):
        self.Sizer.AddMany([editor.label(self), (editor.chooser(self),)])
        self.Sizer.AddMany([(editor.help(self), 0, wx.BOTTOM, 10),
                            wx.Window(self)])
<|endoftext|>