<commit_before># -*- coding: utf-8 -*-
from .util import grouper, parmap
import requests
import json
import urllib
from bs4 import BeautifulSoup

class Api(object):
    """
    """
    ROOT_URL = "https://www.mountainproject.com"
    self.SEARCH_URL = self.ROOT_URL + "/ajax/public/search/results/category"
    DATA_URL = self.ROOT_URL + "/data"
    UP_AREA_IMG = "https://cdn.apstatic.com/mp-img/up.gif"

    def __init__(self, key):
        self.key = key

    def getRoutes(self, routeIds):
        """
        Get all route information for a given set of routeIds
        """
        routes = None
        for route_set in grouper(100, routeIds):
            route_set = filter(None, route_set)
            response = requests.get(
                self.DATA_URL, params={"action": "getRoutes", "key": self.key, "routeIds": ",".join(route_set)})
            if response.ok:
                content = json.loads(response.content)
                if routes:
                    routes["routes"] += content["routes"]
                else:
                    routes = content
            else:
                return None
        return routes

    def _getFAFromRouteHTML(self, soup):
        """
        Extract the First Ascent information for the route.
        """
        try:
            return soup.find("td", text=u"FA:\xA0").nextSibling.text
        except:
            return ""

    def _getParentAreaLink(self, soup):
        """
        Look for a link with the UP_AREA_IMG to get the url to the parent
        """
        return self.ROOT_URL + soup.find(
            "img", {"src": self.UP_AREA_IMG}).parent.get('href')

    def _getGPSCoordinatesFromArea(self, soup):
        """
        Search through a BeautifulSoup interpretation of a webpage looking
        for Location: 	 if that exists then parse the next table element to get
        the GPS in the format LAT, LONG
        """
        gps_coordinates = []
        location_tag = soup.find('td', text=u"Location: ")
        if location_tag:
            # Parse the location tag in the fomat of Lat, Long View
            # Map (Incorrect?). We just want the Lat and
            # Long
            gps_coordinates = location_tag.nextSibling.text.split("View Map")[
                0].strip().split(", ")

        return gps_coordinates

    def _getNearestGPSCoordinateFromRoute(self, soup):
        """
        Search through areas looking for the first one with a GPS tag
        """
        coordinates = []

        while not coordinates:
            response = requests.get(self._getParentAreaLink(soup))
            if response.ok:
                soup = BeautifulSoup(response.content, 'html.parser')
                coordinates = self._getGPSCoordinatesFromArea(soup)
            else:
                raise Exception("Network Connection Failed")

        return coordinates

    def enrichRoute(self, route):
        """
        Scrape MountainProject.com for the first ascent and nearest
        gps data adding that information to the route object.
        """
        response = requests.get(route["url"])
        if response.ok:
            soup = BeautifulSoup(response.content, 'html.parser')
            route["fa"] = self._getFAFromRouteHTML(soup)
            route["gps"] = self._getNearestGPSCoordinateFromRoute(soup)
        return route

    def enrichRoutes(self, routes):
        """
        Run enrich route in parallel for all routes.
        """
        # Enrich all of the routes in parallel
        routes["routes"] = parmap(self.enrichRoute, routes["routes"])
        return routes

    def getToDos(self, startPos, userId):
        """
        Get the users Todo items by userId
        """
        response = requests.get(
            self.DATA_URL, params={"action": "getToDos", "key": self.key, "startPos": startPos, "userId": userId})
        if response.ok:
            return json.loads(response.content)
        else:
            return None

    def getToDosByEmail(self, startPos, email):
        """
        Get the users Todo items by email address
        """
        response = requests.get(
            self.DATA_URL, params={"action": "getToDos", "key": self.key, "startPos": startPos, "email": email})
        if response.ok:
            return json.loads(response.content)
        else:
            return None

    def getTicks(self, startPos, userId):
        """
        Get the users ticks by userId
        """
        response = requests.get(
            self.DATA_URL, params={"action": "getTicks", "key": self.key, "startPos": startPos, "userId": userId})
        if response.ok:
            return json.loads(response.content)
        else:
            return None

    def getTicksByEmail(self, startPos, email):
        """
        Get the users ticks by email
        """
        response = requests.get(
            self.DATA_URL, params={"action": "getTicks", "key": self.key, "startPos": startPos, "email": email})
        if response.ok:
            return json.loads(response.content)
        else:
            return None

    def getUser(self, userId):
        """
        Get the user information by userId
        """
        response = requests.get(
            self.DATA_URL, params={"action": "getUser", "key": self.key, "userId": userId})
        if response.ok:
            return json.loads(response.content)
        else:
            return None

    def getUserByEmail(self, email):
        """
        Get the user information by email address
        """
        response = requests.get(
            self.DATA_URL, params={"action": "getUser", "key": self.key, "email": email})
        if response.ok:
            return json.loads(response.content)
        else:
            return None

    def search(self, query, category, offset, size):
        payload = {"q": urllib.quote_plus(
            query), "c": category, "o": offset, "s": size}
        r = requests.get(self.SEARCH_URL, params=payload)
        if r.ok:
            return json.loads(r.content)
        else:
            raise Exception("Search Failed")

    def search_routes(self, query, offset=0, size=100):
        return self.search(query, "Routes", offset, size)

    def search_routes_for_ids(self, query, offset=0, size=100):
        results = self.search_routes(query, offset, size)
        route_ids = []
        if results["results"]:
            for route_html in results["results"]["Routes"]:
                soup = BeautifulSoup(route_html, 'html.parser')
                link = soup.tr.td.strong.a
                route_ids.append(link.get('href').split("?")[0].split('/')[2])

        return route_ids

    def isearch_all_routes_for_ids(self, query):
        offset = 0
        new_ids = self.search_routes_for_ids(query, offset)
        while len(new_ids) != 0:
            offset += len(new_ids)
            for id in new_ids:
                yield id
            new_ids = self.search_routes_for_ids(query, offset)

    def search_all_routes_for_ids(self, query):
        return list(self.isearch_all_routes_for_ids(query))
<commit_msg>Added convenience functions<commit_after># -*- coding: utf-8 -*-
from .util import grouper, parmap
import requests
import json
import urllib
from bs4 import BeautifulSoup

class Api(object):
    """
    """
    ROOT_URL = "https://www.mountainproject.com"
    SEARCH_URL = ROOT_URL + "/ajax/public/search/results/category"
    DATA_URL = ROOT_URL + "/data"
    UP_AREA_IMG = "https://cdn.apstatic.com/mp-img/up.gif"

    def __init__(self, key):
        self.key = key

    def get_routes(self, routeIds):
        """
        Get all route information for a given set of routeIds
        """
        routes = None
        for route_set in grouper(100, routeIds):
            route_set = filter(None, route_set)
            response = requests.get(
                self.DATA_URL, params={"action": "getRoutes", "key": self.key, "routeIds": ",".join(route_set)})
            if response.ok:
                content = json.loads(response.content)
                if routes:
                    routes["routes"] += content["routes"]
                else:
                    routes = content
            else:
                return None
        return routes

    def _scrape_fa_from_route(self, soup):
        """
        Extract the First Ascent information for the route.
        """
        try:
            return soup.find("td", text=u"FA:\xA0").nextSibling.text
        except:
            return ""

    def _get_parent_area_link(self, soup):
        """
        Look for a link with the UP_AREA_IMG to get the url to the parent
        """
        return self.ROOT_URL + soup.find(
            "img", {"src": self.UP_AREA_IMG}).parent.get('href')

    def _get_gps_from_area(self, soup):
        """
        Search through a BeautifulSoup interpretation of a webpage looking
        for Location: 	 if that exists then parse the next table element to get
        the GPS in the format LAT, LONG
        """
        gps_coordinates = []
        location_tag = soup.find('td', text=u"Location: ")
        if location_tag:
            # Parse the location tag in the fomat of Lat, Long View
            # Map (Incorrect?). We just want the Lat and
            # Long
            gps_coordinates = location_tag.nextSibling.text.split("View Map")[
                0].strip().split(", ")

        return gps_coordinates

    def _get_nearest_gps(self, soup):
        """
        Search through areas looking for the first one with a GPS tag
        """
        coordinates = []

        while not coordinates:
            response = requests.get(self._get_parent_area_link(soup))
            if response.ok:
                soup = BeautifulSoup(response.content, 'html.parser')
                coordinates = self._get_gps_from_area(soup)
            else:
                raise Exception("Network Connection Failed")

        return coordinates

    def enrich_route(self, route):
        """
        Scrape MountainProject.com for the first ascent and nearest
        gps data adding that information to the route object.
        """
        response = requests.get(route["url"])
        if response.ok:
            soup = BeautifulSoup(response.content, 'html.parser')
            route["fa"] = self._scrape_fa_from_route(soup)
            route["gps"] = self._get_nearest_gps(soup)
        return route

    def enrich_routes(self, routes):
        """
        Run enrich route in parallel for all routes.
        """
        # Enrich all of the routes in parallel
        routes["routes"] = parmap(self.enrich_route, routes["routes"])
        return routes

    def get_todos(self, userId, startPos=0):
        """
        Get the users Todo items by userId
        """
        response = requests.get(
            self.DATA_URL, params={"action": "getToDos", "key": self.key, "startPos": startPos, "userId": userId})
        if response.ok:
            return json.loads(response.content)
        else:
            return None

    def get_todos_by_email(self, email, startPos=0):
        """
        Get the users Todo items by email address
        """
        response = requests.get(
            self.DATA_URL, params={"action": "getToDos", "key": self.key, "startPos": startPos, "email": email})
        if response.ok:
            return json.loads(response.content)
        else:
            return None

    def get_ticks(self, userId, startPos=0):
        """
        Get the users ticks by userId
        """
        response = requests.get(
            self.DATA_URL, params={"action": "getTicks", "key": self.key, "startPos": startPos, "userId": userId})
        if response.ok:
            return json.loads(response.content)
        else:
            return None

    def get_ticks_by_email(self, email, startPos=0):
        """
        Get the users ticks by email
        """
        response = requests.get(
            self.DATA_URL, params={"action": "getTicks", "key": self.key, "startPos": startPos, "email": email})
        if response.ok:
            return json.loads(response.content)
        else:
            return None

    def get_user(self, userId):
        """
        Get the user information by userId
        """
        response = requests.get(
            self.DATA_URL, params={"action": "getUser", "key": self.key, "userId": userId})
        if response.ok:
            return json.loads(response.content)
        else:
            return None

    def get_user_by_email(self, email):
        """
        Get the user information by email address
        """
        response = requests.get(
            self.DATA_URL, params={"action": "getUser", "key": self.key, "email": email})
        if response.ok:
            return json.loads(response.content)
        else:
            return None

    def search(self, query, category, offset, size):
        payload = {"q": urllib.quote_plus(
            query), "c": category, "o": offset, "s": size}
        r = requests.get(self.SEARCH_URL, params=payload)
        if r.ok:
            return json.loads(r.content)
        else:
            raise Exception("Search Failed")

    def _search_routes(self, query, offset=0, size=100):
        return self.search(query, "Routes", offset, size)

    def search_routes_for_ids(self, query, offset=0, size=100):
        results = self._search_routes(query, offset, size)
        route_ids = []
        if results["results"]:
            for route_html in results["results"]["Routes"]:
                soup = BeautifulSoup(route_html, 'html.parser')
                link = soup.tr.td.strong.a
                route_ids.append(link.get('href').split("?")[0].split('/')[2])

        return route_ids

    def isearch_all_routes_for_ids(self, query):
        offset = 0
        new_ids = self.search_routes_for_ids(query, offset)
        while len(new_ids) != 0:
            offset += len(new_ids)
            for id in new_ids:
                yield id
            new_ids = self.search_routes_for_ids(query, offset)

    def search_all_routes_for_ids(self, query):
        return list(self.isearch_all_routes_for_ids(query))

    def search_routes(self, query):
        return self.get_routes(self.search_all_routes_for_ids(query))
<|endoftext|>


<commit_before>import builtins
import os
import sys
from io import StringIO
from unittest import mock
import sure
import contexts
from contexts import __main__
from contexts.reporting import cli

class WhenLoadingUpTheModule:
    # this test is just to balance
    # the fact that we're mocking it everywhere else
    def it_should_get_main_from_init(self):
        __main__.main.should.equal(contexts.main)

class MainSharedContext:
    def shared_context(self):
        self.real_main = __main__.main
        self.mock_main = __main__.main = mock.Mock()
        self.real_argv = sys.argv
        self.real_stdout = sys.stdout

        sys.argv = ['run-contexts']
        sys.stdout = mock.Mock()
        sys.stdout.isatty.return_value = True
        os.environ.pop("TEAMCITY_VERSION", None)

    def cleanup_main_and_argv_and_stdout(self):
        __main__.main = self.real_main
        sys.argv = self.real_argv
        sys.stdout = self.real_stdout

class WhenRunningFromCommandLineWithArguments(MainSharedContext):
    @classmethod
    def examples(self):
        yield [], (os.getcwd(), (cli.DotsReporter(sys.stdout), cli.ColouredSummarisingCapturingReporter(sys.stdout), cli.TimedReporter(sys.stdout)), True)
        yield ['-v'], (os.getcwd(), (cli.ColouredVerboseCapturingReporter(sys.stdout),), True)
        yield ['--verbose'], (os.getcwd(), (cli.ColouredVerboseCapturingReporter(sys.stdout),), True)
        yield ['--verbose', '--no-colour'], (os.getcwd(), (cli.StdOutCapturingReporter(sys.stdout),), True)
        yield ['--verbose', '--no-capture'], (os.getcwd(), (cli.ColouredVerboseReporter(sys.stdout),), True)
        yield ['--verbose', '--no-colour', '--no-capture'], (os.getcwd(), (cli.VerboseReporter(sys.stdout),), True)
        yield ['-vs'], (os.getcwd(), (cli.ColouredVerboseReporter(sys.stdout),), True)
        yield ['-q'], (os.getcwd(), (QuietReporterResemblance(),), True)
        yield ['--quiet'], (os.getcwd(), (QuietReporterResemblance(),), True)
        yield ['-s'], (os.getcwd(), (cli.DotsReporter(sys.stdout), cli.ColouredSummarisingReporter(sys.stdout), cli.TimedReporter(sys.stdout)), True)
        yield ['--no-capture'], (os.getcwd(), (cli.DotsReporter(sys.stdout), cli.ColouredSummarisingReporter(sys.stdout), cli.TimedReporter(sys.stdout)), True)
        yield ['--no-colour'], (os.getcwd(), (cli.DotsReporter(sys.stdout), cli.SummarisingCapturingReporter(sys.stdout), cli.TimedReporter(sys.stdout)), True)
        yield ['--no-capture', '--no-colour'], (os.getcwd(), (cli.DotsReporter(sys.stdout), cli.SummarisingReporter(sys.stdout), cli.TimedReporter(sys.stdout)), True)
        yield ['--teamcity'], (os.getcwd(), (contexts.reporting.teamcity.TeamCityReporter(sys.stdout),), True)
        yield ['--no-random'], (os.getcwd(), (cli.DotsReporter(sys.stdout), cli.ColouredSummarisingCapturingReporter(sys.stdout), cli.TimedReporter(sys.stdout)), False)
        yield [os.path.join(os.getcwd(),'made','up','path')], (os.path.join(os.getcwd(),'made','up','path'), (cli.DotsReporter(sys.stdout), cli.ColouredSummarisingCapturingReporter(sys.stdout), cli.TimedReporter(sys.stdout)), True)

    def establish_arguments(self, example):
        argv, self.expected = example
        sys.argv = ['run-contexts'] + argv

    def because_we_call_cmd(self):
        __main__.cmd()

    def it_should_call_main_with_the_correct_arguments(self):
        self.mock_main.assert_called_once_with(*self.expected)

class WhenArgumentsSpecifyMutuallyExclusiveOptions(MainSharedContext):
    @classmethod
    def examples(cls):
        yield ['-q', '-v']
        yield ['-qv']
        yield ['--quiet', '--verbose']
        yield ['-q', '--verbose']
        yield ['--verbose', '-q']

    def establish_that_user_wants_verbose_and_quiet(self, example):
        sys.argv = ['run-contexts'] + example
        self.exception = None

        # argparse will try and print to stderr,
        # we don't want that to happen during the test run
        self.real_stderr = sys.stderr
        sys.stderr = StringIO()

    def because_we_run_from_cmd_line(self):
        try:
            __main__.cmd()
        except BaseException as e:
            self.exception = e

    def it_should_not_call_main(self):
        self.mock_main.mock_calls.should.be.empty

    def it_should_quit(self):
        self.exception.should.be.a('SystemExit')

    def cleanup_stderr_and_argv(self):
        sys.stderr = self.real_stderr

class WhenColoramaIsNotInstalled(MainSharedContext):
    def establish_that_colorama_raises_import_error(self):
        self.real_import = builtins.__import__
        def fake_import(name, *args, **kwargs):
            if name == "colorama":
                raise ImportError
            return self.real_import(name, *args, **kwargs)
        builtins.__import__ = fake_import

    def because_we_run_from_cmd_line(self):
        __main__.cmd()

    def it_should_not_send_a_coloured_reporter_to_main(self):
        self.mock_main.assert_called_once_with(os.getcwd(), (cli.DotsReporter(sys.stdout), cli.SummarisingCapturingReporter(sys.stdout), cli.TimedReporter(sys.stdout)), True)

    def cleanup_import(self):
        builtins.__import__ = self.real_import

class WhenStdOutIsAPipe(MainSharedContext):
    def establish_that_stdout_is_a_pipe(self):
        sys.stdout.isatty.return_value = False

    def because_we_run_from_cmd_line(self):
        __main__.cmd()

    def it_should_not_send_a_coloured_reporter_to_main(self):
        self.mock_main.assert_called_once_with(os.getcwd(), (cli.DotsReporter(sys.stdout), cli.SummarisingCapturingReporter(sys.stdout), cli.TimedReporter(sys.stdout)), True)

###########################################################
# Test helper methods
###########################################################

def args_with(**kwargs):
    defaults = {
        'capture': True,
        'verbosity': 'normal',
        'teamcity': False,
        'shuffle': True,
        'path': os.getcwd(),
        'colour': True
    }
    defaults.update(kwargs)
    return defaults

class QuietReporterResemblance(object):
    """
    Compares equal to quiet reporters
    """
    def __eq__(self, other):
        return type(other) == cli.StdOutCapturingReporter and other.stream != sys.stdout
<commit_msg>Hack to get the tests to run on Windows<commit_after>import builtins
import os
import sys
from io import StringIO
from unittest import mock
import sure
import colorama
import contexts
from contexts import __main__
from contexts.reporting import cli

class WhenLoadingUpTheModule:
    # this test is just to balance
    # the fact that we're mocking it everywhere else
    def it_should_get_main_from_init(self):
        __main__.main.should.equal(contexts.main)

class MainSharedContext:
    def shared_context(self):
        self.real_colorama_init = colorama.init
        colorama.init = mock.Mock()
        self.real_main = __main__.main
        self.mock_main = __main__.main = mock.Mock()
        self.real_argv = sys.argv
        self.real_stdout = sys.stdout

        sys.argv = ['run-contexts']
        sys.stdout = mock.Mock()
        sys.stdout.isatty.return_value = True
        os.environ.pop("TEAMCITY_VERSION", None)

    def cleanup(self):
        __main__.main = self.real_main
        sys.argv = self.real_argv
        sys.stdout = self.real_stdout
        colorama.init = self.real_colorama_init

class WhenRunningFromCommandLineWithArguments(MainSharedContext):
    @classmethod
    def examples(self):
        yield [], [os.getcwd(), (cli.DotsReporter, cli.ColouredSummarisingCapturingReporter, cli.TimedReporter), True]
        yield ['-v'], [os.getcwd(), (cli.ColouredVerboseCapturingReporter,), True]
        yield ['--verbose'], [os.getcwd(), (cli.ColouredVerboseCapturingReporter,), True]
        yield ['--verbose', '--no-colour'], [os.getcwd(), (cli.StdOutCapturingReporter,), True]
        yield ['--verbose', '--no-capture'], [os.getcwd(), (cli.ColouredVerboseReporter,), True]
        yield ['--verbose', '--no-colour', '--no-capture'], [os.getcwd(), (cli.VerboseReporter,), True]
        yield ['-vs'], [os.getcwd(), (cli.ColouredVerboseReporter,), True]
        yield ['-q'], [os.getcwd(), (QuietReporterResemblance,), True]
        yield ['--quiet'], [os.getcwd(), (QuietReporterResemblance,), True]
        yield ['-s'], [os.getcwd(), (cli.DotsReporter, cli.ColouredSummarisingReporter, cli.TimedReporter), True]
        yield ['--no-capture'], [os.getcwd(), (cli.DotsReporter, cli.ColouredSummarisingReporter, cli.TimedReporter), True]
        yield ['--no-colour'], [os.getcwd(), (cli.DotsReporter, cli.SummarisingCapturingReporter, cli.TimedReporter), True]
        yield ['--no-capture', '--no-colour'], [os.getcwd(), (cli.DotsReporter, cli.SummarisingReporter, cli.TimedReporter), True]
        yield ['--teamcity'], [os.getcwd(), (contexts.reporting.teamcity.TeamCityReporter,), True]
        yield ['--no-random'], [os.getcwd(), (cli.DotsReporter, cli.ColouredSummarisingCapturingReporter, cli.TimedReporter), False]
        yield [os.path.join(os.getcwd(),'made','up','path')], [os.path.join(os.getcwd(),'made','up','path'), (cli.DotsReporter, cli.ColouredSummarisingCapturingReporter, cli.TimedReporter), True]

    def establish_arguments(self, example):
        argv, self.expected = example
        self.expected[1] = tuple(cls(sys.stdout) for cls in self.expected[1])
        sys.argv = ['run-contexts'] + argv

    def because_we_call_cmd(self):
        __main__.cmd()

    def it_should_call_main_with_the_correct_arguments(self):
        self.mock_main.assert_called_once_with(*self.expected)

class WhenArgumentsSpecifyMutuallyExclusiveOptions(MainSharedContext):
    @classmethod
    def examples(cls):
        yield ['-q', '-v']
        yield ['-qv']
        yield ['--quiet', '--verbose']
        yield ['-q', '--verbose']
        yield ['--verbose', '-q']

    def establish_that_user_wants_verbose_and_quiet(self, example):
        sys.argv = ['run-contexts'] + example
        self.exception = None

        # argparse will try and print to stderr,
        # we don't want that to happen during the test run
        self.real_stderr = sys.stderr
        sys.stderr = StringIO()

    def because_we_run_from_cmd_line(self):
        try:
            __main__.cmd()
        except BaseException as e:
            self.exception = e

    def it_should_not_call_main(self):
        self.mock_main.mock_calls.should.be.empty

    def it_should_quit(self):
        self.exception.should.be.a('SystemExit')

    def cleanup_stderr_and_argv(self):
        sys.stderr = self.real_stderr

class WhenColoramaIsNotInstalled(MainSharedContext):
    def establish_that_colorama_raises_import_error(self):
        self.real_import = builtins.__import__
        def fake_import(name, *args, **kwargs):
            if name == "colorama":
                raise ImportError
            return self.real_import(name, *args, **kwargs)
        builtins.__import__ = fake_import

    def because_we_run_from_cmd_line(self):
        __main__.cmd()

    def it_should_not_send_a_coloured_reporter_to_main(self):
        self.mock_main.assert_called_once_with(os.getcwd(), (cli.DotsReporter(sys.stdout), cli.SummarisingCapturingReporter(sys.stdout), cli.TimedReporter(sys.stdout)), True)

    def cleanup_import(self):
        builtins.__import__ = self.real_import

class WhenStdOutIsAPipe(MainSharedContext):
    def establish_that_stdout_is_a_pipe(self):
        sys.stdout.isatty.return_value = False

    def because_we_run_from_cmd_line(self):
        __main__.cmd()

    def it_should_not_send_a_coloured_reporter_to_main(self):
        self.mock_main.assert_called_once_with(os.getcwd(), (cli.DotsReporter(sys.stdout), cli.SummarisingCapturingReporter(sys.stdout), cli.TimedReporter(sys.stdout)), True)

###########################################################
# Test helper methods
###########################################################

def args_with(**kwargs):
    defaults = {
        'capture': True,
        'verbosity': 'normal',
        'teamcity': False,
        'shuffle': True,
        'path': os.getcwd(),
        'colour': True
    }
    defaults.update(kwargs)
    return defaults

class QuietReporterResemblance(object):
    """
    Compares equal to quiet reporters
    """
    def __init__(self, not_stream):
        self.not_stream = not_stream
    def __eq__(self, other):
        return type(other) == cli.StdOutCapturingReporter and other.stream != self.not_stream
<|endoftext|>


<commit_before>__license__ = """
Copyright 2016 Parsely, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

__version__ = "2.4.0"

from . import bigquery, docgen, redshift, s3, samples, schema, stream, utils
from six import iteritems

__all__ = [
    "bigquery",
    "docgen",
    "redshift",
    "s3",
    "samples",
    "schema",
    "stream",
    "utils",
]

BOOLEAN_FIELDS = {"flags_is_amp"}

def normalize_keys(input_event_dict, schema=None):
    """Conform events to public schema: correct keys and proper value types.

    @param input_event_dict: A dictionary containing Parse.ly pixel events
    @param schema:  Optional parameter containing the schema to normalize the event_dict keys against
                    IF not specified, this will default to the most recent parsely_raw_data schema
    """
    event_dict = {}
    schema = schema or schema.SCHEMA

    # fix value types
    if input_event_dict.get("metadata.share_urls") is not None and isinstance(
        input_event_dict["metadata.share_urls"], dict
    ):
        input_event_dict["metadata.share_urls"] = (
            list(input_event_dict["metadata.share_urls"].values()) or None
        )

    # replace all "."s in the key with "_"
    input_event_dict = {x.replace('.', '_'): v for x, v in input_event_dict.items()}

    # emit only public schema items
    # ensure all columns are available and null when needed
    # account for all boolean schema defined fields as this is parsely_raw_data specific
    for key in schema:
        if key not in input_event_dict:
            if key in BOOLEAN_FIELDS:
                event_dict[key] = False
            else:
                event_dict[key] = None
        else:
            event_dict[key] = input_event_dict[key]

    event_dict["schema_version"] = __version__

    return event_dict

<commit_msg>increment version<commit_after>__license__ = """
Copyright 2016 Parsely, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

__version__ = "2.4.0"

from . import bigquery, docgen, redshift, s3, samples, schema, stream, utils

__all__ = [
    "bigquery",
    "docgen",
    "redshift",
    "s3",
    "samples",
    "schema",
    "stream",
    "utils",
]

BOOLEAN_FIELDS = {"flags_is_amp"}

def normalize_keys(input_event_dict, schema=None):
    """Conform events to public schema: correct keys and proper value types.

    @param input_event_dict: A dictionary containing Parse.ly pixel events
    @param schema:  Optional parameter containing the schema to normalize the event_dict keys against
                    IF not specified, this will default to the most recent parsely_raw_data schema
    """
    event_dict = {}
    schema = schema or schema.SCHEMA

    # fix value types
    if input_event_dict.get("metadata.share_urls") is not None and isinstance(
        input_event_dict["metadata.share_urls"], dict
    ):
        input_event_dict["metadata.share_urls"] = (
            list(input_event_dict["metadata.share_urls"].values()) or None
        )

    # replace all "."s in the key with "_"
    input_event_dict = {x.replace('.', '_'): v for x, v in input_event_dict.items()}

    # emit only public schema items
    # ensure all columns are available and null when needed
    # account for all boolean schema defined fields as this is parsely_raw_data specific
    for key in schema:
        if key not in input_event_dict:
            if key in BOOLEAN_FIELDS:
                event_dict[key] = False
            else:
                event_dict[key] = None
        else:
            event_dict[key] = input_event_dict[key]

    event_dict["schema_version"] = __version__

    return event_dict

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

__author__ = """Sean Marlow"""
__email__ = 'sean.marlow@suse.com'
__version__ = '1.0.0'
<commit_msg>Remove email.<commit_after># -*- coding: utf-8 -*-

__author__ = """Sean Marlow"""
__version__ = '1.0.0'
<|endoftext|>


<commit_before>import numpy as np
from core import read_events, get_fs

class FindEnd():
    def __init__(self):
        self.keep = True
    def check(self,code):
        if code in '()FfTt]N>#':
            self.keep = False
        return self.keep

def get_stim_start(stim_end_row,digmarks):
    '''
    finds

    '''
    rec,ts = stim_end_row['recording'],stim_end_row['time_samples']
    mask = (
        (digmarks['recording']==rec)
        & (digmarks['time_samples'] < ts)
        & ~digmarks['codes'].str.contains('[RCL]')
        )
    this_trial_mask = (
        digmarks[mask].iloc[::-1]['codes'].apply(FindEnd().check).iloc[::-1]
        & digmarks[mask]['codes'].str.contains('<')
    )
    this_trial = digmarks[mask][this_trial_mask]
    return this_trial.iloc[0]

def _is_not_floatable(arg):
    ''' returns True if arg cannot be converted to float
    '''
    try:
        float(arg)
        return False
    except ValueError:
        return True

def get_stim_info(trial_row,stimulus,fs):
    '''
    finds the stimulus info for a trial.

    Parameters
    -------
    trial_row
        row from a trial
    stimulus
        pandas dataframe of all stimulus events
    fs : float
        sampling rate of block

    Returns
    -------
    digmark row for the response event

    '''
    rec,samps = trial_row['recording'], trial_row['time_samples']
    stim_mask = (
        (stimulus['recording']==rec)
        & (stimulus['time_samples']>(samps-0.1*fs))
        & (stimulus['time_samples']<(samps+fs))
        & ~stimulus['text'].str.contains('date')
        & stimulus['text'].apply(_is_not_floatable) # occlude floats
        )

    if stim_mask.sum()>0:
        return stimulus[stim_mask].iloc[0]
    else:
        return dict(codes=np.nan,time_samples=np.nan,recording=np.nan,text=np.nan)

def get_stim_end(trial_row,digmarks,fs,window=60.0):
    '''
    finds the end of the stimulus event for a trial.

    Parameters
    -------
    trial_row
        row from a trial
    digmarks
        pandas dataframe of all digmark events
    fs : float
        sampling rate of block
    window : float
        time window (in seconds) after the stimulus start in which to look for
        the stimulus end. default: 60.0

    Returns
    -------
    digmark row for the response event

    '''
    rec,samps = trial_row['recording'], trial_row['time_samples']
    resp_mask = (
        (digmarks['recording']==rec)
        & (digmarks['time_samples']>samps)
        & (digmarks['time_samples']<(samps+fs*window))
        & digmarks['codes'].str.contains('[>#]')
        )
    assert digmarks[resp_mask].shape[0]>0
    return digmarks[resp_mask].iloc[0]

def get_response(trial_row,digmarks,fs,window=5.0):
    '''
    finds the response event for a trial.

    Parameters
    -------
    trial_row
        row from a trial
    digmarks
        pandas dataframe of all digmark events
    fs : float
        sampling rate of block
    window : float
        time window (in seconds) after the stimulus end in which to look for
        the response. default: 5.0

    Returns
    -------
    digmark row for the response event

    '''
    rec,samps = trial_row['recording'], trial_row['time_samples']
    try:
        stim_dur = trial_row['stimulus_end'] - trial_row['time_samples']
    except KeyError:
        stim_dur = get_stim_end(rec,samps,fs)['time_samples']-samps
    resp_mask = (
        (digmarks['recording']==rec)
        & (digmarks['time_samples']>(samps+stim_dur))
        & (digmarks['time_samples']<(samps+stim_dur+fs*window))
        & digmarks['codes'].str.contains('[RLN]')
        )
    if digmarks[resp_mask].shape[0]>0:
        return digmarks[resp_mask].iloc[0]
    else:
        return dict(codes=np.nan,time_samples=np.nan,recording=np.nan)

def get_consequence(trial_row,digmarks,fs,window=2.0):
    '''
    finds the consequence event for a trial.

    Parameters
    -------
    trial_row
        row from a trial
    digmarks
        pandas dataframe of all digmark events
    fs : float
        sampling rate of block
    window : float
        time window (in seconds) after the reponse in which to look for the
        consequence. default: 2.0

    Returns
    -------
    digmark row for the consequence event

    '''
    rec,samps = trial_row['recording'], trial_row['time_samples']
    rt = trial_row['response_time']
    bds = rt, rt+fs*window
    resp_mask = (
        (digmarks['recording']==rec)
        & (digmarks['time_samples']>bds[0])
        & (digmarks['time_samples']<bds[1])
        & digmarks['codes'].str.contains('[FfTt]')
        )
    if digmarks[resp_mask].shape[0]>0:
        return digmarks[resp_mask].iloc[0]
    else:
        return dict(codes=np.nan,time_samples=np.nan,recording=np.nan)

def is_correct(consequence):
    '''
    Checks if the consequence indicates that the trial was correct.
    '''
    try:
        return consequence in 'Ff'
    except TypeError:
        return consequence

def get_trials(block_path):
    '''
    returns a pandas dataframe containing trial information for a given block_path

    Parameters
    -------
    block_path : str
        the path to the block

    Returns
    ------

    trials : pandas dataframe

    Columns
    ------

    '''
    digmarks = read_events(block_path,'DigMark')
    stimulus = read_events(block_path,'Stimulus')
    fs = get_fs(block_path)

    stim_end_mask = digmarks['codes'].isin(('>','#'))
    trials = digmarks[stim_end_mask].apply(lambda row: get_stim_start(row,digmarks),axis=1)[:]
    trials.reset_index(inplace=True)
    del trials['index']
    del trials['codes']
    trials['stimulus'] = trials.apply(lambda row: get_stim_info(row,stimulus,fs)['text'],axis=1)
    trials['stimulus_end'] = trials.apply(lambda row: get_stim_end(row,digmarks,fs)['time_samples'],axis=1)
    trials['response'] = trials.apply(lambda row: get_response(row,digmarks,fs)['codes'],axis=1)
    trials['response_time'] = trials.apply(lambda row: get_response(row,digmarks,fs)['time_samples'],axis=1)
    trials['consequence'] = trials.apply(lambda row: get_consequence(row,digmarks,fs)['codes'],axis=1)
    trials['correct'] = trials['consequence'].apply(is_correct)
    return trials<commit_msg>Added docstrings in events.py<commit_after>import numpy as np
from core import read_events, get_fs

class FindEnd():
    def __init__(self):
        self.keep = True
    def check(self,code):
        if code in '()FfTt]N>#':
            self.keep = False
        return self.keep

def get_stim_start(stim_end_row,digmarks):
    '''
    Finds the digmark row corresponding to the beginning of a stimulus

    Parameters
    ------
    stim_end_row : pandas dataframe
        The row of the digmark dataframe corresponding to the end of a stimulus
    digmarks : pandas dataframe
        The digmark dataframe

    Returns
    ------
    this_trial : pandas dataframe
        Row containing the digmark corresponding to the start of the stimulus

    '''
    rec,ts = stim_end_row['recording'],stim_end_row['time_samples']
    mask = (
        (digmarks['recording']==rec)
        & (digmarks['time_samples'] < ts)
        & ~digmarks['codes'].str.contains('[RCL]')
        )
    this_trial_mask = (
        digmarks[mask].iloc[::-1]['codes'].apply(FindEnd().check).iloc[::-1]
        & digmarks[mask]['codes'].str.contains('<')
    )
    this_trial = digmarks[mask][this_trial_mask]
    return this_trial.iloc[0]

def _is_not_floatable(arg):
    ''' returns True if arg cannot be converted to float
    '''
    try:
        float(arg)
        return False
    except ValueError:
        return True

def get_stim_info(trial_row,stimulus,fs):
    '''
    finds the stimulus info for a trial.

    Parameters
    -------
    trial_row
        row from a trial
    stimulus
        pandas dataframe of all stimulus events
    fs : float
        sampling rate of block

    Returns
    -------
    digmark row for the response event

    '''
    rec,samps = trial_row['recording'], trial_row['time_samples']
    stim_mask = (
        (stimulus['recording']==rec)
        & (stimulus['time_samples']>(samps-0.1*fs))
        & (stimulus['time_samples']<(samps+fs))
        & ~stimulus['text'].str.contains('date')
        & stimulus['text'].apply(_is_not_floatable) # occlude floats
        )

    if stim_mask.sum()>0:
        return stimulus[stim_mask].iloc[0]
    else:
        return dict(codes=np.nan,time_samples=np.nan,recording=np.nan,text=np.nan)

def get_stim_end(trial_row,digmarks,fs,window=60.0):
    '''
    finds the end of the stimulus event for a trial.

    Parameters
    -------
    trial_row
        row from a trial
    digmarks
        pandas dataframe of all digmark events
    fs : float
        sampling rate of block
    window : float
        time window (in seconds) after the stimulus start in which to look for
        the stimulus end. default: 60.0

    Returns
    -------
    digmark row for the response event

    '''
    rec,samps = trial_row['recording'], trial_row['time_samples']
    resp_mask = (
        (digmarks['recording']==rec)
        & (digmarks['time_samples']>samps)
        & (digmarks['time_samples']<(samps+fs*window))
        & digmarks['codes'].str.contains('[>#]')
        )
    assert digmarks[resp_mask].shape[0]>0
    return digmarks[resp_mask].iloc[0]

def get_response(trial_row,digmarks,fs,window=5.0):
    '''
    finds the response event for a trial.

    Parameters
    -------
    trial_row
        row from a trial
    digmarks
        pandas dataframe of all digmark events
    fs : float
        sampling rate of block
    window : float
        time window (in seconds) after the stimulus end in which to look for
        the response. default: 5.0

    Returns
    -------
    digmark row for the response event

    '''
    rec,samps = trial_row['recording'], trial_row['time_samples']
    try:
        stim_dur = trial_row['stimulus_end'] - trial_row['time_samples']
    except KeyError:
        stim_dur = get_stim_end(rec,samps,fs)['time_samples']-samps
    resp_mask = (
        (digmarks['recording']==rec)
        & (digmarks['time_samples']>(samps+stim_dur))
        & (digmarks['time_samples']<(samps+stim_dur+fs*window))
        & digmarks['codes'].str.contains('[RLN]')
        )
    if digmarks[resp_mask].shape[0]>0:
        return digmarks[resp_mask].iloc[0]
    else:
        return dict(codes=np.nan,time_samples=np.nan,recording=np.nan)

def get_consequence(trial_row,digmarks,fs,window=2.0):
    '''
    finds the consequence event for a trial.

    Parameters
    -------
    trial_row
        row from a trial
    digmarks
        pandas dataframe of all digmark events
    fs : float
        sampling rate of block
    window : float, optional
        time window (in seconds) after the reponse in which to look for the
        consequence. default: 2.0

    Returns
    -------
    digmark row for the consequence event

    '''
    rec,samps = trial_row['recording'], trial_row['time_samples']
    rt = trial_row['response_time']
    bds = rt, rt+fs*window
    resp_mask = (
        (digmarks['recording']==rec)
        & (digmarks['time_samples']>bds[0])
        & (digmarks['time_samples']<bds[1])
        & digmarks['codes'].str.contains('[FfTt]')
        )
    if digmarks[resp_mask].shape[0]>0:
        return digmarks[resp_mask].iloc[0]
    else:
        return dict(codes=np.nan,time_samples=np.nan,recording=np.nan)

def is_correct(consequence):
    '''
    Checks if the consequence indicates that the trial was correct.
    '''
    try:
        return consequence in 'Ff'
    except TypeError:
        return consequence

def get_trials(block_path):
    '''
    returns a pandas dataframe containing trial information for a given block_path

    Parameters
    -------
    block_path : str
        the path to the block

    Returns
    ------

    trials : pandas dataframe

    Columns
    ------

    '''
    digmarks = read_events(block_path,'DigMark')
    stimulus = read_events(block_path,'Stimulus')
    fs = get_fs(block_path)

    stim_end_mask = digmarks['codes'].isin(('>','#'))
    trials = digmarks[stim_end_mask].apply(lambda row: get_stim_start(row,digmarks),axis=1)[:]
    trials.reset_index(inplace=True)
    del trials['index']
    del trials['codes']
    trials['stimulus'] = trials.apply(lambda row: get_stim_info(row,stimulus,fs)['text'],axis=1)
    trials['stimulus_end'] = trials.apply(lambda row: get_stim_end(row,digmarks,fs)['time_samples'],axis=1)
    trials['response'] = trials.apply(lambda row: get_response(row,digmarks,fs)['codes'],axis=1)
    trials['response_time'] = trials.apply(lambda row: get_response(row,digmarks,fs)['time_samples'],axis=1)
    trials['consequence'] = trials.apply(lambda row: get_consequence(row,digmarks,fs)['codes'],axis=1)
    trials['correct'] = trials['consequence'].apply(is_correct)
    return trials<|endoftext|>


<commit_before>"""test_rpc.py"""
# pylint: disable=missing-docstring,line-too-long,too-many-public-methods

from unittest import TestCase, main
import itertools
import json

from jsonrpcclient import rpc
from jsonrpcclient.rpc import rpc_request, rpc_request_str, sort_request

class TestRPC(TestCase):

    def setUp(self):
        # Monkey patch ID_GENERATOR to ensure it starts with 1
        rpc.ID_GENERATOR = itertools.count(1) # First generated is 1

class TestSortRequest(TestRPC):

    def test(self):
        self.assertEqual(
            '{"jsonrpc": "2.0", "method": "add", "params": [2, 3], "id": 2}',
            json.dumps(sort_request({'id': 2, 'params': [2, 3], 'method': 'add', 'jsonrpc': '2.0'})),
        )

class TestNotifications(TestRPC):

    def test_no_arguments(self):
        self.assertEqual(
            {"jsonrpc": "2.0", "method": "get"},
            rpc_request('get')
        )

    def test_one_positional(self):
        self.assertEqual(
            {"jsonrpc": "2.0", "method": "sqrt", "params": [1]},
            rpc_request('sqrt', 1)
        )

    def test_two_positionals(self):
        self.assertEqual(
            {"jsonrpc": "2.0", "method": "add", "params": [1, 2]},
            rpc_request('add', 1, 2)
        )

    def test_one_keyword(self):
        self.assertEqual(
            {"jsonrpc": "2.0", "method": "find", "params": {"name": "Foo"}},
            rpc_request('find', name='Foo')
        )

    def test_two_keywords(self):
        """Note that keyword arguments are sorted in alphabetical order by the
        keys. This is because they're not received in any order, so we sort
        them, to be sure of *some* order"""
        self.assertEqual(
            {"jsonrpc": "2.0", "method": "find", "params": {"age": 42, "name": "Foo"}},
            rpc_request('find', name='Foo', age=42)
        )

    def test_both_positional_and_keyword(self):
        self.assertEqual(
            {"jsonrpc": "2.0", "method": "find", "params": ["Foo", {"age": 42}]},
            rpc_request('find', 'Foo', age=42)
        )

    def test_dict_params(self):
        self.assertEqual(
            {"jsonrpc": "2.0", "method": "find", "params": {"age": 42, "name": "Foo"}},
            rpc_request('find', name='Foo', age=42)
        )

    def test_list_params(self):
        self.assertEqual(
            {"jsonrpc": "2.0", "method": "find", "params": ["Foo", 42]},
            rpc_request('find', ['Foo', 42])
        )

class TestRequests(TestRPC):

    def test_method_only_requiring_response(self):
        self.assertEqual(
            {"jsonrpc": "2.0", "method": "go", "id": 1},
            rpc_request('go', response=True)
        )

    def test_both_positional_and_keyword_requiring_response(self):
        self.assertEqual(
            {"jsonrpc": "2.0", "method": "go", "params": ["positional", {"keyword": "foo"}], "id": 1},
            rpc_request('go', 'positional', keyword='foo', response=True)
        )

    def test_str(self):
        self.assertEqual(
            '{"jsonrpc": "2.0", "method": "get"}',
            rpc_request_str('get')
        )

if __name__ == '__main__':
    main()
<commit_msg>Add test for incremental id<commit_after>"""test_rpc.py"""
# pylint: disable=missing-docstring,line-too-long,too-many-public-methods

from unittest import TestCase, main
import itertools
import json

from jsonrpcclient import rpc
from jsonrpcclient.rpc import rpc_request, rpc_request_str, sort_request

class TestRPC(TestCase):

    def setUp(self):
        # Monkey patch ID_GENERATOR to ensure it starts with 1
        rpc.ID_GENERATOR = itertools.count(1) # First generated is 1

class TestSortRequest(TestRPC):

    def test(self):
        self.assertEqual(
            '{"jsonrpc": "2.0", "method": "add", "params": [2, 3], "id": 2}',
            json.dumps(sort_request({'id': 2, 'params': [2, 3], 'method': 'add', 'jsonrpc': '2.0'})),
        )

class TestNotifications(TestRPC):

    def test_no_arguments(self):
        self.assertEqual(
            {"jsonrpc": "2.0", "method": "get"},
            rpc_request('get')
        )

    def test_one_positional(self):
        self.assertEqual(
            {"jsonrpc": "2.0", "method": "sqrt", "params": [1]},
            rpc_request('sqrt', 1)
        )

    def test_two_positionals(self):
        self.assertEqual(
            {"jsonrpc": "2.0", "method": "add", "params": [1, 2]},
            rpc_request('add', 1, 2)
        )

    def test_one_keyword(self):
        self.assertEqual(
            {"jsonrpc": "2.0", "method": "find", "params": {"name": "Foo"}},
            rpc_request('find', name='Foo')
        )

    def test_two_keywords(self):
        """Note that keyword arguments are sorted in alphabetical order by the
        keys. This is because they're not received in any order, so we sort
        them, to be sure of *some* order"""
        self.assertEqual(
            {"jsonrpc": "2.0", "method": "find", "params": {"age": 42, "name": "Foo"}},
            rpc_request('find', name='Foo', age=42)
        )

    def test_both_positional_and_keyword(self):
        self.assertEqual(
            {"jsonrpc": "2.0", "method": "find", "params": ["Foo", {"age": 42}]},
            rpc_request('find', 'Foo', age=42)
        )

    def test_dict_params(self):
        self.assertEqual(
            {"jsonrpc": "2.0", "method": "find", "params": {"age": 42, "name": "Foo"}},
            rpc_request('find', name='Foo', age=42)
        )

    def test_list_params(self):
        self.assertEqual(
            {"jsonrpc": "2.0", "method": "find", "params": ["Foo", 42]},
            rpc_request('find', ['Foo', 42])
        )

class TestRequests(TestRPC):

    def test_method_only_requiring_response(self):
        self.assertEqual(
            {"jsonrpc": "2.0", "method": "go", "id": 1},
            rpc_request('go', response=True)
        )

    def test_both_positional_and_keyword_requiring_response(self):
        self.assertEqual(
            {"jsonrpc": "2.0", "method": "go", "params": ["positional", {"keyword": "foo"}], "id": 1},
            rpc_request('go', 'positional', keyword='foo', response=True)
        )

    def test_incremental_id(self):
        self.assertEqual(
            {"jsonrpc": "2.0", "method": "go", "id": 1},
            rpc_request('go', response=True)
        )
        self.assertEqual(
            {"jsonrpc": "2.0", "method": "go", "id": 2},
            rpc_request('go', response=True)
        )

    def test_str(self):
        self.assertEqual(
            '{"jsonrpc": "2.0", "method": "get"}',
            rpc_request_str('get')
        )

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Author: Veronica G. Vergara L.
#
#

from base_scheduler import BaseScheduler

class LSF(BaseScheduler):

    def __init__(self,name='LSF',jobLauncher=None):
        BaseScheduler.__init__(self,name)
        self.__jobLauncher = jobLauncher

    def print_jobLauncher(self):
        print(str(self.__jobLauncher))

if __name__ == '__main__':
    print 'This is the LSF scheduler class'
<commit_msg>updated lsf to use the new init method in the base scheduler class<commit_after>#!/usr/bin/env python
#
# Author: Veronica G. Vergara L.
#
#

from base_scheduler import BaseScheduler

class LSF(BaseScheduler):

    def __init__(self):
        self.__name = 'LSF'
        self.__submitCmd = 'bsub'
        self.__statusCmd = 'bjobs'
        self.__deleteCmd = 'bkill'
        self.__walltimeOpt = '-W'
        self.__numTasksOpt = '-n'
        self.__jobNameOpt = '-N'
        BaseScheduler.__init__(self,self.__name,self.__submitCmd,self.__statusCmd,
                               self.__deleteCmd,self.__walltimeOpt,self.__numTasksOpt,
                               self.__jobNameOpt)

if __name__ == '__main__':
    print 'This is the LSF scheduler class'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Nicolas Bessi. Copyright Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from openerp.osv import orm, fields
from openerp.tools.translate import _
import logging

_logger = logging.getLogger(__name__)

class ResPartner(orm.Model):
    """Adds lastname and firstname, name become a stored function field"""

    _inherit = 'res.partner'

    def init(self, cursor):
        cursor.execute('SELECT id FROM res_partner WHERE lastname IS NOT NULL '
                       'Limit 1')
        if not cursor.fetchone():
            cursor.execute('UPDATE res_partner set lastname = name WHERE name '
                           'IS NOT NULL')
            # Create Sql constraint if table is not empty
            cursor.execute('SELECT id FROM res_partner Limit 1')
            if cursor.fetchone():
                cursor.execute('ALTER TABLE res_partner ALTER COLUMN lastname '
                               'SET NOT NULL')
                _logger.info("NOT NULL constraint for res_partner.lastname "
                             "correctly set")

    def _prepare_name_custom(self, cursor, uid, partner, context=None):
        """
        This function is designed to be inherited in a custom module
        """
        names = (partner.lastname, partner.firstname)
        fullname = " ".join([s for s in names if s])
        return fullname

    def _compute_name_custom(self, cursor, uid, ids, fname, arg, context=None):
        res = {}
        for partner in self.browse(cursor, uid, ids, context=context):
            res[partner.id] = self._prepare_name_custom(
                cursor, uid, partner, context=context)
        return res

    def _write_name(
        self, cursor, uid, partner_id, field_name, field_value, arg,
        context=None
    ):
        """
        Try to reverse the effect of _compute_name_custom:
        * if the partner is not a company and the firstname does not change in
          the new name then firstname remains untouched and lastname is updated
          accordingly
        * otherwise lastname=new name and firstname=False
        In addition an heuristic avoids to keep a firstname without a non-blank
        lastname
        """
        field_value = (
            field_value and not field_value.isspace() and field_value or False)
        vals = {'lastname': field_value, 'firstname': False}
        if field_value:
            flds = self.read(
                cursor, uid, [partner_id], ['firstname', 'is_company'],
                context=context)[0]
            if not flds['is_company']:
                to_check = ' %s' % flds['firstname']
                if field_value.endswith(to_check):
                    ln = field_value[:-len(to_check)].strip()
                    if ln:
                        vals['lastname'] = ln
                        del(vals['firstname'])
                    else:
                        # If the lastname is deleted from the new name
                        # then the firstname becomes the lastname
                        vals['lastname'] = flds['firstname']

        return self.write(cursor, uid, partner_id, vals, context=context)

    def copy_data(self, cr, uid, _id, default=None, context=None):
        """
        Avoid to replicate the firstname into the name when duplicating a
        partner
        """
        default = default or {}
        if not default.get('lastname'):
            default = default.copy()
            default['lastname'] = (
                _('%s (copy)') % self.read(
                    cr, uid, [_id], ['lastname'], context=context
                    )[0]['lastname']
            )
            if default.get('name'):
                del(default['name'])
        return super(ResPartner, self).copy_data(
            cr, uid, _id, default, context=context)

    def create(self, cursor, uid, vals, context=None):
        """
        To support data backward compatibility we have to keep this overwrite
        even if we use fnct_inv: otherwise we can't create entry because
        lastname is mandatory and module will not install if there is demo data
        """
        to_use = vals
        if 'name' in vals:
            corr_vals = vals.copy()
            if vals.get('name'):
                corr_vals['lastname'] = corr_vals['name']
            del(corr_vals['name'])
            to_use = corr_vals
        return super(ResPartner, self).create(
            cursor, uid, to_use, context=context)

    _columns = {'name': fields.function(_compute_name_custom, string="Name",
                                        type="char", store=True,
                                        select=True, readonly=True,
                                        fnct_inv=_write_name),

                'firstname': fields.char("Firstname"),
                'lastname': fields.char("Lastname", required=True)}
<commit_msg>[FIX] IntegrityError: column "lastname" contains null values<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Nicolas Bessi. Copyright Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from openerp.osv import orm, fields
from openerp.tools.translate import _
import logging

_logger = logging.getLogger(__name__)

class ResPartner(orm.Model):
    """Adds lastname and firstname, name become a stored function field"""

    _inherit = 'res.partner'

    def _set_default_value_on_column(self, cr, column_name, context=None):
        res = super(ResPartner, self)._set_default_value_on_column(
            cr, column_name, context=context)
        if column_name == 'lastname':
            cr.execute('SELECT id FROM res_partner WHERE lastname IS NOT NULL '
                       'Limit 1')
            if not cr.fetchone():
                cr.execute('UPDATE res_partner set lastname = name WHERE name '
                           'IS NOT NULL')
                # Create Sql constraint if table is not empty
                cr.execute('SELECT id FROM res_partner Limit 1')
                if cr.fetchone():
                    cr.execute('ALTER TABLE res_partner ALTER COLUMN lastname '
                               'SET NOT NULL')
                    _logger.info("NOT NULL constraint for "
                                 "res_partner.lastname correctly set")
        return res

    def _prepare_name_custom(self, cursor, uid, partner, context=None):
        """
        This function is designed to be inherited in a custom module
        """
        names = (partner.lastname, partner.firstname)
        fullname = " ".join([s for s in names if s])
        return fullname

    def _compute_name_custom(self, cursor, uid, ids, fname, arg, context=None):
        res = {}
        for partner in self.browse(cursor, uid, ids, context=context):
            res[partner.id] = self._prepare_name_custom(
                cursor, uid, partner, context=context)
        return res

    def _write_name(
        self, cursor, uid, partner_id, field_name, field_value, arg,
        context=None
    ):
        """
        Try to reverse the effect of _compute_name_custom:
        * if the partner is not a company and the firstname does not change in
          the new name then firstname remains untouched and lastname is updated
          accordingly
        * otherwise lastname=new name and firstname=False
        In addition an heuristic avoids to keep a firstname without a non-blank
        lastname
        """
        field_value = (
            field_value and not field_value.isspace() and field_value or False)
        vals = {'lastname': field_value, 'firstname': False}
        if field_value:
            flds = self.read(
                cursor, uid, [partner_id], ['firstname', 'is_company'],
                context=context)[0]
            if not flds['is_company']:
                to_check = ' %s' % flds['firstname']
                if field_value.endswith(to_check):
                    ln = field_value[:-len(to_check)].strip()
                    if ln:
                        vals['lastname'] = ln
                        del(vals['firstname'])
                    else:
                        # If the lastname is deleted from the new name
                        # then the firstname becomes the lastname
                        vals['lastname'] = flds['firstname']

        return self.write(cursor, uid, partner_id, vals, context=context)

    def copy_data(self, cr, uid, _id, default=None, context=None):
        """
        Avoid to replicate the firstname into the name when duplicating a
        partner
        """
        default = default or {}
        if not default.get('lastname'):
            default = default.copy()
            default['lastname'] = (
                _('%s (copy)') % self.read(
                    cr, uid, [_id], ['lastname'], context=context
                    )[0]['lastname']
            )
            if default.get('name'):
                del(default['name'])
        return super(ResPartner, self).copy_data(
            cr, uid, _id, default, context=context)

    def create(self, cursor, uid, vals, context=None):
        """
        To support data backward compatibility we have to keep this overwrite
        even if we use fnct_inv: otherwise we can't create entry because
        lastname is mandatory and module will not install if there is demo data
        """
        to_use = vals
        if 'name' in vals:
            corr_vals = vals.copy()
            if vals.get('name'):
                corr_vals['lastname'] = corr_vals['name']
            del(corr_vals['name'])
            to_use = corr_vals
        return super(ResPartner, self).create(
            cursor, uid, to_use, context=context)

    _columns = {'name': fields.function(_compute_name_custom, string="Name",
                                        type="char", store=True,
                                        select=True, readonly=True,
                                        fnct_inv=_write_name),

                'firstname': fields.char("Firstname"),
                'lastname': fields.char("Lastname", required=True)}
<|endoftext|>


<commit_before>__module_name__ = "translator"
__module_version__ = "0.1"
__module_description__ = "Translates from one language to other using Google Translate via YQL."
__module_author__ = "drag <drag.publicgithub@gmail.com>"

import xchat

import json
import urllib2

DEST_LANG = 'en'
AUTOUSER = {}

class TranlateException(Exception):
	pass

"""class TranslatorClass:
# -*- coding: utf-8 -*-
	LANGUAGES = {
		'ENGLISH' : 'en',
		'FRENCH' : 'fr',
		'RUSSIAN' : 'ru'
	}

	LANG_CODES = set(LANGUAGES.values())

	def translate(cls, msg, target, src):
		langs = '';

		if targetLang not in cls.LANG_CODES:
			raise TranslateException('Target language not supported.')

		if srcLang == None:
			langs = '|' + targatLang
		elif srcLang in cls.CODES_SET:
			langs = srcLang + '|' + targetLang
		else:
			raise TranslateException('Source language not supported.')
http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20google.translate%20where%20q%3D%22this%20is%20a%20test%22%20and%20target%3D%22de%22%3B&format=xml&env=http%3A%2F%2Fdatatables.org%2Falltables.env
		url = "http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20google.translate%20where%20q%3D%22" + msg + "%22%20and%20target%3D%22" + target + "%22%20and%20source%3D%22" + src + "%22%3B&format=json&diagnostics=true&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys&callback="
#http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20google.translate%20where%20q%3D%22" + msg + "%22%20and%20target%3D%22" + target + "%22%20and%20source%3D%22" + src + "%22%3B&format=json&diagnostics=true&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys&callback=
		headers = { 'User-Agent': 'Mozilla/5.0' }
		req = urllib2.Request(url, None, headers)
		response = urllib2.urlopen(req)

		result = json.load(response)
		resultArray = result['query']['results']['json']['json'][0]['json']

		str = ""
		if type(resultArray) is dict:
			str += resultArray['json'][0]
		else:
			for subDict in resultArray:
				str += subDict['json'][0]

		return str

class TranslateThread(Thread):
	def __init__(self, queue):
		Threat.__init__(self, target=self.doTranslate())
		self.queue = queue
		self.keepalive == True

	def doTranslate(self):
		global LAST_ERROR

		while True:
			task = self.queue.get()

			if not self.keepalive or task == None:
				break
			try:
				context, user, src, target, text = task
				translation = Translator.translate(text, target, src)
				xchat.prnt("Translation : " + translation)
			except TranslateException, e:
				LAST_ERROR = "[TE: %s] <%s>" %(e, user, text)
			except urllib2.URLError, e:
				LAST_ERROR = "[URL] %s" %e
			except UnicodeError, e:
				LAST_ERROR = "[Encode: %s] <%s> %s" %(e, user, text)

def autoTranslate(word, world_eo1, userdata):
	global AUTOUSER
	global DEST_LANG

	channel = xchat.get_info('channel')
	user = word[1]
	dest = DEST_LANG
	src = None

	if len(word) > 2:
		dest = word[2]
	if len(word) > 3:
		src = word[3]

	AUTOUSER[channel + ' ' + user.lower()] = (dest, src)
	xchat.prnt("User '%s' has been added to auto-translate list" %user)
	return xchat.EAT_ALL

xchat.hook_command("atr", autoTranslate)

def removeAutoTranslate(word, word_eol, userdata):
	global AUTOUSER

	channel = xchat.get_info('channel')
	user = word[1]
	target = channel + ' ' + user

	if AUTOUSER.pop(target, None) != None:
		xchat.prnt("User %s has been removed from the auto-translate list" %user)
	return xchat.EAT_ALL

xchat.hook_command("rmatr", removeAutoTranslate)

def printAutouser(word, word_eol, userdata):
	global AUTOUSER

	channel = xchat.get_info('channel')
	users = [target.split(' ')[1] for target in AUTOUSER.keys()]

	xchat.prnt("Currently automatically translating messages for the folloing users: %s" %(' '.join(users)))
	xchat.EAT_ALL

xchat.hook_command("lsatr", printAutouser)"""

def getPage(words,src,dest):
	url = "http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20google.translate%20where%20q%3D%22" + words +"%22%20and%20target%3D%22" + dest + "%22%20and%20source%3D%22" + src +"%22	3B&format=json&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys&callback="
	headers = { 'User-Agent' : 'Mozilla/5.0' }
	req = urllib2.Request(url, None, headers)
	response = urllib2.urlopen(req)
	return response.read()

def parseJsonResult(resultStr):
	result = json.loads(resultStr)
	resultArray = result['query']['results']['json']['json'][0]['json']
	str=""
	if type(resultArray) is dict:
		str+=resultArray['json'][0]
	else:
		for subDict in resultArray:
			str+=subDict['json'][0]
	return str

def translateTo(word, word_eo1, userdata):
	xchat.prnt("Starting translate");
	if len(word) < 2:
		print "No message found."
	else:
		srcStr = "this is just a test"
		#message = urllib2.quote(message2.encode('utf8'))
		destLanguage = "ru"
		srcLanguage = "en"
# &diagnostics=true&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys&callback=

#		url = "http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20google.translate%20where%20q%3D%22" + message + "%22%20and%20target%3D%22" + target + "%22%20and%20source%3D%22" + src + "%22%3B&format=json&env=http%3A%2F%2Fdatatables.org%2Falltables.env"
#		headers = { 'User-Agent': 'Mozilla/5.0 [en]' }
#		req = urllib2.Request(url, None, headers)
#		response = urllib2.urlopen(req)

#		try:
#			resp = urllib2.urlopen(url)
#			contents = resp.read()
#		except urllib2.HTTPError, error:
#			contents = error.read()
#			xchat.prnt(contents)

#		result = json.load(response)
#		resultArray = result['query']['results']['json']['json'][0]['json']
#		str = ""
#		if type(resultArray) is dict:
#			str += resultArray['json'][0]
#		else:
#			for subDict in resultArray:
#				str += subDict['json'][0]

		page = getPage(urllib2.quote(srcStr.encode('utf8')),srcLanguage,destLanguage)
		result = json.loads(page)
		str = parseJsonResult(result)

		xchat.prnt(str)
		xchat.command('say Translation complete.')
	return xchat.EAT_ALL

xchat.hook_command("tr", translateTo, help="/tr <message> translates message into the language specified.")

def runAutoTranslate(word, word_eol, userdata):
	global AUTOUSER

	user = word[0]
	text = word[1]
	channel = xchat.get_info('channel')
	target = channel + ' ' + user.lower()

	if AUTOUSER.has_key(target):
		xchat.prnt("New message from %s, translating..." %user)

	return xchat.EAT_ALL

xchat.hook_print("Channel Message", runAutoTranslate)
<commit_msg>Cleaned up code<commit_after>__module_name__ = "translator"
__module_version__ = "0.1"
__module_description__ = "Translates from one language to other using Google Translate via YQL."
__module_author__ = "drag"

import xchat

import json
import urllib2

DEST_LANG = 'en'
AUTOUSER = {}

def getPage(words,src,dest):
	url = "http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20google.translate%20where%20q%3D%22" + words +"%22%20and%20target%3D%22" + dest + "%22%20and%20source%3D%22" + src +"%22	3B&format=json&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys&callback="
	headers = { 'User-Agent' : 'Mozilla/5.0' }
	req = urllib2.Request(url, None, headers)
	response = urllib2.urlopen(req)
	return response.read()

def parseJsonResult(resultStr):
	result = json.loads(resultStr)
	resultArray = result['query']['results']['json']['json'][0]['json']
	str=""
	if type(resultArray) is dict:
		str+=resultArray['json'][0]
	else:
		for subDict in resultArray:
			str+=subDict['json'][0]
	return str

def translateTo(word, word_eo1, userdata):
	xchat.prnt("Starting translate");
	srcStr = "this is just a test"
	destLanguage = "ru"
	srcLanguage = "en"

	page = getPage(urllib2.quote(srcStr.encode('utf8')), srcLanguage, destLanguage)
	result = json.loads(page)
	str = parseJsonResult(result)

	print(str)
	xchat.command('say Translation complete.')

	return xchat.EAT_ALL

xchat.hook_command("tr", translateTo, help="/tr <message> translates message into the language specified.")
<|endoftext|>