<commit_before>import requests

class Post:
    """This class represents a post.

    .. note::
        Remember that you need to have access to the post.

    """

    def __init__(self, post_id, client):
        """
        :param post_id: id or guid of the post
        :type post_id: str
        :param client: client object used to authenticate
        :type client: client.Client

        .. note::
            The login function of the client should be called,
            before calling any of the post functions.

        """
        self._client = client
        self.post_id = post_id

    def fetch_data(self):
        r = self._client.session.get(self._client.pod +
                                     '/posts/' +
                                     self.post_id +
                                     '.json')
        if r.status_code == 200:
            return r.json()
        else:
            raise Exception('wrong status code: ' + str(r.status_code))

    def like(self):
        """This function likes a post

        :returns: dict -- json formatted like object.

        """

        data = {'authenticity_token': self._client.get_token()}

        r = self._client.session.post(self._client.pod +
                                      '/posts/' +
                                      self.post_id +
                                      '/likes',
                                      data=data,
                                      headers={'accept': 'application/json'})

        if r.status_code != 201:
            raise Exception(str(r.status_code) + ': Post could not be liked.')

        return r.json()

    def rmlike(self):
        """This function removes a like from a post

        """

        data = {'authenticity_token': self._client.get_token()}

        post_data = self.fetch_data()

        r = self._client.session.delete(self._client.pod + '/posts/' +
                                        self.post_id +
                                        '/likes/' +
                                        str(post_data['interactions']
                                            ['likes'][0]['id']),
                                        data=data)

        if r.status_code != 204:
            raise Exception(str(r.status_code) +
                            ': Like could not be removed.')

    def reshare(self):
        """This function reshares a post

        """

        post_data = self.fetch_data()

        data = {'root_guid': post_data['guid'],
                'authenticity_token': self._client.get_token()}

        r = self._client.session.post(self._client.pod +
                                      '/reshares',
                                      data=data,
                                      headers={'accept': 'application/json'})

        if r.status_code != 201:
            raise Exception(str(r.status_code) +
                            ': Post could not be reshared.')

        return r.json()

    def comment(self, text):
        """This function comments on a post

        :param post_id: id of the post to comment on.
        :type post_id: str
        :param text: text to comment.
        :type text: str

        """

        data = {'text': text,
                'authenticity_token': self._client.get_token()}

        r = self._client.session.post(self._client.pod +
                                      '/posts/' +
                                      self.post_id +
                                      '/comments',
                                      data=data,
                                      headers={'accept': 'application/json'})

        if r.status_code != 201:
            raise Exception(str(r.status_code) +
                            ': Comment could not be posted.')

        return r.json()

    def rmcomment(self, comment_id):
        """This function removes a comment from a post

        :param post_id: id of the post to remove the like from.
        :type post_id: str
        :param like_id: id of the like to remove.
        :type like_id: str

        """

        data = {'authenticity_token': self._client.get_token()}

        r = self._client.session.delete(self._client.pod + '/posts/' +
                                        self.post_id +
                                        '/comments/' +
                                        comment_id,
                                        data=data,
                                        headers={'accept': 'application/json'})

        if r.status_code != 204:
            raise Exception(str(r.status_code) +
                            ': Comment could not be deleted.')
<commit_msg>rename Post.fetch_data() to Post.get_data()<commit_after>import requests

class Post:
    """This class represents a post.

    .. note::
        Remember that you need to have access to the post.

    """

    def __init__(self, post_id, client):
        """
        :param post_id: id or guid of the post
        :type post_id: str
        :param client: client object used to authenticate
        :type client: client.Client

        .. note::
            The login function of the client should be called,
            before calling any of the post functions.

        """
        self._client = client
        self.post_id = post_id

    def get_data(self):
        r = self._client.session.get(self._client.pod +
                                     '/posts/' +
                                     self.post_id +
                                     '.json')
        if r.status_code == 200:
            return r.json()
        else:
            raise Exception('wrong status code: ' + str(r.status_code))

    def like(self):
        """This function likes a post

        :returns: dict -- json formatted like object.

        """

        data = {'authenticity_token': self._client.get_token()}

        r = self._client.session.post(self._client.pod +
                                      '/posts/' +
                                      self.post_id +
                                      '/likes',
                                      data=data,
                                      headers={'accept': 'application/json'})

        if r.status_code != 201:
            raise Exception(str(r.status_code) + ': Post could not be liked.')

        return r.json()

    def rmlike(self):
        """This function removes a like from a post

        """

        data = {'authenticity_token': self._client.get_token()}

        post_data = self.get_data()

        r = self._client.session.delete(self._client.pod + '/posts/' +
                                        self.post_id +
                                        '/likes/' +
                                        str(post_data['interactions']
                                            ['likes'][0]['id']),
                                        data=data)

        if r.status_code != 204:
            raise Exception(str(r.status_code) +
                            ': Like could not be removed.')

    def reshare(self):
        """This function reshares a post

        """

        post_data = self.get_data()

        data = {'root_guid': post_data['guid'],
                'authenticity_token': self._client.get_token()}

        r = self._client.session.post(self._client.pod +
                                      '/reshares',
                                      data=data,
                                      headers={'accept': 'application/json'})

        if r.status_code != 201:
            raise Exception(str(r.status_code) +
                            ': Post could not be reshared.')

        return r.json()

    def comment(self, text):
        """This function comments on a post

        :param post_id: id of the post to comment on.
        :type post_id: str
        :param text: text to comment.
        :type text: str

        """

        data = {'text': text,
                'authenticity_token': self._client.get_token()}

        r = self._client.session.post(self._client.pod +
                                      '/posts/' +
                                      self.post_id +
                                      '/comments',
                                      data=data,
                                      headers={'accept': 'application/json'})

        if r.status_code != 201:
            raise Exception(str(r.status_code) +
                            ': Comment could not be posted.')

        return r.json()

    def rmcomment(self, comment_id):
        """This function removes a comment from a post

        :param post_id: id of the post to remove the like from.
        :type post_id: str
        :param like_id: id of the like to remove.
        :type like_id: str

        """

        data = {'authenticity_token': self._client.get_token()}

        r = self._client.session.delete(self._client.pod + '/posts/' +
                                        self.post_id +
                                        '/comments/' +
                                        comment_id,
                                        data=data,
                                        headers={'accept': 'application/json'})

        if r.status_code != 204:
            raise Exception(str(r.status_code) +
                            ': Comment could not be deleted.')
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function
from ._version import get_versions
import os
try:
    from .__conda_version__ import conda_version
    __version__ = conda_version.replace("'","")
    del conda_version
except ImportError:
    __version__ = get_versions()['version']
    del get_versions

_notebook_loaded = None

def load_notebook(resources=None, verbose=False, force=False):
    ''' Prepare the IPython notebook for displaying Bokeh plots.

    Args:
        resources (Resource, optional) : a resource object describing how and where to load BokehJS from
        verbose (bool, optional) : whether to report detailed settings (default: False)
        force (bool, optional) : whether to skip IPython notebook check (default: False)

    Returns:
        None

    '''
    global _notebook_loaded

    # It's possible the IPython folks will chance things in the future, `force` parameter
    # provides an escape hatch as long as `displaypub` works
    if not force:
        notebook = False
        try:
            notebook = 'notebook' in get_ipython().config['IPKernelApp']['parent_appname']
        except Exception:
            pass
        if not notebook:
            raise RuntimeError('load_notebook() only works inside an IPython Notebook.')

    import IPython.core.displaypub as displaypub
    from .resources import INLINE
    from .templates import NOTEBOOK_LOAD, RESOURCES

    if resources is None:
        resources = INLINE

    plot_resources = RESOURCES.render(
        js_raw = resources.js_raw,
        css_raw = resources.css_raw,
        js_files = resources.js_files,
        css_files = resources.css_files,
    )

    if resources.mode == 'inline':
        js_info = 'inline'
        css_info = 'inline'
    else:
        js_info = resources.js_files[0] if len(resources.js_files) == 1 else resources.js_files
        css_info = resources.css_files[0] if len(resources.css_files) == 1 else resources.css_files

    warnings = ["Warning: " + msg['text'] for msg in resources.messages if msg['type'] == 'warn']

    if _notebook_loaded:
        warnings.append('Warning: BokehJS previously loaded')

    _notebook_loaded = resources

    html = NOTEBOOK_LOAD.render(
        plot_resources = plot_resources,
        logo_url = resources.logo_url,
        verbose = verbose,
        js_info = js_info,
        css_info = css_info,
        bokeh_version = __version__,
        warnings = warnings,
    )
    displaypub.publish_display_data('bokeh', {'text/html': html})

from .settings import settings
from . import sampledata
from .serverconfig import Server, Cloud

def _print_versions():
    """Returns all the versions of software that Bokeh relies on."""
    import platform as pt

    message = """
    Bokeh version: %s
    Python version: %s-%s
    Platform: %s
    """ % (__version__, pt.python_version(),
           pt.python_implementation(), pt.platform())
    return(message)

def print_versions():
    """Print all the versions of software that Bokeh relies on."""
    print(_print_versions())

def report_issue(number=None , owner="ContinuumIO", repo="bokeh",
                 versions=True, browser=True):
    """Opens a new Github issue programmatically.

    This "interactive" function will ask you for some minimal content
    to finally submit a new Github issue, adding essential info about
    the current setup. You can also call it with one specific issue
    number to add the essential info to an already opened issue.

    Parameters
    ----------

    number: int (default=None)
        The issue number if you want to add a new comment to an issue
        already created.

    owner: str (default="ContinuumIO")
        The owner's repository name.

    repo: str (default="bokeh")
        The name of the repository.

    versions: bool (default=True)
        Adds the `_print_versions` content information at the end of
        the body text.

    browser: bool (default=True)
        After submitting the new issue, it opens the issue webpage in
        your default web browser.

    Notes:
        * You can add the GHUSER (Github username) and
        GHPASS (Github password) to your environment to avoid
        filling this info in the interactive prompt.
        * Additionally, you can use this same function to report to any
        other project, just changing the parameters.
    """

    import requests
    import json
    import os
    import webbrowser

    from six.moves import input
    from six.moves.urllib.parse import urljoin

    print("This is the Bokeh reporting engine.\n\n"
          "Next, you will be guided to build the report")

    if number is None:
        title = input('Write the title for the intended issue: ')
        body = input('Write the body for the intended issue: ')
    else:
        body = input('Write your comment here: ')

    ghuser, ghpass = (os.environ.get(x) for x in ["GHUSER", "GHPASS"])
    if ghuser is None and ghpass is None:
        print("You need to add your GHUSER (Github username) and GHPASS (Github password)\n"
              "to the environmentor complete the next lines.")
        environment = input('Do you want to abort to set up the environment variable? ')
        if environment.lower() in ["true", "yes", "y", "on", "1"]:
            return
        else:
            ghuser = input('Write your Github username: ')
            ghpass = input('Write your Github password: ')
    elif ghuser is None and ghpass is not None:
        print("You need to add your GHUSER (Github username) to the environment.")
        return
    elif ghpass is None and ghuser is not None:
        print("You need to add your GHPASS (Github password) to the environment.")
        return

    base = "https://api.github.com"
    if number is None:
        url = "/".join(["repos", owner, repo, "issues"])
        if versions:
            data = {"title": title, "body": body + "\n" + _print_versions()}
        else:
            data = {"title": title, "body": body}
    else:
        url = "/".join(["repos", owner, repo, "issues", str(number), "comments"])
        if versions:
            data = {"body": body + "\n" + _print_versions()}
        else:
            data = {"body": body}
    issues_url = urljoin(base, url)

    print("\nPreview:\n")
    for label, content in sorted(data.items(), reverse=True):
        print('{0}: {1}'.format(label, content))
    value = input('Submit the intended issue/comment? ')
    if value.lower() in ["true", "yes", "y", "on", "1"]:
        r = requests.post(issues_url,
                          auth=(ghuser, ghpass),
                          headers={'Content-Type': 'application/json'},
                          data=json.dumps(data))
        if r.status_code == 201:
            g = requests.get(issues_url)
            if number is None:
                print("Issue successfully submitted.")
                if browser:
                    webbrowser.open_new(g.json()[0].get("html_url"))
            else:
                print("Comment successfully submitted.")
                g = requests.get(issues_url)
                if browser:
                    webbrowser.open_new(g.json()[-1].get("html_url"))
        else:
            print("Something failed, please check your GHUSER and GHPASS.")
    else:
        print("Issue not submitted.")

def test(verbosity=1, xunitfile=None, exit=False):
    """
    Runs the full Bokeh test suite, outputting
    the results of the tests to sys.stdout.

    This uses nose tests to discover which tests to
    run, and runs tests in any 'tests' subdirectory
    within the Bokeh module.

    Parameters
    ----------
    verbosity : int, optional
        Value 0 prints very little, 1 prints a little bit,
        and 2 prints the test names while testing.
    xunitfile : string, optional
        If provided, writes the test results to an xunit
        style xml file. This is useful for running the tests
        in a CI server such as Jenkins.
    exit : bool, optional
        If True, the function will call sys.exit with an
        error code after the tests are finished.
    """
    import nose
    import os
    import sys
    argv = ['nosetests', '--verbosity=%d' % verbosity]
    # Output an xunit file if requested
    if xunitfile:
        argv.extend(['--with-xunit', '--xunit-file=%s' % xunitfile])
    # Set the logging level to warn
    argv.extend(['--logging-level=WARN'])
    # Add all 'tests' subdirectories to the options
    rootdir = os.path.dirname(__file__)
    for root, dirs, files in os.walk(rootdir):
        if 'tests' in dirs:
            testsdir = os.path.join(root, 'tests')
            argv.append(testsdir)
            print('Test dir: %s' % testsdir[len(rootdir)+1:])
    # print versions (handy when reporting problems)
    print_versions()
    sys.stdout.flush()
    # Ask nose to do its thing
    return nose.main(argv=argv, exit=exit)
<commit_msg>remove serverconfig imports from __init__.py<commit_after>from __future__ import absolute_import, print_function
from ._version import get_versions
import os
try:
    from .__conda_version__ import conda_version
    __version__ = conda_version.replace("'","")
    del conda_version
except ImportError:
    __version__ = get_versions()['version']
    del get_versions

_notebook_loaded = None

def load_notebook(resources=None, verbose=False, force=False):
    ''' Prepare the IPython notebook for displaying Bokeh plots.

    Args:
        resources (Resource, optional) : a resource object describing how and where to load BokehJS from
        verbose (bool, optional) : whether to report detailed settings (default: False)
        force (bool, optional) : whether to skip IPython notebook check (default: False)

    Returns:
        None

    '''
    global _notebook_loaded

    # It's possible the IPython folks will chance things in the future, `force` parameter
    # provides an escape hatch as long as `displaypub` works
    if not force:
        notebook = False
        try:
            notebook = 'notebook' in get_ipython().config['IPKernelApp']['parent_appname']
        except Exception:
            pass
        if not notebook:
            raise RuntimeError('load_notebook() only works inside an IPython Notebook.')

    import IPython.core.displaypub as displaypub
    from .resources import INLINE
    from .templates import NOTEBOOK_LOAD, RESOURCES

    if resources is None:
        resources = INLINE

    plot_resources = RESOURCES.render(
        js_raw = resources.js_raw,
        css_raw = resources.css_raw,
        js_files = resources.js_files,
        css_files = resources.css_files,
    )

    if resources.mode == 'inline':
        js_info = 'inline'
        css_info = 'inline'
    else:
        js_info = resources.js_files[0] if len(resources.js_files) == 1 else resources.js_files
        css_info = resources.css_files[0] if len(resources.css_files) == 1 else resources.css_files

    warnings = ["Warning: " + msg['text'] for msg in resources.messages if msg['type'] == 'warn']

    if _notebook_loaded:
        warnings.append('Warning: BokehJS previously loaded')

    _notebook_loaded = resources

    html = NOTEBOOK_LOAD.render(
        plot_resources = plot_resources,
        logo_url = resources.logo_url,
        verbose = verbose,
        js_info = js_info,
        css_info = css_info,
        bokeh_version = __version__,
        warnings = warnings,
    )
    displaypub.publish_display_data('bokeh', {'text/html': html})

from .settings import settings
from . import sampledata

def _print_versions():
    """Returns all the versions of software that Bokeh relies on."""
    import platform as pt

    message = """
    Bokeh version: %s
    Python version: %s-%s
    Platform: %s
    """ % (__version__, pt.python_version(),
           pt.python_implementation(), pt.platform())
    return(message)

def print_versions():
    """Print all the versions of software that Bokeh relies on."""
    print(_print_versions())

def report_issue(number=None , owner="ContinuumIO", repo="bokeh",
                 versions=True, browser=True):
    """Opens a new Github issue programmatically.

    This "interactive" function will ask you for some minimal content
    to finally submit a new Github issue, adding essential info about
    the current setup. You can also call it with one specific issue
    number to add the essential info to an already opened issue.

    Parameters
    ----------

    number: int (default=None)
        The issue number if you want to add a new comment to an issue
        already created.

    owner: str (default="ContinuumIO")
        The owner's repository name.

    repo: str (default="bokeh")
        The name of the repository.

    versions: bool (default=True)
        Adds the `_print_versions` content information at the end of
        the body text.

    browser: bool (default=True)
        After submitting the new issue, it opens the issue webpage in
        your default web browser.

    Notes:
        * You can add the GHUSER (Github username) and
        GHPASS (Github password) to your environment to avoid
        filling this info in the interactive prompt.
        * Additionally, you can use this same function to report to any
        other project, just changing the parameters.
    """

    import requests
    import json
    import os
    import webbrowser

    from six.moves import input
    from six.moves.urllib.parse import urljoin

    print("This is the Bokeh reporting engine.\n\n"
          "Next, you will be guided to build the report")

    if number is None:
        title = input('Write the title for the intended issue: ')
        body = input('Write the body for the intended issue: ')
    else:
        body = input('Write your comment here: ')

    ghuser, ghpass = (os.environ.get(x) for x in ["GHUSER", "GHPASS"])
    if ghuser is None and ghpass is None:
        print("You need to add your GHUSER (Github username) and GHPASS (Github password)\n"
              "to the environmentor complete the next lines.")
        environment = input('Do you want to abort to set up the environment variable? ')
        if environment.lower() in ["true", "yes", "y", "on", "1"]:
            return
        else:
            ghuser = input('Write your Github username: ')
            ghpass = input('Write your Github password: ')
    elif ghuser is None and ghpass is not None:
        print("You need to add your GHUSER (Github username) to the environment.")
        return
    elif ghpass is None and ghuser is not None:
        print("You need to add your GHPASS (Github password) to the environment.")
        return

    base = "https://api.github.com"
    if number is None:
        url = "/".join(["repos", owner, repo, "issues"])
        if versions:
            data = {"title": title, "body": body + "\n" + _print_versions()}
        else:
            data = {"title": title, "body": body}
    else:
        url = "/".join(["repos", owner, repo, "issues", str(number), "comments"])
        if versions:
            data = {"body": body + "\n" + _print_versions()}
        else:
            data = {"body": body}
    issues_url = urljoin(base, url)

    print("\nPreview:\n")
    for label, content in sorted(data.items(), reverse=True):
        print('{0}: {1}'.format(label, content))
    value = input('Submit the intended issue/comment? ')
    if value.lower() in ["true", "yes", "y", "on", "1"]:
        r = requests.post(issues_url,
                          auth=(ghuser, ghpass),
                          headers={'Content-Type': 'application/json'},
                          data=json.dumps(data))
        if r.status_code == 201:
            g = requests.get(issues_url)
            if number is None:
                print("Issue successfully submitted.")
                if browser:
                    webbrowser.open_new(g.json()[0].get("html_url"))
            else:
                print("Comment successfully submitted.")
                g = requests.get(issues_url)
                if browser:
                    webbrowser.open_new(g.json()[-1].get("html_url"))
        else:
            print("Something failed, please check your GHUSER and GHPASS.")
    else:
        print("Issue not submitted.")

def test(verbosity=1, xunitfile=None, exit=False):
    """
    Runs the full Bokeh test suite, outputting
    the results of the tests to sys.stdout.

    This uses nose tests to discover which tests to
    run, and runs tests in any 'tests' subdirectory
    within the Bokeh module.

    Parameters
    ----------
    verbosity : int, optional
        Value 0 prints very little, 1 prints a little bit,
        and 2 prints the test names while testing.
    xunitfile : string, optional
        If provided, writes the test results to an xunit
        style xml file. This is useful for running the tests
        in a CI server such as Jenkins.
    exit : bool, optional
        If True, the function will call sys.exit with an
        error code after the tests are finished.
    """
    import nose
    import os
    import sys
    argv = ['nosetests', '--verbosity=%d' % verbosity]
    # Output an xunit file if requested
    if xunitfile:
        argv.extend(['--with-xunit', '--xunit-file=%s' % xunitfile])
    # Set the logging level to warn
    argv.extend(['--logging-level=WARN'])
    # Add all 'tests' subdirectories to the options
    rootdir = os.path.dirname(__file__)
    for root, dirs, files in os.walk(rootdir):
        if 'tests' in dirs:
            testsdir = os.path.join(root, 'tests')
            argv.append(testsdir)
            print('Test dir: %s' % testsdir[len(rootdir)+1:])
    # print versions (handy when reporting problems)
    print_versions()
    sys.stdout.flush()
    # Ask nose to do its thing
    return nose.main(argv=argv, exit=exit)
<|endoftext|>


<commit_before>import multiprocessing
import logging
import os
import signal
import socket
import sys
import time
import unittest

LOG_PATH = "transport.log"
SERVER_ADDRESS = ('localhost', 3333)
BUFFER_SIZE = 255

log = logging.getLogger(__name__)

class Socket(socket.socket):
    def __init__(self):
        super(Socket, self).__init__(socket.AF_INET, socket.SOCK_STREAM)

def daemon(server_address, execute=lambda request: request, condition=lambda: True):
    ss = Socket()
    ss.bind(server_address)
    ss.listen(1)
    try:
        while condition():
            log.info("SERVER: waiting for a connection...")
            sc, ad = ss.accept()
            log.info("SERVER: connection from %s", ad)
            request = ""
            while True:
                log.info("SERVER: receiving...")
                data = sc.recv(BUFFER_SIZE)
                log.info("SERVER: received '%s'", data)
                if not data:
                    log.info("SERVER: no more data to receive")
                    break
                request += data
            response = execute(request)
            log.info("SERVER: sending '%s' ...", response)
            sc.sendall(response)
            log.info("SERVER: sent")
            log.info("SERVER: closing client socket...")
            sc.close()
            log.info("SERVER: closed client socket")
    finally:
        log.info("SERVER: closing server socket...")
        ss.close()
        log.info("SERVER: closed server socket")

def client(server_address, request):
    ss = Socket()
    log.info("CLIENT: connecting '%s' ...", server_address)
    ss.connect(server_address)
    log.info("CLIENT: connected")
    log.info("CLIENT: sending '%s' ...", request)
    ss.sendall(request)
    ss.shutdown(socket.SHUT_WR)
    log.info("CLIENT: sent")
    response = ""
    while True:
        log.info("CLIENT: receiving...")
        data = ss.recv(BUFFER_SIZE)
        log.info("CLIENT: received '%s'", data)
        if not data:
            log.info("CLIENT: no more data to receive")
            break
        response += data
    log.info("CLIENT: closing...")
    ss.close()
    log.info("CLIENT: closed")
    return response

def init_test_log(path):
    logging.basicConfig(filename=path, level=logging.DEBUG, filemode='a')
    logging.FileHandler(path, mode='w').close() # log cleanup

class LogTestMixin(object):
    def _get_log(self):
        with open(self.LOG_PATH) as f:
            return f.read()

    def setUp(self):
        init_test_log(self.LOG_PATH)
        super(LogTestMixin, self).setUp()

    def assertLogContains(self, expected):
        self.assertIn(expected, self._get_log())

class TestProcess(multiprocessing.Process):
    def kill(self):
        if sys.platform == 'win32':
            self.terminate()
        else:
            os.kill(self.pid, signal.SIGINT)

class TestLog(LogTestMixin, unittest.TestCase):
    LOG_PATH = LOG_PATH

    def test_read_write_ok(self):
        log.info("sample message")
        self.assertLogContains("sample message")

    def test_read_write_mismatch(self):
        log.info("new message")
        self.assertRaisesRegexp(
            AssertionError, r"'sample message' not found in 'INFO:.*:new message",
            self.assertLogContains, "sample message")

def test_daemon(log_path, *args):
    init_test_log(log_path)
    daemon(*args)

class Test(LogTestMixin, unittest.TestCase):
    LOG_PATH = LOG_PATH

    def setUp(self):
        super(Test, self).setUp()
        self.s = TestProcess(target=test_daemon, args=(self.LOG_PATH, SERVER_ADDRESS,))
        self.s.start()
        time.sleep(1)

    def tearDown(self):
        try:
            self.s.kill()
        except OSError, e:
            # process already killed
            pass
        self.s.join()
        super(Test, self).tearDown()

    def test_server_is_listening(self):
        self.assertLogContains("waiting for a connection...")

    def test_server_shutdown(self):
        self.s.kill()
        self.s.join()
        self.assertLogContains("closing server socket...")
        self.assertLogContains("closed server socket")

    def test_client_connect(self):
        self.assertEqual(client(SERVER_ADDRESS, "prova"), "prova")
        time.sleep(1)
        self.assertLogContains("CLIENT: connecting '('localhost', 3333)' ...")
        self.assertLogContains("CLIENT: connected")
        self.assertLogContains("CLIENT: sending 'prova'")
        self.assertLogContains("CLIENT: sent")
        self.assertLogContains("CLIENT: receiving...")
        self.assertLogContains("SERVER: connection from ('127.0.0.1', ")
        self.assertLogContains("SERVER: receiving...")
        self.assertLogContains("SERVER: received 'prova'")
        self.assertLogContains("SERVER: received ''")
        self.assertLogContains("SERVER: no more data to receive")
        self.assertLogContains("SERVER: sending 'prova'")
        self.assertLogContains("SERVER: sent")
        self.assertLogContains("SERVER: closing client socket")
        self.assertLogContains("SERVER: closed client socket")
        self.assertLogContains("CLIENT: received 'prova'")
        self.assertLogContains("CLIENT: received ''")
        self.assertLogContains("CLIENT: no more data to receive")
        self.assertLogContains("CLIENT: closing")
        self.assertLogContains("CLIENT: closed")

if __name__ == '__main__':
    unittest.main()
    # print(globals()[sys.argv[1]](*sys.argv[2:]))
<commit_msg>trasport test refactoring<commit_after>import multiprocessing
import logging
import os
import signal
import socket
import sys
import time
import unittest

LOG_PATH = "transport.log"
SERVER_ADDRESS = ('localhost', 3333)
BUFFER_SIZE = 255

log = logging.getLogger(__name__)

class Socket(socket.socket):
    def __init__(self):
        super(Socket, self).__init__(socket.AF_INET, socket.SOCK_STREAM)

def daemon(server_address, execute=lambda request: request, condition=lambda: True):
    ss = Socket()
    ss.bind(server_address)
    ss.listen(1)
    try:
        while condition():
            log.info("SERVER: waiting for a connection...")
            sc, ad = ss.accept()
            log.info("SERVER: connection from %s", ad)
            request = ""
            while True:
                log.info("SERVER: receiving...")
                data = sc.recv(BUFFER_SIZE)
                log.info("SERVER: received '%s'", data)
                if not data:
                    log.info("SERVER: no more data to receive")
                    break
                request += data
            response = execute(request)
            log.info("SERVER: sending '%s' ...", response)
            sc.sendall(response)
            log.info("SERVER: sent")
            log.info("SERVER: closing client socket...")
            sc.close()
            log.info("SERVER: closed client socket")
    finally:
        log.info("SERVER: closing server socket...")
        ss.close()
        log.info("SERVER: closed server socket")

def client(server_address, request):
    ss = Socket()
    log.info("CLIENT: connecting '%s' ...", server_address)
    ss.connect(server_address)
    log.info("CLIENT: connected")
    log.info("CLIENT: sending '%s' ...", request)
    ss.sendall(request)
    ss.shutdown(socket.SHUT_WR)
    log.info("CLIENT: sent")
    response = ""
    while True:
        log.info("CLIENT: receiving...")
        data = ss.recv(BUFFER_SIZE)
        log.info("CLIENT: received '%s'", data)
        if not data:
            log.info("CLIENT: no more data to receive")
            break
        response += data
    log.info("CLIENT: closing...")
    ss.close()
    log.info("CLIENT: closed")
    return response

def init_test_log(path):
    logging.basicConfig(filename=path, level=logging.DEBUG, filemode='a')
    logging.FileHandler(path, mode='w').close() # log cleanup

class LogTestMixin(object):
    def _get_log(self):
        with open(self.LOG_PATH) as f:
            return f.read()

    def setUp(self):
        init_test_log(self.LOG_PATH)
        super(LogTestMixin, self).setUp()

    def assertLogContains(self, expected):
        self.assertIn(expected, self._get_log())

class TestProcess(multiprocessing.Process):
    def kill(self):
        if sys.platform == 'win32':
            self.terminate()
        else:
            os.kill(self.pid, signal.SIGINT)

class TestLog(LogTestMixin, unittest.TestCase):
    LOG_PATH = LOG_PATH

    def test_read_write_ok(self):
        log.info("sample message")
        self.assertLogContains("sample message")

    def test_read_write_mismatch(self):
        log.info("new message")
        self.assertRaisesRegexp(
            AssertionError, r"'sample message' not found in 'INFO:.*:new message",
            self.assertLogContains, "sample message")

def test_daemon(log_path, *args):
    init_test_log(log_path)
    daemon(*args)

class Test(LogTestMixin, unittest.TestCase):
    LOG_PATH = LOG_PATH

    def setUp(self):
        super(Test, self).setUp()
        self.s = TestProcess(target=test_daemon, args=(self.LOG_PATH, SERVER_ADDRESS,))
        self.s.start()
        time.sleep(1)

    def tearDown(self):
        try:
            self.s.kill()
        except OSError, e:
            # process already killed
            pass
        self.s.join()
        super(Test, self).tearDown()

    def test_server_is_listening(self):
        self.assertLogContains("waiting for a connection...")

    @unittest.skipIf(sys.platform == 'win32', "no clean shutdown on Windows")
    def test_server_shutdown(self):
        self.s.kill()
        self.s.join()
        self.assertLogContains("closing server socket...")
        self.assertLogContains("closed server socket")

    def test_client_connect(self):
        self.assertEqual(client(SERVER_ADDRESS, "prova"), "prova")
        time.sleep(1)
        self.assertLogContains("CLIENT: connecting '('localhost', 3333)' ...")
        self.assertLogContains("CLIENT: connected")
        self.assertLogContains("CLIENT: sending 'prova'")
        self.assertLogContains("CLIENT: sent")
        self.assertLogContains("CLIENT: receiving...")
        self.assertLogContains("SERVER: connection from ('127.0.0.1', ")
        self.assertLogContains("SERVER: receiving...")
        self.assertLogContains("SERVER: received 'prova'")
        self.assertLogContains("SERVER: received ''")
        self.assertLogContains("SERVER: no more data to receive")
        self.assertLogContains("SERVER: sending 'prova'")
        self.assertLogContains("SERVER: sent")
        self.assertLogContains("SERVER: closing client socket")
        self.assertLogContains("SERVER: closed client socket")
        self.assertLogContains("CLIENT: received 'prova'")
        self.assertLogContains("CLIENT: received ''")
        self.assertLogContains("CLIENT: no more data to receive")
        self.assertLogContains("CLIENT: closing")
        self.assertLogContains("CLIENT: closed")

if __name__ == '__main__':
    unittest.main()
    # print(globals()[sys.argv[1]](*sys.argv[2:]))
<|endoftext|>


<commit_before>"""Basic cleanup on lightcurves (trimming, sigma-clipping)."""

import logging

import numpy as np

import k2spin.utils as utils

def trim(time, flux, unc_flux):
    """Remove infs, NaNs, and negative flux values.

    Inputs
    ------
    time, flux, unc_flux: array_like

    Outputs
    -------
    trimmed_time, trimmed_flux, trimmed_unc: arrays
    good: boolean mask, locations that were kept

    """

    good = np.where((np.isfinite(flux)==True) & (flux>0) &
                    (np.isfinite(unc_flux)==True) &
                    (np.isfinite(time)==True))[0]

    trimmed_time = time[good]
    trimmed_flux = flux[good]
    trimmed_unc = unc_flux[good]

    return trimmed_time, trimmed_flux, trimmed_unc, good

def sigma_clip(time, flux, unc_flux, clip_at=6):
    """Perform sigma-clipping on the lightcurve.

    Inputs
    ------
    time, flux, unc_flux: array_like

    clip_at: float (optional)
        how many sigma to clip at. Defaults to 6.

    Outputs
    -------
    clipped_time, clipped_flux, clipped_unc: arrays

    to_keep: boolean mask of locations that were kept
    """

    # Compute statistics on the lightcurve
    med, stdev  = utils.stats(flux, unc_flux)

    # Sigma-clip the lightcurve
    outliers = abs(flux-med)>(stdev*clip_at)
    to_clip = np.where(outliers==True)[0]
    to_keep = np.where(outliers==False)[0]
    logging.debug("Sigma-clipping")
    logging.debug(to_clip)
    clipped_time = np.delete(time, to_clip)
    clipped_flux = np.delete(flux, to_clip)
    clipped_unc = np.delete(unc_flux, to_clip)

    # Return clipped lightcurve
    return clipped_time, clipped_flux, clipped_unc, to_keep

def prep_lc(time, flux, unc_flux, clip_at=6):
    """Trim, sigma-clip, and calculate stats on a lc.

    Inputs
    ------
    time, flux, unc_flux: array_like

    clip_at: float (optional)
        How many sigma to clip at. Defaults to 6.
        Set to None for no sigma clipping

    Outputs
    -------
    clean_time, clean_flux, clean_unc: arrays
    """

    # Trim the lightcurve, remove bad values
    t_time, t_flux, t_unc, t_kept = trim(time, flux, unc_flux)

    # Run sigma-clipping if desired
    if clip_at is not None:
        c_time, c_flux, c_unc, c_kept = sigma_clip(t_time, t_flux, t_unc,
                                                   clip_at=clip_at)
    else:
        c_time, c_flux, c_unc, c_kept = t_time, t_flux, t_unc, t_kept

    all_kept = t_kept[c_kept]

    # Calculate statistics on lightcurve
    c_med, c_stdev = utils.stats(c_flux, c_unc)

    # Return cleaned lightcurve and statistics
    return c_time, c_flux, c_unc, c_med, c_stdev, all_kept
<commit_msg>sigma-clip twice<commit_after>"""Basic cleanup on lightcurves (trimming, sigma-clipping)."""

import logging

import numpy as np

import k2spin.utils as utils

def trim(time, flux, unc_flux):
    """Remove infs, NaNs, and negative flux values.

    Inputs
    ------
    time, flux, unc_flux: array_like

    Outputs
    -------
    trimmed_time, trimmed_flux, trimmed_unc: arrays
    good: boolean mask, locations that were kept

    """

    good = np.where((np.isfinite(flux)==True) & (flux>0) &
                    (np.isfinite(unc_flux)==True) &
                    (np.isfinite(time)==True))[0]

    trimmed_time = time[good]
    trimmed_flux = flux[good]
    trimmed_unc = unc_flux[good]

    return trimmed_time, trimmed_flux, trimmed_unc, good

def sigma_clip(time, flux, unc_flux, clip_at=6):
    """Perform sigma-clipping on the lightcurve.

    Inputs
    ------
    time, flux, unc_flux: array_like

    clip_at: float (optional)
        how many sigma to clip at. Defaults to 6.

    Outputs
    -------
    clipped_time, clipped_flux, clipped_unc: arrays

    to_keep: boolean mask of locations that were kept
    """

    # Compute statistics on the lightcurve
    med, stdev  = utils.stats(flux, unc_flux)

    # Sigma-clip the lightcurve
    outliers = abs(flux-med)>(stdev*clip_at)
    to_clip = np.where(outliers==True)[0]
    to_keep = np.where(outliers==False)[0]
    logging.debug("Sigma-clipping")
    logging.debug(to_clip)
    clipped_time = np.delete(time, to_clip)
    clipped_flux = np.delete(flux, to_clip)
    clipped_unc = np.delete(unc_flux, to_clip)

    # Return clipped lightcurve
    return clipped_time, clipped_flux, clipped_unc, to_keep

def prep_lc(time, flux, unc_flux, clip_at=6):
    """Trim, sigma-clip, and calculate stats on a lc.

    Inputs
    ------
    time, flux, unc_flux: array_like

    clip_at: float (optional)
        How many sigma to clip at. Defaults to 6.
        Set to None for no sigma clipping

    Outputs
    -------
    clean_time, clean_flux, clean_unc: arrays
    """

    # Trim the lightcurve, remove bad values
    t_time, t_flux, t_unc, t_kept = trim(time, flux, unc_flux)

    # Run sigma-clipping if desired, repeat 2X
    if clip_at is not None:
        c_time, c_flux, c_unc, c_kept = sigma_clip(t_time, t_flux, t_unc,
                                                   clip_at=clip_at)
        c_time, c_flux, c_unc, c_kept2 = sigma_clip(c_time, c_flux, c_unc,
                                                    clip_at=clip_at)
        c_kept = c_kept[c_kept2]
    else:
        c_time, c_flux, c_unc, c_kept = t_time, t_flux, t_unc, t_kept

    all_kept = t_kept[c_kept]

    # Calculate statistics on lightcurve
    c_med, c_stdev = utils.stats(c_flux, c_unc)

    # Return cleaned lightcurve and statistics
    return c_time, c_flux, c_unc, c_med, c_stdev, all_kept
<|endoftext|>


<commit_before>"""
FITS File Reader

Notes
-----
PyFITS
    [1] Due to the way PyFITS works with images the header dictionary may
    differ depending on whether is accessed before or after the fits[0].data
    is requested. If the header is read before the data then the original
    header will be returned. If the header is read after the data has been
    accessed then the data will have been scaled and a modified header
    reflecting these changes will be returned: BITPIX may differ and
    BSCALE and B_ZERO may be dropped in the modified version.

    [2] The verify('fix') call attempts to handle violations of the FITS
    standard. For example, nan values will be converted to "nan" strings.
    Attempting to cast a pyfits header to a dictionary while it contains
    invalid header tags will result in an error so verifying it early on
    makes the header easier to work with later.

References
----------
| http://stackoverflow.com/questions/456672/class-factory-in-python
| http://stsdas.stsci.edu/download/wikidocs/The_PyFITS_Handbook.pdf

"""
from __future__ import absolute_import

import os
try:
    import astropy.io.fits as pyfits
except ImportError:
    import pyfits

from sunpy.io.header import FileHeader

__all__ = ['read', 'get_header', 'write']

__author__ = "Keith Hughitt, Stuart Mumford"
__email__ = "keith.hughitt@nasa.gov"

def read(filepath):
    """
    Read a fits file

    Parameters
    ----------
    filepath : string
        The fits file to be read

    Returns
    -------
    pairs : list
        A list of (data, header) tuples

    Notes
    -----
    This routine reads all the HDU's in a fits file and returns a list of the
    data and a FileHeader instance for each one.
    Also all comments in the original file are concatenated into a single
    'comment' key in the returned FileHeader.
    """
    hdulist = pyfits.open(filepath)
    try:
        hdulist.verify('silentfix')

        pairs = []
        for hdu in hdulist:
            comment = "".join(hdu.header.get_comment()).strip()
            history = "".join(hdu.header.get_history()).strip()
            header = FileHeader(hdu.header)
            header.pop('COMMENT', None)
            header.pop('HISTORY', None)
            header['COMMENT'] = comment
            header['HISTORY'] = history
            pairs.append((hdu.data, header))
    finally:
        hdulist.close()

    return pairs

def get_header(filepath):
    """
    Read a fits file and return just the headers for all HDU's

    Parameters
    ----------
    filepath : string
        The file to be read

    Returns
    -------
    headers : list
        A list of FileHeader headers
    """
    hdulist = pyfits.open(filepath)
    try:
        hdulist.verify('silentfix')
        headers= []
        for hdu in hdulist:
            comment = "".join(hdu.header.get_comment()).strip()
            history = "".join(hdu.header.get_history()).strip()
            header = FileHeader(hdu.header)
            header.pop('COMMENT', None)
            header.pop('HISTORY', None)
            header['COMMENT'] = comment
            header['HISTORY'] = history
            headers.append(header)
    finally:
        hdulist.close()
    return headers

def write(fname, data, header, **kwargs):
    """
    Take a data header pair and write a fits file

    Parameters
    ----------
    fname: str
        File name, with extension

    data: ndarray
        n-dimensional data array

    header: dict
        A header dictionary
    """
    #The comments need to be added to the header seperately from the normal
    # kwargs. Find and deal with them:
    fits_header = pyfits.Header()
    # Check Header
    for k,v in header.items():
        if isinstance(v, pyfits.header._HeaderCommentaryCards):
            if k is 'comments':
                fits_header.add_comments(str(v))
            elif k in 'history':
                fits_header.add_history(str(v))
            else:
                fits_header.append(pyfits.Card(k, str(v)))
        else:
            fits_header.append(pyfits.Card(k,v))

    fitskwargs = {'output_verify':'fix'}
    fitskwargs.update(kwargs)
    pyfits.writeto(os.path.expanduser(fname), data, header=fits_header, **fitskwargs)<commit_msg>Silly patch is silly<commit_after>"""
FITS File Reader

Notes
-----
PyFITS
    [1] Due to the way PyFITS works with images the header dictionary may
    differ depending on whether is accessed before or after the fits[0].data
    is requested. If the header is read before the data then the original
    header will be returned. If the header is read after the data has been
    accessed then the data will have been scaled and a modified header
    reflecting these changes will be returned: BITPIX may differ and
    BSCALE and B_ZERO may be dropped in the modified version.

    [2] The verify('fix') call attempts to handle violations of the FITS
    standard. For example, nan values will be converted to "nan" strings.
    Attempting to cast a pyfits header to a dictionary while it contains
    invalid header tags will result in an error so verifying it early on
    makes the header easier to work with later.

References
----------
| http://stackoverflow.com/questions/456672/class-factory-in-python
| http://stsdas.stsci.edu/download/wikidocs/The_PyFITS_Handbook.pdf

"""
from __future__ import absolute_import

import os
try:
    import astropy.io.fits as pyfits
except ImportError:
    import pyfits

from sunpy.io.header import FileHeader

__all__ = ['read', 'get_header', 'write']

__author__ = "Keith Hughitt, Stuart Mumford"
__email__ = "keith.hughitt@nasa.gov"

def read(filepath):
    """
    Read a fits file

    Parameters
    ----------
    filepath : string
        The fits file to be read

    Returns
    -------
    pairs : list
        A list of (data, header) tuples

    Notes
    -----
    This routine reads all the HDU's in a fits file and returns a list of the
    data and a FileHeader instance for each one.
    Also all comments in the original file are concatenated into a single
    'comment' key in the returned FileHeader.
    """
    hdulist = pyfits.open(filepath)
    try:
        hdulist.verify('silentfix')

        pairs = []
        for hdu in hdulist:
            comment = "".join(hdu.header.get_comment()).strip()
            history = "".join(hdu.header.get_history()).strip()
            header = FileHeader(hdu.header)
            header['COMMENT'] = comment
            header['HISTORY'] = history
            pairs.append((hdu.data, header))
    finally:
        hdulist.close()

    return pairs

def get_header(filepath):
    """
    Read a fits file and return just the headers for all HDU's

    Parameters
    ----------
    filepath : string
        The file to be read

    Returns
    -------
    headers : list
        A list of FileHeader headers
    """
    hdulist = pyfits.open(filepath)
    try:
        hdulist.verify('silentfix')
        headers= []
        for hdu in hdulist:
            comment = "".join(hdu.header.get_comment()).strip()
            history = "".join(hdu.header.get_history()).strip()
            header = FileHeader(hdu.header)
            header['COMMENT'] = comment
            header['HISTORY'] = history
            headers.append(header)
    finally:
        hdulist.close()
    return headers

def write(fname, data, header, **kwargs):
    """
    Take a data header pair and write a fits file

    Parameters
    ----------
    fname: str
        File name, with extension

    data: ndarray
        n-dimensional data array

    header: dict
        A header dictionary
    """
    #The comments need to be added to the header seperately from the normal
    # kwargs. Find and deal with them:
    fits_header = pyfits.Header()
    # Check Header
    for k,v in header.items():
        if isinstance(v, pyfits.header._HeaderCommentaryCards):
            if k is 'comments':
                fits_header.add_comments(str(v))
            elif k in 'history':
                fits_header.add_history(str(v))
            else:
                fits_header.append(pyfits.Card(k, str(v)))
        else:
            fits_header.append(pyfits.Card(k,v))

    fitskwargs = {'output_verify':'fix'}
    fitskwargs.update(kwargs)
    pyfits.writeto(os.path.expanduser(fname), data, header=fits_header, **fitskwargs)<|endoftext|>


<commit_before>#!/usr/bin/env python

from lib import assignee_disambiguation
from lib import lawyer_disambiguation
from lib import geoalchemy
<<<<<<< HEAD
import datetime

t1 = datetime.datetime.now()

||||||| merged common ancestors
import datetime

t1 = datetime.datetime.now()

## run geocoding
#print "START: geocode", t1
#import lib.geocode
##print "   - Loc Merge", "\n   -", datetime.datetime.now()-t1
#print"DONE: geocode"
#print "   -", datetime.datetime.now()-t1

=======

>>>>>>> c740e03e1b1ddef82669973da6412b862b56aa88
# run assignee disambiguation and populate the Assignee table
assignee_disambiguation.run_disambiguation()

# run lawyer disambiguation
lawyer_disambiguation.run_disambiguation()

#Run new geocoding
geoalchemy.main()
<commit_msg>Remerged - didn't work properly before<commit_after>#!/usr/bin/env python

from lib import assignee_disambiguation
from lib import lawyer_disambiguation
from lib import geoalchemy

# run assignee disambiguation and populate the Assignee table
assignee_disambiguation.run_disambiguation()

# run lawyer disambiguation
lawyer_disambiguation.run_disambiguation()

#Run new geocoding
geoalchemy.main()
<|endoftext|>


<commit_before>import sys
import re
import json
from Queue import Queue

class Node(object):
    def __init__(self, data):
        self.data = data.rstrip()
        self.children = []

    def add_child(self, obj):
        self.children.append(obj)

class NodeEncoder(json.JSONEncoder):
    def default(self, obj):
        if not isinstance(obj, Node):
            return super(NodeEncoder, self).default(obj)

        return obj.__dict__

def peek_stack(stack):
        # No peek for list? No Stack for python?
        current_parent = stack[len(stack) - 1]
        return current_parent

def main():
    q = Queue()
    s = []
    stop_reading = False

    """
    Detecting depth:
        '|-- ' - A Child for the above node.
        '`-- ' - Last child for the above node.
        '|   ' - Parent is a child.
        '    ' - Parent is a child.
    """
    node_depth = re.compile(r"(\|\-\- |\|   |\`\-\- |    )")

    # Convert input to a queue
    for line in sys.stdin:
        if line is not "\n":
            q.put(line)

    # Grab the root node to create the tree
    tree = Node(q.get())
    s.append(tree)
    last_depth = 1
    last_node = s[0]

    # Start parsing the rest of the tree
    for elem in list(q.queue):
        current_depth = len(node_depth.findall(elem))
        try:
            current_node = Node(re.search("\w+\.?.*$", elem).group(0))
        except:
            current_node = None
            stop_reading = True

        if (current_node is not None) and (stop_reading is False):
            if current_depth == last_depth:
                current_parent = peek_stack(s)
                current_parent.add_child(current_node)
                last_node = current_node
                last_depth = current_depth
            elif current_depth > last_depth:
                s.append(last_node)
                current_parent = peek_stack(s)
                current_parent.add_child(current_node)
                last_node = current_node
                last_depth = current_depth
            elif current_depth < last_depth:
                #print "This must less: " + str((last_depth - current_depth))
                for i in xrange((last_depth - current_depth)):
                    s.pop()
                current_parent = peek_stack(s)
                current_parent.add_child(current_node)
                last_node = current_node
                last_depth = current_depth

    print json.dumps(tree, cls=NodeEncoder)

if __name__ == '__main__':
    main()
<commit_msg>Print pretty json<commit_after>import sys
import re
import json
from Queue import Queue

class Node(object):
    def __init__(self, data):
        self.data = data.rstrip()
        self.children = []

    def add_child(self, obj):
        self.children.append(obj)

class NodeEncoder(json.JSONEncoder):
    def default(self, obj):
        if not isinstance(obj, Node):
            return super(NodeEncoder, self).default(obj)

        return obj.__dict__

def peek_stack(stack):
        # No peek for list? No Stack for python?
        current_parent = stack[len(stack) - 1]
        return current_parent

def main():
    q = Queue()
    s = []
    stop_reading = False

    """
    Detecting depth:
        '|-- ' - A Child for the above node.
        '`-- ' - Last child for the above node.
        '|   ' - Parent is a child.
        '    ' - Parent is a child.
    """
    node_depth = re.compile(r"(\|\-\- |\|   |\`\-\- |    )")

    # Convert input to a queue
    for line in sys.stdin:
        if line is not "\n":
            q.put(line)

    # Grab the root node to create the tree
    tree = Node(q.get())
    s.append(tree)
    last_depth = 1
    last_node = s[0]

    # Start parsing the rest of the tree
    for elem in list(q.queue):
        current_depth = len(node_depth.findall(elem))
        try:
            current_node = Node(re.search("\w+\.?.*$", elem).group(0))
        except:
            current_node = None
            stop_reading = True

        if (current_node is not None) and (stop_reading is False):
            if current_depth == last_depth:
                current_parent = peek_stack(s)
                current_parent.add_child(current_node)
                last_node = current_node
                last_depth = current_depth
            elif current_depth > last_depth:
                s.append(last_node)
                current_parent = peek_stack(s)
                current_parent.add_child(current_node)
                last_node = current_node
                last_depth = current_depth
            elif current_depth < last_depth:
                #print "This must less: " + str((last_depth - current_depth))
                for i in xrange((last_depth - current_depth)):
                    s.pop()
                current_parent = peek_stack(s)
                current_parent.add_child(current_node)
                last_node = current_node
                last_depth = current_depth

    print json.dumps(tree, cls=NodeEncoder, indent=2, separators=(',', ': '))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
    dict
    ~~~~

    Chinese/English Translation

    :date:      09/12/2013
    :author:    Feei <feei@feei.cn>
    :homepage:  https://github.com/wufeifei/dict
    :license:   MIT, see LICENSE for more details.
    :copyright: Copyright (c) 2017 Feei. All rights reserved
"""
from __future__ import unicode_literals
import sys
import json
import re

__name__ = 'dict-cli'
__version__ = '1.3.4'
__description__ = '命令行下中英文翻译工具（Chinese and English translation tools in the command line）'
__keywords__ = 'Translation English2Chinese Chinese2English Command-line'
__author__ = 'Feei'
__contact__ = 'feei@feei.cn'
__url__ = 'https://github.com/wufeifei/dict'
__license__ = 'MIT'

try:
    # For Python 3.0 and later
    from urllib.request import urlopen
    from urllib.parse import quote
except ImportError:
    # Fall back to Python 2's urllib2
    from urllib2 import urlopen
    from urllib import quote

class Dict:
    key = '716426270'
    keyFrom = 'wufeifei'
    api = 'http://fanyi.youdao.com/openapi.do' \
          '?keyfrom=wufeifei&key=716426270&type=data&doctype=json&version=1.1&q='
    content = None

    def __init__(self, argv):
        message = ''
        if len(argv) > 0:
            for s in argv:
                message = message + s + ' '
            self.api = self.api + quote(message.encode('utf-8'))
            self.translate()
        else:
            print('Usage: dict test')

    def translate(self):
        try:
            content = urlopen(self.api).read()
            self.content = json.loads(content)
            self.parse()
        except Exception as e:
            print('ERROR: Network or remote service error!')
            print(e)

    def parse(self):
        code = self.content['errorCode']
        if code == 0:  # Success
            c = None
            try:
                u = self.content['basic']['us-phonetic']  # English
                e = self.content['basic']['uk-phonetic']
            except KeyError:
                try:
                    c = self.content['basic']['phonetic']  # Chinese
                except KeyError:
                    c = 'None'
                u = 'None'
                e = 'None'

            try:
                explains = self.content['basic']['explains']
            except KeyError:
                explains = 'None'

            try:
                phrase = self.content['web']
            except KeyError:
                phrase = 'None'

            print('\033[1;31m################################### \033[0m')
            print('\033[1;31m# \033[0m {0} {1}'.format(
                self.content['query'], self.content['translation'][0]))
            if u != 'None':
                print('\033[1;31m# \033[0m (U: {0} E: {1})'.format(u, e))
            elif c != 'None':
                print('\033[1;31m# \033[0m (Pinyin: {0})'.format(c))
            else:
                print('\033[1;31m# \033[0m')

            print('\033[1;31m# \033[0m')

            if explains != 'None':
                for i in range(0, len(explains)):
                    print('\033[1;31m# \033[0m {0}'.format(explains[i]))
            else:
                print('\033[1;31m# \033[0m Explains None')

            print('\033[1;31m# \033[0m')

            if phrase != 'None':
                for p in phrase:
                    print('\033[1;31m# \033[0m {0} : {1}'.format(
                        p['key'], p['value'][0]))
                    if len(p['value']) > 0:
                        if re.match('[ \u4e00 -\u9fa5]+', p['key']) is None:
                            blank = len(p['key'].encode('gbk'))
                        else:
                            blank = len(p['key'])
                        for i in p['value'][1:]:
                            print('\033[1;31m# \033[0m {0} {1}'.format(
                                ' ' * (blank + 3), i))

            print('\033[1;31m################################### \033[0m')
            # Phrase
            # for i in range(0, len(self.content['web'])):
            #     print self.content['web'][i]['key'], ':'
            #     for j in range(0, len(self.content['web'][i]['value'])):
            #         print self.content['web'][i]['value'][j]
        elif code == 20:  # Text to long
            print('WORD TO LONG')
        elif code == 30:  # Trans error
            print('TRANSLATE ERROR')
        elif code == 40:  # Don't support this language
            print('CAN\'T SUPPORT THIS LANGUAGE')
        elif code == 50:  # Key failed
            print('KEY FAILED')
        elif code == 60:  # Don't have this word
            print('DO\'T HAVE THIS WORD')

def main():
    Dict(sys.argv[1:])

if __name__ == '__main__':
    main()
<commit_msg>improves unicode and utf-8<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
    dict
    ~~~~

    Chinese/English Translation

    :date:      09/12/2013
    :author:    Feei <feei@feei.cn>
    :homepage:  https://github.com/wufeifei/dict
    :license:   MIT, see LICENSE for more details.
    :copyright: Copyright (c) 2017 Feei. All rights reserved
"""
from __future__ import unicode_literals
import sys
import json
import re

__name__ = 'dict-cli'
__version__ = '1.3.4'
__description__ = '命令行下中英文翻译工具（Chinese and English translation tools in the command line）'
__keywords__ = 'Translation English2Chinese Chinese2English Command-line'
__author__ = 'Feei'
__contact__ = 'feei@feei.cn'
__url__ = 'https://github.com/wufeifei/dict'
__license__ = 'MIT'

try:
    # For Python 3.0 and later
    from urllib.request import urlopen
    from urllib.parse import quote
except ImportError:
    # Fall back to Python 2's urllib2
    from urllib2 import urlopen
    from urllib import quote

class Dict:
    key = '716426270'
    keyFrom = 'wufeifei'
    api = 'http://fanyi.youdao.com/openapi.do' \
          '?keyfrom=wufeifei&key=716426270&type=data&doctype=json&version=1.1&q='
    content = None

    def __init__(self, argv):
        message = ''
        if len(argv) > 0:
            for s in argv:
                message = message + s + ' '
            self.api = self.api + quote(message.encode('utf-8'))
            self.translate()
        else:
            print('Usage: dict test')

    def translate(self):
        try:
            content = urlopen(self.api).read()
            self.content = json.loads(content.decode('utf-8'))
            self.parse()
        except Exception as e:
            print('ERROR: Network or remote service error!')
            print(e)

    def parse(self):
        code = self.content['errorCode']
        if code == 0:  # Success
            c = None
            try:
                u = self.content['basic']['us-phonetic']  # English
                e = self.content['basic']['uk-phonetic']
            except KeyError:
                try:
                    c = self.content['basic']['phonetic']  # Chinese
                except KeyError:
                    c = 'None'
                u = 'None'
                e = 'None'

            try:
                explains = self.content['basic']['explains']
            except KeyError:
                explains = 'None'

            try:
                phrase = self.content['web']
            except KeyError:
                phrase = 'None'

            print('\033[1;31m################################### \033[0m')
            print('\033[1;31m# \033[0m {0} {1}'.format(
                self.content['query'], self.content['translation'][0]))
            if u != 'None':
                print('\033[1;31m# \033[0m (U: {0} E: {1})'.format(u, e))
            elif c != 'None':
                print('\033[1;31m# \033[0m (Pinyin: {0})'.format(c))
            else:
                print('\033[1;31m# \033[0m')

            print('\033[1;31m# \033[0m')

            if explains != 'None':
                for i in range(0, len(explains)):
                    print('\033[1;31m# \033[0m {0}'.format(explains[i]))
            else:
                print('\033[1;31m# \033[0m Explains None')

            print('\033[1;31m# \033[0m')

            if phrase != 'None':
                for p in phrase:
                    print('\033[1;31m# \033[0m {0} : {1}'.format(
                        p['key'], p['value'][0]))
                    if len(p['value']) > 0:
                        if re.match('[ \u4e00 -\u9fa5]+', p['key']) is None:
                            blank = len(p['key'].encode('gbk'))
                        else:
                            blank = len(p['key'])
                        for i in p['value'][1:]:
                            print('\033[1;31m# \033[0m {0} {1}'.format(
                                ' ' * (blank + 3), i))

            print('\033[1;31m################################### \033[0m')
            # Phrase
            # for i in range(0, len(self.content['web'])):
            #     print self.content['web'][i]['key'], ':'
            #     for j in range(0, len(self.content['web'][i]['value'])):
            #         print self.content['web'][i]['value'][j]
        elif code == 20:  # Text to long
            print('WORD TO LONG')
        elif code == 30:  # Trans error
            print('TRANSLATE ERROR')
        elif code == 40:  # Don't support this language
            print('CAN\'T SUPPORT THIS LANGUAGE')
        elif code == 50:  # Key failed
            print('KEY FAILED')
        elif code == 60:  # Don't have this word
            print('DO\'T HAVE THIS WORD')

def main():
    Dict(sys.argv[1:])

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#! /usr/bin/python2 -tt
# ______________________________________________________________________________
#
#  CLEAN
# ______________________________________________________________________________
# By Jules Baratoux on Wed Aug 28 2013
# Just a simple project to learn python
#
# NAME
#        clean - remove dirt from directories
#
# SYNOPSIS
#        clean [options] [directories]
#
# DESCRIPTION
#        Remove  dirt files from directories. Multiples directories can be
#        set. By default clean the current directory.
#
# OPTIONS
#       -d, --maxdepth levels
#               Descend at most  levels (a non-negative integer) levels
#               of  directories  below   the  command  line  arguments.
#
#       -f, --force
#               ignore nonexistent files, never prompt
#
#       -i      Prompt before every removal
#
#       -I      prompt once  before removing more than  three files, or
#               when  removing recursively.   Less  intrusive than  -i,
#               while still giving protection against most mistakes
#
#       -L, --follow
#               Follow symbolic
#
#       -H, --no-follow
#               Do not  follow symbolic links,  except while processing
#               the  command  line  arguments.  This  is  the   default
#               behaviour.
#
#       -R, --recursive
#               Clean directories and their subdirectories recursively
#
#       -h, --help
#               Display this help and exit
#
#       -v, --verbose
#               Explain what is being done
# ______________________________________________________________________________

import getopt, sys, os, re

class Param(object):
    def __init__(self, follow=False, recursive=False, verbose=False, maxdepth=3):
        self.follow = follow
        self.recursive = recursive
        self.verbose = verbose
        self.maxdepth = maxdepth
        self.rm_options = ""
# ______________________________________________________________________________

param = Param()

def main():
    try:
        options, argv = getopt.gnu_getopt(sys.argv[1:], "d:fiILHRhv", ["maxdepth=", "force", "follow",
                                                                       "no-follow", "help", "recursive", "verbose"])
    except getopt.GetoptError as err:
        print str(err)
        usage()
        sys.exit(2)

    for option, arg in options:
        if option in ("-h", "--help"):
            usage()
            sys.exit()
        elif option in ("-d", "--maxdepth"):
            param.maxdepth = arg
        elif option in ("-f", "--force"):
            param.rm_options += "--force "
        elif option in ("-I"):
            param.rm_options += "-I "
        elif option in ("-i"):
            param.rm_options += "-i "
        elif option in ("-R", "--recursive"):
            param.recursive = True
        elif option in ("-v", "--verbose"):
            param.verbose = True
            param.rm_options += "--verbose "
        elif option in ("-L", "--follow"):
            param.follow = True
        elif option in ("-H", "--no-follow"):
            param.follow = False
        else:
            print option
            assert False, "unhandled option"

    if len(argv) > 0:
        for arg in argv:
            clean(arg)
    else:
        clean('.')
# ______________________________________________________________________________

def resolve_link(pathname):
    pathname = os.path._resolve_link(pathname)
    return "." if pathname is None else pathname

def isdir(pathname):
    try:
        return os.path.isdir(pathname) or (os.path.islink(pathname) and os.path.isdir(resolve_link(pathname)))
    except OSError as e:
        print str(e)
        return False

def isfile(pathname):
    try:
        return os.path.isfile(pathname) or (os.path.islink(pathname) and os.path.isfile(resolve_link(pathname)))
    except OSError as e:
        print str(e)
        return False

def islink(pathname):
    return os.path.islink(pathname)
# ______________________________________________________________________________

def file_handler(path):
    if re.match( "([^~]+~$)"             + '|' +
                 "(#[^#]+#$)"            + '|' +
                r"(\.#[^#]+$)"           + '|' +
                r"(.+\.stackdump$)", os.path.basename(path)):
        remove(path)

def dir_handler(pathname, depth):
    depth += 1;
    for name in os.listdir(pathname):
        name = pathname + "/" + name
        if (depth <= param.maxdepth) and (isdir(name) and param.recursive) and (not islink(name) or param.follow):
            try:
                dir_handler(name, depth)
            except OSError as e:
                print str(e)
        elif isfile(name) or isdir(name):
            file_handler(name)

def clean(pathname, depth = 0):
    if isdir(pathname):
        dir_handler(pathname, depth)
    if isfile(pathname):
        file_handler(pathname)
# ______________________________________________________________________________

def remove(filename):
    os.system("rm " + param.rm_options + filename)

def usage():
    synopsis = "usage: "

    text = []
    f = open(sys.argv[0], 'r')
    if f:
        flag = False
        for line in f:
            if flag:
                synopsis += line[9:-1]
                break
            elif line == "# SYNOPSIS\n":
                flag = True;

        flag = False
        for line in f:
            if flag:
                if line[0:4] == "# ____..."[0:4]:
                    break
                text.append(line[1:-1])
            elif line == "# OPTIONS\n":
                flag = True

    f.close()
    print synopsis, "\n"
    for line in text:
        print line
# ______________________________________________________________________________

if __name__ == "__main__":
    main()
<commit_msg>fix readlink<commit_after>#! /usr/bin/python2 -tt
# ______________________________________________________________________________
#
#  CLEAN
# ______________________________________________________________________________
# By Jules Baratoux on Wed Aug 28 2013
# Just a simple project to learn python
#
# NAME
#        clean - remove dirt from directories
#
# SYNOPSIS
#        clean [options] [directories]
#
# DESCRIPTION
#        Remove  dirt files from directories. Multiples directories can be
#        set. By default clean the current directory.
#
# OPTIONS
#       -d, --maxdepth levels
#               Descend at most  levels (a non-negative integer) levels
#               of  directories  below   the  command  line  arguments.
#
#       -f, --force
#               ignore nonexistent files, never prompt
#
#       -i      Prompt before every removal
#
#       -I      prompt once  before removing more than  three files, or
#               when  removing recursively.   Less  intrusive than  -i,
#               while still giving protection against most mistakes
#
#       -L, --follow
#               Follow symbolic
#
#       -H, --no-follow
#               Do not  follow symbolic links,  except while processing
#               the  command  line  arguments.  This  is  the   default
#               behaviour.
#
#       -R, --recursive
#               Clean directories and their subdirectories recursively
#
#       -h, --help
#               Display this help and exit
#
#       -v, --verbose
#               Explain what is being done
# ______________________________________________________________________________

import getopt, sys, os, re

class Param(object):
    def __init__(self, follow=False, recursive=False, verbose=False, maxdepth=3):
        self.follow = follow
        self.recursive = recursive
        self.verbose = verbose
        self.maxdepth = maxdepth
        self.rm_options = ""
# ______________________________________________________________________________

param = Param()

def main():
    try:
        options, argv = getopt.gnu_getopt(sys.argv[1:], "d:fiILHRhv", ["maxdepth=", "force", "follow",
                                                                       "no-follow", "help", "recursive", "verbose"])
    except getopt.GetoptError as err:
        print str(err)
        usage()
        sys.exit(2)

    for option, arg in options:
        if option in ("-h", "--help"):
            usage()
            sys.exit()
        elif option in ("-d", "--maxdepth"):
            param.maxdepth = arg
        elif option in ("-f", "--force"):
            param.rm_options += "--force "
        elif option in ("-I"):
            param.rm_options += "-I "
        elif option in ("-i"):
            param.rm_options += "-i "
        elif option in ("-R", "--recursive"):
            param.recursive = True
        elif option in ("-v", "--verbose"):
            param.verbose = True
            param.rm_options += "--verbose "
        elif option in ("-L", "--follow"):
            param.follow = True
        elif option in ("-H", "--no-follow"):
            param.follow = False
        else:
            print option
            assert False, "unhandled option"

    if len(argv) > 0:
        for arg in argv:
            clean(arg)
    else:
        clean('.')
# ______________________________________________________________________________

def resolve_link(pathname):
    from os import readlink;
    pathname = readlink(pathname)
    return "." if pathname is None else pathname

def isdir(pathname):
    try:
        return os.path.isdir(pathname) or (os.path.islink(pathname) and os.path.isdir(resolve_link(pathname)))
    except OSError as e:
        print str(e)
        return False

def isfile(pathname):
    try:
        return os.path.isfile(pathname) or (os.path.islink(pathname) and os.path.isfile(resolve_link(pathname)))
    except OSError as e:
        print str(e)
        return False

def islink(pathname):
    return os.path.islink(pathname)
# ______________________________________________________________________________

def file_handler(path):
    if re.match( "([^~]+~$)"             + '|' +
                 "(#[^#]+#$)"            + '|' +
                r"(\.#[^#]+$)"           + '|' +
                r"(.+\.stackdump$)", os.path.basename(path)):
        remove(path)

def dir_handler(pathname, depth):
    depth += 1;
    for name in os.listdir(pathname):
        name = pathname + "/" + name
        if (depth <= param.maxdepth) and (isdir(name) and param.recursive) and (not islink(name) or param.follow):
            try:
                dir_handler(name, depth)
            except OSError as e:
                print str(e)
        elif isfile(name) or isdir(name):
            file_handler(name)

def clean(pathname, depth = 0):
    if isdir(pathname):
        dir_handler(pathname, depth)
    if isfile(pathname):
        file_handler(pathname)
# ______________________________________________________________________________

def remove(filename):
    os.system("rm " + param.rm_options + filename)

def usage():
    synopsis = "usage: "

    text = []
    f = open(sys.argv[0], 'r')
    if f:
        flag = False
        for line in f:
            if flag:
                synopsis += line[9:-1]
                break
            elif line == "# SYNOPSIS\n":
                flag = True;

        flag = False
        for line in f:
            if flag:
                if line[0:4] == "# ____..."[0:4]:
                    break
                text.append(line[1:-1])
            elif line == "# OPTIONS\n":
                flag = True

    f.close()
    print synopsis, "\n"
    for line in text:
        print line
# ______________________________________________________________________________

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># encoding: utf8
import datetime
import csv
import os
import hashlib
import locale

from sqlalchemy.orm import sessionmaker
from sqlalchemy import create_engine
from trombi import mappings

DATADIR = os.path.join(os.path.dirname(__file__), 'data')
Session = sessionmaker(autoflush=False)

locale.setlocale(locale.LC_ALL, 'fr_FR.UTF-8')

def get_hash(session, name):
    res = session.query(mappings.Hash).filter_by(name=name).first()
    if res is None:
        return None
    return res.value

def set_hash(session, name, value):
    hash = session.query(mappings.Hash).filter_by(name=name).first()
    if hash is None:
        hash = mappings.Hash(name, value)
    else:
        hash.value = value

    session.add(hash)
    session.commit()

def cvs2table(session, name, callback, delimiter=';'):
    existing_hash = get_hash(session, name)
    csvfile = os.path.join(DATADIR, u'%s.csv' % name)

    with open(csvfile) as f:
        file_hash = hashlib.md5(f.read()).hexdigest()

    if file_hash != existing_hash:
        with open(csvfile, 'rb') as f:
            reader = csv.reader(f, delimiter=delimiter)
            for index, row in enumerate(reader):
                obj = callback(index, row)
                if obj is not None:
                    session.add(obj)

        session.commit()
        set_hash(session, name, file_hash)

def init(sqluri='sqlite:////tmp/acr.db', fill=True):
    engine = create_engine(sqluri)
    Session.configure(bind=engine)
    mappings.Base.metadata.create_all(engine)

    if not fill:
        return

    session = Session()

    # Membership
    def _create_membership(index, row):
        if index == 0:
            return
        membership = mappings.Membership()
        membership.label = unicode(row[0], 'utf8')
        membership.price = int(row[1])
        return membership

    cvs2table(session, u"membership", _create_membership)

    # Categories
    def _create_cat(index, row):
        if index == 0:
            return
        cat = mappings.Category()
        cat.code = unicode(row[0], 'utf8')
        cat.label = unicode(row[1], 'utf8')
        cat.min_age = int(row[2])
        cat.max_age = int(row[3])
        return cat

    cvs2table(session, u"categories", _create_cat)

    # Cities
    done = []

    def _create_city(index, row):
        if index == 0:
            return
        label = unicode(row[1], 'utf8')
        zipcode = int(row[2])
        if (label, zipcode) in done:
            return
        city = mappings.City(label, zipcode)
        done.append((label, zipcode))
        return city

    cvs2table(session, u"communes", _create_city, delimiter='\t')

    # Members
    def _create_member(index, row):
        if index in (0, 1):
            return

        row = [unicode(value, 'utf8').strip() for value in row]
        if all(value == '' for value in row):
            return

        member = mappings.Member()
        member.permissions = "User"
        member.lastname = row[1]
        member.firstname = row[2]
        member.address = row[3]

        try:
            zipcode = int(row[4])
            label = row[5]
            if u'SAINT ' in label:  # XXX bof
                label = label.replace(u'SAINT', u'ST')

            res = session.query(mappings.City).filter_by(zipcode=zipcode,
                                                         label=label)
            city = res.one()
            if city:
                member.city = city

        except ValueError:
            pass

        member.phone = row[6].replace(' ', '')
        member.birthday = datetime.datetime.strptime(row[7],
                                                     '%m/%d/%y')
        if member.birthday.year > 2016:

            member.birthday = datetime.datetime(member.birthday.year-100,
                                                member.birthday.month,
                                                member.birthday.day)

        member.category_code = row[8]
        gender = row[9]
        if gender in (u'H', u'M'):
            gender = u'M'

        member.gender = gender
        member.email = row[10]

        try:
            price = float(row[14].replace(',', '.'))
            res = session.query(mappings.Membership).filter_by(price=price).first()
            if res:
                member.membership_label = res.label
            else:
                raise ValueError()
        except ValueError:
            label = row[11].split('\n')[-1]
            res = session.query(mappings.Membership).filter_by(label=label).first()
            if res:
                member.membership_label = label
            else:
                member.membership_label = u'Simple'
        member.licence = row[12]
        try:
            cert = datetime.datetime.strptime(row[13], '%m/%d/%y')
            member.medical_certificate_date = cert
        except ValueError:
            pass

        last_upd = row[-1].encode('utf8').lower().split('-')

        if last_upd[0] == 'fev':
            last_upd[0] = 'fév'

        if last_upd[0] == 'dec':
            last_upd[0] = 'déc'

        if last_upd[0] == 'sep':
            last_upd[0] = 'sept'

        try:
            member.last_updated = datetime.datetime.strptime('-'.join(last_upd), '%b-%y')
        except ValueError:
            pass
        member.is_published = member.has_paid = True    # XXX
        member.password = u''
        return member

    cvs2table(session, u"adherents", _create_member)
    return engine, session
<commit_msg>fix comma<commit_after># encoding: utf8
import datetime
import csv
import os
import hashlib
import locale

from sqlalchemy.orm import sessionmaker
from sqlalchemy import create_engine
from trombi import mappings

DATADIR = os.path.join(os.path.dirname(__file__), 'data')
Session = sessionmaker(autoflush=False)

locale.setlocale(locale.LC_ALL, 'fr_FR.UTF-8')

def get_hash(session, name):
    res = session.query(mappings.Hash).filter_by(name=name).first()
    if res is None:
        return None
    return res.value

def set_hash(session, name, value):
    hash = session.query(mappings.Hash).filter_by(name=name).first()
    if hash is None:
        hash = mappings.Hash(name, value)
    else:
        hash.value = value

    session.add(hash)
    session.commit()

def cvs2table(session, name, callback, delimiter=';'):
    existing_hash = get_hash(session, name)
    csvfile = os.path.join(DATADIR, u'%s.csv' % name)

    with open(csvfile) as f:
        file_hash = hashlib.md5(f.read()).hexdigest()

    if file_hash != existing_hash:
        with open(csvfile, 'rb') as f:
            reader = csv.reader(f, delimiter=delimiter)
            for index, row in enumerate(reader):
                obj = callback(index, row)
                if obj is not None:
                    session.add(obj)

        session.commit()
        set_hash(session, name, file_hash)

def init(sqluri='sqlite:////tmp/acr.db', fill=True):
    engine = create_engine(sqluri)
    Session.configure(bind=engine)
    mappings.Base.metadata.create_all(engine)

    if not fill:
        return

    session = Session()

    # Membership
    def _create_membership(index, row):
        if index == 0:
            return
        membership = mappings.Membership()
        membership.label = unicode(row[0], 'utf8')
        membership.price = int(row[1])
        return membership

    cvs2table(session, u"membership", _create_membership)

    # Categories
    def _create_cat(index, row):
        if index == 0:
            return
        cat = mappings.Category()
        cat.code = unicode(row[0], 'utf8')
        cat.label = unicode(row[1], 'utf8')
        cat.min_age = int(row[2])
        cat.max_age = int(row[3])
        return cat

    cvs2table(session, u"categories", _create_cat)

    # Cities
    done = []

    def _create_city(index, row):
        if index == 0:
            return
        label = unicode(row[1], 'utf8')
        zipcode = int(row[2])
        if (label, zipcode) in done:
            return
        city = mappings.City(label, zipcode)
        done.append((label, zipcode))
        return city

    cvs2table(session, u"communes", _create_city, delimiter='\t')

    # Members
    def _create_member(index, row):
        if index in (0, 1):
            return

        row = [unicode(value, 'utf8').strip() for value in row]
        if all(value == '' for value in row):
            return

        member = mappings.Member()
        member.permissions = "User"
        member.lastname = row[1]
        member.firstname = row[2]
        member.address = row[3]

        try:
            zipcode = int(row[4])
            label = row[5]
            if u'SAINT ' in label:  # XXX bof
                label = label.replace(u'SAINT', u'ST')

            res = session.query(mappings.City).filter_by(zipcode=zipcode,
                                                         label=label)
            city = res.one()
            if city:
                member.city = city

        except ValueError:
            pass

        member.phone = row[6].replace(' ', '')
        member.birthday = datetime.datetime.strptime(row[7],
                                                     '%m/%d/%y')
        if member.birthday.year > 2016:

            member.birthday = datetime.datetime(member.birthday.year-100,
                                                member.birthday.month,
                                                member.birthday.day)

        member.category_code = row[8]
        gender = row[9]
        if gender in (u'H', u'M'):
            gender = u'M'

        member.gender = gender
        member.email = row[10].replace(',', '.')

        try:
            price = float(row[14].replace(',', '.'))
            res = session.query(mappings.Membership).filter_by(price=price).first()
            if res:
                member.membership_label = res.label
            else:
                raise ValueError()
        except ValueError:
            label = row[11].split('\n')[-1]
            res = session.query(mappings.Membership).filter_by(label=label).first()
            if res:
                member.membership_label = label
            else:
                member.membership_label = u'Simple'
        member.licence = row[12]
        try:
            cert = datetime.datetime.strptime(row[13], '%m/%d/%y')
            member.medical_certificate_date = cert
        except ValueError:
            pass

        last_upd = row[-1].encode('utf8').lower().split('-')

        if last_upd[0] == 'fev':
            last_upd[0] = 'fév'

        if last_upd[0] == 'dec':
            last_upd[0] = 'déc'

        if last_upd[0] == 'sep':
            last_upd[0] = 'sept'

        try:
            member.last_updated = datetime.datetime.strptime('-'.join(last_upd), '%b-%y')
        except ValueError:
            pass
        member.is_published = member.has_paid = True    # XXX
        member.password = u''
        return member

    cvs2table(session, u"adherents", _create_member)
    return engine, session
<|endoftext|>


<commit_before>#!/usr/bin/env Python
from lib import alchemy
from lib import assignee_disambiguation
import datetime

t1 = datetime.datetime.now()

## run geocoding
#print "START: geocode", t1
#import lib.geocode
##print "   - Loc Merge", "\n   -", datetime.datetime.now()-t1
#print"DONE: geocode"
#print "   -", datetime.datetime.now()-t1

# run assignee disambiguation and populate the Assignee table
assignee_disambiguation.run_disambiguation()

#def run_org_clean():
#    org = orgClean.orgClean(db = 'assignee.sqlite3', fld = 'assigneeAsc', table = 'assignee_1', other = "")
#    org.disambig()
#    print"DONE: orgClean"
#    #print "   -", datetime.datetime.now()-t1
#
#    # Copy assignee num from grp to assignee table
#    s.merge(key=[['AsgNum', 'AsgNum2']], on=['AssigneeAsc'], tableFrom='grp')
#    print "DONE: Replaced Asgnum!", "\n   -", datetime.datetime.now()-t1
#    s.c.execute("""update assignee_1 set City = cc(city, country, 'city'), Country = cc(city, country, 'ctry');""")
#    s.attach('hashTbl.sqlite3')
#    s.merge(key=['NCity', 'NState', 'NCountry', 'NZipcode', 'NLat', 'NLong'],
#            on=['City', 'State', 'Country'],
#            tableFrom='locMerge', db='db')
#    s.commit()
#    print "DONE: Asg Locationize!", "\n   -", datetime.datetime.now()-t1
#
#
#run_org_clean()
#s.close()
#
#
# ###########################
####                       ###
###     I N V E N T O R     ##
####                       ###
# ###########################
#
## adds geocoding to inventors
#
#def handle_inventor():
#
#    ## Clean inventor: ascit(Firstname, Lastname, Street)
#    ## Create new table inventor_1 to hold prepped data
#
#    i = SQLite.SQLite(db = 'inventor.sqlite3', tbl = 'inventor_1')
#    i.conn.create_function("ascit", 1, fwork.ascit)
#    i.conn.create_function("cc",    3, locFunc.cityctry)
#    i.c.execute('drop table if exists inventor_1')
#    i.replicate(tableTo = 'inventor_1', table = 'inventor')
#    i.c.execute('insert or ignore into inventor_1 select * from inventor  %s' % (debug and "LIMIT 2500" or ""))
#
#    i.c.execute("""
#            UPDATE  inventor_1
#               SET  firstname = ascit(firstname),
#                    lastname  = ascit(lastname),
#                    street    = ascit(street),
#                    City      = cc(city, country, 'city'),
#                    Country   = cc(city, country, 'ctry');
#                """)
#
#    i.commit()
#    print 'inventor_1 created'
#
#    i.attach('hashTbl.sqlite3')
#    i.merge(key=['NCity', 'NState', 'NCountry', 'NZipcode', 'NLat', 'NLong'],
#            on=['City', 'State', 'Country'],
#            tableFrom='locMerge',
#            db='db')
#
##     i.merge(key=['NCity', 'NState', 'NCountry', 'NZipcode', 'NLat', 'NLong'],
##             on=['City', 'State', 'Country', 'Zipcode'],
##             tableFrom='locMerge',
##             db='db')
#
#    i.commit()
#    i.close()
#    print "DONE: Inv Locationize!", "\n   -", datetime.datetime.now()-t1
#
#handle_inventor()
#
#
# ###########################
####                       ###
###        C L A S S        ##
####                       ###
# ###########################
#
## Clean up classes
## see CleanDataSet.py --> classes()
## FIXME: Module importing not allowed in function.
## TODO: get rid of in refactor
#import lib.CleanDataset as CleanDataset
#CleanDataset.classes()
#print "DONE: Classes!", "\n   -", datetime.datetime.now()-t1
#
#
#
# ###########################
####                       ###
###       P A T E N T       ##
####                       ###
# ###########################
#
## normalizes the application date and grant date
#def handle_patent():
#    p = SQLite.SQLite(db = 'patent.sqlite3', tbl = 'patent')
#    p.conn.create_function('dVert', 1, senAdd.dateVert)
#    p.c.execute("""update patent set AppDate=dVert(AppDate), GDate=dVert(GDate);""")
#    p.commit()
#    p.close()
#    print "DONE: Patent Date!", "\n   -", datetime.datetime.now()-t1
#
#handle_patent()
<commit_msg>fix case sensitive bug with calling clean.py as shell script<commit_after>#!/usr/bin/env python

from lib import alchemy
from lib import assignee_disambiguation
import datetime

t1 = datetime.datetime.now()

## run geocoding
#print "START: geocode", t1
#import lib.geocode
##print "   - Loc Merge", "\n   -", datetime.datetime.now()-t1
#print"DONE: geocode"
#print "   -", datetime.datetime.now()-t1

# run assignee disambiguation and populate the Assignee table
assignee_disambiguation.run_disambiguation()

#def run_org_clean():
#    org = orgClean.orgClean(db = 'assignee.sqlite3', fld = 'assigneeAsc', table = 'assignee_1', other = "")
#    org.disambig()
#    print"DONE: orgClean"
#    #print "   -", datetime.datetime.now()-t1
#
#    # Copy assignee num from grp to assignee table
#    s.merge(key=[['AsgNum', 'AsgNum2']], on=['AssigneeAsc'], tableFrom='grp')
#    print "DONE: Replaced Asgnum!", "\n   -", datetime.datetime.now()-t1
#    s.c.execute("""update assignee_1 set City = cc(city, country, 'city'), Country = cc(city, country, 'ctry');""")
#    s.attach('hashTbl.sqlite3')
#    s.merge(key=['NCity', 'NState', 'NCountry', 'NZipcode', 'NLat', 'NLong'],
#            on=['City', 'State', 'Country'],
#            tableFrom='locMerge', db='db')
#    s.commit()
#    print "DONE: Asg Locationize!", "\n   -", datetime.datetime.now()-t1
#
#
#run_org_clean()
#s.close()
#
#
# ###########################
####                       ###
###     I N V E N T O R     ##
####                       ###
# ###########################
#
## adds geocoding to inventors
#
#def handle_inventor():
#
#    ## Clean inventor: ascit(Firstname, Lastname, Street)
#    ## Create new table inventor_1 to hold prepped data
#
#    i = SQLite.SQLite(db = 'inventor.sqlite3', tbl = 'inventor_1')
#    i.conn.create_function("ascit", 1, fwork.ascit)
#    i.conn.create_function("cc",    3, locFunc.cityctry)
#    i.c.execute('drop table if exists inventor_1')
#    i.replicate(tableTo = 'inventor_1', table = 'inventor')
#    i.c.execute('insert or ignore into inventor_1 select * from inventor  %s' % (debug and "LIMIT 2500" or ""))
#
#    i.c.execute("""
#            UPDATE  inventor_1
#               SET  firstname = ascit(firstname),
#                    lastname  = ascit(lastname),
#                    street    = ascit(street),
#                    City      = cc(city, country, 'city'),
#                    Country   = cc(city, country, 'ctry');
#                """)
#
#    i.commit()
#    print 'inventor_1 created'
#
#    i.attach('hashTbl.sqlite3')
#    i.merge(key=['NCity', 'NState', 'NCountry', 'NZipcode', 'NLat', 'NLong'],
#            on=['City', 'State', 'Country'],
#            tableFrom='locMerge',
#            db='db')
#
##     i.merge(key=['NCity', 'NState', 'NCountry', 'NZipcode', 'NLat', 'NLong'],
##             on=['City', 'State', 'Country', 'Zipcode'],
##             tableFrom='locMerge',
##             db='db')
#
#    i.commit()
#    i.close()
#    print "DONE: Inv Locationize!", "\n   -", datetime.datetime.now()-t1
#
#handle_inventor()
#
#
# ###########################
####                       ###
###        C L A S S        ##
####                       ###
# ###########################
#
## Clean up classes
## see CleanDataSet.py --> classes()
## FIXME: Module importing not allowed in function.
## TODO: get rid of in refactor
#import lib.CleanDataset as CleanDataset
#CleanDataset.classes()
#print "DONE: Classes!", "\n   -", datetime.datetime.now()-t1
#
#
#
# ###########################
####                       ###
###       P A T E N T       ##
####                       ###
# ###########################
#
## normalizes the application date and grant date
#def handle_patent():
#    p = SQLite.SQLite(db = 'patent.sqlite3', tbl = 'patent')
#    p.conn.create_function('dVert', 1, senAdd.dateVert)
#    p.c.execute("""update patent set AppDate=dVert(AppDate), GDate=dVert(GDate);""")
#    p.commit()
#    p.close()
#    print "DONE: Patent Date!", "\n   -", datetime.datetime.now()-t1
#
#handle_patent()
<|endoftext|>


<commit_before>#!/usr/bin/python
import sys
import os
import subprocess
import glob

subprocess.call('rm -rf Modules', shell=True)
subprocess.call('rm -rf *lib_hao_0', shell=True)
subprocess.call('rm -rf *lib_hao_1', shell=True)
<commit_msg>change clean.py<commit_after>#!/usr/bin/python
import sys
import os
import subprocess
import glob

subprocess.call('rm -rf Modules', shell=True)
subprocess.call('rm -rf *lib_hao_*', shell=True)
<|endoftext|>