<commit_before>#!/usr/bin/env python
# encoding: utf-8
'''
pydoop -- a simple concurrent job execution library
'''

import sys
import os
from optparse import OptionParser
import select
import fcntl
from functools import partial
import errno
try:
    from importlib import import_module as import_mod
except:
    import_mod = __import__

def set_nonblocking(fd):
    val = fcntl.fcntl(fd, fcntl.F_GETFL)
    fcntl.fcntl(fd, fcntl.F_SETFL, val | os.O_NONBLOCK)

def import_func(mod_file, func_name):
    mod_name = os.path.basename(mod_file).split('.')
    mod_name = mod_name[0]

    try:
        mod = import_mod(mod_name)
    except ImportError:
        print 'cannot import module', mod_name
        return None

    try:
        entry_func = mod.__dict__.get(func_name)
    except TypeError:
        return None

    return entry_func

class _StopDispatch(Exception):
    pass

class EventLoop(object):
    EV_IN, EV_OUT = 1, 2

    def __init__(self):
        self.__on_exit = None

    def add_event(self, fd, events, cb):
        raise NotImplementedError()

    def del_event(self, fd):
        raise NotImplementedError()

    def dispatch(self):
        try:
            self._do_dispatch()
        finally:
            if self.__on_exit:
                self.__on_exit()

    def _do_dispatch(self):
        raise NotImplementedError()

    def stop_dispatch(self):
        raise _StopDispatch()

    def set_on_exit_cb(self, cb):
        self.__on_exit = cb

class SelectLoop(EventLoop):
    def __init__(self):
        EventLoop.__init__(self)
        self.__rfds = []
        self.__wfds = []
        self.__rfd_cbs = {}
        self.__wfd_cbs = {}

    def add_event(self, fd, events, cb):
        if not isinstance(events, (list, set)):
            events = [events]

        if EventLoop.EV_IN in events and fd not in self.__rfd_cbs:
                self.__rfds.append(fd)
                self.__rfd_cbs[fd] = cb
        if EventLoop.EV_OUT in events and fd not in self.__wfd_cbs:
                self.__wfds.append(fd)
                self.__wfd_cbs[fd] = cb

    def del_event(self, fd):
        try:
            self.__rfds.remove(fd)
            del self.__rfd_cbs[fd]
        except ValueError:
            pass

        try:
            self.__wfds.remove(fd)
            del self.__wfd_cbs[fd]
        except ValueError:
            pass

    def _do_dispatch(self):
        while True:
            rfds, wfds, _ = select.select(self.__rfds, self.__wfds, [])
            try:
                for rfd in rfds:
                    self.__rfd_cbs[rfd](rfd, EventLoop.EV_IN, self)

                for wfd in wfds:
                    self.__wfd_cbs[wfd](wfd, EventLoop.EV_OUT, self)
            except _StopDispatch:
                break

if 'epoll' in select.__dict__:
    class EpollLoop(EventLoop):
        def __init__(self):
            EventLoop.__init__(self)
            self.__epoll = select.epoll()
            self.__ev_map = {EventLoop.EV_IN: select.EPOLLIN,
                             EventLoop.EV_OUT: select.EPOLLOUT}
            self.__rev_ev_map = {select.EPOLLIN: EventLoop.EV_IN,
                                 select.EPOLLOUT: EventLoop.EV_OUT}
            self.__fd_cbs = {}

        def __get_fileno(self, fd):
            fd_no = fd
            if not isinstance(fd_no, int):
                fd_no = fd_no.fileno()
            return fd_no

        def add_event(self, fd, events, cb):
            if not isinstance(events, (list, set)):
                events = [events]

            fd_no = self.__get_fileno(fd)
            if fd_no in self.__fd_cbs:
                return

            ep_event = 0
            for ev in events:
                if ev in self.__ev_map:
                    ep_event = ep_event | self.__ev_map[ev]

            self.__fd_cbs[fd_no] = (events, cb)
            self.__epoll.register(fd_no, ep_event)

        def del_event(self, fd):
            fd_no = self.__get_fileno(fd)
            if fd_no in self.__fd_cbs:
                del self.__fd_cbs[fd_no]
            self.__epoll.unregister(fd_no)

        def _do_dispatch(self):
            while True:
                events = self.__epoll.poll()
                event = []
                try:
                    for fileno, ep_event in events:
                        if ep_event & select.EPOLLIN:
                            event.append(EventLoop.EV_IN)
                        if ep_event & select.EPOLLOUT:
                            event.append(EventLoop.EV_OUT)
                        #event = self.__rev_ev_map[ep_event]
                        self.__fd_cbs[fileno][1](fileno, event, self)
                except _StopDispatch:
                    break

    _event_loop = EpollLoop()
else:
    _event_loop = SelectLoop()

class FdBuffer(object):

    def __init__(self):
        self.__buffer = ""

    def set_content(self, content):
        self.__buffer = content

    def content(self):
        return self.__buffer

    def skip(self, n):
        self.__buffer = self.__buffer[n:]

    def empty(self):
        return len(self.__buffer) == 0

finished_children_num = 0
def write_child_pipe(fd, _, ev_loop, rfd, fd_buf):
    global finished_children_num
    if fd_buf.empty():
        try:
            line = rfd.readline()
        except ValueError:
            line = None

        if line:
            fd_buf.set_content(line)
        else:
            os.close(fd)

    while not fd_buf.empty():
        try:
            n = os.write(fd, fd_buf.content())
            fd_buf.skip(n)
        except OSError, e:
            if e.errno == errno.EPIPE: # child has closed the pipe
                ev_loop.del_event(fd)
                os.close(fd)
            elif e.errno in [errno.EAGAIN, errno.EWOULDBLOCK]:
                return

def close_all_fds(fds):
    for fd in fds:
        try:
            os.close(fd)
        except OSError:
            pass

def child_main(entry_func, rpipe):
    line = rpipe.readline()
    while line:
        entry_func(line)
        line = rpipe.readline()

    rpipe.close()
    return 0

class Pool(object):

    def __init__(self, worker_num=4):
        self.__worker_num = worker_num

    def run(self, proc_func, infd):
        self.finished_children_num = 0
        child_pids = []
        for _i in xrange(self.__worker_num):
            try:
                data_rfd, data_wfd = os.pipe()
                life_rfd, life_wfd = os.pipe()
                pid = os.fork()
            except OSError, e:
                print >> sys.stderr, e.strerror
                break

            if pid == 0:
                os.close(data_wfd)
                os.close(life_rfd)
                for _, fds in child_pids:
                    for fd in fds:
                        os.close(fd)

                rpipe = os.fdopen(data_rfd, 'r')
                try:
                    sys.exit(child_main(proc_func, rpipe))
                except:
                    sys.exit(1)
            else:
                os.close(data_rfd)
                os.close(life_wfd)
                set_nonblocking(data_wfd)
                set_nonblocking(life_rfd)
                write_cb = partial(write_child_pipe, rfd=infd, fd_buf=FdBuffer())
                _event_loop.add_event(data_wfd, EventLoop.EV_OUT, write_cb)
                _event_loop.add_event(life_rfd, EventLoop.EV_IN, self.read_life_signal)
                child_pids.append((pid, (data_wfd, life_rfd)))

        _event_loop.set_on_exit_cb(partial(close_all_fds,
                                           [fds[0] for _, fds in child_pids]))

        try:
            _event_loop.dispatch()
        except KeyboardInterrupt:
            print 'User requests exit.'

        print 'All children have been exited'

    def read_life_signal(self, fd, _, ev_loop):
        try:
            os.read(fd, 1)
        except:
            pass

        os.close(fd)
        pid, exit_status = os.wait()
        print 'child %d exit' % (pid),
        if os.WIFEXITED(exit_status):
            print 'normally'
        else:
            print 'imnormally'

        self.finished_children_num += 1
        if self.finished_children_num == self.__worker_num:
            ev_loop.stop_dispatch()

def main(argv=None):
    '''Command line options.'''

    program_name = os.path.basename(sys.argv[0])

    if argv is None:
        argv = sys.argv[1:]
    try:
        # setup option parser
        parser = OptionParser(usage = "%prog [options] JOBFILE INFILE")
        parser.add_option("-o", "--out", dest="outfile",
                          help="set output path", metavar="FILE")
        parser.add_option("-w", "--worker-num", dest="worker_num",
                          help="number of workers", metavar="NUM",
                          type="int", default=4)
        parser.add_option("-f", "--func", dest="func",
                          help="entry function in the JOBFILE")
    except Exception, e:
        indent = len(program_name) * " "
        sys.stderr.write(program_name + ": " + repr(e) + "\n")
        sys.stderr.write(indent + "  for help use --help")
        return 2

    (opts, args) = parser.parse_args(argv)

    if len(args) < 2:
        parser.error('Please input JOBFILE and INFILE')

    worker_num = opts.worker_num
    job_file = args[0]
    in_file = args[1]

    # MAIN BODY #
    # first try to import the specified function
    assert opts.func
    if opts.func:
        child_entry_func = import_func(job_file, opts.func)
        if child_entry_func is None:
            parser.error('Cannot import function ' + opts.func)

    try:
        infd = open(in_file)
    except:
        parser.error('Cannot open input file ' + in_file)

    pool = Pool(worker_num)
    pool.run(child_entry_func, infd)

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>Remove useless global finished_children_num.<commit_after>#!/usr/bin/env python
# encoding: utf-8
'''
pydoop -- a simple concurrent job execution library
'''

import sys
import os
from optparse import OptionParser
import select
import fcntl
from functools import partial
import errno
try:
    from importlib import import_module as import_mod
except:
    import_mod = __import__

def set_nonblocking(fd):
    val = fcntl.fcntl(fd, fcntl.F_GETFL)
    fcntl.fcntl(fd, fcntl.F_SETFL, val | os.O_NONBLOCK)

def import_func(mod_file, func_name):
    mod_name = os.path.basename(mod_file).split('.')
    mod_name = mod_name[0]

    try:
        mod = import_mod(mod_name)
    except ImportError:
        print 'cannot import module', mod_name
        return None

    try:
        entry_func = mod.__dict__.get(func_name)
    except TypeError:
        return None

    return entry_func

class _StopDispatch(Exception):
    pass

class EventLoop(object):
    EV_IN, EV_OUT = 1, 2

    def __init__(self):
        self.__on_exit = None

    def add_event(self, fd, events, cb):
        raise NotImplementedError()

    def del_event(self, fd):
        raise NotImplementedError()

    def dispatch(self):
        try:
            self._do_dispatch()
        finally:
            if self.__on_exit:
                self.__on_exit()

    def _do_dispatch(self):
        raise NotImplementedError()

    def stop_dispatch(self):
        raise _StopDispatch()

    def set_on_exit_cb(self, cb):
        self.__on_exit = cb

class SelectLoop(EventLoop):
    def __init__(self):
        EventLoop.__init__(self)
        self.__rfds = []
        self.__wfds = []
        self.__rfd_cbs = {}
        self.__wfd_cbs = {}

    def add_event(self, fd, events, cb):
        if not isinstance(events, (list, set)):
            events = [events]

        if EventLoop.EV_IN in events and fd not in self.__rfd_cbs:
                self.__rfds.append(fd)
                self.__rfd_cbs[fd] = cb
        if EventLoop.EV_OUT in events and fd not in self.__wfd_cbs:
                self.__wfds.append(fd)
                self.__wfd_cbs[fd] = cb

    def del_event(self, fd):
        try:
            self.__rfds.remove(fd)
            del self.__rfd_cbs[fd]
        except ValueError:
            pass

        try:
            self.__wfds.remove(fd)
            del self.__wfd_cbs[fd]
        except ValueError:
            pass

    def _do_dispatch(self):
        while True:
            rfds, wfds, _ = select.select(self.__rfds, self.__wfds, [])
            try:
                for rfd in rfds:
                    self.__rfd_cbs[rfd](rfd, EventLoop.EV_IN, self)

                for wfd in wfds:
                    self.__wfd_cbs[wfd](wfd, EventLoop.EV_OUT, self)
            except _StopDispatch:
                break

if 'epoll' in select.__dict__:
    class EpollLoop(EventLoop):
        def __init__(self):
            EventLoop.__init__(self)
            self.__epoll = select.epoll()
            self.__ev_map = {EventLoop.EV_IN: select.EPOLLIN,
                             EventLoop.EV_OUT: select.EPOLLOUT}
            self.__rev_ev_map = {select.EPOLLIN: EventLoop.EV_IN,
                                 select.EPOLLOUT: EventLoop.EV_OUT}
            self.__fd_cbs = {}

        def __get_fileno(self, fd):
            fd_no = fd
            if not isinstance(fd_no, int):
                fd_no = fd_no.fileno()
            return fd_no

        def add_event(self, fd, events, cb):
            if not isinstance(events, (list, set)):
                events = [events]

            fd_no = self.__get_fileno(fd)
            if fd_no in self.__fd_cbs:
                return

            ep_event = 0
            for ev in events:
                if ev in self.__ev_map:
                    ep_event = ep_event | self.__ev_map[ev]

            self.__fd_cbs[fd_no] = (events, cb)
            self.__epoll.register(fd_no, ep_event)

        def del_event(self, fd):
            fd_no = self.__get_fileno(fd)
            if fd_no in self.__fd_cbs:
                del self.__fd_cbs[fd_no]
            self.__epoll.unregister(fd_no)

        def _do_dispatch(self):
            while True:
                events = self.__epoll.poll()
                event = []
                try:
                    for fileno, ep_event in events:
                        if ep_event & select.EPOLLIN:
                            event.append(EventLoop.EV_IN)
                        if ep_event & select.EPOLLOUT:
                            event.append(EventLoop.EV_OUT)
                        #event = self.__rev_ev_map[ep_event]
                        self.__fd_cbs[fileno][1](fileno, event, self)
                except _StopDispatch:
                    break

    _event_loop = EpollLoop()
else:
    _event_loop = SelectLoop()

class FdBuffer(object):

    def __init__(self):
        self.__buffer = ""

    def set_content(self, content):
        self.__buffer = content

    def content(self):
        return self.__buffer

    def skip(self, n):
        self.__buffer = self.__buffer[n:]

    def empty(self):
        return len(self.__buffer) == 0

def write_child_pipe(fd, _, ev_loop, rfd, fd_buf):
    if fd_buf.empty():
        try:
            line = rfd.readline()
        except ValueError:
            line = None

        if line:
            fd_buf.set_content(line)
        else:
            os.close(fd)

    while not fd_buf.empty():
        try:
            n = os.write(fd, fd_buf.content())
            fd_buf.skip(n)
        except OSError, e:
            if e.errno == errno.EPIPE: # child has closed the pipe
                ev_loop.del_event(fd)
                os.close(fd)
            elif e.errno in [errno.EAGAIN, errno.EWOULDBLOCK]:
                return

def close_all_fds(fds):
    for fd in fds:
        try:
            os.close(fd)
        except OSError:
            pass

def child_main(entry_func, rpipe):
    line = rpipe.readline()
    while line:
        entry_func(line)
        line = rpipe.readline()

    rpipe.close()
    return 0

class Pool(object):

    def __init__(self, worker_num=4):
        self.__worker_num = worker_num

    def run(self, proc_func, infd):
        self.finished_children_num = 0
        child_pids = []
        for _i in xrange(self.__worker_num):
            try:
                data_rfd, data_wfd = os.pipe()
                life_rfd, life_wfd = os.pipe()
                pid = os.fork()
            except OSError, e:
                print >> sys.stderr, e.strerror
                break

            if pid == 0:
                os.close(data_wfd)
                os.close(life_rfd)
                for _, fds in child_pids:
                    for fd in fds:
                        os.close(fd)

                rpipe = os.fdopen(data_rfd, 'r')
                try:
                    sys.exit(child_main(proc_func, rpipe))
                except:
                    sys.exit(1)
            else:
                os.close(data_rfd)
                os.close(life_wfd)
                set_nonblocking(data_wfd)
                set_nonblocking(life_rfd)
                write_cb = partial(write_child_pipe, rfd=infd, fd_buf=FdBuffer())
                _event_loop.add_event(data_wfd, EventLoop.EV_OUT, write_cb)
                _event_loop.add_event(life_rfd, EventLoop.EV_IN, self.read_life_signal)
                child_pids.append((pid, (data_wfd, life_rfd)))

        _event_loop.set_on_exit_cb(partial(close_all_fds,
                                           [fds[0] for _, fds in child_pids]))

        try:
            _event_loop.dispatch()
        except KeyboardInterrupt:
            print 'User requests exit.'

        print 'All children have been exited'

    def read_life_signal(self, fd, _, ev_loop):
        try:
            os.read(fd, 1)
        except:
            pass

        os.close(fd)
        pid, exit_status = os.wait()
        print 'child %d exit' % (pid),
        if os.WIFEXITED(exit_status):
            print 'normally'
        else:
            print 'imnormally'

        self.finished_children_num += 1
        if self.finished_children_num == self.__worker_num:
            ev_loop.stop_dispatch()

def main(argv=None):
    '''Command line options.'''

    program_name = os.path.basename(sys.argv[0])

    if argv is None:
        argv = sys.argv[1:]
    try:
        # setup option parser
        parser = OptionParser(usage = "%prog [options] JOBFILE INFILE")
        parser.add_option("-o", "--out", dest="outfile",
                          help="set output path", metavar="FILE")
        parser.add_option("-w", "--worker-num", dest="worker_num",
                          help="number of workers", metavar="NUM",
                          type="int", default=4)
        parser.add_option("-f", "--func", dest="func",
                          help="entry function in the JOBFILE")
    except Exception, e:
        indent = len(program_name) * " "
        sys.stderr.write(program_name + ": " + repr(e) + "\n")
        sys.stderr.write(indent + "  for help use --help")
        return 2

    (opts, args) = parser.parse_args(argv)

    if len(args) < 2:
        parser.error('Please input JOBFILE and INFILE')

    worker_num = opts.worker_num
    job_file = args[0]
    in_file = args[1]

    # MAIN BODY #
    # first try to import the specified function
    assert opts.func
    if opts.func:
        child_entry_func = import_func(job_file, opts.func)
        if child_entry_func is None:
            parser.error('Cannot import function ' + opts.func)

    try:
        infd = open(in_file)
    except:
        parser.error('Cannot open input file ' + in_file)

    pool = Pool(worker_num)
    pool.run(child_entry_func, infd)

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#pymill.py

import pycurl2 as pycurl
import cStringIO
import json

class Pymill():
    """
    These are the parameters each object type contains

    Card:
    id: unique card ID
    card_type: visa, mastercard, (maybe one day american express)
    country: country the card was issued in
    expire_month: (2ch)
    expire_year: (4ch)
    card_holder: name of cardholder
    last4: last 4 digits of card
    created_at: unixtime
    updated_at: unixtime

    Transaction:
    id: unique transaction ID
    amount: amount charged in EuroCENTS
    status: open, pending, closed or refunded
    description: user-selected description of the transaction
    livemode: true or false depending on whether the transaction is real or in test mode
    creditcard: a card object (see above)
    clients: if a preset client (see below) was used to make the transaction. Otherwise null
    created_at: unixtime
    updated_at: unixtime

    Refund:
    id: unique refund ID
    transaction: The unique transaction ID of the transaction being refunded
    amount: amount refunded in EuroCENTS
    status: open, pending or refunded
    description: user-selected description of the refund
    livemode: true or false depending on whether the transaction is real or in test mode
    created_at: unixtime
    updated_at: unixtime

    Client:
    id: unique id for this client
    email: client's email address (optional)
    description: description of this client (optional)
    created_at: unix timestamp identifying time of creation
    updated_at: unix timestamp identifying time of last change
    creditcard: cc object (optional)
    subscription: subscription object (optional)

    Offer:
    id: unique offer identifier
    name: freely controllable offer name
    amount: The amount, in EuroCENTS, to be charged every time the offer period passes. Note that ODD values will NOT work in test mode.
    interval: "week", "month", or "year". The client will be charged every time the interval passes
    trial_period_days: Number of days before the first charge. (optional)
    """

    def __init__(self, privatekey):
        """
        Initialize a new paymill interface connection. Requires a private key.
        """
        self.c = pycurl.Curl()
        self.c.setopt(pycurl.NOSIGNAL, 1)
        self.c.setopt(pycurl.CONNECTTIMEOUT, 30)
        self.c.setopt(pycurl.USERPWD, '%s:' % (privatekey,))
        self.bridge = "https://api.paymill.de/v2/"

    def _post(self, url, params=()):
        """
        Posts a request with parameters to a given url
        url: The URL of the entity to post to.
        params: a tuple of (name, value) tuples of parameters

        Returns None
        """
        self.c.setopt(self.c.URL, url)
        if params is not ():
            p = str("&".join([i[0] + "=" + i[1] for i in params]))
            print p
            self.c.setopt(pycurl.CUSTOMREQUEST, "POST")
            self.c.setopt(self.c.POSTFIELDS, p)
        self.c.perform()

    def _apicall(self, url, params=(), cr="GET", ch=None):
        """
        Call an API endpoint.
        url: The URL of the entity to post to.
        params: a tuple of (name, value) tuples of parameters
        cr: The request type to be made. If parameters are passed, this will be ignored and treated as POST

        Returns a dictionary object populated with the json returned.
        """
        if ch is not None:
            self.c.setopt(pycurl.HTTPHEADER, ch)
        self.c.setopt(pycurl.CUSTOMREQUEST, cr)
        buf = cStringIO.StringIO()
        self.c.setopt(self.c.WRITEFUNCTION, buf.write)
        self._post(url, params)
        s = buf.getvalue()
        buf.close()
        if ch is not None:
            return s
        return json.loads(s)

    def newcard(self, token):
        """
        Create a credit card from a given token.
        token: string Unique credit card token

        Returns: a dict with a member "data" containing a dict representing a CC
        """
        return self._apicall(self.bridge + "payments", (("token", token),))

    def getcarddetails(self, cardid):
        """
        Get the details of a credit card from its id.
        cardid: string Unique id for the credit card

        Returns: a dict with a member "data" containing a dict representing a CC
        """
        return self._apicall(self.bridge + "payments" + str(cardid))

    def getcards(self):
        """
        List all stored cards.

        Returns: a dict with a member "data" which is an array of dicts, each representing a CC
        """
        return self._apicall(self.bridge + "payments")

    def delcard(self, cardid):
        """
        Delete a stored CC
        cardid: Unique id for the CC to be deleted

        Returns: a dict with an member "data" containing an empty array
        """
        return self._apicall(self.bridge + "payments/%s" % (str(cardid),), cr="DELETE")

    def transact(self, amount=0, currency="eur", description=None, token=None, client=None, payment=None):
        """
        Create a transaction (charge a card). You must provide an amount, and exactly one funding source.
        The amount is in Eurocents, and the funding source can be a client, a token, or a card id.
        amount: The amount (in Euro CENTS) to be charged. For example, 240 will charge 2 euros and 40 cents, NOT 240 euros.
        currency: Must be "eur" if given (optional)
        description: A short description of the transaction (optional)
        token: A token generated by the paymill bridge js library
        client: A client id number
        creditcard: A CC id number.

        Returns: None if one of the required parameters is missing. A dict with a member "data" containing a transaction dict otherwise.
        """

        p = []
        if token is not None:
            p += [("token", token)]
        elif client is not None:
            p += [("client", client)]
        elif payment is not None:
            p += [("payment", payment)]
        else:
            return None
        if amount == 0:
            return None
        if description is not None:
            p += [("description", description)]
        p += [("amount", str(amount))]
        p += [("currency", currency)]
        return self._apicall(self.bridge + "transactions", tuple(p))

    def gettrandetails(self, tranid):
        """
        Get details on a transaction.
        tranid: string Unique id for the transaction

        Returns: a dict representing a transaction
        """
        return self._apicall(self.bridge + "transactions/" + str(cardid))

    def gettrans(self):
        """
        List all transactions.

        Returns: a dict with a member "data" which is an array of dicts, each representing a transaction
        """
        return self._apicall(self.bridge + "transactions/")

    def refund(self, tranid, amount, description=None):
        """
        Refunds an already performed transaction.
        tranid: string Unique transaction id
        amount: The amount in cents that are to be refunded
        description: A description of the refund (optional)

        Returns: a dict with a member "data" which is a dict representing a refund, or None if the amount is 0
        """
        if amount == 0:
            return None
        p = [("amount", str(amount))]
        if description is not None:
            p += [("description", description)]
        return self._apicall(self.bridge + "refund/" + str(tranid), tuple(p))

    def getrefdetails(self, refid):
        """
        Get the details of a refund from its id.
        refid: string Unique id for the refund

        Returns: a dict with a member "data" which is a dict representing a refund
        """
        return self._apicall(self.bridge + "refunds/" + str(refid))

    def getrefs(self):
        """
        List all stored refunds.

        Returns: a dict with a member "data" which is an array of dicts, each representing a refund
        """
        return self._apicall(self.bridge + "refunds/")

    def newclient(email, description=None):
        """
        Creates a new client.
        email: client's email address
        description: description of this client (optional)

        Returns: a dict with a member "data" which is a dict representing a client.
        """
        p = [("email", str(email))]
        if description is not None:
            p += [("description", description)]
        return self._apicall(self.bridge + "clients", tuple(p))

    def getclientdetails(self, cid):
        """
        Get the details of a client from its id.
        cid: string Unique id for the client

        Returns: a dict with a member "data" which is a dict representing a client
        """
        return self._apicall(self.bridge + "clients/" + str(cid))

    def updateclient(self, cid, email, description=None):
        """
        Updates the details of a client.
        cid: string Unique client id
        email: The email of the client
        description: A description of the client (optional)

        Returns: a dict with a member "data" which is a dict representing a client
        """
        p = [("email", str(email))]
        if description is not None:
            p += [("description", description)]
        return self._apicall(self.bridge + "clients/" + str(cid), tuple(p))

    def delclient(self, cid):
        """
        Delete a stored client
        cid: Unique id for the client to be deleted

        Returns: a dict with an member "data" containing an empty array
        """
        return self._apicall(self.bridge + "clients/%s" % (str(cid),), cr="DELETE")

    def getclients(self):
        """
        List all stored clients.

        Returns: a dict with a member "data" which is an array of dicts, each representing a client
        """
        return self._apicall(self.bridge + "clients/")

    def exportclients(self):
        """
        Export all stored clients in CSV form

        Returns: the contents of the CSV file
        """
        return self._apicall(self.bridge + "clients/", ch=["Accept: text/csv"])

    def newoffer(self, amount, interval="month", currency="eur", name=None):
        """
        Creates a new offer
        amount: The amount in cents that are to be charged every interval
        interval: MUST be either "week", "month" or "year"
        currency: Must be "eur" if given (optional)
        name: A name for this offer

        Returns: a dict with a member "data" which is a dict representing
            an offer, or None if the amount is 0 or the interval is invalid
        """
        if amount == 0:
            return None
        p = [("amount", str(amount))]
        if interval not in ["week", "month", "year"]:
            return None
        p += [("interval", str(interval))]
        if name is not None:
            p += [("name", name)]
        return self._apicall(self.bridge + "offers", tuple(p))

    def getofferdetails(self, oid):
        """
        Get the details of an offer from its id.
        oid: string Unique id for the offer

        Returns: a dict with a member "data" which is a dict representing an offer
        """
        return self._apicall(self.bridge + "offers/" + str(oid))

    def updateoffer(self, oid, name):
        """
        Updates the details of an offer. Only the name may be changed
        oid: string Unique offer id
        name: The new name of the offer

        Returns: a dict with a member "data" which is a dict representing an offer
        """
        p = [("name", str(name))]
        return self._apicall(self.bridge + "offers/" + str(oid), tuple(p))

    def deloffer(self, oid):
        """
        Delete a stored offer. May only be done if no subscriptions to this offer are active.
        oid: Unique id for the offer to be deleted

        Returns: a dict with an member "data" containing an empty array
        """
        return self._apicall(self.bridge + "offers/%s" % (str(oid),), cr="DELETE")

    def getoffers(self):
        """
        List all stored offers.

        Returns: a dict with a member "data" which is an array of dicts, each representing an offer
        """
        return self._apicall(self.bridge + "offers/")

    def newsub(self, client, offer):
        """
        Subscribes a client to an offer
        client: The id of the client
        offer: The id of the offer

        Returns: a dict with a member "data" which is a dict representing a subscription
        """
        if amount == 0:
            return None
        p = [("offer", str(offer)), ("token", str(client))]
        return self._apicall(self.bridge + "subscriptions", tuple(p))

    def getsubdetails(self, sid):
        """
        Get the details of a subscription from its id.
        sid: string Unique id for the subscription

        Returns: a dict with a member "data" which is a dict representing a subscription
        """
        return self._apicall(self.bridge + "subscriptions/" + str(sid))

    def cancelsubafter(self, sid, cancel=True):
        """
        Cancels a subscription after its interval ends
        sid: string Unique subscription id
        cancel: If True, the subscription will be cancelled at the end of its interval. Set to False to undo.

        Returns: a dict with a member "data" which is a dict representing a subscription
        """
        if cancel:
            p = [("cancel_at_period_end", "true")]
        else:
            p = [("cancel_at_period_end", "false")]
        return self._apicall(self.bridge + "subscriptions/" + str(sid), tuple(p))

    def cancelsubnow(self, sid):
        """
        Cancel a subscription immediately. Pending transactions will still be charged.
        sid: Unique subscription id

        Returns: a dict with an member "data"
        """
        return self._apicall(self.bridge + "subscriptions/%s" % (str(sid),), cr="DELETE")

    def getsubs(self):
        """
        List all stored subscriptions.

        Returns: a dict with a member "data" which is an array of dicts, each representing a subscription
        """
        return self._apicall(self.bridge + "subscriptions/")

if __name__ == "__main__":
    p = Pymill("YOURPRIVATEKEYHERE")
    cc = (p.getcards())["data"][0]["id"]
    print p.getcarddetails(cc)
    #print p.transact(amount=300,card=cc,description="pymilltest")
<commit_msg>fixing missing self in newclient function<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#pymill.py

import pycurl2 as pycurl
import cStringIO
import json

class Pymill():
    """
    These are the parameters each object type contains

    Card:
    id: unique card ID
    card_type: visa, mastercard, (maybe one day american express)
    country: country the card was issued in
    expire_month: (2ch)
    expire_year: (4ch)
    card_holder: name of cardholder
    last4: last 4 digits of card
    created_at: unixtime
    updated_at: unixtime

    Transaction:
    id: unique transaction ID
    amount: amount charged in EuroCENTS
    status: open, pending, closed or refunded
    description: user-selected description of the transaction
    livemode: true or false depending on whether the transaction is real or in test mode
    creditcard: a card object (see above)
    clients: if a preset client (see below) was used to make the transaction. Otherwise null
    created_at: unixtime
    updated_at: unixtime

    Refund:
    id: unique refund ID
    transaction: The unique transaction ID of the transaction being refunded
    amount: amount refunded in EuroCENTS
    status: open, pending or refunded
    description: user-selected description of the refund
    livemode: true or false depending on whether the transaction is real or in test mode
    created_at: unixtime
    updated_at: unixtime

    Client:
    id: unique id for this client
    email: client's email address (optional)
    description: description of this client (optional)
    created_at: unix timestamp identifying time of creation
    updated_at: unix timestamp identifying time of last change
    creditcard: cc object (optional)
    subscription: subscription object (optional)

    Offer:
    id: unique offer identifier
    name: freely controllable offer name
    amount: The amount, in EuroCENTS, to be charged every time the offer period passes. Note that ODD values will NOT work in test mode.
    interval: "week", "month", or "year". The client will be charged every time the interval passes
    trial_period_days: Number of days before the first charge. (optional)
    """

    def __init__(self, privatekey):
        """
        Initialize a new paymill interface connection. Requires a private key.
        """
        self.c = pycurl.Curl()
        self.c.setopt(pycurl.NOSIGNAL, 1)
        self.c.setopt(pycurl.CONNECTTIMEOUT, 30)
        self.c.setopt(pycurl.USERPWD, '%s:' % (privatekey,))
        self.bridge = "https://api.paymill.de/v2/"

    def _post(self, url, params=()):
        """
        Posts a request with parameters to a given url
        url: The URL of the entity to post to.
        params: a tuple of (name, value) tuples of parameters

        Returns None
        """
        self.c.setopt(self.c.URL, url)
        if params is not ():
            p = str("&".join([i[0] + "=" + i[1] for i in params]))
            print p
            self.c.setopt(pycurl.CUSTOMREQUEST, "POST")
            self.c.setopt(self.c.POSTFIELDS, p)
        self.c.perform()

    def _apicall(self, url, params=(), cr="GET", ch=None):
        """
        Call an API endpoint.
        url: The URL of the entity to post to.
        params: a tuple of (name, value) tuples of parameters
        cr: The request type to be made. If parameters are passed, this will be ignored and treated as POST

        Returns a dictionary object populated with the json returned.
        """
        if ch is not None:
            self.c.setopt(pycurl.HTTPHEADER, ch)
        self.c.setopt(pycurl.CUSTOMREQUEST, cr)
        buf = cStringIO.StringIO()
        self.c.setopt(self.c.WRITEFUNCTION, buf.write)
        self._post(url, params)
        s = buf.getvalue()
        buf.close()
        if ch is not None:
            return s
        return json.loads(s)

    def newcard(self, token):
        """
        Create a credit card from a given token.
        token: string Unique credit card token

        Returns: a dict with a member "data" containing a dict representing a CC
        """
        return self._apicall(self.bridge + "payments", (("token", token),))

    def getcarddetails(self, cardid):
        """
        Get the details of a credit card from its id.
        cardid: string Unique id for the credit card

        Returns: a dict with a member "data" containing a dict representing a CC
        """
        return self._apicall(self.bridge + "payments" + str(cardid))

    def getcards(self):
        """
        List all stored cards.

        Returns: a dict with a member "data" which is an array of dicts, each representing a CC
        """
        return self._apicall(self.bridge + "payments")

    def delcard(self, cardid):
        """
        Delete a stored CC
        cardid: Unique id for the CC to be deleted

        Returns: a dict with an member "data" containing an empty array
        """
        return self._apicall(self.bridge + "payments/%s" % (str(cardid),), cr="DELETE")

    def transact(self, amount=0, currency="eur", description=None, token=None, client=None, payment=None):
        """
        Create a transaction (charge a card). You must provide an amount, and exactly one funding source.
        The amount is in Eurocents, and the funding source can be a client, a token.
        amount: The amount (in Euro CENTS) to be charged. For example, 240 will charge 2 euros and 40 cents, NOT 240 euros.
        currency: Must be "eur" if given (optional)
        description: A short description of the transaction (optional)
        token: A token generated by the paymill bridge js library
        client: A client id number

        Returns: None if one of the required parameters is missing. A dict with a member "data" containing a transaction dict otherwise.
        """

        p = []
        if token is not None:
            p += [("token", token)]
        elif client is not None:
            p += [("client", client)]
        elif payment is not None:
            p += [("payment", payment)]
        else:
            return None
        if amount == 0:
            return None
        if description is not None:
            p += [("description", description)]
        p += [("amount", str(amount))]
        p += [("currency", currency)]
        return self._apicall(self.bridge + "transactions", tuple(p))

    def gettrandetails(self, tranid):
        """
        Get details on a transaction.
        tranid: string Unique id for the transaction

        Returns: a dict representing a transaction
        """
        return self._apicall(self.bridge + "transactions/" + str(cardid))

    def gettrans(self):
        """
        List all transactions.

        Returns: a dict with a member "data" which is an array of dicts, each representing a transaction
        """
        return self._apicall(self.bridge + "transactions/")

    def refund(self, tranid, amount, description=None):
        """
        Refunds an already performed transaction.
        tranid: string Unique transaction id
        amount: The amount in cents that are to be refunded
        description: A description of the refund (optional)

        Returns: a dict with a member "data" which is a dict representing a refund, or None if the amount is 0
        """
        if amount == 0:
            return None
        p = [("amount", str(amount))]
        if description is not None:
            p += [("description", description)]
        return self._apicall(self.bridge + "refund/" + str(tranid), tuple(p))

    def getrefdetails(self, refid):
        """
        Get the details of a refund from its id.
        refid: string Unique id for the refund

        Returns: a dict with a member "data" which is a dict representing a refund
        """
        return self._apicall(self.bridge + "refunds/" + str(refid))

    def getrefs(self):
        """
        List all stored refunds.

        Returns: a dict with a member "data" which is an array of dicts, each representing a refund
        """
        return self._apicall(self.bridge + "refunds/")

    def newclient(self, email, description=None):
        """
        Creates a new client.
        email: client's email address
        description: description of this client (optional)

        Returns: a dict with a member "data" which is a dict representing a client.
        """
        p = [("email", str(email))]
        if description is not None:
            p += [("description", description)]
        return self._apicall(self.bridge + "clients", tuple(p))

    def getclientdetails(self, cid):
        """
        Get the details of a client from its id.
        cid: string Unique id for the client

        Returns: a dict with a member "data" which is a dict representing a client
        """
        return self._apicall(self.bridge + "clients/" + str(cid))

    def updateclient(self, cid, email, description=None):
        """
        Updates the details of a client.
        cid: string Unique client id
        email: The email of the client
        description: A description of the client (optional)

        Returns: a dict with a member "data" which is a dict representing a client
        """
        p = [("email", str(email))]
        if description is not None:
            p += [("description", description)]
        return self._apicall(self.bridge + "clients/" + str(cid), tuple(p))

    def delclient(self, cid):
        """
        Delete a stored client
        cid: Unique id for the client to be deleted

        Returns: a dict with an member "data" containing an empty array
        """
        return self._apicall(self.bridge + "clients/%s" % (str(cid),), cr="DELETE")

    def getclients(self):
        """
        List all stored clients.

        Returns: a dict with a member "data" which is an array of dicts, each representing a client
        """
        return self._apicall(self.bridge + "clients/")

    def exportclients(self):
        """
        Export all stored clients in CSV form

        Returns: the contents of the CSV file
        """
        return self._apicall(self.bridge + "clients/", ch=["Accept: text/csv"])

    def newoffer(self, amount, interval="month", currency="eur", name=None):
        """
        Creates a new offer
        amount: The amount in cents that are to be charged every interval
        interval: MUST be either "week", "month" or "year"
        currency: Must be "eur" if given (optional)
        name: A name for this offer

        Returns: a dict with a member "data" which is a dict representing
            an offer, or None if the amount is 0 or the interval is invalid
        """
        if amount == 0:
            return None
        p = [("amount", str(amount))]
        if interval not in ["week", "month", "year"]:
            return None
        p += [("interval", str(interval))]
        if name is not None:
            p += [("name", name)]
        return self._apicall(self.bridge + "offers", tuple(p))

    def getofferdetails(self, oid):
        """
        Get the details of an offer from its id.
        oid: string Unique id for the offer

        Returns: a dict with a member "data" which is a dict representing an offer
        """
        return self._apicall(self.bridge + "offers/" + str(oid))

    def updateoffer(self, oid, name):
        """
        Updates the details of an offer. Only the name may be changed
        oid: string Unique offer id
        name: The new name of the offer

        Returns: a dict with a member "data" which is a dict representing an offer
        """
        p = [("name", str(name))]
        return self._apicall(self.bridge + "offers/" + str(oid), tuple(p))

    def deloffer(self, oid):
        """
        Delete a stored offer. May only be done if no subscriptions to this offer are active.
        oid: Unique id for the offer to be deleted

        Returns: a dict with an member "data" containing an empty array
        """
        return self._apicall(self.bridge + "offers/%s" % (str(oid),), cr="DELETE")

    def getoffers(self):
        """
        List all stored offers.

        Returns: a dict with a member "data" which is an array of dicts, each representing an offer
        """
        return self._apicall(self.bridge + "offers/")

    def newsub(self, client, offer):
        """
        Subscribes a client to an offer
        client: The id of the client
        offer: The id of the offer

        Returns: a dict with a member "data" which is a dict representing a subscription
        """
        if amount == 0:
            return None
        p = [("offer", str(offer)), ("token", str(client))]
        return self._apicall(self.bridge + "subscriptions", tuple(p))

    def getsubdetails(self, sid):
        """
        Get the details of a subscription from its id.
        sid: string Unique id for the subscription

        Returns: a dict with a member "data" which is a dict representing a subscription
        """
        return self._apicall(self.bridge + "subscriptions/" + str(sid))

    def cancelsubafter(self, sid, cancel=True):
        """
        Cancels a subscription after its interval ends
        sid: string Unique subscription id
        cancel: If True, the subscription will be cancelled at the end of its interval. Set to False to undo.

        Returns: a dict with a member "data" which is a dict representing a subscription
        """
        if cancel:
            p = [("cancel_at_period_end", "true")]
        else:
            p = [("cancel_at_period_end", "false")]
        return self._apicall(self.bridge + "subscriptions/" + str(sid), tuple(p))

    def cancelsubnow(self, sid):
        """
        Cancel a subscription immediately. Pending transactions will still be charged.
        sid: Unique subscription id

        Returns: a dict with an member "data"
        """
        return self._apicall(self.bridge + "subscriptions/%s" % (str(sid),), cr="DELETE")

    def getsubs(self):
        """
        List all stored subscriptions.

        Returns: a dict with a member "data" which is an array of dicts, each representing a subscription
        """
        return self._apicall(self.bridge + "subscriptions/")

if __name__ == "__main__":
    p = Pymill("YOURPRIVATEKEYHERE")
    cc = (p.getcards())["data"][0]["id"]
    print p.getcarddetails(cc)
    #print p.transact(amount=300,card=cc,description="pymilltest")
<|endoftext|>


<commit_before>#Copyright (c) 2009 Eugene Kaznacheev <qetzal@gmail.com>

#Permission is hereby granted, free of charge, to any person
#obtaining a copy of this software and associated documentation
#files (the "Software"), to deal in the Software without
#restriction, including without limitation the rights to use,
#copy, modify, merge, publish, distribute, sublicense, and/or sell
#copies of the Software, and to permit persons to whom the
#Software is furnished to do so, subject to the following
#conditions:

#The above copyright notice and this permission notice shall be
#included in all copies or substantial portions of the Software.

#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
#OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
#NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
#HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
#WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
#OTHER DEALINGS IN THE SOFTWARE.

"""
Fetches weather reports from Google Weather, Yahoo Wheather and NOAA
"""

import urllib2, urllib
from xml.dom import minidom

GOOGLE_WEATHER_URL = 'http://www.google.com/ig/api?weather=%s&hl=%s'

YAHOO_WEATHER_URL = 'http://xml.weather.yahoo.com/forecastrss?p=%s&u=%s'
YAHOO_WEATHER_NS = 'http://xml.weather.yahoo.com/ns/rss/1.0'

NOAA_WEATHER_URL = 'http://www.weather.gov/xml/current_obs/%s.xml'

GISMETEO_WEATHER_URL = 'http://informer.gismeteo.ru/xml/%s.xml'

def get_weather_from_google(location_id, hl = ''):
        """
        Fetches weather report from Google

        Parameters
          location_id: a zip code (10001); city name, state (weather=woodland,PA); city name, country (weather=london,england); or possibly others.
          hl: the language parameter (language code)

        Returns:
          weather_data: a dictionary of weather data.
        """

        url = GOOGLE_WEATHER_URL % (location_id, hl)
        handler = urllib2.urlopen(url)
        dom = minidom.parse(handler)
        handler.close()

        weather_data = {}
        weather_dom = dom.getElementsByTagName('weather')[0]

        data_structure = {
                        'forecast_information': ('city', 'postal_code', 'latitude_e6', 'longitude_e6', 'forecast_date', 'current_date_time', 'unit_system'),
                        'current_conditions': ('condition','temp_f', 'temp_c', 'humidity', 'wind_condition', 'icon')
        }

        forecast_conditions_structure = ('day_of_week', 'low', 'high', 'icon', 'condition')
        for (tag, list_of_tags2) in data_structure.iteritems():
                tmp_conditions = {}
                for tag2 in list_of_tags2:
                        tmp_conditions[tag2] =  weather_dom.getElementsByTagName(tag)[0].getElementsByTagName(tag2)[0].getAttribute('data')
                weather_data[tag] = tmp_conditions

        forecast_conditions = ('day_of_week', 'low', 'high', 'icon', 'condition')

        forecasts = []
        for forecast in dom.getElementsByTagName('forecast_conditions'):
                tmp_forecast = {}
                for tag in forecast_conditions:
                        tmp_forecast[tag] = forecast.getElementsByTagName(tag)[0].getAttribute('data')
                forecasts.append(tmp_forecast)
                weather_data['forecasts'] = forecasts

        dom.unlink()

        return weather_data

def get_weather_from_yahoo(location_id, units = 'metric'):
        """
Fetches weather report from Yahoo!

Parameters
location_id: A five digit US zip code or location ID. To find your location ID,
browse or search for your city from the Weather home page(http://weather.yahoo.com/)
The weather ID is in the URL for the forecast page for that city. You can also get the location ID by entering your zip code on the home page. For example, if you search for Los Angeles on the Weather home page, the forecast page for that city is http://weather.yahoo.com/forecast/USCA0638.html. The location ID is USCA0638.

units: type of units. 'metric' for metric and '' for  non-metric
Note that choosing metric units changes all the weather units to metric, for example, wind speed will be reported as kilometers per hour and barometric pressure as millibars.

Returns:
weather_data: a dictionary of weather data. See  http://developer.yahoo.com/weather/#channel
        """

        if units == 'metric':
                unit = 'c'
        else:
                 unit = 'f'
        url = YAHOO_WEATHER_URL % (location_id, unit)
        handler = urllib2.urlopen(url)
        dom = minidom.parse(handler)
        handler.close()

        weather_data = {}
        weather_data['title'] = dom.getElementsByTagName('title')[0].firstChild.data
        weather_data['link'] = dom.getElementsByTagName('link')[0].firstChild.data

        ns_data_structure = {
                'location': ('city', 'region', 'country'),
                'units': ('temperature', 'distance', 'pressure', 'speed'),
                'wind': ('chill', 'direction', 'speed'),
                'atmosphere': ('humidity', 'visibility', 'pressure', 'rising'),
                'astronomy': ('sunrise', 'sunset'),
                'condition': ('text', 'code', 'temp', 'date')

        }

        for (tag, attrs) in ns_data_structure.iteritems():
                weather_data[tag] = xml_get_ns_yahoo_tag(dom, YAHOO_WEATHER_NS, tag, attrs)

        weather_data['geo'] = {}
        weather_data['geo']['lat'] = dom.getElementsByTagName('geo:lat')[0].firstChild.data
        weather_data['geo']['long'] = dom.getElementsByTagName('geo:long')[0].firstChild.data

        weather_data['condition']['title'] = dom.getElementsByTagName('item')[0].getElementsByTagName('title')[0].firstChild.data
        weather_data['html_description'] = dom.getElementsByTagName('item')[0].getElementsByTagName('description')[0].firstChild.data

        forecasts = []
        for forecast in dom.getElementsByTagNameNS(YAHOO_WEATHER_NS, 'forecast'):
                forecasts.append(xml_get_attrs(forecast,('date', 'low', 'high', 'text', 'code')))
        weather_data['forecasts'] = forecasts

        dom.unlink()

        return weather_data

def get_weather_from_noaa(station_id):
        """
Fetches weather report from NOAA: National Oceanic and Atmospheric Administration (United States)

Parameters
station_id: To find your station ID, perform the following steps:
1. Open this URL: http://www.weather.gov/xml/current_obs/seek.php?state=az&Find=Find
2. Select the necessary state state. Click 'Find'.
3. Find the necessary station in the 'Observation Location' column.
4. The station ID is in the URL for the weather page for that station.
For example if the weather page is http://weather.noaa.gov/weather/current/KPEO.html -- the station ID is KPEO.

Other way to get the station ID: use this library: http://code.google.com/p/python-weather/ and 'Weather.location2station' function.

Returns:
weather_data: a dictionary of weather data.

(useful icons: http://www.weather.gov/xml/current_obs/weather.php)
        """

        url = NOAA_WEATHER_URL % (station_id)
        handler = urllib2.urlopen(url)
        dom = minidom.parse(handler)
        handler.close()

        data_structure = ('suggested_pickup',
                                'suggested_pickup_period',
                                'location',
                                'station_id',
                                'latitude',
                                'longitude',
                                'observation_time',
                                'observation_time_rfc822',
                                'weather',
                                'temperature_string',
                                'temp_f',
                                'temp_c',
                                'relative_humidity',
                                'wind_string',
                                'wind_dir',
                                'wind_degrees',
                                'wind_mph',
                                'wind_gust_mph',
                                'pressure_string',
                                'pressure_mb',
                                'pressure_in',
                                'dewpoint_string',
                                'dewpoint_f',
                                'dewpoint_c',
                                'heat_index_string',
                                'heat_index_f',
                                'heat_index_c',
                                'windchill_string',
                                'windchill_f',
                                'windchill_c',
                                'icon_url_base',
                                'icon_url_name',
                                'two_day_history_url',
                                'ob_url'
                                )
	weather_data = {}
        for tag in data_structure:
               weather_data[tag] =  dom.getElementsByTagName('current_observation')[0].getElementsByTagName(tag)[0].firstChild.data

        dom.unlink()
        return weather_data

def get_weather_from_gismeteo(location_id):
	"""
Fetches weather report from GisMeteo (russian weather service)

Parameters
location_id: A location ID of the necessary place. To find your location ID,
browse or search for your city from the Gismeteo page: http://informer.gismeteo.ru/xml.html?index=27612%CC%EE%F1%EA%E2%E0&&lang=ru (the page is in Russian)
The location ID is in the URL for the XML URL for that city.
For example, if you search for New York on the GisMeteo page, the XML URLfor that city is
http://informer.gismeteo.ru/xml/72503_1.xml. The location ID is 72503_1.

Returns:
weather_data: a dictionary of weather data. See http://informer.gismeteo.ru/xml.html?index=27612%CC%EE%F1%EA%E2%E0&&lang=ru (at the page bottom)
	"""
	url = GISMETEO_WEATHER_URL % (location_id)
	handler = urllib2.urlopen(url)
	dom = minidom.parse(handler)
	handler.close()

	forecast_data_structure = {
			'PHENOMENA': ('cloudiness','precipitation', 'rpower', 'spower'),
			'PRESSURE': ('max','min'),
			'TEMPERATURE': ('max','min'),
			'WIND': ('max','min', 'direction'),
			'RELWET': ('max','min'),
			'HEAT': ('max','min'),
			}
	town_tag_attr = ('index', 'sname', 'latitude','longitude')
	forecast_tag_attr = ('day', 'month', 'year', 'hour', 'tod', 'predict', 'weekday')

	weather_data = {}

    	for attrs in town_tag_attr:
    		weather_data['town'] = xml_get_attrs(dom.getElementsByTagName('TOWN')[0], attrs)

        forecasts = []
        for forecast in dom.getElementsByTagName('FORECAST'):
        	_tmp_forecast = {}
		_tmp_forecast.update(xml_get_attrs(forecast, forecast_tag_attr))
		for (tag, attrs) in forecast_data_structure.iteritems():
			_tmp_forecast[tag] = xml_get_attrs(forecast.getElementsByTagName[tag][0], attrs)
			forecasts.append(_tmp_forecast)

        weather_data['forecasts'] = forecasts

def xml_get_ns_yahoo_tag(dom, YAHOO_WEATHER_NS, tag, attrs):
        """
        Parses the necessary tag and returns the dictionary with values

        Parameters:
        dom - DOM
        YAHOO_WEATHER_NS - namespace
        tag - necessary tag
        attrs - tuple of attributes

        Returns: a dictionary of elements
        """
        element = dom.getElementsByTagNameNS(YAHOO_WEATHER_NS, tag)[0]
        return xml_get_attrs(element,attrs)

def xml_get_attrs(xml_element, attrs):
        """
        Returns the list of necessary attributes

        Parameters:
        element: xml element
        attrs: tuple of attributes

        Return: a dictionary of elements
        """

        result = {}
        for attr in attrs:
                result[attr] = xml_element.getAttribute(attr)
        return result
<commit_msg><commit_after>#Copyright (c) 2009 Eugene Kaznacheev <qetzal@gmail.com>

#Permission is hereby granted, free of charge, to any person
#obtaining a copy of this software and associated documentation
#files (the "Software"), to deal in the Software without
#restriction, including without limitation the rights to use,
#copy, modify, merge, publish, distribute, sublicense, and/or sell
#copies of the Software, and to permit persons to whom the
#Software is furnished to do so, subject to the following
#conditions:

#The above copyright notice and this permission notice shall be
#included in all copies or substantial portions of the Software.

#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
#OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
#NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
#HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
#WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
#OTHER DEALINGS IN THE SOFTWARE.

"""
Fetches weather reports from Google Weather, Yahoo Wheather and NOAA
"""

import urllib2, urllib
from xml.dom import minidom

GOOGLE_WEATHER_URL = 'http://www.google.com/ig/api?weather=%s&hl=%s'

YAHOO_WEATHER_URL = 'http://xml.weather.yahoo.com/forecastrss?p=%s&u=%s'
YAHOO_WEATHER_NS = 'http://xml.weather.yahoo.com/ns/rss/1.0'

NOAA_WEATHER_URL = 'http://www.weather.gov/xml/current_obs/%s.xml'

GISMETEO_WEATHER_URL = 'http://informer.gismeteo.ru/xml/%s.xml'

def get_weather_from_google(location_id, hl = ''):
        """
        Fetches weather report from Google

        Parameters
          location_id: a zip code (10001); city name, state (weather=woodland,PA); city name, country (weather=london,england); or possibly others.
          hl: the language parameter (language code)

        Returns:
          weather_data: a dictionary of weather data.
        """

        url = GOOGLE_WEATHER_URL % (location_id, hl)
        handler = urllib2.urlopen(url)
        dom = minidom.parse(handler)
        handler.close()

        weather_data = {}
        weather_dom = dom.getElementsByTagName('weather')[0]

        data_structure = {
                        'forecast_information': ('city', 'postal_code', 'latitude_e6', 'longitude_e6', 'forecast_date', 'current_date_time', 'unit_system'),
                        'current_conditions': ('condition','temp_f', 'temp_c', 'humidity', 'wind_condition', 'icon')
        }

        forecast_conditions_structure = ('day_of_week', 'low', 'high', 'icon', 'condition')
        for (tag, list_of_tags2) in data_structure.iteritems():
                tmp_conditions = {}
                for tag2 in list_of_tags2:
                        tmp_conditions[tag2] =  weather_dom.getElementsByTagName(tag)[0].getElementsByTagName(tag2)[0].getAttribute('data')
                weather_data[tag] = tmp_conditions

        forecast_conditions = ('day_of_week', 'low', 'high', 'icon', 'condition')

        forecasts = []
        for forecast in dom.getElementsByTagName('forecast_conditions'):
                tmp_forecast = {}
                for tag in forecast_conditions:
                        tmp_forecast[tag] = forecast.getElementsByTagName(tag)[0].getAttribute('data')
                forecasts.append(tmp_forecast)
                weather_data['forecasts'] = forecasts

        dom.unlink()

        return weather_data

def get_weather_from_yahoo(location_id, units = 'metric'):
        """
Fetches weather report from Yahoo!

Parameters
location_id: A five digit US zip code or location ID. To find your location ID,
browse or search for your city from the Weather home page(http://weather.yahoo.com/)
The weather ID is in the URL for the forecast page for that city. You can also get the location ID by entering your zip code on the home page. For example, if you search for Los Angeles on the Weather home page, the forecast page for that city is http://weather.yahoo.com/forecast/USCA0638.html. The location ID is USCA0638.

units: type of units. 'metric' for metric and '' for  non-metric
Note that choosing metric units changes all the weather units to metric, for example, wind speed will be reported as kilometers per hour and barometric pressure as millibars.

Returns:
weather_data: a dictionary of weather data. See  http://developer.yahoo.com/weather/#channel
        """

        if units == 'metric':
                unit = 'c'
        else:
                 unit = 'f'
        url = YAHOO_WEATHER_URL % (location_id, unit)
        handler = urllib2.urlopen(url)
        dom = minidom.parse(handler)
        handler.close()

        weather_data = {}
        weather_data['title'] = dom.getElementsByTagName('title')[0].firstChild.data
        weather_data['link'] = dom.getElementsByTagName('link')[0].firstChild.data

        ns_data_structure = {
                'location': ('city', 'region', 'country'),
                'units': ('temperature', 'distance', 'pressure', 'speed'),
                'wind': ('chill', 'direction', 'speed'),
                'atmosphere': ('humidity', 'visibility', 'pressure', 'rising'),
                'astronomy': ('sunrise', 'sunset'),
                'condition': ('text', 'code', 'temp', 'date')

        }

        for (tag, attrs) in ns_data_structure.iteritems():
                weather_data[tag] = xml_get_ns_yahoo_tag(dom, YAHOO_WEATHER_NS, tag, attrs)

        weather_data['geo'] = {}
        weather_data['geo']['lat'] = dom.getElementsByTagName('geo:lat')[0].firstChild.data
        weather_data['geo']['long'] = dom.getElementsByTagName('geo:long')[0].firstChild.data

        weather_data['condition']['title'] = dom.getElementsByTagName('item')[0].getElementsByTagName('title')[0].firstChild.data
        weather_data['html_description'] = dom.getElementsByTagName('item')[0].getElementsByTagName('description')[0].firstChild.data

        forecasts = []
        for forecast in dom.getElementsByTagNameNS(YAHOO_WEATHER_NS, 'forecast'):
                forecasts.append(xml_get_attrs(forecast,('date', 'low', 'high', 'text', 'code')))
        weather_data['forecasts'] = forecasts

        dom.unlink()

        return weather_data

def get_weather_from_noaa(station_id):
        """
Fetches weather report from NOAA: National Oceanic and Atmospheric Administration (United States)

Parameters
station_id: To find your station ID, perform the following steps:
1. Open this URL: http://www.weather.gov/xml/current_obs/seek.php?state=az&Find=Find
2. Select the necessary state state. Click 'Find'.
3. Find the necessary station in the 'Observation Location' column.
4. The station ID is in the URL for the weather page for that station.
For example if the weather page is http://weather.noaa.gov/weather/current/KPEO.html -- the station ID is KPEO.

Other way to get the station ID: use this library: http://code.google.com/p/python-weather/ and 'Weather.location2station' function.

Returns:
weather_data: a dictionary of weather data.

(useful icons: http://www.weather.gov/xml/current_obs/weather.php)
        """

        url = NOAA_WEATHER_URL % (station_id)
        handler = urllib2.urlopen(url)
        dom = minidom.parse(handler)
        handler.close()

        data_structure = ('suggested_pickup',
                                'suggested_pickup_period',
                                'location',
                                'station_id',
                                'latitude',
                                'longitude',
                                'observation_time',
                                'observation_time_rfc822',
                                'weather',
                                'temperature_string',
                                'temp_f',
                                'temp_c',
                                'relative_humidity',
                                'wind_string',
                                'wind_dir',
                                'wind_degrees',
                                'wind_mph',
                                'wind_gust_mph',
                                'pressure_string',
                                'pressure_mb',
                                'pressure_in',
                                'dewpoint_string',
                                'dewpoint_f',
                                'dewpoint_c',
                                'heat_index_string',
                                'heat_index_f',
                                'heat_index_c',
                                'windchill_string',
                                'windchill_f',
                                'windchill_c',
                                'icon_url_base',
                                'icon_url_name',
                                'two_day_history_url',
                                'ob_url'
                                )
	weather_data = {}
        for tag in data_structure:
               weather_data[tag] =  dom.getElementsByTagName('current_observation')[0].getElementsByTagName(tag)[0].firstChild.data

        dom.unlink()
        return weather_data

def get_weather_from_gismeteo(location_id):
	"""
Fetches weather report from GisMeteo (russian weather service)

Parameters
location_id: A location ID of the necessary place. To find your location ID,
browse or search for your city from the Gismeteo page: http://informer.gismeteo.ru/xml.html?index=27612%CC%EE%F1%EA%E2%E0&&lang=ru (the page is in Russian)
The location ID is in the URL for the XML URL for that city.
For example, if you search for New York on the GisMeteo page, the XML URLfor that city is
http://informer.gismeteo.ru/xml/72503_1.xml. The location ID is 72503_1.

Returns:
weather_data: a dictionary of weather data. See http://informer.gismeteo.ru/xml.html?index=27612%CC%EE%F1%EA%E2%E0&&lang=ru (at the page bottom)
	"""
	url = GISMETEO_WEATHER_URL % (location_id)
	handler = urllib2.urlopen(url)
	dom = minidom.parse(handler)
	handler.close()

	forecast_data_structure = {
			'PHENOMENA': ('cloudiness','precipitation', 'rpower', 'spower'),
			'PRESSURE': ('max','min'),
			'TEMPERATURE': ('max','min'),
			'WIND': ('max','min', 'direction'),
			'RELWET': ('max','min'),
			'HEAT': ('max','min'),
			}
	town_tag_attr = ('index', 'sname', 'latitude','longitude')
	forecast_tag_attr = ('day', 'month', 'year', 'hour', 'tod', 'predict', 'weekday')

	weather_data = {}

    	for attrs in town_tag_attr:
    		weather_data['town'] = xml_get_attrs(dom.getElementsByTagName('TOWN')[0], attrs)

        forecasts = []
	xml_forecats_node  = dom.getElementsByTagName('FORECAST')

        for i  in range(xml_forecats_node.length):
		forecast = dom.getElementsByTagName('MMWEATHER')[0].getElementsByTagName('REPORT')[0].getElementsByTagName('TOWN')[0].getElementsByTagName('FORECAST').item(i)
        	_tmp_forecast = {}
		_tmp_forecast.update(xml_get_attrs(forecast, forecast_tag_attr))
		forecast = xml_forecats_node.item(i)
		for (tag, attrs) in forecast_data_structure.iteritems():
			_tmp_forecast[tag] = xml_get_attrs(forecast, attrs)
			forecasts.append(_tmp_forecast)

        weather_data['forecasts'] = forecasts
	return weather_data

def xml_get_ns_yahoo_tag(dom, YAHOO_WEATHER_NS, tag, attrs):
        """
        Parses the necessary tag and returns the dictionary with values

        Parameters:
        dom - DOM
        YAHOO_WEATHER_NS - namespace
        tag - necessary tag
        attrs - tuple of attributes

        Returns: a dictionary of elements
        """
        element = dom.getElementsByTagNameNS(YAHOO_WEATHER_NS, tag)[0]
        return xml_get_attrs(element,attrs)

def xml_get_attrs(xml_element, attrs):
        """
        Returns the list of necessary attributes

        Parameters:
        element: xml element
        attrs: tuple of attributes

        Return: a dictionary of elements
        """

        result = {}
        for attr in attrs:
                result[attr] = xml_element.getAttribute(attr)
        return result
<|endoftext|>


<commit_before>import argparse
import json
import os

import converters

INFO_JSON = "./.info.json"

def default_ymir_ap():
    ap = argparse.ArgumentParser()
    ap.add_argument("-i", "--input", nargs = "+", help = "input file (text or gzipped) or a folder with input files (of the same format) or a list of space-separated files and/or folders in any combinations", type = str, default = "")
    ap.add_argument("-f", "--format", help = "format of input files (tcR, MiTCR, MiGEC, etc.) as an alias or as a Python 3 class from your module linked to the '$YMIR_HOME/converters' package. For a list of possible input formats and their aliases in this Ymir distribution run $python3 pyymir.py -f", type = str, default = "")
    ap.add_argument("-m", "--model", help = "either an alias of the one from available models in Ymir or a path to a folder with a model's .json file. For a list of available models in this Ymir distribution run $python3 pyymir.py -m", type = str, default = "")
    return ap

def parse_input(args):
    """
    Parse the input argument and return a list of files (possibly found in the input folders).
    """

    def _get_files_from_folder(folderpath):
        # return [folderpath + "/" + f for f in os.listdir(folderpath) if f[0] != "." and "ymir_in" not in f]
        return [folderpath + "/" + f for f in os.listdir(folderpath) if f[0] != "."]

    args_input = args.input

    res = []
    flag = True

    if len(args_input):
        for x in args_input:
            if os.path.exists(x):
                if os.path.isdir(x):
                    res.extend(_get_files_from_folder(x))
                else:
                    res.append(x)
            else:
                print('\tERROR: file "', x, '" not found )\':', sep = "")
                flag = False
    else:
        print("\tERROR: please specify at least one input file ]:")
        flag = False

    return res, flag

def parse_model(args):
    """
    Check if the given string is an alias of the one of available models or
    it is a path to a folder with a model.
    If it's an alias, than return a path to this model.
    If it's a path then check for a model.json file in this folder.
    """

    args_model = args.model

    res = ""
    flag = True

    if args_model:
        cur_models = {}

        with open(INFO_JSON) as f:
            jsdata = json.load(f)["models"]["values"]
            cur_models = {jsdata[key]["alias"] : jsdata[key]["path"] for key in jsdata}

        if args_model in cur_models:
            res = cur_models[args_model]
        else:
            if os.path.exists(args_model + "/model.json"):
                res = args_model
            else:
                print('\tERROR: model "', args_model, '" not found o:<', sep = "")
    else:
        print("\tERROR: please specify a generation model >:")
        flag = False

    return res, flag

def parse_format(args):
    """
    Parse format and return a class for convert one repertoire format to Ymir's format.
    """

    args_format = args.format

    res = None
    flag = True

    if len(args_format):
        with open(INFO_JSON) as f:
            jsdata = json.load(f)["formats"]["values"]
            # print(jsdata)
            if args_format in jsdata:
                res = jsdata[args_format]["pyclass"]
            else:
                # if format is custom than search for it
                print("\tERROR: unknown format or python converter class o:")

            res = getattr(converters, res)
    else:
        print("\tERROR: please specify a format for input files :3")
        flag = False

    return res, flag

def parse_output_files(infiles, args):
    """
    Create the output folder for output files.
    """

    args_out = args.output
    if not os.path.exists(args_out):
        os.makedirs(args_out)
    return [args_out + "/" + x[x.rfind("/") + 1:] + ".ymir_out.txt" for x in infiles], True

def parse_output_models(infiles, args):
    """
    Create the output folder and all inner folders for each model for input files.
    """

    args_out = args.output
    res = []
    for x in infiles:
        new_x = args_out + "/" + x[x.rfind("/") + 1:x.rfind(".")] + ".model/"
        if not os.path.exists(new_x):
            os.makedirs(new_x)
        res.append(new_x)
    return res, True

def convert(filepath, converter):
    out_file = ""
    flag = True
    if converter is converters.YmirConverter:
        out_file = filepath
        flag = True
    else:
        out_file = filepath[:filepath.rfind(".")] + ".ymir_in" + filepath[filepath.rfind("."):]
        print("Converting '", filepath, "' to '", out_file, "'", "...", end = "\t", sep = "")
        c = converter()
        flag = c.convert(file_in=filepath, file_out=out_file)
        if flag: print("Done.")
    return out_file, flag

def extract_info(arg, jsdata):
    """
    Extract information from Ymir's general info .json file.
    """

    def _pretty_dict(key, d, shift):
        if type(d) is list:
            for key, val in sorted(d):
                _pretty_dict(key, val, shift + 1)
        elif type(d) is dict:
            for key, val in sorted(d.items()):
                _pretty_dict(key, val, shift + 1)
        else:
            if key == "alias":
                print("  " * shift, '"', d, '" :\t', sep = '', end = '')
            elif key == "comment":
                print(d)
            else:
                print("  " * shift, key, ": ", d, sep = '')

    print(jsdata[arg]["title"])
    for val in jsdata[arg]["values"]:
        _pretty_dict(val, jsdata[arg]["values"][val], 1)
    print()

if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument("-v", "--version", help = "Display an information about the current Ymir distribution", action = "store_true")
    ap.add_argument("-a", "--algos", help = "List of available algorithms for the statistical inference with their parameters in this Ymir distribution", action = "store_true")
    ap.add_argument("-f", "--formats", help = "list of possible input formats in this Ymir distribution", action = "store_true")
    ap.add_argument("-m", "--models", help = "List of available models in this Ymir distribution", action = "store_true")
    ap.add_argument("-s", "--scripts", help = "List of available ready-to-use scripts", action = "store_true")
    args = ap.parse_args()

    jsdata = None
    with open(INFO_JSON) as f:
        jsdata = json.load(f)
        extract_info("ymir", jsdata)
        some_true = False
        for arg, val in vars(args).items():
            if val and arg != "version":
                extract_info(arg, jsdata)
                some_true = True
        if not some_true:
            for arg, val in vars(args).items():
                if arg != "version":
                    extract_info(arg, jsdata)
<commit_msg>add correct names for files with .gz<commit_after>import argparse
import json
import os

import converters

INFO_JSON = "./.info.json"

def default_ymir_ap():
    ap = argparse.ArgumentParser()
    ap.add_argument("-i", "--input", nargs = "+", help = "input file (text or gzipped) or a folder with input files (of the same format) or a list of space-separated files and/or folders in any combinations", type = str, default = "")
    ap.add_argument("-f", "--format", help = "format of input files (tcR, MiTCR, MiGEC, etc.) as an alias or as a Python 3 class from your module linked to the '$YMIR_HOME/converters' package. For a list of possible input formats and their aliases in this Ymir distribution run $python3 pyymir.py -f", type = str, default = "")
    ap.add_argument("-m", "--model", help = "either an alias of the one from available models in Ymir or a path to a folder with a model's .json file. For a list of available models in this Ymir distribution run $python3 pyymir.py -m", type = str, default = "")
    return ap

def parse_input(args):
    """
    Parse the input argument and return a list of files (possibly found in the input folders).
    """

    def _get_files_from_folder(folderpath):
        # return [folderpath + "/" + f for f in os.listdir(folderpath) if f[0] != "." and "ymir_in" not in f]
        return [folderpath + "/" + f for f in os.listdir(folderpath) if f[0] != "."]

    args_input = args.input

    res = []
    flag = True

    if len(args_input):
        for x in args_input:
            if os.path.exists(x):
                if os.path.isdir(x):
                    res.extend(_get_files_from_folder(x))
                else:
                    res.append(x)
            else:
                print('\tERROR: file "', x, '" not found )\':', sep = "")
                flag = False
    else:
        print("\tERROR: please specify at least one input file ]:")
        flag = False

    return res, flag

def parse_model(args):
    """
    Check if the given string is an alias of the one of available models or
    it is a path to a folder with a model.
    If it's an alias, than return a path to this model.
    If it's a path then check for a model.json file in this folder.
    """

    args_model = args.model

    res = ""
    flag = True

    if args_model:
        cur_models = {}

        with open(INFO_JSON) as f:
            jsdata = json.load(f)["models"]["values"]
            cur_models = {jsdata[key]["alias"] : jsdata[key]["path"] for key in jsdata}

        if args_model in cur_models:
            res = cur_models[args_model]
        else:
            if os.path.exists(args_model + "/model.json"):
                res = args_model
            else:
                print('\tERROR: model "', args_model, '" not found o:<', sep = "")
    else:
        print("\tERROR: please specify a generation model >:")
        flag = False

    return res, flag

def parse_format(args):
    """
    Parse format and return a class for convert one repertoire format to Ymir's format.
    """

    args_format = args.format

    res = None
    flag = True

    if len(args_format):
        with open(INFO_JSON) as f:
            jsdata = json.load(f)["formats"]["values"]
            # print(jsdata)
            if args_format in jsdata:
                res = jsdata[args_format]["pyclass"]
            else:
                # if format is custom than search for it
                print("\tERROR: unknown format or python converter class o:")

            res = getattr(converters, res)
    else:
        print("\tERROR: please specify a format for input files :3")
        flag = False

    return res, flag

def parse_output_files(infiles, args):
    """
    Create the output folder for output files.
    """

    args_out = args.output
    if not os.path.exists(args_out):
        os.makedirs(args_out)
    return [args_out + "/" + x[x.rfind("/") + 1:] + ".ymir_out.txt" for x in infiles], True

def parse_output_models(infiles, args):
    """
    Create the output folder and all inner folders for each model for input files.
    """

    args_out = args.output
    res = []
    for x in infiles:
        new_x = args_out + "/" + x[x.rfind("/") + 1:x.rfind(".")] + ".model/"
        if not os.path.exists(new_x):
            os.makedirs(new_x)
        res.append(new_x)
    return res, True

def convert(filepath, converter):
    out_file = ""
    flag = True
    if converter is converters.YmirConverter:
        out_file = filepath
        flag = True
    else:
        out_file = filepath[:filepath.rfind(".")] + ".ymir_in" + filepath[filepath.rfind("."):]
        if out_file.endswith('.gz'): out_file = out_file[:-2] + 'txt'
        print("Converting '", filepath, "' to '", out_file, "'", "...", end = "\t", sep = "")
        c = converter()
        flag = c.convert(file_in=filepath, file_out=out_file)
        if flag: print("Done.")
    return out_file, flag

def extract_info(arg, jsdata):
    """
    Extract information from Ymir's general info .json file.
    """

    def _pretty_dict(key, d, shift):
        if type(d) is list:
            for key, val in sorted(d):
                _pretty_dict(key, val, shift + 1)
        elif type(d) is dict:
            for key, val in sorted(d.items()):
                _pretty_dict(key, val, shift + 1)
        else:
            if key == "alias":
                print("  " * shift, '"', d, '" :\t', sep = '', end = '')
            elif key == "comment":
                print(d)
            else:
                print("  " * shift, key, ": ", d, sep = '')

    print(jsdata[arg]["title"])
    for val in jsdata[arg]["values"]:
        _pretty_dict(val, jsdata[arg]["values"][val], 1)
    print()

if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument("-v", "--version", help = "Display an information about the current Ymir distribution", action = "store_true")
    ap.add_argument("-a", "--algos", help = "List of available algorithms for the statistical inference with their parameters in this Ymir distribution", action = "store_true")
    ap.add_argument("-f", "--formats", help = "list of possible input formats in this Ymir distribution", action = "store_true")
    ap.add_argument("-m", "--models", help = "List of available models in this Ymir distribution", action = "store_true")
    ap.add_argument("-s", "--scripts", help = "List of available ready-to-use scripts", action = "store_true")
    args = ap.parse_args()

    jsdata = None
    with open(INFO_JSON) as f:
        jsdata = json.load(f)
        extract_info("ymir", jsdata)
        some_true = False
        for arg, val in vars(args).items():
            if val and arg != "version":
                extract_info(arg, jsdata)
                some_true = True
        if not some_true:
            for arg, val in vars(args).items():
                if arg != "version":
                    extract_info(arg, jsdata)
<|endoftext|>


<commit_before>import numpy as np
import scipy.interpolate as spi
import matplotlib.pyplot as plt

class PDF(object):

    def __init__(self, truth=None):
        self.truth = truth
        self.quantiles = None
        self.difs = None
        self.mids = None
        self.quantvals = None
        self.interpolator = None

    def evaluate(self, loc):

        return

    def integrate(self, limits):

        return

    def quantize(self, percent=1., number=None):
        """
        Computes an array of evenly-spaced quantiles.

        Parameters
        ----------
        percent : float
            The separation of the requested quantiles, in percent
        num_points : int
            The number of quantiles to compute.

        Returns
        -------
        self.quantiles : ndarray, float
            The quantile points.

        Comments
        --------
        Quantiles of a PDF could be a useful approximate way to store it. This method computes the quantiles, and stores them in the
        `self.quantiles` attribute.

        Uses the `.ppf` method of the `rvs_continuous` distribution
        object stored in `self.truth`. This calculates the inverse CDF.
        See `the Scipy docs <https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.rv_continuous.ppf.html#scipy.stats.rv_continuous.ppf>`_ for details.
        """
        if number is not None:
            # Compute the spacing of the quantiles:
            quantum = 1.0 / float(number+1)
        else:
            quantum = percent/100.0
            # Over-write the number of quantiles:
            number = np.ceil(100.0 / percent) - 1
            assert number > 0

        points = np.linspace(0.0+quantum, 1.0-quantum, number)
        print("Calculating quantiles: ", points)
        self.quantiles = self.truth.ppf(points)
        print("Result: ", self.quantiles)
        return self.quantiles

    def interpolate(self):#, number=100, grid=None):
        """
        Returns interpolator based on quantiles.

        Parameters
        ----------
        None

        Returns
        -------
        self.interpolator: interpolator
            The interpolating function
        """
#         if self.interpolator is None:

        if self.quantiles is None:
            self.quantiles = self.quantize()
        self.difs = self.quantiles[1:]-self.quantiles[:-1]
        self.mids = (self.quantiles[1:]+self.quantiles[:-1])/2.
        self.quantvals = (1.0/(len(self.quantiles)+1))/self.difs

        print("Creating interpolator")
        self.interpolator = spi.interp1d(self.mids, self.quantvals, fill_value="extrapolate")

#         if grid is None:
#             grid = np.linspace(min(self.mids), max(self.mids), number)

        return self.interpolator

    def approximate(self, points):

        """
        Interpolates the quantiles.

        Parameters
        ----------
        number: int
            The number of points over which to interpolate, bounded by the quantile value endpoints
        points: ndarray
            The value(s) at which to evaluate the interpolated function

        Returns
        -------
        points: ndarray, float
            The input grid upon which to interpolate
        interpolated : ndarray, float
            The interpolated points.

        Comments
        --------
        Extrapolation is linear while values are positive; otherwise, extrapolation returns 0.

        Notes
        -----
        Example:
        > x, y = p.approximate(np.linspace(-1., 1., 100))
        """

        if self.interpolator is None:
            self.interpolator = self.interpolate()
        #print("Grid: ", x)
        interpolated = self.interpolator(points)
        interpolated[interpolated<0.] = 0.
        #print("Result: ", interpolated)

        return (points, interpolated)

    def plot(self, limits, points=None):
        """
        Plot the PDF, in various ways.

        Parameters
        ----------
        limits : tuple, float
            Range over which to plot the PDF
        points: ndarray
            The value(s) at which to evaluate the interpolator

        Notes
        -----
        What this method plots depends on what information about the PDF is stored in it: the more properties the PDF has, the more exciting the plot!
        """
        x = np.linspace(limits[0], limits[1], 100)

        if self.truth is not None:
            plt.plot(x, self.truth.pdf(x), color='k', linestyle='-', lw=1.0, alpha=1.0, label='True PDF')

        if self.quantiles is not None:
            y = [0., 1.]
            plt.vlines(self.quantiles, y[0], y[1], color='k', linestyle='--', lw=1.0, alpha=1., label='Quantiles')

        if points is not None:
            (grid, interpolated) = self.approximate(points)
            plt.plot(grid, interpolated, color='r', linestyle=':', lw=2.0, alpha=1.0, label='Interpolated PDF')

        plt.legend()
        plt.xlabel('x')
        plt.ylabel('Probability density')
        plt.savefig('plot.png')

        return
<commit_msg>interpolate() no longer returns anything since approximate() now does all the work<commit_after>import numpy as np
import scipy.interpolate as spi
import matplotlib.pyplot as plt

class PDF(object):

    def __init__(self, truth=None):
        self.truth = truth
        self.quantiles = None
        self.difs = None
        self.mids = None
        self.quantvals = None
        self.interpolator = None

    def evaluate(self, loc):

        return

    def integrate(self, limits):

        return

    def quantize(self, percent=1., number=None):
        """
        Computes an array of evenly-spaced quantiles.

        Parameters
        ----------
        percent : float
            The separation of the requested quantiles, in percent
        num_points : int
            The number of quantiles to compute.

        Returns
        -------
        self.quantiles : ndarray, float
            The quantile points.

        Comments
        --------
        Quantiles of a PDF could be a useful approximate way to store it. This method computes the quantiles, and stores them in the
        `self.quantiles` attribute.

        Uses the `.ppf` method of the `rvs_continuous` distribution
        object stored in `self.truth`. This calculates the inverse CDF.
        See `the Scipy docs <https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.rv_continuous.ppf.html#scipy.stats.rv_continuous.ppf>`_ for details.
        """
        if number is not None:
            # Compute the spacing of the quantiles:
            quantum = 1.0 / float(number+1)
        else:
            quantum = percent/100.0
            # Over-write the number of quantiles:
            number = np.ceil(100.0 / percent) - 1
            assert number > 0

        points = np.linspace(0.0+quantum, 1.0-quantum, number)
        print("Calculating quantiles: ", points)
        self.quantiles = self.truth.ppf(points)
        print("Result: ", self.quantiles)
        return self.quantiles

    def interpolate(self):#, number=100, grid=None):
        """
        Returns interpolator based on quantiles.

        Parameters
        ----------
        None

        Returns
        -------
        self.interpolator: interpolator
            The interpolating function
        """
#         if self.interpolator is None:

        if self.quantiles is None:
            self.quantiles = self.quantize()
        self.difs = self.quantiles[1:]-self.quantiles[:-1]
        self.mids = (self.quantiles[1:]+self.quantiles[:-1])/2.
        self.quantvals = (1.0/(len(self.quantiles)+1))/self.difs

        print("Creating interpolator")
        self.interpolator = spi.interp1d(self.mids, self.quantvals, fill_value="extrapolate")

#         if grid is None:
#             grid = np.linspace(min(self.mids), max(self.mids), number)

        return

    def approximate(self, points):

        """
        Interpolates the quantiles.

        Parameters
        ----------
        number: int
            The number of points over which to interpolate, bounded by the quantile value endpoints
        points: ndarray
            The value(s) at which to evaluate the interpolated function

        Returns
        -------
        points: ndarray, float
            The input grid upon which to interpolate
        interpolated : ndarray, float
            The interpolated points.

        Comments
        --------
        Extrapolation is linear while values are positive; otherwise, extrapolation returns 0.

        Notes
        -----
        Example:
        > x, y = p.approximate(np.linspace(-1., 1., 100))
        """

        if self.interpolator is None:
            self.interpolator = self.interpolate()
        #print("Grid: ", x)
        interpolated = self.interpolator(points)
        interpolated[interpolated<0.] = 0.
        #print("Result: ", interpolated)

        return (points, interpolated)

    def plot(self, limits, points=None):
        """
        Plot the PDF, in various ways.

        Parameters
        ----------
        limits : tuple, float
            Range over which to plot the PDF
        points: ndarray
            The value(s) at which to evaluate the interpolator

        Notes
        -----
        What this method plots depends on what information about the PDF is stored in it: the more properties the PDF has, the more exciting the plot!
        """
        x = np.linspace(limits[0], limits[1], 100)

        if self.truth is not None:
            plt.plot(x, self.truth.pdf(x), color='k', linestyle='-', lw=1.0, alpha=1.0, label='True PDF')

        if self.quantiles is not None:
            y = [0., 1.]
            plt.vlines(self.quantiles, y[0], y[1], color='k', linestyle='--', lw=1.0, alpha=1., label='Quantiles')

        if points is not None:
            (grid, interpolated) = self.approximate(points)
            plt.plot(grid, interpolated, color='r', linestyle=':', lw=2.0, alpha=1.0, label='Interpolated PDF')

        plt.legend()
        plt.xlabel('x')
        plt.ylabel('Probability density')
        plt.savefig('plot.png')

        return
<|endoftext|>