<commit_before>
import tornado.ioloop
import tornado.web
from tornado import gen

import pymongo
import pymongo.errors as perr

import ujson
import jsonschema

from analysisbucket.server import utils

loop = tornado.ioloop.IOLoop.instance()

def db_connect(database, host, port):
    """Helper function to deal with stateful connections to motor.
    Connection established lazily.

    Parameters
    ----------
    database: str
        The name of database pymongo creates and/or connects
    host: str
        Name/address of the server that mongo daemon lives
    port: int
        Port num of the server

    Returns pymongo.MotorDatabase
    -------
        Async server object which comes in handy as server has to juggle multiple clients
        and makes no difference for a single client compared to pymongo
    """
    client = pymongo.MongoClient(host=host, port=port)
    database = client[database]
    return database

class DefaultHandler(tornado.web.RequestHandler):
    """DefaultHandler which takes care of CORS for @hslepicka js gui. Does not hurt, one day we might need this"""
    @tornado.web.asynchronous
    def set_default_headers(self):
        self.set_header('Access-Control-Allow-Origin', '*')
        self.set_header('Access-Control-Allow-Methods', 'POST, GET, OPTIONS')
        self.set_header('Access-Control-Max-Age', 1000)
        self.set_header('Access-Control-Allow-Headers', '*')
        self.set_header('Content-type', 'application/json')

    def data_received(self, chunk):
        """Abstract method, here to show it exists explicitly. Useful for streaming client"""
        pass

class AnalysisHeaderHandler(DefaultHandler):
    """Handler for analysis_header insert and query operations.
    Uses traditional RESTful lingo. get for querying and post for inserts

   Methods
    -------
    get()
        Query analysis_header documents. Query params are jsonified for type preservation
    post()
        Insert a analysis_header document.Same validation method as bluesky, secondary
        safety net.
    """

    @tornado.web.asynchronous
    def get(self):
        database = self.settings['db']
        query = utils.unpack_params(self)
        _id = query.pop('_id', None)
        find_last = query.pop('find_last', None)
        if _id:
            raise tornado.web.HTTPError(500, 'No ObjectId based search supported')
        if find_last:
            docs = database.analysis_header.find().sort('time', direction=pymongo.DESCENDING)
        else:
            docs = database.analysis_header.find(query)

        if not docs:
            raise tornado.web.HTTPError(500, reason='No results found for query')
        else:
            utils.return2client(self, docs)

    @tornado.web.asynchronous
    def post(self):
        database = self.settings['db']
        data = ujson.loads(self.request.body.decode("utf-8"))
        jsonschema.validate(data, utils.schemas['analysis_header'])
        try:
            result = database.analysis_header.insert(data)
        except perr.PyMongoError:
            raise tornado.web.HTTPError(500,
                                        status='Unable to insert the document')

        database.analysis_header.create_index([('uid', pymongo.ASCENDING)],
                                       unique=True, background=True)
        database.analysis_header.create_index([('time', pymongo.ASCENDING)],
                                        unique=False)

        if not result:
            raise tornado.web.HTTPError(500)
        else:
            utils.return2client(self, data)<commit_msg>WIP:Added analysis tail collection handler support<commit_after>
import tornado.ioloop
import tornado.web
from tornado import gen

import pymongo
import pymongo.errors as perr

import ujson
import jsonschema

from analysisbucket.server import utils

loop = tornado.ioloop.IOLoop.instance()

def db_connect(database, host, port):
    """Helper function to deal with stateful connections to motor.
    Connection established lazily.

    Parameters
    ----------
    database: str
        The name of database pymongo creates and/or connects
    host: str
        Name/address of the server that mongo daemon lives
    port: int
        Port num of the server

    Returns pymongo.MotorDatabase
    -------
        Async server object which comes in handy as server has to juggle multiple clients
        and makes no difference for a single client compared to pymongo
    """
    client = pymongo.MongoClient(host=host, port=port)
    database = client[database]
    return database

class DefaultHandler(tornado.web.RequestHandler):
    """DefaultHandler which takes care of CORS for @hslepicka js gui. Does not hurt, one day we might need this"""
    @tornado.web.asynchronous
    def set_default_headers(self):
        self.set_header('Access-Control-Allow-Origin', '*')
        self.set_header('Access-Control-Allow-Methods', 'POST, GET, OPTIONS')
        self.set_header('Access-Control-Max-Age', 1000)
        self.set_header('Access-Control-Allow-Headers', '*')
        self.set_header('Content-type', 'application/json')

    def data_received(self, chunk):
        """Abstract method, here to show it exists explicitly. Useful for streaming client"""
        pass

class AnalysisHeaderHandler(DefaultHandler):
    """Handler for analysis_header insert and query operations.
    Uses traditional RESTful lingo. get for querying and post for inserts

   Methods
    -------
    get()
        Query analysis_header documents. Query params are jsonified for type preservation
    post()
        Insert a analysis_header document.Same validation method as bluesky, secondary
        safety net.
    """

    @tornado.web.asynchronous
    def get(self):
        database = self.settings['db']
        query = utils.unpack_params(self)
        _id = query.pop('_id', None)
        find_last = query.pop('find_last', None)
        if _id:
            raise tornado.web.HTTPError(500, 'No ObjectId based search supported')
        if find_last:
            docs = database.analysis_header.find().sort('time',
                                direction=pymongo.DESCENDING)
        else:
            docs = database.analysis_header.find(query)

        if not docs:
            raise tornado.web.HTTPError(500, reason='No results found for query')
        else:
            utils.return2client(self, docs)

    @tornado.web.asynchronous
    def post(self):
        database = self.settings['db']
        data = ujson.loads(self.request.body.decode("utf-8"))
        jsonschema.validate(data, utils.schemas['analysis_header'])
        try:
            result = database.analysis_header.insert(data)
        except perr.PyMongoError:
            raise tornado.web.HTTPError(500,
                                        status='Unable to insert the document')

        database.analysis_header.create_index([('uid', pymongo.ASCENDING)],
                                       unique=True, background=True)
        database.analysis_header.create_index([('time', pymongo.ASCENDING)],
                                        unique=False)

        if not result:
            raise tornado.web.HTTPError(500)
        else:
            utils.return2client(self, data)<|endoftext|>


<commit_before>#!/usr/bin/env python2

# Shamelessly ripped POC from https://gist.github.com/takeshixx/10107280
# CVE-2014-0160

import sys
import struct
import socket
import time
import select
import re
import argparse

v = 0

def h2bin(x):
    return x.replace(' ', '').replace('\n', '').decode('hex')

hello = h2bin('''
16 03 02 00  dc 01 00 00 d8 03 02 53
43 5b 90 9d 9b 72 0b bc  0c bc 2b 92 a8 48 97 cf
bd 39 04 cc 16 0a 85 03  90 9f 77 04 33 d4 de 00
00 66 c0 14 c0 0a c0 22  c0 21 00 39 00 38 00 88
00 87 c0 0f c0 05 00 35  00 84 c0 12 c0 08 c0 1c
c0 1b 00 16 00 13 c0 0d  c0 03 00 0a c0 13 c0 09
c0 1f c0 1e 00 33 00 32  00 9a 00 99 00 45 00 44
c0 0e c0 04 00 2f 00 96  00 41 c0 11 c0 07 c0 0c
c0 02 00 05 00 04 00 15  00 12 00 09 00 14 00 11
00 08 00 06 00 03 00 ff  01 00 00 49 00 0b 00 04
03 00 01 02 00 0a 00 34  00 32 00 0e 00 0d 00 19
00 0b 00 0c 00 18 00 09  00 0a 00 16 00 17 00 08
00 06 00 07 00 14 00 15  00 04 00 05 00 12 00 13
00 01 00 02 00 03 00 0f  00 10 00 11 00 23 00 00
00 0f 00 01 01
''')

hb = h2bin('''
18 03 02 00 03
01 40 00
''')

def hexdump(s):
    for b in xrange(0, len(s), 16):
        lin = [c for c in s[b : b + 16]]
        hxdat = ' '.join('%02X' % ord(c) for c in lin)
        pdat = ''.join((c if 32 <= ord(c) <= 126 else '.' )for c in lin)
        print ' %04x: %-48s %s' % (b, hxdat, pdat)
    print

def recvall(s, length, timeout=5):
    endtime = time.time() + timeout
    rdata = ''
    remain = length
    while remain > 0:
        rtime = endtime - time.time()
        if rtime < 0:
            return None
        r, w, e = select.select([s], [], [], 5)
        if s in r:
            try:
                data = s.recv(remain)
            except socket.error:
                return None
            # EOF?
            if not data:
                return None
            rdata += data
            remain -= len(data)
    return rdata

def recvmsg(s):
    hdr = recvall(s, 5)
    if hdr is None:
        if v > 0: print 'Unexpected EOF receiving record header - server closed connection'
        return None, None, None
    typ, ver, ln = struct.unpack('>BHH', hdr)
    pay = recvall(s, ln, 10)
    if pay is None:
        if v > 0: print 'Unexpected EOF receiving record payload - server closed connection'
        return None, None, None
    if v > 1: print ' ... received message: type = %d, ver = %04x, length = %d' % (typ, ver, len(pay))
    return typ, ver, pay

def hit_hb(s):
    s.send(hb)
    while True:
        typ, ver, pay = recvmsg(s)
        if typ is None:
            return False

        if typ == 24:
            if v > 1: print 'Received heartbeat response:'
            if v > 2: hexdump(pay)
            if len(pay) > 3:
                return True
            else:
                return False
            return False

        if typ == 21:
            if v > 2: hexdump(pay)
            if v > 1: print 'Server returned error, likely not vulnerable'
            return False

def scan_hb(address, port):

    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.connect((address, port))

    s.send(hello)

    while True:
        typ, ver, pay = recvmsg(s)
        if typ == None:
            return False
        if typ == 22 and ord(pay[0]) == 0x0E:
            break

    s.send(hb)
    return hit_hb(s)

def scan_port(host, port):
    if v > 0:
        print "Scanning %s:%d" % (host, port)
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
        s.connect((host, port))
        s.close()
        return 0
    except socket.timeout:
        if v > 2: print "Socket timed out on %s:%d" % (host, port)
        return 1
    except:
        return 2

def write_vul_address(outFile, target):
    with open(outFile, "a") as output:
        output.write(target + "\n")
        output.close()

def main():
    global v
    parser = argparse.ArgumentParser(description="Basic scanner for the Heartbeat vulnerability")
    group = parser.add_mutually_exclusive_group(required=True)
    parser.add_argument("-o", "--output", help="File with vulnerable hosts")
    parser.add_argument("-p", "--port", type=int, default=443, help="TCP port the scan default=443")
    parser.add_argument("--timeout", type=int, default=1, help="Set timeout default=1 second")
    parser.add_argument("-v", dest="verbosity", action="count", default=0, help="Prints verbose output")
    group.add_argument("-f", "--file", help="File with hosts to scan")
    group.add_argument("-i", "--ips", metavar="ip", nargs="+")

    args = parser.parse_args()

    socket.setdefaulttimeout(args.timeout)
    if args.verbosity > 0: v = args.verbosity

    if args.file:
        with open(args.file) as inFile:
            targets = [ target.strip() for target in inFile.readlines() ]
    else:
        targets = args.ips

    for target in targets:
        result =  scan_port(target, args.port)
        if result == 0:
            if scan_hb(target, args.port):
                if args.output:
                    write_vul_address(args.output, target)
                print "Vulnerable:", target
            else:
                print "Patched:", target
        elif result == 1:
            print "Down:", target
        else:
            print "Closed:", target

if __name__ == '__main__':
    main()
<commit_msg>Made some argparse changes. Like ips is now a positional argument<commit_after>#!/usr/bin/env python2

# Shamelessly ripped POC from https://gist.github.com/takeshixx/10107280
# CVE-2014-0160

import sys
import struct
import socket
import time
import select
import re
import argparse

v = 0
usage = """sage: hb-scan.py [-h] [-o OUTPUT] [-p PORT] [--timeout TIMEOUT] [-v]
                  ([-f FILE] | [ip [ip ...]] )
"""

def h2bin(x):
    return x.replace(' ', '').replace('\n', '').decode('hex')

hello = h2bin('''
16 03 02 00  dc 01 00 00 d8 03 02 53
43 5b 90 9d 9b 72 0b bc  0c bc 2b 92 a8 48 97 cf
bd 39 04 cc 16 0a 85 03  90 9f 77 04 33 d4 de 00
00 66 c0 14 c0 0a c0 22  c0 21 00 39 00 38 00 88
00 87 c0 0f c0 05 00 35  00 84 c0 12 c0 08 c0 1c
c0 1b 00 16 00 13 c0 0d  c0 03 00 0a c0 13 c0 09
c0 1f c0 1e 00 33 00 32  00 9a 00 99 00 45 00 44
c0 0e c0 04 00 2f 00 96  00 41 c0 11 c0 07 c0 0c
c0 02 00 05 00 04 00 15  00 12 00 09 00 14 00 11
00 08 00 06 00 03 00 ff  01 00 00 49 00 0b 00 04
03 00 01 02 00 0a 00 34  00 32 00 0e 00 0d 00 19
00 0b 00 0c 00 18 00 09  00 0a 00 16 00 17 00 08
00 06 00 07 00 14 00 15  00 04 00 05 00 12 00 13
00 01 00 02 00 03 00 0f  00 10 00 11 00 23 00 00
00 0f 00 01 01
''')

hb = h2bin('''
18 03 02 00 03
01 40 00
''')

def hexdump(s):
    for b in xrange(0, len(s), 16):
        lin = [c for c in s[b : b + 16]]
        hxdat = ' '.join('%02X' % ord(c) for c in lin)
        pdat = ''.join((c if 32 <= ord(c) <= 126 else '.' )for c in lin)
        print ' %04x: %-48s %s' % (b, hxdat, pdat)
    print

def recvall(s, length, timeout=5):
    endtime = time.time() + timeout
    rdata = ''
    remain = length
    while remain > 0:
        rtime = endtime - time.time()
        if rtime < 0:
            return None
        r, w, e = select.select([s], [], [], 5)
        if s in r:
            try:
                data = s.recv(remain)
            except socket.error:
                return None
            # EOF?
            if not data:
                return None
            rdata += data
            remain -= len(data)
    return rdata

def recvmsg(s):
    hdr = recvall(s, 5)
    if hdr is None:
        if v > 0: print 'Unexpected EOF receiving record header - server closed connection'
        return None, None, None
    typ, ver, ln = struct.unpack('>BHH', hdr)
    pay = recvall(s, ln, 10)
    if pay is None:
        if v > 0: print 'Unexpected EOF receiving record payload - server closed connection'
        return None, None, None
    if v > 1: print ' ... received message: type = %d, ver = %04x, length = %d' % (typ, ver, len(pay))
    return typ, ver, pay

def hit_hb(s):
    s.send(hb)
    while True:
        typ, ver, pay = recvmsg(s)
        if typ is None:
            return False

        if typ == 24:
            if v > 1: print 'Received heartbeat response:'
            if v > 2: hexdump(pay)
            if len(pay) > 3:
                return True
            else:
                return False
            return False

        if typ == 21:
            if v > 2: hexdump(pay)
            if v > 1: print 'Server returned error, likely not vulnerable'
            return False

def scan_hb(address, port):

    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.connect((address, port))

    s.send(hello)

    while True:
        typ, ver, pay = recvmsg(s)
        if typ == None:
            return False
        if typ == 22 and ord(pay[0]) == 0x0E:
            break

    s.send(hb)
    return hit_hb(s)

def scan_port(host, port):
    if v > 0:
        print "Scanning %s:%d" % (host, port)
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
        s.connect((host, port))
        s.close()
        return 0
    except socket.timeout:
        if v > 2: print "Socket timed out on %s:%d" % (host, port)
        return 1
    except:
        return 2

def write_vul_address(outFile, target):
    with open(outFile, "a") as output:
        output.write(target + "\n")
        output.close()

def main():
    global v
    parser = argparse.ArgumentParser(description="Basic scanner for the Heartbeat vulnerability", usage=usage)
    parser.add_argument("-o", "--output", help="File to write vulnerable hosts to.")
    parser.add_argument("-p", "--port", type=int, default=443, help="TCP port the scan. default=443")
    parser.add_argument("--timeout", type=int, default=1, help="Set timeout in seconds. default=1")
    parser.add_argument("-v", dest="verbosity", action="count", default=0, help="Prints verbose output")
    parser.add_argument("-f", "--file", help="File with hosts to scan")
    parser.add_argument("ips", metavar="ip", nargs="*")

    args = parser.parse_args()

    if args.file and args.ips:
        parser.error("Must choose file or ip list")
    if not (args.file or args.ips):
        parser.error("Must provide ips or an input file")

    socket.setdefaulttimeout(args.timeout)
    if args.verbosity > 0: v = args.verbosity

    if args.file:
        with open(args.file) as inFile:
            targets = [ target.strip() for target in inFile.readlines() ]
    else:
        targets = args.ips

    for target in targets:
        result =  scan_port(target, args.port)
        if result == 0:
            if scan_hb(target, args.port):
                if args.output:
                    write_vul_address(args.output, target)
                print "Vulnerable:", target
            else:
                print "Patched:", target
        elif result == 1:
            print "Down:", target
        else:
            print "Closed:", target

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import os
import tempfile
import re

import pexpect

from abi_error import AbiConverterError, AbiCommandError
from cm.converters.pandoc_converters import do_tidy

TYPES_IN  = {'602': '602',       'abw': 'abw',       'aw': 'aw',
             'awt': 'awt',       'cwk': 'cwk',       'dbk': 'dbk',
             'doc': 'doc',       'docm': 'docm',     'docx': 'docx',
             'dot': 'dot',       'dotm': 'dotm',     'dotx': 'dotx',
             'fo': 'fo',         'htm': 'htm',       'html': 'html',
             'hwp': 'hwp',       'isc': 'isc',       'iscii': 'iscii',
             'kwd': 'kwd',       'mif': 'mif',       'odt': 'odt',
             'opml': 'opml',     'ott': 'ott',       'pdb': 'pdb',
             'pdf': 'pdf',       'rtf': 'rtf',       'sdw': 'sdw',
             'stw': 'stw',       'sxw': 'sxw',       'text': 'text',
             'txt': 'txt',       'wml': 'wml',       'wp': 'wp',
             'wpd': 'wpd',       'wri': 'wri',       'xhtml': 'xhtml',
             'xml': 'xml',       'zabw': 'zabw'}

TYPES_OUT = {'abw': 'abw',       'aw': 'aw',         'awt': 'awt',
             'dbk': 'dbk',       'doc': 'doc',       'docx': 'docx',
             'eml': 'eml',       'fo': 'fo',         'html': 'html',
             'isc': 'isc',       'iscii': 'iscii',   'kwd': 'kwd',
             'latex': 'latex',   'mht': 'mht',       'mif': 'mif',
             'nroff': 'nroff',   'nws': 'nws',       'odt': 'odt',
             'pdb': 'pdb',       'pdf': 'pdf',       'ps': 'ps',
             'rtf': 'rtf',       'sxw': 'sxw',       'text': 'text',
             'txt': 'txt',       'wml': 'wml',       'xml': 'xml',
             'xml2ps': 'xml2ps', 'zabw': 'zabw'}

class AbiFileConverter(object):
    """This let's you convert between all filetypes supperted by the
    AbiWord program. Import type isn't checked, as AbiWord doesn't check
    on extension, but on metadata.
    """

    def __init__(self, timeout=60):
        self.id = None
        self.timeout = timeout
        self._start_abiword()

    def _start_abiword(self):
        """
        Start abiword with the AbiCommand plugin, if not already started
        """

        # find the abiword executable
        abicommand = None
        for dir in os.environ['PATH'].split(':'):
            if os.path.isfile(os.path.join(dir, 'abiword')):
                abicommand = os.path.join(dir, 'abiword')
        if not abicommand:
            raise AbiConverterError('Can not find abiword executable')

        # start the abiword executable
        try:
            self.child = pexpect.spawn(abicommand + ' --plugin AbiCommand')
            self.child.expect(
                    'AbiWord command line plugin: Type "quit" to exit', 10)
        except:
            raise AbiConverterError('Can not open abiword executable')

    def stop_abiword(self):
        """
        Stop the running abiword, kill it if necessary
        """
        self.child.sendline('quit')
        if self._is_running():
            os.kill(self.child.pid, 9)

    def _is_running(self):
        """
        Test to see if abiword is running
        """
        try:
            self.child.sendline('writepid /dev/null')
            self.child.expect('OK', 1)
            return True
        except:
            return False

    def convert_file(self, in_file, out_file=None, type=None):
        """
        Convert a file. If out_file is not specified, a byte string is
        returned. If type is not specified, the file extension from out_file is
        used to determine the type. If this fails, the type 'text' is used.
        Return value is -1 if an error occurred.
        """
        # is the out_file specified?
        return_bytes = False
        if out_file is None:
            out_file = tempfile.mktemp(prefix="abiconvert_")
            return_bytes = True

        # is the type specified
        type = TYPES_OUT.get(
            type or os.path.splitext(out_file)[1][1:], 'txt')

        # do the conversion
        try:
          self._perform_conversion(in_file, out_file, type)
        except:
          raise

        # return a byte string if no out_file is specified
        if return_bytes:
            fp = open(out_file,  'r')
            bytes = fp.read()
            fp.close()
            os.remove(out_file)
            return bytes

    def _perform_conversion(self, in_file, out_file, type):
        """
        Do the actual conversion
        """
        # make sure we are up and running
        if not self._is_running:
            self._start_abiword()

        # convert the file
        cmd = 'convert %s %s %s' % (os.path.abspath(in_file),
                                    os.path.abspath(out_file), type)
        self.child.sendline(cmd)

        # Check for errors
        i = 1

        try:
          i = self.child.expect(['OK', pexpect.TIMEOUT])
          if i != 0:
            raise AbiCommandError('Error performing AbiCommand: %s' %cmd)
        except:
          raise

    def convert_to_html(self, input):
        """
        Convert input file to HTML
        """

        from tempfile import mkstemp,mkdtemp

        THE_OUTDIR = "outdir"
        THE_OUTFILE = "outfile"
        THE_INDIR = "indir"
        THE_INFILE = "infile"

        infile = None
        outfile = None
        out_f = None
        try:
          # create in/out files
          temp_dir = mkdtemp(prefix="cm_")

          # in
          indir_name = os.path.join(temp_dir, THE_INDIR)
          os.mkdir(indir_name)
          infile_name = os.path.join(indir_name, THE_INFILE)

          # out
          outdir_name = os.path.join(temp_dir, THE_OUTDIR)
          os.mkdir(outdir_name)
          outfile_name = os.path.join(outdir_name, THE_OUTFILE)

          # write infile
          infile = open(infile_name,'w')
          if type(input) == unicode:
            input = input.encode('utf8')
          infile.write(input)
          infile.close()

          # fix perms
          # TODO group permission should suffice
          os.chmod(temp_dir, 0755) # read
          os.chmod(indir_name, 0755) # read
          os.chmod(infile_name, 0755) # read
          os.chmod(outdir_name, 0777) # read / write

          # Do the job
          try:
            self.convert_file(infile_name, outfile_name, 'html')
          except:
            raise

          out_f = open(outfile_name,'r')
          output = out_f.read()

          # load other files (useful only for html)
          img_res = []
          if os.path.isdir(outdir_name + '/' + THE_OUTFILE + '_files'):
            image_names = [name for name in os.listdir(outdir_name + '/' + THE_OUTFILE + '_files') if name != THE_OUTFILE]
            for image_name in image_names:
              img_res.append(os.path.join(outdir_name + '/' + THE_OUTFILE + '_files', image_name))

            # clean images paths
            output = re.sub(r'<img(.+src=")outfile_files/([^"]+")', r'<img\1\2', output)
            output = re.sub(r'<img(.+)style="width:[\d\.]+mm"', r'<img\1', output)
          return output,img_res

        except Exception as inst:
          pass

        finally:
          try:
            if out_f:
                out_f.close()
            if infile:
                infile.close()
          except:
            pass
          if inst:
            raise inst

    def convert_from_html(self, input, format):
        """
        Convert input file from HTML
        """

        from tempfile import mkstemp,mkdtemp

        THE_OUTDIR = "outdir"
        THE_OUTFILE = "outfile"
        THE_INDIR = "indir"
        THE_INFILE = "infile"

        infile = None
        outfile = None
        out_f = None
        try:
          # create in/out files
          temp_dir = mkdtemp(prefix="cm_")

          # in
          indir_name = os.path.join(temp_dir, THE_INDIR)
          os.mkdir(indir_name)
          infile_name = os.path.join(indir_name, THE_INFILE + '.html')

          # out
          outdir_name = os.path.join(temp_dir, THE_OUTDIR)
          os.mkdir(outdir_name)
          outfile_name = os.path.join(outdir_name, THE_OUTFILE)

          # write infile
          infile = open(infile_name,'w')
          if type(input) == unicode:
            input = input.encode('utf8')
          infile.write(input)
          infile.close()

          # fix perms
          # TODO group permission should suffice
          os.chmod(temp_dir, 0755) # read
          os.chmod(indir_name, 0755) # read
          os.chmod(infile_name, 0755) # read
          os.chmod(outdir_name, 0777) # read / write

          # Do the job
          self.convert_file(infile_name, outfile_name, format)

          out_f = open(outfile_name,'r')
          output = out_f.read()
          return output

        finally:
          try:
            if out_f:
                out_f.close()
            if infile:
                infile.close()
            top = temp_dir
            for root, dirs, files in os.walk(top, topdown=False):
                for name in files:
                    os.remove(os.path.join(root, name))
                for name in dirs:
                    os.rmdir(os.path.join(root, name))
            os.rmdir(top)
          except:
            pass

    def add_html_header(self, body):
        """
        Add an HTML header to an HTML body
        """

        if '<html' in body and '<body' in body:
          full_html = body
        else:
          full_html = """

        <meta http-equiv="content-type" content="text/html; charset=utf-8" />

        %s

""" %body

        # Adds some style to fix Abiword default margins for paragraphs.
        from BeautifulSoup import BeautifulSoup
        import cssutils
        soup = BeautifulSoup(full_html)
        for p in soup.findAll(['p', 'div', 'ul', 'ol', 'dl']):
          try:
            css = p['style']
            s = cssutils.parseStyle(css)
            if s.getProperty('margin') == None:
              if s.getProperty('margin-top') == None:
                s.setProperty('margin-top', '10pt')
              if s.getProperty('margin-bottom') == None:
                s.setProperty('margin-bottom', '10pt')
            p['style'] = s.cssText

          except KeyError:
            p['style'] = 'margin-top: 10pt; margin-bottom: 10pt;';

        return do_tidy(unicode(soup))
<commit_msg>Fix abiword bug with pre+code tags.<commit_after>import os
import tempfile
import re

import pexpect

from abi_error import AbiConverterError, AbiCommandError
from cm.converters.pandoc_converters import do_tidy

TYPES_IN  = {'602': '602',       'abw': 'abw',       'aw': 'aw',
             'awt': 'awt',       'cwk': 'cwk',       'dbk': 'dbk',
             'doc': 'doc',       'docm': 'docm',     'docx': 'docx',
             'dot': 'dot',       'dotm': 'dotm',     'dotx': 'dotx',
             'fo': 'fo',         'htm': 'htm',       'html': 'html',
             'hwp': 'hwp',       'isc': 'isc',       'iscii': 'iscii',
             'kwd': 'kwd',       'mif': 'mif',       'odt': 'odt',
             'opml': 'opml',     'ott': 'ott',       'pdb': 'pdb',
             'pdf': 'pdf',       'rtf': 'rtf',       'sdw': 'sdw',
             'stw': 'stw',       'sxw': 'sxw',       'text': 'text',
             'txt': 'txt',       'wml': 'wml',       'wp': 'wp',
             'wpd': 'wpd',       'wri': 'wri',       'xhtml': 'xhtml',
             'xml': 'xml',       'zabw': 'zabw'}

TYPES_OUT = {'abw': 'abw',       'aw': 'aw',         'awt': 'awt',
             'dbk': 'dbk',       'doc': 'doc',       'docx': 'docx',
             'eml': 'eml',       'fo': 'fo',         'html': 'html',
             'isc': 'isc',       'iscii': 'iscii',   'kwd': 'kwd',
             'latex': 'latex',   'mht': 'mht',       'mif': 'mif',
             'nroff': 'nroff',   'nws': 'nws',       'odt': 'odt',
             'pdb': 'pdb',       'pdf': 'pdf',       'ps': 'ps',
             'rtf': 'rtf',       'sxw': 'sxw',       'text': 'text',
             'txt': 'txt',       'wml': 'wml',       'xml': 'xml',
             'xml2ps': 'xml2ps', 'zabw': 'zabw'}

class AbiFileConverter(object):
    """This let's you convert between all filetypes supperted by the
    AbiWord program. Import type isn't checked, as AbiWord doesn't check
    on extension, but on metadata.
    """

    def __init__(self, timeout=60):
        self.id = None
        self.timeout = timeout
        self._start_abiword()

    def _start_abiword(self):
        """
        Start abiword with the AbiCommand plugin, if not already started
        """

        # find the abiword executable
        abicommand = None
        for dir in os.environ['PATH'].split(':'):
            if os.path.isfile(os.path.join(dir, 'abiword')):
                abicommand = os.path.join(dir, 'abiword')
        if not abicommand:
            raise AbiConverterError('Can not find abiword executable')

        # start the abiword executable
        try:
            self.child = pexpect.spawn(abicommand + ' --plugin AbiCommand')
            self.child.expect(
                    'AbiWord command line plugin: Type "quit" to exit', 10)
        except:
            raise AbiConverterError('Can not open abiword executable')

    def stop_abiword(self):
        """
        Stop the running abiword, kill it if necessary
        """
        self.child.sendline('quit')
        if self._is_running():
            os.kill(self.child.pid, 9)

    def _is_running(self):
        """
        Test to see if abiword is running
        """
        try:
            self.child.sendline('writepid /dev/null')
            self.child.expect('OK', 1)
            return True
        except:
            return False

    def convert_file(self, in_file, out_file=None, type=None):
        """
        Convert a file. If out_file is not specified, a byte string is
        returned. If type is not specified, the file extension from out_file is
        used to determine the type. If this fails, the type 'text' is used.
        Return value is -1 if an error occurred.
        """
        # is the out_file specified?
        return_bytes = False
        if out_file is None:
            out_file = tempfile.mktemp(prefix="abiconvert_")
            return_bytes = True

        # is the type specified
        type = TYPES_OUT.get(
            type or os.path.splitext(out_file)[1][1:], 'txt')

        # do the conversion
        try:
          self._perform_conversion(in_file, out_file, type)
        except:
          raise

        # return a byte string if no out_file is specified
        if return_bytes:
            fp = open(out_file,  'r')
            bytes = fp.read()
            fp.close()
            os.remove(out_file)
            return bytes

    def _perform_conversion(self, in_file, out_file, type):
        """
        Do the actual conversion
        """
        # make sure we are up and running
        if not self._is_running:
            self._start_abiword()

        # convert the file
        cmd = 'convert %s %s %s' % (os.path.abspath(in_file),
                                    os.path.abspath(out_file), type)
        self.child.sendline(cmd)

        # Check for errors
        i = 1

        try:
          i = self.child.expect(['OK', pexpect.TIMEOUT])
          if i != 0:
            raise AbiCommandError('Error performing AbiCommand: %s' %cmd)
        except:
          raise

    def convert_to_html(self, input):
        """
        Convert input file to HTML
        """

        from tempfile import mkstemp,mkdtemp

        THE_OUTDIR = "outdir"
        THE_OUTFILE = "outfile"
        THE_INDIR = "indir"
        THE_INFILE = "infile"

        infile = None
        outfile = None
        out_f = None
        try:
          # create in/out files
          temp_dir = mkdtemp(prefix="cm_")

          # in
          indir_name = os.path.join(temp_dir, THE_INDIR)
          os.mkdir(indir_name)
          infile_name = os.path.join(indir_name, THE_INFILE)

          # out
          outdir_name = os.path.join(temp_dir, THE_OUTDIR)
          os.mkdir(outdir_name)
          outfile_name = os.path.join(outdir_name, THE_OUTFILE)

          # write infile
          infile = open(infile_name,'w')
          if type(input) == unicode:
            input = input.encode('utf8')
          infile.write(input)
          infile.close()

          # fix perms
          # TODO group permission should suffice
          os.chmod(temp_dir, 0755) # read
          os.chmod(indir_name, 0755) # read
          os.chmod(infile_name, 0755) # read
          os.chmod(outdir_name, 0777) # read / write

          # Do the job
          try:
            self.convert_file(infile_name, outfile_name, 'html')
          except:
            raise

          out_f = open(outfile_name,'r')
          output = out_f.read()

          # load other files (useful only for html)
          img_res = []
          if os.path.isdir(outdir_name + '/' + THE_OUTFILE + '_files'):
            image_names = [name for name in os.listdir(outdir_name + '/' + THE_OUTFILE + '_files') if name != THE_OUTFILE]
            for image_name in image_names:
              img_res.append(os.path.join(outdir_name + '/' + THE_OUTFILE + '_files', image_name))

            # clean images paths
            output = re.sub(r'<img(.+src=")outfile_files/([^"]+")', r'<img\1\2', output)
            output = re.sub(r'<img(.+)style="width:[\d\.]+mm"', r'<img\1', output)
          return output,img_res

        except Exception as inst:
          pass

        finally:
          try:
            if out_f:
                out_f.close()
            if infile:
                infile.close()
          except:
            pass
          if inst:
            raise inst

    def convert_from_html(self, input, format):
        """
        Convert input file from HTML
        """

        from tempfile import mkstemp,mkdtemp

        THE_OUTDIR = "outdir"
        THE_OUTFILE = "outfile"
        THE_INDIR = "indir"
        THE_INFILE = "infile"

        infile = None
        outfile = None
        out_f = None
        try:
          # create in/out files
          temp_dir = mkdtemp(prefix="cm_")

          # in
          indir_name = os.path.join(temp_dir, THE_INDIR)
          os.mkdir(indir_name)
          infile_name = os.path.join(indir_name, THE_INFILE + '.html')

          # out
          outdir_name = os.path.join(temp_dir, THE_OUTDIR)
          os.mkdir(outdir_name)
          outfile_name = os.path.join(outdir_name, THE_OUTFILE)

          # write infile
          infile = open(infile_name,'w')
          if type(input) == unicode:
            input = input.encode('utf8')

          # fix abiword bug with pre code tags
          input = re.sub(r'
\s+', r'
', input)
          input = re.sub(r'\s+
', r'
', input)

          infile.write(input)
          infile.close()

          # fix perms
          # TODO group permission should suffice
          os.chmod(temp_dir, 0755) # read
          os.chmod(indir_name, 0755) # read
          os.chmod(infile_name, 0755) # read
          os.chmod(outdir_name, 0777) # read / write

          # Do the job
          self.convert_file(infile_name, outfile_name, format)

          out_f = open(outfile_name,'r')
          output = out_f.read()
          return output

        finally:
          try:
            if out_f:
                out_f.close()
            if infile:
                infile.close()
            top = temp_dir
            for root, dirs, files in os.walk(top, topdown=False):
                for name in files:
                    os.remove(os.path.join(root, name))
                for name in dirs:
                    os.rmdir(os.path.join(root, name))
            os.rmdir(top)
          except:
            pass

    def add_html_header(self, body):
        """
        Add an HTML header to an HTML body
        """

        if '<html' in body and '<body' in body:
          full_html = body
        else:
          full_html = """

        <meta http-equiv="content-type" content="text/html; charset=utf-8" />

        %s

""" %body

        # Adds some style to fix Abiword default margins for paragraphs.
        from BeautifulSoup import BeautifulSoup
        import cssutils
        soup = BeautifulSoup(full_html)
        for p in soup.findAll(['p', 'div', 'ul', 'ol', 'dl']):
          try:
            css = p['style']
            s = cssutils.parseStyle(css)
            if s.getProperty('margin') == None:
              if s.getProperty('margin-top') == None:
                s.setProperty('margin-top', '10pt')
              if s.getProperty('margin-bottom') == None:
                s.setProperty('margin-bottom', '10pt')
            p['style'] = s.cssText

          except KeyError:
            p['style'] = 'margin-top: 10pt; margin-bottom: 10pt;';

        return do_tidy(unicode(soup))
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
.. module:: parts
   :synopsis: The part object.

.. moduleauthor:: Hazen Babcock
"""

import numpy

import openldraw.lcad_lib.colorsParser as colorsParser

# Load colors and create dictionaries.
all_colors = colorsParser.loadColors()

lcad_name_dict = {}
for color_group in all_colors:
    for color in color_group:
        lcad_name_dict[color.name.lower()] = color

def formatNumber(a_number, precision):
    f_string = "{0:." + str(precision) + "f}"
    return f_string.format(a_number)

def toColor(color):
    if isinstance(color, int):
        return str(color)
    else:
        return lcad_name_dict[color.lower()].code

# LDraw primitives.
class LDraw(object):

    def __init__(self, matrix, coords, color):
        self.coords = coords
        self.color = toColor(color)
        self.step = 0

        # Transform coordinates using current transformation matrix.
        for i in range(len(coords)/3):
            vec = numpy.array([self.coords[3*i],
                               self.coords[3*i+1],
                               self.coords[3*i+2],
                               1.0])
            loc = numpy.dot(matrix, vec)
            self.coords[3*i] = loc[0]
            self.coords[3*i+1] = loc[1]
            self.coords[3*i+2] = loc[2]

    def toLDraw(self):
        ld_str = self.prefix + self.color + " "
        ld_str += " ".join(map(lambda(x): formatNumber(x, 2), self.coords))
        return ld_str

class Line(LDraw):

    def __init__(self, matrix, coords, color):
        LDraw.__init__(self, matrix, coords, color)
        self.prefix = "2 "

class OptionalLine(LDraw):

    def __init__(self, matrix, coords, color):
        LDraw.__init__(self, matrix, coords, color)
        self.prefix = "5 "

class Quadrilateral(LDraw):

    def __init__(self, matrix, coords, color):
        LDraw.__init__(self, matrix, coords, color)
        self.prefix = "4 "

class Triangle(LDraw):

    def __init__(self, matrix, coords, color):
        LDraw.__init__(self, matrix, coords, color)
        self.prefix = "3 "

# LDraw parts.
class Part(object):

    def __init__(self, matrix, part_id, part_color, step):

        self.matrix = matrix.copy()
        self.part_color = toColor(part_color)
        self.part_id = part_id
        self.step = step

        #self.loc = numpy.array([0.0, 0.0, 0.0, 1.0])
        #self.loc = numpy.dot(self.model_matrix, self.loc)

    def toLDraw(self):
        """
        Return a string in ldraw format.

        :returns: str.
        """
        ld_str = "1 "

        # color
        ld_str += self.part_color + " "

        # xyz
        ld_str += formatNumber(self.matrix[0,3], 2) + " "
        ld_str += formatNumber(self.matrix[1,3], 2) + " "
        ld_str += formatNumber(self.matrix[2,3], 2) + " "

        # abcdefghi
        for i in range(3):
            for j in range(3):
                ld_str += formatNumber(self.matrix[i,j], 3) + " "

        # part
        if not (".dat" in self.part_id) and not (".ldr" in self.part_id):
            ld_str += self.part_id + ".dat"
        else:
            ld_str += self.part_id

        return ld_str

#
# The MIT License
#
# Copyright (c) 2014 Hazen Babcock
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
<commit_msg>Remove trailing zeros and '-0' from output.<commit_after>#!/usr/bin/env python
"""
.. module:: parts
   :synopsis: The part object.

.. moduleauthor:: Hazen Babcock
"""

import numpy

import openldraw.lcad_lib.colorsParser as colorsParser

# Load colors and create dictionaries.
all_colors = colorsParser.loadColors()

lcad_name_dict = {}
for color_group in all_colors:
    for color in color_group:
        lcad_name_dict[color.name.lower()] = color

def formatNumber(a_number, precision):
    f_string = "{0:." + str(precision) + "f}"
    s = f_string.format(a_number)

    # Remove trailing zeros.
    s = s.rstrip('0').rstrip('.') if '.' in s else s

    # Remove leading minus (if zero).
    if (s == "-0"):
        s = "0"
    return s

def toColor(color):
    if isinstance(color, int):
        return str(color)
    else:
        return lcad_name_dict[color.lower()].code

# LDraw primitives.
class LDraw(object):

    def __init__(self, matrix, coords, color):
        self.coords = coords
        self.color = toColor(color)
        self.step = 0

        # Transform coordinates using current transformation matrix.
        for i in range(len(coords)/3):
            vec = numpy.array([self.coords[3*i],
                               self.coords[3*i+1],
                               self.coords[3*i+2],
                               1.0])
            loc = numpy.dot(matrix, vec)
            self.coords[3*i] = loc[0]
            self.coords[3*i+1] = loc[1]
            self.coords[3*i+2] = loc[2]

    def toLDraw(self):
        ld_str = self.prefix + self.color + " "
        ld_str += " ".join(map(lambda(x): formatNumber(x, 2), self.coords))
        return ld_str

class Line(LDraw):

    def __init__(self, matrix, coords, color):
        LDraw.__init__(self, matrix, coords, color)
        self.prefix = "2 "

class OptionalLine(LDraw):

    def __init__(self, matrix, coords, color):
        LDraw.__init__(self, matrix, coords, color)
        self.prefix = "5 "

class Quadrilateral(LDraw):

    def __init__(self, matrix, coords, color):
        LDraw.__init__(self, matrix, coords, color)
        self.prefix = "4 "

class Triangle(LDraw):

    def __init__(self, matrix, coords, color):
        LDraw.__init__(self, matrix, coords, color)
        self.prefix = "3 "

# LDraw parts.
class Part(object):

    def __init__(self, matrix, part_id, part_color, step):

        self.matrix = matrix.copy()
        self.part_color = toColor(part_color)
        self.part_id = part_id
        self.step = step

        #self.loc = numpy.array([0.0, 0.0, 0.0, 1.0])
        #self.loc = numpy.dot(self.model_matrix, self.loc)

    def toLDraw(self):
        """
        Return a string in ldraw format.

        :returns: str.
        """
        ld_str = "1 "

        # color
        ld_str += self.part_color + " "

        # xyz
        ld_str += formatNumber(self.matrix[0,3], 2) + " "
        ld_str += formatNumber(self.matrix[1,3], 2) + " "
        ld_str += formatNumber(self.matrix[2,3], 2) + " "

        # abcdefghi
        for i in range(3):
            for j in range(3):
                ld_str += formatNumber(self.matrix[i,j], 3) + " "

        # part
        if not (".dat" in self.part_id) and not (".ldr" in self.part_id):
            ld_str += self.part_id + ".dat"
        else:
            ld_str += self.part_id

        return ld_str

#
# The MIT License
#
# Copyright (c) 2014 Hazen Babcock
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
<|endoftext|>


<commit_before>"""
Filename:     remove_drift.py
Author:       Damien Irving, irving.damien@gmail.com
Description:  Remove drift from a data series

"""

# Import general Python modules

import sys, os, pdb
import argparse
import numpy
import iris
import cf_units
from iris.experimental.equalise_cubes import equalise_attributes

# Import my modules

cwd = os.getcwd()
repo_dir = '/'
for directory in cwd.split('/')[1:]:
    repo_dir = os.path.join(repo_dir, directory)
    if directory == 'climate-analysis':
        break

modules_dir = os.path.join(repo_dir, 'modules')
sys.path.append(modules_dir)

try:
    import general_io as gio
except ImportError:
    raise ImportError('Must run this script from anywhere within the climate-analysis git repo')

# Define functions

def apply_polynomial(x_data, coefficient_data):
    """Evaluate cubic polynomial.

    Args:
      x_data (numpy.ndarray): One dimensional x-axis data
      coefficient_data (numpy.ndarray): Multi-dimensional coefficient array (e.g. lat, lon, depth)

    """

    coefficient_dict = {}
    if coefficient_data.ndim == 1:
        coefficient_dict['a'] = coefficient_data[0]
        coefficient_dict['b'] = coefficient_data[1]
        coefficient_dict['c'] = coefficient_data[2]
        coefficient_dict['d'] = coefficient_data[3]
    else:
        while x_data.ndim < coefficient_data.ndim:
            x_data = x_data[..., numpy.newaxis]
        for index, coefficient in enumerate(['a', 'b', 'c', 'd']):
            coef = coefficient_data[index, ...]
            coef = numpy.repeat(coef[numpy.newaxis, ...], x_data.shape[0], axis=0)
            assert x_data.ndim == coef.ndim
            coefficient_dict[coefficient] = coef

    result = coefficient_dict['a'] + coefficient_dict['b'] * x_data + coefficient_dict['c'] * x_data**2 + coefficient_dict['d'] * x_data**3

    return result

def check_attributes(data_attrs, control_attrs):
    """Make sure the correct control run has been used."""

    assert data_attrs['parent_experiment_id'] in [control_attrs['experiment_id'], 'N/A']

    control_rip = 'r%si%sp%s' %(control_attrs['realization'],
                                control_attrs['initialization_method'],
                                control_attrs['realization'])
    assert data_attrs['parent_experiment_rip'] in [control_rip, 'N/A']

def time_adjustment(first_data_cube, coefficient_cube):
    """Determine the adjustment that needs to be made to time axis."""

    branch_time_value = first_data_cube.attributes['branch_time'] #FIXME: Add half a time step?
    branch_time_unit = coefficient_cube.attributes['time_unit']
    branch_time_calendar = coefficient_cube.attributes['time_calendar']
    data_time_coord = first_data_cube.coord('time')

    new_unit = cf_units.Unit(branch_time_unit, calendar=branch_time_calendar)
    data_time_coord.convert_units(new_unit)

    first_experiment_time = data_time_coord.points[0]
    time_diff = first_experiment_time - branch_time_value

    return time_diff, first_experiment_time, new_unit

def main(inargs):
    """Run the program."""

    first_data_cube = iris.load_cube(inargs.data_files[0])
    coefficient_cube = iris.load_cube(inargs.coefficient_file)

    time_diff, first_experiment_time, new_time_unit = time_adjustment(first_data_cube, coefficient_cube)
    del first_data_cube

    new_cubelist = []
    for filename in inargs.data_files:

        data_cube = iris.load_cube(filename)
        check_attributes(data_cube.attributes, coefficient_cube.attributes)

        # Sync the data time axis with the coefficient time axis
        time_coord = data_cube.coord('time')
        time_coord.convert_units(new_time_unit)

        assert time_coord.points[0] >= first_experiment_time
        time_values = time_coord.points - time_diff

        # Remove the drift
        drift_signal = apply_polynomial(time_values, coefficient_cube.data)
        new_cube = data_cube - drift_signal
        new_cube.metadata = data_cube.metadata

        if inargs.outfile[-3:] == '.nc':
            new_cubelist.append(new_cube)
        elif inargs.outfile[-1] == '/':
            infile = filename.split('/')[-1]
            outfile = inargs.outfile + infile
            metadata_dict = {infile: data_cube.attributes['history'],
                             inargs.coefficient_file: coefficient_cube.attributes['history']}
            new_cube.attributes['history'] = gio.write_metadata(file_info=metadata_dict)

            iris.save(new_cube, outfile)
            print 'output:', outfile

    if inargs.outfile[-3:] == '.nc':
        new_cubelist = iris.cube.CubeList(new_cubelist)
        equalise_attributes(new_cubelist)
        new_cubelist = new_cubelist.concatenate_cube()

        metadata_dict = {inargs.data_files[0]: data_cubelist[0].attributes['history'],
                        inargs.coefficient_file: coefficient_cube.attributes['history']}
        new_cubelist.attributes['history'] = gio.write_metadata(file_info=metadata_dict)

        iris.save(new_cubelist, inargs.outfile)

if __name__ == '__main__':

    extra_info ="""
example:

author:
    Damien Irving, irving.damien@gmail.com
notes:
    Generate the polynomial coefficients first from calc_drift_coefficients.py

"""

    description='Remove drift from a data series'
    parser = argparse.ArgumentParser(description=description,
                                     epilog=extra_info,
                                     argument_default=argparse.SUPPRESS,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument("data_files", type=str, nargs='*', help="Input data files, in chronological order (needs to include whole experiment to get time axis correct)")
    parser.add_argument("coefficient_file", type=str, help="Input coefficient file")
    parser.add_argument("outfile", type=str, help="Give a path instead of a file name if you want an output file corresponding to each input file")

    args = parser.parse_args()

    main(args)
<commit_msg>added assertion for incorrect outfile information<commit_after>"""
Filename:     remove_drift.py
Author:       Damien Irving, irving.damien@gmail.com
Description:  Remove drift from a data series

"""

# Import general Python modules

import sys, os, pdb
import argparse
import numpy
import iris
import cf_units
from iris.experimental.equalise_cubes import equalise_attributes

# Import my modules

cwd = os.getcwd()
repo_dir = '/'
for directory in cwd.split('/')[1:]:
    repo_dir = os.path.join(repo_dir, directory)
    if directory == 'climate-analysis':
        break

modules_dir = os.path.join(repo_dir, 'modules')
sys.path.append(modules_dir)

try:
    import general_io as gio
except ImportError:
    raise ImportError('Must run this script from anywhere within the climate-analysis git repo')

# Define functions

def apply_polynomial(x_data, coefficient_data):
    """Evaluate cubic polynomial.

    Args:
      x_data (numpy.ndarray): One dimensional x-axis data
      coefficient_data (numpy.ndarray): Multi-dimensional coefficient array (e.g. lat, lon, depth)

    """

    coefficient_dict = {}
    if coefficient_data.ndim == 1:
        coefficient_dict['a'] = coefficient_data[0]
        coefficient_dict['b'] = coefficient_data[1]
        coefficient_dict['c'] = coefficient_data[2]
        coefficient_dict['d'] = coefficient_data[3]
    else:
        while x_data.ndim < coefficient_data.ndim:
            x_data = x_data[..., numpy.newaxis]
        for index, coefficient in enumerate(['a', 'b', 'c', 'd']):
            coef = coefficient_data[index, ...]
            coef = numpy.repeat(coef[numpy.newaxis, ...], x_data.shape[0], axis=0)
            assert x_data.ndim == coef.ndim
            coefficient_dict[coefficient] = coef

    result = coefficient_dict['a'] + coefficient_dict['b'] * x_data + coefficient_dict['c'] * x_data**2 + coefficient_dict['d'] * x_data**3

    return result

def check_attributes(data_attrs, control_attrs):
    """Make sure the correct control run has been used."""

    assert data_attrs['parent_experiment_id'] in [control_attrs['experiment_id'], 'N/A']

    control_rip = 'r%si%sp%s' %(control_attrs['realization'],
                                control_attrs['initialization_method'],
                                control_attrs['realization'])
    assert data_attrs['parent_experiment_rip'] in [control_rip, 'N/A']

def time_adjustment(first_data_cube, coefficient_cube):
    """Determine the adjustment that needs to be made to time axis."""

    branch_time_value = first_data_cube.attributes['branch_time'] #FIXME: Add half a time step?
    branch_time_unit = coefficient_cube.attributes['time_unit']
    branch_time_calendar = coefficient_cube.attributes['time_calendar']
    data_time_coord = first_data_cube.coord('time')

    new_unit = cf_units.Unit(branch_time_unit, calendar=branch_time_calendar)
    data_time_coord.convert_units(new_unit)

    first_experiment_time = data_time_coord.points[0]
    time_diff = first_experiment_time - branch_time_value

    return time_diff, first_experiment_time, new_unit

def main(inargs):
    """Run the program."""

    first_data_cube = iris.load_cube(inargs.data_files[0])
    coefficient_cube = iris.load_cube(inargs.coefficient_file)

    time_diff, first_experiment_time, new_time_unit = time_adjustment(first_data_cube, coefficient_cube)
    del first_data_cube

    new_cubelist = []
    for filename in inargs.data_files:

        data_cube = iris.load_cube(filename)
        check_attributes(data_cube.attributes, coefficient_cube.attributes)

        # Sync the data time axis with the coefficient time axis
        time_coord = data_cube.coord('time')
        time_coord.convert_units(new_time_unit)

        assert time_coord.points[0] >= first_experiment_time
        time_values = time_coord.points - time_diff

        # Remove the drift
        drift_signal = apply_polynomial(time_values, coefficient_cube.data)
        new_cube = data_cube - drift_signal
        new_cube.metadata = data_cube.metadata

        assert (inargs.outfile[-3:] == '.nc') or (inargs.outfile[-1] == '/')

        if inargs.outfile[-3:] == '.nc':
            new_cubelist.append(new_cube)
        elif inargs.outfile[-1] == '/':
            infile = filename.split('/')[-1]
            outfile = inargs.outfile + infile
            metadata_dict = {infile: data_cube.attributes['history'],
                             inargs.coefficient_file: coefficient_cube.attributes['history']}
            new_cube.attributes['history'] = gio.write_metadata(file_info=metadata_dict)

            iris.save(new_cube, outfile)
            print 'output:', outfile

    if inargs.outfile[-3:] == '.nc':
        new_cubelist = iris.cube.CubeList(new_cubelist)
        equalise_attributes(new_cubelist)
        new_cubelist = new_cubelist.concatenate_cube()

        metadata_dict = {inargs.data_files[0]: data_cubelist[0].attributes['history'],
                        inargs.coefficient_file: coefficient_cube.attributes['history']}
        new_cubelist.attributes['history'] = gio.write_metadata(file_info=metadata_dict)

        iris.save(new_cubelist, inargs.outfile)

if __name__ == '__main__':

    extra_info ="""
example:

author:
    Damien Irving, irving.damien@gmail.com
notes:
    Generate the polynomial coefficients first from calc_drift_coefficients.py

"""

    description='Remove drift from a data series'
    parser = argparse.ArgumentParser(description=description,
                                     epilog=extra_info,
                                     argument_default=argparse.SUPPRESS,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument("data_files", type=str, nargs='*', help="Input data files, in chronological order (needs to include whole experiment to get time axis correct)")
    parser.add_argument("coefficient_file", type=str, help="Input coefficient file")
    parser.add_argument("outfile", type=str, help="Give a path instead of a file name if you want an output file corresponding to each input file")

    args = parser.parse_args()

    main(args)
<|endoftext|>


<commit_before>"""
Helpers
=======

Useful functions and classes that don't really fit elsewhere.
"""

def is_update_modifier(doc):
    """Return true if and only if `doc` is an update modifier.

    This function only checks the first key it finds.  `doc` is assumed to
    be either a valid update modifier or a replacement document.  The result of
    passing a bad modifier or document is undefined.
    """
    if not doc:
        return False

    # To save time just check the first key.
    key = doc.iterkeys().next()
    return key.startswith('$')

def value_or_result(potential):
    """Return `potential` or it's return value.

    If `potential` is callable then the result of calling it without arguments
    is returned instead.

    """
    if callable(potential):
        return potential()
    return potential

class ConversionDict(dict):
    """A :class:`dict` subclass that wraps contents on lookup.

    >>> cd = ConversionDict({'x': 5, 'y': 6}, {'x': str})
    >>> cd['x']
    '5'
    >>> cd.x
    '5'
    >>> cd['y']
    6
    >>> cd.y
    6

    :param content: is the dictionary content.
    :param conversions: is a dictionary containing conversions to be applied on
        lookup.  If no conversion is present in `conversions` the value is
        returned unchanged.
    """

    NONKEY_ATTRS = set(['_conversions'])
    """Attributes which should not be treated as keys in the dictionary.

    Classes extending :class:`ConversionDict` should add the names of any
    attributes they wish to keep that should not be set into the underlying
    :class:`dict`.
    """

    def __init__(self, content, conversions):
        dict.__init__(self, content)
        self._conversions = conversions

    def __getitem__(self, key):
        value = dict.__getitem__(self, key)
        conversion = self.__get_conversion(key)
        return self.__convert_value(value, conversion)

    @staticmethod
    def __convert_value(value, conversion):
        """Convert `value` using `conversion`.

        If `conversion` is ``None`` then `value` is returned unchanged.
        """
        if isinstance(value, dict) and not isinstance(value, ConversionDict):
            return ConversionDict(value, conversion)

        if conversion is None:
            return value

        if isinstance(conversion, dict):
            return ConversionDict(value, conversion)

        if isinstance(value, list):
            return [conversion(x) for x in value]

        return conversion(value)

    def __get_conversion(self, key):
        if self._conversions:
            return self._conversions.get(key)

    def __getattr__(self, name):
        """Get the `name` attribute.

        If `name` is a :class:`dict` then it will be wrapped in a new
        :class:`ConversionDict`.  This allows chained dot notation to reference
        items in embedded dictionaries.
        """
        if name in self:
            return self.__getitem__(name)

        raise AttributeError('{0.__name__!r} has no attribute {1!r}.'.format(
            type(self), name))

    def __setattr__(self, name, value):
        """Set an attribute.

        Put an item in this dictionary unless `name` is a contained in
        :data:`NONKEY_ATTRS`.  Or some other attribute already set on either
        the instance or the class (which allows the tests to dingus out methods
        on instances).
        """
        if name in self.NONKEY_ATTRS or name in dir(self):
            object.__setattr__(self, name, value)
        else:
            self.__setitem__(name, value)

    def iteritems(self):
        """Iterate the items in this :class:`ConversionDict`.

        The underlying :meth:`dict.iteritems` must be wrapped such that the
        values returned by this generator are converted to appropriate types.
        """
        for key, value in dict.iteritems(self):
            conversion = self.__get_conversion(key)
            yield key, self.__convert_value(value, conversion)

    def itervalues(self):
        """Iterate the values in this :class:`ConversionDict`.

        The values are converted to the appropriate type before returning.
        """
        for key, value in self.iteritems():
            yield value

    def items(self):
        """Return a list of the items in this :class:`ConversionDict`.
        """
        return [item for item in self.iteritems()]

    def values(self):
        """Return a list of the values in this :class:`ConversionDict`.
        """
        return [value for value in self.itervalues()]
<commit_msg>helpers: Remove unnecessary conditional from ConversionDict<commit_after>"""
Helpers
=======

Useful functions and classes that don't really fit elsewhere.
"""

def is_update_modifier(doc):
    """Return true if and only if `doc` is an update modifier.

    This function only checks the first key it finds.  `doc` is assumed to
    be either a valid update modifier or a replacement document.  The result of
    passing a bad modifier or document is undefined.
    """
    if not doc:
        return False

    # To save time just check the first key.
    key = doc.iterkeys().next()
    return key.startswith('$')

def value_or_result(potential):
    """Return `potential` or it's return value.

    If `potential` is callable then the result of calling it without arguments
    is returned instead.

    """
    if callable(potential):
        return potential()
    return potential

class ConversionDict(dict):
    """A :class:`dict` subclass that wraps contents on lookup.

    >>> cd = ConversionDict({'x': 5, 'y': 6}, {'x': str})
    >>> cd['x']
    '5'
    >>> cd.x
    '5'
    >>> cd['y']
    6
    >>> cd.y
    6

    :param content: is the dictionary content.
    :param conversions: is a dictionary containing conversions to be applied on
        lookup.  If no conversion is present in `conversions` the value is
        returned unchanged.
    """

    NONKEY_ATTRS = set(['_conversions'])
    """Attributes which should not be treated as keys in the dictionary.

    Classes extending :class:`ConversionDict` should add the names of any
    attributes they wish to keep that should not be set into the underlying
    :class:`dict`.
    """

    def __init__(self, content, conversions):
        dict.__init__(self, content)
        self._conversions = conversions

    def __getitem__(self, key):
        value = dict.__getitem__(self, key)
        conversion = self.__get_conversion(key)
        return self.__convert_value(value, conversion)

    @staticmethod
    def __convert_value(value, conversion):
        """Convert `value` using `conversion`.

        If `conversion` is ``None`` then `value` is returned unchanged.
        """
        if conversion is None:
            return value

        if isinstance(conversion, dict):
            return ConversionDict(value, conversion)

        if isinstance(value, list):
            return [conversion(x) for x in value]

        return conversion(value)

    def __get_conversion(self, key):
        if self._conversions:
            return self._conversions.get(key)

    def __getattr__(self, name):
        """Get the `name` attribute.

        If `name` is a :class:`dict` then it will be wrapped in a new
        :class:`ConversionDict`.  This allows chained dot notation to reference
        items in embedded dictionaries.
        """
        if name in self:
            return self.__getitem__(name)

        raise AttributeError('{0.__name__!r} has no attribute {1!r}.'.format(
            type(self), name))

    def __setattr__(self, name, value):
        """Set an attribute.

        Put an item in this dictionary unless `name` is a contained in
        :data:`NONKEY_ATTRS`.  Or some other attribute already set on either
        the instance or the class (which allows the tests to dingus out methods
        on instances).
        """
        if name in self.NONKEY_ATTRS or name in dir(self):
            object.__setattr__(self, name, value)
        else:
            self.__setitem__(name, value)

    def iteritems(self):
        """Iterate the items in this :class:`ConversionDict`.

        The underlying :meth:`dict.iteritems` must be wrapped such that the
        values returned by this generator are converted to appropriate types.
        """
        for key, value in dict.iteritems(self):
            conversion = self.__get_conversion(key)
            yield key, self.__convert_value(value, conversion)

    def itervalues(self):
        """Iterate the values in this :class:`ConversionDict`.

        The values are converted to the appropriate type before returning.
        """
        for key, value in self.iteritems():
            yield value

    def items(self):
        """Return a list of the items in this :class:`ConversionDict`.
        """
        return [item for item in self.iteritems()]

    def values(self):
        """Return a list of the values in this :class:`ConversionDict`.
        """
        return [value for value in self.itervalues()]
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# file: ntpclient.py
# vim:fileencoding=utf-8:fdm=marker:ft=python
#
# Author: R.F. Smith <rsmith@xs4all.nl>
# Created: 2017-11-16 19:33:50 +0100
# Last modified: 2018-09-09T14:10:52+0200
"""Simple NTP query program."""

from contextlib import closing
from datetime import datetime
from socket import socket, AF_INET, SOCK_DGRAM
import os
import struct
import time

# See e.g. # https://www.cisco.com/c/en/us/about/press/internet-protocol-journal/back-issues/table-contents-58/154-ntp.html
# From left to right:
# * No leap second adjustment = 0 (2 bits)
# * protocol version 3 (3 bits)
# * client packet = 3 (3 bits)
# In [1]: hex((0 & 0b11) << 6 | (3 & 0b111) << 3 | (3 & 0b111))
# Out[1]: '0x1b'
_query = b'\x1b' + 47 * b'\0'

def get_ntp_time(host="pool.ntp.org", port=123):
    fmt = "!12I"
    with closing(socket(AF_INET, SOCK_DGRAM)) as s:
        s.sendto(_query, (host, port))
        msg, address = s.recvfrom(1024)
    unpacked = struct.unpack(fmt, msg[0:struct.calcsize(fmt)])
    # Return the average of receive and transmit timestamps.
    # Note that 2208988800 is the difference in seconds between the
    # UNIX epoch 1970-1-1 and the NTP epoch 1900-1-1.
    # See: (datetime.datetime(1970,1,1) - datetime.datetim:::::e(1900,1,1)).total_seconds()
    t2 = unpacked[8] + float(unpacked[9]) / 2**32 - 2208988800
    t3 = unpacked[10] + float(unpacked[11]) / 2**32 - 2208988800
    return (t2 + t3) / 2

if __name__ == "__main__":
    res = None
    t1 = time.clock_gettime(time.CLOCK_REALTIME)
    ntptime = get_ntp_time('nl.pool.ntp.org')
    t4 = time.clock_gettime(time.CLOCK_REALTIME)
    # It is not guaranteed that the NTP time is *exactly* in the middle of both
    # local times. But it is a reasonable simplification.
    localtime = (t1 + t4) / 2
    if os.geteuid() == 0:
        time.clock_settime(time.CLOCK_REALTIME, ntptime)
        res = 'Time set to NTP time.'
    diff = localtime - ntptime
    localtime = datetime.fromtimestamp(localtime)
    ntptime = datetime.fromtimestamp(ntptime)
    print('Local time value:', localtime.strftime('%a %b %d %H:%M:%S.%f %Y'))
    print('NTP time value:', ntptime.strftime('%a %b %d %H:%M:%S.%f %Y'))
    print('Local time - ntp time: {:.6f} s'.format(diff))
    if res:
        print(res)
<commit_msg>Add -q. Set MIT license.<commit_after>#!/usr/bin/env python3
# file: ntpclient.py
# vim:fileencoding=utf-8:fdm=marker:ft=python
#
# Copyright © 2018 R.F. Smith <rsmith@xs4all.nl>.
# SPDX-License-Identifier: MIT
# Created: 2017-11-16 19:33:50 +0100
# Last modified: 2018-09-09T21:11:09+0200
"""
Simple NTP query program. This program does not strive for high accuracy.
Use this only as a client, not for a time server!
"""

from contextlib import closing
from datetime import datetime
from socket import socket, AF_INET, SOCK_DGRAM
import os
import struct
import sys
import time

# See e.g. # https://www.cisco.com/c/en/us/about/press/internet-protocol-journal/back-issues/table-contents-58/154-ntp.html
# From left to right:
# * No leap second adjustment = 0 (2 bits)
# * protocol version 3 (3 bits)
# * client packet = 3 (3 bits)
# In [1]: hex((0 & 0b11) << 6 | (3 & 0b111) << 3 | (3 & 0b111))
# Out[1]: '0x1b'
_query = b'\x1b' + 47 * b'\0'

def get_ntp_time(host="pool.ntp.org", port=123):
    fmt = "!12I"
    with closing(socket(AF_INET, SOCK_DGRAM)) as s:
        s.sendto(_query, (host, port))
        msg, address = s.recvfrom(1024)
    unpacked = struct.unpack(fmt, msg[0:struct.calcsize(fmt)])
    # Return the average of receive and transmit timestamps.
    # Note that 2208988800 is the difference in seconds between the
    # UNIX epoch 1970-1-1 and the NTP epoch 1900-1-1.
    # See: (datetime.datetime(1970,1,1) - datetime.datetime(1900,1,1)).total_seconds()
    t2 = unpacked[8] + float(unpacked[9]) / 2**32 - 2208988800
    t3 = unpacked[10] + float(unpacked[11]) / 2**32 - 2208988800
    return (t2 + t3) / 2

def main(argv):
    """
    Entry point for ntpclient.py.

    Arguments:
        argv: command line arguments
    """
    res = None
    quiet = False
    if '-q' in argv:
        quiet = True
    t1 = time.clock_gettime(time.CLOCK_REALTIME)
    ntptime = get_ntp_time('nl.pool.ntp.org')
    t4 = time.clock_gettime(time.CLOCK_REALTIME)
    # It is not guaranteed that the NTP time is *exactly* in the middle of both
    # local times. But it is a reasonable simplification.
    localtime = (t1 + t4) / 2
    if os.geteuid() == 0:
        time.clock_settime(time.CLOCK_REALTIME, ntptime)
        res = 'Time set to NTP time.'
    diff = localtime - ntptime
    localtime = datetime.fromtimestamp(localtime)
    ntptime = datetime.fromtimestamp(ntptime)
    if not quiet:
        print('Local time value:', localtime.strftime('%a %b %d %H:%M:%S.%f %Y'))
        print('NTP time value:', ntptime.strftime('%a %b %d %H:%M:%S.%f %Y'))
        print('Local time - ntp time: {:.6f} s'.format(diff))
        if res:
            print(res)

if __name__ == '__main__':
    main(sys.argv[1:])
<|endoftext|>


<commit_before>t = var.trees[0]
a = var.alignments[0]
t.data = Data()

t.model.dump()

print('\nAfter optimizing, the composition of the model for the non-root nodes is:')
print(t.model.parts[0].comps[0].val)
print('...and the composition of the root model is:')
print(t.model.parts[0].comps[1].val)
t.write()
func.reseedCRandomizer(os.getpid())

# The char "symbols", AAs in this case, are available as a.symbols; that is why
# I gave a name to var.alignments[0].  Also available as
# d.parts[partNum].symbols, so d.parts[0].symbols are also 'arndcqeghilkmfpstwyv'

counts = [0] * 20
for rep in range(100):
    ancSt = t.ancestralStateDraw()
    for i in range(20):
        ch = a.symbols[i] # 'arndcqeghilkmfpstwyv'
        cnt = ancSt.count(ch)
        counts[i] += cnt
        mySum = float(sum(counts))
print("\nsymbol optimized      draws")
for i in range(20):
    print("  %s      %.5f     %.4f" % (a.symbols[i], t.model.parts[0].comps[1].val[i], counts[i]/mySum))
<commit_msg>Added Zeldovich T_opt estimator<commit_after>t = var.trees[0]
a = var.alignments[0]
t.data = Data()

t.model.dump()

print('\nAfter optimizing, the composition of the model for the non-root nodes is:')
print(t.model.parts[0].comps[0].val)
print('...and the composition of the root model is:')
print(t.model.parts[0].comps[1].val)
t.write()
func.reseedCRandomizer(os.getpid())

# The char "symbols", AAs in this case, are available as a.symbols; that is why
# I gave a name to var.alignments[0].  Also available as
# d.parts[partNum].symbols, so d.parts[0].symbols are also 'arndcqeghilkmfpstwyv'

counts = [0] * 20
for rep in range(100):
    ancSt = t.ancestralStateDraw()
    for i in range(20):
        ch = a.symbols[i] # 'arndcqeghilkmfpstwyv'
        cnt = ancSt.count(ch)
        counts[i] += cnt
        mySum = float(sum(counts))
print("\nsymbol optimized      draws")
for i in range(20):
    print("  %s      %.5f     %.4f" % (a.symbols[i], t.model.parts[0].comps[1].val[i], counts[i]/mySum))

#calculate predicted OGT according to Zeldovich
f_ivywrel = 0
for i in range(20):
    if a.symbols[i] in 'ivywrel':
        f_ivywrel += t.model.parts[0].comps[1].val[i]
print("F(IVYWREL) = " + str(f_ivywrel))
print("T_opt estimate according to Zeldovich: " + str(937.0*float(f_ivywrel) - 335.0))
<|endoftext|>


<commit_before># coding: utf-8
import json
from flask import Flask, request
import sys
from musica import *
from collections import OrderedDict
import unicodedata

app = Flask(__name__)

ARTISTA_ID = 0
MUSICA_ID = 1
ARTISTA = 2
MUSICA = 3
GENERO = 4
POPULARIDADE = 5
TOM = 6
SEQ_FAMOSA = 7
CIFRA = 8

TAM_PAGINA = 100

def init():
    reload(sys)
    sys.setdefaultencoding('utf8')

    f = open('../data/top/dataset_final.csv')
    f.readline()

    musicas_dict = {}
    global generos
    global musicas
    global sequencias
    generos = set()
    sequencias = {'BmGDA' : 0,
            'CGAmF' : 1,
            'EmG' : 2,
            'CA7DmG7' : 3,
            'GmF' : 4,
            'CC7FFm' : 5}

    for line in f:
        line = line.replace('"', '').replace('NA', '')[:-1]

        musica = line.split(',')
        musica[POPULARIDADE] = int(musica[POPULARIDADE].replace('.', ''))
        musica[CIFRA] = list(musica[CIFRA].split(';')) if musica[CIFRA] != '' else list()
        musica[SEQ_FAMOSA] = musica[SEQ_FAMOSA].split(";")

        if musica[GENERO] != '' :
            generos.add(musica[GENERO])

        musica_obj = Musica(musica[ARTISTA_ID], musica[ARTISTA], musica[MUSICA_ID], musica[MUSICA],
                musica[GENERO], int(musica[POPULARIDADE]), musica[SEQ_FAMOSA], musica[TOM], musica[CIFRA])
        musicas_dict[musica_obj.id_unico_musica] = musica_obj

    # dictionary sorted by popularity
    musicas = OrderedDict(sorted(musicas_dict.items(), key=lambda x: x[1].popularidade, reverse = True))
    generos = list(generos)

    f.close()

def apply_filtro_generos(lista_musicas, lista_generos):
    match = []
    for m in lista_musicas:
        if m.genero in lista_generos:
            match.append(m)
    return match

@app.route('/search')
def busca():
    generos_tag = request.args.get('generos', [])
    generos_key = generos
    if generos_tag:
        generos_key = generos_tag.encode('utf-8').split(',')

    pagina_tag = request.args.get('pagina','1')

    key = request.args.get('key').lower()
    key = remover_combinantes(key)

    out = []
    for musica in musicas.values():
        text = '%s %s' % (musica.nome_artista.lower(), musica.nome_musica.lower())
        if key in remover_combinantes(unicode(text)) and musica.genero in generos_key:
            matches = {
                'id_unico_musica' : musica.id_unico_musica,
                'id_artista' : musica.id_artista,
                'id_musica' : musica.id_musica,
                'nome_artista' : musica.nome_artista,
                'nome_musica' : musica.nome_musica,
                'genero' : musica.genero,
                'url' : musica.url,
            }
            out.append(matches)
    return json.dumps(get_pagina(out, pagina_tag))

def remover_combinantes(string):
    string = unicodedata.normalize('NFD', string)
    return u''.join(ch for ch in string if unicodedata.category(ch) != 'Mn')

@app.route('/musica')
def get_musicas():
    return json.dumps([v.__dict__ for v in musicas.values()])

@app.route('/genero')
def get_generos():
    return json.dumps(list(generos))

@app.route('/musica/<m_id>/')
def get_musica(m_id):
    return json.dumps(musicas[m_id].__dict__)

@app.route('/similares')
def get_similares():
    # tratando request
    acordes_tag = request.args.get('acordes')
    id_musica_tag = request.args.get('id_unico_musica')
    sequencia_tag = request.args.get('sequencia')
    pagina_tag = request.args.get('pagina','1')

    # se não existir, filtra por todos.
    generos_tag = request.args.get('generos')
    generos_key = generos
    if generos_tag:
        generos_key = generos_tag.encode('utf-8').split(',')

    acordes = []
    if acordes_tag:
        acordes = acordes_tag.encode('utf-8').split(',')
    elif id_musica_tag:
        musica = musicas[id_musica_tag]
        acordes = musica.acordes
    elif sequencia_tag:
        acordes = sequencia_tag.encode('utf-8').replace(',','')
        similares = []

        if acordes in sequencias:
            id_seq = sequencias[acordes]
            similares = get_pagina(get_similares_por_sequencia(id_seq, generos_key), pagina_tag)
        return json.dumps(similares)

    similares = get_similares(acordes, generos_key)

    return json.dumps(get_pagina(similares, pagina_tag))

def get_similares_por_sequencia(id_seq, generos_key):
    similares = []
    for musica in musicas.values():
        if str(id_seq) in musica.seqs_famosas and musica.genero in generos_key:
            similar = {
                'id_unico_musica' : musica.id_unico_musica,
                'id_artista' : musica.id_artista,
                'id_musica' : musica.id_musica,
                'nome_artista' : musica.nome_artista,
                'nome_musica' : musica.nome_musica,
                'popularidade' : musica.popularidade,
                'acordes' : musica.acordes,
                'genero' : musica.genero,
                'url' : musica.url,
            }
            similares.append(similar)

    return similares

def get_pagina(colecao, pagina_tag):
    sl = (int(pagina_tag) - 1)*TAM_PAGINA
    return colecao[sl:sl+TAM_PAGINA]

def get_similares(acordes, generos_key):
    similares = []
    for musica in musicas.values():
        inter = set(acordes).intersection(set(musica.acordes))
        diff = set(musica.acordes) - set(acordes)

        # somente as que tiverem interseção e as que forem
        # dos generos solicitados.
        if len(inter) > 0 and musica.genero in generos_key:
            similar = {
                    'id_unico_musica' : musica.id_unico_musica,
                    'id_artista' : musica.id_artista,
                    'id_musica' : musica.id_musica,
                    'nome_artista' : musica.nome_artista,
                    'nome_musica' : musica.nome_musica,
                    'popularidade' : musica.popularidade,
                    'acordes' : musica.acordes,
                    'genero' : musica.genero,
                    'url' : musica.url,
                    'diferenca' : list(diff),
                    'intersecao' : list(inter)
            }

            similares.append(similar)

    # ordenados por menor diferença, maior interseção e maior popularidade.
    return sorted(similares, key=lambda x: (len(x['diferenca']), -len(x['intersecao'])))

@app.after_request
def add_header(response):
    response.headers['Access-Control-Allow-Origin'] = '*'
    return response

if __name__ == '__main__':
    init()
    app.run(debug=True)

<commit_msg>removendo código morto<commit_after># coding: utf-8
import json
from flask import Flask, request
import sys
from musica import *
from collections import OrderedDict
import unicodedata

app = Flask(__name__)

ARTISTA_ID = 0
MUSICA_ID = 1
ARTISTA = 2
MUSICA = 3
GENERO = 4
POPULARIDADE = 5
TOM = 6
SEQ_FAMOSA = 7
CIFRA = 8

TAM_PAGINA = 100

def init():
    reload(sys)
    sys.setdefaultencoding('utf8')

    f = open('../data/top/dataset_final.csv')
    f.readline()

    musicas_dict = {}
    global generos
    global musicas
    global sequencias
    generos = set()
    sequencias = {'BmGDA' : 0,
            'CGAmF' : 1,
            'EmG' : 2,
            'CA7DmG7' : 3,
            'GmF' : 4,
            'CC7FFm' : 5}

    for line in f:
        line = line.replace('"', '').replace('NA', '')[:-1]

        musica = line.split(',')
        musica[POPULARIDADE] = int(musica[POPULARIDADE].replace('.', ''))
        musica[CIFRA] = list(musica[CIFRA].split(';')) if musica[CIFRA] != '' else list()
        musica[SEQ_FAMOSA] = musica[SEQ_FAMOSA].split(";")

        if musica[GENERO] != '' :
            generos.add(musica[GENERO])

        musica_obj = Musica(musica[ARTISTA_ID], musica[ARTISTA], musica[MUSICA_ID], musica[MUSICA],
                musica[GENERO], int(musica[POPULARIDADE]), musica[SEQ_FAMOSA], musica[TOM], musica[CIFRA])
        musicas_dict[musica_obj.id_unico_musica] = musica_obj

    # dictionary sorted by popularity
    musicas = OrderedDict(sorted(musicas_dict.items(), key=lambda x: x[1].popularidade, reverse = True))
    generos = list(generos)

    f.close()

@app.route('/search')
def busca():
    generos_tag = request.args.get('generos', [])
    generos_key = generos
    if generos_tag:
        generos_key = generos_tag.encode('utf-8').split(',')

    pagina_tag = request.args.get('pagina','1')

    key = request.args.get('key').lower()
    key = remover_combinantes(key)

    out = []
    for musica in musicas.values():
        text = '%s %s' % (musica.nome_artista.lower(), musica.nome_musica.lower())
        if key in remover_combinantes(unicode(text)) and musica.genero in generos_key:
            matches = {
                'id_unico_musica' : musica.id_unico_musica,
                'id_artista' : musica.id_artista,
                'id_musica' : musica.id_musica,
                'nome_artista' : musica.nome_artista,
                'nome_musica' : musica.nome_musica,
                'genero' : musica.genero,
                'url' : musica.url,
            }
            out.append(matches)
    return json.dumps(get_pagina(out, pagina_tag))

def remover_combinantes(string):
    string = unicodedata.normalize('NFD', string)
    return u''.join(ch for ch in string if unicodedata.category(ch) != 'Mn')

@app.route('/musica')
def get_musicas():
    return json.dumps([v.__dict__ for v in musicas.values()])

@app.route('/genero')
def get_generos():
    return json.dumps(list(generos))

@app.route('/musica/<m_id>/')
def get_musica(m_id):
    return json.dumps(musicas[m_id].__dict__)

@app.route('/similares')
def get_similares():
    # tratando request
    acordes_tag = request.args.get('acordes')
    id_musica_tag = request.args.get('id_unico_musica')
    sequencia_tag = request.args.get('sequencia')
    pagina_tag = request.args.get('pagina','1')

    # se não existir, filtra por todos.
    generos_tag = request.args.get('generos')
    generos_key = generos
    if generos_tag:
        generos_key = generos_tag.encode('utf-8').split(',')

    acordes = []
    if acordes_tag:
        acordes = acordes_tag.encode('utf-8').split(',')
    elif id_musica_tag:
        musica = musicas[id_musica_tag]
        acordes = musica.acordes
    elif sequencia_tag:
        acordes = sequencia_tag.encode('utf-8').replace(',','')
        similares = []

        if acordes in sequencias:
            id_seq = sequencias[acordes]
            similares = get_pagina(get_similares_por_sequencia(id_seq, generos_key), pagina_tag)
        return json.dumps(similares)

    similares = get_similares(acordes, generos_key)

    return json.dumps(get_pagina(similares, pagina_tag))

def get_similares_por_sequencia(id_seq, generos_key):
    similares = []
    for musica in musicas.values():
        if str(id_seq) in musica.seqs_famosas and musica.genero in generos_key:
            similar = {
                'id_unico_musica' : musica.id_unico_musica,
                'id_artista' : musica.id_artista,
                'id_musica' : musica.id_musica,
                'nome_artista' : musica.nome_artista,
                'nome_musica' : musica.nome_musica,
                'popularidade' : musica.popularidade,
                'acordes' : musica.acordes,
                'genero' : musica.genero,
                'url' : musica.url,
            }
            similares.append(similar)

    return similares

def get_pagina(colecao, pagina_tag):
    sl = (int(pagina_tag) - 1)*TAM_PAGINA
    return colecao[sl:sl+TAM_PAGINA]

def get_similares(acordes, generos_key):
    similares = []
    for musica in musicas.values():
        inter = set(acordes).intersection(set(musica.acordes))
        diff = set(musica.acordes) - set(acordes)

        # somente as que tiverem interseção e as que forem
        # dos generos solicitados.
        if len(inter) > 0 and musica.genero in generos_key:
            similar = {
                    'id_unico_musica' : musica.id_unico_musica,
                    'id_artista' : musica.id_artista,
                    'id_musica' : musica.id_musica,
                    'nome_artista' : musica.nome_artista,
                    'nome_musica' : musica.nome_musica,
                    'popularidade' : musica.popularidade,
                    'acordes' : musica.acordes,
                    'genero' : musica.genero,
                    'url' : musica.url,
                    'diferenca' : list(diff),
                    'intersecao' : list(inter)
            }

            similares.append(similar)

    # ordenados por menor diferença, maior interseção e maior popularidade.
    return sorted(similares, key=lambda x: (len(x['diferenca']), -len(x['intersecao'])))

@app.after_request
def add_header(response):
    response.headers['Access-Control-Allow-Origin'] = '*'
    return response

if __name__ == '__main__':
    init()
    app.run(debug=True)

<|endoftext|>


<commit_before>from __future__ import unicode_literals

import frappe, os
from frappe import _

from frappe.utils.pdf import get_pdf,cleanup
from PyPDF2 import PdfFileWriter

no_cache = 1

base_template_path = "templates/www/printview.html"
standard_format = "templates/print_formats/standard.html"

@frappe.whitelist()
def download_multi_pdf(doctype, name, format=None):
	"""
	Concatenate multiple docs as PDF .

	Returns a PDF compiled by concatenating multiple documents. The documents
	can be from a single DocType or multiple DocTypes

	Note: The design may seem a little weird, but it exists exists to
		ensure backward compatibility. The correct way to use this function is to
		pass a dict to doctype as described below

	NEW FUNCTIONALITY
	=================
	Parameters:
	doctype (dict):
		key (string): DocType name
		value (list): of strings of doc names which need to be concatenated and printed
	name (string):
		name of the pdf which is generated
	format:
		Print Format to be used

	Returns:
	PDF: A PDF generated by the concatenation of the mentioned input docs

	OLD FUNCTIONALITY - soon to be deprecated
	=========================================
	Parameters:
	doctype (string):
		name of the DocType to which the docs belong which need to be printed
	name (string or list):
		If string the name of the doc which needs to be printed
		If list the list of strings of doc names which needs to be printed
	format:
		Print Format to be used

	Returns:
	PDF: A PDF generated by the concatenation of the mentioned input docs
	"""

	import json
	output = PdfFileWriter()

	if not isinstance(doctype, dict):
		result = json.loads(name)

		# Concatenating pdf files
		for i, ss in enumerate(result):
			output = frappe.get_print(doctype, ss, format, as_pdf = True, output = output)
		frappe.local.response.filename = "{doctype}.pdf".format(doctype=doctype.replace(" ", "-").replace("/", "-"))
	else:
		for doctype_name in doctype:
			for doc_name in doctype[doctype_name]:
				try:
					output = frappe.get_print(doctype_name, doc_name, format, as_pdf = True, output = output)
				except Exception:
					frappe.log_error("Permission Error on doc {} of doctype {}".format(doc_name, doctype_name))
		frappe.local.response.filename = "{}.pdf".format(name)

	frappe.local.response.filecontent = read_multi_pdf(output)
	frappe.local.response.type = "download"

def read_multi_pdf(output):
	# Get the content of the merged pdf files
	fname = os.path.join("/tmp", "frappe-pdf-{0}.pdf".format(frappe.generate_hash()))
	output.write(open(fname,"wb"))

	with open(fname, "rb") as fileobj:
		filedata = fileobj.read()

	return filedata

@frappe.whitelist()
def download_pdf(doctype, name, format=None, doc=None, no_letterhead=0):
	html = frappe.get_print(doctype, name, format, doc=doc, no_letterhead=no_letterhead)
	frappe.local.response.filename = "{name}.pdf".format(name=name.replace(" ", "-").replace("/", "-"))
	frappe.local.response.filecontent = get_pdf(html)
	frappe.local.response.type = "pdf"

@frappe.whitelist()
def report_to_pdf(html, orientation="Landscape"):
	frappe.local.response.filename = "report.pdf"
	frappe.local.response.filecontent = get_pdf(html, {"orientation": orientation})
	frappe.local.response.type = "pdf"

@frappe.whitelist()
def print_by_server(doctype, name, print_format=None, doc=None, no_letterhead=0):
	print_settings = frappe.get_doc("Print Settings")
	try:
		import cups
	except ImportError:
		frappe.throw(_("You need to install pycups to use this feature!"))
		return
	try:
		cups.setServer(print_settings.server_ip)
		cups.setPort(print_settings.port)
		conn = cups.Connection()
		output = PdfFileWriter()
		output = frappe.get_print(doctype, name, print_format, doc=doc, no_letterhead=no_letterhead, as_pdf = True, output = output)
		file = os.path.join("/", "tmp", "frappe-pdf-{0}.pdf".format(frappe.generate_hash()))
		output.write(open(file,"wb"))
		conn.printFile(print_settings.printer_name,file , name, {})
	except IOError as e:
		if ("ContentNotFoundError" in e.message
			or "ContentOperationNotPermittedError" in e.message
			or "UnknownContentError" in e.message
			or "RemoteHostClosedError" in e.message):
			frappe.throw(_("PDF generation failed"))
	except cups.IPPError:
		frappe.throw(_("Printing failed"))
	finally:
		cleanup(file,{})
<commit_msg>fix: Pass no_letterhead to download_multi_pdf<commit_after>from __future__ import unicode_literals

import frappe, os
from frappe import _

from frappe.utils.pdf import get_pdf,cleanup
from PyPDF2 import PdfFileWriter

no_cache = 1

base_template_path = "templates/www/printview.html"
standard_format = "templates/print_formats/standard.html"

@frappe.whitelist()
def download_multi_pdf(doctype, name, format=None, no_letterhead=0):
	"""
	Concatenate multiple docs as PDF .

	Returns a PDF compiled by concatenating multiple documents. The documents
	can be from a single DocType or multiple DocTypes

	Note: The design may seem a little weird, but it exists exists to
		ensure backward compatibility. The correct way to use this function is to
		pass a dict to doctype as described below

	NEW FUNCTIONALITY
	=================
	Parameters:
	doctype (dict):
		key (string): DocType name
		value (list): of strings of doc names which need to be concatenated and printed
	name (string):
		name of the pdf which is generated
	format:
		Print Format to be used

	Returns:
	PDF: A PDF generated by the concatenation of the mentioned input docs

	OLD FUNCTIONALITY - soon to be deprecated
	=========================================
	Parameters:
	doctype (string):
		name of the DocType to which the docs belong which need to be printed
	name (string or list):
		If string the name of the doc which needs to be printed
		If list the list of strings of doc names which needs to be printed
	format:
		Print Format to be used

	Returns:
	PDF: A PDF generated by the concatenation of the mentioned input docs
	"""

	import json
	output = PdfFileWriter()

	if not isinstance(doctype, dict):
		result = json.loads(name)

		# Concatenating pdf files
		for i, ss in enumerate(result):
			output = frappe.get_print(doctype, ss, format, as_pdf = True, output = output, no_letterhead=no_letterhead)
		frappe.local.response.filename = "{doctype}.pdf".format(doctype=doctype.replace(" ", "-").replace("/", "-"))
	else:
		for doctype_name in doctype:
			for doc_name in doctype[doctype_name]:
				try:
					output = frappe.get_print(doctype_name, doc_name, format, as_pdf = True, output = output, no_letterhead=no_letterhead)
				except Exception:
					frappe.log_error("Permission Error on doc {} of doctype {}".format(doc_name, doctype_name))
		frappe.local.response.filename = "{}.pdf".format(name)

	frappe.local.response.filecontent = read_multi_pdf(output)
	frappe.local.response.type = "download"

def read_multi_pdf(output):
	# Get the content of the merged pdf files
	fname = os.path.join("/tmp", "frappe-pdf-{0}.pdf".format(frappe.generate_hash()))
	output.write(open(fname,"wb"))

	with open(fname, "rb") as fileobj:
		filedata = fileobj.read()

	return filedata

@frappe.whitelist()
def download_pdf(doctype, name, format=None, doc=None, no_letterhead=0):
	html = frappe.get_print(doctype, name, format, doc=doc, no_letterhead=no_letterhead)
	frappe.local.response.filename = "{name}.pdf".format(name=name.replace(" ", "-").replace("/", "-"))
	frappe.local.response.filecontent = get_pdf(html)
	frappe.local.response.type = "pdf"

@frappe.whitelist()
def report_to_pdf(html, orientation="Landscape"):
	frappe.local.response.filename = "report.pdf"
	frappe.local.response.filecontent = get_pdf(html, {"orientation": orientation})
	frappe.local.response.type = "pdf"

@frappe.whitelist()
def print_by_server(doctype, name, print_format=None, doc=None, no_letterhead=0):
	print_settings = frappe.get_doc("Print Settings")
	try:
		import cups
	except ImportError:
		frappe.throw(_("You need to install pycups to use this feature!"))
		return
	try:
		cups.setServer(print_settings.server_ip)
		cups.setPort(print_settings.port)
		conn = cups.Connection()
		output = PdfFileWriter()
		output = frappe.get_print(doctype, name, print_format, doc=doc, no_letterhead=no_letterhead, as_pdf = True, output = output)
		file = os.path.join("/", "tmp", "frappe-pdf-{0}.pdf".format(frappe.generate_hash()))
		output.write(open(file,"wb"))
		conn.printFile(print_settings.printer_name,file , name, {})
	except IOError as e:
		if ("ContentNotFoundError" in e.message
			or "ContentOperationNotPermittedError" in e.message
			or "UnknownContentError" in e.message
			or "RemoteHostClosedError" in e.message):
			frappe.throw(_("PDF generation failed"))
	except cups.IPPError:
		frappe.throw(_("Printing failed"))
	finally:
		cleanup(file,{})
<|endoftext|>