<commit_before>#!/usr/local/bin/python

import sys, getopt
from Bio import AlignIO
from Bio.Alphabet import IUPAC
from Bio.Nexus import Nexus
from Heathpy import write_phylip

def main(argv):
  usage = 'ConvertAln -i <infile> -x <informat> -o <outfile> -f <outformat>'
  infile = ''
  informat = ''
  outfile = ''
  outformat = ''
  try:
     opts, args = getopt.getopt(argv,"hi:x:o:f:",["infile=", "informat=", "outfile=", "outformat="])
  except getopt.GetoptError:
     sys.exit(usage)
  for opt, arg in opts:
     if opt == '-h':
        print usage
        sys.exit()
     elif opt in ("-i", "--infile"):
        infile = arg
     elif opt in ("-x", "--informat"):
        informat = arg
     elif opt in ("-o", "--outfile"):
        outfile = arg
     elif opt in ("-f", "--outformat"):
        outformat = arg
  if not infile:
    sys.exit("must specify infile! %s" % usage)
  if not outformat:
    sys.exit("must specify format to convert to! %s" % usage)

  if not informat:
    informat = guess_format(infile)

  if not outfile:
    if '.' in infile:
      outfile = '.'.join((infile.split('.')[:-1] + [get_extension(outformat)]))
    else:
      outfile = '.'.join((infile, get_extension(outformat)))
  if infile == 'pipe' or infile == 'stdin' or infile == 'STDIN' or infile == '|':
    infile = sys.stdin
  if outformat == 'phylip':
    alignment=AlignIO.read(infile, informat, alphabet=IUPAC.ambiguous_dna)
    if outfile == 'pipe' or outfile == 'stdout' or outfile == 'STDOUT' or outfile == '|' or outfile == '>':
      write_phylip(alignment, sys.stdout)
    else:
      out_fh = open(outfile, 'w')
      write_phylip(alignment, out_fh)
      out_fh.close()

  else:
    if outfile == 'pipe' or outfile == 'stdout' or outfile == 'STDOUT' or outfile == '|' or outfile == '>':
      outfile = sys.stdout
    if outformat == 'nexus':
      alignment=AlignIO.read(infile, informat, alphabet=IUPAC.ambiguous_dna)
      write_nexus(alignment, outfile)
    else:
      AlignIO.convert(infile, informat, outfile, outformat, alphabet=IUPAC.ambiguous_dna)

def write_nexus(alignment, outfile):
  minimal_record = "#NEXUS\nbegin data; dimensions ntax=0 nchar=0; format datatype=%s; end;" % "dna"
  n = Nexus.Nexus(minimal_record)
  n.alphabet = alignment._alphabet
  for record in alignment:
    n.add_sequence(record.id, record.seq.tostring())
    n.write_nexus_data(outfile, interleave=False)

def guess_format(file):
  ext = file.split('.')[-1]
  if ext[0] == 'f' and ext[-1] == 'q': format = "fastq"
  elif ext[0:2] == 'fa': format = "fasta"
  elif ext == 'xmfa': format = "xmfa"
  elif ext == 'phy': format = "phylip"
  elif ext == 'aln': format = "clustal"
  elif ext == 'gb': format =  "genbank"
  elif ext == 'nxs': format =  "nexus"
  elif ext == 'nex': format =  "nexus"
  elif ext == 'maf': format =  "maf"
  else: sys.exit("extension not recognized! Use -x option to specify explicitly")
  return format

def get_extension(format):
  if format == "fasta": extension = "fa"
  elif format == "xmfa": extension = "xmfa"
  elif format == "fastq": extension = "fq"
  elif format == "phylip": extension = "phy"
  elif format == "clustal": extension = "aln"
  elif format == "genbank": extension = "gbk"
  elif format == "nexus": extension = "nex"
  else: sys.exit("format not recognized! Please select one of 'clustalw', 'fasta', 'fastq', 'genbank', 'nexus', 'phylip' or 'xmfa'")
  return extension

if __name__ == "__main__":
  if sys.argv[1] == 'self-test':
    from os import path, system
    print "Converting fasta to phylip..."
    infile = path.join(path.expanduser("~"), 'Perl', 't', 'ConvertAln.t.fa')
    main(('-i', infile, '-f', 'phylip'))
    system('open %s' % path.join(path.expanduser("~"), 'Perl', 't', 'ConvertAln.t.phy'))
    print "Converting phylip to nexus..."
    infile = path.join(path.expanduser("~"), 'Perl', 't', 'ConvertAln.t.phy')
    main(('-i', infile, '-f', 'nexus'))
    system('open %s' % path.join(path.expanduser("~"), 'Perl', 't', 'ConvertAln.t.nex'))
    print "Converting nexus to clustal..."
    infile = path.join(path.expanduser("~"), 'Perl', 't', 'ConvertAln.t.nex')
    main(('-i', infile, '-f', 'clustal'))
    system('open %s' % path.join(path.expanduser("~"), 'Perl', 't', 'ConvertAln.t.aln'))

  else:
    main(sys.argv[1:])

<commit_msg>added code to remove sequences consisting of only gaps<commit_after>#!/usr/local/bin/python

import sys, getopt
from Bio.Align import MultipleSeqAlignment
from Bio import AlignIO
from Bio.Alphabet import IUPAC
from Bio.Nexus import Nexus
from Heathpy import write_phylip

def main(argv):
  usage = 'ConvertAln -i <infile> -x <informat> -o <outfile> -f <outformat>'
  infile = ''
  informat = ''
  outfile = ''
  outformat = ''
  try:
     opts, args = getopt.getopt(argv,"hi:x:o:f:",["infile=", "informat=", "outfile=", "outformat="])
  except getopt.GetoptError:
     sys.exit(usage)
  for opt, arg in opts:
     if opt == '-h':
        print usage
        sys.exit()
     elif opt in ("-i", "--infile"):
        infile = arg
     elif opt in ("-x", "--informat"):
        informat = arg
     elif opt in ("-o", "--outfile"):
        outfile = arg
     elif opt in ("-f", "--outformat"):
        outformat = arg
  if not infile:
    sys.exit("must specify infile! %s" % usage)
  if not outformat:
    sys.exit("must specify format to convert to! %s" % usage)

  if not informat:
    informat = guess_format(infile)

  if not outfile:
    if '.' in infile:
      outfile = '.'.join((infile.split('.')[:-1] + [get_extension(outformat)]))
    else:
      outfile = '.'.join((infile, get_extension(outformat)))
  if infile == 'pipe' or infile == 'stdin' or infile == 'STDIN' or infile == '|':
    infile = sys.stdin
  if outformat == 'phylip':
    alignment=AlignIO.read(infile, informat, alphabet=IUPAC.ambiguous_dna)
    alignment = remove_blank(alignment)
    if outfile == 'pipe' or outfile == 'stdout' or outfile == 'STDOUT' or outfile == '|' or outfile == '>':
      write_phylip(alignment, sys.stdout)
    else:
      out_fh = open(outfile, 'w')
      write_phylip(alignment, out_fh)
      out_fh.close()

  else:
    if outfile == 'pipe' or outfile == 'stdout' or outfile == 'STDOUT' or outfile == '|' or outfile == '>':
      outfile = sys.stdout
    if outformat == 'nexus':
      alignment=AlignIO.read(infile, informat, alphabet=IUPAC.ambiguous_dna)
      write_nexus(alignment, outfile)
    else:
      AlignIO.convert(infile, informat, outfile, outformat, alphabet=IUPAC.ambiguous_dna)

def write_nexus(alignment, outfile):
  minimal_record = "#NEXUS\nbegin data; dimensions ntax=0 nchar=0; format datatype=%s; end;" % "dna"
  n = Nexus.Nexus(minimal_record)
  n.alphabet = alignment._alphabet
  for record in alignment:
    n.add_sequence(record.id, record.seq.tostring())
    n.write_nexus_data(outfile, interleave=False)

def guess_format(file):
  ext = file.split('.')[-1]
  if ext[0] == 'f' and ext[-1] == 'q': format = "fastq"
  elif ext[0:2] == 'fa': format = "fasta"
  elif ext == 'xmfa': format = "xmfa"
  elif ext == 'phy': format = "phylip"
  elif ext == 'aln': format = "clustal"
  elif ext == 'gb': format =  "genbank"
  elif ext == 'nxs': format =  "nexus"
  elif ext == 'nex': format =  "nexus"
  elif ext == 'maf': format =  "maf"
  else: sys.exit("extension not recognized! Use -x option to specify explicitly")
  return format

def get_extension(format):
  if format == "fasta": extension = "fa"
  elif format == "xmfa": extension = "xmfa"
  elif format == "fastq": extension = "fq"
  elif format == "phylip": extension = "phy"
  elif format == "clustal": extension = "aln"
  elif format == "genbank": extension = "gbk"
  elif format == "nexus": extension = "nex"
  else: sys.exit("format not recognized! Please select one of 'clustalw', 'fasta', 'fastq', 'genbank', 'nexus', 'phylip' or 'xmfa'")
  return extension

def remove_blank(alignment):
  sequences = []
  for seq in alignment:
    if len(seq.seq.ungap(gap='-')) > 0:
      sequences.append(seq)
  return MultipleSeqAlignment(sequences)

if __name__ == "__main__":
  if sys.argv[1] == 'self-test':
    from os import path, system
    print "Converting fasta to phylip..."
    infile = path.join(path.expanduser("~"), 'Perl', 't', 'ConvertAln.t.fa')
    main(('-i', infile, '-f', 'phylip'))
    system('open %s' % path.join(path.expanduser("~"), 'Perl', 't', 'ConvertAln.t.phy'))
    print "Converting phylip to nexus..."
    infile = path.join(path.expanduser("~"), 'Perl', 't', 'ConvertAln.t.phy')
    main(('-i', infile, '-f', 'nexus'))
    system('open %s' % path.join(path.expanduser("~"), 'Perl', 't', 'ConvertAln.t.nex'))
    print "Converting nexus to clustal..."
    infile = path.join(path.expanduser("~"), 'Perl', 't', 'ConvertAln.t.nex')
    main(('-i', infile, '-f', 'clustal'))
    system('open %s' % path.join(path.expanduser("~"), 'Perl', 't', 'ConvertAln.t.aln'))

  else:
    main(sys.argv[1:])

<|endoftext|>


<commit_before>import os
from PyQt5.QtCore import (
    Qt,
    QVariant,
    QStringListModel
)

from PyQt5.QtWidgets import (
    QVBoxLayout,
    QTableWidget,
    QHeaderView,
    QTableWidgetItem,
    QDockWidget,
    QWidget,
    QTableView,
    QListView,
)

from cemu.memory import (
    MemorySection,
    MemoryLayoutEntryType,
)

from cemu.utils import (
    ishex,
)

from typing import List, Tuple, Any

class MemoryMappingWidget(QDockWidget):
    def __init__(self, parent, *args, **kwargs):
        super(MemoryMappingWidget, self).__init__("Memory map", parent)
        self.log = self.parentWidget().log
        layout = QVBoxLayout()
        self.memory_mapping = [
            MemorySection(".text",  0x00040000, 0x1000, "READ|EXEC",  None),
            MemorySection(".data",  0x00060000, 0x1000, "READ|WRITE", None),
            MemorySection(".stack", 0x00080000, 0x4000, "READ|WRITE", None),
            MemorySection(".misc",  0x00070000, 0x1000, "ALL",        None),
        ]
        model = QStringListModel([ str(x) for x in self.memory_mapping ])
        view = QListView()
        view.setModel(model)
        layout.addWidget(view)
        widget = QWidget(self)
        widget.setLayout(layout)
        self.setWidget(widget)
        return

    @property
    def maps(self) -> List[MemoryLayoutEntryType]:
        self.__maps = [ entry.export() for entry in self.memory_mapping ]
        return self.__maps<commit_msg>fixes issue #50 : added buttons to control the sections passed to unicorn<commit_after>import os
from PyQt5.QtCore import (
    Qt,
    QVariant,
    QStringListModel
)

from PyQt5.QtWidgets import (
    QVBoxLayout,
    QTableWidget,
    QHeaderView,
    QTableWidgetItem,
    QDockWidget,
    QWidget,
    QTableView,
    QListView,
    QPushButton,
    QGridLayout,
    QHBoxLayout,
    QLabel,
    QLineEdit,
    QMessageBox,
    QCheckBox,
)

from cemu.memory import (
    MemorySection,
    MemoryLayoutEntryType,
)

from cemu.utils import (
    ishex,
)

from typing import List, Tuple, Any

class MemoryMappingWidget(QDockWidget):
    def __init__(self, parent, *args, **kwargs):
        super(MemoryMappingWidget, self).__init__("Memory map", parent)
        self.log = self.parentWidget().log
        layout = QVBoxLayout()
        self.__memory_mapping = [
            MemorySection(".text",  0x00040000, 0x1000, "READ|EXEC"),
            MemorySection(".data",  0x00060000, 0x1000, "READ|WRITE"),
            MemorySection(".stack", 0x00080000, 0x4000, "READ|WRITE"),
            MemorySection(".misc",  0x00070000, 0x1000, "ALL"),
        ]
        self.model = QStringListModel()
        self.model.setStringList(self.memory_mapping_str)
        self.view = QListView()
        self.view.setModel(self.model)
        layout.addWidget(self.view)
        buttons = QHBoxLayout()
        btn_add = QPushButton("Add Section")
        buttons.addWidget(btn_add)
        btn_add.clicked.connect(self.onAddSectionButtonClicked)
        buttons.addStretch(1)
        btn_del = QPushButton("Remove Section")
        btn_del.clicked.connect(self.onDeleteSectionButtonClicked)
        buttons.addWidget(btn_del)
        layout.addLayout(buttons)
        w = QWidget(self)
        w.setLayout(layout)
        self.setWidget(w)
        return

    @property
    def memory_mapping_str(self) -> None:
        """
        Generator for the string view model
        """
        for entry in self.__memory_mapping:
            yield str(entry)

    def updateView(self) -> bool:
        """
        Refresh the view
        """
        self.model.setStringList(self.memory_mapping_str)
        return True

    @property
    def maps(self) -> List[MemoryLayoutEntryType]:
        """
        Exports the memory mapping to a format usable for Unicorn
        """
        self.__maps = [ entry.export() for entry in self.__memory_mapping ]
        return self.__maps

    def onAddSectionButtonClicked(self) -> None:
        """
        Callback associated with the click of the "Add Section" button
        """
        self.add_or_edit_section_popup()
        return

    def onDeleteSectionButtonClicked(self) -> None:
        """
        Callback associated with the click of the "Remove Section" button
        """
        idx = self.view.currentIndex()
        line_content = idx.data()
        for i, sect in enumerate(self.__memory_mapping):
            if line_content == str(sect):
                del self.__memory_mapping[i]
                self.updateView()
                break
        return

    def add_or_edit_section_popup(self) -> None:
        """
        Popup that present a form with all info to submit to create a new section
        """
        msgbox = QMessageBox(self)
        wid = QWidget()

        name = QLabel("Name")
        nameEdit = QLineEdit()

        startAddress = QLabel("Start Address (hex)")
        startAddressEdit = QLineEdit()

        size = QLabel("Size (hex)")
        sizeEdit = QLineEdit()

        perm = QLabel("Permissions")
        permCheck = QWidget()
        permCheckLayout = QHBoxLayout()
        perm_read_btn = QCheckBox("R")
        perm_write_btn = QCheckBox("W")
        perm_exec_btn = QCheckBox("X")
        permCheckLayout.addWidget(perm_read_btn)
        permCheckLayout.addWidget(perm_write_btn)
        permCheckLayout.addWidget(perm_exec_btn)
        permCheck.setLayout(permCheckLayout)

        grid = QGridLayout()
        grid.setSpacing(10)

        grid.addWidget(name, 1, 0)
        grid.addWidget(nameEdit, 1, 1)

        grid.addWidget(startAddress, 2, 0)
        grid.addWidget(startAddressEdit, 2, 1)

        grid.addWidget(size, 3, 0)
        grid.addWidget(sizeEdit, 3, 1)

        grid.addWidget(perm, 4, 0)
        grid.addWidget(permCheck, 4, 1)

        msgbox.setLayout(grid)
        msgbox.setGeometry(300, 300, 350, 300)

        msgbox.setWindowTitle("Add/edit section")
        layout = msgbox.layout()
        wid.setLayout(grid)
        wid.setMinimumWidth(400)
        layout.addWidget(wid)

        msgbox.setStandardButtons(QMessageBox.Ok | QMessageBox.Cancel)

        ret = msgbox.exec()
        if ret == QMessageBox.Ok:
            section_name = nameEdit.text()
            section_address = int(startAddressEdit.text(), 16)
            section_size = int(sizeEdit.text(), 16)
            section_perm = []
            if perm_read_btn.isChecked(): section_perm.append("READ")
            if perm_write_btn.isChecked(): section_perm.append("WRITE")
            if perm_exec_btn.isChecked(): section_perm.append("EXEC")
            section = MemorySection(section_name, section_address, section_size, "|".join(section_perm))
            self.__memory_mapping.append(section)
            self.updateView()
        return
<|endoftext|>


<commit_before>import re

from datetime import datetime

# HTTP statuses
# non-annotted statuses are defined in RFC7231
# see: http://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml
HTTP_STATUSES = {
    100: "Continue",
    101: "Switching Protocols",
    102: "Processing", #RFC2518

    200: "OK",
    201: "Created",
    202: "Accepted",
    203: "Non-Authoritative Information",
    204: "No Content",
    205: "Reset Content",
    206: "Partial Content", #RFC7233
    207: "Multi-Status", #RFC4918
    208: "Already Reported", #RFC5842
    226: "IM Used", #RFC3229

    300: "Multiple Choices",
    301: "Moved Permanently",
    302: "Moved Temporarily",
    303: "See Other",
    304: "Not Modified", #RFC7232
    305: "Use Proxy",
    307: "Temporary redirect",
    308: "Permanent Redirect", #RFC7538

    400: "Bad Request",
    401: "Unauthorized",
    402: "Payment Required",
    403: "Forbidden",
    404: "Not Found",
    405: "Method Not Allowed",
    406: "Not Acceptable",
    407: "Proxy Authentification Required", #RFC7235
    408: "Request Time-out",
    409: "Conflict",
    410: "Gone",
    411: "Length Required",
    412: "Precondition Failed", #RFC7232
    413: "Payload Too Large",
    414: "URI Too Long",
    415: "Unsupported Media Type",
    416: "Range Not Satisfiable", #RFC7233
    417: "Exception Failed",
    421: "Misdirected Request", #RFC7540
    422: "Unprocessable Entity", #RFC4918
    423: "Locked", #RFC4918
    424: "Failed Dependency", #RFC4918
    426: "Upgrade Required",
    428: "Precondition Required", #RFC6585
    429: "Too Many Request", #RFC6585
    431: "Request Header Fields Too Large", #RFC6585

    500: "Internal Server Error",
    501: "Not Implemented",
    502: "Bad Gateway",
    503: "Service Unavailable",
    504: "Gateway Time-out",
    505: "HTTP Version Not Supported",
    506: "Variant Also Negociates", #RFC2295
    507: "Insufficient Storage", #RFC4918
    508: "Loop Detected", #RFC5842
    510: "Not Extended", #RFC2774
    511: "Network Authentification Required" #RFC6585
}

HTTP_METHODS = frozenset((
    "GET", "HEAD", "POST", "OPTIONS", "CONNECT", "TRACE", "PUT", "PATCH", "DELETE"
))

#=============================#
# RFC 1123 datetime functions #
#=============================#

WEEKDAY = ("Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun")
MONTH = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")

_rfc1123_datetime = r"^[A-Z][a-z]{2}, ([0-9]{1,2}) ([A-Z][a-z]{2}) ([0-9]{2}|[0-9]{4}) ([0-9]{2}):([0-9]{2}):([0-9]{2}) GMT$"
RFC1123_DATETIME_REGEX = re.compile(_rfc1123_datetime)

def is_rfc1123_datetime(value):
    assert isinstance(value, str)
    return RFC1123_DATETIME_REGEX.match(value) is not None

def rfc1123_datetime_encode(dt):
    assert isinstance(dt, datetime)

    return "{0}, {1} {2} {3} {4:02}:{5:02}:{6:02} {7}".format(
        WEEKDAY[dt.weekday()],
        dt.day,
        MONTH[dt.month - 1],
        dt.year,
        dt.hour,
        dt.minute,
        dt.second,
        "GMT"
    )

def rfc1123_datetime_decode(string):
    match = RFC1123_DATETIME_REGEX.match(string)

    if not match:
        msg = "{} is not a valid RFC1123 date".format(string)
        raise ValueError(msg)

    groups = match.groups()

    return datetime(
        year = int(groups[2]) if len(groups[2]) == 4 else int("19" + groups[2]),
        month = MONTH.index(groups[1]) + 1,
        day = int(groups[0]),
        hour = int(groups[3]),
        minute = int(groups[4]),
        second = int(groups[5])
    )
<commit_msg>added async iterator for body reading<commit_after>import io
import re

from asyncio import coroutine
from datetime import datetime

if "StopAsyncIteration" not in dir(__builtins__):
    from asyncioplus.utils import StopAsyncIteration

# HTTP statuses
# non-annotted statuses are defined in RFC7231
# see: http://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml
HTTP_STATUSES = {
    100: "Continue",
    101: "Switching Protocols",
    102: "Processing", #RFC2518

    200: "OK",
    201: "Created",
    202: "Accepted",
    203: "Non-Authoritative Information",
    204: "No Content",
    205: "Reset Content",
    206: "Partial Content", #RFC7233
    207: "Multi-Status", #RFC4918
    208: "Already Reported", #RFC5842
    226: "IM Used", #RFC3229

    300: "Multiple Choices",
    301: "Moved Permanently",
    302: "Moved Temporarily",
    303: "See Other",
    304: "Not Modified", #RFC7232
    305: "Use Proxy",
    307: "Temporary redirect",
    308: "Permanent Redirect", #RFC7538

    400: "Bad Request",
    401: "Unauthorized",
    402: "Payment Required",
    403: "Forbidden",
    404: "Not Found",
    405: "Method Not Allowed",
    406: "Not Acceptable",
    407: "Proxy Authentification Required", #RFC7235
    408: "Request Time-out",
    409: "Conflict",
    410: "Gone",
    411: "Length Required",
    412: "Precondition Failed", #RFC7232
    413: "Payload Too Large",
    414: "URI Too Long",
    415: "Unsupported Media Type",
    416: "Range Not Satisfiable", #RFC7233
    417: "Exception Failed",
    421: "Misdirected Request", #RFC7540
    422: "Unprocessable Entity", #RFC4918
    423: "Locked", #RFC4918
    424: "Failed Dependency", #RFC4918
    426: "Upgrade Required",
    428: "Precondition Required", #RFC6585
    429: "Too Many Request", #RFC6585
    431: "Request Header Fields Too Large", #RFC6585

    500: "Internal Server Error",
    501: "Not Implemented",
    502: "Bad Gateway",
    503: "Service Unavailable",
    504: "Gateway Time-out",
    505: "HTTP Version Not Supported",
    506: "Variant Also Negociates", #RFC2295
    507: "Insufficient Storage", #RFC4918
    508: "Loop Detected", #RFC5842
    510: "Not Extended", #RFC2774
    511: "Network Authentification Required" #RFC6585
}

HTTP_METHODS = frozenset((
    "GET", "HEAD", "POST", "OPTIONS", "CONNECT", "TRACE", "PUT", "PATCH", "DELETE"
))

#=============================#
# RFC 1123 datetime functions #
#=============================#

WEEKDAY = ("Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun")
MONTH = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")

_rfc1123_datetime = r"^[A-Z][a-z]{2}, ([0-9]{1,2}) ([A-Z][a-z]{2}) ([0-9]{2}|[0-9]{4}) ([0-9]{2}):([0-9]{2}):([0-9]{2}) GMT$"
RFC1123_DATETIME_REGEX = re.compile(_rfc1123_datetime)

def is_rfc1123_datetime(value):
    assert isinstance(value, str)
    return RFC1123_DATETIME_REGEX.match(value) is not None

def rfc1123_datetime_encode(dt):
    assert isinstance(dt, datetime)

    return "{0}, {1} {2} {3} {4:02}:{5:02}:{6:02} {7}".format(
        WEEKDAY[dt.weekday()],
        dt.day,
        MONTH[dt.month - 1],
        dt.year,
        dt.hour,
        dt.minute,
        dt.second,
        "GMT"
    )

def rfc1123_datetime_decode(string):
    match = RFC1123_DATETIME_REGEX.match(string)

    if not match:
        msg = "{} is not a valid RFC1123 date".format(string)
        raise ValueError(msg)

    groups = match.groups()

    return datetime(
        year = int(groups[2]) if len(groups[2]) == 4 else int("19" + groups[2]),
        month = MONTH.index(groups[1]) + 1,
        day = int(groups[0]),
        hour = int(groups[3]),
        minute = int(groups[4]),
        second = int(groups[5])
    )

class BufferedBodyReader:
    def __init__(self, reader, body_size = None, block_size = io.DEFAULT_BUFFER_SIZE):
        self.reader = reader
        self.body_size = body_size
        self.block_size = block_size

        self.current_block = 0

        if self.body_size is not None:
            self.block_count, self.last_block_size = divmod(self.body_size, self.block_size)

        self.bytes_read = 0

    @property
    def is_complete(self):
        return self.bytes_read == self.body_size

    @coroutine
    def __aiter__(self):
        return self

    @coroutine
    def __anext__(self):
        if self.body_size is not None:
            if self.current_block < self.block_count:
                block_size = self.block_size
            elif self.current_block == self.block_count:
                block_size = self.last_block_size
            elif self.current_block > self.block_count:
                block_size = 0
        else:
            block_size = self.block_size

        if block_size == 0:
            raise StopAsyncIteration

        block = yield from self.reader.read(block_size)

        if block == b"":
            raise StopAsyncIteration

        self.bytes_read += len(block)
        self.current_block += 1

        return block

class ChunkedBodyReader:
    def __init__(self, reader):
        self.reader = reader

        self.current_chunk = 0
        # self.body_size = 0

    @coroutine
    def __aiter__(self):
        return self

    @coroutine
    def __anext__(self):
        chunk_header = yield from self.reader.read_until(b"\r\n")
        chunk_size = int(chunk_header, base = 16)

        if chunk_size == 0:
            raise StopAsyncIteration

        chunk = yield from self.reader.read(chunk_size + 2)
        chunk, check_crlf = chunk[:-2], chunk[-2:]

        if check_crlf != b"\r\n":
            raise Exception("chunk not followed by CRLF")

        if chunk == b"":
            # TODO: parse trailer headers
            raise StopAsyncIteration

        # self.body_size += len(chunk)
        self.current_chunk += 1

        return chunk
<|endoftext|>


<commit_before>import re
from bush import color
from bush.aws.base import AWSBase

class EC2(AWSBase):
    USAGE = """%prog ec2 <Command> [options]

Commands
    * ls
    * images
    * start
    * stop
    * remove
    """
    SUB_COMMANDS = ['ls', 'images', 'start', 'stop', 'remove']

    COLUMNS_HELP = """select columns, comma separated.
availability_zone,
image_id,
instance_id,
instance_type,
key_name,
launch_time,
private_dns_name,
private_ip_address,
public_dns_name,
public_ip_address,
security_groups,
state,
tag_Name
    """
    TAG_RE = re.compile(r'^tag_(.+)')

    def __init__(self, options):
        super().__init__(options, 'ec2')

    def __get_instances(self):
        filter_name = ''
        filter_values = ''
        options = self.options

        if options.instance_id:
            filter_name = 'instance-id'
            filter_values = options.instance_id.split(',')
        elif options.tag_name:
            filter_name = 'tag:Name'
            filter_values = options.tag_name.split(',')
        elif options.filter_name and options.filter_values:
            filter_name = options.filter_name
            filter_values = options.filter_values.split(',')
        else:
            self.instances = self.resource.instances.all()
            return

        f = {'Name': filter_name, 'Values': filter_values}
        self.instances = self.resource.instances.filter(Filters=[f])

    def __get_max_len(self, attr):
        max_len = len(attr)
        for instance in self.instances:
            try:
                val = getattr(instance, attr)
            except:
                val = ''
            if not type(val) is str:
                val = ''

            if max_len < len(val):
                max_len = len(val)

        return max_len

    def __get_max_len_from_tag(self, key, header):
        max_len = len(header)
        for instance in self.instances:
            val = self.__get_tag_value(instance.tags, key)

            if max_len < len(val):
                max_len = len(val)

        return max_len

    def __get_max_len_from_state(self):
        max_len = 5  # <state>
        for instance in self.instances:
            val = instance.state['Name']

            if max_len < len(val):
                max_len = len(val)

        return max_len

    def __get_max_len_from_security_groups(self):
        max_len = 15  # <security_groups>
        for instance in self.instances:
            val = self.__get_security_group_names(instance.security_groups)

            if max_len < len(val):
                max_len = len(val)

        return max_len

    def __get_tag_value(self, tags, key):
        if tags is None:
            return ''

        for tag in tags:
            if tag['Key'] == key:
                return tag['Value']

        return ''

    def __get_state(self, instance):
        code = instance.state['Code']
        state = instance.state['Name']
        # ref http://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_InstanceState.html
        if code == 16:
            return color.green(state)
        elif code == 48 or code == 80:
            return color.red(state)
        elif code == 0 or code == 32 or code == 64:
            return color.yellow(state)
        else:
            return state

    def __get_state_with_image(self, image):
        state = image.state
        if state == 'available':
            return color.green(state)
        elif state == 'pending':
            return color.yellow(state)
        elif state == 'failed':
            return color.red(state)
        else:
            return state

    def __get_security_group_names(self, groups):
        names = []
        if groups:
            for g in groups:
                names.append(g['GroupName'])

        return ', '.join(names)

    def __get_list_format(self, headers):
        formats = []
        for i, header in enumerate(headers):
            tag_key = EC2.TAG_RE.match(header)
            if tag_key:
                len = self.__get_max_len_from_tag(tag_key.groups()[0], header)
            elif header == 'state':
                len = self.__get_max_len_from_state()
            elif header == 'availability_zone':
                len = 17  # <availability_zone>
            elif header == 'launch_time':
                len = 19  # YYYY/mm/dd HH:MM:SS
            elif header == 'security_groups':
                len = self.__get_max_len_from_security_groups()
            else:
                len = self.__get_max_len(header)
            formats.append('{%s:<%s}' % (i, len + 1))

        return ''.join(formats)

    def ls(self):
        default_columns = [
            'instance_id',
            'instance_type',
            'tag_Name',
            'public_ip_address',
            'private_ip_address',
            'state'
        ]

        other_columns = [
            'availability_zone',
            'image_id',
            'key_name',
            'launch_time',
            'private_dns_name',
            'public_dns_name',
            'security_groups',
        ]

        all_columns = default_columns + other_columns
        if self.options.columns:
            columns = []
            # unique
            for c in self.options.columns.split(','):
                if c not in columns:
                    columns.append(c)
        else:
            columns = default_columns

        headers = [h for h in columns if h in all_columns]

        self.__get_instances()

        i_info = []
        for instance in self.instances:
            line = {}
            for h in headers:
                tag_key = EC2.TAG_RE.match(h)
                if tag_key:
                    val = self.__get_tag_value(instance.tags,
                                               tag_key.groups()[0])
                elif h == 'availability_zone':
                    val = instance.placement['AvailabilityZone']
                elif h == 'launch_time':
                    if instance.launch_time:
                        time_format = '%Y/%m/%d %H:%M:%S'
                        val = instance.launch_time.strftime(time_format)
                elif h == 'security_groups':
                    val = self.__get_security_group_names(
                            instance.security_groups)
                elif h == 'state':
                    val = self.__get_state(instance)
                else:
                    val = getattr(instance, h) or ''
                line[h] = val or '-'

            i_info.append(line)

        list_format = self.__get_list_format(headers)
        header = list_format.format(*headers)

        order_by = self.options.order_by
        if order_by and order_by in headers:
            i_info = sorted(i_info, key=lambda x: x[order_by])
            if self.options.order == 'desc':
                i_info.reverse()

        page = []
        page.append(header)
        page.append('-' * (len(header) - 1))

        for line in i_info:
            l = []
            for k in line:
                l.append(line[k])
            page.append(list_format.format(*l))

        return page

    def images(self):
        columns = [
            'image_id',
            'name',
            'description',
            'creation_date',
            'state'
        ]

        if self.options.image_id:
            image_ids = self.options.image_id.split(',')
            images = self.resource.images.filter(ImageIds=image_ids)
        else:
            owners = ['self']
            images = self.resource.images.filter(Owners=owners)

        i_info = []
        for image in images:
            line = {}
            for column in columns:
                if column == 'state':
                    val = self.__get_state_with_image(image)
                else:
                    val = getattr(image, column)
                line[column] = val or '-'

            i_info.append(line)

        formats = []
        for i, column in enumerate(columns):
            l = len(column)
            for image in images:
                val = getattr(image, column) or ''
                if len(val) > l:
                    l = len(val)

            formats.append('{%s:<%s}' % (i, l + 1))
        list_format = ''.join(formats)
        header = list_format.format(*columns)

        i_info = sorted(i_info, key=lambda x: x['name'])

        page = []
        page.append(header)
        page.append('-' * (len(header) - 1))

        for line in i_info:
            l = []
            for k in line:
                l.append(line[k])
            page.append(list_format.format(*l))

        return page

    def start_instances(self):
        self.options.columns = 'instance_id,tag_Name,state'
        self.__get_instances()

        instance_ids = []
        will_starts = []
        for instance in self.instances:
            will_start = {}
            state = instance.state['Code']
            if state == 80:
                instance_ids.append(instance.instance_id)
                will_start['instance_id'] = instance.instance_id
                will_start['tag_Name'] = self.__get_tag_value(
                    instance.tags, 'Name'
                )
                will_starts.append(will_start)

        if len(instance_ids) == 0:
            return ['Not found stopped instances']

        sorted_instances = sorted(
            will_starts, key=lambda x: x['instance_id']
        )

        startings = self.client.start_instances(
            InstanceIds=instance_ids
            )['StartingInstances']
        sorted_startings = sorted(startings, key=lambda x: x['InstanceId'])

        formats = []
        for i, column in enumerate(['instance_id', 'tag_Name']):
            max_len = len(max(w[column] for w in will_starts))
            formats.append('{%s:<%s}' % (i, max_len + 1))

        headers = ['instance_id', 'tag_Name']
        list_format = ''.join(formats)
        header = list_format.format(*headers)
        page = [header, '-' * (len(header) - 1)]

        for i, starting in enumerate(sorted_startings):
            if starting['CurrentState']['Code'] == 0:
                page.append(list_format.format(
                    sorted_instances[i]['instance_id'],
                    sorted_instances[i]['tag_Name']
                ))

        return page

    def stop_instances(self):
        self.options.columns = 'instance_id,tag_Name,state'
        self.__get_instances()

        instance_ids = []
        will_stops = []
        for instance in self.instances:
            will_stop = {}
            state = instance.state['Code']
            if state == 16:
                instance_ids.append(instance.instance_id)
                will_stop['instance_id'] = instance.instance_id
                will_stop['tag_Name'] = self.__get_tag_value(
                    instance.tags, 'Name'
                )
                will_stops.append(will_stop)

        if len(instance_ids) == 0:
            return ['Not found running instances']

        sorted_instances = sorted(will_stops, key=lambda x: x['instance_id'])

        stoppings = self.client.stop_instances(
            InstanceIds=instance_ids
            )['StoppingInstances']
        sorted_stoppings = sorted(stoppings, key=lambda x: x['InstanceId'])

        formats = []
        for i, column in enumerate(['instance_id', 'tag_Name']):
            max_len = len(max(w[column] for w in will_stops))
            formats.append('{%s:<%s}' % (i, max_len + 1))

        headers = ['instance_id', 'tag_Name']
        list_format = ''.join(formats)
        header = list_format.format(*headers)
        page = [header, '-' * (len(header) - 1)]

        for i, stopping in enumerate(sorted_stoppings):
            if stopping['CurrentState']['Code'] == 64:
                page.append(list_format.format(
                    sorted_instances[i]['instance_id'],
                    sorted_instances[i]['tag_Name']
                ))

        return page

    def terminate_instances(self):
        self.options.columns = 'instance_id,tag_Name,state'
        self.__get_instances()

        instance_ids = []
        will_terminates = []
        for instance in self.instances:
            will_terminate = {}
            state = instance.state['Code']
            if state == 16 or state == 80:
                instance_ids.append(instance.instance_id)
                will_terminate['instance_id'] = instance.instance_id
                will_terminate['tag_Name'] = self.__get_tag_value(
                    instance.tags, 'Name'
                )
                will_terminates.append(will_terminate)

        if len(instance_ids) == 0:
            return ['Not found running or stopped instances']

        sorted_instances = sorted(
            will_terminates, key=lambda x: x['instance_id']
        )

        terminating = self.client.terminate_instances(
            InstanceIds=instance_ids
            )['TerminatingInstances']
        sorted_stoppings = sorted(terminating, key=lambda x: x['InstanceId'])

        formats = []
        for i, column in enumerate(['instance_id', 'tag_Name']):
            max_len = len(max(w[column] for w in will_terminates))
            formats.append('{%s:<%s}' % (i, max_len + 1))

        headers = ['instance_id', 'tag_Name']
        list_format = ''.join(formats)
        header = list_format.format(*headers)
        page = [header, '-' * (len(header) - 1)]

        for i, stopping in enumerate(sorted_stoppings):
            page.append(list_format.format(
                sorted_instances[i]['instance_id'],
                sorted_instances[i]['tag_Name']
            ))

        return page
<commit_msg>Require filter start/stop/terminate EC2 instance<commit_after>import re
from bush import color
from bush.aws.base import AWSBase

class EC2(AWSBase):
    USAGE = """%prog ec2 <Command> [options]

Commands
    * ls
    * images
    * start
    * stop
    * remove
    """
    SUB_COMMANDS = ['ls', 'images', 'start', 'stop', 'remove']

    COLUMNS_HELP = """select columns, comma separated.
availability_zone,
image_id,
instance_id,
instance_type,
key_name,
launch_time,
private_dns_name,
private_ip_address,
public_dns_name,
public_ip_address,
security_groups,
state,
tag_Name
    """
    TAG_RE = re.compile(r'^tag_(.+)')

    def __init__(self, options):
        super().__init__(options, 'ec2')

    def __get_instances(self):
        filter_name = ''
        filter_values = ''
        options = self.options

        if options.instance_id:
            filter_name = 'instance-id'
            filter_values = options.instance_id.split(',')
        elif options.tag_name:
            filter_name = 'tag:Name'
            filter_values = options.tag_name.split(',')
        elif options.filter_name and options.filter_values:
            filter_name = options.filter_name
            filter_values = options.filter_values.split(',')
        else:
            self.instances = self.resource.instances.all()
            return

        f = {'Name': filter_name, 'Values': filter_values}
        self.instances = self.resource.instances.filter(Filters=[f])

    def __get_max_len(self, attr):
        max_len = len(attr)
        for instance in self.instances:
            try:
                val = getattr(instance, attr)
            except:
                val = ''
            if not type(val) is str:
                val = ''

            if max_len < len(val):
                max_len = len(val)

        return max_len

    def __get_max_len_from_tag(self, key, header):
        max_len = len(header)
        for instance in self.instances:
            val = self.__get_tag_value(instance.tags, key)

            if max_len < len(val):
                max_len = len(val)

        return max_len

    def __get_max_len_from_state(self):
        max_len = 5  # <state>
        for instance in self.instances:
            val = instance.state['Name']

            if max_len < len(val):
                max_len = len(val)

        return max_len

    def __get_max_len_from_security_groups(self):
        max_len = 15  # <security_groups>
        for instance in self.instances:
            val = self.__get_security_group_names(instance.security_groups)

            if max_len < len(val):
                max_len = len(val)

        return max_len

    def __get_tag_value(self, tags, key):
        if tags is None:
            return ''

        for tag in tags:
            if tag['Key'] == key:
                return tag['Value']

        return ''

    def __get_state(self, instance):
        code = instance.state['Code']
        state = instance.state['Name']
        # ref http://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_InstanceState.html
        if code == 16:
            return color.green(state)
        elif code == 48 or code == 80:
            return color.red(state)
        elif code == 0 or code == 32 or code == 64:
            return color.yellow(state)
        else:
            return state

    def __get_state_with_image(self, image):
        state = image.state
        if state == 'available':
            return color.green(state)
        elif state == 'pending':
            return color.yellow(state)
        elif state == 'failed':
            return color.red(state)
        else:
            return state

    def __get_security_group_names(self, groups):
        names = []
        if groups:
            for g in groups:
                names.append(g['GroupName'])

        return ', '.join(names)

    def __get_list_format(self, headers):
        formats = []
        for i, header in enumerate(headers):
            tag_key = EC2.TAG_RE.match(header)
            if tag_key:
                len = self.__get_max_len_from_tag(tag_key.groups()[0], header)
            elif header == 'state':
                len = self.__get_max_len_from_state()
            elif header == 'availability_zone':
                len = 17  # <availability_zone>
            elif header == 'launch_time':
                len = 19  # YYYY/mm/dd HH:MM:SS
            elif header == 'security_groups':
                len = self.__get_max_len_from_security_groups()
            else:
                len = self.__get_max_len(header)
            formats.append('{%s:<%s}' % (i, len + 1))

        return ''.join(formats)

    def __is_non_filter(self):
        options = self.options
        iid = options.instance_id
        name = options.tag_name
        fil = options.filter_name and options.filter_values

        if iid or name or fil:
            return False
        return True

    def ls(self):
        default_columns = [
            'instance_id',
            'instance_type',
            'tag_Name',
            'public_ip_address',
            'private_ip_address',
            'state'
        ]

        other_columns = [
            'availability_zone',
            'image_id',
            'key_name',
            'launch_time',
            'private_dns_name',
            'public_dns_name',
            'security_groups',
        ]

        all_columns = default_columns + other_columns
        if self.options.columns:
            columns = []
            # unique
            for c in self.options.columns.split(','):
                if c not in columns:
                    columns.append(c)
        else:
            columns = default_columns

        headers = [h for h in columns if h in all_columns]

        self.__get_instances()

        i_info = []
        for instance in self.instances:
            line = {}
            for h in headers:
                tag_key = EC2.TAG_RE.match(h)
                if tag_key:
                    val = self.__get_tag_value(instance.tags,
                                               tag_key.groups()[0])
                elif h == 'availability_zone':
                    val = instance.placement['AvailabilityZone']
                elif h == 'launch_time':
                    if instance.launch_time:
                        time_format = '%Y/%m/%d %H:%M:%S'
                        val = instance.launch_time.strftime(time_format)
                elif h == 'security_groups':
                    val = self.__get_security_group_names(
                            instance.security_groups)
                elif h == 'state':
                    val = self.__get_state(instance)
                else:
                    val = getattr(instance, h) or ''
                line[h] = val or '-'

            i_info.append(line)

        list_format = self.__get_list_format(headers)
        header = list_format.format(*headers)

        order_by = self.options.order_by
        if order_by and order_by in headers:
            i_info = sorted(i_info, key=lambda x: x[order_by])
            if self.options.order == 'desc':
                i_info.reverse()

        page = []
        page.append(header)
        page.append('-' * (len(header) - 1))

        for line in i_info:
            l = []
            for k in line:
                l.append(line[k])
            page.append(list_format.format(*l))

        return page

    def images(self):
        columns = [
            'image_id',
            'name',
            'description',
            'creation_date',
            'state'
        ]

        if self.options.image_id:
            image_ids = self.options.image_id.split(',')
            images = self.resource.images.filter(ImageIds=image_ids)
        else:
            owners = ['self']
            images = self.resource.images.filter(Owners=owners)

        i_info = []
        for image in images:
            line = {}
            for column in columns:
                if column == 'state':
                    val = self.__get_state_with_image(image)
                else:
                    val = getattr(image, column)
                line[column] = val or '-'

            i_info.append(line)

        formats = []
        for i, column in enumerate(columns):
            l = len(column)
            for image in images:
                val = getattr(image, column) or ''
                if len(val) > l:
                    l = len(val)

            formats.append('{%s:<%s}' % (i, l + 1))
        list_format = ''.join(formats)
        header = list_format.format(*columns)

        i_info = sorted(i_info, key=lambda x: x['name'])

        page = []
        page.append(header)
        page.append('-' * (len(header) - 1))

        for line in i_info:
            l = []
            for k in line:
                l.append(line[k])
            page.append(list_format.format(*l))

        return page

    def start_instances(self):
        if self.__is_non_filter():
            return ['Filter id or name required.']

        self.options.columns = 'instance_id,tag_Name,state'
        self.__get_instances()

        instance_ids = []
        will_starts = []
        for instance in self.instances:
            will_start = {}
            state = instance.state['Code']
            if state == 80:
                instance_ids.append(instance.instance_id)
                will_start['instance_id'] = instance.instance_id
                will_start['tag_Name'] = self.__get_tag_value(
                    instance.tags, 'Name'
                )
                will_starts.append(will_start)

        if len(instance_ids) == 0:
            return ['Not found stopped instances']

        sorted_instances = sorted(
            will_starts, key=lambda x: x['instance_id']
        )

        startings = self.client.start_instances(
            InstanceIds=instance_ids
            )['StartingInstances']
        sorted_startings = sorted(startings, key=lambda x: x['InstanceId'])

        formats = []
        for i, column in enumerate(['instance_id', 'tag_Name']):
            max_len = len(max(w[column] for w in will_starts))
            formats.append('{%s:<%s}' % (i, max_len + 1))

        headers = ['instance_id', 'tag_Name']
        list_format = ''.join(formats)
        header = list_format.format(*headers)
        page = [header, '-' * (len(header) - 1)]

        for i, starting in enumerate(sorted_startings):
            if starting['CurrentState']['Code'] == 0:
                page.append(list_format.format(
                    sorted_instances[i]['instance_id'],
                    sorted_instances[i]['tag_Name']
                ))

        return page

    def stop_instances(self):
        if self.__is_non_filter():
            return ['Filter id or name required.']

        self.options.columns = 'instance_id,tag_Name,state'
        self.__get_instances()

        instance_ids = []
        will_stops = []
        for instance in self.instances:
            will_stop = {}
            state = instance.state['Code']
            if state == 16:
                instance_ids.append(instance.instance_id)
                will_stop['instance_id'] = instance.instance_id
                will_stop['tag_Name'] = self.__get_tag_value(
                    instance.tags, 'Name'
                )
                will_stops.append(will_stop)

        if len(instance_ids) == 0:
            return ['Not found running instances']

        sorted_instances = sorted(will_stops, key=lambda x: x['instance_id'])

        stoppings = self.client.stop_instances(
            InstanceIds=instance_ids
            )['StoppingInstances']
        sorted_stoppings = sorted(stoppings, key=lambda x: x['InstanceId'])

        formats = []
        for i, column in enumerate(['instance_id', 'tag_Name']):
            max_len = len(max(w[column] for w in will_stops))
            formats.append('{%s:<%s}' % (i, max_len + 1))

        headers = ['instance_id', 'tag_Name']
        list_format = ''.join(formats)
        header = list_format.format(*headers)
        page = [header, '-' * (len(header) - 1)]

        for i, stopping in enumerate(sorted_stoppings):
            if stopping['CurrentState']['Code'] == 64:
                page.append(list_format.format(
                    sorted_instances[i]['instance_id'],
                    sorted_instances[i]['tag_Name']
                ))

        return page

    def terminate_instances(self):
        if self.__is_non_filter():
            return ['Filter id or name required.']

        self.options.columns = 'instance_id,tag_Name,state'
        self.__get_instances()

        instance_ids = []
        will_terminates = []
        for instance in self.instances:
            will_terminate = {}
            state = instance.state['Code']
            if state == 16 or state == 80:
                instance_ids.append(instance.instance_id)
                will_terminate['instance_id'] = instance.instance_id
                will_terminate['tag_Name'] = self.__get_tag_value(
                    instance.tags, 'Name'
                )
                will_terminates.append(will_terminate)

        if len(instance_ids) == 0:
            return ['Not found running or stopped instances']

        sorted_instances = sorted(
            will_terminates, key=lambda x: x['instance_id']
        )

        terminating = self.client.terminate_instances(
            InstanceIds=instance_ids
            )['TerminatingInstances']
        sorted_terminating = sorted(terminating, key=lambda x: x['InstanceId'])

        formats = []
        for i, column in enumerate(['instance_id', 'tag_Name']):
            max_len = len(max(w[column] for w in will_terminates))
            formats.append('{%s:<%s}' % (i, max_len + 1))

        headers = ['instance_id', 'tag_Name']
        list_format = ''.join(formats)
        header = list_format.format(*headers)
        page = [header, '-' * (len(header) - 1)]

        for i, stopping in enumerate(sorted_terminating):
            page.append(list_format.format(
                sorted_instances[i]['instance_id'],
                sorted_instances[i]['tag_Name']
            ))

        return page
<|endoftext|>


<commit_before>from __future__ import print_function
import os
import numpy as np
from astropy.io import fits
from astropy import log

def ph2cogrid(ntemp=50, trange=[10,200], abundances=(1.2e-9,),
              logdensities=(4,5), opr=3,
              deltav=5.0, # km/s
             ):
    """
    Create a model grid with some ``ntemp`` of temperatures in the range
    ``trange``.
    """
    import pyradex

    temperatures=np.linspace(trange[0],trange[1],ntemp)

    # initial density; will be modified later
    density = 1e4

    fortho = opr/(1.+opr)

    R = pyradex.Radex(species='ph2co-h2',
                      abundance=abundances[0],
                      collider_densities={'H2':density},
                      deltav=deltav,
                      column=None,
                      temperature=temperatures[0],
                      )

    Xarr = {}
    for abundance in abundances:
        Xarr[abundance] = {}

        densities = [10**x for x in logdensities]
        ratio1 = {d:[] for d in densities}
        ratio2 = {d:[] for d in densities}
        f1 = {d:[] for d in densities}
        f2 = {d:[] for d in densities}
        f3 = {d:[] for d in densities}

        for density in densities:
            #R.density = {'H2': density}
            R.density = {'oH2':density*fortho,'pH2':density*(1.-fortho)}
            for temperature in temperatures:
                R.temperature = temperature
                R.abundance = abundance
                log.info("niter={0} col={1:0.1f} dens={2:0.1f} tem={3:0.1f} abund={4:0.2g}"
                         .format(R.run_radex(validate_colliders=False,
                                          reload_molfile=False),
                              np.log10(R.column.value),
                              np.log10(R.density['oH2'].value),
                              R.temperature,
                              R.abundance))

                #import ipdb; ipdb.set_trace()

                F1 = R.T_B[2]  # 218.222192 3_0_3
                F2 = R.T_B[12] # 218.760066 3_2_1
                F3 = R.T_B[9]  # 218.475632 3_2_2

                ratio1[density].append(F2/F1)
                ratio2[density].append(F3/F1)
                f3[density].append(F3)
                f2[density].append(F2)
                f1[density].append(F1)
            print()

        f1 = {d:np.array([x.value for x in f1[d]]) for d in densities}
        f2 = {d:np.array([x.value for x in f2[d]]) for d in densities}
        f3 = {d:np.array([x.value for x in f3[d]]) for d in densities}
        ratio1 = {d:np.array(ratio1[d]) for d in densities}
        ratio2 = {d:np.array(ratio2[d]) for d in densities}

        Xarr[abundance] = {'flux1':f1,
                           'flux2':f2,
                           'flux3':f3,
                           'ratio1':ratio1,
                           'ratio2':ratio2}

    return Xarr

class TemperatureMapper(object):
    """
    For lazier evaluation of temperature mapping function
    """
    def __init__(self, trange=[10,300], ntemp=100, logdensities=(4,4.5,5),
                 abundances=(1e-8, 1.2e-9, 1e-10), **kwargs):
        self.trange = trange
        self.ntemp = ntemp
        self.kwargs = kwargs
        self.kwargs['logdensities'] = logdensities
        self.kwargs['abundances'] = abundances

    def init(self):
        self.Xarr = ph2cogrid(trange=self.trange, ntemp=self.ntemp,
                              **self.kwargs)
        self.temperatures = np.linspace(self.trange[0], self.trange[1],
                                        self.ntemp)

    def get_mapper(self, lineid, tmin=np.nan, tmax=np.nan,
                   density=1e4, abundance=1.2e-9):
        if not hasattr(self,'temperatures'):
            self.init()

        rationame = {'321220': 'ratio1',
                     '322221': 'ratio2'}[lineid]

        # ugly hack because ph2co is indexed with floats
        # Use FIXED abundance, FIXED column, FIXED density
        ratios = self.Xarr[abundance][rationame][density]

        def ratio_to_tem(r):
            inds = np.argsort(ratios)
            return np.interp(r, ratios[inds], self.temperatures[inds], tmin,
                             tmax)

        return ratio_to_tem

    def __call__(self, x, lineid='321220', **kwargs):
        return self.get_mapper(lineid, **kwargs)(x)

if 'tm' not in locals():
    tm = TemperatureMapper(trange=[10,300],ntemp=100)
<commit_msg>remove temperature_mapper<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python

import sys

from centinel import experiment_runner
from centinel.serverconnection import ServerConnection
from centinel.utils.colors import bcolors

selection = sys.argv
selection.pop(0)

print bcolors.OKBLUE + 'Starting the exepriments.' + bcolors.ENDC
experiment_runner.run(selection)
print bcolors.OKGREEN + 'Experiments done.' + bcolors.ENDC

print bcolors.OKBLUE + 'Starting the exepriments.' + bcolors.ENDC
serverconn = ServerConnection()
if not serverconn.connect():
    print bcolors.FAIL + 'Results not submitted.' + bcolors.ENDC
    exit(1)

serverconn.sync_results()

serverconn.disconnect()<commit_msg>minor errors<commit_after>#!/usr/bin/env python

import sys

from centinel import experiment_runner
from centinel.serverconnection import ServerConnection
from centinel.utils.colors import bcolors

selection = sys.argv
selection.pop(0)

print bcolors.OKBLUE + 'Starting the exepriments.' + bcolors.ENDC
experiment_runner.run(selection)
print bcolors.OKGREEN + 'Experiments done.' + bcolors.ENDC

print bcolors.OKBLUE + 'Starting sync.' + bcolors.ENDC
serverconn = ServerConnection()
if not serverconn.connect():
    print bcolors.FAIL + 'Results not synced.' + bcolors.ENDC
    exit(1)

serverconn.sync_results()

serverconn.disconnect()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import sys
from web.fetch import Fetcher
from django.core.management.base import BaseCommand
import leancloud
from settings import LEAN_CLOUD_ID, LEAN_CLOUD_SECRET

reload(sys)
sys.setdefaultencoding("utf-8")

class Command(BaseCommand):
    def handle(self, *args, **options):
        leancloud.init(LEAN_CLOUD_ID, LEAN_CLOUD_SECRET)

        query = leancloud.Query('Chairman')

        allDataCompleted = False
        batch = 0
        limit = 1000
        while not allDataCompleted:
            query.limit(limit)
            query.skip(batch * limit)
            query.add_ascending('createdAt')
            resultList = query.find()
            if len(resultList) < limit:
                allDataCompleted = True
                leancloud.Object.destroy_all(resultList)
            batch += 1

        fetcher = Fetcher()
        fetcher.fetch_cc()
        fetcher.fetch_douyu()
        fetcher.fetch_longzhu()
        # fetcher.fetch_quanmin()
        fetcher.fetch_xiongmao()
        fetcher.fetch_zhanqi()
        fetcher.fetch_huya()

        for chairman in fetcher.chairmans:
            try:
                chairman.save()
            except Exception, e:
                print e
<commit_msg>comment quanmin<commit_after># -*- coding: utf-8 -*-
import sys
from web.fetch import Fetcher
from django.core.management.base import BaseCommand
import leancloud
from settings import LEAN_CLOUD_ID, LEAN_CLOUD_SECRET

reload(sys)
sys.setdefaultencoding("utf-8")

class Command(BaseCommand):
    def handle(self, *args, **options):
        try:
            leancloud.init(LEAN_CLOUD_ID, LEAN_CLOUD_SECRET)

            query = leancloud.Query('Chairman')

            allDataCompleted = False
            batch = 0
            limit = 1000
            while not allDataCompleted:
                query.limit(limit)
                query.skip(batch * limit)
                query.add_ascending('createdAt')
                resultList = query.find()
                if len(resultList) < limit:
                    allDataCompleted = True
                    leancloud.Object.destroy_all(resultList)
                batch += 1

            fetcher = Fetcher()
            fetcher.fetch_cc()
            fetcher.fetch_douyu()
            fetcher.fetch_longzhu()
            # fetcher.fetch_quanmin()
            fetcher.fetch_xiongmao()
            fetcher.fetch_zhanqi()
            fetcher.fetch_huya()

            for chairman in fetcher.chairmans:
                chairman.save()
        except Exception, e:
            print e
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

#This software is distributed under the Creative Commons license (CC0) version 1.0. A copy of this license should have been distributed with this software.
#The license can also be read online: <https://creativecommons.org/publicdomain/zero/1.0/>. If this online license differs from the license provided with this software, the license provided with this software should be applied.

"""
Provides the implementation of real numbers as a data type definition.

Real numbers are serialised using the JSON format for floating point numbers.
"""

import io #To decode UTF-8 streaming.
import json #We're using JSON to serialise floating point numbers in a readable format quickly.

import luna.plugins #To access the data API for raising SerialisationExceptions.
import luna.stream #To provide the serialised streams in their expected format.

def deserialise(serialised):
	"""
	Interprets a sequence of bytes that represents a real number.
	:param serialised: A sequence of bytes that represents a real number.
	:return: The real number that was being represented by the sequence of
	bytes.
	"""
	try:
		instance = json.load(io.TextIOWrapper(serialised, encoding="utf_8"))
	except UnicodeDecodeError as e:
		raise luna.plugins.api("data").SerialisationException("The serialised sequence is not proper UTF-8, so it doesn't represent a real number.") from e
	except json.JSONDecodeError as e:
		raise luna.plugins.api("data").SerialisationException("The serialised sequence does not represent a JSON-format document and therefore doesn't represent a real number.") from e
	if type(instance) != float:
		raise luna.plugins.api("data").SerialisationException("The serialised sequence does not represent a real number, but an instance of type {instance_type}.".format(instance_type=type(instance)))
	return instance

def is_instance(instance):
	"""
	Detects whether some object is a real number.
	:param instance: The instance of which to check whether it is a real number.
	:return: ``True`` if the object is a real number, or ``False`` if it isn't.
	"""
	return type(instance) == float

def is_serialised(serialised):
	"""
	Detects whether a byte stream represents a real number.
	:param serialised: A byte stream which must be identified as being a real
	number or not.
	:return: ``True`` if the stream likely represents a real number, or
	``False`` if it does not.
	"""
	#This works with a simple finite state automaton in a linear fashion:
	#initial -> integer_start -> integer -> fractional_start -> fractional -> exponent_initial -> exponent_start -> exponent
	#Each state represents what character is expected next.
	#The FSA solution is not pretty, but it's the only way it could be made to work with a possibly infinite byte stream.
	state = "initial"
	for byte in serialised:
		if state == "initial": #Initial state: May be a negative sign or integer_start.
			if byte == b"-"[0]:
				state = "integer_start"
			elif byte >= b"0"[0] and byte <= b"9"[0]:
				state = "integer"
			else:
				return False
		elif state == "integer_start": #First character of integer. An integer must have at least 1 digit.
			if byte >= b"0"[0] and byte <= b"9"[0]:
				state = "integer"
			else:
				return False
		elif state == "integer": #Consecutive characters of the integer. May be a period, indicating start of fractional, or an E, indicating start of exponent.
			if byte >= b"0"[0] and byte <= b"9"[0]:
				pass #Still integer.
			if byte == b"."[0]:
				state = "fractional_start"
			elif byte == b"e"[0] or byte == b"E"[0]:
				state = "exponent_initial"
			else:
				return False
		elif state == "fractional_start": #Start of fractional part.
			if byte >= b"0"[0] and byte <= b"9"[0]:
				state = "fractional"
			else:
				return False
		elif state == "fractional": #Continuation of factional part. May be an E, indicating start of exponent.
			if byte >= b"0"[0] and byte <= b"9"[0]:
				pass #Still fractional part.
			elif byte == b"e"[0] or byte == b"E"[0]:
				state = "exponent_initial"
			else:
				return False
		elif state == "exponent_initial": #Initial state of exponent, may be negative or a number.
			if byte == b"-"[0]:
				state = "exponent_start"
			elif byte >= b"0"[0] and byte <= b"9"[0]:
				state = "exponent"
			else:
				return False
		elif state == "exponent_start": #First character of an exponent. Not an end state.
			if byte >= b"0"[0] and byte <= b"9"[0]:
				state = "exponent"
			else:
				return False
		elif state == "exponent": #Continuation of an exponent.
			if byte >= b"0"[0] and byte <= b"9"[0]:
				pass #Still exponent.
			else:
				return False
	return state == "fractional" or state == "exponent" #Allowable end states.

def serialise(instance):
	"""
	Serialises a real number to a sequence of bytes.
	:param instance: The real number to serialise.
	:return: A sequence of bytes representing the real number.
	"""
	try:
		output = luna.stream.BytesStreamReader(json.dumps(instance).encode("utf_8"))
	except TypeError as e:
		raise luna.plugins.api("data").SerialisationException("Trying to serialise an object that is not a real number.") from e
	return output<commit_msg>Fix parsing integer part in is_serialised<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

#This software is distributed under the Creative Commons license (CC0) version 1.0. A copy of this license should have been distributed with this software.
#The license can also be read online: <https://creativecommons.org/publicdomain/zero/1.0/>. If this online license differs from the license provided with this software, the license provided with this software should be applied.

"""
Provides the implementation of real numbers as a data type definition.

Real numbers are serialised using the JSON format for floating point numbers.
"""

import io #To decode UTF-8 streaming.
import json #We're using JSON to serialise floating point numbers in a readable format quickly.

import luna.plugins #To access the data API for raising SerialisationExceptions.
import luna.stream #To provide the serialised streams in their expected format.

def deserialise(serialised):
	"""
	Interprets a sequence of bytes that represents a real number.
	:param serialised: A sequence of bytes that represents a real number.
	:return: The real number that was being represented by the sequence of
	bytes.
	"""
	try:
		instance = json.load(io.TextIOWrapper(serialised, encoding="utf_8"))
	except UnicodeDecodeError as e:
		raise luna.plugins.api("data").SerialisationException("The serialised sequence is not proper UTF-8, so it doesn't represent a real number.") from e
	except json.JSONDecodeError as e:
		raise luna.plugins.api("data").SerialisationException("The serialised sequence does not represent a JSON-format document and therefore doesn't represent a real number.") from e
	if type(instance) != float:
		raise luna.plugins.api("data").SerialisationException("The serialised sequence does not represent a real number, but an instance of type {instance_type}.".format(instance_type=type(instance)))
	return instance

def is_instance(instance):
	"""
	Detects whether some object is a real number.
	:param instance: The instance of which to check whether it is a real number.
	:return: ``True`` if the object is a real number, or ``False`` if it isn't.
	"""
	return type(instance) == float

def is_serialised(serialised):
	"""
	Detects whether a byte stream represents a real number.
	:param serialised: A byte stream which must be identified as being a real
	number or not.
	:return: ``True`` if the stream likely represents a real number, or
	``False`` if it does not.
	"""
	#This works with a simple finite state automaton in a linear fashion:
	#initial -> integer_start -> integer -> fractional_start -> fractional -> exponent_initial -> exponent_start -> exponent
	#Each state represents what character is expected next.
	#The FSA solution is not pretty, but it's the only way it could be made to work with a possibly infinite byte stream.
	state = "initial"
	for byte in serialised:
		if state == "initial": #Initial state: May be a negative sign or integer_start.
			if byte == b"-"[0]:
				state = "integer_start"
			elif byte >= b"0"[0] and byte <= b"9"[0]:
				state = "integer"
			else:
				return False
		elif state == "integer_start": #First character of integer. An integer must have at least 1 digit.
			if byte >= b"0"[0] and byte <= b"9"[0]:
				state = "integer"
			else:
				return False
		elif state == "integer": #Consecutive characters of the integer. May be a period, indicating start of fractional, or an E, indicating start of exponent.
			if byte >= b"0"[0] and byte <= b"9"[0]:
				pass #Still integer.
			elif byte == b"."[0]:
				state = "fractional_start"
			elif byte == b"e"[0] or byte == b"E"[0]:
				state = "exponent_initial"
			else:
				return False
		elif state == "fractional_start": #Start of fractional part.
			if byte >= b"0"[0] and byte <= b"9"[0]:
				state = "fractional"
			else:
				return False
		elif state == "fractional": #Continuation of factional part. May be an E, indicating start of exponent.
			if byte >= b"0"[0] and byte <= b"9"[0]:
				pass #Still fractional part.
			elif byte == b"e"[0] or byte == b"E"[0]:
				state = "exponent_initial"
			else:
				return False
		elif state == "exponent_initial": #Initial state of exponent, may be negative or a number.
			if byte == b"-"[0]:
				state = "exponent_start"
			elif byte >= b"0"[0] and byte <= b"9"[0]:
				state = "exponent"
			else:
				return False
		elif state == "exponent_start": #First character of an exponent. Not an end state.
			if byte >= b"0"[0] and byte <= b"9"[0]:
				state = "exponent"
			else:
				return False
		elif state == "exponent": #Continuation of an exponent.
			if byte >= b"0"[0] and byte <= b"9"[0]:
				pass #Still exponent.
			else:
				return False
	return state == "fractional" or state == "exponent" #Allowable end states.

def serialise(instance):
	"""
	Serialises a real number to a sequence of bytes.
	:param instance: The real number to serialise.
	:return: A sequence of bytes representing the real number.
	"""
	try:
		output = luna.stream.BytesStreamReader(json.dumps(instance).encode("utf_8"))
	except TypeError as e:
		raise luna.plugins.api("data").SerialisationException("Trying to serialise an object that is not a real number.") from e
	return output<|endoftext|>


<commit_before># coding: utf-8
from pupa.scrape import Scraper

from utils import lxmlize, CanadianLegislator as Legislator

import re

COUNCIL_PAGE = 'http://nslegislature.ca/index.php/people/members/'

PARTIES = {
    'Liberal': 'Nova Scotia Liberal Party',
    'PC': 'Progressive Conservative Association of Nova Scotia',
    'NDP': 'Nova Scotia New Democratic Party'
}

def get_party(abbreviation):
  """Return a political party based on party abbreviation"""
  return PARTIES[abbreviation]

class NovaScotiaPersonScraper(Scraper):

  def get_people(self):
    page = lxmlize(COUNCIL_PAGE)
    for row in page.xpath('//div[@id="content"]/table/tbody/tr'):
      full_name, party_abbr, post = row.xpath('./td//text()')[:3]
      name = ' '.join(reversed(full_name.split(',')))
      detail_url = row[0][0].attrib['href']
      image, phone = get_details(detail_url)
      p = Legislator(name=name, post_id=post, role='MLA',
          party=get_party(party_abbr), image=image)
      p.add_source(COUNCIL_PAGE)
      p.add_source(detail_url)
      p.add_contact('voice', phone, 'legislature')
      yield p

def get_details(url):
  page = lxmlize(url)
  image = page.xpath('string(//img[@class="portrait"]/@src)')
  # this is generated by JS, grrr
  #email = page.xpath('string(//span[contains(@id, "eeEncEmail")])')
  phone = page.xpath('string(//dd[@class="numbers"]/text())').split(': ')[1]
  return image, phone

<commit_msg>ca_ns emails<commit_after># coding: utf-8
from pupa.scrape import Scraper

from utils import lxmlize, CanadianLegislator as Legislator

import re

COUNCIL_PAGE = 'http://nslegislature.ca/index.php/people/members/'

PARTIES = {
    'Liberal': 'Nova Scotia Liberal Party',
    'PC': 'Progressive Conservative Association of Nova Scotia',
    'NDP': 'Nova Scotia New Democratic Party'
}

def get_party(abbreviation):
  """Return a political party based on party abbreviation"""
  return PARTIES[abbreviation]

class NovaScotiaPersonScraper(Scraper):

  def get_people(self):
    page = lxmlize(COUNCIL_PAGE)
    for row in page.xpath('//div[@id="content"]/table/tbody/tr'):
      full_name, party_abbr, post = row.xpath('./td//text()')[:3]
      name = ' '.join(reversed(full_name.split(',')))
      detail_url = row[0][0].attrib['href']
      image, phone, email = get_details(detail_url)
      p = Legislator(name=name, post_id=post, role='MLA',
          party=get_party(party_abbr), image=image)
      p.add_source(COUNCIL_PAGE)
      p.add_source(detail_url)
      p.add_contact('voice', phone, 'legislature')
      p.add_contact('email', email, None)
      yield p

def get_details(url):
  page = lxmlize(url)
  image = page.xpath('string(//img[@class="portrait"]/@src)')
  phone = page.xpath('string(//dd[@class="numbers"]/text())').split(': ')[1]
  email_js = page.xpath('string(//dd/script)')
  email_addr = process_email(email_js)
  return image, phone, email_addr

def process_email(js):
  charcodes = reversed(re.findall(r"]='(.+?)'", js))
  def convert_char(code):
    try:
      return chr(int(code))
    except ValueError:
      return code
  content = ''.join(convert_char(code) for code in charcodes)
  return re.search(r'>(.+)<', content).group(1)

<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
$Id$

This file is part of the sqlmap project, http://sqlmap.sourceforge.net.

Copyright (c) 2007-2010 Bernardo Damele A. G. <bernardo.damele@gmail.com>
Copyright (c) 2006 Daniele Bellucci <daniele.bellucci@gmail.com>

sqlmap is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free
Software Foundation version 2 of the License.

sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
details.

You should have received a copy of the GNU General Public License along
with sqlmap; if not, write to the Free Software Foundation, Inc., 51
Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
"""

try:
    import pyodbc
except ImportError, _:
    pass

from lib.core.data import conf
from lib.core.data import logger
from lib.core.exception import sqlmapConnectionException
from lib.core.exception import sqlmapUnsupportedFeatureException
from lib.core.settings import IS_WIN

from plugins.generic.connector import Connector as GenericConnector

class Connector(GenericConnector):
    """
    Homepage: http://pyodbc.googlecode.com/
    User guide: http://code.google.com/p/pyodbc/wiki/GettingStarted
    API: http://code.google.com/p/pyodbc/w/list
    Debian package: python-pyodbc
    License: MIT
    """

    def __init__(self):
        GenericConnector.__init__(self)

    def connect(self):
        if not IS_WIN:
            errMsg  = "currently, direct connection to Microsoft Access database(s) "
            errMsg += "is restricted to Windows platforms"
            raise sqlmapUnsupportedFeatureException, errMsg

        self.initConnection()
        self.checkFileDb()

        try:
            self.connector = pyodbc.connect('Driver={Microsoft Access Driver (*.mdb)};Dbq=%s;Uid=Admin;Pwd=;' % self.db)
        except (pyodbc.Error, pyodbc.OperationalError), msg:
            raise sqlmapConnectionException, msg[1]

        self.setCursor()
        self.connected()

    def fetchall(self):
        try:
            return self.cursor.fetchall()
        except pyodbc.ProgrammingError, msg:
            logger.log(8, msg[1])
            return None

    def execute(self, query):
        try:
            self.cursor.execute(query)
        except (pyodbc.OperationalError, pyodbc.ProgrammingError), msg:
            logger.log(8, msg[1])
        except pyodbc.Error, msg:
            raise sqlmapConnectionException, msg[1]

        self.connector.commit()

    def select(self, query):
        self.execute(query)
        return self.fetchall()
<commit_msg>just a test<commit_after>#!/usr/bin/env python

"""
$Id$

Copyright (c) 2006-2010 sqlmap developers (http://sqlmap.sourceforge.net/)
See the file doc/COPYING for copying permission.
"""

try:
    import pyodbc
except ImportError, _:
    pass

from lib.core.data import conf
from lib.core.data import logger
from lib.core.exception import sqlmapConnectionException
from lib.core.exception import sqlmapUnsupportedFeatureException
from lib.core.settings import IS_WIN

from plugins.generic.connector import Connector as GenericConnector

class Connector(GenericConnector):
    """
    Homepage: http://pyodbc.googlecode.com/
    User guide: http://code.google.com/p/pyodbc/wiki/GettingStarted
    API: http://code.google.com/p/pyodbc/w/list
    Debian package: python-pyodbc
    License: MIT
    """

    def __init__(self):
        GenericConnector.__init__(self)

    def connect(self):
        if not IS_WIN:
            errMsg  = "currently, direct connection to Microsoft Access database(s) "
            errMsg += "is restricted to Windows platforms"
            raise sqlmapUnsupportedFeatureException, errMsg

        self.initConnection()
        self.checkFileDb()

        try:
            self.connector = pyodbc.connect('Driver={Microsoft Access Driver (*.mdb)};Dbq=%s;Uid=Admin;Pwd=;' % self.db)
        except (pyodbc.Error, pyodbc.OperationalError), msg:
            raise sqlmapConnectionException, msg[1]

        self.setCursor()
        self.connected()

    def fetchall(self):
        try:
            return self.cursor.fetchall()
        except pyodbc.ProgrammingError, msg:
            logger.log(8, msg[1])
            return None

    def execute(self, query):
        try:
            self.cursor.execute(query)
        except (pyodbc.OperationalError, pyodbc.ProgrammingError), msg:
            logger.log(8, msg[1])
        except pyodbc.Error, msg:
            raise sqlmapConnectionException, msg[1]

        self.connector.commit()

    def select(self, query):
        self.execute(query)
        return self.fetchall()
<|endoftext|>


<commit_before>import datetime as dt
import pickle
import random
from collections import namedtuple

import pandas as pd

from caar.configparser_read import INSIDE_DEVICE_ID,                         \
    INSIDE_LOG_DATE, INSIDE_DEGREES, CYCLE_DEVICE_ID, CYCLE_START_TIME,      \
    CYCLE_END_TIME, OUTSIDE_LOCATION_ID, OUTSIDE_LOG_DATE, OUTSIDE_DEGREES

Cycle = namedtuple('Cycle', ['thermo_id', 'cycle_mode', 'start_time'])
Inside = namedtuple('Inside', ['thermo_id', 'log_date'])
Outside = namedtuple('Outside', ['location_id', 'log_date'])

def create_inside_df(dict_or_pickle_file, thermo_ids=None):
    """Returns pandas DataFrame containing thermostat id, time stamps and
    inside temperatures at the time of cooling (or heating) cycles starting
    and ending.
    """
    multi_ids, vals = _records_as_lists_of_tuples(dict_or_pickle_file,
                                                  'thermo_id', 'log_date',
                                                  ids=thermo_ids)
    inside_df = _create_multi_index_df([INSIDE_DEVICE_ID, INSIDE_LOG_DATE], multi_ids,
                                       [INSIDE_DEGREES], vals)
    return inside_df

def create_cycles_df(dict_or_pickle_file, thermo_ids=None):
    """Returns pandas DataFrame containing thermostat ids, time stamps
    and indoor temperatures.
    """
    multi_ids, vals = _records_as_lists_of_tuples(dict_or_pickle_file,
                                                  'thermo_id', 'start_time',
                                                  ids=thermo_ids)
    cycles_df = _create_multi_index_df([CYCLE_DEVICE_ID, CYCLE_START_TIME],
                                       multi_ids, [CYCLE_END_TIME], vals)
    return cycles_df

def create_outside_df(dict_or_pickle_file, location_ids=None):
    """Returns pandas DataFrame containing location ids, time stamps and
    outdoor temperatures.
    """
    multi_ids, vals = _records_as_lists_of_tuples(dict_or_pickle_file,
                                                  'location_id', 'log_date',
                                                  ids=location_ids)
    outside_df = _create_multi_index_df([OUTSIDE_LOCATION_ID, OUTSIDE_LOG_DATE],
                                        multi_ids, [OUTSIDE_DEGREES], vals)
    return outside_df

def _records_as_lists_of_tuples(dict_or_pickle_file, id_field, time_field, ids=None):
    """Returns tuple containing
    1) a list of named tuples containing thermostat (or outdoor location) ids
    and time stamps and
    2) a list of either indoor (or outdoor) temperatures, or the ending time
    of a cycle, based on input of a pickle file containing a dict.
    """
    if isinstance(dict_or_pickle_file, dict):
        records = dict_or_pickle_file
    else:
        try:
            with open(dict_or_pickle_file, 'rb') as cp:
                records = pickle.load(cp)
        except ValueError:
            print('The first argument must be a pickle file or dict.')

    record_keys_copy = list(records.keys())
    random_record_key = random.choice(record_keys_copy)
    random_record = records[random_record_key]
    data_type = _determine_if_temperature_or_time_data(random_record)
    if ids is not None:
        for record_key in record_keys_copy:
            # Discard record if it is not among the desired ids.
            if getattr(record_key, id_field) not in ids:
                records.pop(record_key, None)
    multi_ids = []
    vals = []
    # inside and outside temperatures have temperature data
    if data_type == 'temperature':
        multi_ids, vals = _temps_multi_ids(records, id_field, time_field)
    # time_stamp data (the data value detected is end time) is associated
    # with cycling data
    elif data_type == 'time_stamp':
        multi_ids, vals = _cycles_multi_ids(records, id_field, time_field)
    return (multi_ids, vals)

def _determine_if_temperature_or_time_data(record_data):
    """Returns 'temperature' or 'time' (a string) based on the content of the
    value in a dict item.
    """
    if isinstance(record_data, tuple):
        try:
            assert isinstance(_datetime_from_string(record_data[0]), dt.datetime)
        except ValueError:
            print('The data field from the tuple does not match the given time format.')
        else:
            return 'time_stamp'
    else:
        try:
            isinstance(record_data, str)
        except ValueError:
            print('The data field is not a string within a tuple or a string.')
        else:
            return 'temperature'

def _temps_multi_ids(records, id_field, time_field):
    """Returns tuple containing
    1) a list of named tuples containing thermostat ids and time stamps and
    2) a list of either indoor (or outdoor) temperatures, based on items
    (records) in a dict.
    """
    multi_ids = []
    vals = []
    for k, v in records.items():
        record_id, time_stamp = [getattr(k, f) for f in [id_field, time_field]]
        time = _datetime_from_string(time_stamp)
        multi_ids.append(tuple([record_id, time]))
        temperature = int(v)
        vals.append(temperature)
    return (multi_ids, vals)

def _cycles_multi_ids(records, id_field, time_field):
    """Returns tuple containing
    1) a list of named tuples containing outdoor location ids and time stamps and
    2) a list of ending times of cycles, based on a dict.
    """
    multi_ids = []
    vals = []
    for k, v in records.items():
        record_id, time_stamp = [getattr(k, f) for f in [id_field, time_field]]
        time = _datetime_from_string(time_stamp)
        multi_ids.append(tuple([record_id, time]))
        end_time = _datetime_from_string(v[0])
        vals.append(end_time)
    return (multi_ids, vals)

def _datetime_from_string(time_string):
    return dt.datetime.strptime(time_string, '%Y-%m-%d %H:%M:%S')

def _create_multi_index_df(multiindex_names, multi_ids, column_names, values):
    """Returns MultiIndex pandas dataframe in which the index columns are for
    an id and time stamp and the value is for a temperature or a time stamp
    indicating the end of a cycle.
    """
    multiindex_columns = tuple(multiindex_names)
    multicols = pd.MultiIndex.from_tuples(multi_ids, names=multiindex_columns)
    df = pd.DataFrame(values, index=multicols, columns=column_names)
    df.sortlevel(inplace=True, sort_remaining=True)
    return df
<commit_msg>Updated documentation of dataframe creation functions<commit_after>import datetime as dt
import pickle
import random
from collections import namedtuple

import pandas as pd

from caar.configparser_read import INSIDE_DEVICE_ID,                         \
    INSIDE_LOG_DATE, INSIDE_DEGREES, CYCLE_DEVICE_ID, CYCLE_START_TIME,      \
    CYCLE_END_TIME, OUTSIDE_LOCATION_ID, OUTSIDE_LOG_DATE, OUTSIDE_DEGREES

Cycle = namedtuple('Cycle', ['thermo_id', 'cycle_mode', 'start_time'])
Inside = namedtuple('Inside', ['thermo_id', 'log_date'])
Outside = namedtuple('Outside', ['location_id', 'log_date'])

def create_inside_df(dict_or_pickle_file, thermo_ids=None):
    """Returns pandas DataFrame containing thermostat id, time stamps and
    inside temperatures at the time of cooling (or heating) cycles starting
    and ending.

    Args:
        dict_or_pickle_file (dict or str): Must be created with dict_from_file() or pickle_from_file() function.

        thermo_ids (Optional: list of ints): Thermostat IDs. If no argument is specified, all IDs from the first arg will be in the DataFrame.

    Returns:
        inside_idf (pandas DataFrame): DataFrame has multi-index based on the ID(s) and timestamps.
    """
    multi_ids, vals = _records_as_lists_of_tuples(dict_or_pickle_file,
                                                  'thermo_id', 'log_date',
                                                  ids=thermo_ids)
    inside_df = _create_multi_index_df([INSIDE_DEVICE_ID, INSIDE_LOG_DATE], multi_ids,
                                       [INSIDE_DEGREES], vals)
    return inside_df

def create_cycles_df(dict_or_pickle_file, thermo_ids=None):
    """Returns pandas DataFrame containing thermostat ids and cycle beginning time stamps as multi-part indexes, and
    cycle ending times as values.

    Args:
        dict_or_pickle_file (dict or str): Must be created with dict_from_file() or pickle_from_file() function.

        thermo_ids (Optional: list of ints): Thermostat IDs. If no  argument is specified, all IDs from the first arg will be in the DataFrame.

    Returns:
        cycles_df (pandas DataFrame): DataFrame has multi-index based on the ID(s) and timestamps.
    """
    multi_ids, vals = _records_as_lists_of_tuples(dict_or_pickle_file,
                                                  'thermo_id', 'start_time',
                                                  ids=thermo_ids)
    cycles_df = _create_multi_index_df([CYCLE_DEVICE_ID, CYCLE_START_TIME],
                                       multi_ids, [CYCLE_END_TIME], vals)
    return cycles_df

def create_outside_df(dict_or_pickle_file, location_ids=None):
    """Returns pandas DataFrame containing records with location ids and time stamps as multi-part indexes and
    outdoor temperatures as values.

    Args:
        dict_or_pickle_file (dict or str): Must be created with dict_from_file() or pickle_from_file() function.

        location_ids (Optional: list of ints): If no Location ID(s) argument is specified, all IDs from the first arg will be in the DataFrame.

    Returns:
        outside_df (pandas DataFrame): DataFrame has multi-index based on the ID(s) and timestamps.
    """
    multi_ids, vals = _records_as_lists_of_tuples(dict_or_pickle_file,
                                                  'location_id', 'log_date',
                                                  ids=location_ids)
    outside_df = _create_multi_index_df([OUTSIDE_LOCATION_ID, OUTSIDE_LOG_DATE],
                                        multi_ids, [OUTSIDE_DEGREES], vals)
    return outside_df

def _records_as_lists_of_tuples(dict_or_pickle_file, id_field, time_field, ids=None):
    """Returns tuple containing
    1) a list of named tuples containing thermostat (or outdoor location) ids
    and time stamps and
    2) a list of either indoor (or outdoor) temperatures, or the ending time
    of a cycle, based on input of a pickle file containing a dict.
    """
    if isinstance(dict_or_pickle_file, dict):
        records = dict_or_pickle_file
    else:
        try:
            with open(dict_or_pickle_file, 'rb') as cp:
                records = pickle.load(cp)
        except ValueError:
            print('The first argument must be a pickle file or dict.')

    record_keys_copy = list(records.keys())
    random_record_key = random.choice(record_keys_copy)
    random_record = records[random_record_key]
    data_type = _determine_if_temperature_or_time_data(random_record)
    if ids is not None:
        for record_key in record_keys_copy:
            # Discard record if it is not among the desired ids.
            if getattr(record_key, id_field) not in ids:
                records.pop(record_key, None)
    multi_ids = []
    vals = []
    # inside and outside temperatures have temperature data
    if data_type == 'temperature':
        multi_ids, vals = _temps_multi_ids(records, id_field, time_field)
    # time_stamp data (the data value detected is end time) is associated
    # with cycling data
    elif data_type == 'time_stamp':
        multi_ids, vals = _cycles_multi_ids(records, id_field, time_field)
    return (multi_ids, vals)

def _determine_if_temperature_or_time_data(record_data):
    """Returns 'temperature' or 'time' (a string) based on the content of the
    value in a dict item.
    """
    if isinstance(record_data, tuple):
        try:
            assert isinstance(_datetime_from_string(record_data[0]), dt.datetime)
        except ValueError:
            print('The data field from the tuple does not match the given time format.')
        else:
            return 'time_stamp'
    else:
        try:
            isinstance(record_data, str)
        except ValueError:
            print('The data field is not a string within a tuple or a string.')
        else:
            return 'temperature'

def _temps_multi_ids(records, id_field, time_field):
    """Returns tuple containing
    1) a list of named tuples containing thermostat ids and time stamps and
    2) a list of either indoor (or outdoor) temperatures, based on items
    (records) in a dict.
    """
    multi_ids = []
    vals = []
    for k, v in records.items():
        record_id, time_stamp = [getattr(k, f) for f in [id_field, time_field]]
        time = _datetime_from_string(time_stamp)
        multi_ids.append(tuple([record_id, time]))
        temperature = int(v)
        vals.append(temperature)
    return (multi_ids, vals)

def _cycles_multi_ids(records, id_field, time_field):
    """Returns tuple containing
    1) a list of named tuples containing outdoor location ids and time stamps and
    2) a list of ending times of cycles, based on a dict.
    """
    multi_ids = []
    vals = []
    for k, v in records.items():
        record_id, time_stamp = [getattr(k, f) for f in [id_field, time_field]]
        time = _datetime_from_string(time_stamp)
        multi_ids.append(tuple([record_id, time]))
        end_time = _datetime_from_string(v[0])
        vals.append(end_time)
    return (multi_ids, vals)

def _datetime_from_string(time_string):
    return dt.datetime.strptime(time_string, '%Y-%m-%d %H:%M:%S')

def _create_multi_index_df(multiindex_names, multi_ids, column_names, values):
    """Returns MultiIndex pandas dataframe in which the index columns are for
    an id and time stamp and the value is for a temperature or a time stamp
    indicating the end of a cycle.
    """
    multiindex_columns = tuple(multiindex_names)
    multicols = pd.MultiIndex.from_tuples(multi_ids, names=multiindex_columns)
    df = pd.DataFrame(values, index=multicols, columns=column_names)
    df.sortlevel(inplace=True, sort_remaining=True)
    return df
<|endoftext|>