<commit_before>import imaplib
import io
import re
import email
import chardet
import base64
import quopri
import time
from datetime import datetime
from email.header import decode_header
from imbox.utils import str_encode, str_decode

import logging

logger = logging.getLogger(__name__)

class Struct:
    def __init__(self, **entries):
        self.__dict__.update(entries)

    def keys(self):
        return self.__dict__.keys()

    def __repr__(self):
        return str(self.__dict__)

def decode_mail_header(value, default_charset='us-ascii'):
    """
    Decode a header value into a unicode string.
    """
    try:
        headers = decode_header(value)
    except email.errors.HeaderParseError:
        return str_decode(str_encode(value, default_charset, 'replace'), default_charset)
    else:
        for index, (text, charset) in enumerate(headers):
            try:
                logger.debug("Mail header no. {index}: {data} encoding {charset}".format(
                    index=index,
                    data=str_decode(text, charset or 'utf-8', 'replace'),
                    charset=charset))
                headers[index] = str_decode(text, charset or default_charset,
                                            'replace')
            except LookupError:
                # if the charset is unknown, force default
                headers[index] = str_decode(text, default_charset, 'replace')

        return ''.join(headers)

def get_mail_addresses(message, header_name):
    """
    Retrieve all email addresses from one message header.
    """
    headers = [h for h in message.get_all(header_name, [])]
    addresses = email.utils.getaddresses(headers)

    for index, (address_name, address_email) in enumerate(addresses):
        addresses[index] = {'name': decode_mail_header(address_name),
                            'email': address_email}
        logger.debug("{} Mail address in message: <{}> {}".format(
            header_name.upper(), address_name, address_email))
    return addresses

def decode_param(param):
    name, v = param.split('=', 1)
    values = v.split('\n')
    value_results = []
    for value in values:
        match = re.findall(r'=\?((?:\w|-)+)\?([QB])\?(.+?)\?=', value)
        if match:
            for encoding, type_, code in match:
                if type_ == 'Q':
                    value = quopri.decodestring(code)
                elif type_ == 'B':
                    value = code.encode()
                    missing_padding = len(value) % 4

                    if missing_padding:
                        value += b"=" * (4 - missing_padding)

                value = str_encode(value, encoding)

            value_results.append(value)

            if value_results:
                v = ''.join(value_results)

    logger.debug("Decoded parameter {} - {}".format(name, v))
    return name, v

def parse_attachment(message_part):
    # Check again if this is a valid attachment
    content_disposition = message_part.get("Content-Disposition", None)
    if content_disposition is not None and not message_part.is_multipart():
        dispositions = [
            disposition.strip()
            for disposition in content_disposition.split(";")
            if disposition.strip()
        ]

        if dispositions[0].lower() in ["attachment", "inline"]:
            file_data = message_part.get_payload(decode=True)

            attachment = {
                'content-type': message_part.get_content_type(),
                'size': len(file_data),
                'content': io.BytesIO(file_data),
                'content-id': message_part.get("Content-ID", None)
            }
            filename = message_part.get_param('name')
            filename_parts = []
            for param in dispositions[1:]:
                if param:
                    name, value = decode_param(param)

                    # Check for split filename
                    s_name = name.rstrip('*').split("*")
                    if s_name[0] == 'filename':
                        # If this is a split file name - use the number after the * as an index to insert this part
                        if len(s_name) > 1 and s_name[1] != '':
                            filename_parts.insert(int(s_name[1]),value[1:-1] if value.startswith('"') else value)
                        else:
                            filename_parts.insert(0,value[1:-1] if value.startswith('"') else value)

                    if 'create-date' in name:
                        attachment['create-date'] = value

            attachment['filename'] = "".join(filename_parts)
            return attachment

    return None

def decode_content(message):
    content = message.get_payload(decode=True)
    charset = message.get_content_charset('utf-8')
    try:
        return content.decode(charset, 'ignore')
    except LookupError:
        encoding = chardet.detect(content).get('encoding')
        if encoding:
            return content.decode(encoding, 'ignore')
        return content
    except AttributeError:
        return content

def fetch_email_by_uid(uid, connection, parser_policy):
    message, data = connection.uid('fetch', uid, '(BODY.PEEK[] FLAGS)')
    logger.debug("Fetched message for UID {}".format(int(uid)))

    raw_headers = data[0][0] + data[1]
    raw_email = data[0][1]

    email_object = parse_email(raw_email, policy=parser_policy)
    flags = parse_flags(raw_headers.decode())
    email_object.__dict__['flags'] = flags

    return email_object

def parse_flags(headers):
    """Copied from https://github.com/girishramnani/gmail/blob/master/gmail/message.py"""
    if len(headers) == 0:
        return []
    headers = bytes(headers, "ascii")
    return list(imaplib.ParseFlags(headers))

def parse_email(raw_email, policy=None):
    if isinstance(raw_email, bytes):
        raw_email = str_encode(raw_email, 'utf-8', errors='ignore')
    if policy is not None:
        email_parse_kwargs = dict(policy=policy)
    else:
        email_parse_kwargs = {}

    try:
        email_message = email.message_from_string(
            raw_email, **email_parse_kwargs)
    except UnicodeEncodeError:
        email_message = email.message_from_string(
            raw_email.encode('utf-8'), **email_parse_kwargs)
    maintype = email_message.get_content_maintype()
    parsed_email = {'raw_email': raw_email}

    body = {
        "plain": [],
        "html": []
    }
    attachments = []

    if maintype in ('multipart', 'image'):
        logger.debug("Multipart message. Will process parts.")
        for part in email_message.walk():
            content_type = part.get_content_type()
            part_maintype = part.get_content_maintype()
            content_disposition = part.get('Content-Disposition', None)
            if content_disposition or not part_maintype == "text":
                content = part.get_payload(decode=True)
            else:
                content = decode_content(part)

            is_inline = content_disposition is None \
                or content_disposition.startswith("inline")
            if content_type == "text/plain" and is_inline:
                body['plain'].append(content)
            elif content_type == "text/html" and is_inline:
                body['html'].append(content)
            elif content_disposition:
                attachment = parse_attachment(part)
                if attachment:
                    attachments.append(attachment)

    elif maintype == 'text':
        payload = decode_content(email_message)
        body['plain'].append(payload)

    parsed_email['attachments'] = attachments

    parsed_email['body'] = body
    email_dict = dict(email_message.items())

    parsed_email['sent_from'] = get_mail_addresses(email_message, 'from')
    parsed_email['sent_to'] = get_mail_addresses(email_message, 'to')
    parsed_email['cc'] = get_mail_addresses(email_message, 'cc')
    parsed_email['bcc'] = get_mail_addresses(email_message, 'bcc')

    value_headers_keys = ['subject', 'date', 'message-id']
    key_value_header_keys = ['received-spf',
                             'mime-version',
                             'x-spam-status',
                             'x-spam-score',
                             'content-type']

    parsed_email['headers'] = []
    for key, value in email_dict.items():

        if key.lower() in value_headers_keys:
            valid_key_name = key.lower().replace('-', '_')
            parsed_email[valid_key_name] = decode_mail_header(value)

        if key.lower() in key_value_header_keys:
            parsed_email['headers'].append({'Name': key,
                                            'Value': value})

    if parsed_email.get('date'):
        parsed_email['parsed_date'] = email.utils.parsedate_to_datetime(parsed_email['date'])

    logger.info("Downloaded and parsed mail '{}' with {} attachments".format(
        parsed_email.get('subject'), len(parsed_email.get('attachments'))))
    return Struct(**parsed_email)
<commit_msg>Local variable 'filename' value is not used<commit_after>import imaplib
import io
import re
import email
import chardet
import base64
import quopri
import time
from datetime import datetime
from email.header import decode_header
from imbox.utils import str_encode, str_decode

import logging

logger = logging.getLogger(__name__)

class Struct:
    def __init__(self, **entries):
        self.__dict__.update(entries)

    def keys(self):
        return self.__dict__.keys()

    def __repr__(self):
        return str(self.__dict__)

def decode_mail_header(value, default_charset='us-ascii'):
    """
    Decode a header value into a unicode string.
    """
    try:
        headers = decode_header(value)
    except email.errors.HeaderParseError:
        return str_decode(str_encode(value, default_charset, 'replace'), default_charset)
    else:
        for index, (text, charset) in enumerate(headers):
            try:
                logger.debug("Mail header no. {index}: {data} encoding {charset}".format(
                    index=index,
                    data=str_decode(text, charset or 'utf-8', 'replace'),
                    charset=charset))
                headers[index] = str_decode(text, charset or default_charset,
                                            'replace')
            except LookupError:
                # if the charset is unknown, force default
                headers[index] = str_decode(text, default_charset, 'replace')

        return ''.join(headers)

def get_mail_addresses(message, header_name):
    """
    Retrieve all email addresses from one message header.
    """
    headers = [h for h in message.get_all(header_name, [])]
    addresses = email.utils.getaddresses(headers)

    for index, (address_name, address_email) in enumerate(addresses):
        addresses[index] = {'name': decode_mail_header(address_name),
                            'email': address_email}
        logger.debug("{} Mail address in message: <{}> {}".format(
            header_name.upper(), address_name, address_email))
    return addresses

def decode_param(param):
    name, v = param.split('=', 1)
    values = v.split('\n')
    value_results = []
    for value in values:
        match = re.findall(r'=\?((?:\w|-)+)\?([QB])\?(.+?)\?=', value)
        if match:
            for encoding, type_, code in match:
                if type_ == 'Q':
                    value = quopri.decodestring(code)
                elif type_ == 'B':
                    value = code.encode()
                    missing_padding = len(value) % 4

                    if missing_padding:
                        value += b"=" * (4 - missing_padding)

                value = str_encode(value, encoding)

            value_results.append(value)

            if value_results:
                v = ''.join(value_results)

    logger.debug("Decoded parameter {} - {}".format(name, v))
    return name, v

def parse_attachment(message_part):
    # Check again if this is a valid attachment
    content_disposition = message_part.get("Content-Disposition", None)
    if content_disposition is not None and not message_part.is_multipart():
        dispositions = [
            disposition.strip()
            for disposition in content_disposition.split(";")
            if disposition.strip()
        ]

        if dispositions[0].lower() in ["attachment", "inline"]:
            file_data = message_part.get_payload(decode=True)

            attachment = {
                'content-type': message_part.get_content_type(),
                'size': len(file_data),
                'content': io.BytesIO(file_data),
                'content-id': message_part.get("Content-ID", None)
            }
            filename_parts = []
            for param in dispositions[1:]:
                if param:
                    name, value = decode_param(param)

                    # Check for split filename
                    s_name = name.rstrip('*').split("*")
                    if s_name[0] == 'filename':
                        # If this is a split file name - use the number after the * as an index to insert this part
                        if len(s_name) > 1 and s_name[1] != '':
                            filename_parts.insert(int(s_name[1]),value[1:-1] if value.startswith('"') else value)
                        else:
                            filename_parts.insert(0,value[1:-1] if value.startswith('"') else value)

                    if 'create-date' in name:
                        attachment['create-date'] = value

            attachment['filename'] = "".join(filename_parts)
            return attachment

    return None

def decode_content(message):
    content = message.get_payload(decode=True)
    charset = message.get_content_charset('utf-8')
    try:
        return content.decode(charset, 'ignore')
    except LookupError:
        encoding = chardet.detect(content).get('encoding')
        if encoding:
            return content.decode(encoding, 'ignore')
        return content
    except AttributeError:
        return content

def fetch_email_by_uid(uid, connection, parser_policy):
    message, data = connection.uid('fetch', uid, '(BODY.PEEK[] FLAGS)')
    logger.debug("Fetched message for UID {}".format(int(uid)))

    raw_headers = data[0][0] + data[1]
    raw_email = data[0][1]

    email_object = parse_email(raw_email, policy=parser_policy)
    flags = parse_flags(raw_headers.decode())
    email_object.__dict__['flags'] = flags

    return email_object

def parse_flags(headers):
    """Copied from https://github.com/girishramnani/gmail/blob/master/gmail/message.py"""
    if len(headers) == 0:
        return []
    headers = bytes(headers, "ascii")
    return list(imaplib.ParseFlags(headers))

def parse_email(raw_email, policy=None):
    if isinstance(raw_email, bytes):
        raw_email = str_encode(raw_email, 'utf-8', errors='ignore')
    if policy is not None:
        email_parse_kwargs = dict(policy=policy)
    else:
        email_parse_kwargs = {}

    try:
        email_message = email.message_from_string(
            raw_email, **email_parse_kwargs)
    except UnicodeEncodeError:
        email_message = email.message_from_string(
            raw_email.encode('utf-8'), **email_parse_kwargs)
    maintype = email_message.get_content_maintype()
    parsed_email = {'raw_email': raw_email}

    body = {
        "plain": [],
        "html": []
    }
    attachments = []

    if maintype in ('multipart', 'image'):
        logger.debug("Multipart message. Will process parts.")
        for part in email_message.walk():
            content_type = part.get_content_type()
            part_maintype = part.get_content_maintype()
            content_disposition = part.get('Content-Disposition', None)
            if content_disposition or not part_maintype == "text":
                content = part.get_payload(decode=True)
            else:
                content = decode_content(part)

            is_inline = content_disposition is None \
                or content_disposition.startswith("inline")
            if content_type == "text/plain" and is_inline:
                body['plain'].append(content)
            elif content_type == "text/html" and is_inline:
                body['html'].append(content)
            elif content_disposition:
                attachment = parse_attachment(part)
                if attachment:
                    attachments.append(attachment)

    elif maintype == 'text':
        payload = decode_content(email_message)
        body['plain'].append(payload)

    parsed_email['attachments'] = attachments

    parsed_email['body'] = body
    email_dict = dict(email_message.items())

    parsed_email['sent_from'] = get_mail_addresses(email_message, 'from')
    parsed_email['sent_to'] = get_mail_addresses(email_message, 'to')
    parsed_email['cc'] = get_mail_addresses(email_message, 'cc')
    parsed_email['bcc'] = get_mail_addresses(email_message, 'bcc')

    value_headers_keys = ['subject', 'date', 'message-id']
    key_value_header_keys = ['received-spf',
                             'mime-version',
                             'x-spam-status',
                             'x-spam-score',
                             'content-type']

    parsed_email['headers'] = []
    for key, value in email_dict.items():

        if key.lower() in value_headers_keys:
            valid_key_name = key.lower().replace('-', '_')
            parsed_email[valid_key_name] = decode_mail_header(value)

        if key.lower() in key_value_header_keys:
            parsed_email['headers'].append({'Name': key,
                                            'Value': value})

    if parsed_email.get('date'):
        parsed_email['parsed_date'] = email.utils.parsedate_to_datetime(parsed_email['date'])

    logger.info("Downloaded and parsed mail '{}' with {} attachments".format(
        parsed_email.get('subject'), len(parsed_email.get('attachments'))))
    return Struct(**parsed_email)
<|endoftext|>


<commit_before>from __future__ import print_function

import requests
import os
import datetime
import base64
import config
import platform

class Commands(object):

	def __init__(self):
		self.platform = platform.system()

	SCREENSHOT_COMMANDS = {
		"Linux": "scrot -s {0}",
		"Darwin": "screencapture -s {0}",
	}

	CLIPBOARD_COMMANDS = {
		"Linux": "xsel --clipboard --input",
		"Darwin": "pbcopy",
	}

	NOTIFY_COMMANDS = {
		"Linux": 'notify-send "{0}"',
		"Darwin": "say screenshot copied",
	}

	def take(self):
		return self.SCREENSHOT_COMMANDS.get(self.platform)

	def copy_to_clipboard(self):
		return self.CLIPBOARD_COMMANDS.get(self.platform)

	def send_notification(self):
		return self.NOTIFY_COMMANDS.get(self.platform)

class ScreenShotUploader(object):

	def __init__(self):
		self.commands = Commands()
		self.client_id = config.get_client_id()
		self.target_path = None

	def get_file_name(self):
		return datetime.datetime.now().strftime("%Y_%M_%d_%H%M%s.png")

	def take_screenshot(self, target_path=None):
		if not self.target_path:
			self.target_path = os.path.expanduser("~/imgur_uploads")
			if not os.path.exists(self.target_path):
					os.mkdir(self.target_path)
			os.chdir(self.target_path)

		file_name = self.get_file_name()

		os.system(self.commands.take().format(file_name))

		self.upload_to_imgur(file_name)

	def upload_to_imgur(self, file_name):

		response = requests.post(
		    "https://api.imgur.com/3/upload.json",
		    headers={"Authorization": "Client-ID {0}".format(self.client_id)},
		    data={
		        'image': base64.b64encode(open(os.path.join(self.target_path, file_name), 'rb').read()),
		        'type': 'base64',
		        'name': file_name,
		    }
		)

		data = response.json()
		if data.get("status") == 200:

			print(data.get("data").get("link"))

			os.system("echo {0} | {1}".format(
				data.get("data").get("link"),
				self.commands.copy_to_clipboard())
			)

			os.system(self.commands.send_notification().format(
				"Upload completed. Image url pasted to clipboard.")
			)

		else:
			if 'error' in data.get("data"):
				os.system(self.commands.send_notification().format(
					print(data.get("data").get("error")))
				)

def main():
	ss = ScreenShotUploader()
	ss.take_screenshot()

if __name__ == '__main__':
	main()
<commit_msg>remove unneeded tab<commit_after>from __future__ import print_function

import requests
import os
import datetime
import base64
import config
import platform

class Commands(object):

	def __init__(self):
		self.platform = platform.system()

	SCREENSHOT_COMMANDS = {
		"Linux": "scrot -s {0}",
		"Darwin": "screencapture -s {0}",
	}

	CLIPBOARD_COMMANDS = {
		"Linux": "xsel --clipboard --input",
		"Darwin": "pbcopy",
	}

	NOTIFY_COMMANDS = {
		"Linux": 'notify-send "{0}"',
		"Darwin": "say screenshot copied",
	}

	def take(self):
		return self.SCREENSHOT_COMMANDS.get(self.platform)

	def copy_to_clipboard(self):
		return self.CLIPBOARD_COMMANDS.get(self.platform)

	def send_notification(self):
		return self.NOTIFY_COMMANDS.get(self.platform)

class ScreenShotUploader(object):

	def __init__(self):
		self.commands = Commands()
		self.client_id = config.get_client_id()
		self.target_path = None

	def get_file_name(self):
		return datetime.datetime.now().strftime("%Y_%M_%d_%H%M%s.png")

	def take_screenshot(self, target_path=None):
		if not self.target_path:
			self.target_path = os.path.expanduser("~/imgur_uploads")
			if not os.path.exists(self.target_path):
				os.mkdir(self.target_path)
			os.chdir(self.target_path)

		file_name = self.get_file_name()

		os.system(self.commands.take().format(file_name))

		self.upload_to_imgur(file_name)

	def upload_to_imgur(self, file_name):

		response = requests.post(
		    "https://api.imgur.com/3/upload.json",
		    headers={"Authorization": "Client-ID {0}".format(self.client_id)},
		    data={
		        'image': base64.b64encode(open(os.path.join(self.target_path, file_name), 'rb').read()),
		        'type': 'base64',
		        'name': file_name,
		    }
		)

		data = response.json()
		if data.get("status") == 200:

			print(data.get("data").get("link"))

			os.system("echo {0} | {1}".format(
				data.get("data").get("link"),
				self.commands.copy_to_clipboard())
			)

			os.system(self.commands.send_notification().format(
				"Upload completed. Image url pasted to clipboard.")
			)

		else:
			if 'error' in data.get("data"):
				os.system(self.commands.send_notification().format(
					print(data.get("data").get("error")))
				)

def main():
	ss = ScreenShotUploader()
	ss.take_screenshot()

if __name__ == '__main__':
	main()
<|endoftext|>


<commit_before>from __future__ import print_function

import requests
import os
import datetime
import base64
import config
import platform

class Commands(object):

	def __init__(self):
		self.platform = platform.system()

	SCREENSHOT_COMMANDS = {
		"Linux": "scrot -s {0}",
		"Darwin": "screencapture -s {0}",
	}

	CLIPBOARD_COMMANDS = {
		"Linux": "xsel --clipboard --input",
		"Darwin": "pbcopy",
	}

	NOTIFY_COMMANDS = {
		"Linux": 'notify-send "{0}"',
		"Darwin": "say screenshot copied",
	}

	def take(self):
		return self.SCREENSHOT_COMMANDS.get(self.platform)

	def copy_to_clipboard(self):
		return self.CLIPBOARD_COMMANDS.get(self.platform)

	def send_notification(self):
		return self.NOTIFY_COMMANDS.get(self.platform)

class ScreenShotUploader(object):

	def __init__(self):
		self.commands = Commands()
		self.client_id = config.get_client_id()
		self.target_path = None

	def get_file_name(self):
		return datetime.datetime.now().strftime("%Y_%M_%d_%H%M%s.png")

	def take_screenshot(self, target_path=None):
		if not self.target_path:
			self.target_path = os.path.expanduser("~/imgur_uploads")
			if not os.path.exists(self.target_path):
					os.mkdir(self.target_path)
			os.chdir(self.target_path)

		file_name =  self.get_file_name()

		os.system(self.commands.take().format(file_name))

		self.upload_to_imgur(file_name)

	def upload_to_imgur(self, file_name):

		response = requests.post(
		    "https://api.imgur.com/3/upload.json",
		    headers = {"Authorization": "Client-ID {0}".format(self.client_id)},
		    data = {
		        'image': base64.b64encode(open(os.path.join(self.target_path, file_name), 'rb').read()),
		        'type': 'base64',
		        'name': file_name,
		    }
		)

		data = response.json()
		if data.get("status") == 200:
			print(data.get("data").get("link"))
			os.system("echo {0} | {1}".format(data.get("data").get("link"), self.commands.copy_to_clipboard()))
			os.system(self.commands.send_notification().format("Upload completed. Image url pasted to clipboard."))

		else:
			if 'error' in data.get("data"):
				print(data.get("data").get("error"))

def main():
	ss = ScreenShotUploader()
	ss.take_screenshot()

if __name__ == '__main__':
	main()
<commit_msg>replace annoying say command with "ding" sound<commit_after>from __future__ import print_function

import requests
import os
import datetime
import base64
import config
import platform

class Commands(object):

	def __init__(self):
		self.platform = platform.system()

	SCREENSHOT_COMMANDS = {
		"Linux": "scrot -s {0}",
		"Darwin": "screencapture -s {0}",
	}

	CLIPBOARD_COMMANDS = {
		"Linux": "xsel --clipboard --input",
		"Darwin": "pbcopy",
	}

	NOTIFY_COMMANDS = {
		"Linux": 'notify-send "{0}"',
		"Darwin": "afplay /System/Library/Sounds/Glass.aiff",
	}

	def take(self):
		return self.SCREENSHOT_COMMANDS.get(self.platform)

	def copy_to_clipboard(self):
		return self.CLIPBOARD_COMMANDS.get(self.platform)

	def send_notification(self):
		return self.NOTIFY_COMMANDS.get(self.platform)

class ScreenShotUploader(object):

	def __init__(self):
		self.commands = Commands()
		self.client_id = config.get_client_id()
		self.target_path = None

	def get_file_name(self):
		return datetime.datetime.now().strftime("%Y_%M_%d_%H%M%s.png")

	def take_screenshot(self, target_path=None):
		if not self.target_path:
			self.target_path = os.path.expanduser("~/imgur_uploads")
			if not os.path.exists(self.target_path):
					os.mkdir(self.target_path)
			os.chdir(self.target_path)

		file_name =  self.get_file_name()

		os.system(self.commands.take().format(file_name))

		self.upload_to_imgur(file_name)

	def upload_to_imgur(self, file_name):

		response = requests.post(
		    "https://api.imgur.com/3/upload.json",
		    headers = {"Authorization": "Client-ID {0}".format(self.client_id)},
		    data = {
		        'image': base64.b64encode(open(os.path.join(self.target_path, file_name), 'rb').read()),
		        'type': 'base64',
		        'name': file_name,
		    }
		)

		data = response.json()
		if data.get("status") == 200:
			print(data.get("data").get("link"))
			os.system("echo {0} | {1}".format(data.get("data").get("link"), self.commands.copy_to_clipboard()))
			os.system(self.commands.send_notification().format("Upload completed. Image url pasted to clipboard."))

		else:
			if 'error' in data.get("data"):
				print(data.get("data").get("error"))

def main():
	ss = ScreenShotUploader()
	ss.take_screenshot()

if __name__ == '__main__':
	main()
<|endoftext|>


<commit_before>#!/usr/bin/python

## Binary Analysis Tool
## Copyright 2012 Armijn Hemel for Tjaldur Software Governance Solutions
## Licensed under Apache 2.0, see LICENSE file for details

'''
This is a plugin for the Binary Analysis Tool. It generates images of results
of the ranking scan, like piecharts and version charts.

This should be run as a postrun scan

Parameters for configuration file:

* BAT_IMAGEDIR :: location to where images should be written
'''

import os, os.path, sys, subprocess, array, cPickle, tempfile, copy
from PIL import Image
import matplotlib
matplotlib.use('cairo')
import pylab

def generateImages(filename, unpackreport, leafscans, scantempdir, toplevelscandir, envvars={}):
	if not unpackreport.has_key('sha256'):
		return

	scanenv = os.environ.copy()
	if envvars != None:
		for en in envvars.split(':'):
			try:
				(envname, envvalue) = en.split('=')
				scanenv[envname] = envvalue
			except Exception, e:
				pass

	imagedir = scanenv.get('BAT_IMAGEDIR', "%s/%s" % (toplevelscandir, "images"))
	try:
		os.stat(imagedir)
	except:
		## BAT_IMAGEDIR does not exist
		try:
			os.makedirs(imagedir)
		except Exception, e:
			return

	## generate piechart and version information
	if leafscans.has_key('ranking'):
		## the ranking result is (res, dynamicRes, variablepvs)
		(res, dynamicRes, variablepvs) = leafscans['ranking']
		if res == None and dynamicRes == {}:
			return
		if res['reports'] != []:
			piedata = []
			pielabels = []
			totals = 0.0
			others = 0.0
			for j in res['reports']:
				## less than half a percent, that's not significant anymore
				if j[3] < 0.5:
					totals += j[3]
					others += j[3]
					if totals <= 99.0:
						continue
				if totals >= 99.0:
					pielabels.append("others")
					piedata.append(others + 100.0 - totals)
					break
				else:
					pielabels.append(j[1])
					piedata.append(j[3])
					totals += j[3]
			pylab.figure(1, figsize=(6.5,6.5))
			ax = pylab.axes([0.2, 0.15, 0.6, 0.6])

			pylab.pie(piedata, labels=pielabels)

			pylab.savefig('%s/%s-piechart.png' % (imagedir, unpackreport['sha256']))
			pylab.gcf().clear()
			for j in res['reports']:
				if j[4] != {}:
					'''
					j_sorted = sorted(j[4], key=lambda x: j[4][x])
					max_y = j[4][j_sorted[-1]]
					xvalues = []
					yvalues = []
					for v in j_sorted:
						xvalues.append(v)
						yvalues.append(j[4][v])
						print >>sys.stderr, v, j[4][v], j[1], xvalues, yvalues, "max", max_y

					figsize = len(xvalues) * 1.0
					pylab.gcf().set_size_inches(figsize, 7)

					pylab.xlabel('version')
					pylab.ylabel('matches')
					pylab.title("Unique matches for %s" % j[1])
					## leave some space at the top
					pylab.gca().set_ylim(top=max_y + 1)
					x = pylab.arange(len(xvalues))
					b = pylab.bar(x, yvalues, width=0.6)
					for bb in b:
						print >>sys.stderr, bb.get_width(), bb.get_height()
					## center the text
					pylab.xticks(x+0.3, xvalues, rotation=270)

					pylab.savefig('%s/%s-%s-version.png' % (imagedir, unpackreport['sha256'], j[1]))
					pylab.gcf().clear()
					'''
					tmppickle = tempfile.mkstemp()
					pickledata = []
					vals = list(set(j[4].values()))
					vals.sort()
					for v in vals:
						j_sorted = filter(lambda x: x[1] == v, j[4].items())
						j_sorted.sort()
						for v in j_sorted:
							pickledata.append(v)
					cPickle.dump(pickledata, os.fdopen(tmppickle[0], 'w'))
					p = subprocess.Popen(['bat-generate-version-chart.py', '-i', tmppickle[1], '-o', '%s/%s-%s-version.png' % (imagedir, unpackreport['sha256'], j[1])], stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
					(stanout, stanerr) = p.communicate()
					if p.returncode != 0:
						print >>sys.stderr, stanerr
					os.unlink(tmppickle[1])

		## generate version information
		## the ranking result is (res, dynamicRes, variablepvs)
		if dynamicRes.has_key('packages'):
			for package in dynamicRes['packages']:
				packagedata = copy.copy(dynamicRes['packages'][package])
				tmppickle = tempfile.mkstemp()
				pickledata = []
				p_sorted = sorted(packagedata, key=lambda x: x[1])
				for v in p_sorted:
					pickledata.append(v)
				cPickle.dump(pickledata, os.fdopen(tmppickle[0], 'w'))
				p = subprocess.Popen(['bat-generate-version-chart.py', '-i', tmppickle[1], '-o', '%s/%s-%s-funcversion.png' % (imagedir, unpackreport['sha256'], package)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
				(stanout, stanerr) = p.communicate()
				if p.returncode != 0:
					print >>sys.stderr, stanerr
				os.unlink(tmppickle[1])
<commit_msg>sort hits for function name matches<commit_after>#!/usr/bin/python

## Binary Analysis Tool
## Copyright 2012 Armijn Hemel for Tjaldur Software Governance Solutions
## Licensed under Apache 2.0, see LICENSE file for details

'''
This is a plugin for the Binary Analysis Tool. It generates images of results
of the ranking scan, like piecharts and version charts.

This should be run as a postrun scan

Parameters for configuration file:

* BAT_IMAGEDIR :: location to where images should be written
'''

import os, os.path, sys, subprocess, array, cPickle, tempfile, copy
from PIL import Image
import matplotlib
matplotlib.use('cairo')
import pylab

def generateImages(filename, unpackreport, leafscans, scantempdir, toplevelscandir, envvars={}):
	if not unpackreport.has_key('sha256'):
		return

	scanenv = os.environ.copy()
	if envvars != None:
		for en in envvars.split(':'):
			try:
				(envname, envvalue) = en.split('=')
				scanenv[envname] = envvalue
			except Exception, e:
				pass

	imagedir = scanenv.get('BAT_IMAGEDIR', "%s/%s" % (toplevelscandir, "images"))
	try:
		os.stat(imagedir)
	except:
		## BAT_IMAGEDIR does not exist
		try:
			os.makedirs(imagedir)
		except Exception, e:
			return

	## generate piechart and version information
	if leafscans.has_key('ranking'):
		## the ranking result is (res, dynamicRes, variablepvs)
		(res, dynamicRes, variablepvs) = leafscans['ranking']
		if res == None and dynamicRes == {}:
			return
		if res['reports'] != []:
			piedata = []
			pielabels = []
			totals = 0.0
			others = 0.0
			for j in res['reports']:
				## less than half a percent, that's not significant anymore
				if j[3] < 0.5:
					totals += j[3]
					others += j[3]
					if totals <= 99.0:
						continue
				if totals >= 99.0:
					pielabels.append("others")
					piedata.append(others + 100.0 - totals)
					break
				else:
					pielabels.append(j[1])
					piedata.append(j[3])
					totals += j[3]
			pylab.figure(1, figsize=(6.5,6.5))
			ax = pylab.axes([0.2, 0.15, 0.6, 0.6])

			pylab.pie(piedata, labels=pielabels)

			pylab.savefig('%s/%s-piechart.png' % (imagedir, unpackreport['sha256']))
			pylab.gcf().clear()
			for j in res['reports']:
				if j[4] != {}:
					'''
					j_sorted = sorted(j[4], key=lambda x: j[4][x])
					max_y = j[4][j_sorted[-1]]
					xvalues = []
					yvalues = []
					for v in j_sorted:
						xvalues.append(v)
						yvalues.append(j[4][v])
						print >>sys.stderr, v, j[4][v], j[1], xvalues, yvalues, "max", max_y

					figsize = len(xvalues) * 1.0
					pylab.gcf().set_size_inches(figsize, 7)

					pylab.xlabel('version')
					pylab.ylabel('matches')
					pylab.title("Unique matches for %s" % j[1])
					## leave some space at the top
					pylab.gca().set_ylim(top=max_y + 1)
					x = pylab.arange(len(xvalues))
					b = pylab.bar(x, yvalues, width=0.6)
					for bb in b:
						print >>sys.stderr, bb.get_width(), bb.get_height()
					## center the text
					pylab.xticks(x+0.3, xvalues, rotation=270)

					pylab.savefig('%s/%s-%s-version.png' % (imagedir, unpackreport['sha256'], j[1]))
					pylab.gcf().clear()
					'''
					tmppickle = tempfile.mkstemp()
					pickledata = []
					vals = list(set(j[4].values()))
					vals.sort()
					for v in vals:
						j_sorted = filter(lambda x: x[1] == v, j[4].items())
						j_sorted.sort()
						for v2 in j_sorted:
							pickledata.append(v2)
					cPickle.dump(pickledata, os.fdopen(tmppickle[0], 'w'))
					p = subprocess.Popen(['bat-generate-version-chart.py', '-i', tmppickle[1], '-o', '%s/%s-%s-version.png' % (imagedir, unpackreport['sha256'], j[1])], stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
					(stanout, stanerr) = p.communicate()
					if p.returncode != 0:
						print >>sys.stderr, stanerr
					os.unlink(tmppickle[1])

		## generate version information
		## the ranking result is (res, dynamicRes, variablepvs)
		if dynamicRes.has_key('packages'):
			for package in dynamicRes['packages']:
				packagedata = copy.copy(dynamicRes['packages'][package])
				tmppickle = tempfile.mkstemp()
				pickledata = []
				p_sorted = sorted(packagedata, key=lambda x: x[1])
				vals = list(set(map(lambda x: x[1], p_sorted)))
				vals.sort()
				for v in vals:
					j_sorted = filter(lambda x: x[1] == v, p_sorted)
					j_sorted.sort()
					for v2 in j_sorted:
						pickledata.append(v2)
				cPickle.dump(pickledata, os.fdopen(tmppickle[0], 'w'))
				p = subprocess.Popen(['bat-generate-version-chart.py', '-i', tmppickle[1], '-o', '%s/%s-%s-funcversion.png' % (imagedir, unpackreport['sha256'], package)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
				(stanout, stanerr) = p.communicate()
				if p.returncode != 0:
					print >>sys.stderr, stanerr
				os.unlink(tmppickle[1])
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
import collections
from consort.makers.ConsortObject import ConsortObject
from abjad.tools import scoretools
from abjad.tools.topleveltools import inspect_
from abjad.tools.topleveltools import iterate

class RegisterAgent(ConsortObject):
    r'''An register agent.
    '''

    ### CLASS VARIABLES ###

    __slots__ = (
        '_division_inflections',
        '_global_inflections',
        '_octavations',
        '_phrase_inflections',
        )

    ### INITIALIZER ###

    def __init__(
        self,
        division_inflections=None,
        global_inflections=None,
        octavations=None,
        phrase_inflections=None,
        ):
        from plague_water import makers
        pass

    ### SPECIAL METHODS ###

    def __call__(
        self,
        music,
        seed=0,
        ):
        assert isinstance(music, scoretools.Container)

    ### PUBLIC METHODS ###

    @staticmethod
    def iterate_score(
        score,
        ):
        from consort import makers
        counter = collections.Counter()
        for voice in iterate(score).by_class(scoretools.Voice):
            for container in voice:
                prototype = makers.MusicSpecifier
                music_specifier = inspect_(container).get_effective(prototype)
                register_agent = music_specifier.register_agent
                if register_agent is None:
                    continue
                seed = counter[register_agent]
                register_agent(
                    container,
                    seed=seed,
                    )
                counter[register_agent] += 1

    ### PUBLIC PROPERTIES ###

    @property
    def division_inflections(self):
        return self._division_inflections

    @property
    def global_inflections(self):
        return self._global_inflections

    @property
    def phrase_inflections(self):
        return self._phrase_inflections

    @property
    def octavations(self):
        return self._octavations
<commit_msg>Simplified RegisterAgent.<commit_after># -*- encoding: utf-8 -*-
import collections
from abjad.tools import datastructuretools
from abjad.tools import pitchtools
from abjad.tools import scoretools
from abjad.tools.topleveltools import inspect_
from abjad.tools.topleveltools import iterate
from consort.makers.ConsortObject import ConsortObject

class RegisterAgent(ConsortObject):
    r'''An register agent.
    '''

    ### CLASS VARIABLES ###

    __slots__ = (
        '_register',
        )

    ### INITIALIZER ###

    def __init__(
        self,
        register=None,
        ):
        if register is not None:
            register = pitchtools.NamedPitch(register)
        self._register = register

    ### SPECIAL METHODS ###

    def __call__(
        self,
        music,
        seed=0,
        ):
        assert isinstance(music, scoretools.Container)
        octave_transposition_mapping = self.octave_transposition_mapping
        iterator = iterate(music).by_logical_tie(pitched=True)
        for i, logical_tie in enumerate(iterator):
            i += seed
            octave = pitchtools.Octave(self.octavations[i])
            named_pitch = logical_tie.head.written_pitch
            named_pitch_class = named_pitch.named_pitch_class
            octavated_pitch = pitchtools.NamedPitch(named_pitch_class, octave)
            new_pitch = octave_transposition_mapping([octavated_pitch])
            for note in logical_tie:
                note.written_pitch = new_pitch

    ### PUBLIC METHODS ###

    @staticmethod
    def iterate_score(
        score,
        ):
        from consort import makers
        counter = collections.Counter()
        for voice in iterate(score).by_class(scoretools.Voice):
            for container in voice:
                prototype = makers.MusicSpecifier
                music_specifier = inspect_(container).get_effective(prototype)
                register_agent = music_specifier.register_agent
                if register_agent is None:
                    continue
                seed = counter[music_specifier]
                register_agent(
                    container,
                    seed=seed,
                    )
                counter[music_specifier] += 1

    ### PUBLIC PROPERTIES ###

    @property
    def octavations(self):
        return datastructuretools.CyclicTuple((
            2, 7, 1, 8, 2, 8, 1, 8, 2, 8, 4, 5, 9, 0, 4, 5, 2, 3, 5, 3, 6, 0,
            2, 8, 7, 4, 7, 1, 3, 5, 2, 6, 6, 2, 4, 9, 7, 7, 5, 7, 2, 4, 7, 0,
            ))

    @property
    def register(self):
        return self._register

    @property
    def octave_transposition_mapping(self):
        return pitchtools.OctaveTranspositionMapping([
            ('[C0, C5)', self.register),
            ('[C5, C10)', self.register + 6),
            ])
<|endoftext|>


<commit_before>import json
import os
import logging
import sys
from pyssdb import pyssdb

CONFIG_FILE = 1
OUTPUT_DIR  = 2

ARGUMENT_LIST = [
    [CONFIG_FILE, "-f", "<config_file_name>", True],
    [OUTPUT_DIR, "-o", "<output_directory>", True]
]

def PrintUsage():

    print("\n", __file__, "\n")
    for argItem in ARGUMENT_LIST:
        if argItem[3] == True:
            print("    ", argItem[1], argItem[2])
        else:
            print("    ", argItem[1], argItem[2], "[OPTIONAL]")

def GetArguments(argv):

    arguments = dict()
    idx, argc = 0, len(argv)

    while idx < argc:
        for argItem in ARGUMENT_LIST:
            if (argv[idx] == argItem[1]) and (idx < argc - 1):
                idx = idx + 1
                arguments[argItem[0]] = argv[idx]
        idx = idx + 1

    for argItem in ARGUMENT_LIST:
        if (argItem[3] == True) and (argItem[0] not in arguments):
            PrintUsage()
            exit(0)

    return arguments

def LoadConfig(filename):

    fd = open(filename)
    data = fd.read()
    fd.close()

    return data

if __name__ == '__main__':

    args = GetArguments(sys.argv)

    logging.basicConfig(level=logging.DEBUG, format="%(asctime)s-%(thread)06d-%(levelname)s: %(message)s", datefmt="%Y%m%d-%H%M%S")

    confData = LoadConfig(args[CONFIG_FILE])
    agentConf = json.loads(confData)

    try:

        logging.info("Connecting to %s:%i(%i) ...", agentConf["SSDB"]["Host"], agentConf["SSDB"]["Port"], agentConf["SSDB"]["Timeout"])
        confSSDB = pyssdb.Client(host = agentConf["SSDB"]["Host"], port = agentConf["SSDB"]["Port"], socket_timeout = agentConf["SSDB"]["Timeout"])

        logging.info("Sending credential ...")
        confSSDB.auth(agentConf["SSDB"]["Passcode"])

        zoneCount = confSSDB.hsize("DNS-Zones")
        logging.info("Total zone file: %d", zoneCount)

        zoneNameList = confSSDB.hkeys("DNS-Zones", "", "", zoneCount)

        for zoneName in zoneNameList:
            zoneContent = confSSDB.hget("DNS-Zones", zoneName.decode("utf-8"))
            outputFile = args[OUTPUT_DIR].rstrip("/") + "/" + zoneName.decode("utf-8").rstrip(".")
            logging.info("  Output %s to %s", zoneName.decode("utf-8"), outputFile)
            zonefd = open(outputFile, mode = "wt")
            zonefd.write(zoneContent.decode("utf-8"))
            zonefd.close()

        logging.info("Disconnecting from SSDB ...")
        confSSDB.disconnect()

    except Exception as ex:
        print(ex)
<commit_msg>path.join<commit_after>import json
import os
import logging
import sys
from pyssdb import pyssdb

CONFIG_FILE = 1
OUTPUT_DIR  = 2

ARGUMENT_LIST = [
    [CONFIG_FILE, "-f", "<config_file_name>", True],
    [OUTPUT_DIR, "-o", "<output_directory>", True]
]

def PrintUsage():

    print("\n", __file__, "\n")
    for argItem in ARGUMENT_LIST:
        if argItem[3] == True:
            print("    ", argItem[1], argItem[2])
        else:
            print("    ", argItem[1], argItem[2], "[OPTIONAL]")
    print("")

def GetArguments(argv):

    arguments = dict()
    idx, argc = 0, len(argv)

    while idx < argc:
        for argItem in ARGUMENT_LIST:
            if (argv[idx] == argItem[1]) and (idx < argc - 1):
                idx = idx + 1
                arguments[argItem[0]] = argv[idx]
        idx = idx + 1

    for argItem in ARGUMENT_LIST:
        if (argItem[3] == True) and (argItem[0] not in arguments):
            PrintUsage()
            exit(0)

    return arguments

def LoadConfig(filename):

    fd = open(filename)
    data = fd.read()
    fd.close()

    return data

if __name__ == '__main__':

    args = GetArguments(sys.argv)

    logging.basicConfig(level=logging.DEBUG, format="%(asctime)s-%(thread)06d-%(levelname)s: %(message)s", datefmt="%Y%m%d-%H%M%S")

    confData = LoadConfig(args[CONFIG_FILE])
    agentConf = json.loads(confData)

    try:

        logging.info("Connecting to %s:%i(%i) ...", agentConf["SSDB"]["Host"], agentConf["SSDB"]["Port"], agentConf["SSDB"]["Timeout"])
        confSSDB = pyssdb.Client(host = agentConf["SSDB"]["Host"], port = agentConf["SSDB"]["Port"], socket_timeout = agentConf["SSDB"]["Timeout"])

        logging.info("Sending credential ...")
        confSSDB.auth(agentConf["SSDB"]["Passcode"])

        zoneCount = confSSDB.hsize("DNS-Zones")
        logging.info("Total zone file: %d", zoneCount)

        zoneNameList = confSSDB.hkeys("DNS-Zones", "", "", zoneCount)

        for zoneName in zoneNameList:
            zoneContent = confSSDB.hget("DNS-Zones", zoneName.decode("utf-8"))
            outputFile = os.path.join(args[OUTPUT_DIR], zoneName.decode("utf-8") + "zone")
            logging.info("  Output %s to %s", zoneName.decode("utf-8"), outputFile)

            zonefd = open(outputFile, mode = "wt")
            zonefd.write(zoneContent.decode("utf-8"))
            zonefd.close()

    except Exception as ex:
        print(ex)

    finally:
        if confSSDB != None:
            logging.info("Disconnecting from SSDB ...")
            confSSDB.disconnect()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
    Mango records system logic functions.
'''

# This file is part of mango.

# Distributed under the terms of the last AGPL License.
# The full license is in the file LICENCE, distributed as part of this software.

__author__ = 'Jean Chassoul'

import logging

import arrow
import motor

import uuid

# import numpy as np
import pandas as pd

from tornado import gen

from mango.messages import records
from mango.messages import reports

from mango.tools import clean_structure
from mango.tools import clean_results
from mango.tools import check_times

class Records(object):
    '''
        Records resources
    '''

    @gen.coroutine
    def get_record(self, account, record_uuid):
        '''
            Get a detail record
        '''
        if not account:
            record = yield self.db.records.find_one({'uuid':record_uuid},{'_id':0})
        else:

            # change accountcode to account, because the accountcode is a uuid
            # and we're expecting an account name.

            record = yield self.db.records.find_one({'uuid':record_uuid,
                                                     'account':account},
                                                    {'_id':0})
        try:
            if record:
                record = records.Record(record)
                record.validate()
        except Exception, e:
            logging.exception(e) # catch some daemon here!
            raise e
        finally:
            raise gen.Return(record)

    @gen.coroutine
    def get_record_list(self, account, start, end, lapse, page_num):
        '''
            Get detail records
        '''
        page_num = int(page_num)
        page_size = self.settings['page_size']
        record_list = []

        if not account:
            query = self.db.records.find({'public':True})
        elif type(account) is list:
            accounts = [{'accountcode':a, 'assigned': True} for a in account]
            query = self.db.records.find({'$or':accounts})
        else:
            query = self.db.records.find({'accountcode':account,
                                        'assigned':True})

        query = query.sort([('uuid', -1)]).skip(page_num * page_size).limit(page_size)

        try:

            while (yield query.fetch_next):
                result = query.next_object()
                record_list.append(records.Record(result))

        except Exception, e:
            logging.exception(e)
            raise e

        try:
            struct = {'results': record_list}
            message = reports.BaseResult(struct)
            message.validate()
            message = clean_results(message)
        except Exception, e:
            logging.exception(e)
            raise e
        finally:
            raise gen.Return(message)

    @gen.coroutine
    def get_unassigned_records(self, start, end, lapse, page_num):
        '''
            Get unassigned record detail records
        '''
        page_num = int(page_num)
        page_size = self.settings['page_size']
        result = []

        # or $exist = false ?

        query = self.db.records.find({'assigned':False})
        query = query.sort([('uuid', -1)]).skip(page_num * page_size).limit(page_size)

        try:
            for record in (yield query.to_list()):
                result.append(records.Record(record))

            struct = {'results':result}

            results = reports.BaseResult(struct)
            results.validate()
        except Exception, e:
            logging.exception(e)
            raise e

        results = clean_results(results)
        raise gen.Return(results)

    @gen.coroutine
    def get_summaries(self, account, start, end, lapse, page_num):
        '''
            Get summaries
        '''
        times = yield check_times(start, end)

        if lapse:
            logging.info('get summaries lapse %s' % (lapse))

    @gen.coroutine
    def get_summary(self, account, start, end, lapse):
        '''
            Get summary
        '''

        times = yield check_times(start, end)

        if lapse:
            logging.info('get summary lapse %s' % (lapse))

        # MongoDB aggregation match operator
        if type(account) is list:
            match = {
                'assigned':True,
                'start':{'$gte':times.get('start'), '$lt':times.get('end')},
                '$or':[{'accountcode':a} for a in account]
            }
        else:
            match = {
                'accountcode':account,
                'assigned': True,
                'start': {'$gte':times.get('start'), '$lt': times.get('end')}
            }

        # MongoDB aggregation project operator
        project = {
            "_id" : 0,

            # record timestamps
            "start":1,
            "answer":1,
            "end":1,

            # record duration seconds
            "duration" : 1,
            # record billing seconds
            "billsec" : 1,

            # project id's timestamp stuff?
            "year" : {
                "$year" : "$start"
            },
            "month" : {
                "$month" : "$start"
            },
            "week" : {
                "$week" : "$start"
            },
            "day" : {
                "$dayOfMonth" : "$start"
            },
            "hour" : {
                "$hour" : "$start"
            },
            "minute" : {
                "$minute" : "$start"
            },
            "second" : {
                "$second" : "$start"
            }
        }

        # MongoDB aggregation group operator

        # R&D on group by accountcode, account, uuid, or something else ...

        group = {
            '_id': {
                'start': '$start',
                'answer': '$answer',
                'end': '$end',
                'year': '$year',
                'month': '$month',
                'week':'$week',
                'day': '$day',
                'hour':'$hour',
                'minute': '$minute',
                'second': '$second',
            },

            'records': {
                '$sum':1
            },

            'average': {
                '$avg':'$billsec'
            },

            'duration': {
                '$sum':'$duration'
            },

            'billing': {
                '$sum':'$billsec'
            }
        }

        # MongoDB aggregation pipeline
        pipeline = [
            {'$match':match},
            {'$project':project},
            {'$group':group}
        ]

        result = yield self.db.records.aggregate(pipeline)

        raise gen.Return(result.get('result'))

    @gen.coroutine
    def new_detail_record(self, struct, db=None):
        '''
            Create a new record entry
        '''
        if not db:
            db = self.db
        try:
            # if not type str convert to str
            logging.info(struct)
            struct['strdate'] = struct.get('strdate', '')

            record = records.Record(struct)
            record.validate()
        except Exception, e:
            logging.exception(e)
            raise e

        record = clean_structure(record)

        result = yield db.records.insert(record)

        message = {
            'uniqueid':struct.get('uniqueid'),
            'uuid':record.get('uuid')
        }

        raise gen.Return(message)

    @gen.coroutine
    def set_assigned_flag(self, account, record_uuid):
        '''
            Set the record assigned flag
        '''
        logging.info('set_assigned_flag account: %s, record: %s' % (account, record_uuid))

        result = yield self.db.records.update(
                                {'uuid':record_uuid,
                                 'accountcode':account},
                                {'$set': {'assigned': True}})

        raise gen.Return(result)

    @gen.coroutine
    def remove_record(self, record_uuid):
        '''
            Remove a record entry
        '''
        result = yield self.db.records.remove({'uuid':record_uuid})
        raise gen.Return(result)

    @gen.coroutine
    def replace_record(self, struct):
        '''
            Replace a existent record entry
        '''
        # put implementation
        pass

    @gen.coroutine
    def resource_options(self):
        '''
            Return resource options
        '''
        # options implementation
        pass

    @gen.coroutine
    def modify_record(self, struct):
        '''
            Modify a existent record entry
        '''
        # patch implementation
        pass<commit_msg>monta<commit_after># -*- coding: utf-8 -*-
'''
    Mango records system logic functions.
'''

# This file is part of mango.

# Distributed under the terms of the last AGPL License.
# The full license is in the file LICENCE, distributed as part of this software.

__author__ = 'Jean Chassoul'

import logging

import arrow
import motor

import uuid

# import numpy as np
import pandas as pd

from tornado import gen

from mango.messages import records
from mango.messages import reports

from mango.tools import clean_structure
from mango.tools import clean_results
from mango.tools import check_times

class Records(object):
    '''
        Records resources
    '''

    @gen.coroutine
    def get_record(self, account, record_uuid):
        '''
            Get a detail record
        '''
        if not account:
            record = yield self.db.records.find_one({'uuid':record_uuid},{'_id':0})
        else:

            # change accountcode to account, because the accountcode is a uuid
            # and we're expecting an account name.

            record = yield self.db.records.find_one({'uuid':record_uuid,
                                                     'account':account},
                                                    {'_id':0})
        try:
            if record:
                record = records.Record(record)
                record.validate()
        except Exception, e:
            logging.exception(e) # catch some daemon here!
            raise e
        finally:
            raise gen.Return(record)

    @gen.coroutine
    def get_record_list(self, account, start, end, lapse, page_num):
        '''
            Get detail records
        '''
        page_num = int(page_num)
        page_size = self.settings['page_size']
        record_list = []

        if not account:
            query = self.db.records.find({'public':True})
        elif type(account) is list:
            accounts = [{'accountcode':a, 'assigned': True} for a in account]
            query = self.db.records.find({'$or':accounts})
        else:
            query = self.db.records.find({'accountcode':account,
                                        'assigned':True})

        query = query.sort([('uuid', -1)]).skip(page_num * page_size).limit(page_size)

        try:

            while (yield query.fetch_next):
                result = query.next_object()
                record_list.append(records.Record(result))

        except Exception, e:
            logging.exception(e)
            raise e

        try:
            struct = {'results': record_list}
            message = reports.BaseResult(struct)
            message.validate()
            message = clean_results(message)
        except Exception, e:
            logging.exception(e)
            raise e
        finally:
            raise gen.Return(message)

    @gen.coroutine
    def get_unassigned_records(self, start, end, lapse, page_num):
        '''
            Get unassigned record detail records
        '''
        page_num = int(page_num)
        page_size = self.settings['page_size']
        result = []

        # or $exist = false ?

        query = self.db.records.find({'assigned':False})
        query = query.sort([('uuid', -1)]).skip(page_num * page_size).limit(page_size)

        try:
            for record in (yield query.to_list()):
                result.append(records.Record(record))

            struct = {'results':result}

            results = reports.BaseResult(struct)
            results.validate()
        except Exception, e:
            logging.exception(e)
            raise e

        results = clean_results(results)
        raise gen.Return(results)

    @gen.coroutine
    def get_summaries(self, account, start, end, lapse, page_num):
        '''
            Get summaries
        '''
        times = yield check_times(start, end)

        if lapse:
            logging.info('get summaries lapse %s' % (lapse))

    @gen.coroutine
    def get_summary(self, account, start, end, lapse):
        '''
            Get summary
        '''

        times = yield check_times(start, end)

        if lapse:
            logging.info('get summary lapse %s' % (lapse))

        # MongoDB aggregation match operator
        if type(account) is list:
            match = {
                'assigned':True,
                'start':{'$gte':times.get('start'), '$lt':times.get('end')},
                '$or':[{'accountcode':a} for a in account]
            }
        else:
            match = {
                'accountcode':account,
                'assigned': True,
                'start': {'$gte':times.get('start'), '$lt': times.get('end')}
            }

        # MongoDB aggregation project operator
        project = {
            "_id" : 0,

            # record timestamps
            "start":1,
            "answer":1,
            "end":1,

            # record duration seconds
            "duration" : 1,
            # record billing seconds
            "billsec" : 1,

            # project id's timestamp stuff?
            "year" : {
                "$year" : "$start"
            },
            "month" : {
                "$month" : "$start"
            },
            "week" : {
                "$week" : "$start"
            },
            "day" : {
                "$dayOfMonth" : "$start"
            },
            "hour" : {
                "$hour" : "$start"
            },
            "minute" : {
                "$minute" : "$start"
            },
            "second" : {
                "$second" : "$start"
            }
        }

        # MongoDB aggregation group operator

        # R&D on group by accountcode, account, uuid, or something else ...

        group = {
            '_id': {
                'start': '$start',
                'answer': '$answer',
                'end': '$end',
                'year': '$year',
                'month': '$month',
                'week':'$week',
                'day': '$day',
                'hour':'$hour',
                'minute': '$minute',
                'second': '$second',
            },

            'records': {
                '$sum':1
            },

            'average': {
                '$avg':'$billsec'
            },

            'duration': {
                '$sum':'$duration'
            },

            'billing': {
                '$sum':'$billsec'
            }
        }

        # MongoDB aggregation pipeline
        pipeline = [
            {'$match':match},
            {'$project':project},
            {'$group':group}
        ]

        result = yield self.db.records.aggregate(pipeline)

        raise gen.Return(result.get('result'))

    @gen.coroutine
    def new_detail_record(self, struct, db=None):
        '''
            Create a new record entry
        '''
        if not db:
            db = self.db
        try:
            # if not type str convert to str

            logging.info(struct)
            logging.info(struct)
            logging.info(struct)

            struct['strdate'] = struct.get('strdate', '')

            record = records.Record(struct)
            record.validate()
        except Exception, e:
            logging.exception(e)
            raise e

        record = clean_structure(record)

        result = yield db.records.insert(record)

        message = {
            'uniqueid':struct.get('uniqueid'),
            'uuid':record.get('uuid')
        }

        raise gen.Return(message)

    @gen.coroutine
    def set_assigned_flag(self, account, record_uuid):
        '''
            Set the record assigned flag
        '''
        logging.info('set_assigned_flag account: %s, record: %s' % (account, record_uuid))

        result = yield self.db.records.update(
                                {'uuid':record_uuid,
                                 'accountcode':account},
                                {'$set': {'assigned': True}})

        raise gen.Return(result)

    @gen.coroutine
    def remove_record(self, record_uuid):
        '''
            Remove a record entry
        '''
        result = yield self.db.records.remove({'uuid':record_uuid})
        raise gen.Return(result)

    @gen.coroutine
    def replace_record(self, struct):
        '''
            Replace a existent record entry
        '''
        # put implementation
        pass

    @gen.coroutine
    def resource_options(self):
        '''
            Return resource options
        '''
        # options implementation
        pass

    @gen.coroutine
    def modify_record(self, struct):
        '''
            Modify a existent record entry
        '''
        # patch implementation
        pass<|endoftext|>


<commit_before>"""Extends BaseHTTPRequestHandler with SSL certificate generation."""

import socket
import certutils

openssl_import_error = None
try:
  # Requires: pyOpenSSL 0.13+
  from OpenSSL import SSL
except ImportError, e:
  openssl_import_error = e

cert_store = None

class WrappedConnection(object):

  def __init__(self, obj):
    self._wrapped_obj = obj

  def __getattr__(self, attr):
    if attr in self.__dict__:
      return getattr(self, attr)
    return getattr(self._wrapped_obj, attr)

  def recv(self, buflen=1024, flags=0):
    try:
      return self._wrapped_obj.recv(buflen, flags)
    except SSL.SysCallError, e:
      if e.args[1] == 'Unexpected EOF':
        return ''
      raise
    except SSL.ZeroReturnError:
      return ''

def set_ca_cert(ca_cert):
  global cert_store
  cert_store = certutils.CertStore(ca_cert, cert_dir=None)

class SslHandshakeHandler(object):
  """Handles Server Name Indication (SNI) using dummy certs."""

  def __init__(self, request, client_address, server):
    pass

  def setup(self):
    """Sets up connection providing the certificate to the client."""
    self.server_name = None
    # One of: One of SSLv2_METHOD, SSLv3_METHOD, SSLv23_METHOD, or TLSv1_METHOD
    method = SSL.SSLv23_METHOD
    context = SSL.Context(method)
    def handle_servername(connection):
      """A SNI callback that happens during do_handshake()."""
      try:
        host = connection.get_servername()
        if host:
          self.server_name = host
          cert = cert_store.get_cert(host)
          new_context = SSL.Context(SSL.SSLv23_METHOD)
          new_context.use_certificate_file(cert)
          new_context.use_privatekey_file(cert_store.ca_cert)
          connection.set_context(new_context)
          return new_context
        # else: fail with 'no shared cipher'
        # TODO(mruthven): move cert generation to after fetch so the host name
        # can be gotten from the server.
      except Exception, e:
        # Do not leak any exceptions or else openssl crashes.
        print('Exception in SNI handler', e)

    context.set_tlsext_servername_callback(handle_servername)
    self.connection = WrappedConnection(SSL.Connection(context,
                                                       self.connection))
    self.connection.set_accept_state()
    try:
      self.connection.do_handshake()
    except SSL.Error, v:
      self.connection.shutdown()
      self.connection.close()
      raise Exception('SSL handshake error: %s' % str(v))

    # Re-wrap the read/write streams with our new connection.
    self.rfile = socket._fileobject(self.connection, 'rb', self.rbufsize,
                                    close=False)
    self.wfile = socket._fileobject(self.connection, 'wb', self.wbufsize,
                                    close=False)

  def finish(self):
    self.connection.shutdown()
    self.connection.close()

def wrap_handler(handler_class, cert_file):
  """Wraps a BaseHTTPHandler with SSL MITM certificates."""
  if openssl_import_error:
    raise openssl_import_error
  set_ca_cert(cert_file)

  class WrappedHandler(SslHandshakeHandler, handler_class):

    def setup(self):
      handler_class.setup(self)
      SslHandshakeHandler.setup(self)

    def finish(self):
      handler_class.finish(self)
      SslHandshakeHandler.finish(self)
  return WrappedHandler

<commit_msg>removing server_name from SslHandshakeHandler<commit_after>"""Extends BaseHTTPRequestHandler with SSL certificate generation."""

import socket
import certutils

openssl_import_error = None
try:
  # Requires: pyOpenSSL 0.13+
  from OpenSSL import SSL
except ImportError, e:
  openssl_import_error = e

cert_store = None

class WrappedConnection(object):

  def __init__(self, obj):
    self._wrapped_obj = obj

  def __getattr__(self, attr):
    if attr in self.__dict__:
      return getattr(self, attr)
    return getattr(self._wrapped_obj, attr)

  def recv(self, buflen=1024, flags=0):
    try:
      return self._wrapped_obj.recv(buflen, flags)
    except SSL.SysCallError, e:
      if e.args[1] == 'Unexpected EOF':
        return ''
      raise
    except SSL.ZeroReturnError:
      return ''

def set_ca_cert(ca_cert):
  global cert_store
  cert_store = certutils.CertStore(ca_cert, cert_dir=None)

class SslHandshakeHandler:
  """Handles Server Name Indication (SNI) using dummy certs."""

  def setup(self):
    """Sets up connection providing the certificate to the client."""
    # One of: One of SSLv2_METHOD, SSLv3_METHOD, SSLv23_METHOD, or TLSv1_METHOD
    method = SSL.SSLv23_METHOD
    context = SSL.Context(method)
    def handle_servername(connection):
      """A SNI callback that happens during do_handshake()."""
      try:
        host = connection.get_servername()
        if host:
          cert = cert_store.get_cert(host)
          new_context = SSL.Context(SSL.SSLv23_METHOD)
          new_context.use_certificate_file(cert)
          new_context.use_privatekey_file(cert_store.ca_cert)
          connection.set_context(new_context)
          return new_context
        # else: fail with 'no shared cipher'
        # TODO(mruthven): move cert generation to after fetch so the host name
        # can be gotten from the server.
      except Exception, e:
        # Do not leak any exceptions or else openssl crashes.
        logging.error('Exception in SNI handler', e)

    context.set_tlsext_servername_callback(handle_servername)
    self.connection = WrappedConnection(SSL.Connection(context,
                                                       self.connection))
    self.connection.set_accept_state()
    try:
      self.connection.do_handshake()
    except SSL.Error, v:
      self.connection.shutdown()
      self.connection.close()
      raise Exception('SSL handshake error: %s' % str(v))

    # Re-wrap the read/write streams with our new connection.
    self.rfile = socket._fileobject(self.connection, 'rb', self.rbufsize,
                                    close=False)
    self.wfile = socket._fileobject(self.connection, 'wb', self.wbufsize,
                                    close=False)

  def finish(self):
    self.connection.shutdown()
    self.connection.close()

def wrap_handler(handler_class, cert_file):
  """Wraps a BaseHTTPHandler with SSL MITM certificates."""
  if openssl_import_error:
    raise openssl_import_error
  set_ca_cert(cert_file)

  class WrappedHandler(SslHandshakeHandler, handler_class):

    def setup(self):
      handler_class.setup(self)
      SslHandshakeHandler.setup(self)

    def finish(self):
      handler_class.finish(self)
      SslHandshakeHandler.finish(self)
  return WrappedHandler

<|endoftext|>


<commit_before>"""
distribution.py
Author: Suhan Gui
Credit: Stack Overflow

Assignment: Character Distribution

Write and submit a Python program (distribution.py) that computes and displays
the distribution of characters in a given sample of text.

Output of your program should look like this:

Please enter a string of text (the bigger the better): The rain in Spain stays mainly in the plain.
The distribution of characters in "The rain in Spain stays mainly in the plain." is:
iiiiii
nnnnnn
aaaaa
sss
ttt
ee
hh
ll
pp
yy
m
r

Notice about this example:

* The text: 'The rain ... plain' is provided by the user as input to your program.
* Uppercase characters are converted to lowercase
* Spaces and punctuation marks are ignored completely.
* Characters that are more common appear first in the list.
* Where the same number of characters occur, the lines are ordered alphabetically.
  For example, in the printout above, the letters e, h, l, p and y both occur twice
  in the text and they are listed in the output in alphabetical order.
* Letters that do not occur in the text are not listed in the output at all.
"""
from collections import Counter
import string
import operator

word=input("Please enter a string of text (the bigger the better): ")
potato=word.lower()
print('The distribution of characters in "{0}" is:'.format(word))

doge=dict(map(lambda letter:(letter,len(potato)-len(potato.replace(letter,''))),potato))
sorted_doge = sorted(doge.items(), key=operator.itemgetter(0))

print(sorted_doge)<commit_msg>Updated from Brython Server: 10/8/2015 9:27:45 PM<commit_after>"""
distribution.py
Author: Suhan Gui
Credit: Stack Overflow & Nils Kingston

Assignment: Character Distribution

Write and submit a Python program (distribution.py) that computes and displays
the distribution of characters in a given sample of text.

Output of your program should look like this:

Please enter a string of text (the bigger the better): The rain in Spain stays mainly in the plain.
The distribution of characters in "The rain in Spain stays mainly in the plain." is:
iiiiii
nnnnnn
aaaaa
sss
ttt
ee
hh
ll
pp
yy
m
r

Notice about this example:

* The text: 'The rain ... plain' is provided by the user as input to your program.
* Uppercase characters are converted to lowercase
* Spaces and punctuation marks are ignored completely.
* Characters that are more common appear first in the list.
* Where the same number of characters occur, the lines are ordered alphabetically.
  For example, in the printout above, the letters e, h, l, p and y both occur twice
  in the text and they are listed in the output in alphabetical order.
* Letters that do not occur in the text are not listed in the output at all.
"""
from collections import Counter
import string
import operator

word=input("Please enter a string of text (the bigger the better): ")
potato=word.lower()
print('The distribution of characters in "{0}" is:'.format(word))

doge=dict(map(lambda letter:(letter,len(potato)-len(potato.replace(letter,''))),potato))
sorted_doge = sorted(doge.items(), key=lambda.itemgetter(0)))))

for x in sorted_doge:
    if x[0]!=0:
        print(x[0]*x[1])<|endoftext|>


<commit_before>"""
distribution.py
Author: Avery Wallis
Credit: Ethan, Payton, Doniel

Assignment:

Write and submit a Python program (distribution.py) that computes and displays
the distribution of characters in a given sample of text.

Output of your program should look like this:

Please enter a string of text (the bigger the better): The rain in Spain stays mainly in the plain.
The distribution of characters in "The rain in Spain stays mainly in the plain." is:
iiiiii
nnnnnn
aaaaa
sss
ttt
ee
hh
ll
pp
yy
m
r

Notice about this example:

* The text: 'The rain ... plain' is provided by the user as input to your program.
* Uppercase characters are converted to lowercase
* Spaces and punctuation marks are ignored completely.
* Characters that are more common appear first in the list.
* Where the same number of characters occur, the lines are ordered alphabetically.
  For example, in the printout above, the letters e, h, l, p and y both occur twice
  in the text and they are listed in the output in alphabetical order.
* Letters that do not occur in the text are not listed in the output at all.
"""
string= str(input("Please enter a string of text (the bigger the better): "))
string1=string.lower()
print('The distribution of characters in "'+string1+ '" is:')

alph='abcdefghijklmnopqrstuvwxyz'
result=[]
listnum=[]

for c in alph:
    r = string.count(c)
    if not r==0:
        t=(r*c)
        result.append(t)
        listnum.append(r)

lists=zip(listnum, result)
print(list(lists))
lists=sorted(lists, key=lambda listnum: listnum[0])   # sort by number
lists.sort(reverse=True)
print(list(lists))
print(list(lists(result[-1])))

'''
https://docs.python.org/3/howto/sorting.html#sortinghowto
'''<commit_msg>Updated from Brython Server: 2/26/2016 2:12:46 PM<commit_after>"""
distribution.py
Author: Avery Wallis
Credit: Ethan, Payton, Doniel

Assignment:

Write and submit a Python program (distribution.py) that computes and displays
the distribution of characters in a given sample of text.

Output of your program should look like this:

Please enter a string of text (the bigger the better): The rain in Spain stays mainly in the plain.
The distribution of characters in "The rain in Spain stays mainly in the plain." is:
iiiiii
nnnnnn
aaaaa
sss
ttt
ee
hh
ll
pp
yy
m
r

Notice about this example:

* The text: 'The rain ... plain' is provided by the user as input to your program.
* Uppercase characters are converted to lowercase
* Spaces and punctuation marks are ignored completely.
* Characters that are more common appear first in the list.
* Where the same number of characters occur, the lines are ordered alphabetically.
  For example, in the printout above, the letters e, h, l, p and y both occur twice
  in the text and they are listed in the output in alphabetical order.
* Letters that do not occur in the text are not listed in the output at all.
"""
string= str(input("Please enter a string of text (the bigger the better): "))
string1=string.lower()
print('The distribution of characters in "'+string1+ '" is:')

alph='abcdefghijklmnopqrstuvwxyz'
result=[]
listnum=[]

for c in alph:
    r = string.count(c)
    if not r==0:
        t=(r*c)
        result.append(t)
        listnum.append(r)

lists=zip(listnum, result)
print(list(lists))
lists=sorted(lists, key=lambda listnum: listnum[0])   # sort by number
lists.sort(reverse=True)
print(list(lists))
print(list(lists[-1]))

'''
https://docs.python.org/3/howto/sorting.html#sortinghowto
'''<|endoftext|>


<commit_before>"""
distribution.py
Author: Dina
Credit: none

Assignment:

Write and submit a Python program (distribution.py) that computes and displays
the distribution of characters in a given sample of text.

Output of your program should look like this:

Please enter a string of text (the bigger the better): The rain in Spain stays mainly in the plain.
The distribution of characters in "The rain in Spain stays mainly in the plain." is:
iiiiii
nnnnnn
aaaaa
sss
ttt
ee
hh
ll
pp
yy
m
r

Notice about this example:

* The text: 'The rain ... plain' is provided by the user as input to your program.
* Uppercase characters are converted to lowercase
* Spaces and punctuation marks are ignored completely.
* Characters that are more common appear first in the list.
* Where the same number of characters occur, the lines are ordered alphabetically.
  For example, in the printout above, the letters e, h, l, p and y both occur twice
  in the text and they are listed in the output in alphabetical order.
* Letters that do not occur in the text are not listed in the output at all.
"""
text = input("Please enter a string of text (the bigger the better):")
print("The distribution of characters in " + text + " is:")
text = text.lower()
length = len(text)
letter = 0
used = []
dis = []

while letter < length:
    b = text[letter]
    #print(used)
    letter = letter+1
    if not( b in used or b == ' ' or b == '-' or b == '.' or b== ','):
        a = text.count(b)
        while a >= 1:
            dis.append(a)
            a = a-1
        used.append(b)
        print(zip(dis, used))

#print (b, end = '')

#print("hello")

<commit_msg>Updated from Brython Server: 10/19/2015 2:46:07 PM<commit_after>"""
distribution.py
Author: Dina
Credit: none

Assignment:

Write and submit a Python program (distribution.py) that computes and displays
the distribution of characters in a given sample of text.

Output of your program should look like this:

Please enter a string of text (the bigger the better): The rain in Spain stays mainly in the plain.
The distribution of characters in "The rain in Spain stays mainly in the plain." is:
iiiiii
nnnnnn
aaaaa
sss
ttt
ee
hh
ll
pp
yy
m
r

Notice about this example:

* The text: 'The rain ... plain' is provided by the user as input to your program.
* Uppercase characters are converted to lowercase
* Spaces and punctuation marks are ignored completely.
* Characters that are more common appear first in the list.
* Where the same number of characters occur, the lines are ordered alphabetically.
  For example, in the printout above, the letters e, h, l, p and y both occur twice
  in the text and they are listed in the output in alphabetical order.
* Letters that do not occur in the text are not listed in the output at all.
"""
text = input("Please enter a string of text (the bigger the better):")
print("The distribution of characters in " + text + " is:")
text = text.lower()
length = len(text)
letter = 0
used = []
dis = []

while letter < length:
    b = text[letter]
    letter = letter+1
    if not( b in used or b == ' ' or b == '-' or b == '.' or b== ','):
        dis.append(text.count(b))
        used.append(b)
        print(zip(dis, used))

c = zip(dis,used)
print(c)
print("hello")

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import aiopg
from psycopg2.extras import DictCursor
from typing import List, Dict

class DB:

    def __init__(self, config: Dict):
        self.config = config
        self.dbs = {}

    async def init(self, loop=None):
        for _key, _creds in self.config.items():
            self.dbs[_key] = {}
            for _role, _dsn in _creds.items():
                self.dbs[_key][_role] = await aiopg.create_pool(
                    _dsn, minsize=1, maxsize=10, timeout=5, loop=loop
                )

    async def shutdown(self, app):
        # close connections

        for k, db in self.dbs.items():
            for role, item in db.items():
                if item is not None:
                    item.close()

        for k, db in self.dbs.items():
            for role, item in db.items():
                if item is not None:
                    await item.wait_closed()

    async def execute(self, db_name: str, query: str, values: List, _type: str):
        """
        Execute SQL query in connection pool
        :param db_name:
        :param query:
        :param values:
        :param _type:
        :return:
        """

        if _type not in ('select', 'insert', 'update', 'delete'):
            raise RuntimeError(
                'Wrong request type {}'.format(_type)
            )
        if not self.dbs[db_name]['master']:
            raise RuntimeError(
                'db {} master is not initialized'.format(db_name)
            )

        pool = self.dbs[db_name]['master']
        if _type == 'select' and 'slave' in self.dbs[db_name]:
            pool = self.dbs[db_name]['slave']

        async with pool.acquire() as conn:
            async with conn.cursor(cursor_factory=DictCursor) as cursor:
                await cursor.execute(query, values)

                if _type == 'select':
                    data = await cursor.fetchall()
                else:
                    cursor.connection.commit()
                    data = cursor.rowcount
        return data

def setup(app):
    db = DB(app.settings.DATABASE)
    app.loop.run_until_complete(db.init())
    app.db = db
    app.on_cleanup.append(db.shutdown)
<commit_msg>start rewrite execute method<commit_after># -*- coding: utf-8 -*-
import warnings

import aiopg
from psycopg2.extras import DictCursor
from typing import List, Dict

class DB:

    def __init__(self, config: Dict):
        self.config = config
        self.dbs = {}

    async def init(self, loop=None):
        for _key, _creds in self.config.items():
            self.dbs[_key] = {}
            for _role, _dsn in _creds.items():
                self.dbs[_key][_role] = await aiopg.create_pool(
                    _dsn, minsize=1, maxsize=10, timeout=5, loop=loop
                )

    async def shutdown(self, app):
        # close connections

        for k, db in self.dbs.items():
            for role, item in db.items():
                if item is not None:
                    item.close()

        for k, db in self.dbs.items():
            for role, item in db.items():
                if item is not None:
                    await item.wait_closed()

    async def execute(self, db_name: str, query: str, values: List, _type: str):
        """
        Execute SQL query in connection pool
        :param db_name:
        :param query:
        :param values:
        :param _type:
        :return:
        """
        warnings.warn("Use single methods!", DeprecationWarning)

        if _type not in ('select', 'insert', 'update', 'delete'):
            raise RuntimeError(
                'Wrong request type {}'.format(_type)
            )
        if not self.dbs[db_name]['master']:
            raise RuntimeError(
                'db {} master is not initialized'.format(db_name)
            )

        pool = self.dbs[db_name]['master']
        if _type == 'select' and 'slave' in self.dbs[db_name]:
            pool = self.dbs[db_name]['slave']

        async with pool.acquire() as conn:
            async with conn.cursor(cursor_factory=DictCursor) as cursor:
                await cursor.execute(query, values)

                if _type == 'select':
                    data = await cursor.fetchall()
                else:
                    cursor.connection.commit()
                    data = cursor.rowcount
        return data

    async def select(self, db_name: str, query:str, values: List):
        return await self._execute(db_name, query, values, 'select')

    async def insert(self, db_name: str, query:str, values: List, returning:bool = False):
        return await self._execute(db_name, query, values, 'insert', returning)

    async def update(self, db_name: str, query:str, values: List, returning:bool = False):
        return await self._execute(db_name, query, values, 'update', returning)

    async def delete(self, db_name: str, query:str, values: List):
        return await self._execute(db_name, query, values, 'delete')

    async def _execute(self, db_name: str, query: str, values: List, _type: str, returning: bool = False):
        """
        Execute SQL query in connection pool
        :param db_name:
        :param query:
        :param values:
        :param _type:
        :return:
        """

        if _type not in ('select', 'insert', 'update', 'delete'):
            raise RuntimeError(
                'Wrong request type {}'.format(_type)
            )
        if not self.dbs[db_name]['master']:
            raise RuntimeError(
                'db {} master is not initialized'.format(db_name)
            )

        pool = self.dbs[db_name]['master']
        if _type == 'select' and 'slave' in self.dbs[db_name]:
            pool = self.dbs[db_name]['slave']

        async with pool.acquire() as conn:
            async with conn.cursor(cursor_factory=DictCursor) as cursor:
                await cursor.execute(query, values)
                if _type == 'select':
                    return await cursor.fetchall()
                else:
                    cursor.connection.commit()
                    if returning:
                        data = await cursor.fetchone()
                    else:
                        data = cursor.rowcount
        return data

def setup(app):
    db = DB(app.settings.DATABASE)
    app.loop.run_until_complete(db.init())
    app.db = db
    app.on_cleanup.append(db.shutdown)
<|endoftext|>


<commit_before>import django

from functools import partial

from django.shortcuts import render
from django.template.loader import render_to_string
from django.template.response import TemplateResponse as BaseTemplateResponse

if django.VERSION < (1, 6):
    from django.conf.urls.defaults import include, url
else:
    from django.conf.urls import include, url

if django.VERSION < (2, 0):
    from django.core.urlresolvers import reverse, NoReverseMatch
else:
    from django.urls import reverse, NoReverseMatch

def patterns(*urls):
    if django.VERSION < (1, 6):
        from django.conf.urls.defaults import patterns
        return patterns('', *urls)
    elif django.VERSION < (1, 10):
        from django.conf.urls import patterns
        return patterns('', *urls)
    return list(urls)

if django.VERSION >= (1, 9):
    if django.VERSION < (2, 0):
        from django.template.library import parse_bits
    else:
        def parse_bits(parser, bits, params, varargs, varkw, defaults,
                       takes_context, name):
            from django.template import library
            return library.parse_bits(
                parser=parser, bits=bits, params=params, varargs=varargs,
                varkw=varkw, defaults=defaults, kwonly=(), kwonly_defaults=(),
                takes_context=takes_context, name=name)

    def generic_tag_compiler(parser, token, params, varargs, varkw, defaults,
                             name, takes_context, node_class):
        """
        Returns a template.Node subclass.

        This got inlined into django.template.library since Django since 1.9, this here
        is a copypasta replacement:
        https://github.com/django/django/blob/stable/1.8.x/django/template/base.py#L1089
        """
        bits = token.split_contents()[1:]
        args, kwargs = parse_bits(parser, bits, params, varargs, varkw,
                                  defaults, takes_context, name)
        return node_class(takes_context, args, kwargs)
else:
    from django.template.base import parse_bits
    from django.template.base import generic_tag_compiler  # noqa

if django.VERSION >= (1, 8):
    # Always use the Django template engine on Django 1.8.
    render_to_string = partial(render_to_string, using='django')
    render = partial(render, using='django')

    class TemplateResponse(BaseTemplateResponse):
        def __init__(self, *args, **kwargs):
            kwargs['using'] = 'django'
            super(TemplateResponse, self).__init__(*args, **kwargs)
else:
    TemplateResponse = BaseTemplateResponse

if django.VERSION < (1, 9):
    from django.core.cache import get_cache
else:
    from django.core.cache import caches
    def get_cache(name):
        return caches[name]

__all__ = ['render_to_string',
           'render',
           'patterns',
           'include',
           'url',
           'reverse',
           'NoReverseMatch',
           'generic_tag_compiler',
           'parse_bits',
           'get_cache']
<commit_msg>make lint<commit_after>import django

from functools import partial

from django.shortcuts import render
from django.template.loader import render_to_string
from django.template.response import TemplateResponse as BaseTemplateResponse

if django.VERSION < (1, 6):
    from django.conf.urls.defaults import include, url
else:
    from django.conf.urls import include, url

if django.VERSION < (2, 0):
    from django.core.urlresolvers import reverse, NoReverseMatch
else:
    from django.urls import reverse, NoReverseMatch

def patterns(*urls):
    if django.VERSION < (1, 6):
        from django.conf.urls.defaults import patterns
        return patterns('', *urls)
    elif django.VERSION < (1, 10):
        from django.conf.urls import patterns
        return patterns('', *urls)
    return list(urls)

if django.VERSION >= (1, 9):
    if django.VERSION < (2, 0):
        from django.template.library import parse_bits
    else:
        def parse_bits(parser, bits, params, varargs, varkw, defaults,
                       takes_context, name):
            from django.template import library
            return library.parse_bits(
                parser=parser, bits=bits, params=params, varargs=varargs,
                varkw=varkw, defaults=defaults, kwonly=(), kwonly_defaults=(),
                takes_context=takes_context, name=name)

    def generic_tag_compiler(parser, token, params, varargs, varkw, defaults,
                             name, takes_context, node_class):
        """
        Returns a template.Node subclass.

        This got inlined into django.template.library since Django since 1.9, this here
        is a copypasta replacement:
        https://github.com/django/django/blob/stable/1.8.x/django/template/base.py#L1089
        """
        bits = token.split_contents()[1:]
        args, kwargs = parse_bits(parser, bits, params, varargs, varkw,
                                  defaults, takes_context, name)
        return node_class(takes_context, args, kwargs)
else:
    from django.template.base import parse_bits
    from django.template.base import generic_tag_compiler  # noqa

if django.VERSION >= (1, 8):
    # Always use the Django template engine on Django 1.8.
    render_to_string = partial(render_to_string, using='django')
    render = partial(render, using='django')

    class TemplateResponse(BaseTemplateResponse):
        def __init__(self, *args, **kwargs):
            kwargs['using'] = 'django'
            super(TemplateResponse, self).__init__(*args, **kwargs)
else:
    TemplateResponse = BaseTemplateResponse

if django.VERSION < (1, 9):
    from django.core.cache import get_cache
else:
    from django.core.cache import caches

    def get_cache(name):
        return caches[name]

__all__ = ['render_to_string',
           'render',
           'patterns',
           'include',
           'url',
           'reverse',
           'NoReverseMatch',
           'generic_tag_compiler',
           'parse_bits',
           'get_cache']
<|endoftext|>