<commit_before>import datetime
import sys
import copy
import threading

from django.conf import settings
from django.contrib import messages
from django.core.exceptions import ImproperlyConfigured
from django.core.mail import EmailMultiAlternatives
from django.http import HttpResponse, HttpResponseRedirect
from django.template import loader
from django.utils import six
if six.PY2:
    # python 2.x need to work with unicode
    from django.utils.encoding import \
        smart_unicode as smart_text, \
        force_unicode as force_text
else:
    from django.utils.encoding import force_text, smart_text
from django.utils.html import escape
from django.utils.translation import ugettext as _
from django.utils.xmlutils import SimplerXMLGenerator
from django.db.models import BooleanField, NullBooleanField

from xadmin.plugins.utils import get_context_dict
from xadmin.sites import site
from xadmin.views import BaseAdminPlugin, ListAdminView
from xadmin.util import json
from xadmin.views.list import ALL_VAR

try:
    import unicodecsv
except ImportError:
    unicodecsv = None

try:
    import xlwt
    has_xlwt = True
except:
    has_xlwt = False

try:
    import xlsxwriter
    has_xlsxwriter = True
except:
    has_xlsxwriter = False

class ExportMenuPlugin(BaseAdminPlugin):

    list_export = ('xlsx', 'xls', 'csv', 'xml', 'json')
    export_names = {'xlsx': 'Excel 2007', 'xls': 'Excel', 'csv': 'CSV',
                    'xml': 'XML', 'json': 'JSON'}

    export_menu_block_template = 'xadmin/blocks/model_list.top_toolbar.exports.html'

    def init_request(self, *args, **kwargs):
        self.list_export = [
            f for f in self.list_export
            if (f != 'xlsx' or has_xlsxwriter) and (f != 'xls' or has_xlwt)]

    def block_top_toolbar(self, context, nodes):
        if self.list_export:
            context.update({
                'show_export_all': self.admin_view.paginator.count > self.admin_view.list_per_page and not ALL_VAR in self.admin_view.request.GET,
                'form_params': self.admin_view.get_form_params({'_do_': 'export'}, ('export_type',)),
                'export_types': [{'type': et, 'name': self.export_names[et]} for et in self.list_export],
            })
            nodes.append(loader.render_to_string(self.export_menu_block_template,
                                                 context=get_context_dict(context)))

class ExportPlugin(BaseAdminPlugin):

    export_mimes = {'xlsx': 'application/vnd.ms-excel',
                    'xls': 'application/vnd.ms-excel', 'csv': 'text/csv',
                    'xml': 'application/xhtml+xml', 'json': 'application/json'}

    export_unicode_csv = False
    export_unicode_encoding = "utf-8"
    export_email_config = {}

    def init_request(self, *args, **kwargs):
        return self.request.GET.get('_do_') == 'export'

    def _format_value(self, o):
        if (o.field is None and getattr(o.attr, 'boolean', False)) or \
           (o.field and isinstance(o.field, (BooleanField, NullBooleanField))):
                value = o.value
        elif str(o.text).startswith(""):
            value = escape(str(o.text)[25:-7])
        else:
            value = escape(str(o.text))
        return value

    def _options_is_on(self, name):
        """Checks if the option value is on. Return bool"""
        return getattr(self.request, self.request.method).get(name, 'off') == 'on'

    def _get_objects(self, context):
        headers = [c for c in context['result_headers'].cells if c.export]
        rows = context['results']

        return [dict([
            (force_text(headers[i].text), self._format_value(o)) for i, o in
            enumerate(filter(lambda c:getattr(c, 'export', False), r.cells))]) for r in rows]

    def _get_datas(self, context):
        rows = context['results']

        new_rows = [[self._format_value(o) for o in
            filter(lambda c:getattr(c, 'export', False), r.cells)] for r in rows]
        new_rows.insert(0, [force_text(c.text) for c in context['result_headers'].cells if c.export])
        return new_rows

    def get_xlsx_export(self, context):
        datas = self._get_datas(context)
        output = six.BytesIO()
        export_header = self._options_is_on('export_xlsx_header')
        model_name = self.opts.verbose_name
        book = xlsxwriter.Workbook(output)
        sheet = book.add_worksheet(
            u"%s %s" % (_(u'Sheet'), force_text(model_name)))
        styles = {'datetime': book.add_format({'num_format': 'yyyy-mm-dd hh:mm:ss'}),
                  'date': book.add_format({'num_format': 'yyyy-mm-dd'}),
                  'time': book.add_format({'num_format': 'hh:mm:ss'}),
                  'header': book.add_format({'font': 'name Times New Roman', 'color': 'red', 'bold': 'on', 'num_format': '#,##0.00'}),
                  'default': book.add_format()}

        if not export_header:
            datas = datas[1:]
        for rowx, row in enumerate(datas):
            for colx, value in enumerate(row):
                if export_header and rowx == 0:
                    cell_style = styles['header']
                else:
                    if isinstance(value, datetime.datetime):
                        cell_style = styles['datetime']
                    elif isinstance(value, datetime.date):
                        cell_style = styles['date']
                    elif isinstance(value, datetime.time):
                        cell_style = styles['time']
                    else:
                        cell_style = styles['default']
                sheet.write(rowx, colx, value, cell_style)
        book.close()

        output.seek(0)
        return output.getvalue()

    def get_xls_export(self, context):
        datas = self._get_datas(context)
        output = six.BytesIO()
        export_header = self._options_is_on('export_xls_header')
        model_name = self.opts.verbose_name
        book = xlwt.Workbook(encoding=self.export_unicode_encoding)
        sheet = book.add_sheet(u"%s %s" % (_(u'Sheet'), force_text(model_name)))
        styles = {'datetime': xlwt.easyxf(num_format_str='yyyy-mm-dd hh:mm:ss'),
                  'date': xlwt.easyxf(num_format_str='yyyy-mm-dd'),
                  'time': xlwt.easyxf(num_format_str='hh:mm:ss'),
                  'header': xlwt.easyxf('font: name Times New Roman, color-index red, bold on', num_format_str='#,##0.00'),
                  'default': xlwt.Style.default_style}

        if not export_header:
            datas = datas[1:]
        for rowx, row in enumerate(datas):
            for colx, value in enumerate(row):
                if export_header and rowx == 0:
                    cell_style = styles['header']
                else:
                    if isinstance(value, datetime.datetime):
                        cell_style = styles['datetime']
                    elif isinstance(value, datetime.date):
                        cell_style = styles['date']
                    elif isinstance(value, datetime.time):
                        cell_style = styles['time']
                    else:
                        cell_style = styles['default']
                sheet.write(rowx, colx, value, style=cell_style)
        book.save(output)

        output.seek(0)
        return output.getvalue()

    def _format_csv_text(self, t):
        if isinstance(t, bool):
            return _('Yes') if t else _('No')
        t = t.replace('"', '""').replace(',', '\,')
        cls_str = str if six.PY3 else basestring
        if isinstance(t, cls_str):
            t = '"%s"' % t
        return t

    def get_csv_export(self, context):
        if self.export_unicode_csv:
            return self.get_unicode_csv_export(context)

        datas = self._get_datas(context)
        stream = []

        if self._options_is_on('export_csv_header'):
            datas = datas[1:]

        for row in datas:
            stream.append(','.join(map(self._format_csv_text, row)))

        return '\r\n'.join(stream)

    def get_unicode_csv_export(self, context):
        """Exports the data in the configured encoding. Default utf8"""
        if unicodecsv is None:
            raise ImproperlyConfigured("Need to install module \"unicodecsv\" "
                                       "in order to export csv as unicode.")
        datas = self._get_datas(context)
        stream = six.BytesIO()
        writer = unicodecsv.writer(stream, encoding=self.export_unicode_encoding)
        writer.writerows(datas)
        return stream.getvalue()

    def _to_xml(self, xml, data):
        if isinstance(data, (list, tuple)):
            for item in data:
                xml.startElement("row", {})
                self._to_xml(xml, item)
                xml.endElement("row")
        elif isinstance(data, dict):
            for key, value in six.iteritems(data):
                key = key.replace(' ', '_')
                xml.startElement(key, {})
                self._to_xml(xml, value)
                xml.endElement(key)
        else:
            xml.characters(smart_text(data))

    def get_xml_export(self, context):
        results = self._get_objects(context)

        stream = six.BytesIO()

        xml = SimplerXMLGenerator(stream, self.export_unicode_encoding)
        xml.startDocument()
        xml.startElement("objects", {})

        self._to_xml(xml, results)

        xml.endElement("objects")
        xml.endDocument()

        return stream.getvalue().split((b'\n'))[1]

    def get_json_export(self, context):
        results = self._get_objects(context)
        return json.dumps({'objects': results}, ensure_ascii=False,
                          indent=(self._options_is_on('export_json_format') and 4 or None))

    def send_mail(self, user, data, context):
        """Send the data file by email"""
        filename, content, file_mimetype = self._get_file_spec(data, context)
        export_email_config = {
            'subject': _('Exported file delivery'),
            'message': _('The file is attached.'),
            'from_email': settings.DEFAULT_FROM_EMAIL,
            'recipient_list': [user.email],
            'fail_silently': True,
            'html_message': False
        }
        export_email_config.update(self.export_email_config)

        html_message = export_email_config.pop('html_message', False)
        fail_silently = export_email_config.pop('fail_silently', True)

        # compat
        export_email_config['body'] = export_email_config.pop('message', '')
        export_email_config['to'] = export_email_config.pop('recipient_list', None)

        mail = EmailMultiAlternatives(**export_email_config)
        if html_message:
            mail.attach_alternative(html_message, 'text/html')

        mail.attach(filename, content, file_mimetype)
        mail.send(fail_silently=fail_silently)
        return True

    def _get_file_spec(self, data, context):
        file_type = data.get('export_type', 'csv')
        content = getattr(self, 'get_%s_export' % file_type)(context)
        filename = u"{0:s}.{1:s}".format(self.opts.verbose_name.replace(' ', '_'),
                                         file_type)
        file_mimetype = self.export_mimes[file_type]
        return filename, content, file_mimetype

    def get_response(self, response, context, *args, **kwargs):
        if self._options_is_on('export_to_email'):
            user = self.request.user
            email = user.email if hasattr(user, 'email') else None
            if email is not None:
                th = threading.Thread(target=self.send_mail,
                                      args=(self.request.user,
                                            copy.deepcopy(self.request.GET),
                                            context.copy()))
                th.start()
                messages.success(self.request, _("The file is sent to your email: "
                                                 "{0:s}".format(email)))
            else:
                messages.warning(self.request, _("Your account does not have an email address."))
            return HttpResponseRedirect(self.request.path)

        filename, content, file_mimetype = self._get_file_spec(self.request.GET, context)

        response = HttpResponse(content_type="{0:s}; charset={1:s}".format(file_mimetype, self.export_unicode_encoding))
        filename_format = u'attachment; filename="{0:s}"'.format(filename)
        response['Content-Disposition'] = filename_format.encode(self.export_unicode_encoding)

        response.write(content)
        return response

    # View Methods
    def get_result_list(self, __):
        if self._options_is_on('all'):
            self.admin_view.list_per_page = sys.maxsize
        return __()

    def result_header(self, item, field_name, row):
        item.export = not item.attr or field_name == '__str__' or getattr(item.attr, 'allow_export', True)
        return item

    def result_item(self, item, obj, field_name, row):
        item.export = item.field or field_name == '__str__' or getattr(item.attr, 'allow_export', True)
        return item

site.register_plugin(ExportMenuPlugin, ListAdminView)
site.register_plugin(ExportPlugin, ListAdminView)
<commit_msg>Sending email with possibility inheritance.<commit_after>import datetime
import sys
import copy
import threading

from django.conf import settings
from django.contrib import messages
from django.core.exceptions import ImproperlyConfigured
from django.core.mail import EmailMultiAlternatives
from django.http import HttpResponse, HttpResponseRedirect
from django.template import loader
from django.utils import six
if six.PY2:
    # python 2.x need to work with unicode
    from django.utils.encoding import \
        smart_unicode as smart_text, \
        force_unicode as force_text
else:
    from django.utils.encoding import force_text, smart_text
from django.utils.html import escape
from django.utils.translation import ugettext as _
from django.utils.xmlutils import SimplerXMLGenerator
from django.db.models import BooleanField, NullBooleanField

from xadmin.plugins.utils import get_context_dict
from xadmin.sites import site
from xadmin.views import BaseAdminPlugin, ListAdminView
from xadmin.util import json
from xadmin.views.list import ALL_VAR

try:
    import unicodecsv
except ImportError:
    unicodecsv = None

try:
    import xlwt
    has_xlwt = True
except:
    has_xlwt = False

try:
    import xlsxwriter
    has_xlsxwriter = True
except:
    has_xlsxwriter = False

class ExportMenuPlugin(BaseAdminPlugin):

    list_export = ('xlsx', 'xls', 'csv', 'xml', 'json')
    export_names = {'xlsx': 'Excel 2007', 'xls': 'Excel', 'csv': 'CSV',
                    'xml': 'XML', 'json': 'JSON'}

    export_menu_block_template = 'xadmin/blocks/model_list.top_toolbar.exports.html'

    def init_request(self, *args, **kwargs):
        self.list_export = [
            f for f in self.list_export
            if (f != 'xlsx' or has_xlsxwriter) and (f != 'xls' or has_xlwt)]

    def block_top_toolbar(self, context, nodes):
        if self.list_export:
            context.update({
                'show_export_all': self.admin_view.paginator.count > self.admin_view.list_per_page and not ALL_VAR in self.admin_view.request.GET,
                'form_params': self.admin_view.get_form_params({'_do_': 'export'}, ('export_type',)),
                'export_types': [{'type': et, 'name': self.export_names[et]} for et in self.list_export],
            })
            nodes.append(loader.render_to_string(self.export_menu_block_template,
                                                 context=get_context_dict(context)))

class ExportPlugin(BaseAdminPlugin):

    export_mimes = {'xlsx': 'application/vnd.ms-excel',
                    'xls': 'application/vnd.ms-excel', 'csv': 'text/csv',
                    'xml': 'application/xhtml+xml', 'json': 'application/json'}

    export_unicode_csv = False
    export_unicode_encoding = "utf-8"
    export_email_config = {}

    def init_request(self, *args, **kwargs):
        return self.request.GET.get('_do_') == 'export'

    def _format_value(self, o):
        if (o.field is None and getattr(o.attr, 'boolean', False)) or \
           (o.field and isinstance(o.field, (BooleanField, NullBooleanField))):
                value = o.value
        elif str(o.text).startswith(""):
            value = escape(str(o.text)[25:-7])
        else:
            value = escape(str(o.text))
        return value

    def _options_is_on(self, name):
        """Checks if the option value is on. Return bool"""
        return getattr(self.request, self.request.method).get(name, 'off') == 'on'

    def _get_objects(self, context):
        headers = [c for c in context['result_headers'].cells if c.export]
        rows = context['results']

        return [dict([
            (force_text(headers[i].text), self._format_value(o)) for i, o in
            enumerate(filter(lambda c:getattr(c, 'export', False), r.cells))]) for r in rows]

    def _get_datas(self, context):
        rows = context['results']

        new_rows = [[self._format_value(o) for o in
            filter(lambda c:getattr(c, 'export', False), r.cells)] for r in rows]
        new_rows.insert(0, [force_text(c.text) for c in context['result_headers'].cells if c.export])
        return new_rows

    def get_xlsx_export(self, context):
        datas = self._get_datas(context)
        output = six.BytesIO()
        export_header = self._options_is_on('export_xlsx_header')
        model_name = self.opts.verbose_name
        book = xlsxwriter.Workbook(output)
        sheet = book.add_worksheet(
            u"%s %s" % (_(u'Sheet'), force_text(model_name)))
        styles = {'datetime': book.add_format({'num_format': 'yyyy-mm-dd hh:mm:ss'}),
                  'date': book.add_format({'num_format': 'yyyy-mm-dd'}),
                  'time': book.add_format({'num_format': 'hh:mm:ss'}),
                  'header': book.add_format({'font': 'name Times New Roman', 'color': 'red', 'bold': 'on', 'num_format': '#,##0.00'}),
                  'default': book.add_format()}

        if not export_header:
            datas = datas[1:]
        for rowx, row in enumerate(datas):
            for colx, value in enumerate(row):
                if export_header and rowx == 0:
                    cell_style = styles['header']
                else:
                    if isinstance(value, datetime.datetime):
                        cell_style = styles['datetime']
                    elif isinstance(value, datetime.date):
                        cell_style = styles['date']
                    elif isinstance(value, datetime.time):
                        cell_style = styles['time']
                    else:
                        cell_style = styles['default']
                sheet.write(rowx, colx, value, cell_style)
        book.close()

        output.seek(0)
        return output.getvalue()

    def get_xls_export(self, context):
        datas = self._get_datas(context)
        output = six.BytesIO()
        export_header = self._options_is_on('export_xls_header')
        model_name = self.opts.verbose_name
        book = xlwt.Workbook(encoding=self.export_unicode_encoding)
        sheet = book.add_sheet(u"%s %s" % (_(u'Sheet'), force_text(model_name)))
        styles = {'datetime': xlwt.easyxf(num_format_str='yyyy-mm-dd hh:mm:ss'),
                  'date': xlwt.easyxf(num_format_str='yyyy-mm-dd'),
                  'time': xlwt.easyxf(num_format_str='hh:mm:ss'),
                  'header': xlwt.easyxf('font: name Times New Roman, color-index red, bold on', num_format_str='#,##0.00'),
                  'default': xlwt.Style.default_style}

        if not export_header:
            datas = datas[1:]
        for rowx, row in enumerate(datas):
            for colx, value in enumerate(row):
                if export_header and rowx == 0:
                    cell_style = styles['header']
                else:
                    if isinstance(value, datetime.datetime):
                        cell_style = styles['datetime']
                    elif isinstance(value, datetime.date):
                        cell_style = styles['date']
                    elif isinstance(value, datetime.time):
                        cell_style = styles['time']
                    else:
                        cell_style = styles['default']
                sheet.write(rowx, colx, value, style=cell_style)
        book.save(output)

        output.seek(0)
        return output.getvalue()

    def _format_csv_text(self, t):
        if isinstance(t, bool):
            return _('Yes') if t else _('No')
        t = t.replace('"', '""').replace(',', '\,')
        cls_str = str if six.PY3 else basestring
        if isinstance(t, cls_str):
            t = '"%s"' % t
        return t

    def get_csv_export(self, context):
        if self.export_unicode_csv:
            return self.get_unicode_csv_export(context)

        datas = self._get_datas(context)
        stream = []

        if self._options_is_on('export_csv_header'):
            datas = datas[1:]

        for row in datas:
            stream.append(','.join(map(self._format_csv_text, row)))

        return '\r\n'.join(stream)

    def get_unicode_csv_export(self, context):
        """Exports the data in the configured encoding. Default utf8"""
        if unicodecsv is None:
            raise ImproperlyConfigured("Need to install module \"unicodecsv\" "
                                       "in order to export csv as unicode.")
        datas = self._get_datas(context)
        stream = six.BytesIO()
        writer = unicodecsv.writer(stream, encoding=self.export_unicode_encoding)
        writer.writerows(datas)
        return stream.getvalue()

    def _to_xml(self, xml, data):
        if isinstance(data, (list, tuple)):
            for item in data:
                xml.startElement("row", {})
                self._to_xml(xml, item)
                xml.endElement("row")
        elif isinstance(data, dict):
            for key, value in six.iteritems(data):
                key = key.replace(' ', '_')
                xml.startElement(key, {})
                self._to_xml(xml, value)
                xml.endElement(key)
        else:
            xml.characters(smart_text(data))

    def get_xml_export(self, context):
        results = self._get_objects(context)

        stream = six.BytesIO()

        xml = SimplerXMLGenerator(stream, self.export_unicode_encoding)
        xml.startDocument()
        xml.startElement("objects", {})

        self._to_xml(xml, results)

        xml.endElement("objects")
        xml.endDocument()

        return stream.getvalue().split((b'\n'))[1]

    def get_json_export(self, context):
        results = self._get_objects(context)
        return json.dumps({'objects': results}, ensure_ascii=False,
                          indent=(self._options_is_on('export_json_format') and 4 or None))

    def send_mail(self, user, data, context):
        """Send the data file by email"""
        email_config = {
            'subject': _('Exported file delivery'),
            'message': _('The file is attached.'),
            'from_email': settings.DEFAULT_FROM_EMAIL,
            'recipient_list': [user.email],
            'fail_silently': True,
            'html_message': False
        }
        email_config.update(self.export_email_config)

        def send_mail_async(config, data, context):
            filename, content, file_mimetype = self._get_file_spec(data, context)

            html_message = config.pop('html_message', False)
            fail_silently = config.pop('fail_silently', True)

            # compat
            config['body'] = config.pop('message', '')
            config['to'] = config.pop('recipient_list', None)

            mail = EmailMultiAlternatives(**config)
            if html_message:
                mail.attach_alternative(html_message, 'text/html')

            mail.attach(filename, content, file_mimetype)
            mail.send(fail_silently=fail_silently)

        thargs = (email_config.copy(),
                  copy.deepcopy(data),
                  context.copy())
        th = threading.Thread(target=send_mail_async, args=thargs)
        th.start()

    def _get_file_spec(self, data, context):
        file_type = data.get('export_type', 'csv')
        content = getattr(self, 'get_%s_export' % file_type)(context)
        filename = u"{0:s}.{1:s}".format(self.opts.verbose_name.replace(' ', '_'),
                                         file_type)
        file_mimetype = self.export_mimes[file_type]
        return filename, content, file_mimetype

    def get_response(self, response, context, *args, **kwargs):
        request = self.request
        if self._options_is_on('export_to_email'):
            user = request.user
            email = user.email if hasattr(user, 'email') else None
            if email is not None:
                self.send_mail(user, request.GET, context)
                messages.success(request, _("The file is sent to your email: "
                                            "{0:s}".format(email)))
            else:
                messages.warning(request, _("Your account does not have an email address."))
            return HttpResponseRedirect(request.path)

        filename, content, file_mimetype = self._get_file_spec(request.GET, context)

        response = HttpResponse(content_type="{0:s}; charset={1:s}".format(file_mimetype, self.export_unicode_encoding))
        filename_format = u'attachment; filename="{0:s}"'.format(filename)
        response['Content-Disposition'] = filename_format.encode(self.export_unicode_encoding)

        response.write(content)
        return response

    # View Methods
    def get_result_list(self, __):
        if self._options_is_on('all'):
            self.admin_view.list_per_page = sys.maxsize
        return __()

    def result_header(self, item, field_name, row):
        item.export = not item.attr or field_name == '__str__' or getattr(item.attr, 'allow_export', True)
        return item

    def result_item(self, item, obj, field_name, row):
        item.export = item.field or field_name == '__str__' or getattr(item.attr, 'allow_export', True)
        return item

site.register_plugin(ExportMenuPlugin, ListAdminView)
site.register_plugin(ExportPlugin, ListAdminView)
<|endoftext|>


<commit_before># coding: utf-8

import socket

from contextlib import closing
from .api import Intoxicated

class Proxy(object):
    """ Represents a Proxy object """

    def __init__(self, **kwargs):
        self.name = kwargs["name"]
        self.upstream = kwargs["upstream"]
        self.api_server = kwargs["api_server"]
        self.enabled = kwargs["enabled"]
        self.listen = kwargs["listen"]

    def destroy(self):
        """ Destroy a Toxiproxy proxy """

        return bool(self.api_server.delete("/proxies/%s" % self.name))

    def disable(self):
        """
        Disables a Toxiproxy - this will drop all active connections and
        stop the proxy from listening.
        """

        return self.__enable_proxy(False)

    def enable(self):
        """
        Enables a Toxiproxy - this will cause the proxy to start listening again.
        """

        return self.__enable_proxy(True)

    def __enable_proxy(self, enabled=False):
        """ Enables or Disable a proxy """

        # Lets build a dictionary to send the data to the Toxiproxy server
        json = {
            "enabled": enabled,
        }

        proxy_info = self.api_server.post("/proxies/%s" % self.name, json=json).json()
        self.__update_proxy(**proxy_info)

        return True

    def __update_proxy(self, **proxy_info):
        """ Update proxy information after some action """

        self.__dict__.update(**proxy_info)

class Toxiproxy(object):
    """ Represents a Toxiproxy server """

    def __init__(self, server_host="127.0.0.1", server_port=8474):
        """ Toxiproxy constructor """

        self.api_server = Intoxicated(server_host, server_port)
        self.proxies = []

    def running(self):
        """ Test if the toxiproxy server is running """

        with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
            is_running = True if sock.connect_ex((self.api_server.host, self.api_server.port)) == 0 else False

        return is_running

    def version(self):
        """ Get the toxiproxy server version """

        if self.running() is True:
            return self.api_server.get("/version").content
        else:
            return False

    def reset(self):
        """ Re-enables all proxies and disables all toxics. """

        return bool(self.api_server.post("/reset"))

    def create(self, upstream, name, listen=None, enabled=None):
        """ Create a toxiproxy proxy """

        # Lets build a dictionary to send the data to the Toxiproxy server
        json = {
            "upstream": upstream,
            "name": name
        }

        if listen is not None:
            json["listen"] = listen
        if enabled is not None:
            json["enabled"] = enabled

        proxy_info = self.api_server.post("/proxies", json=json).json()
        proxy_info["api_server"] = self.api_server

        # Lets create a Proxy object to hold all its data
        proxy = Proxy(**proxy_info)

        # Add the new proxy to the toxiproxy proxies collection
        self.proxies.append(proxy)

        return proxy

    def destroy(self, proxy):
        """ Delete a toxiproxy proxy """

        if proxy.destroy() is True:
            self.proxies.remove(proxy)
            return True
        else:
            return False
<commit_msg>Some refactoring to make it easier to handle proxies<commit_after># coding: utf-8

import socket

from contextlib import closing
from future.utils import raise_with_traceback
from .api import Intoxicated
from .exceptions import ProxyExists

class Proxy(object):
    """ Represents a Proxy object """

    def __init__(self, **kwargs):
        self.name = kwargs["name"]
        self.upstream = kwargs["upstream"]
        self.api_server = kwargs["api_server"]
        self.enabled = kwargs["enabled"]
        self.listen = kwargs["listen"]

    def destroy(self):
        """ Destroy a Toxiproxy proxy """

        return bool(self.api_server.delete("/proxies/%s" % self.name))

    def disable(self):
        """
        Disables a Toxiproxy - this will drop all active connections and
        stop the proxy from listening.
        """

        return self.__enable_proxy(False)

    def enable(self):
        """
        Enables a Toxiproxy - this will cause the proxy to start listening again.
        """

        return self.__enable_proxy(True)

    def __enable_proxy(self, enabled=False):
        """ Enables or Disable a proxy """

        # Lets build a dictionary to send the data to the Toxiproxy server
        json = {
            "enabled": enabled,
        }

        proxy_info = self.api_server.post("/proxies/%s" % self.name, json=json).json()
        self.__update_proxy(**proxy_info)

        return True

    def __update_proxy(self, **proxy_info):
        """ Update proxy information after some action """

        self.__dict__.update(**proxy_info)

class Toxiproxy(object):
    """ Represents a Toxiproxy server """

    def __init__(self, server_host="127.0.0.1", server_port=8474):
        """ Toxiproxy constructor """

        self.api_server = Intoxicated(server_host, server_port)
        self.proxies = {}

    def get_proxy(self, proxy_name):
        """ Retrive a proxy if it exists """

        if proxy_name in self.proxies:
            return self.proxies[proxy_name]
        else:
            return None

    def running(self):
        """ Test if the toxiproxy server is running """

        with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
            is_running = True if sock.connect_ex((self.api_server.host, self.api_server.port)) == 0 else False

        return is_running

    def version(self):
        """ Get the toxiproxy server version """

        if self.running() is True:
            return self.api_server.get("/version").content
        else:
            return False

    def reset(self):
        """ Re-enables all proxies and disables all toxics. """

        return bool(self.api_server.post("/reset"))

    def create(self, upstream, name, listen=None, enabled=None):
        """ Create a toxiproxy proxy """

        if name in self.proxies:
            raise_with_traceback(ProxyExists("This proxy already exists."))

        # Lets build a dictionary to send the data to the Toxiproxy server
        json = {
            "upstream": upstream,
            "name": name
        }

        if listen is not None:
            json["listen"] = listen
        if enabled is not None:
            json["enabled"] = enabled

        proxy_info = self.api_server.post("/proxies", json=json).json()
        proxy_info["api_server"] = self.api_server

        # Lets create a Proxy object to hold all its data
        proxy = Proxy(**proxy_info)

        # Add the new proxy to the toxiproxy proxies collection
        self.proxies.update({proxy.name: proxy})

        return proxy

    def destroy(self, proxy):
        """ Delete a toxiproxy proxy """

        if proxy.destroy() is True:
            del self.proxies[proxy.name]
            return True
        else:
            return False
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Copyright (c) 2017 Aaron LI
# MIT license
#
# Create image from OSKAR simulated visibility data using `WSClean`.
# WSClean: https://sourceforge.net/p/wsclean/
#
# 2017-09-01
#

import os
import sys
import re
import argparse
import subprocess
import time
import tempfile

def printlog(msg, logfile=None, **kwargs):
    if logfile:
        files = [sys.stdout, logfile]
    else:
        files = [sys.stdout]
    for f in files:
        print(msg, file=f, **kwargs)

def wsclean(args, dryrun=False, logfile=None):
    """
    Run the WSClean imager with the provided arguments.

    NOTE
    ----
    * All the WSClean output is also captured and tee'd into a log file
      if specified.  However, the WSClean's fine progress report doesn't
      work due to the buffered I/O...
    * A randomly generated temporary directory is specified, to avoid the
      conflict when running multiple WSClean's on the same MeasurementSet.
    """
    tmpdir = tempfile.TemporaryDirectory()
    cmd = [
        "wsclean", "-tempdir", tmpdir.name,
    ] + [str(arg) for arg in args]  # NOTE: Convert all arguments to strings
    printlog("CMD: %s" % " ".join(cmd), logfile=logfile)
    if dryrun:
        print(">>> DRY RUN MODE <<<")
        tmpdir.cleanup()
        return

    t1 = time.perf_counter()
    with subprocess.Popen(cmd, bufsize=0,  # unbuffered
                          stdout=subprocess.PIPE,
                          stderr=subprocess.STDOUT) as proc:
        for out in proc.stdout:
            printlog(out.decode("utf-8"), logfile=logfile, end="", flush=True)
        retcode = proc.wait()
        if retcode:
            raise subprocess.CalledProcessError(retcode, cmd)
    t2 = time.perf_counter()
    printlog("-----------------------------------------------------------",
             logfile=logfile)
    printlog("WSClean running time: %.1f [min]" % ((t2-t1)/60),
             logfile=logfile)
    printlog("-----------------------------------------------------------",
             logfile=logfile)
    tmpdir.cleanup()

def main():
    parser = argparse.ArgumentParser(
        description="Run WSClean with more handy arguments")
    parser.add_argument("-a", "--args", dest="args",
                        help="additional arguments for WSClean, " +
                        "in a quoted string separated by space, e.g.," +
                        "' -simulate-noise 0.001' (NOTE the beginning space!)")
    parser.add_argument("-d", "--dirty", dest="dirty", action="store_true",
                        help="only create the dirty image by overriding " +
                        "--niter to 0")
    parser.add_argument("-n", "--dry-run", dest="dryrun", action="store_true",
                        help="do not actually run WSClean")
    parser.add_argument("--update-model", dest="update_model",
                        action="store_true",
                        help="write/update the MODEL_DATA column in MS")
    parser.add_argument("--save-weights", dest="save_weights",
                        action="store_true",
                        help="save gridded weights in <name>-weights.fits")
    parser.add_argument("--save-uv", dest="save_uv",
                        action="store_true",
                        help="save gridded uv plane (i.e., FFT of the " +
                        "residual image) in <name>-uv-{real,imag}.fits")
    parser.add_argument("--circular-beam", dest="circular_beam",
                        action="store_true",
                        help="force the fitted beam to be circular, i.e., " +
                        "BMIN == BMAJ")
    parser.add_argument("--beam-size", dest="beam_size", type=float,
                        help="specify the circular beam size (FWHM) in " +
                        "[arcsec] for restoring the clean components; " +
                        "implies --circular-beam")
    parser.add_argument("--wlayers", dest="wlayers", type=int,
                        help="specify the number of w-layers to use")
    parser.add_argument("--simulate-noise", dest="simu_noise", type=float,
                        help="[mJy] replace the given MS with visibilities " +
                        "sampling from a Gaussian distribution of specified " +
                        "sigma to simulate the thermal noise images")
    parser.add_argument("--uv-range", dest="uv_range", default=":",
                        help="uv range [lambda] (i.e., baseline lengths) " +
                        "used for imaging; syntax: '<min>:<max>' " +
                        "(default: ':', i.e., all uv/baselines)")
    parser.add_argument("-w", "--weight", dest="weight", default="uniform",
                        choices=["uniform", "natural", "briggs"],
                        help="weighting method (default: 'uniform')")
    parser.add_argument("-B", "--briggs", dest="briggs",
                        type=float, default=0.0,
                        help="Briggs robustness parameter (default: 0); " +
                        "-1 (uniform) -> 1 (natural)")
    parser.add_argument("-#", "--niter", dest="niter",
                        type=float, default=5e5,
                        help="maximum number of CLEAN iterations " +
                        "(default: 500,000)")
    parser.add_argument("--gain", dest="gain", type=float, default=0.1,
                        help="CLEAN gain for each minor iteration " +
                        "(default: 0.1)")
    parser.add_argument("--mgain", dest="mgain", type=float, default=0.85,
                        help="CLEAN gain for major iterations " +
                        "(default: 0.85)")
    parser.add_argument("-s", "--size", dest="size", type=int,
                        required=True,
                        help="output image size (pixel number on a side)")
    parser.add_argument("-p", "--pixelsize", dest="pixelsize", type=float,
                        required=True,
                        help="output image pixel size [arcsec]")
    parser.add_argument("-G", "--taper-gaus", dest="taper_gaus", type=float,
                        help="taper the weights with a Gaussian function " +
                        "to reduce the contribution of long baselines. " +
                        "Gaussian beam size in [arcsec].")
    parser.add_argument("--fit-spec-order", dest="fit_spec_order", type=int,
                        help="do joined-channel CLEAN by fitting the " +
                        "spectra with [order] polynomial in normal-space")
    #
    exgrp = parser.add_mutually_exclusive_group()
    exgrp.add_argument("-S", "--threshold-nsigma", dest="threshold_nsigma",
                       type=float, default=2.0,
                       help="estimate the noise level <sigma> and stop at " +
                       "nsigma*<sigma> (default: 2.0 <sigma>)")
    exgrp.add_argument("-t", "--threshold", dest="threshold", type=float,
                       help="stopping CLEAN threshold [mJy]")
    #
    parser.add_argument("-N", "--name", dest="name", required=True,
                        help="filename prefix for the output files")
    parser.add_argument("-m", "--ms", nargs="+", help="input visibility MSes")
    args = parser.parse_args()

    nms = len(args.ms)  # i.e., number of MS == number of channels

    cmdargs = [
        "-verbose",
        "-log-time",
        "-pol", "XX",  # OSKAR "Scalar" simulation only gives "XX" component
        "-make-psf",  # always make the PSF, even no cleaning performed
    ]

    if args.dirty:
        cmdargs += ["-niter", 0]  # make dirty image only
    else:
        cmdargs += ["-niter", int(args.niter)]

    if args.weight == "uniform":
        cmdargs += ["-weight", "uniform",
                    "-weighting-rank-filter", 3]
    elif args.weight == "briggs":
        cmdargs += ["-weight", "briggs", args.briggs]
    else:
        cmdargs += ["-weight", args.weight]  # natural
    cmdargs += ["-gain", args.gain]
    cmdargs += ["-mgain", args.mgain]
    cmdargs += ["-size", args.size, args.size]
    cmdargs += ["-scale", "{0}asec".format(args.pixelsize)]

    if args.fit_spec_order:
        cmdargs += ["-joinchannels", "-channelsout", nms,
                    "-fit-spectral-pol", args.fit_spec_order+1]

    if args.update_model:
        cmdargs += ["-update-model-required"]
    else:
        cmdargs += ["-no-update-model-required"]

    if args.save_weights:
        cmdargs += ["-saveweights"]
    if args.save_uv:
        cmdargs += ["-saveuv"]
    if args.circular_beam:
        cmdargs += ["-circularbeam"]
    if args.beam_size:
        cmdargs += ["-beamsize", args.beam_size]
    if args.wlayers:
        cmdargs += ["-nwlayers", args.wlayers]
    if args.simu_noise:
        cmdargs += ["-simulate-noise", args.simu_noise*1e-3]  # [mJy] -> [Jy]

    # uv/baseline range
    uvmin, uvmax = args.uv_range.strip().split(":")
    if uvmin:
        cmdargs += ["-minuv-l", float(uvmin)]
    if uvmax:
        cmdargs += ["-maxuv-l", float(uvmax)]

    if args.threshold:
        cmdargs += ["-threshold", args.threshold*1e-3]  # [mJy] -> [Jy]
    else:
        cmdargs += ["-auto-threshold", args.threshold_nsigma]

    if args.taper_gaus:
        cmdargs += ["-taper-gaussian", args.taper_gaus]

    # additional WSClean arguments
    if args.args:
        extra_args = re.split(r"\s+", args.args.strip())
        print("Additional WSClean arguments:", extra_args)
        cmdargs += extra_args

    nameprefix = args.name.rstrip("-_")
    cmdargs += ["-name", nameprefix]
    cmdargs += args.ms

    if args.dryrun:
        logfile = None
    else:
        logfilename = nameprefix + "-wsclean.log"
        logfile = open(logfilename, "w")
        logfile.write(" ".join(sys.argv) + "\n")
    wsclean(cmdargs, dryrun=args.dryrun, logfile=logfile)

    if args.dirty and not args.dryrun:
        # Remove the output "-image" since it is identical to "-dirty"
        os.remove(nameprefix+"-image.fits")

    if logfile:
        logfile.close()

if __name__ == "__main__":
    main()
<commit_msg>astro/wsclean.py: update parameters against w.r.t. WSClean 2.5<commit_after>#!/usr/bin/env python3
#
# Copyright (c) 2017-2018 Aaron LI
# MIT license
#
# Create image from OSKAR simulated visibility data using `WSClean`.
# WSClean: https://sourceforge.net/p/wsclean/
#
# 2017-09-01
#

import os
import sys
import re
import argparse
import subprocess
import time
import tempfile

def printlog(msg, logfile=None, **kwargs):
    if logfile:
        files = [sys.stdout, logfile]
    else:
        files = [sys.stdout]
    for f in files:
        print(msg, file=f, **kwargs)

def wsclean(args, dryrun=False, logfile=None):
    """
    Run the WSClean imager with the provided arguments.

    NOTE
    ----
    * All the WSClean output is also captured and tee'd into a log file
      if specified.  However, the WSClean's fine progress report doesn't
      work due to the buffered I/O...
    * A randomly generated temporary directory is specified, to avoid the
      conflict when running multiple WSClean's on the same MeasurementSet.
    """
    tmpdir = tempfile.TemporaryDirectory()
    cmd = [
        "wsclean", "-temp-dir", tmpdir.name,
    ] + [str(arg) for arg in args]  # NOTE: Convert all arguments to strings
    printlog("CMD: %s" % " ".join(cmd), logfile=logfile)
    if dryrun:
        print(">>> DRY RUN MODE <<<")
        tmpdir.cleanup()
        return

    t1 = time.perf_counter()
    with subprocess.Popen(cmd, bufsize=0,  # unbuffered
                          stdout=subprocess.PIPE,
                          stderr=subprocess.STDOUT) as proc:
        for out in proc.stdout:
            printlog(out.decode("utf-8"), logfile=logfile, end="", flush=True)
        retcode = proc.wait()
        if retcode:
            raise subprocess.CalledProcessError(retcode, cmd)
    t2 = time.perf_counter()
    printlog("-----------------------------------------------------------",
             logfile=logfile)
    printlog("WSClean running time: %.1f [min]" % ((t2-t1)/60),
             logfile=logfile)
    printlog("-----------------------------------------------------------",
             logfile=logfile)
    tmpdir.cleanup()

def main():
    parser = argparse.ArgumentParser(
        description="Run WSClean with more handy arguments")
    parser.add_argument("-a", "--args", dest="args",
                        help="additional arguments for WSClean, " +
                        "in a quoted string separated by space, e.g.," +
                        "' -simulate-noise 0.001' (NOTE the beginning space!)")
    parser.add_argument("-d", "--dirty", dest="dirty", action="store_true",
                        help="only create the dirty image by overriding " +
                        "--niter to 0")
    parser.add_argument("-n", "--dry-run", dest="dryrun", action="store_true",
                        help="do not actually run WSClean")
    parser.add_argument("--update-model", dest="update_model",
                        action="store_true",
                        help="write/update the MODEL_DATA column in MS")
    parser.add_argument("--save-weights", dest="save_weights",
                        action="store_true",
                        help="save gridded weights in <name>-weights.fits")
    parser.add_argument("--save-uv", dest="save_uv",
                        action="store_true",
                        help="save gridded uv plane (i.e., FFT of the " +
                        "residual image) in <name>-uv-{real,imag}.fits")
    parser.add_argument("--circular-beam", dest="circular_beam",
                        action="store_true",
                        help="force the fitted beam to be circular, i.e., " +
                        "BMIN == BMAJ")
    parser.add_argument("--beam-size", dest="beam_size", type=float,
                        help="specify the circular beam size (FWHM) in " +
                        "[arcsec] for restoring the clean components; " +
                        "implies --circular-beam")
    parser.add_argument("--wlayers", dest="wlayers", type=int,
                        help="specify the number of w-layers to use")
    parser.add_argument("--simulate-noise", dest="simu_noise", type=float,
                        help="[mJy] replace the given MS with visibilities " +
                        "sampling from a Gaussian distribution of specified " +
                        "sigma to simulate the thermal noise images")
    parser.add_argument("--uv-range", dest="uv_range", default=":",
                        help="uv range [lambda] (i.e., baseline lengths) " +
                        "used for imaging; syntax: '<min>:<max>' " +
                        "(default: ':', i.e., all uv/baselines)")
    parser.add_argument("-w", "--weight", dest="weight", default="uniform",
                        choices=["uniform", "natural", "briggs"],
                        help="weighting method (default: 'uniform')")
    parser.add_argument("-B", "--briggs", dest="briggs",
                        type=float, default=0.0,
                        help="Briggs robustness parameter (default: 0); " +
                        "-1 (uniform) -> 1 (natural)")
    parser.add_argument("-#", "--niter", dest="niter",
                        type=float, default=5e5,
                        help="maximum number of CLEAN iterations " +
                        "(default: 500,000)")
    parser.add_argument("--gain", dest="gain", type=float, default=0.1,
                        help="CLEAN gain for each minor iteration " +
                        "(default: 0.1)")
    parser.add_argument("--mgain", dest="mgain", type=float, default=0.85,
                        help="CLEAN gain for major iterations " +
                        "(default: 0.85)")
    parser.add_argument("-s", "--size", dest="size", type=int,
                        required=True,
                        help="output image size (pixel number on a side)")
    parser.add_argument("-p", "--pixelsize", dest="pixelsize", type=float,
                        required=True,
                        help="output image pixel size [arcsec]")
    parser.add_argument("-G", "--taper-gaus", dest="taper_gaus", type=float,
                        help="taper the weights with a Gaussian function " +
                        "to reduce the contribution of long baselines. " +
                        "Gaussian beam size in [arcsec].")
    parser.add_argument("--fit-spec-order", dest="fit_spec_order", type=int,
                        help="do joined-channel CLEAN by fitting the " +
                        "spectra with [order] polynomial in normal-space")
    #
    exgrp = parser.add_mutually_exclusive_group()
    exgrp.add_argument("-S", "--threshold-nsigma", dest="threshold_nsigma",
                       type=float, default=2.0,
                       help="estimate the noise level <sigma> and stop at " +
                       "nsigma*<sigma> (default: 2.0 <sigma>)")
    exgrp.add_argument("-t", "--threshold", dest="threshold", type=float,
                       help="stopping CLEAN threshold [mJy]")
    #
    parser.add_argument("-N", "--name", dest="name", required=True,
                        help="filename prefix for the output files")
    parser.add_argument("-m", "--ms", nargs="+", help="input visibility MSes")
    args = parser.parse_args()

    nms = len(args.ms)  # i.e., number of MS == number of channels

    cmdargs = [
        "-verbose",
        "-log-time",
        "-pol", "XX",  # OSKAR "Scalar" simulation only gives "XX" component
        "-make-psf",  # always make the PSF, even no cleaning performed
    ]

    if args.dirty:
        cmdargs += ["-niter", 0]  # make dirty image only
    else:
        cmdargs += ["-niter", int(args.niter)]

    if args.weight == "uniform":
        cmdargs += ["-weight", "uniform",
                    "-weighting-rank-filter", 3]
    elif args.weight == "briggs":
        cmdargs += ["-weight", "briggs", args.briggs]
    else:
        cmdargs += ["-weight", args.weight]  # natural
    cmdargs += ["-gain", args.gain]
    cmdargs += ["-mgain", args.mgain]
    cmdargs += ["-size", args.size, args.size]
    cmdargs += ["-scale", "{0}asec".format(args.pixelsize)]

    if args.fit_spec_order:
        cmdargs += ["-joinchannels", "-channelsout", nms,
                    "-fit-spectral-pol", args.fit_spec_order+1]

    if args.update_model:
        cmdargs += ["-update-model-required"]
    else:
        cmdargs += ["-no-update-model-required"]

    if args.save_weights:
        cmdargs += ["-saveweights"]
    if args.save_uv:
        cmdargs += ["-saveuv"]
    if args.circular_beam:
        cmdargs += ["-circular-beam"]
    if args.beam_size:
        cmdargs += ["-beamsize", args.beam_size]
    if args.wlayers:
        cmdargs += ["-nwlayers", args.wlayers]
    if args.simu_noise:
        cmdargs += ["-simulate-noise", args.simu_noise*1e-3]  # [mJy] -> [Jy]

    # uv/baseline range
    uvmin, uvmax = args.uv_range.strip().split(":")
    if uvmin:
        cmdargs += ["-minuv-l", float(uvmin)]
    if uvmax:
        cmdargs += ["-maxuv-l", float(uvmax)]

    if args.threshold:
        cmdargs += ["-threshold", args.threshold*1e-3]  # [mJy] -> [Jy]
    else:
        cmdargs += ["-auto-threshold", args.threshold_nsigma]

    if args.taper_gaus:
        cmdargs += ["-taper-gaussian", args.taper_gaus]

    # additional WSClean arguments
    if args.args:
        extra_args = re.split(r"\s+", args.args.strip())
        print("Additional WSClean arguments:", extra_args)
        cmdargs += extra_args

    nameprefix = args.name.rstrip("-_")
    cmdargs += ["-name", nameprefix]
    cmdargs += args.ms

    if args.dryrun:
        logfile = None
    else:
        logfilename = nameprefix + "-wsclean.log"
        logfile = open(logfilename, "w")
        logfile.write(" ".join(sys.argv) + "\n")
    wsclean(cmdargs, dryrun=args.dryrun, logfile=logfile)

    if args.dirty and not args.dryrun:
        # Remove the output "-image" since it is identical to "-dirty"
        os.remove(nameprefix+"-image.fits")

    if logfile:
        logfile.close()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># This file is for training on Cloud ML Engine with XGBoost.

# [START setup]
import datetime
import os
import subprocess
import sys
import pandas as pd
import xgboost as xgb

# Fill in your Cloud Storage bucket name
BUCKET_ID = <YOUR_BUCKET_NAME>
# [END setup]

# [START download-data]
iris_data_filename = 'iris_data.csv'
iris_target_filename = 'iris_target.csv'
data_dir = 'gs://cloud-samples-data/ml-engine/iris'

# gsutil outputs everything to stderr so we need to divert it to stdout.
subprocess.check_call(['gsutil', 'cp', os.path.join(data_dir,
                                                    iris_data_filename),
                       iris_data_filename], stderr=sys.stdout)
subprocess.check_call(['gsutil', 'cp', os.path.join(data_dir,
                                                    iris_target_filename),
                       iris_target_filename], stderr=sys.stdout)
# [END download-data]

# [START load-into-pandas]
# Load data into pandas
iris_data = pd.read_csv(iris_data_filename).values
iris_target = pd.read_csv(iris_target_filename).values
iris_target = iris_target.reshape((iris_target.size,))
# [END load-into-pandas]

# [START train-and-save-model]
# Load data into DMatrix object
dtrain = xgb.DMatrix(iris_data, label=iris_target)

# Train XGBoost model
bst = xgb.train({}, dtrain, 20)

# Export the classifier to a file
model = 'model.bst'
bst.save_model(model)
# [END train-and-save-model]

# [START upload-model]
# Upload the saved model file to Cloud Storage
model_path = os.path.join('gs://', BUCKET_ID, datetime.datetime.now().strftime(
    'iris_%Y%m%d_%H%M%S'), model)
subprocess.check_call(['gsutil', 'cp', model, model_path], stderr=sys.stdout)
# [END upload-model]
<commit_msg>Update iris_training.py<commit_after># This file is for training on Cloud ML Engine with XGBoost.

# [START setup]
import datetime
import os
import subprocess
import sys
import pandas as pd
import xgboost as xgb

# Fill in your Cloud Storage bucket name
BUCKET_ID = '<YOUR_BUCKET_NAME>'
# [END setup]

# [START download-data]
iris_data_filename = 'iris_data.csv'
iris_target_filename = 'iris_target.csv'
data_dir = 'gs://cloud-samples-data/ml-engine/iris'

# gsutil outputs everything to stderr so we need to divert it to stdout.
subprocess.check_call(['gsutil', 'cp', os.path.join(data_dir,
                                                    iris_data_filename),
                       iris_data_filename], stderr=sys.stdout)
subprocess.check_call(['gsutil', 'cp', os.path.join(data_dir,
                                                    iris_target_filename),
                       iris_target_filename], stderr=sys.stdout)
# [END download-data]

# [START load-into-pandas]
# Load data into pandas
iris_data = pd.read_csv(iris_data_filename).values
iris_target = pd.read_csv(iris_target_filename).values
iris_target = iris_target.reshape((iris_target.size,))
# [END load-into-pandas]

# [START train-and-save-model]
# Load data into DMatrix object
dtrain = xgb.DMatrix(iris_data, label=iris_target)

# Train XGBoost model
bst = xgb.train({}, dtrain, 20)

# Export the classifier to a file
model = 'model.bst'
bst.save_model(model)
# [END train-and-save-model]

# [START upload-model]
# Upload the saved model file to Cloud Storage
model_path = os.path.join('gs://', BUCKET_ID, datetime.datetime.now().strftime(
    'iris_%Y%m%d_%H%M%S'), model)
subprocess.check_call(['gsutil', 'cp', model, model_path], stderr=sys.stdout)
# [END upload-model]
<|endoftext|>


<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst

# this indicates whether or not we are in the package's setup.py
try:
    _ASTROPY_SETUP_
except NameError:
    from sys import version_info
    if version_info[0] >= 3:
        import builtins
    else:
        import __builtin__ as builtins
    builtins._ASTROPY_SETUP_ = False
    del version_info
    del builtins

try:
    from .version import version as __version__
except ImportError:
    __version__ = ''
try:
    from .version import githash as __githash__
except ImportError:
    __githash__ = ''

# set up the test command

def _get_test_runner():
    from astropy.tests.helper import TestRunner
    return TestRunner(__path__[0])

def test(package=None, test_path=None, args=None, plugins=None,
         verbose=False, pastebin=None, remote_data=False, pep8=False,
         pdb=False, coverage=False, open_files=False, **kwargs):
    """
    Run the tests using py.test. A proper set of arguments is constructed and
    passed to `pytest.main`.

    Parameters
    ----------
    package : str, optional
        The name of a specific package to test, e.g. 'io.fits' or 'utils'.
        If nothing is specified all default tests are run.

    test_path : str, optional
        Specify location to test by path. May be a single file or
        directory. Must be specified absolutely or relative to the
        calling directory.

    args : str, optional
        Additional arguments to be passed to `pytest.main` in the `args`
        keyword argument.

    plugins : list, optional
        Plugins to be passed to `pytest.main` in the `plugins` keyword
        argument.

    verbose : bool, optional
        Convenience option to turn on verbose output from py.test. Passing
        True is the same as specifying `-v` in `args`.

    pastebin : {'failed','all',None}, optional
        Convenience option for turning on py.test pastebin output. Set to
        'failed' to upload info for failed tests, or 'all' to upload info
        for all tests.

    remote_data : bool, optional
        Controls whether to run tests marked with @remote_data. These
        tests use online data and are not run by default. Set to True to
        run these tests.

    pep8 : bool, optional
        Turn on PEP8 checking via the pytest-pep8 plugin and disable normal
        tests. Same as specifying `--pep8 -k pep8` in `args`.

    pdb : bool, optional
        Turn on PDB post-mortem analysis for failing tests. Same as
        specifying `--pdb` in `args`.

    coverage : bool, optional
        Generate a test coverage report.  The result will be placed in
        the directory htmlcov.

    open_files : bool, optional
        Fail when any tests leave files open.  Off by default, because
        this adds extra run time to the test suite.  Works only on
        platforms with a working `lsof` command.

    parallel : int, optional
        When provided, run the tests in parallel on the specified
        number of CPUs.  If parallel is negative, it will use the all
        the cores on the machine.  Requires the `pytest-xdist` plugin
        is installed. Only available when using Astropy 0.3 or later.

    kwargs
        Any additional keywords passed into this function will be passed
        on to the astropy test runner.  This allows use of test-related
        functionality implemented in later versions of astropy without
        explicitly updating the package template.

    See Also
    --------
    pytest.main : py.test function wrapped by `run_tests`.

    """
    test_runner = _get_test_runner()
    return test_runner.run_tests(
        package=package, test_path=test_path, args=args,
        plugins=plugins, verbose=verbose, pastebin=pastebin,
        remote_data=remote_data, pep8=pep8, pdb=pdb,
        coverage=coverage, open_files=open_files, **kwargs)

if not _ASTROPY_SETUP_:

    import os
    from warnings import warn
    from astropy import config

    # add these here so we only need to cleanup the namespace at the end
    config_dir = None

    if not os.environ.get('ASTROPY_SKIP_CONFIG_UPDATE', False):
        config_dir = os.path.dirname(__file__)
        try:
            config.configuration.update_default_config(__package__, config_dir)
        except config.configuration.ConfigurationDefaultMissingError as e:
            wmsg = (e.args[0] + " Cannot install default profile. If you are "
                    "importing from source, this is expected.")
            warn(config.configuration.ConfigurationDefaultMissingWarning(wmsg))
            del e

    from distutils.version import LooseVersion

    import astropy.coordinates

    # Monkeypatching for plural symbols
    if LooseVersion(astropy.version.version) < LooseVersion('0.3.dev4957'):

        # unfortunately, astropy:py3 and astropy:py2 have different versioning systems
        # this is a hack to avoid monkeypatching for astropy:py3 that has dev versions
        # much lower than astropy:py2
        if astropy.version.minor >= 3:
            from astropy.extern import six
            if six.PY3:
                do_monkeypatch = False
            else:
                do_monkeypatch = True
            del six
        else: # if astropy version is old, just do the patch... this CANNOT be right.
            do_monkeypatch = True

        if do_monkeypatch:

            warn("You are using an 'old' version of astropy prior to the change "
                 "that made all units singular.  astropy is being monkeypatched "
                 "such that degrees and degree are both allow and hours and hour "
                 "are both allowed.  This is NOT normal astropy behavior.")

            def _Angle_degrees_monkeypatch(self):
                return self.degrees
            astropy.coordinates.Angle.degree = property(_Angle_degrees_monkeypatch)
            def _Angle_hours_monkeypatch(self):
                return self.hours
            astropy.coordinates.Angle.hour = property(_Angle_hours_monkeypatch)

    del os, warn, config_dir, LooseVersion  # clean up namespace
<commit_msg>remove monkeypatch for old, unsupported astropy<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst

# this indicates whether or not we are in the package's setup.py
try:
    _ASTROPY_SETUP_
except NameError:
    from sys import version_info
    if version_info[0] >= 3:
        import builtins
    else:
        import __builtin__ as builtins
    builtins._ASTROPY_SETUP_ = False
    del version_info
    del builtins

try:
    from .version import version as __version__
except ImportError:
    __version__ = ''
try:
    from .version import githash as __githash__
except ImportError:
    __githash__ = ''

# set up the test command

def _get_test_runner():
    from astropy.tests.helper import TestRunner
    return TestRunner(__path__[0])

def test(package=None, test_path=None, args=None, plugins=None,
         verbose=False, pastebin=None, remote_data=False, pep8=False,
         pdb=False, coverage=False, open_files=False, **kwargs):
    """
    Run the tests using py.test. A proper set of arguments is constructed and
    passed to `pytest.main`.

    Parameters
    ----------
    package : str, optional
        The name of a specific package to test, e.g. 'io.fits' or 'utils'.
        If nothing is specified all default tests are run.

    test_path : str, optional
        Specify location to test by path. May be a single file or
        directory. Must be specified absolutely or relative to the
        calling directory.

    args : str, optional
        Additional arguments to be passed to `pytest.main` in the `args`
        keyword argument.

    plugins : list, optional
        Plugins to be passed to `pytest.main` in the `plugins` keyword
        argument.

    verbose : bool, optional
        Convenience option to turn on verbose output from py.test. Passing
        True is the same as specifying `-v` in `args`.

    pastebin : {'failed','all',None}, optional
        Convenience option for turning on py.test pastebin output. Set to
        'failed' to upload info for failed tests, or 'all' to upload info
        for all tests.

    remote_data : bool, optional
        Controls whether to run tests marked with @remote_data. These
        tests use online data and are not run by default. Set to True to
        run these tests.

    pep8 : bool, optional
        Turn on PEP8 checking via the pytest-pep8 plugin and disable normal
        tests. Same as specifying `--pep8 -k pep8` in `args`.

    pdb : bool, optional
        Turn on PDB post-mortem analysis for failing tests. Same as
        specifying `--pdb` in `args`.

    coverage : bool, optional
        Generate a test coverage report.  The result will be placed in
        the directory htmlcov.

    open_files : bool, optional
        Fail when any tests leave files open.  Off by default, because
        this adds extra run time to the test suite.  Works only on
        platforms with a working `lsof` command.

    parallel : int, optional
        When provided, run the tests in parallel on the specified
        number of CPUs.  If parallel is negative, it will use the all
        the cores on the machine.  Requires the `pytest-xdist` plugin
        is installed. Only available when using Astropy 0.3 or later.

    kwargs
        Any additional keywords passed into this function will be passed
        on to the astropy test runner.  This allows use of test-related
        functionality implemented in later versions of astropy without
        explicitly updating the package template.

    See Also
    --------
    pytest.main : py.test function wrapped by `run_tests`.

    """
    test_runner = _get_test_runner()
    return test_runner.run_tests(
        package=package, test_path=test_path, args=args,
        plugins=plugins, verbose=verbose, pastebin=pastebin,
        remote_data=remote_data, pep8=pep8, pdb=pdb,
        coverage=coverage, open_files=open_files, **kwargs)

if not _ASTROPY_SETUP_:

    import os
    from warnings import warn
    from astropy import config

    # add these here so we only need to cleanup the namespace at the end
    config_dir = None

    if not os.environ.get('ASTROPY_SKIP_CONFIG_UPDATE', False):
        config_dir = os.path.dirname(__file__)
        try:
            config.configuration.update_default_config(__package__, config_dir)
        except config.configuration.ConfigurationDefaultMissingError as e:
            wmsg = (e.args[0] + " Cannot install default profile. If you are "
                    "importing from source, this is expected.")
            warn(config.configuration.ConfigurationDefaultMissingWarning(wmsg))
            del e

    del os, warn, config_dir # clean up namespace
<|endoftext|>


<commit_before>from .utils import to_bytes

class Message:
    """A message recieved from the IRC network."""

    def __init__(self, raw_message):
        self.raw = raw_message
        self.prefix, self.command, self.params, self.suffix = self.elements()

    def elements(self):
        """
        Split the raw message into it's component parts.

        Adapted from http://stackoverflow.com/a/930706/400691
        """
        message = self.raw.strip()

        prefix = b''
        if message[0:1] == b':':  # Odd slicing required for bytes
            prefix, message = message[1:].split(b' ', 1)

        suffix = b''
        if b' :' in message:
            message, suffix = message.split(b' :', 1)

        command, *params = message.split()
        params = list(filter(None, params))

        return prefix, command, params, suffix

def message_bytes(command, prefix=b'', params=None, suffix=b''):
    command = to_bytes(command)
    prefix = to_bytes(prefix)
    params = list(map(to_bytes, params or []))
    suffix = to_bytes(suffix)

    message = command
    if prefix:
        message = b':' + prefix + b' ' + message
    if params:
        params = b' '.join(params)
        message = message + b' ' + params
    if suffix:
        message = message + b' :' + suffix
    return message
<commit_msg>Better comment<commit_after>from .utils import to_bytes

class Message:
    """A message recieved from the IRC network."""

    def __init__(self, raw_message):
        self.raw = raw_message
        self.prefix, self.command, self.params, self.suffix = self.elements()

    def elements(self):
        """
        Split the raw message into it's component parts.

        Adapted from http://stackoverflow.com/a/930706/400691
        """
        message = self.raw.strip()

        prefix = b''
        # Odd slicing required for bytes to avoid getting int instead of char
        # http://stackoverflow.com/q/28249597/400691
        if message[0:1] == b':':
            prefix, message = message[1:].split(b' ', 1)

        suffix = b''
        if b' :' in message:
            message, suffix = message.split(b' :', 1)

        command, *params = message.split()
        params = list(filter(None, params))

        return prefix, command, params, suffix

def message_bytes(command, prefix=b'', params=None, suffix=b''):
    command = to_bytes(command)
    prefix = to_bytes(prefix)
    params = list(map(to_bytes, params or []))
    suffix = to_bytes(suffix)

    message = command
    if prefix:
        message = b':' + prefix + b' ' + message
    if params:
        params = b' '.join(params)
        message = message + b' ' + params
    if suffix:
        message = message + b' :' + suffix
    return message
<|endoftext|>


<commit_before>from django.db import models

class Acl(models.Model):
    """Acl Model"""
    aclName = models.CharField(blank=True, maxlength=100)
    aclMatch = models.CharField(blank=True, maxlength=100)

    class Admin:
        list_display = ('',)
        search_fields = ('',)

    def __str__(self):
        return self.aclName

class View(models.Model):
    """View Model"""
    viewName = models.CharField(blank=True, maxlength=100)
    matchClients = models.CharField(blank=True, maxlength=100)
    tsigs = models.CharField(blank=True, maxlength=100)

    class Admin:
        list_display = ('',)
        search_fields = ('',)

    def __str__(self):
        return self.viewName

class Domain(models.Model):
    """Domain Model"""
    view = models.ForeignKey(View)
    domains = models.CharField(blank=True, maxlength=100)

    class Admin:
        list_display = ('',)
        search_fields = ('',)

    def __str__(self):
        return "Domain"
<commit_msg>拆分了Model,用于保存数据<commit_after>from django.db import models

class Acl(models.Model):
    """Acl Model"""
    aclName = models.CharField(blank=True, maxlength=100)

    class Admin:
        list_display = ('aclName',)
        search_fields = ('',)

    def __str__(self):
        return self.aclName

class AclMatch(models.Model):
    """AclMatch Model"""
    acl = models.ForeignKey(Acl)
    aclMatch = models.CharField(blank=True, maxlength=100)

    class Admin:
        list_display = ('acl','aclMatch')
        search_fields = ('',)

    def __str__(self):
        return "AclMatch"

class View(models.Model):
    """View Model"""
    viewName = models.CharField(blank=True, maxlength=100)

    class Admin:
        list_display = ('viewName',)
        search_fields = ('',)

    def __str__(self):
        return self.viewName

class ViewMatch(models.Model):
    """ViewMatch Model"""
    view = models.ForeignKey(View)
    viewMatchClient = models.CharField(blank=True, maxlength=100)

    class Admin:
        list_display = ('view','viewMatchClient')
        search_fields = ('',)

    def __str__(self):
        return "ViewMatch"

class ViewTsig(models.Model):
    """ViewTsig Model"""
    view = models.ForeignKey(View)
    tsig = models.CharField(blank=True, maxlength=100)

    class Admin:
        list_display = ('view','tsig')
        search_fields = ('',)

    def __str__(self):
        return "ViewTsig"

class Domain(models.Model):
    """Domain Model"""
    view = models.ForeignKey(View)
    domain = models.CharField(blank=True, maxlength=100)

    class Admin:
        list_display = ('view','domain')
        search_fields = ('',)

    def __str__(self):
        return "Domain"
<|endoftext|>


<commit_before>
def get_variant_display_headers(mall, project, indiv_id_list=None):
    """
    Get the list of header fields to display in a variants table
    Should match below
    TODO: should take annotation spec
    """
    headers = [
        'genes',
        'chr',
        'pos',
        'ref',
        'alt',
        'worst_annotation',
    ]
    headers.extend(project.get_reference_population_slugs())
    headers.extend([
        'polyphen',
        'sift',
        'muttaster',
        'fathmm',
    ])

    if indiv_id_list:
        for indiv_id in indiv_id_list:
            headers.append(indiv_id)
            headers.append(indiv_id+'_gq')
            headers.append(indiv_id+'_dp')

    return headers

def get_display_fields_for_variant(mall, project, variant, indiv_id_list=None):
    """
    Return a list of strings that can be output as a tsv or spreadsheet
    """
    fields = []
    genes = [mall.reference.get_gene_symbol(gene_id) for gene_id in variant.coding_gene_ids]
    fields.append(','.join(genes))
    fields.extend([
        variant.chr,
        str(variant.pos),
        variant.ref,
        variant.alt,
        variant.annotation.get('vep_group', '.'),
    ])
    for ref_population_slug in project.get_reference_population_slugs():
        fields.append(variant.annotation['freqs'][ref_population_slug])
    for field_key in ['polyphen', 'sift', 'muttaster', 'fathmm']:
        fields.append(variant.annotation[field_key])
    if indiv_id_list is None:
        indiv_id_list = []
    for indiv_id in indiv_id_list:
        genotype = variant.get_genotype(indiv_id)
        if genotype is None:
            fields.extend(['.', '.', '.'])
        else:
            if genotype.num_alt == 0:
                fields.append("%s/%s" % (variant.ref, variant.ref))
            elif genotype.num_alt == 1:
                fields.append("%s/%s" % (variant.ref, variant.alt))
            elif genotype.num_alt == 2:
                fields.append("%s/%s" % (variant.alt, variant.alt))
            else:
                fields.append("./.")

            fields.append(str(genotype.gq) if genotype.gq is not None else '.')
            fields.append(genotype.extras['dp'] if genotype.extras.get('dp') is not None else '.')
    return fields<commit_msg>fixed handling for missing polyphen annotations<commit_after>
def get_variant_display_headers(mall, project, indiv_id_list=None):
    """
    Get the list of header fields to display in a variants table
    Should match below
    TODO: should take annotation spec
    """
    headers = [
        'genes',
        'chr',
        'pos',
        'ref',
        'alt',
        'worst_annotation',
    ]
    headers.extend(project.get_reference_population_slugs())
    headers.extend([
        'polyphen',
        'sift',
        'muttaster',
        'fathmm',
    ])

    if indiv_id_list:
        for indiv_id in indiv_id_list:
            headers.append(indiv_id)
            headers.append(indiv_id+'_gq')
            headers.append(indiv_id+'_dp')

    return headers

def get_display_fields_for_variant(mall, project, variant, indiv_id_list=None):
    """
    Return a list of strings that can be output as a tsv or spreadsheet
    """
    fields = []
    genes = [mall.reference.get_gene_symbol(gene_id) for gene_id in variant.coding_gene_ids]
    fields.append(','.join(genes))
    fields.extend([
        variant.chr,
        str(variant.pos),
        variant.ref,
        variant.alt,
        variant.annotation.get('vep_group', '.'),
    ])
    for ref_population_slug in project.get_reference_population_slugs():
        fields.append(variant.annotation['freqs'][ref_population_slug])
    for field_key in ['polyphen', 'sift', 'muttaster', 'fathmm']:
        fields.append(variant.annotation.get(field_key, ''))
    if indiv_id_list is None:
        indiv_id_list = []
    for indiv_id in indiv_id_list:
        genotype = variant.get_genotype(indiv_id)
        if genotype is None:
            fields.extend(['.', '.', '.'])
        else:
            if genotype.num_alt == 0:
                fields.append("%s/%s" % (variant.ref, variant.ref))
            elif genotype.num_alt == 1:
                fields.append("%s/%s" % (variant.ref, variant.alt))
            elif genotype.num_alt == 2:
                fields.append("%s/%s" % (variant.alt, variant.alt))
            else:
                fields.append("./.")

            fields.append(str(genotype.gq) if genotype.gq is not None else '.')
            fields.append(genotype.extras['dp'] if genotype.extras.get('dp') is not None else '.')
    return fields<|endoftext|>