<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import json
import pytest
from django import forms
from django.core.urlresolvers import reverse
from django.utils.encoding import smart_text
from mock import Mock

from saleor.dashboard.product.forms import (ProductClassForm,
                                            ProductClassSelectorForm,
                                            ProductForm)
from saleor.product.models import (Product, ProductAttribute, ProductClass,
                                   ProductVariant)

@pytest.mark.integration
@pytest.mark.django_db
def test_stock_record_update_works(admin_client, product_in_stock):
    variant = product_in_stock.variants.get()
    stock = variant.stock.order_by('-quantity_allocated').first()
    quantity = stock.quantity
    quantity_allocated = stock.quantity_allocated
    url = reverse(
        'dashboard:variant-stock-update', kwargs={
            'product_pk': product_in_stock.pk,
            'variant_pk': variant.pk,
            'stock_pk': stock.pk})
    admin_client.post(url, {
        'variant': stock.variant_id, 'location': stock.location.id,
        'cost_price': stock.cost_price.net,
        'quantity': quantity + 5})
    new_stock = variant.stock.get(pk=stock.pk)
    assert new_stock.quantity == quantity + 5
    assert new_stock.quantity_allocated == quantity_allocated

def test_valid_product_class_form(color_attribute, size_attribute):
    data = {'name': "Testing Class",
            'product_attributes': [color_attribute.pk],
            'variant_attributes': [size_attribute.pk],
            'has_variants': True}
    form = ProductClassForm(data)
    assert form.is_valid()

    # Don't allow same attribute in both fields
    data['variant_attributes'] = [color_attribute.pk, size_attribute.pk]
    data['product_attributes'] = [size_attribute.pk]
    form = ProductClassForm(data)
    assert not form.is_valid()

def test_variantless_product_class_form(color_attribute, size_attribute):
    data = {'name': "Testing Class",
            'product_attributes': [color_attribute.pk],
            'variant_attributes': [],
            'has_variants': False}
    form = ProductClassForm(data)
    assert form.is_valid()

    # Don't allow variant attributes when no variants
    data = {'name': "Testing Class",
            'product_attributes': [color_attribute.pk],
            'variant_attributes': [size_attribute.pk],
            'has_variants': False}
    form = ProductClassForm(data)
    assert not form.is_valid()

def test_edit_used_product_class(db):
    product_class = ProductClass.objects.create(name='New class',
                                                has_variants=True)
    product = Product.objects.create(
        name='Test product', price=10, product_class=product_class)
    ProductVariant.objects.create(product=product, sku='1234')

    # When all products have only one variant you can change
    # has_variants to false
    assert product.variants.all().count() == 1
    data = {'name': product_class.name,
            'product_attributes': product_class.product_attributes.all(),
            'variant_attributes': product_class.variant_attributes.all(),
            'has_variants': False}
    form = ProductClassForm(data, instance=product_class)
    assert form.is_valid()

    data = {'name': product_class.name,
            'product_attributes': product_class.product_attributes.all(),
            'variant_attributes': product_class.variant_attributes.all(),
            'has_variants': True}
    form = ProductClassForm(data, instance=product_class)
    assert form.is_valid()

    # Test has_variants validator which prevents turning off when product
    # has multiple variants
    ProductVariant.objects.create(product=product, sku='12345')
    assert product.variants.all().count() == 2
    data = {'name': product_class.name,
            'product_attributes': product_class.product_attributes.all(),
            'variant_attributes': product_class.variant_attributes.all(),
            'has_variants': False}
    form = ProductClassForm(data, instance=product_class)
    assert not form.is_valid()
    assert 'has_variants' in form.errors.keys()

def test_change_attributes_in_product_form(db, product_in_stock,
                                           color_attribute):
    product = product_in_stock
    product_class = product.product_class
    text_attribute = ProductAttribute.objects.create(slug='author',
                                                     name='Author')
    product_class.product_attributes.add(text_attribute)
    color_value = color_attribute.values.first()
    new_author = 'Main Tester'
    new_color = color_value.pk
    data = {'name': product.name,
            'price': product.price.gross,
            'categories': [c.pk for c in product.categories.all()],
            'description': 'description',
            'attribute-author': new_author,
            'attribute-color': new_color}
    form = ProductForm(data, instance=product)
    assert form.is_valid()
    product = form.save()
    assert product.get_attribute(color_attribute.pk) == smart_text(new_color)
    assert product.get_attribute(text_attribute.pk) == new_author

def test_get_formfield_name_with_unicode_characters(db):
    text_attribute = ProductAttribute.objects.create(slug=u'ąęαβδηθλμπ',
                                                     name=u'ąęαβδηθλμπ')
    assert text_attribute.get_formfield_name() == 'attribute-ąęαβδηθλμπ'

def test_view_product_toggle_publish(db, admin_client, product_in_stock):
    product = product_in_stock
    url = reverse('dashboard:product-publish', kwargs={'pk': product.pk})
    response = admin_client.post(url)
    assert response.status_code == 200
    data = {'success': True, 'is_published': False}
    assert json.loads(response.content.decode('utf8')) == data
    admin_client.post(url)
    product.refresh_from_db()
    assert product.is_published

def test_view_product_delete(db, admin_client, product_in_stock):
    product = product_in_stock
    url = reverse('dashboard:product-delete', kwargs={'pk': product.pk})
    response = admin_client.post(url)
    assert response.status_code == 302
    assert not Product.objects.filter(pk=product.pk)
<commit_msg>Add test for product class deletion view<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import json
import pytest
from django import forms
from django.core.urlresolvers import reverse
from django.utils.encoding import smart_text
from mock import Mock

from saleor.dashboard.product.forms import (ProductClassForm,
                                            ProductClassSelectorForm,
                                            ProductForm)
from saleor.product.models import (Product, ProductAttribute, ProductClass,
                                   ProductVariant)

@pytest.mark.integration
@pytest.mark.django_db
def test_stock_record_update_works(admin_client, product_in_stock):
    variant = product_in_stock.variants.get()
    stock = variant.stock.order_by('-quantity_allocated').first()
    quantity = stock.quantity
    quantity_allocated = stock.quantity_allocated
    url = reverse(
        'dashboard:variant-stock-update', kwargs={
            'product_pk': product_in_stock.pk,
            'variant_pk': variant.pk,
            'stock_pk': stock.pk})
    admin_client.post(url, {
        'variant': stock.variant_id, 'location': stock.location.id,
        'cost_price': stock.cost_price.net,
        'quantity': quantity + 5})
    new_stock = variant.stock.get(pk=stock.pk)
    assert new_stock.quantity == quantity + 5
    assert new_stock.quantity_allocated == quantity_allocated

def test_valid_product_class_form(color_attribute, size_attribute):
    data = {'name': "Testing Class",
            'product_attributes': [color_attribute.pk],
            'variant_attributes': [size_attribute.pk],
            'has_variants': True}
    form = ProductClassForm(data)
    assert form.is_valid()

    # Don't allow same attribute in both fields
    data['variant_attributes'] = [color_attribute.pk, size_attribute.pk]
    data['product_attributes'] = [size_attribute.pk]
    form = ProductClassForm(data)
    assert not form.is_valid()

def test_variantless_product_class_form(color_attribute, size_attribute):
    data = {'name': "Testing Class",
            'product_attributes': [color_attribute.pk],
            'variant_attributes': [],
            'has_variants': False}
    form = ProductClassForm(data)
    assert form.is_valid()

    # Don't allow variant attributes when no variants
    data = {'name': "Testing Class",
            'product_attributes': [color_attribute.pk],
            'variant_attributes': [size_attribute.pk],
            'has_variants': False}
    form = ProductClassForm(data)
    assert not form.is_valid()

def test_edit_used_product_class(db):
    product_class = ProductClass.objects.create(name='New class',
                                                has_variants=True)
    product = Product.objects.create(
        name='Test product', price=10, product_class=product_class)
    ProductVariant.objects.create(product=product, sku='1234')

    # When all products have only one variant you can change
    # has_variants to false
    assert product.variants.all().count() == 1
    data = {'name': product_class.name,
            'product_attributes': product_class.product_attributes.all(),
            'variant_attributes': product_class.variant_attributes.all(),
            'has_variants': False}
    form = ProductClassForm(data, instance=product_class)
    assert form.is_valid()

    data = {'name': product_class.name,
            'product_attributes': product_class.product_attributes.all(),
            'variant_attributes': product_class.variant_attributes.all(),
            'has_variants': True}
    form = ProductClassForm(data, instance=product_class)
    assert form.is_valid()

    # Test has_variants validator which prevents turning off when product
    # has multiple variants
    ProductVariant.objects.create(product=product, sku='12345')
    assert product.variants.all().count() == 2
    data = {'name': product_class.name,
            'product_attributes': product_class.product_attributes.all(),
            'variant_attributes': product_class.variant_attributes.all(),
            'has_variants': False}
    form = ProductClassForm(data, instance=product_class)
    assert not form.is_valid()
    assert 'has_variants' in form.errors.keys()

def test_change_attributes_in_product_form(db, product_in_stock,
                                           color_attribute):
    product = product_in_stock
    product_class = product.product_class
    text_attribute = ProductAttribute.objects.create(slug='author',
                                                     name='Author')
    product_class.product_attributes.add(text_attribute)
    color_value = color_attribute.values.first()
    new_author = 'Main Tester'
    new_color = color_value.pk
    data = {'name': product.name,
            'price': product.price.gross,
            'categories': [c.pk for c in product.categories.all()],
            'description': 'description',
            'attribute-author': new_author,
            'attribute-color': new_color}
    form = ProductForm(data, instance=product)
    assert form.is_valid()
    product = form.save()
    assert product.get_attribute(color_attribute.pk) == smart_text(new_color)
    assert product.get_attribute(text_attribute.pk) == new_author

def test_get_formfield_name_with_unicode_characters(db):
    text_attribute = ProductAttribute.objects.create(slug=u'ąęαβδηθλμπ',
                                                     name=u'ąęαβδηθλμπ')
    assert text_attribute.get_formfield_name() == 'attribute-ąęαβδηθλμπ'

def test_view_product_toggle_publish(db, admin_client, product_in_stock):
    product = product_in_stock
    url = reverse('dashboard:product-publish', kwargs={'pk': product.pk})
    response = admin_client.post(url)
    assert response.status_code == 200
    data = {'success': True, 'is_published': False}
    assert json.loads(response.content.decode('utf8')) == data
    admin_client.post(url)
    product.refresh_from_db()
    assert product.is_published

def test_view_product_delete(db, admin_client, product_in_stock):
    product = product_in_stock
    url = reverse('dashboard:product-delete', kwargs={'pk': product.pk})
    response = admin_client.post(url)
    assert response.status_code == 302
    assert not Product.objects.filter(pk=product.pk)

def test_view_product_class_delete(db, admin_client, product_in_stock):
    product_class = product_in_stock.product_class
    url = reverse('dashboard:product-class-delete', kwargs={'pk': product_class.pk})
    response = admin_client.post(url)
    assert response.status_code == 302
    assert not ProductClass.objects.filter(pk=product_class.pk)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# This file is part of Shuup.
#
# Copyright (c) 2012-2016, Shoop Ltd. All rights reserved.
#
# This source code is licensed under the AGPLv3 license found in the
# LICENSE file in the root directory of this source tree.
from __future__ import unicode_literals

from django.utils.html import escape
from django.utils.translation import ugettext_lazy as _

from shuup.admin.utils.picotable import (
    ChoicesFilter, Column, DateRangeFilter, MultiFieldTextFilter, RangeFilter,
    TextFilter
)
from shuup.admin.utils.views import PicotableListView
from shuup.core.models import Order, OrderStatus, PaymentStatus, ShippingStatus
from shuup.utils.i18n import format_money, get_locally_formatted_datetime

class OrderListView(PicotableListView):
    model = Order
    columns = [
        Column("identifier", _(u"Order"), linked=True, filter_config=TextFilter(operator="startswith")),
        Column("order_date", _(u"Order Date"), display="format_order_date", filter_config=DateRangeFilter()),
        Column(
            "customer_name", _(u"Customer"), display="format_customer_name",
            filter_config=MultiFieldTextFilter(filter_fields=("customer__email", "customer__name"))
        ),
        Column("status", _(u"Status"), filter_config=ChoicesFilter(choices=OrderStatus.objects.all())),
        Column("payment_status", _(u"Payment Status"), filter_config=ChoicesFilter(choices=PaymentStatus.choices)),
        Column("shipping_status", _(u"Shipping Status"), filter_config=ChoicesFilter(choices=ShippingStatus.choices)),
        Column(
            "taxful_total_price", _(u"Total"), sort_field="taxful_total_price_value",
            display="format_taxful_total_price", class_name="text-right",
            filter_config=RangeFilter(field_type="number", filter_field="taxful_total_price_value")
        ),
    ]

    def get_queryset(self):
        return super(OrderListView, self).get_queryset().exclude(deleted=True)

    def format_customer_name(self, instance, *args, **kwargs):
        return instance.get_customer_name() or ""

    def format_order_date(self, instance, *args, **kwargs):
        return get_locally_formatted_datetime(instance.order_date)

    def format_taxful_total_price(self, instance, *args, **kwargs):
        return escape(format_money(instance.taxful_total_price))

    def get_object_abstract(self, instance, item):
        return [
            {"text": "%s" % instance, "class": "header"},
            {"title": _(u"Total"), "text": item["taxful_total_price"]},
            {"title": _(u"Status"), "text": item["status"]}
        ]
<commit_msg>Admin: Fix bug in sort and add new fields to customer name filter<commit_after># -*- coding: utf-8 -*-
# This file is part of Shuup.
#
# Copyright (c) 2012-2016, Shoop Ltd. All rights reserved.
#
# This source code is licensed under the AGPLv3 license found in the
# LICENSE file in the root directory of this source tree.
from __future__ import unicode_literals

from django.utils.html import escape
from django.utils.translation import ugettext_lazy as _

from shuup.admin.utils.picotable import (
    ChoicesFilter, Column, DateRangeFilter, MultiFieldTextFilter, RangeFilter,
    TextFilter
)
from shuup.admin.utils.views import PicotableListView
from shuup.core.models import Order, OrderStatus, PaymentStatus, ShippingStatus
from shuup.utils.i18n import format_money, get_locally_formatted_datetime

class OrderListView(PicotableListView):
    model = Order
    columns = [
        Column("identifier", _(u"Order"), linked=True, filter_config=TextFilter(operator="startswith")),
        Column("order_date", _(u"Order Date"), display="format_order_date", filter_config=DateRangeFilter()),
        Column(
            "customer", _(u"Customer"), display="format_customer_name",
            filter_config=MultiFieldTextFilter(
                filter_fields=(
                    "customer__email", "customer__name", "billing_address__name",
                    "shipping_address__name", "orderer__name"
                )
            )
        ),
        Column("status", _(u"Status"), filter_config=ChoicesFilter(choices=OrderStatus.objects.all())),
        Column("payment_status", _(u"Payment Status"), filter_config=ChoicesFilter(choices=PaymentStatus.choices)),
        Column("shipping_status", _(u"Shipping Status"), filter_config=ChoicesFilter(choices=ShippingStatus.choices)),
        Column(
            "taxful_total_price", _(u"Total"), sort_field="taxful_total_price_value",
            display="format_taxful_total_price", class_name="text-right",
            filter_config=RangeFilter(field_type="number", filter_field="taxful_total_price_value")
        ),
    ]

    def get_queryset(self):
        return super(OrderListView, self).get_queryset().exclude(deleted=True)

    def format_customer_name(self, instance, *args, **kwargs):
        return instance.get_customer_name() or ""

    def format_order_date(self, instance, *args, **kwargs):
        return get_locally_formatted_datetime(instance.order_date)

    def format_taxful_total_price(self, instance, *args, **kwargs):
        return escape(format_money(instance.taxful_total_price))

    def get_object_abstract(self, instance, item):
        return [
            {"text": "%s" % instance, "class": "header"},
            {"title": _(u"Total"), "text": item["taxful_total_price"]},
            {"title": _(u"Status"), "text": item["status"]}
        ]
<|endoftext|>


<commit_before>"""
GeoDjango introspection rules
"""

import django
from django.conf import settings

from south.modelsinspector import add_introspection_rules

def has_spatialite():
    "Checks for the presence of SpataiLite"
    try:
        from ctypes.util import find_library
    except ImportError:
        return False
    from django.conf import settings
    return bool(getattr(settings, 'SPATIALITE_LIBRARY_PATH', find_library('spatialite')))

def has_geos():
    try:
        from django.contrib.gis.geos import libgeos
    except ImportError:
        return False
    else:
        return True

# First, work out if GIS is enabled
# (If it isn't importing the field will fail)
has_gis = has_geos() and \
          ((settings.DATABASE_ENGINE in ["postgresql", "postgresql_psycopg2", "mysql"]) or \
          (settings.DATABASE_ENGINE == "sqlite3" and has_spatialite()))

if has_gis:
    # Alright,import the field
    from django.contrib.gis.db.models.fields import GeometryField

    # Make some introspection rules
    if django.VERSION[0] == 1 and django.VERSION[1] >= 1:
        # Django 1.1's gis module renamed these.
        rules = [
            (
                (GeometryField, ),
                [],
                {
                    "srid": ["srid", {"default": 4326}],
                    "spatial_index": ["spatial_index", {"default": True}],
                    "dim": ["dim", {"default": 2}],
                },
            ),
        ]
    else:
        rules = [
            (
                (GeometryField, ),
                [],
                {
                    "srid": ["_srid", {"default": 4326}],
                    "spatial_index": ["_spatial_index", {"default": True}],
                    "dim": ["_dim", {"default": 2}],
                },
            ),
        ]

    # Install them
    add_introspection_rules(rules, ["^django\.contrib\.gis"])<commit_msg>Catch more errors for the GeoDjango import test<commit_after>"""
GeoDjango introspection rules
"""

import django
from django.conf import settings

from south.modelsinspector import add_introspection_rules

def has_spatialite():
    "Checks for the presence of SpataiLite"
    try:
        from ctypes.util import find_library
    except ImportError:
        return False
    from django.conf import settings
    return bool(getattr(settings, 'SPATIALITE_LIBRARY_PATH', find_library('spatialite')))

def has_geos():
    try:
        from django.contrib.gis.geos import libgeos
    except (ImportError, OSError):
        return False
    else:
        return True

# First, work out if GIS is enabled
# (If it isn't importing the field will fail)
has_gis = has_geos() and \
          ((settings.DATABASE_ENGINE in ["postgresql", "postgresql_psycopg2", "mysql"]) or \
          (settings.DATABASE_ENGINE == "sqlite3" and has_spatialite()))

if has_gis:
    # Alright,import the field
    from django.contrib.gis.db.models.fields import GeometryField

    # Make some introspection rules
    if django.VERSION[0] == 1 and django.VERSION[1] >= 1:
        # Django 1.1's gis module renamed these.
        rules = [
            (
                (GeometryField, ),
                [],
                {
                    "srid": ["srid", {"default": 4326}],
                    "spatial_index": ["spatial_index", {"default": True}],
                    "dim": ["dim", {"default": 2}],
                },
            ),
        ]
    else:
        rules = [
            (
                (GeometryField, ),
                [],
                {
                    "srid": ["_srid", {"default": 4326}],
                    "spatial_index": ["_spatial_index", {"default": True}],
                    "dim": ["_dim", {"default": 2}],
                },
            ),
        ]

    # Install them
    add_introspection_rules(rules, ["^django\.contrib\.gis"])<|endoftext|>


<commit_before>#! /usr/bin/env python

from __future__ import print_function, absolute_import, division

'''
Contains functions that facilitate the downloading of COS data
via XML requests to MAST.

Use:
    This script is intended to be imported by other modules that
    download COS data.
'''

__author__ = "Jo Taylor"
__date__ = "04-13-2016"
__maintainer__ = "Jo Taylor"
__email__ = "jotaylor@stsci.edu"

import re
import pickle
import os
import urllib
import string
from datetime import datetime
import time
import pdb
import stat
try:
    from http.client import HTTPSConnection
except ImportError:
    from httplib import HTTPSConnection

from .manualabor import work_laboriously
from .SignStsciRequest import SignStsciRequest
from .logging_dec import log_function

MAX_RETRIEVAL = 20
BASE_DIR = "/grp/hst/cos2/smov_testing"
MYUSER = "jotaylor"
PERM_755 = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH
REQUEST_TEMPLATE = string.Template('\
<?xml version=\"1.0\"?> \n \
<!DOCTYPE distributionRequest SYSTEM \"http://dmswww.stsci.edu/dtd/sso/distribution.dtd\"> \n \
  <distributionRequest> \n \
    <head> \n \
      <requester userId = \"$archive_user\" email = \"$email\" source = "starview"/> \n \
      <delivery> \n \
        <ftp hostName = \"$ftp_host\" loginName = \"$ftp_user\" directory = \"$ftp_dir\" secure = \"true\" /> \n \
      </delivery> \n \
      <process compression = \"none\"/> \n \
    </head> \n \
    <body> \n \
      <include> \n \
        <select> \n \
          $suffix\n \
        </select> \n \
        $datasets \n \
      </include> \n \
    </body> \n \
  </distributionRequest> \n' )

#-----------------------------------------------------------------------------#
#-----------------------------------------------------------------------------#

#@log_function
def build_xml_request(dest_dir, datasets):
    '''
    Build the xml request string.

    Parameters:
        dest_dir : string
            The path fo the directory in which the data will be downloaded.
        datasets : list
            A list of datasets to be requested.

    Returns:
        xml_request : string
            The xml request string.
    '''
    archive_user = MYUSER
    email = MYUSER + "@stsci.edu"
    host = "plhstins1.stsci.edu"
    ftp_user = MYUSER
    ftp_dir = dest_dir
    # Not currently using suffix dependence.
    suffix = "<suffix name=\"*\" />"
    dataset_str = "\n"
    for item in datasets:
        dataset_str = "{0}          <rootname>{1}</rootname>\n".format(dataset_str,item)

    request_str = REQUEST_TEMPLATE.safe_substitute(
        archive_user = archive_user,
        email = email,
        ftp_host = host,
        ftp_dir =ftp_dir,
        ftp_user = ftp_user,
        suffix = suffix,
        datasets = dataset_str)
    xml_request = string.Template(request_str)
    xml_request = xml_request.template

    return xml_request

#-----------------------------------------------------------------------------#
#-----------------------------------------------------------------------------#

#@log_function
def submit_xml_request(xml_file):
    '''
    Submit the xml request to MAST.

    Parameters:
        xml_file : string
            The xml request string.

    Returns:
        response : httplib object
            The xml request submission result.
    '''
    home = os.environ.get("HOME")
    user = os.environ.get("USER")

    signer = SignStsciRequest()
    request_xml_str = signer.signRequest("{0}/.ssh/privkey.pem".format(home),
                                         xml_file)
    params = urllib.urlencode({
        "dadshost": "dmsops1.stsci.edu",
        "dadsport": 4703,
        "mission": "HST",
        "request": request_xml_str})
    headers = {"Accept": "text/html",
               "User-Agent": "{0}PythonScript".format(user)}
    req = HTTPSConnection("archive.stsci.edu")
    req.request("POST", "/cgi-bin/dads.cgi", params, headers)
    response = req.getresponse().read()

    return response

#-----------------------------------------------------------------------------#
#-----------------------------------------------------------------------------#

#@log_function
def retrieve_data(dest_dir, datasets):
    '''
    For a given list of datasets, submit an xml request to MAST
    to retrieve them to a specified directory.

    Parameters:
        dest_dir : string
            The path to the directory to which the data will be downloaded.
        datasets : list
            A list of datasets to be requested.

    Returns:
        tracking_ids : list
            The tracking IDs for all submitted for a given program.
    '''
    dataset_lists = (datasets[i:i+MAX_RETRIEVAL]
                     for i in xrange(0, len(datasets), MAX_RETRIEVAL))
    tracking_ids = []
    for item in dataset_lists:
        try:
            xml_file = build_xml_request(dest_dir, item)
            result = submit_xml_request(xml_file)
#            print(xml_file)
#            print(result)
            tmp_id = re.search("("+MYUSER+"[0-9]{5})", result).group()
            tracking_ids.append(tmp_id)
        # If you can't get a ID, MAST is most likely down.
        except AttributeError:
            print("Something is wrong on the MAST side...")
            print("Unsuccesful request for {0}".format(item))
            print("Continuing to next dataset.")
            pass
    return tracking_ids

#-----------------------------------------------------------------------------#
#-----------------------------------------------------------------------------#

#@log_function
def everything_retrieved(tracking_id):
    '''
    Check every 15 minutes to see if all submitted datasets have been
    retrieved. Based on code from J. Ely.

    Parameters:
        tracking_id : string
            A submission ID string..

    Returns:
        done : bool
            Boolean specifying is data is retrieved or not.
        killed : bool
            Boolean specifying is request was killed.
    '''

    done = False
    killed = False
#    print tracking_id
    status_url = "http://archive.stsci.edu/cgi-bin/reqstat?reqnum=={0}".format(tracking_id)
    # In case connection to URL is down.
    tries = 5
    while tries > 0:
        try:
            urllines = urllib.urlopen(status_url).readlines()
        except IOError:
            print("Something went wrong connecting to {0}.".format(status_url))
            tries -= 1
            time.sleep(30)
            killed = True
        else:
            tries = -100
            for line in urllines:
                if "State" in line:
                    if "COMPLETE" in line:
                        done = True
                        killed = False
                    elif "KILLED" in line:
                        killed = True

    return done, killed

#-----------------------------------------------------------------------------#
#-----------------------------------------------------------------------------#

#@log_function
def cycle_thru(prop_dict, prop, all_tracking_ids_tmp):
    '''
    For a given proposal, determine path to put data.
    Retrieve data for a proposal and keep track of the tracking IDs
    for each submission (each proposal may have multiple submissions
    depending on number of datasets in proposal).

    Parameters:
        prop_dict : dictionary
            A dictionary whose keys are proposal IDs and keys are
            dataset names for each ID.
        prop : int
            The proposal ID for which to retrieve data.
        all_tracking_ids_tmp : list
            Running tally of all tracking IDs for all propsals retrieved
            to date.

    Returns:
        all_tracking_ids : list
            Running tally of all tracking IDs for all propsals retrieved
            to date.
    '''

    prop_dir = os.path.join(BASE_DIR, str(prop))
    if not os.path.exists(prop_dir):
        os.chmod(BASE_DIR, PERM_755)
        os.mkdir(prop_dir)
    os.chmod(prop_dir, PERM_755)
    print("I am retrieving {0} dataset(s) for {1}".format(len(prop_dict[prop]),prop))
    ind_id = retrieve_data(prop_dir, prop_dict[prop])
    for item in ind_id:
        all_tracking_ids_tmp.append(item)

    return all_tracking_ids_tmp

#-----------------------------------------------------------------------------#
#-----------------------------------------------------------------------------#

#@log_function
def run_all_retrievals(prop_dict=None, pkl_file=None, run_labor=True):
    '''
    Open the pickle file containing a dictionary of all missing COS data
    to be retrieved. It is set up to handle all situations (if run daily=few
    programs, if re-retrieving all data=hundreds of programs). If there
    are more than 100 programs to be retrieved, stop every 100 programs and
    run manualabor to calibrate and zip files. If this is not done, you will
    QUICKLY run out of storage when running manualabor later.
    Split the total number of programs into groups, once a defined number
    of programs have been retrieved (int_num), add more to the queue.
    Using this method, there are always pending programs (as opposed to
    retrieving X, waiting until finished, then starting another X).
    For each proposal in a group, use cycle_thru to request all datasets.
    If there are too many datasets, it will split it up for manageability.

    Paramaters:
        pkl_file : string
            Name of pickle file with data to be retrieved.

    Returns:
        Nothing
    '''

    if pkl_file:
        prop_dict = pickle.load(open(pkl_file, "rb"))
    prop_dict_keys = prop_dict.keys()
    pstart = 0
    pend = 10 # should be 10
    int_num = 5 # should be 5
    century = 50 # should be 50
    all_tracking_ids = []
    end_msg = "\nAll data from {0} programs were successfully " \

    if pend > len(prop_dict_keys):
        for prop in prop_dict_keys:
            all_tracking_ids = cycle_thru(prop_dict, prop, all_tracking_ids)
        print(end_msg.format(len(prop_dict_keys)))

    # While the number of processed programs is less than total programs
    while pend < len(prop_dict_keys):

        # If there are more than N*50 programs to be retrieved, stop
        # and calibrate and zip.
        if pend > century:
            century += 50 # should be 50
            print("Pausing retrieval to calibrate and zip current data")
            work_laboriously(prl=True)
        # For each proposal (prop) in the current grouping (total number
        # of programs split up for manageability), retrieve data for it.
        for prop in prop_dict_keys[pstart:pend]:
            all_tracking_ids = cycle_thru(prop_dict, prop, all_tracking_ids)
        # This while loop keeps track of how many submitted programs in the
        # current group (defined as from pstart:pend) have been entirely
        # retrieved, stored in counter. Once the number of finished
        # programs reaches (total# - int_num), another int_num of programs
        # are added to the queue. While data not retrieved, wait 15 minutes
        # before checking again.
        counter = []
        num_ids = len(all_tracking_ids)
        while sum(counter) < (num_ids - int_num):
            counter = []
            for tracking_id in all_tracking_ids:
                status,badness = everything_retrieved(tracking_id)
                counter.append(status)
                if badness:
                    print("!"*70)
                    print("RUH ROH!!! Request {0} was killed or cannot be connected!".format(tracking_id))
                    counter.append(badness)
            current_retrieved = [all_tracking_ids[i] for i in
                                 xrange(len(counter)) if not counter[i]]
            if sum(counter) < (num_ids - int_num):
                print(datetime.now())
                print("Data not yet delivered for {0}. Checking again in "
                      "5 minutes".format(current_retrieved))
                time.sleep(350)
        # When total# - int_num programs have been retrieved, add int_num more
        # to queue.
        else:
            print("\nData delivered, adding {0} program(s) to queue".
                  format(int_num))
            pstart = pend
            pend += int_num
    # When pend > total # of programs, it does not mean all have been
    # retrieved. Check, and retrieve if so.
    else:
        "delivered. ".format(len(prop_dict_keys))
        if (len(prop_dict_keys) - (pend-int_num)) > 0:
            for prop in prop_dict_keys[pend-int_num:]:
                all_tracking_ids = cycle_thru(prop_dict, prop, all_tracking_ids)
            print(end_msg)
        else:
            print(end_msg)
    if run_labor:
        print("Beginning manual labor now...")
        work_laboriously(prl=True)

#-----------------------------------------------------------------------------#
#-----------------------------------------------------------------------------#

if __name__ == "__main__":
    pkl_file = "filestoretrieve.p"
    try:
        run_all_retrievals(prop_dict=None, pkl_file=pkl_file, run_labor=True)
    except Exception as e:
        print(Exception, e)
<commit_msg>Changed name of everything_retrieved function and added new function to streamline<commit_after>#! /usr/bin/env python

from __future__ import print_function, absolute_import, division

'''
Contains functions that facilitate the downloading of COS data
via XML requests to MAST.

Use:
    This script is intended to be imported by other modules that
    download COS data.
'''

__author__ = "Jo Taylor"
__date__ = "04-13-2016"
__maintainer__ = "Jo Taylor"
__email__ = "jotaylor@stsci.edu"

import re
import pickle
import os
import urllib
import string
from datetime import datetime
import time
import pdb
import stat
try:
    from http.client import HTTPSConnection
except ImportError:
    from httplib import HTTPSConnection

from .manualabor import work_laboriously
from .SignStsciRequest import SignStsciRequest
from .logging_dec import log_function

MAX_RETRIEVAL = 20
BASE_DIR = "/grp/hst/cos2/smov_testing"
MYUSER = "jotaylor"
PERM_755 = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH
REQUEST_TEMPLATE = string.Template('\
<?xml version=\"1.0\"?> \n \
<!DOCTYPE distributionRequest SYSTEM \"http://dmswww.stsci.edu/dtd/sso/distribution.dtd\"> \n \
  <distributionRequest> \n \
    <head> \n \
      <requester userId = \"$archive_user\" email = \"$email\" source = "starview"/> \n \
      <delivery> \n \
        <ftp hostName = \"$ftp_host\" loginName = \"$ftp_user\" directory = \"$ftp_dir\" secure = \"true\" /> \n \
      </delivery> \n \
      <process compression = \"none\"/> \n \
    </head> \n \
    <body> \n \
      <include> \n \
        <select> \n \
          $suffix\n \
        </select> \n \
        $datasets \n \
      </include> \n \
    </body> \n \
  </distributionRequest> \n' )

#-----------------------------------------------------------------------------#
#-----------------------------------------------------------------------------#

#@log_function
def build_xml_request(dest_dir, datasets):
    '''
    Build the xml request string.

    Parameters:
        dest_dir : string
            The path fo the directory in which the data will be downloaded.
        datasets : list
            A list of datasets to be requested.

    Returns:
        xml_request : string
            The xml request string.
    '''
    archive_user = MYUSER
    email = MYUSER + "@stsci.edu"
    host = "plhstins1.stsci.edu"
    ftp_user = MYUSER
    ftp_dir = dest_dir
    # Not currently using suffix dependence.
    suffix = "<suffix name=\"*\" />"
    dataset_str = "\n"
    for item in datasets:
        dataset_str = "{0}          <rootname>{1}</rootname>\n".format(dataset_str,item)

    request_str = REQUEST_TEMPLATE.safe_substitute(
        archive_user = archive_user,
        email = email,
        ftp_host = host,
        ftp_dir =ftp_dir,
        ftp_user = ftp_user,
        suffix = suffix,
        datasets = dataset_str)
    xml_request = string.Template(request_str)
    xml_request = xml_request.template

    return xml_request

#-----------------------------------------------------------------------------#
#-----------------------------------------------------------------------------#

#@log_function
def submit_xml_request(xml_file):
    '''
    Submit the xml request to MAST.

    Parameters:
        xml_file : string
            The xml request string.

    Returns:
        response : httplib object
            The xml request submission result.
    '''
    home = os.environ.get("HOME")
    user = os.environ.get("USER")

    signer = SignStsciRequest()
    request_xml_str = signer.signRequest("{0}/.ssh/privkey.pem".format(home),
                                         xml_file)
    params = urllib.urlencode({
        "dadshost": "dmsops1.stsci.edu",
        "dadsport": 4703,
        "mission": "HST",
        "request": request_xml_str})
    headers = {"Accept": "text/html",
               "User-Agent": "{0}PythonScript".format(user)}
    req = HTTPSConnection("archive.stsci.edu")
    req.request("POST", "/cgi-bin/dads.cgi", params, headers)
    response = req.getresponse().read()

    return response

#-----------------------------------------------------------------------------#
#-----------------------------------------------------------------------------#

#@log_function
def retrieve_data(dest_dir, datasets):
    '''
    For a given list of datasets, submit an xml request to MAST
    to retrieve them to a specified directory.

    Parameters:
        dest_dir : string
            The path to the directory to which the data will be downloaded.
        datasets : list
            A list of datasets to be requested.

    Returns:
        tracking_ids : list
            The tracking IDs for all submitted for a given program.
    '''
    dataset_lists = (datasets[i:i+MAX_RETRIEVAL]
                     for i in xrange(0, len(datasets), MAX_RETRIEVAL))
    tracking_ids = []
    for item in dataset_lists:
        try:
            xml_file = build_xml_request(dest_dir, item)
            result = submit_xml_request(xml_file)
#            print(xml_file)
#            print(result)
            tmp_id = re.search("("+MYUSER+"[0-9]{5})", result).group()
            tracking_ids.append(tmp_id)
        # If you can't get a ID, MAST is most likely down.
        except AttributeError:
            print("Something is wrong on the MAST side...")
            print("Unsuccesful request for {0}".format(item))
            print("Continuing to next dataset.")
            pass
    return tracking_ids

#-----------------------------------------------------------------------------#
#-----------------------------------------------------------------------------#

#@log_function
def check_id_status(tracking_id):
    '''
    Check every 15 minutes to see if all submitted datasets have been
    retrieved. Based on code from J. Ely.

    Parameters:
        tracking_id : string
            A submission ID string..

    Returns:
        done : bool
            Boolean specifying is data is retrieved or not.
        killed : bool
            Boolean specifying is request was killed.
    '''

    done = False
    killed = False
#    print tracking_id
    status_url = "http://archive.stsci.edu/cgi-bin/reqstat?reqnum=={0}".format(tracking_id)
    # In case connection to URL is down.
    tries = 5
    while tries > 0:
        try:
            urllines = urllib.urlopen(status_url).readlines()
        except IOError:
            print("Something went wrong connecting to {0}.".format(status_url))
            tries -= 1
            time.sleep(30)
            killed = True
        else:
            tries = -100
            for line in urllines:
                if "State" in line:
                    if "COMPLETE" in line:
                        done = True
                        killed = False
                    elif "KILLED" in line:
                        killed = True

    return done, killed

#-----------------------------------------------------------------------------#
#-----------------------------------------------------------------------------#

#@log_function
def cycle_thru(prop_dict, prop, all_tracking_ids_tmp):
    '''
    For a given proposal, determine path to put data.
    Retrieve data for a proposal and keep track of the tracking IDs
    for each submission (each proposal may have multiple submissions
    depending on number of datasets in proposal).

    Parameters:
        prop_dict : dictionary
            A dictionary whose keys are proposal IDs and keys are
            dataset names for each ID.
        prop : int
            The proposal ID for which to retrieve data.
        all_tracking_ids_tmp : list
            Running tally of all tracking IDs for all propsals retrieved
            to date.

    Returns:
        all_tracking_ids_tmp : list
            Running tally of all tracking IDs for all propsals retrieved
            to date.
    '''

    prop_dir = os.path.join(BASE_DIR, str(prop))
    if not os.path.exists(prop_dir):
        os.chmod(BASE_DIR, PERM_755)
        os.mkdir(prop_dir)
    os.chmod(prop_dir, PERM_755)
    print("I am retrieving {0} dataset(s) for {1}".format(len(prop_dict[prop]),prop))
    ind_id = retrieve_data(prop_dir, prop_dict[prop])
    for item in ind_id:
        all_tracking_ids_tmp.append(item)

    return all_tracking_ids_tmp

#-----------------------------------------------------------------------------#
#-----------------------------------------------------------------------------#

def check_data_retrieval(all_tracking_ids):
    counter = []
    for tracking_id in all_tracking_ids:
        status,badness = check_id_status(tracking_id)
        counter.append(status)
        if badness:
            print("!"*70)
            print("RUH ROH!!! Request {0} was killed or cannot be connected!".format(tracking_id))
            counter.append(badness)
    current_retrieved = [all_tracking_ids[i] for i in
                         xrange(len(counter)) if not counter[i]]

    return counter, current_retrieved
#-----------------------------------------------------------------------------#
#-----------------------------------------------------------------------------#

#@log_function
def run_all_retrievals(prop_dict=None, pkl_file=None, run_labor=True):
    '''
    Open the pickle file containing a dictionary of all missing COS data
    to be retrieved. It is set up to handle all situations (if run daily=few
    programs, if re-retrieving all data=hundreds of programs). If there
    are more than 100 programs to be retrieved, stop every 100 programs and
    run manualabor to calibrate and zip files. If this is not done, you will
    QUICKLY run out of storage when running manualabor later.
    Split the total number of programs into groups, once a defined number
    of programs have been retrieved (int_num), add more to the queue.
    Using this method, there are always pending programs (as opposed to
    retrieving X, waiting until finished, then starting another X).
    For each proposal in a group, use cycle_thru to request all datasets.
    If there are too many datasets, it will split it up for manageability.

    Paramaters:
        pkl_file : string
            Name of pickle file with data to be retrieved.

    Returns:
        Nothing
    '''

    if pkl_file:
        prop_dict = pickle.load(open(pkl_file, "rb"))
    prop_dict_keys = prop_dict.keys()
    pstart = 0
    pend = 10 # should be 10
    int_num = 5 # should be 5
    century = 50 # should be 50
    all_tracking_ids = []
    end_msg = "\nAll data from {0} programs were successfully "

    # If less than pend programs were requested, do not enter while loop.
    if pend > len(prop_dict_keys):
        for prop in prop_dict_keys:
            all_tracking_ids = cycle_thru(prop_dict, prop, all_tracking_ids)
        counter = []
        num_ids = len(all_tracking_ids)
        while sum(counter) < len(all_tracking_ids):
            counter,current_retrieved = check_data_retrieval(all_tracking_ids)
            if sum(counter) < len(all_tracking_ids):
                print(datetime.now())
                print("Data not yet delivered for {0}. Checking again in "
                      "5 minutes".format(current_retrieved))
                time.sleep(350)
        else:
            print(end_msg.format(len(prop_dict_keys)))

    # While the number of processed programs is less than total programs
    while pend < len(prop_dict_keys):

        # If there are more than N*50 programs to be retrieved, stop
        # and calibrate and zip.
        if pend > century:
            century += 50 # should be 50
            print("Pausing retrieval to calibrate and zip current data")
            work_laboriously(prl=True)
        # For each proposal (prop) in the current grouping (total number
        # of programs split up for manageability), retrieve data for it.
        for prop in prop_dict_keys[pstart:pend]:
            all_tracking_ids = cycle_thru(prop_dict, prop, all_tracking_ids)
        # This while loop keeps track of how many submitted programs in the
        # current group (defined as from pstart:pend) have been entirely
        # retrieved, stored in counter. Once the number of finished
        # programs reaches (total# - int_num), another int_num of programs
        # are added to the queue. While data not retrieved, wait 5 minutes
        # before checking again.
        counter = []
        num_ids = len(all_tracking_ids)
        while sum(counter) < (num_ids - int_num):
            counter = []
            for tracking_id in all_tracking_ids:
                status,badness = check_id_status(tracking_id)
                counter.append(status)
                if badness:
                    print("!"*70)
                    print("RUH ROH!!! Request {0} was killed or cannot be connected!".format(tracking_id))
                    counter.append(badness)
            current_retrieved = [all_tracking_ids[i] for i in
                                 xrange(len(counter)) if not counter[i]]
            if sum(counter) < (num_ids - int_num):
                print(datetime.now())
                print("Data not yet delivered for {0}. Checking again in "
                      "5 minutes".format(current_retrieved))
                time.sleep(350)
        # When total# - int_num programs have been retrieved, add int_num more
        # to queue.
        else:
            print("\nData delivered, adding {0} program(s) to queue".
                  format(int_num))
            pstart = pend
            pend += int_num
    # When pend > total # of programs, it does not mean all have been
    # retrieved. Check, and retrieve if so.
    else:
        "delivered. ".format(len(prop_dict_keys))
        if (len(prop_dict_keys) - (pend-int_num)) > 0:
            for prop in prop_dict_keys[pend-int_num:]:
                all_tracking_ids = cycle_thru(prop_dict, prop, all_tracking_ids)
            print(end_msg)
        else:
            print(end_msg)
    if run_labor:
        print("Beginning manual labor now...")
        work_laboriously(prl=True)

#-----------------------------------------------------------------------------#
#-----------------------------------------------------------------------------#

if __name__ == "__main__":
    pkl_file = "filestoretrieve.p"
    try:
        run_all_retrievals(prop_dict=None, pkl_file=pkl_file, run_labor=True)
    except Exception as e:
        print(Exception, e)
<|endoftext|>


<commit_before>
import ast
import errno
import os
import platform
import shutil
import subprocess
import sys

from distutils import log
from distutils.command.build_ext import build_ext
from distutils.command.sdist import sdist
from distutils.errors import DistutilsError

def makedirs(path):
    try:
        os.makedirs(path)
    except OSError as e:
        if e.errno==errno.EEXIST and os.path.isdir(path) and os.access(path, os.R_OK | os.W_OK | os.X_OK):
            return
        raise

def rmtree(path):
    try:
        shutil.rmtree(path)
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise

def exec_process(cmdline, silent=True, input=None, **kwargs):
    """Execute a subprocess and returns the returncode, stdout buffer and stderr buffer.
    Optionally prints stdout and stderr while running."""
    try:
        sub = subprocess.Popen(args=cmdline, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
        stdout, stderr = sub.communicate(input=input)
        returncode = sub.returncode
        if not silent:
            sys.stdout.write(stdout)
            sys.stderr.write(stderr)
    except OSError as e:
        if e.errno == errno.ENOENT:
            raise DistutilsError('"%s" is not present on this system' % cmdline[0])
        else:
            raise
    if returncode != 0:
        raise DistutilsError('Got return value %d while executing "%s", stderr output was:\n%s' % (returncode, " ".join(cmdline), stderr.decode("utf-8").rstrip("\n")))
    return stdout

def prepare_windows_env(env):
    env.pop('VS120COMNTOOLS', None)
    env.pop('VS110COMNTOOLS', None)
    if sys.version_info < (3, 3):
        env.pop('VS100COMNTOOLS', None)

    if env.get('PYTHON'):
        return  # Already manually set by user.

    if (2, 6) <= sys.version_info[:2] <= (2, 7):
        env['PYTHON'] = sys.executable
        return  # The current executable is fine.

    # Try if `python` on PATH is the right one. If we would execute
    # `python` directly the current executable might be used so we
    # delegate this to cmd.
    cmd = ['cmd.exe', '/C',  'python', '-c', 'import sys; '
           'v = str(sys.version_info[:2]); sys.stdout.write(v); '
           'sys.stdout.flush()']
    try:
        sub = subprocess.Popen(cmd, stdout=subprocess.PIPE)
        stdout, _ = sub.communicate()
        version = ast.literal_eval(stdout.decode('utf-8').strip())
        if (2, 6) <= version <= (2, 7):
            return  # Python on PATH is fine
    except OSError:
        pass

    # Check default install locations
    for v in ('26', '27'):
        path = os.path.join('%SYSTEMDRIVE%', 'Python%s' % v, 'python.exe')
        path = os.path.expandvars(path)
        if os.path.isfile(path):
            log.info('Using "%s" to build libuv...' % path)
            env['PYTHON'] = path
            return

    raise DistutilsError('No appropriate Python version found. An '
                         'installation of 2.6 or 2.7 is required to '
                         'build libuv. You can set the environment '
                         'variable "PYTHON" to point to a custom '
                         'installation location.')

class libuv_build_ext(build_ext):
    libuv_dir      = os.path.join('deps', 'libuv')
    libuv_repo     = 'https://github.com/joyent/libuv.git'
    libuv_branch   = 'master'
    libuv_revision = 'f326820'
    libuv_patches  = []

    user_options = build_ext.user_options
    user_options.extend([
        ("libuv-clean-compile", None, "Clean libuv tree before compilation"),
        ("libuv-force-fetch", None, "Remove libuv (if present) and fetch it again"),
        ("libuv-verbose-build", None, "Print output of libuv build process")
    ])
    boolean_options = build_ext.boolean_options
    boolean_options.extend(["libuv-clean-compile", "libuv-force-fetch", "libuv-verbose-build"])

    def initialize_options(self):
        build_ext.initialize_options(self)
        self.libuv_clean_compile = 0
        self.libuv_force_fetch = 0
        self.libuv_verbose_build = 0

    def build_extensions(self):
        self.force = self.libuv_force_fetch or self.libuv_clean_compile
        if sys.platform == 'win32':
            self.libuv_lib = os.path.join(self.libuv_dir, 'Release', 'lib', 'libuv.lib')
        else:
            self.libuv_lib = os.path.join(self.libuv_dir, '.libs', 'libuv.a')
        self.get_libuv()
        # Set compiler options
        self.extensions[0].extra_objects.extend([self.libuv_lib])
        self.compiler.add_include_dir(os.path.join(self.libuv_dir, 'include'))
        if sys.platform.startswith('linux'):
            self.compiler.add_library('rt')
        elif sys.platform == 'win32':
            self.extensions[0].define_macros.append(('WIN32', 1))
            self.extensions[0].extra_link_args.extend(['/NODEFAULTLIB:libcmt', '/LTCG'])
            self.compiler.add_library('advapi32')
            self.compiler.add_library('iphlpapi')
            self.compiler.add_library('psapi')
            self.compiler.add_library('ws2_32')
        elif sys.platform.startswith('freebsd'):
            self.compiler.add_library('kvm')
        build_ext.build_extensions(self)

    def get_libuv(self):
        #self.debug_mode =  bool(self.debug) or hasattr(sys, 'gettotalrefcount')
        def download_libuv():
            log.info('Downloading libuv...')
            makedirs(self.libuv_dir)
            exec_process(['git', 'clone', '-b', self.libuv_branch, self.libuv_repo, self.libuv_dir])
            exec_process(['git', 'reset', '--hard', self.libuv_revision], cwd=self.libuv_dir)
        def patch_libuv():
            if self.libuv_patches:
                log.info('Patching libuv...')
                for patch_file in self.libuv_patches:
                    exec_process(['patch', '--forward', '-d', self.libuv_dir, '-p0', '-i', os.path.abspath(patch_file)])
        def build_libuv():
            cflags = '-fPIC'
            env = os.environ.copy()
            env['CFLAGS'] = ' '.join(x for x in (cflags, env.get('CFLAGS', None), env.get('ARCHFLAGS', None)) if x)
            log.info('Building libuv...')
            if sys.platform == 'win32':
                prepare_windows_env(env)
                libuv_arch = {'32bit': 'x86', '64bit': 'x64'}[platform.architecture()[0]]
                exec_process(['cmd.exe', '/C', 'vcbuild.bat', libuv_arch, 'release'], cwd=self.libuv_dir, env=env, shell=True, silent=not self.libuv_verbose_build)
            else:
                exec_process(['sh', 'autogen.sh'], cwd=self.libuv_dir, env=env, silent=not self.libuv_verbose_build)
                exec_process(['./configure'], cwd=self.libuv_dir, env=env, silent=not self.libuv_verbose_build)
                exec_process(['make'], cwd=self.libuv_dir, env=env, silent=not self.libuv_verbose_build)
        if self.libuv_force_fetch:
            rmtree('deps')
        if not os.path.exists(self.libuv_dir):
            try:
                download_libuv()
            except BaseException:
                rmtree('deps')
                raise
            patch_libuv()
            build_libuv()
        else:
            if self.libuv_clean_compile:
                if sys.platform == 'win32':
                    env = os.environ.copy()
                    prepare_windows_env(env)
                    exec_process(['cmd.exe', '/C', 'vcbuild.bat', 'clean'], cwd=self.libuv_dir, env=env, shell=True)
                    rmtree(os.path.join(self.libuv_dir, 'Release'))
                else:
                    exec_process(['make', 'clean'], cwd=self.libuv_dir)
                    exec_process(['make', 'distclean'], cwd=self.libuv_dir)
            if not os.path.exists(self.libuv_lib):
                log.info('libuv needs to be compiled.')
                build_libuv()
            else:
                log.info('No need to build libuv.')

class libuv_sdist(sdist):
    libuv_dir      = os.path.join('deps', 'libuv')
    libuv_repo     = libuv_build_ext.libuv_repo
    libuv_branch   = libuv_build_ext.libuv_branch
    libuv_revision = libuv_build_ext.libuv_revision
    libuv_patches  = libuv_build_ext.libuv_patches

    def initialize_options(self):
        sdist.initialize_options(self)
        rmtree('deps')
        makedirs(self.libuv_dir)
        log.info('Downloading libuv...')
        exec_process(['git', 'clone', '-b', self.libuv_branch, self.libuv_repo, self.libuv_dir])
        exec_process(['git', 'checkout', self.libuv_revision], cwd=self.libuv_dir)
        if self.libuv_patches:
            log.info('Patching libuv...')
            for patch_file in self.libuv_patches:
                exec_process(['patch', '--forward', '-d', self.libuv_dir, '-p0', '-i', os.path.abspath(patch_file)])
        rmtree(os.path.join(self.libuv_dir, '.git'))

<commit_msg>Raised libuv revision<commit_after>
import ast
import errno
import os
import platform
import shutil
import subprocess
import sys

from distutils import log
from distutils.command.build_ext import build_ext
from distutils.command.sdist import sdist
from distutils.errors import DistutilsError

def makedirs(path):
    try:
        os.makedirs(path)
    except OSError as e:
        if e.errno==errno.EEXIST and os.path.isdir(path) and os.access(path, os.R_OK | os.W_OK | os.X_OK):
            return
        raise

def rmtree(path):
    try:
        shutil.rmtree(path)
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise

def exec_process(cmdline, silent=True, input=None, **kwargs):
    """Execute a subprocess and returns the returncode, stdout buffer and stderr buffer.
    Optionally prints stdout and stderr while running."""
    try:
        sub = subprocess.Popen(args=cmdline, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
        stdout, stderr = sub.communicate(input=input)
        returncode = sub.returncode
        if not silent:
            sys.stdout.write(stdout)
            sys.stderr.write(stderr)
    except OSError as e:
        if e.errno == errno.ENOENT:
            raise DistutilsError('"%s" is not present on this system' % cmdline[0])
        else:
            raise
    if returncode != 0:
        raise DistutilsError('Got return value %d while executing "%s", stderr output was:\n%s' % (returncode, " ".join(cmdline), stderr.decode("utf-8").rstrip("\n")))
    return stdout

def prepare_windows_env(env):
    env.pop('VS120COMNTOOLS', None)
    env.pop('VS110COMNTOOLS', None)
    if sys.version_info < (3, 3):
        env.pop('VS100COMNTOOLS', None)

    if env.get('PYTHON'):
        return  # Already manually set by user.

    if (2, 6) <= sys.version_info[:2] <= (2, 7):
        env['PYTHON'] = sys.executable
        return  # The current executable is fine.

    # Try if `python` on PATH is the right one. If we would execute
    # `python` directly the current executable might be used so we
    # delegate this to cmd.
    cmd = ['cmd.exe', '/C',  'python', '-c', 'import sys; '
           'v = str(sys.version_info[:2]); sys.stdout.write(v); '
           'sys.stdout.flush()']
    try:
        sub = subprocess.Popen(cmd, stdout=subprocess.PIPE)
        stdout, _ = sub.communicate()
        version = ast.literal_eval(stdout.decode('utf-8').strip())
        if (2, 6) <= version <= (2, 7):
            return  # Python on PATH is fine
    except OSError:
        pass

    # Check default install locations
    for v in ('26', '27'):
        path = os.path.join('%SYSTEMDRIVE%', 'Python%s' % v, 'python.exe')
        path = os.path.expandvars(path)
        if os.path.isfile(path):
            log.info('Using "%s" to build libuv...' % path)
            env['PYTHON'] = path
            return

    raise DistutilsError('No appropriate Python version found. An '
                         'installation of 2.6 or 2.7 is required to '
                         'build libuv. You can set the environment '
                         'variable "PYTHON" to point to a custom '
                         'installation location.')

class libuv_build_ext(build_ext):
    libuv_dir      = os.path.join('deps', 'libuv')
    libuv_repo     = 'https://github.com/joyent/libuv.git'
    libuv_branch   = 'master'
    libuv_revision = '69f9f6f'
    libuv_patches  = []

    user_options = build_ext.user_options
    user_options.extend([
        ("libuv-clean-compile", None, "Clean libuv tree before compilation"),
        ("libuv-force-fetch", None, "Remove libuv (if present) and fetch it again"),
        ("libuv-verbose-build", None, "Print output of libuv build process")
    ])
    boolean_options = build_ext.boolean_options
    boolean_options.extend(["libuv-clean-compile", "libuv-force-fetch", "libuv-verbose-build"])

    def initialize_options(self):
        build_ext.initialize_options(self)
        self.libuv_clean_compile = 0
        self.libuv_force_fetch = 0
        self.libuv_verbose_build = 0

    def build_extensions(self):
        self.force = self.libuv_force_fetch or self.libuv_clean_compile
        if sys.platform == 'win32':
            self.libuv_lib = os.path.join(self.libuv_dir, 'Release', 'lib', 'libuv.lib')
        else:
            self.libuv_lib = os.path.join(self.libuv_dir, '.libs', 'libuv.a')
        self.get_libuv()
        # Set compiler options
        self.extensions[0].extra_objects.extend([self.libuv_lib])
        self.compiler.add_include_dir(os.path.join(self.libuv_dir, 'include'))
        if sys.platform.startswith('linux'):
            self.compiler.add_library('rt')
        elif sys.platform == 'win32':
            self.extensions[0].define_macros.append(('WIN32', 1))
            self.extensions[0].extra_link_args.extend(['/NODEFAULTLIB:libcmt', '/LTCG'])
            self.compiler.add_library('advapi32')
            self.compiler.add_library('iphlpapi')
            self.compiler.add_library('psapi')
            self.compiler.add_library('ws2_32')
        elif sys.platform.startswith('freebsd'):
            self.compiler.add_library('kvm')
        build_ext.build_extensions(self)

    def get_libuv(self):
        #self.debug_mode =  bool(self.debug) or hasattr(sys, 'gettotalrefcount')
        def download_libuv():
            log.info('Downloading libuv...')
            makedirs(self.libuv_dir)
            exec_process(['git', 'clone', '-b', self.libuv_branch, self.libuv_repo, self.libuv_dir])
            exec_process(['git', 'reset', '--hard', self.libuv_revision], cwd=self.libuv_dir)
        def patch_libuv():
            if self.libuv_patches:
                log.info('Patching libuv...')
                for patch_file in self.libuv_patches:
                    exec_process(['patch', '--forward', '-d', self.libuv_dir, '-p0', '-i', os.path.abspath(patch_file)])
        def build_libuv():
            cflags = '-fPIC'
            env = os.environ.copy()
            env['CFLAGS'] = ' '.join(x for x in (cflags, env.get('CFLAGS', None), env.get('ARCHFLAGS', None)) if x)
            log.info('Building libuv...')
            if sys.platform == 'win32':
                prepare_windows_env(env)
                libuv_arch = {'32bit': 'x86', '64bit': 'x64'}[platform.architecture()[0]]
                exec_process(['cmd.exe', '/C', 'vcbuild.bat', libuv_arch, 'release'], cwd=self.libuv_dir, env=env, shell=True, silent=not self.libuv_verbose_build)
            else:
                exec_process(['sh', 'autogen.sh'], cwd=self.libuv_dir, env=env, silent=not self.libuv_verbose_build)
                exec_process(['./configure'], cwd=self.libuv_dir, env=env, silent=not self.libuv_verbose_build)
                exec_process(['make'], cwd=self.libuv_dir, env=env, silent=not self.libuv_verbose_build)
        if self.libuv_force_fetch:
            rmtree('deps')
        if not os.path.exists(self.libuv_dir):
            try:
                download_libuv()
            except BaseException:
                rmtree('deps')
                raise
            patch_libuv()
            build_libuv()
        else:
            if self.libuv_clean_compile:
                if sys.platform == 'win32':
                    env = os.environ.copy()
                    prepare_windows_env(env)
                    exec_process(['cmd.exe', '/C', 'vcbuild.bat', 'clean'], cwd=self.libuv_dir, env=env, shell=True)
                    rmtree(os.path.join(self.libuv_dir, 'Release'))
                else:
                    exec_process(['make', 'clean'], cwd=self.libuv_dir)
                    exec_process(['make', 'distclean'], cwd=self.libuv_dir)
            if not os.path.exists(self.libuv_lib):
                log.info('libuv needs to be compiled.')
                build_libuv()
            else:
                log.info('No need to build libuv.')

class libuv_sdist(sdist):
    libuv_dir      = os.path.join('deps', 'libuv')
    libuv_repo     = libuv_build_ext.libuv_repo
    libuv_branch   = libuv_build_ext.libuv_branch
    libuv_revision = libuv_build_ext.libuv_revision
    libuv_patches  = libuv_build_ext.libuv_patches

    def initialize_options(self):
        sdist.initialize_options(self)
        rmtree('deps')
        makedirs(self.libuv_dir)
        log.info('Downloading libuv...')
        exec_process(['git', 'clone', '-b', self.libuv_branch, self.libuv_repo, self.libuv_dir])
        exec_process(['git', 'checkout', self.libuv_revision], cwd=self.libuv_dir)
        if self.libuv_patches:
            log.info('Patching libuv...')
            for patch_file in self.libuv_patches:
                exec_process(['patch', '--forward', '-d', self.libuv_dir, '-p0', '-i', os.path.abspath(patch_file)])
        rmtree(os.path.join(self.libuv_dir, '.git'))

<|endoftext|>


<commit_before>from aiohttp import ClientSession
import asyncio
import json
import logging
from uuid import uuid4
from enum import Enum

from tukio.task import register
from tukio.task.holder import TaskHolder
from tukio.workflow import WorkflowExecState, Workflow

from .utils import runtime

log = logging.getLogger(__name__)

class Status(Enum):
    RUNNING = 'running'
    DONE = 'done'
    TIMEOUT = 'timeout'

@register('trigger_workflow', 'execute')
class TriggerWorkflowTask(TaskHolder):

    WORKFLOW_URL = 'http://{host}{nyuki_api}/v1/workflow/instances'
    SCHEMA = {
        'type': 'object',
        'required': ['nyuki_api', 'template'],
        'properties': {
            'nyuki_api': {'type': 'string', 'description': 'nyuki_api'},
            'template': {'type': 'string', 'description': 'template_id'},
            'draft': {'type': 'boolean'},
            'await_completion': {'type': 'boolean', 'default': True},
            'timeout': {'type': 'integer', 'minimum': 1, 'default': 60}
        },
        'dependencies': {
            'await_completion': ['timeout']
        },
        'additionalProperties': False
    }

    def __init__(self, config):
        super().__init__(config)
        self.template = self.config['template']
        self.draft = self.config.get('draft', False)
        self.blocking = self.config.get('await_completion', False)
        self.timeout = self.config.get('timeout', 60)
        # Workflow URL
        self.nyuki_api = self.config.get('nyuki_api') or ''
        if self.nyuki_api and not self.nyuki_api.startswith('/'):
            self.nyuki_api = '/' + self.nyuki_api
        self.url = self.WORKFLOW_URL.format(
            host=runtime.config.get('http_host', 'localhost'),
            nyuki_api=self.nyuki_api
        )

        self._task = None
        self._triggered = {
            'workflow_holder': self.nyuki_api.split('/')[1],
            'workflow_template_id': self.template,
            'workflow_exec_id': None,
            'workflow_draft': self.draft,
            'status': None
        }

    def report(self):
        return self._triggered

    async def teardown(self):
        exec_id = self._triggered['workflow_exec_id']
        holder = self._triggered['workflow_holder']
        uri = '{}@{}'.format(exec_id, holder)
        if not exec_id:
            log.debug('No triggered workflow to cancel')
            return

        async with ClientSession() as session:
            params = {
                'url': '{}/{}'.format(self.url, exec_id),
                'headers': {'Content-Type': 'application/json'}
            }
            async with session.delete(**params) as response:
                response = response.status

        if response != 200:
            log.debug('Failed to cancel triggered workflow {}'.format(uri))
        else:
            log.debug('Triggered workflow {} has been cancelled'.format(uri))

    async def async_exec(self, topic, data):
        log.debug(
            "Received data for async trigger_workflow in '%s': %s", topic, data
        )
        if data['type'] in [WorkflowExecState.end.value, WorkflowExecState.error.value]:
            self.async_future.set_result(data)
            asyncio.ensure_future(runtime.bus.unsubscribe(topic))

    async def execute(self, event):
        """
        Entrypoint execution method.
        """
        data = event.data
        self._task = asyncio.Task.current_task()

        # Send the HTTP request
        log.info('Request %s to process template %s', self.nyuki_api, self.template)
        log.debug(
            'Request details: url=%s, draft=%s, data=%s',
            self.url, self.draft, data
        )

        # Handle blocking trigger_workflow using mqtt
        workflow = Workflow.current_workflow()
        headers = {
            'Content-Type': 'application/json',
            'Referer': '{}/{}'.format(runtime.bus.name, workflow.uid)
        }

        if self.blocking:
            topic = '{}/async/{}'.format(runtime.bus.name, str(uuid4())[:8])
            headers['X-Surycat-Async-Topic'] = topic
            self.async_future = asyncio.Future()
            await runtime.bus.subscribe(topic, self.async_exec)
            asyncio.get_event_loop().call_later(
                self.timeout,
                asyncio.ensure_future,
                runtime.bus.unsubscribe(topic)
            )

        async with ClientSession() as session:
            params = {
                'url': self.url,
                'headers': headers,
                'data': json.dumps({
                    'id': self.template,
                    'draft': self.draft,
                    'inputs': data
                })
            }
            async with session.put(**params) as response:
                # Response validity
                if response.status != 200:
                    raise RuntimeError(
                        "Can't process remote workflow template {} on {}".format(
                            self.template, self.nyuki_api
                        )
                    )
                resp_body = await response.json()

        instance = resp_body['exec']['id']
        self._triggered['workflow_exec_id'] = instance
        log.debug('Request sent successfully to {}', self.nyuki_api)

        if not self.blocking:
            self._triggered['status'] = Status.DONE.value
            self._task.dispatch_progress(self.report())

        # Block until task completed
        if self.blocking:
            self._triggered['status'] = Status.RUNNING.value
            self._task.dispatch_progress(self.report())
            log.info('Waiting for %s@%s to complete', instance, self.nyuki_api)
            try:
                await asyncio.wait_for(self.async_future, self.timeout)
                self._triggered['status'] = Status.DONE.value
                log.info('Instance %s@%s is done', instance, self.nyuki_api)
            except asyncio.TimeoutError:
                self._triggered['status'] = Status.TIMEOUT.value
                log.info('Instance %s@%s has timeouted', instance, self.nyuki_api)
            self._task.dispatch_progress({'status': self._triggered['status']})

        return data
<commit_msg>Awaiting completion by default as specified in the schema<commit_after>from aiohttp import ClientSession
import asyncio
import json
import logging
from uuid import uuid4
from enum import Enum

from tukio.task import register
from tukio.task.holder import TaskHolder
from tukio.workflow import WorkflowExecState, Workflow

from .utils import runtime

log = logging.getLogger(__name__)

class Status(Enum):
    RUNNING = 'running'
    DONE = 'done'
    TIMEOUT = 'timeout'

@register('trigger_workflow', 'execute')
class TriggerWorkflowTask(TaskHolder):

    WORKFLOW_URL = 'http://{host}{nyuki_api}/v1/workflow/instances'
    SCHEMA = {
        'type': 'object',
        'required': ['nyuki_api', 'template'],
        'properties': {
            'nyuki_api': {'type': 'string', 'description': 'nyuki_api'},
            'template': {'type': 'string', 'description': 'template_id'},
            'draft': {'type': 'boolean'},
            'await_completion': {'type': 'boolean', 'default': True},
            'timeout': {'type': 'integer', 'minimum': 1, 'default': 60}
        },
        'dependencies': {
            'await_completion': ['timeout']
        },
        'additionalProperties': False
    }

    def __init__(self, config):
        super().__init__(config)
        self.template = self.config['template']
        self.draft = self.config.get('draft', False)
        self.blocking = self.config.get('await_completion', True)
        self.timeout = self.config.get('timeout', 60)
        # Workflow URL
        self.nyuki_api = self.config.get('nyuki_api') or ''
        if self.nyuki_api and not self.nyuki_api.startswith('/'):
            self.nyuki_api = '/' + self.nyuki_api
        self.url = self.WORKFLOW_URL.format(
            host=runtime.config.get('http_host', 'localhost'),
            nyuki_api=self.nyuki_api
        )

        self._task = None
        self._triggered = {
            'workflow_holder': self.nyuki_api.split('/')[1],
            'workflow_template_id': self.template,
            'workflow_exec_id': None,
            'workflow_draft': self.draft,
            'status': None
        }

    def report(self):
        return self._triggered

    async def teardown(self):
        exec_id = self._triggered['workflow_exec_id']
        holder = self._triggered['workflow_holder']
        uri = '{}@{}'.format(exec_id, holder)
        if not exec_id:
            log.debug('No triggered workflow to cancel')
            return

        async with ClientSession() as session:
            params = {
                'url': '{}/{}'.format(self.url, exec_id),
                'headers': {'Content-Type': 'application/json'}
            }
            async with session.delete(**params) as response:
                response = response.status

        if response != 200:
            log.debug('Failed to cancel triggered workflow {}'.format(uri))
        else:
            log.debug('Triggered workflow {} has been cancelled'.format(uri))

    async def async_exec(self, topic, data):
        log.debug(
            "Received data for async trigger_workflow in '%s': %s", topic, data
        )
        if data['type'] in [WorkflowExecState.end.value, WorkflowExecState.error.value]:
            self.async_future.set_result(data)
            asyncio.ensure_future(runtime.bus.unsubscribe(topic))

    async def execute(self, event):
        """
        Entrypoint execution method.
        """
        data = event.data
        self._task = asyncio.Task.current_task()

        # Send the HTTP request
        log.info('Request %s to process template %s', self.nyuki_api, self.template)
        log.debug(
            'Request details: url=%s, draft=%s, data=%s',
            self.url, self.draft, data
        )

        # Handle blocking trigger_workflow using mqtt
        workflow = Workflow.current_workflow()
        headers = {
            'Content-Type': 'application/json',
            'Referer': '{}/{}'.format(runtime.bus.name, workflow.uid)
        }

        if self.blocking:
            topic = '{}/async/{}'.format(runtime.bus.name, str(uuid4())[:8])
            headers['X-Surycat-Async-Topic'] = topic
            self.async_future = asyncio.Future()
            await runtime.bus.subscribe(topic, self.async_exec)
            asyncio.get_event_loop().call_later(
                self.timeout,
                asyncio.ensure_future,
                runtime.bus.unsubscribe(topic)
            )

        async with ClientSession() as session:
            params = {
                'url': self.url,
                'headers': headers,
                'data': json.dumps({
                    'id': self.template,
                    'draft': self.draft,
                    'inputs': data
                })
            }
            async with session.put(**params) as response:
                # Response validity
                if response.status != 200:
                    raise RuntimeError(
                        "Can't process remote workflow template {} on {}".format(
                            self.template, self.nyuki_api
                        )
                    )
                resp_body = await response.json()

        instance = resp_body['exec']['id']
        self._triggered['workflow_exec_id'] = instance
        log.debug('Request sent successfully to {}', self.nyuki_api)

        if not self.blocking:
            self._triggered['status'] = Status.DONE.value
            self._task.dispatch_progress(self.report())

        # Block until task completed
        if self.blocking:
            self._triggered['status'] = Status.RUNNING.value
            self._task.dispatch_progress(self.report())
            log.info('Waiting for %s@%s to complete', instance, self.nyuki_api)
            try:
                await asyncio.wait_for(self.async_future, self.timeout)
                self._triggered['status'] = Status.DONE.value
                log.info('Instance %s@%s is done', instance, self.nyuki_api)
            except asyncio.TimeoutError:
                self._triggered['status'] = Status.TIMEOUT.value
                log.info('Instance %s@%s has timeouted', instance, self.nyuki_api)
            self._task.dispatch_progress({'status': self._triggered['status']})

        return data
<|endoftext|>