<commit_before># -*- coding: utf-8 -*-
from cadnano.cnenum import PointType
from cadnano.cnenum import LatticeType
# from cadnano.cnenum import StrandType
from cadnano.fileio.lattice import HoneycombDnaPart
# from cadnano.fileio.lattice import SquareDnaPart

FORMAT_VERSION = "2.0"
ROW_OFFSET = 2
COL_OFFSET = 3
DEFAULT_ROWS = 30
DEFAULT_COLS = 32
lattice_type = LatticeType.HONEYCOMB
positionToLatticeCoord = HoneycombDnaPart.positionToLatticeCoordRound

def encodeDocument(document):

    part = next(document.getParts())
    radius = part.radius()

    # Determine offset to approximately center the shape
    xLL, yLL, xUR, yUR = part.getVirtualHelixOriginLimits()
    rowLL, colLL = positionToLatticeCoord(radius, xLL, yLL, False, False)
    rowUR, colUR = positionToLatticeCoord(radius, xUR, yUR, True, True)
    row_range = [i for i in range(rowUR+ROW_OFFSET, rowLL-1, -1)]
    col_range = [i for i in range(colLL-COL_OFFSET, colUR+1)]

    # Attempt to shift the design to the center of the lattice
    # height = abs(rowLL-rowUR)
    # width = abs(colLL-colUR)
    # delta_row = (DEFAULT_ROWS-height)//2
    # if delta_row % 2 == 1:
    #     delta_row += 1
    # delta_col = (DEFAULT_COLS-width)//2
    # print("Shifting each (row,col) by ({0},{1})".format(delta_row, delta_col))

    insertions = part.insertions()
    vh_order = part.getVirtualHelixOrder()
    name = "legacy-export-cn25"
    max_base_idx = max([part.maxBaseIdx(id_num) for id_num in vh_order])

    print("Translating vh:(row,col) to legacy coordinates...")

    # iterate through virtualhelix list
    vh_list = []
    for id_num in vh_order:
        # vh = part.getVirtualHelix(id_num)
        fwd_ss, rev_ss = part.getStrandSets(id_num)
        # print(vh, fwd_ss, rev_ss)

        # insertions and skips
        insertion_dict = insertions[id_num]
        insts = [0 for i in range(max_base_idx)]
        skips = [0 for i in range(max_base_idx)]
        for idx, insertion in insertion_dict.items():
            if insertion.isSkip():
                skips[idx] = insertion.length()
            else:
                insts[idx] = insertion.length()
        # if sum(insts) != 0:
        #     print("insts", insts)
        # if sum(skips) != 0:
        #     print("skips", skips)

        if id_num % 2 == 0:
            scaf_ss, stap_ss = fwd_ss, rev_ss
        else:
            scaf_ss, stap_ss = rev_ss, fwd_ss  # noqa

        # colors
        stap_colors = []
        for strand in stap_ss:
            if strand.connection5p() is None:
                c = str(strand.oligo().getColor())[1:]  # drop the hash
                stap_colors.append([strand.idx5Prime(), int(c, 16)])
        # print(stap_colors)

        # convert x,y coordinates to new (row, col)
        vh_x, vh_y = part.getVirtualHelixOrigin(id_num)
        row, col = positionToLatticeCoord(radius, vh_x, vh_y, False, False)
        new_row = row_range.index(row)
        new_col = col_range.index(col)
        print("{0:>2}: ({1:>2},{2:>2}) -> ({3:>2},{4:>2})".format(id_num, row, col, new_row, new_col))

        vh_dict = {"row": new_row,
                   "col": new_col,
                   "num": id_num,
                   "scaf": getLegacyStrandSetArray(scaf_ss, max_base_idx),
                   "stap": getLegacyStrandSetArray(stap_ss, max_base_idx),
                   "loop": insts,
                   "skip": skips,
                   "scafLoop": [],
                   "stapLoop": [],
                   "stap_colors": stap_colors}
        vh_list.append(vh_dict)
    # bname = basename(str(fname))
    obj = {"name": name, "vstrands": vh_list}
    return obj

def getLegacyStrandSetArray(ss, max_base_idx):
    """Given a strandset and max_base_idx, return legacy serialization array format."""
    num = ss.idNum()
    ret = [[-1, -1, -1, -1] for i in range(max_base_idx)]
    if ss.isForward():
        for strand in ss.strands():
            lo, hi = strand.idxs()
            assert strand.idx5Prime() == lo and strand.idx3Prime() == hi
            # map the first base (5' xover if necessary)
            s5p = strand.connection5p()
            if s5p is not None:
                ret[lo][0] = s5p.idNum()
                ret[lo][1] = s5p.idx3Prime()
            ret[lo][2] = num
            ret[lo][3] = lo + 1
            # map the internal bases
            for idx in range(lo + 1, hi):
                ret[idx][0] = num
                ret[idx][1] = idx - 1
                ret[idx][2] = num
                ret[idx][3] = idx + 1
            # map the last base (3' xover if necessary)
            ret[hi][0] = num
            ret[hi][1] = hi - 1
            s3p = strand.connection3p()
            if s3p is not None:
                ret[hi][2] = s3p.idNum()
                ret[hi][3] = s3p.idx5Prime()
            # end if
        # end for
    # end if
    else:
        for strand in ss.strands():
            lo, hi = strand.idxs()
            assert strand.idx3Prime() == lo and strand.idx5Prime() == hi
            # map the first base (3' xover if necessary)
            ret[lo][0] = num
            ret[lo][1] = lo + 1
            s3p = strand.connection3p()
            if s3p is not None:
                ret[lo][2] = s3p.idNum()
                ret[lo][3] = s3p.idx5Prime()
            # map the internal bases
            for idx in range(lo + 1, hi):
                ret[idx][0] = num
                ret[idx][1] = idx + 1
                ret[idx][2] = num
                ret[idx][3] = idx - 1
            # map the last base (5' xover if necessary)
            ret[hi][2] = num
            ret[hi][3] = hi - 1
            s5p = strand.connection5p()
            if s5p is not None:
                ret[hi][0] = s5p.idNum()
                ret[hi][1] = s5p.idx3Prime()
            # end if
        # end for
    return ret
# end def
<commit_msg>cleanup v2encode comments & print statements<commit_after># -*- coding: utf-8 -*-
from cadnano.cnenum import LatticeType
from cadnano.fileio.lattice import HoneycombDnaPart

FORMAT_VERSION = "2.0"
ROW_OFFSET = 2
COL_OFFSET = 3
# DEFAULT_ROWS = 30
# DEFAULT_COLS = 32
lattice_type = LatticeType.HONEYCOMB
positionToLatticeCoord = HoneycombDnaPart.positionToLatticeCoordRound

def encodeDocument(document):

    part = next(document.getParts())
    radius = part.radius()

    # Determine offset to approximately center the shape
    xLL, yLL, xUR, yUR = part.getVirtualHelixOriginLimits()
    rowLL, colLL = positionToLatticeCoord(radius, xLL, yLL, False, False)
    rowUR, colUR = positionToLatticeCoord(radius, xUR, yUR, True, True)
    row_range = [i for i in range(rowUR+ROW_OFFSET, rowLL-1, -1)]
    col_range = [i for i in range(colLL-COL_OFFSET, colUR+1)]

    # Attempt to shift the design to the center of the lattice
    # height = abs(rowLL-rowUR)
    # width = abs(colLL-colUR)
    # delta_row = (DEFAULT_ROWS-height)//2
    # if delta_row % 2 == 1:
    #     delta_row += 1
    # delta_col = (DEFAULT_COLS-width)//2
    # print("Shifting each (row,col) by ({0},{1})".format(delta_row, delta_col))

    insertions = part.insertions()
    vh_order = part.getVirtualHelixOrder()
    name = "legacy-export-cn25"
    max_base_idx = max([part.maxBaseIdx(id_num) for id_num in vh_order])

    print("Translating vh:(row,col) to legacy coordinates...")

    # Iterate through virtualhelix list
    vh_list = []
    for id_num in vh_order:
        fwd_ss, rev_ss = part.getStrandSets(id_num)

        # Insertions and skips
        insertion_dict = insertions[id_num]
        insts = [0 for i in range(max_base_idx)]
        skips = [0 for i in range(max_base_idx)]
        for idx, insertion in insertion_dict.items():
            if insertion.isSkip():
                skips[idx] = insertion.length()
            else:
                insts[idx] = insertion.length()

        if id_num % 2 == 0:
            scaf_ss, stap_ss = fwd_ss, rev_ss
        else:
            scaf_ss, stap_ss = rev_ss, fwd_ss  # noqa

        # Colors
        stap_colors = []
        for strand in stap_ss:
            if strand.connection5p() is None:
                c = str(strand.oligo().getColor())[1:]  # drop the hash
                stap_colors.append([strand.idx5Prime(), int(c, 16)])

        # Convert x,y coordinates to new (row, col)
        vh_x, vh_y = part.getVirtualHelixOrigin(id_num)
        row, col = positionToLatticeCoord(radius, vh_x, vh_y, False, False)
        new_row = row_range.index(row)
        new_col = col_range.index(col)
        print("{0:>2}: ({1:>2},{2:>2}) -> ({3:>2},{4:>2})".format(id_num, row, col, new_row, new_col))

        # Put everything together in a new dict
        vh_dict = {"row": new_row,
                   "col": new_col,
                   "num": id_num,
                   "scaf": getLegacyStrandSetArray(scaf_ss, max_base_idx),
                   "stap": getLegacyStrandSetArray(stap_ss, max_base_idx),
                   "loop": insts,
                   "skip": skips,
                   "scafLoop": [],
                   "stapLoop": [],
                   "stap_colors": stap_colors}
        vh_list.append(vh_dict)
    obj = {"name": name, "vstrands": vh_list}
    return obj

def getLegacyStrandSetArray(ss, max_base_idx):
    """Given a strandset and max_base_idx, return legacy serialization array format."""
    num = ss.idNum()
    ret = [[-1, -1, -1, -1] for i in range(max_base_idx)]
    if ss.isForward():
        for strand in ss.strands():
            lo, hi = strand.idxs()
            assert strand.idx5Prime() == lo and strand.idx3Prime() == hi
            # map the first base (5' xover if necessary)
            s5p = strand.connection5p()
            if s5p is not None:
                ret[lo][0] = s5p.idNum()
                ret[lo][1] = s5p.idx3Prime()
            ret[lo][2] = num
            ret[lo][3] = lo + 1
            # map the internal bases
            for idx in range(lo + 1, hi):
                ret[idx][0] = num
                ret[idx][1] = idx - 1
                ret[idx][2] = num
                ret[idx][3] = idx + 1
            # map the last base (3' xover if necessary)
            ret[hi][0] = num
            ret[hi][1] = hi - 1
            s3p = strand.connection3p()
            if s3p is not None:
                ret[hi][2] = s3p.idNum()
                ret[hi][3] = s3p.idx5Prime()
            # end if
        # end for
    # end if
    else:
        for strand in ss.strands():
            lo, hi = strand.idxs()
            assert strand.idx3Prime() == lo and strand.idx5Prime() == hi
            # map the first base (3' xover if necessary)
            ret[lo][0] = num
            ret[lo][1] = lo + 1
            s3p = strand.connection3p()
            if s3p is not None:
                ret[lo][2] = s3p.idNum()
                ret[lo][3] = s3p.idx5Prime()
            # map the internal bases
            for idx in range(lo + 1, hi):
                ret[idx][0] = num
                ret[idx][1] = idx + 1
                ret[idx][2] = num
                ret[idx][3] = idx - 1
            # map the last base (5' xover if necessary)
            ret[hi][2] = num
            ret[hi][3] = hi - 1
            s5p = strand.connection5p()
            if s5p is not None:
                ret[hi][0] = s5p.idNum()
                ret[hi][1] = s5p.idx3Prime()
            # end if
        # end for
    return ret
# end def
<|endoftext|>


<commit_before>from django.http import Http404
from calaccess_raw import get_model_list
from django.core.urlresolvers import reverse
from django.template.defaultfilters import slugify
from bakery.views import (
    BuildableArchiveIndexView,
    BuildableYearArchiveView,
    BuildableDetailView,
    BuildableListView,
    BuildableRedirectView,
    Buildable404View,
)
from calaccess_raw.models.tracking import RawDataVersion, RawDataFile

#
# Version based archives
#

class VersionArchiveIndex(BuildableArchiveIndexView):
    """
    A list of the latest versions of CAL-ACCESS in our archive
    """
    model = RawDataVersion
    date_field = "release_datetime"
    template_name = "calaccess_website/version_archive.html"
    build_path = "index.html"

class VersionYearArchiveList(BuildableYearArchiveView):
    """
    A list of all versions of CAL-ACCESS in a given year
    """
    model = RawDataVersion
    date_field = "release_datetime"
    make_object_list = True
    template_name = "calaccess_website/version_archive_year.html"

class VersionDetail(BuildableDetailView):
    """
    A detail page with everything about an individual CAL-ACCESS version
    """
    model = RawDataVersion
    template_name = 'calaccess_website/version_detail.html'

    def get_url(self, obj):
        return reverse('version_detail', kwargs=dict(pk=obj.pk))

class LatestVersion(BuildableRedirectView):
    """
    Redirect to the detail page of the latest CAL-ACCESS version
    """
    build_path = "versions/latest/index.html"
    pattern_name = 'version_detail'

    def get_redirect_url(self, *args, **kwargs):
        try:
            obj = RawDataVersion.objects.latest('release_datetime')
        except RawDataVersion.DoesNotExist:
            raise Http404
        return reverse("version_detail", kwargs=dict(pk=obj.pk))

#
# Raw data file based archives
#

class RawDataFileList(BuildableListView):
    queryset = get_model_list()
    template_name = 'calaccess_website/raw_data_files_list.html'
    context_object_name = 'raw data files'

class RawDataFileDetail(BuildableDetailView):
    """
    A detail page with everything we know about the provided raw data file.
    """
    template_name = 'calaccess_website/rawdatafile_detail.html'

    def get_queryset(self):
        """
        Returns a list of the raw data files as a key dictionary
        with the URL slug as the keys.
        """
        return dict((slugify(m().db_table), m) for m in get_model_list())

    def set_kwargs(self, obj):
        self.kwargs = {
            'file_name': obj
        }

    def get_object(self):
        """
        Returns the file model from the CAL-ACCESS raw data app that
        matches the provided slug.

        Raises a 404 error if one is not found
        """
        key = self.kwargs['file_name']
        try:
            return self.get_queryset()[key.lower()]
        except KeyError:
            raise Http404

    def get_url(self, obj):
        return reverse('rawdatafile_detail', kwargs=dict(file_name=obj))

    def get_context_data(self, **kwargs):
        """
        Add some extra bits to the template's context
        """
        context = super(RawDataFileDetail, self).get_context_data(**kwargs)
        # Pull all previous versions of the provided file
        context['version_list'] = RawDataFile.objects.filter(
            file_name=self.kwargs['file_name'].upper()
        ).order_by('-version__release_datetime')
        return context

    def build_queryset(self):
        [self.build_object(o) for o in self.get_queryset()]

#
# Extra stuff
#

class CalAccess404View(Buildable404View):
    build_path = "404.html"
    template_name = "calaccess_website/404.html"
<commit_msg>Add build path to RawDataFileList view<commit_after>from django.http import Http404
from calaccess_raw import get_model_list
from django.core.urlresolvers import reverse
from django.template.defaultfilters import slugify
from bakery.views import (
    BuildableArchiveIndexView,
    BuildableYearArchiveView,
    BuildableDetailView,
    BuildableListView,
    BuildableRedirectView,
    Buildable404View,
)
from calaccess_raw.models.tracking import RawDataVersion, RawDataFile

#
# Version based archives
#

class VersionArchiveIndex(BuildableArchiveIndexView):
    """
    A list of the latest versions of CAL-ACCESS in our archive
    """
    model = RawDataVersion
    date_field = "release_datetime"
    template_name = "calaccess_website/version_archive.html"
    build_path = "index.html"

class VersionYearArchiveList(BuildableYearArchiveView):
    """
    A list of all versions of CAL-ACCESS in a given year
    """
    model = RawDataVersion
    date_field = "release_datetime"
    make_object_list = True
    template_name = "calaccess_website/version_archive_year.html"

class VersionDetail(BuildableDetailView):
    """
    A detail page with everything about an individual CAL-ACCESS version
    """
    model = RawDataVersion
    template_name = 'calaccess_website/version_detail.html'

    def get_url(self, obj):
        return reverse('version_detail', kwargs=dict(pk=obj.pk))

class LatestVersion(BuildableRedirectView):
    """
    Redirect to the detail page of the latest CAL-ACCESS version
    """
    build_path = "versions/latest/index.html"
    pattern_name = 'version_detail'

    def get_redirect_url(self, *args, **kwargs):
        try:
            obj = RawDataVersion.objects.latest('release_datetime')
        except RawDataVersion.DoesNotExist:
            raise Http404
        return reverse("version_detail", kwargs=dict(pk=obj.pk))

#
# Raw data file based archives
#

class RawDataFileList(BuildableListView):
    queryset = get_model_list()
    template_name = 'calaccess_website/raw_data_files_list.html'
    context_object_name = 'raw data files'
    build_path = "raw-data-files/index.html"

class RawDataFileDetail(BuildableDetailView):
    """
    A detail page with everything we know about the provided raw data file.
    """
    template_name = 'calaccess_website/rawdatafile_detail.html'

    def get_queryset(self):
        """
        Returns a list of the raw data files as a key dictionary
        with the URL slug as the keys.
        """
        return dict((slugify(m().db_table), m) for m in get_model_list())

    def set_kwargs(self, obj):
        self.kwargs = {
            'file_name': obj
        }

    def get_object(self):
        """
        Returns the file model from the CAL-ACCESS raw data app that
        matches the provided slug.

        Raises a 404 error if one is not found
        """
        key = self.kwargs['file_name']
        try:
            return self.get_queryset()[key.lower()]
        except KeyError:
            raise Http404

    def get_url(self, obj):
        return reverse('rawdatafile_detail', kwargs=dict(file_name=obj))

    def get_context_data(self, **kwargs):
        """
        Add some extra bits to the template's context
        """
        context = super(RawDataFileDetail, self).get_context_data(**kwargs)
        # Pull all previous versions of the provided file
        context['version_list'] = RawDataFile.objects.filter(
            file_name=self.kwargs['file_name'].upper()
        ).order_by('-version__release_datetime')
        return context

    def build_queryset(self):
        [self.build_object(o) for o in self.get_queryset()]

#
# Extra stuff
#

class CalAccess404View(Buildable404View):
    build_path = "404.html"
    template_name = "calaccess_website/404.html"
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from .helpers import cheskHashOnReceiveResult
from django import forms
from django.conf import settings
from django.core.urlresolvers import reverse

class _BaseForm(forms.Form):

    eshopId = forms.CharField(initial=settings.INTELLECTMONEY_SHOPID)
    orderId = forms.CharField(max_length=50)

    def clean_eshopId(self):
        eshopId = self.cleaned_data['eshopId']
        if eshopId != settings.INTELLECTMONEY_SHOPID:
            raise forms.ValidationError(u'Неверный eshopId')
        return eshopId

class _BasePaymentForm(_BaseForm):

    CURRENCY_CHOICES = map(lambda x: (x, x), ['RUR', 'TST'])
    serviceName = forms.CharField(label=u'Payment Description', required=False)
    recipientAmount = forms.DecimalField(max_digits=10, decimal_places=2)
    recipientCurrency = forms.ChoiceField(
        choices=CURRENCY_CHOICES,
        initial=settings.INTELLECTMONEY_DEBUG and 'TST' or 'RUR'
    )
    userName = forms.CharField(max_length=255, required=False)
    email = forms.EmailField(required=False)

class IntellectMoneyForm(_BasePaymentForm):
    """Payment request form."""

    PREFERENCE_CHOICES = [
        ('inner', 'IntellectMoney'),
        ('bankCard', 'Visa/MasterCard'),
        ('exchangers', u'Internet Exchangers'),
        ('terminals', u'Terminals'),
        ('bank', u'Bank'),
    ]

    successUrl = forms.CharField(
        required=False, max_length=512,
        initial=settings.INTELLECTMONEY_SUCCESS_URL
    )
    failUrl = forms.CharField(
        required=False, max_length=512,
        initial=settings.INTELLECTMONEY_FAIL_URL
    )
    preference = forms.ChoiceField(
        label=u'Payment Method', choices=PREFERENCE_CHOICES
    )
    expireDate = forms.DateTimeField(required=False)
    hash = forms.CharField(required=False)

class ResultUrlForm(_BasePaymentForm):

    STATUS_CHOICES = [
        (3, u'Создан счет к оплате (СКО) за покупку'),
        (4, u'СКО аннулирован, деньги возвращены пользователю'),
        (7, u'СКО частично оплачен'),
        (5, u'СКО полностью оплачен'),
        (6, u'Cумма заблокирована на СКО, ожидается запрос на списание'),
    ]

    paymentId = forms.CharField(label=u'IntellectMoney Payment ID')
    paymentData = forms.DateTimeField(input_formats=['%Y-%m-%d %H:%M:%S'])
    paymentStatus= forms.TypedChoiceField(choices=STATUS_CHOICES, coerce=int)
    eshopAccount = forms.CharField()
    hash = forms.CharField()
    secretKey = forms.CharField()

    def __init__(self, *args, **kwargs):
        super(ResultUrlForm, self).__init__(*args, **kwargs)
        if settings.INTELLECTMONEY_SEND_SECRETKEY:
            self.fields['hash'].required = False
            self.fields['secretKey'].required = True
        else:
            self.fields['hash'].required = True
            self.fields['secretKey'].required = False

    def clean_secretKey(self):
        secretKey = self.cleaned_data['secretKey']
        if settings.INTELLECTMONEY_SEND_SECRETKEY:
            if secretKey != settings.INTELLECTMONEY_SECRETKEY:
                raise forms.ValidationError(u'Неверное значение')
        return secretKey

    def clean(self):
        data = self.cleaned_data
        if not settings.INTELLECTMONEY_SEND_SECRETKEY:
            if not cheskHashOnReceiveResult(data):
                raise forms.ValidationError(u'Неверный hash')
        return data

class AcceptingForm(_BaseForm):

    ACTION_CHOICES = [
        ('Refund', 'Refund'),
        ('ToPaid', 'ToPaid')
    ]

    action = forms.ChoiceField(choices=ACTION_CHOICES)
    secretKey = forms.CharField()
<commit_msg>Способ оплаты необязателен<commit_after># -*- coding: utf-8 -*-
from .helpers import cheskHashOnReceiveResult
from django import forms
from django.conf import settings
from django.core.urlresolvers import reverse

class _BaseForm(forms.Form):

    eshopId = forms.CharField(initial=settings.INTELLECTMONEY_SHOPID)
    orderId = forms.CharField(max_length=50)

    def clean_eshopId(self):
        eshopId = self.cleaned_data['eshopId']
        if eshopId != settings.INTELLECTMONEY_SHOPID:
            raise forms.ValidationError(u'Неверный eshopId')
        return eshopId

class _BasePaymentForm(_BaseForm):

    CURRENCY_CHOICES = map(lambda x: (x, x), ['RUR', 'TST'])
    serviceName = forms.CharField(label=u'Payment Description', required=False)
    recipientAmount = forms.DecimalField(max_digits=10, decimal_places=2)
    recipientCurrency = forms.ChoiceField(
        choices=CURRENCY_CHOICES,
        initial=settings.INTELLECTMONEY_DEBUG and 'TST' or 'RUR'
    )
    userName = forms.CharField(max_length=255, required=False)
    email = forms.EmailField(required=False)

class IntellectMoneyForm(_BasePaymentForm):
    """Payment request form."""

    PREFERENCE_CHOICES = [
        ('inner', 'IntellectMoney'),
        ('bankCard', 'Visa/MasterCard'),
        ('exchangers', u'Internet Exchangers'),
        ('terminals', u'Terminals'),
        ('bank', u'Bank'),
    ]

    successUrl = forms.CharField(
        required=False, max_length=512,
        initial=settings.INTELLECTMONEY_SUCCESS_URL
    )
    failUrl = forms.CharField(
        required=False, max_length=512,
        initial=settings.INTELLECTMONEY_FAIL_URL
    )
    preference = forms.ChoiceField(
        label=u'Payment Method', choices=PREFERENCE_CHOICES, required=False
    )
    expireDate = forms.DateTimeField(required=False)
    hash = forms.CharField(required=False)

class ResultUrlForm(_BasePaymentForm):

    STATUS_CHOICES = [
        (3, u'Создан счет к оплате (СКО) за покупку'),
        (4, u'СКО аннулирован, деньги возвращены пользователю'),
        (7, u'СКО частично оплачен'),
        (5, u'СКО полностью оплачен'),
        (6, u'Cумма заблокирована на СКО, ожидается запрос на списание'),
    ]

    paymentId = forms.CharField(label=u'IntellectMoney Payment ID')
    paymentData = forms.DateTimeField(input_formats=['%Y-%m-%d %H:%M:%S'])
    paymentStatus= forms.TypedChoiceField(choices=STATUS_CHOICES, coerce=int)
    eshopAccount = forms.CharField()
    hash = forms.CharField()
    secretKey = forms.CharField()

    def __init__(self, *args, **kwargs):
        super(ResultUrlForm, self).__init__(*args, **kwargs)
        if settings.INTELLECTMONEY_SEND_SECRETKEY:
            self.fields['hash'].required = False
            self.fields['secretKey'].required = True
        else:
            self.fields['hash'].required = True
            self.fields['secretKey'].required = False

    def clean_secretKey(self):
        secretKey = self.cleaned_data['secretKey']
        if settings.INTELLECTMONEY_SEND_SECRETKEY:
            if secretKey != settings.INTELLECTMONEY_SECRETKEY:
                raise forms.ValidationError(u'Неверное значение')
        return secretKey

    def clean(self):
        data = self.cleaned_data
        if not settings.INTELLECTMONEY_SEND_SECRETKEY:
            if not cheskHashOnReceiveResult(data):
                raise forms.ValidationError(u'Неверный hash')
        return data

class AcceptingForm(_BaseForm):

    ACTION_CHOICES = [
        ('Refund', 'Refund'),
        ('ToPaid', 'ToPaid')
    ]

    action = forms.ChoiceField(choices=ACTION_CHOICES)
    secretKey = forms.CharField()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Author: [AbhishekKr <abhikumar@gmail.com>](http://abhishekkr.github.io)
"""

import os
import yaml

def refresh_host_yaml(host_yaml, type={}):
    """ Prepares empty host yaml if not present. """
    with open(host_yaml, "w") as f:
            yaml.dump(type, f)

def state_to_yaml(host_yaml, res, state=None):
    """ Update Host yaml log file states. """
    module = res['invocation']['module_name']

    if not os.path.exists(host_yaml):
        refresh_host_yaml(host_yaml)

    with open(host_yaml) as f:
        newdct = yaml.load(f)

    newdct[str(module)] = { "state": str(state) }

    with open(host_yaml, "w") as f:
        yaml.dump(newdct, f)

def get_host_yaml(host_log, log_dir, host):
    """ Update hostlog yaml with host name. Return path for host yaml to be log created. """
    host = host.split("->")[0].strip()

    with open(host_log) as f:
        host_list = yaml.load(f)

    if host not in host_list:
        host_list.append(host)

    with open(host_log, "w") as f:
        yaml.dump(host_list, f)

    return os.path.join(log_dir.strip(), host)

class CallbackModule(object):
    """
    This plugin makes use of the following environment variables:
        DASHR_LOG_DIRECTORY (else uses /var/log/ansible to create dashr logs)
    """

    def __init__(self):
        self.dashr_log_directory = os.getenv('DASHR_LOG_DIRECTORY', '/var/log/ansible')

        self.dashr_hostlog_directory = os.path.join(self.dashr_log_directory, "hosts")
        if not os.path.exists(self.dashr_hostlog_directory):
            os.makedirs(self.dashr_hostlog_directory)

        self.dashr_hostlog = os.path.join(self.dashr_log_directory, "dashr_log_hostlist.yaml")
        if not os.path.exists(self.dashr_hostlog):
            refresh_host_yaml(self.dashr_hostlog, [])

    def on_any(self, *args, **kwargs):
        pass

    def runner_on_failed(self, host, res, ignore_errors=False):
        host_yaml = get_host_yaml(self.dashr_hostlog, self.dashr_hostlog_directory, host)
        state_to_yaml(host_yaml , res, "failed")

    def runner_on_ok(self, host, res):
        host_yaml = get_host_yaml(self.dashr_hostlog, self.dashr_hostlog_directory, host)
        state_to_yaml(host_yaml, res, "ok")

    def runner_on_error(self, host, msg):
        pass

    def runner_on_skipped(self, host, item=None):
        pass

    def runner_on_unreachable(self, host, res):
        host_yaml = get_host_yaml(self.dashr_hostlog, self.dashr_hostlog_directory, host)
        state_to_yaml(host_yaml, res, "unreachable")

    def runner_on_no_hosts(self):
        pass

    def runner_on_async_poll(self, host, res, jid, clock):
        host_yaml = get_host_yaml(self.dashr_hostlog, self.dashr_hostlog_directory, host)
        state_to_yaml(host_yaml, res)

    def runner_on_async_ok(self, host, res, jid):
        host_yaml = get_host_yaml(self.dashr_hostlog, self.dashr_hostlog_directory, host)
        state_to_yaml(host_yaml, res, "ok")

    def runner_on_async_failed(self, host, res, jid):
        host_yaml = get_host_yaml(self.dashr_hostlog, self.dashr_hostlog_directory, host)
        state_to_yaml(host_yaml, res, "failed")

    def playbook_on_start(self):
        pass

    def playbook_on_notify(self, host, handler):
        pass

    def playbook_on_no_hosts_matched(self):
        pass

    def playbook_on_no_hosts_remaining(self):
        pass

    def playbook_on_task_start(self, name, is_conditional):
        pass

    def playbook_on_vars_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, salt=None, default=None):
        pass

    def playbook_on_setup(self):
        pass

    def playbook_on_import_for_host(self, host, imported_file):
        pass

    def playbook_on_not_import_for_host(self, host, missing_file):
        pass

    def playbook_on_play_start(self, pattern):
        pass

    def playbook_on_stats(self, stats):
        pass
<commit_msg>[callback] updated to record desired details if present of tasks<commit_after>#!/usr/bin/env python

"""
Author: [AbhishekKr <abhikumar@gmail.com>](http://abhishekkr.github.io)
"""

import os
import yaml

EXTRA_DETAILS = ['cmd', 'command', 'start', 'end', 'delta', 'msg', 'stdout', 'stderr']

def prepare_extra_details(res):
    """ Add EXTRA_DETAIL field matched details to task yaml. """
    details = {}
    if type(res) == type(dict()):
      for field in EXTRA_DETAILS:
        if field in res.keys():
            details[field] = res[field].encode('utf-8')
    return details

def refresh_host_yaml(host_yaml, type={}):
    """ Prepares empty host yaml if not present. """
    with open(host_yaml, "w") as f:
            yaml.dump(type, f)

def state_to_yaml(host_yaml, res, state=None):
    """ Update Host yaml log file states. """
    module = res['invocation']['module_name']

    if not os.path.exists(host_yaml):
        refresh_host_yaml(host_yaml)

    with open(host_yaml) as f:
        newdct = yaml.load(f)

    newdct[str(module)] = { "state": str(state), "details": prepare_extra_details(res) }

    with open(host_yaml, "w") as f:
        yaml.dump(newdct, f)

def get_host_yaml(host_log, log_dir, host):
    """ Update hostlog yaml with host name. Return path for host yaml to be log created. """
    host = host.split("->")[0].strip()

    with open(host_log) as f:
        host_list = yaml.load(f)

    if host not in host_list:
        host_list.append(host)

    with open(host_log, "w") as f:
        yaml.dump(host_list, f)

    return os.path.join(log_dir.strip(), host)

class CallbackModule(object):
    """
    This plugin makes use of the following environment variables:
        DASHR_LOG_DIRECTORY (else uses /var/log/ansible to create dashr logs)
    """

    def __init__(self):
        self.dashr_log_directory = os.getenv('DASHR_LOG_DIRECTORY', '/var/log/ansible')

        self.dashr_hostlog_directory = os.path.join(self.dashr_log_directory, "hosts")
        if not os.path.exists(self.dashr_hostlog_directory):
            os.makedirs(self.dashr_hostlog_directory)

        self.dashr_hostlog = os.path.join(self.dashr_log_directory, "dashr_log_hostlist.yaml")
        if not os.path.exists(self.dashr_hostlog):
            refresh_host_yaml(self.dashr_hostlog, [])

    def on_any(self, *args, **kwargs):
        pass

    def runner_on_failed(self, host, res, ignore_errors=False):
        host_yaml = get_host_yaml(self.dashr_hostlog, self.dashr_hostlog_directory, host)
        state_to_yaml(host_yaml , res, "failed")

    def runner_on_ok(self, host, res):
        host_yaml = get_host_yaml(self.dashr_hostlog, self.dashr_hostlog_directory, host)
        state_to_yaml(host_yaml, res, "ok")

    def runner_on_error(self, host, msg):
        pass

    def runner_on_skipped(self, host, item=None):
        pass

    def runner_on_unreachable(self, host, res):
        host_yaml = get_host_yaml(self.dashr_hostlog, self.dashr_hostlog_directory, host)
        state_to_yaml(host_yaml, res, "unreachable")

    def runner_on_no_hosts(self):
        pass

    def runner_on_async_poll(self, host, res, jid, clock):
        host_yaml = get_host_yaml(self.dashr_hostlog, self.dashr_hostlog_directory, host)
        state_to_yaml(host_yaml, res)

    def runner_on_async_ok(self, host, res, jid):
        host_yaml = get_host_yaml(self.dashr_hostlog, self.dashr_hostlog_directory, host)
        state_to_yaml(host_yaml, res, "ok")

    def runner_on_async_failed(self, host, res, jid):
        host_yaml = get_host_yaml(self.dashr_hostlog, self.dashr_hostlog_directory, host)
        state_to_yaml(host_yaml, res, "failed")

    def playbook_on_start(self):
        pass

    def playbook_on_notify(self, host, handler):
        pass

    def playbook_on_no_hosts_matched(self):
        pass

    def playbook_on_no_hosts_remaining(self):
        pass

    def playbook_on_task_start(self, name, is_conditional):
        pass

    def playbook_on_vars_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, salt=None, default=None):
        pass

    def playbook_on_setup(self):
        pass

    def playbook_on_import_for_host(self, host, imported_file):
        pass

    def playbook_on_not_import_for_host(self, host, missing_file):
        pass

    def playbook_on_play_start(self, pattern):
        pass

    def playbook_on_stats(self, stats):
        pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2015 Ericsson AB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json

from calvin.csparser.parser import calvin_parser
from calvin.actorstore.store import DocumentationStore
import json

class Checker(object):
    # FIXME: Provide additional checks making use of
    #        - actor_def.inport_names and actor_def.outport_names
    #        - make use of arg_type (STRING, NUMBER, etc.)
    #        - analyze the actions wrt port use and token consumption:
    #          for f in actor_def.action_priority:
    #              print f.__name__, [x.cell_contents for x in f.__closure__]
    #
    def __init__(self, cs_info):
        super(Checker, self).__init__()
        self.ds = DocumentationStore()
        self.cs_info = cs_info
        self.constants = self.cs_info['constants']
        self.local_actors = self.cs_info['components']
        self.errors = []
        self.warnings = []
        self.check()

    def issue(self, fmt, **info):
        return {'reason':fmt.format(**info), 'line':info.get('line', 0), 'col':info.get('col', 0)}

    def append_error(self, fmt, **info):
        issue = self.issue(fmt, **info)
        self.errors.append(issue)

    def append_warning(self, fmt, **info):
        issue = self.issue(fmt, **info)
        self.warnings.append(issue)

    def check(self):
        """
        Check constants, local components, and program, in that order.
        Generate error and warning issues as they are encountered.
        """
        self.check_constants()
        for comp in self.local_actors.values():
            self.check_component(comp)
        self.check_structure(self.cs_info['structure'])

    def check_constants(self):
        """Verify that all constant definitions evaluate to a value."""
        for constant in self.constants:
            try:
                self.lookup_constant(constant)
            except KeyError as e:
                fmt = "Constant '{name}' is undefined"
                self.append_error(fmt, name=e.args[0])

    def check_component(self, comp_def):
        """
        Check connections, structure, and argument for local component
        """
        defs = self.get_definition(comp_def['name'])
        self.check_connections(None, defs, comp_def['structure']['connections'])
        self.check_structure(comp_def['structure'])
        self.check_component_arguments(comp_def)

    def check_component_arguments(self, comp_def):
        # FIXME: Compare to check_arguments and extend?
        # Check for unused arguments and issue warning, not error
        args = set(comp_def['arg_identifiers'])
        used_args = set([])
        for a in comp_def['structure']['actors'].values():
            used_args.update(a['args'].keys())
        unused_args = args - used_args
        for u in unused_args:
            fmt = "Unused argument: '{param}'"
            self.append_warning(fmt, line=comp_def['dbg_line'], param=u)

    def get_definition(self, actor_type):
        """
        Get the actor/component definition from the docstore.
        For local components, let the docstore generate the definition.
        """
        if actor_type in self.local_actors:
            return self.ds.component_docs("local."+actor_type, self.local_actors[actor_type])
        return self.ds.actor_docs(actor_type)

    def lookup_constant(self, identifier):
        """
        Return value for constant 'identifier' by recursively looking for a value.
        Raise an exception if not found
        """
        kind, value = self.constants[identifier]
        if kind != "IDENTIFIER":
            return value
        return self.lookup_constant(value)

    def dbg_lines(self, s):
        try:
            return [x['dbg_line'] for x in s] + [0]
        except:
            return [0]

    def _verify_port_names(self, definition, connections, actor=None):
        """Look for misspelled port names."""
        # A little transformation is required depending on actor vs. component and port direction
        # # component inports:
        # invalid_ports = [(c['src_port'], 'in', c['dbg_line']) for c in connections if c['src'] == "." and c['src_port'] not in ports]
        # # component outports:
        # invalid_ports = [(c['dst_port'], 'out', c['dbg_line']) for c in connections if c['dst'] == "." and c['dst_port'] not in ports]
        # # actor inports:
        # invalid_ports = [(c['dst_port'], 'in', c['dbg_line']) for c in connections if c['dst'] == actor and c['dst_port'] not in ports]
        # # actor outports:
        # invalid_ports = [(c['src_port'], 'out', c['dbg_line']) for c in connections if c['src'] == actor and c['src_port'] not in ports]
        retval = []
        is_component = actor is None
        actor = '.' if is_component else actor
        for port_dir in ['in', 'out']:
            if is_component:
                target, port = ('dst', 'dst_port') if port_dir == 'out' else ('src', 'src_port')
            else:
                target, port = ('dst', 'dst_port') if port_dir == 'in' else ('src', 'src_port')
            def_dir = 'inputs' if port_dir == 'in' else 'outputs'
            ports = [p for p, _ in definition[def_dir]]
            invalid_ports = [(c[port], port_dir, c['dbg_line']) for c in connections if c[target] == actor and c[port] not in ports]
            retval.extend(invalid_ports)

        return retval

    def check_component_connections(self, definition, connections):
        invalid_ports = self._verify_port_names(definition, connections)
        for port, port_dir, line in invalid_ports:
            fmt = "Component {name} has no {port_dir}port '{port}'"
            self.append_error(fmt, line=line, port=port, port_dir=port_dir, **definition)

        inport_names = [p for p, _ in definition['inputs']]
        outport_names = [p for p, _ in definition['outputs']]

        # outports should have exactly one connection
        for port in outport_names:
            incoming = [c for c in connections if c['dst'] == "." and c['dst_port'] == port]
            if len(incoming) == 0:
                fmt = "Component {name} is missing connection to outport '{port}'"
                self.append_error(fmt, line=max(self.dbg_lines(connections)), port=port, **definition)
            if len(incoming) > 1:
                fmt = "Component {name} has multiple connections to outport '{port}'"
                for c in incoming:
                    self.append_error(fmt, line=c['dbg_line'], port=port, **definition)
        # inports should have at least one connection
        for port in inport_names:
            outgoing = [c for c in connections if c['src'] == "." and c['src_port'] == port]
            if len(outgoing) < 1:
                fmt = "Component {name} is missing connection to inport '{port}'"
                self.append_error(fmt, line=max(self.dbg_lines(connections)), port=port, **definition)

    def check_actor_connections(self, actor, definition, connections):

        invalid_ports = self._verify_port_names(definition, connections, actor)
        for port, port_dir, line in invalid_ports:
            fmt = "Actor {actor} ({ns}.{name}) has no {port_dir}port '{port}'"
            self.append_error(fmt, line=line, port=port, port_dir=port_dir, actor=actor, **definition)

        inport_names = [p for p, _ in definition['inputs']]
        outport_names = [p for p, _ in definition['outputs']]
        # inports should have exactly one connection
        for port in inport_names:
            incoming = [c for c in connections if c['dst'] == actor and c['dst_port'] == port]
            if len(incoming) == 0:
                fmt = "Actor {actor} ({ns}.{name}) is missing connection to inport '{port}'"
                self.append_error(fmt, line=max(self.dbg_lines(connections)), port=port, actor=actor, **definition)
            if len(incoming) > 1:
                fmt = "Actor {actor} ({ns}.{name}) has multiple connections to inport '{port}'"
                for c in incoming:
                    self.append_error(fmt, line=c['dbg_line'], port=port, actor=actor, **definition)
        # outports should have at least one connection
        for port in outport_names:
            outgoing = [c for c in connections if c['src'] == actor and c['src_port'] == port]
            if not outgoing:
                fmt = "Actor {actor} ({ns}.{name}) is missing connection to outport '{port}'"
                self.append_error(fmt, line=max(self.dbg_lines(connections)), port=port, actor=actor, **definition)

    def check_connections(self, actor, definition, connections):
        if actor:
            self.check_actor_connections(actor, definition, connections)
        else:
            self.check_component_connections(definition, connections)

    def expand_arguments(self, declaration):
        """
        Check the the arguments for constants that must be expanded
        Append errors if definition is missing.
        """
        for argname, (kind, value) in declaration['args'].iteritems():
            if kind != "IDENTIFIER":
                continue
            try:
                self.lookup_constant(value)
            except:
                fmt = "Undefined identifier: '{param}'"
                self.append_error(fmt, line=declaration['dbg_line'], param=value)

    def check_arguments(self, definition, declaration):
        mandatory = set(definition['args']['mandatory'])
        defined = set(declaration['args'].keys())
        missing = mandatory - defined
        # Case 1: Missing parameters
        for m in missing:
            fmt = "Missing argument: '{param}'"
            self.append_error(fmt, line=declaration['dbg_line'], param=m)
        # FIXME: Case 2: Unused parameter
        # FIXME: Case 3: value for arg is IDENTIFIER rather than VALUE, and not defined in constants
        # self.expand_arguments(declaration)

    def check_structure(self, structure):
        actors = structure['actors'].keys()

        # Look for undefined actors
        src_actors = {c['src'] for c in structure['connections'] if c['src'] != "."}
        dst_actors = {c['dst'] for c in structure['connections'] if c['dst'] != "."}
        # Implicit src actors are defined by a constant on the inport
        implicit_src_actors = {c['src'] for c in structure['connections'] if c['src'] is None}
        all_actors = src_actors | dst_actors
        undefined_actors = all_actors - (set(actors) | implicit_src_actors)
        for actor in undefined_actors:
            fmt = "Undefined actor: '{actor}'"
            lines = [c['dbg_line'] for c in structure['connections'] if c['src'] == actor or c['dst'] == actor]
            for line in lines:
                self.append_error(fmt, line=line, actor=actor)

        # Note: Unused actors will be caught when checking connections

        # Check if actor exists
        for actor in actors:
            actor_type = structure['actors'][actor]['actor_type']
            definition = self.get_definition(actor_type)
            if not definition:
                fmt = "Unknown actor type: '{type}'"
                self.append_error(fmt, type=actor_type, line=structure['actors'][actor]['dbg_line'])
                continue
            # We have actor definition, check that it is fully connected
            self.check_connections(actor, definition, structure['connections'])
            self.check_arguments(definition, structure['actors'][actor])

def check(cs_info):
    clint = Checker(cs_info)
    return clint.errors, clint.warnings

if __name__ == '__main__':
    import sys
    import os
    import json

    if len(sys.argv) < 2:
        script = 'inline'
        source_text = \
"""# Test script
        component Count(len) -> seq {
            src : std.Constant(data="hup", n=len)
            src.token > .seq
        }

        src: Count(len=5)
        snk : io.StandardOut()
        src.seq > snk.token
"""
    else:
        script = sys.argv[1]
        script = os.path.expanduser(script)
        try:
            with open(script, 'r') as source:
                source_text = source.read()
        except:
            print "Error: Could not read file: '%s'" % script
            sys.exit(1)

    result, errors, warnings = calvin_parser(source_text, script)
    if errors:
        print "{reason} {script} [{line}:{col}]".format(script=script, **error)
    else:
        errors, warnings = check(result)
        print "errors:", [x['reason'] for x in errors]
        print "warnings:", [x['reason'] for x in warnings]
<commit_msg>Introduce at least one connection check.<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2015 Ericsson AB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json

from calvin.csparser.parser import calvin_parser
from calvin.actorstore.store import DocumentationStore
import json

class Checker(object):
    # FIXME: Provide additional checks making use of
    #        - actor_def.inport_names and actor_def.outport_names
    #        - make use of arg_type (STRING, NUMBER, etc.)
    #        - analyze the actions wrt port use and token consumption:
    #          for f in actor_def.action_priority:
    #              print f.__name__, [x.cell_contents for x in f.__closure__]
    #
    def __init__(self, cs_info):
        super(Checker, self).__init__()
        self.ds = DocumentationStore()
        self.cs_info = cs_info
        self.constants = self.cs_info['constants']
        self.local_actors = self.cs_info['components']
        self.errors = []
        self.warnings = []
        self.check()

    def issue(self, fmt, **info):
        return {'reason':fmt.format(**info), 'line':info.get('line', 0), 'col':info.get('col', 0)}

    def append_error(self, fmt, **info):
        issue = self.issue(fmt, **info)
        self.errors.append(issue)

    def append_warning(self, fmt, **info):
        issue = self.issue(fmt, **info)
        self.warnings.append(issue)

    def check(self):
        """
        Check constants, local components, and program, in that order.
        Generate error and warning issues as they are encountered.
        """
        self.check_constants()
        for comp in self.local_actors.values():
            self.check_component(comp)
        self.check_structure(self.cs_info['structure'])

    def check_constants(self):
        """Verify that all constant definitions evaluate to a value."""
        for constant in self.constants:
            try:
                self.lookup_constant(constant)
            except KeyError as e:
                fmt = "Constant '{name}' is undefined"
                self.append_error(fmt, name=e.args[0])

    def check_component(self, comp_def):
        """
        Check connections, structure, and argument for local component
        """
        defs = self.get_definition(comp_def['name'])
        self.check_connections(None, defs, comp_def['structure']['connections'])
        self.check_structure(comp_def['structure'])
        self.check_component_arguments(comp_def)

    def check_component_arguments(self, comp_def):
        # FIXME: Compare to check_arguments and extend?
        # Check for unused arguments and issue warning, not error
        args = set(comp_def['arg_identifiers'])
        used_args = set([])
        for a in comp_def['structure']['actors'].values():
            used_args.update(a['args'].keys())
        unused_args = args - used_args
        for u in unused_args:
            fmt = "Unused argument: '{param}'"
            self.append_warning(fmt, line=comp_def['dbg_line'], param=u)

    def get_definition(self, actor_type):
        """
        Get the actor/component definition from the docstore.
        For local components, let the docstore generate the definition.
        """
        if actor_type in self.local_actors:
            return self.ds.component_docs("local."+actor_type, self.local_actors[actor_type])
        return self.ds.actor_docs(actor_type)

    def lookup_constant(self, identifier):
        """
        Return value for constant 'identifier' by recursively looking for a value.
        Raise an exception if not found
        """
        kind, value = self.constants[identifier]
        if kind != "IDENTIFIER":
            return value
        return self.lookup_constant(value)

    def dbg_lines(self, s):
        try:
            return [x['dbg_line'] for x in s] + [0]
        except:
            return [0]

    def _verify_port_names(self, definition, connections, actor=None):
        """Look for misspelled port names."""
        # A little transformation is required depending on actor vs. component and port direction
        # # component inports:
        # invalid_ports = [(c['src_port'], 'in', c['dbg_line']) for c in connections if c['src'] == "." and c['src_port'] not in ports]
        # # component outports:
        # invalid_ports = [(c['dst_port'], 'out', c['dbg_line']) for c in connections if c['dst'] == "." and c['dst_port'] not in ports]
        # # actor inports:
        # invalid_ports = [(c['dst_port'], 'in', c['dbg_line']) for c in connections if c['dst'] == actor and c['dst_port'] not in ports]
        # # actor outports:
        # invalid_ports = [(c['src_port'], 'out', c['dbg_line']) for c in connections if c['src'] == actor and c['src_port'] not in ports]
        retval = []
        is_component = actor is None
        actor = '.' if is_component else actor
        for port_dir in ['in', 'out']:
            if is_component:
                target, port = ('dst', 'dst_port') if port_dir == 'out' else ('src', 'src_port')
            else:
                target, port = ('dst', 'dst_port') if port_dir == 'in' else ('src', 'src_port')
            def_dir = 'inputs' if port_dir == 'in' else 'outputs'
            ports = [p for p, _ in definition[def_dir]]
            invalid_ports = [(c[port], port_dir, c['dbg_line']) for c in connections if c[target] == actor and c[port] not in ports]
            retval.extend(invalid_ports)

        return retval

    def check_atleast_one_connection(self, ports, connections, actor=None):
        # inports should have at least one connection
        retval = []
        is_component = actor is None
        actor = '.' if is_component else actor
        for port in ports:
            pc = [c for c in connections if c['src'] == actor and c['src_port'] == port]
            if len(pc) < 1:
                retval.append(port)
        return retval

    def check_component_connections(self, definition, connections):
        invalid_ports = self._verify_port_names(definition, connections)
        for port, port_dir, line in invalid_ports:
            fmt = "Component {name} has no {port_dir}port '{port}'"
            self.append_error(fmt, line=line, port=port, port_dir=port_dir, **definition)

        # outports should have exactly one connection
        outport_names = [p for p, _ in definition['outputs']]
        for port in outport_names:
            incoming = [c for c in connections if c['dst'] == "." and c['dst_port'] == port]
            if len(incoming) == 0:
                fmt = "Component {name} is missing connection to outport '{port}'"
                self.append_error(fmt, line=max(self.dbg_lines(connections)), port=port, **definition)
            if len(incoming) > 1:
                fmt = "Component {name} has multiple connections to outport '{port}'"
                for c in incoming:
                    self.append_error(fmt, line=c['dbg_line'], port=port, **definition)

        # inports should have at least one connection
        inport_names = [p for p, _ in definition['inputs']]
        bad_ports = self.check_atleast_one_connection(inport_names, connections)
        for port in bad_ports:
            fmt = "Component {name} is missing connection to inport '{port}'"
            self.append_error(fmt, line=max(self.dbg_lines(connections)), port=port, **definition)

    def check_actor_connections(self, actor, definition, connections):

        invalid_ports = self._verify_port_names(definition, connections, actor)
        for port, port_dir, line in invalid_ports:
            fmt = "Actor {actor} ({ns}.{name}) has no {port_dir}port '{port}'"
            self.append_error(fmt, line=line, port=port, port_dir=port_dir, actor=actor, **definition)

        # inports should have exactly one connection
        inport_names = [p for p, _ in definition['inputs']]
        for port in inport_names:
            incoming = [c for c in connections if c['dst'] == actor and c['dst_port'] == port]
            if len(incoming) == 0:
                fmt = "Actor {actor} ({ns}.{name}) is missing connection to inport '{port}'"
                self.append_error(fmt, line=max(self.dbg_lines(connections)), port=port, actor=actor, **definition)
            if len(incoming) > 1:
                fmt = "Actor {actor} ({ns}.{name}) has multiple connections to inport '{port}'"
                for c in incoming:
                    self.append_error(fmt, line=c['dbg_line'], port=port, actor=actor, **definition)
        # outports should have at least one connection
        outport_names = [p for p, _ in definition['outputs']]
        bad_ports = self.check_atleast_one_connection(outport_names, connections, actor)
        for port in bad_ports:
            fmt = "Actor {actor} ({ns}.{name}) is missing connection to outport '{port}'"
            self.append_error(fmt, line=max(self.dbg_lines(connections)), port=port, actor=actor, **definition)

    def check_connections(self, actor, definition, connections):
        if actor:
            self.check_actor_connections(actor, definition, connections)
        else:
            self.check_component_connections(definition, connections)

    def expand_arguments(self, declaration):
        """
        Check the the arguments for constants that must be expanded
        Append errors if definition is missing.
        """
        for argname, (kind, value) in declaration['args'].iteritems():
            if kind != "IDENTIFIER":
                continue
            try:
                self.lookup_constant(value)
            except:
                fmt = "Undefined identifier: '{param}'"
                self.append_error(fmt, line=declaration['dbg_line'], param=value)

    def check_arguments(self, definition, declaration):
        mandatory = set(definition['args']['mandatory'])
        defined = set(declaration['args'].keys())
        missing = mandatory - defined
        # Case 1: Missing parameters
        for m in missing:
            fmt = "Missing argument: '{param}'"
            self.append_error(fmt, line=declaration['dbg_line'], param=m)
        # FIXME: Case 2: Unused parameter
        # FIXME: Case 3: value for arg is IDENTIFIER rather than VALUE, and not defined in constants
        # self.expand_arguments(declaration)

    def check_structure(self, structure):
        actors = structure['actors'].keys()

        # Look for undefined actors
        src_actors = {c['src'] for c in structure['connections'] if c['src'] != "."}
        dst_actors = {c['dst'] for c in structure['connections'] if c['dst'] != "."}
        # Implicit src actors are defined by a constant on the inport
        implicit_src_actors = {c['src'] for c in structure['connections'] if c['src'] is None}
        all_actors = src_actors | dst_actors
        undefined_actors = all_actors - (set(actors) | implicit_src_actors)
        for actor in undefined_actors:
            fmt = "Undefined actor: '{actor}'"
            lines = [c['dbg_line'] for c in structure['connections'] if c['src'] == actor or c['dst'] == actor]
            for line in lines:
                self.append_error(fmt, line=line, actor=actor)

        # Note: Unused actors will be caught when checking connections

        # Check if actor exists
        for actor in actors:
            actor_type = structure['actors'][actor]['actor_type']
            definition = self.get_definition(actor_type)
            if not definition:
                fmt = "Unknown actor type: '{type}'"
                self.append_error(fmt, type=actor_type, line=structure['actors'][actor]['dbg_line'])
                continue
            # We have actor definition, check that it is fully connected
            self.check_connections(actor, definition, structure['connections'])
            self.check_arguments(definition, structure['actors'][actor])

def check(cs_info):
    clint = Checker(cs_info)
    return clint.errors, clint.warnings

if __name__ == '__main__':
    import sys
    import os
    import json

    if len(sys.argv) < 2:
        script = 'inline'
        source_text = \
"""# Test script
        component Count(len) -> seq {
            src : std.Constant(data="hup", n=len)
            src.token > .seq
        }

        src: Count(len=5)
        snk : io.StandardOut()
        src.seq > snk.token
"""
    else:
        script = sys.argv[1]
        script = os.path.expanduser(script)
        try:
            with open(script, 'r') as source:
                source_text = source.read()
        except:
            print "Error: Could not read file: '%s'" % script
            sys.exit(1)

    result, errors, warnings = calvin_parser(source_text, script)
    if errors:
        print "{reason} {script} [{line}:{col}]".format(script=script, **error)
    else:
        errors, warnings = check(result)
        print "errors:", [x['reason'] for x in errors]
        print "warnings:", [x['reason'] for x in warnings]
<|endoftext|>


<commit_before># Copyright 2016-2017 Onestein (<http://www.onestein.eu>)
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

{
    'name': 'Holidays expiration',
    'summary': 'Automatic management of holidays expiration',
    'author': 'Onestein',
    'website': 'http://www.onestein.eu',
    'category': 'Human Resources',
    'version': '11.0.1.0.0',
    'license': 'AGPL-3',
    'depends': [
        'hr_holidays',
        'mail',
    ],
    'data': [
        'data/hr_holidays_data.xml',
        'views/hr_holidays.xml',
        'views/res_company.xml',
        'data/hr_holidays_cron.xml',
    ],
}
<commit_msg>Update __manifest__.py<commit_after># Copyright 2016-2018 Onestein (<http://www.onestein.eu>)
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

{
    'name': 'Holidays expiration',
    'summary': 'Automatic management of holidays expiration. Leave days can have a expiary date per leave type. E-mail warnings can be set.',
    'images': [static/description/banner.png],
    'author': 'Onestein',
    'website': 'http://www.onestein.eu',
    'category': 'Human Resources',
    'version': '11.0.1.0.0',
    'license': 'AGPL-3',
    'depends': [
        'hr_holidays',
        'mail',
    ],
    'data': [
        'data/hr_holidays_data.xml',
        'views/hr_holidays.xml',
        'views/res_company.xml',
        'data/hr_holidays_cron.xml',
    ],
}
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Copyright (C) 2020 VyOS maintainers and contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 or later as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os

from fnmatch import fnmatch
from sys import exit

from vyos.config import Config
from vyos.configdict import dict_merge
from vyos.configverify import verify_bridge_vrf
from vyos.template import render
from vyos.util import call
from vyos.xml import defaults
from vyos import ConfigError
from vyos import airbag
airbag.enable()

# XXX: workaround for https://phabricator.vyos.net/T2660
default_values = {'backup' : {'distance' : '10'}}

def check_kmod():
    modules = ['option', 'usb_wwan', 'usbserial']
    for module in modules:
        if not os.path.exists(f'/sys/module/{module}'):
            if call(f'modprobe {module}') != 0:
                raise ConfigError(f'Loading Kernel module {module} failed')

def find_device_file(device):
    """ Recurively search /dev for the given device file and return its full path.
        If no device file was found 'None' is returned """
    for root, dirs, files in os.walk('/dev'):
        for basename in files:
            if fnmatch(basename, device):
                return os.path.join(root, basename)

    return None

def get_config():
    """ Retrive CLI config as dictionary. Dictionary can never be empty,
    as at least the interface name will be added or a deleted flag """
    conf = Config()

    # determine tagNode instance
    if 'VYOS_TAGNODE_VALUE' not in os.environ:
        raise ConfigError('Interface (VYOS_TAGNODE_VALUE) not specified')

    # retrieve interface default values
    base = ['interfaces', 'wirelessmodem']
    # XXX: workaround for https://phabricator.vyos.net/T2660
    #default_values = defaults(base)

    ifname = os.environ['VYOS_TAGNODE_VALUE']
    base = base + [ifname]

    wwan = conf.get_config_dict(base, key_mangling=('-', '_'))
    # Check if interface has been removed
    if wwan == {}:
        wwan.update({'deleted' : ''})

    # We have gathered the dict representation of the CLI, but there are
    # default options which we need to update into the dictionary
    # retrived.
    wwan = dict_merge(default_values, wwan)

    # Add interface instance name into dictionary
    wwan.update({'ifname': ifname})

    return wwan

def verify(wwan):
    if 'deleted' in wwan.keys():
        return None

    if not 'apn' in wwan.keys():
        raise ConfigError('No APN configured for "{ifname}"'.format(**wwan))

    if not 'device' in wwan.keys():
        raise ConfigError('Physical "device" must be configured')

    # we can not use isfile() here as Linux device files are no regular files
    # thus the check will return False
    if not os.path.exists(find_device_file(wwan['device'])):
        raise ConfigError('Device "{device}" does not exist'.format(**wwan))

    verify_bridge_vrf(wwan)

    return None

def generate(wwan):
    # set up configuration file path variables where our templates will be
    # rendered into
    ifname = wwan['ifname']
    config_wwan = f'/etc/ppp/peers/{ifname}'
    config_wwan_chat = f'/etc/ppp/peers/chat.{ifname}'
    script_wwan_pre_up = f'/etc/ppp/ip-pre-up.d/1010-vyos-wwan-{ifname}'
    script_wwan_ip_up = f'/etc/ppp/ip-up.d/1010-vyos-wwan-{ifname}'
    script_wwan_ip_down = f'/etc/ppp/ip-down.d/1010-vyos-wwan-{ifname}'

    config_files = [config_wwan, config_wwan_chat, script_wwan_pre_up,
                    script_wwan_ip_up, script_wwan_ip_down]

    # Always hang-up WWAN connection prior generating new configuration file
    call(f'systemctl stop ppp@{ifname}.service')

    if 'deleted' in wwan:
        # Delete PPP configuration files
        for file in config_files:
            if os.path.exists(file):
                os.unlink(file)

    else:
        wwan['device'] = find_device_file(wwan['device'])

        # Create PPP configuration files
        render(config_wwan, 'wwan/peer.tmpl', wwan)
        # Create PPP chat script
        render(config_wwan_chat, 'wwan/chat.tmpl', wwan)

        # generated script file must be executable

        # Create script for ip-pre-up.d
        render(script_wwan_pre_up, 'wwan/ip-pre-up.script.tmpl',
               wwan, permission=0o755)
        # Create script for ip-up.d
        render(script_wwan_ip_up, 'wwan/ip-up.script.tmpl',
               wwan, permission=0o755)
        # Create script for ip-down.d
        render(script_wwan_ip_down, 'wwan/ip-down.script.tmpl',
               wwan, permission=0o755)

    return None

def apply(wwan):
    if 'deleted' in wwan.keys():
        # bail out early
        return None

    if not 'disable' in wwan.keys():
        # "dial" WWAN connection
        call('systemctl start ppp@{ifname}.service'.format(**wwan))

    return None

if __name__ == '__main__':
    try:
        check_kmod()
        c = get_config()
        verify(c)
        generate(c)
        apply(c)
    except ConfigError as e:
        print(e)
        exit(1)
<commit_msg>wwan: T2660: remove workaround - use XML backed default dict<commit_after>#!/usr/bin/env python3
#
# Copyright (C) 2020 VyOS maintainers and contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 or later as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os

from fnmatch import fnmatch
from sys import exit

from vyos.config import Config
from vyos.configdict import dict_merge
from vyos.configverify import verify_bridge_vrf
from vyos.template import render
from vyos.util import call
from vyos.xml import defaults
from vyos import ConfigError
from vyos import airbag
airbag.enable()

def check_kmod():
    modules = ['option', 'usb_wwan', 'usbserial']
    for module in modules:
        if not os.path.exists(f'/sys/module/{module}'):
            if call(f'modprobe {module}') != 0:
                raise ConfigError(f'Loading Kernel module {module} failed')

def find_device_file(device):
    """ Recurively search /dev for the given device file and return its full path.
        If no device file was found 'None' is returned """
    for root, dirs, files in os.walk('/dev'):
        for basename in files:
            if fnmatch(basename, device):
                return os.path.join(root, basename)

    return None

def get_config():
    """ Retrive CLI config as dictionary. Dictionary can never be empty,
    as at least the interface name will be added or a deleted flag """
    conf = Config()

    # determine tagNode instance
    if 'VYOS_TAGNODE_VALUE' not in os.environ:
        raise ConfigError('Interface (VYOS_TAGNODE_VALUE) not specified')

    # retrieve interface default values
    base = ['interfaces', 'wirelessmodem']
    default_values = defaults(base)

    ifname = os.environ['VYOS_TAGNODE_VALUE']
    base = base + [ifname]

    wwan = conf.get_config_dict(base, key_mangling=('-', '_'))
    # Check if interface has been removed
    if wwan == {}:
        wwan.update({'deleted' : ''})

    # We have gathered the dict representation of the CLI, but there are
    # default options which we need to update into the dictionary
    # retrived.
    wwan = dict_merge(default_values, wwan)

    # Add interface instance name into dictionary
    wwan.update({'ifname': ifname})

    return wwan

def verify(wwan):
    if 'deleted' in wwan.keys():
        return None

    if not 'apn' in wwan.keys():
        raise ConfigError('No APN configured for "{ifname}"'.format(**wwan))

    if not 'device' in wwan.keys():
        raise ConfigError('Physical "device" must be configured')

    # we can not use isfile() here as Linux device files are no regular files
    # thus the check will return False
    if not os.path.exists(find_device_file(wwan['device'])):
        raise ConfigError('Device "{device}" does not exist'.format(**wwan))

    verify_bridge_vrf(wwan)

    return None

def generate(wwan):
    # set up configuration file path variables where our templates will be
    # rendered into
    ifname = wwan['ifname']
    config_wwan = f'/etc/ppp/peers/{ifname}'
    config_wwan_chat = f'/etc/ppp/peers/chat.{ifname}'
    script_wwan_pre_up = f'/etc/ppp/ip-pre-up.d/1010-vyos-wwan-{ifname}'
    script_wwan_ip_up = f'/etc/ppp/ip-up.d/1010-vyos-wwan-{ifname}'
    script_wwan_ip_down = f'/etc/ppp/ip-down.d/1010-vyos-wwan-{ifname}'

    config_files = [config_wwan, config_wwan_chat, script_wwan_pre_up,
                    script_wwan_ip_up, script_wwan_ip_down]

    # Always hang-up WWAN connection prior generating new configuration file
    call(f'systemctl stop ppp@{ifname}.service')

    if 'deleted' in wwan:
        # Delete PPP configuration files
        for file in config_files:
            if os.path.exists(file):
                os.unlink(file)

    else:
        wwan['device'] = find_device_file(wwan['device'])

        # Create PPP configuration files
        render(config_wwan, 'wwan/peer.tmpl', wwan)
        # Create PPP chat script
        render(config_wwan_chat, 'wwan/chat.tmpl', wwan)

        # generated script file must be executable

        # Create script for ip-pre-up.d
        render(script_wwan_pre_up, 'wwan/ip-pre-up.script.tmpl',
               wwan, permission=0o755)
        # Create script for ip-up.d
        render(script_wwan_ip_up, 'wwan/ip-up.script.tmpl',
               wwan, permission=0o755)
        # Create script for ip-down.d
        render(script_wwan_ip_down, 'wwan/ip-down.script.tmpl',
               wwan, permission=0o755)

    return None

def apply(wwan):
    if 'deleted' in wwan.keys():
        # bail out early
        return None

    if not 'disable' in wwan.keys():
        # "dial" WWAN connection
        call('systemctl start ppp@{ifname}.service'.format(**wwan))

    return None

if __name__ == '__main__':
    try:
        check_kmod()
        c = get_config()
        verify(c)
        generate(c)
        apply(c)
    except ConfigError as e:
        print(e)
        exit(1)
<|endoftext|>


<commit_before>import os
import sys
from urllib2 import HTTPError, urlopen
from json import load, loads, dump
from codecs import getreader
import requests
from requests.packages.urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
import gippy
import boto3
from jinja2 import Environment, FileSystemLoader, select_autoescape

if sys.version_info < (3, 0):
    from HTMLParser import HTMLParser
else:
    from html.parser import HTMLParser

PROVIDER = os.getenv('PROVIDER', 'LPDAAC_ECS')
PRODUCT = os.getenv('PRODUCT', 'MCD43A4.006')
PAGE_SIZE = int(os.getenv('PAGE_SIZE', '1'))
AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')

# for jinja2 templating
jinja_env = Environment(
    loader=FileSystemLoader(['templates', 'lib/templates']),
    autoescape=select_autoescape(['html', 'xml'])
)
template = jinja_env.get_template('index.html')

def query_cmr(start_date, end_date):
    """ Search CMR database for spectral MODIS tiles matching a temporal range,
    defined by a start date and end date. Returns metadata containing the URL of
    each image.
    """

    prod, ver = PRODUCT.split('.')
    query = 'https://cmr.earthdata.nasa.gov/search/granules.json?' + \
            'provider={0}&short_name={1}&version={2}'.format(
             PROVIDER, prod, ver) + \
            '&online_only=true&page_size={0}&sort_key=start_date'.format(
             PAGE_SIZE) + \
            '&temporal={0}T00:00:00Z,{1}T23:59:00Z'.format(
             start_date, end_date)
    try:
        response = urlopen(query)
    except HTTPError as error:
        raise ValueError('The CMR server returned the following error:',
                         error.read())

    json_data = load(getreader("utf-8")(response))
    tile_meta = []
    for entry in json_data['feed']['entry']:
        date = entry['time_start'].split('T')[0]
        entry_meta = {'date': date}
        for url_entry in entry['links']:
            if url_entry['type'] == 'application/x-hdfeos':
                entry_meta['url'] = url_entry['href']
            if url_entry['type'] == 'image/jpeg':
                entry_meta['thumb'] = url_entry['href']

        tile_meta.append(entry_meta)
    response.close()

    return tile_meta

def set_retries(retries):
    s = requests.Session()
    r = Retry(total=retries, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504])
    s.mount('http://', HTTPAdapter(max_retries=r))
    s.mount('https://', HTTPAdapter(max_retries=r))
    return s

class LinkFinder(HTMLParser):
    def __init__(self):
        self.reset()
        self.download_link = None

    def handle_starttag(self, tag, attrs):
        if attrs and attrs[0][0] == 'href':
            self.download_link = attrs[0][1]

def get_stream(session, url, auth):
    """ Traverse redirects to get the final url """
    stream = session.get(url, auth=auth, allow_redirects=False, stream=True)
    if stream.status_code == 302:
        link = LinkFinder()
        link.feed(stream.text)
        print('trying new link ' + link.download_link)
        get_stream(session, link.download_link, auth)
    elif stream.status_code == 200:
        print(stream.text)
        print('actually downloading')
        return stream
    else:
        raise Exception("Earthdata Authentication Error")

def download(url, path='./'):
    """ Get URL and save with some name """
    fout = os.path.join(path, os.path.basename(url))
    auth = (os.environ.get('EARTHDATA_USER'), os.environ.get('EARTHDATA_PASS'))
    # download as stream
    session = set_retries(5)
    stream = get_stream(session, url, auth)
    chunk_size = 1024
    # try:
    with open(fout, 'wb') as f:
        for chunk in stream.iter_content(chunk_size):
            f.write(chunk)
    # except:
    #     raise Exception("Problem downloading %s" % stream)

    return fout

def convert_to_geotiff(hdf, path='./'):
    file_names = []
    img = gippy.GeoImage(hdf, True)
    # write out metadata
    metadata_fname = 'metadata.json'
    with open(metadata_fname, 'w') as outfile:
        print('Writing metadata')
        dump(img.meta(), outfile, sort_keys=True, indent=4, ensure_ascii=False)
        file_names.append(metadata_fname)
    # save each band as a TIF
    for i, band in enumerate(img):
        fname = hdf.replace('.hdf', '') + '_B' + str(i).zfill(2) + '.TIF'
        print('Writing ' + fname)
        a = gippy.GeoImage.create_from(img, fname, nb=1)
        a.add_band(img[i])
        a.save(fname)
        file_names.append(fname)

    return file_names

def get_product_name(filename):
    return filename.split('.')[0] + '.' + filename.split('.')[3]

def get_tile_id(filename):
    return filename.split('.')[2].replace('h', '').replace('v', '/')

def get_date(filename):
    return filename.split('.')[1].replace('A', '')

def get_s3_folder(filename):
    return '%s/%s/%s' % (get_product_name(filename), get_tile_id(filename),
                         get_date(filename))

def push_to_s3(filename, bucket, folder):
    """ Copy file to S3 """
    s3 = boto3.client(
        's3',
        aws_access_key_id=AWS_ACCESS_KEY_ID,
        aws_secret_access_key=AWS_SECRET_ACCESS_KEY
    )
    key = '%s/%s' % (folder, os.path.basename(filename))
    with open(filename, 'rb') as f:
        resp = s3.put_object(Bucket=bucket, Key=key, Body=f, ACL='public-read')
    return 's3://%s/%s' % (bucket, key)

def make_index(thumb, product, files):
    print template.render(thumb=thumb, product=product, files=files)
<commit_msg>Make string updates, update html rendering<commit_after>import os
import sys
from urllib2 import HTTPError, urlopen
from json import load, loads, dump
from codecs import getreader
import requests
from requests.packages.urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
import gippy
import boto3
from jinja2 import Environment, FileSystemLoader, select_autoescape

if sys.version_info < (3, 0):
    from HTMLParser import HTMLParser
else:
    from html.parser import HTMLParser

PROVIDER = os.getenv('PROVIDER', 'LPDAAC_ECS')
PRODUCT = os.getenv('PRODUCT', 'MCD43A4.006')
PAGE_SIZE = int(os.getenv('PAGE_SIZE', '1'))
AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')

# for jinja2 templating
jinja_env = Environment(
    loader=FileSystemLoader(['templates', 'lib/templates']),
    autoescape=select_autoescape(['html', 'xml'])
)
template = jinja_env.get_template('index.html')

def query_cmr(start_date, end_date):
    """ Search CMR database for spectral MODIS tiles matching a temporal range,
    defined by a start date and end date. Returns metadata containing the URL of
    each image.
    """

    prod, ver = PRODUCT.split('.')
    query = 'https://cmr.earthdata.nasa.gov/search/granules.json?' + \
            'provider={0}&short_name={1}&version={2}'.format(
             PROVIDER, prod, ver) + \
            '&online_only=true&page_size={0}&sort_key=start_date'.format(
             PAGE_SIZE) + \
            '&temporal={0}T00:00:00Z,{1}T23:59:00Z'.format(
             start_date, end_date)
    print('searching ' + query)
    try:
        response = urlopen(query)
    except HTTPError as error:
        raise ValueError('The CMR server returned the following error:',
                         error.read())

    json_data = load(getreader("utf-8")(response))
    tile_meta = []
    for entry in json_data['feed']['entry']:
        date = entry['time_start'].split('T')[0]
        entry_meta = {'date': date}
        for url_entry in entry['links']:
            if url_entry.get('type') == 'application/x-hdfeos':
                entry_meta['url'] = url_entry['href']
            if url_entry.get('type') == 'image/jpeg':
                entry_meta['thumb'] = url_entry['href']

        tile_meta.append(entry_meta)
    response.close()

    return tile_meta

def set_retries(retries):
    s = requests.Session()
    r = Retry(total=retries, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504])
    s.mount('http://', HTTPAdapter(max_retries=r))
    s.mount('https://', HTTPAdapter(max_retries=r))
    return s

class LinkFinder(HTMLParser):
    def __init__(self):
        self.reset()
        self.download_link = None

    def handle_starttag(self, tag, attrs):
        if attrs and attrs[0][0] == 'href':
            self.download_link = attrs[0][1]

def get_stream(session, url, auth):
    """ Traverse redirects to get the final url """
    stream = session.get(url, auth=auth, allow_redirects=False, stream=True)
    if stream.status_code == 302:
        link = LinkFinder()
        link.feed(stream.text)
        print('trying new link ' + link.download_link)
        get_stream(session, link.download_link, auth)
    elif stream.status_code == 200:
        print(stream.text)
        print('actually downloading')
        return stream
    else:
        raise Exception("Earthdata Authentication Error")

def download(url, path='./'):
    """ Get URL and save with some name """
    fout = os.path.join(path, os.path.basename(url))
    auth = (os.environ.get('EARTHDATA_USER'), os.environ.get('EARTHDATA_PASS'))
    # download as stream
    session = set_retries(5)
    stream = get_stream(session, url, auth)
    chunk_size = 1024
    # try:
    with open(fout, 'wb') as f:
        for chunk in stream.iter_content(chunk_size):
            f.write(chunk)
    # except:
    #     raise Exception("Problem downloading %s" % stream)

    return fout

def convert_to_geotiff(hdf, path='./'):
    file_names = []
    img = gippy.GeoImage(hdf, True)
    # write out metadata
    metadata_fname = 'metadata.json'
    with open(metadata_fname, 'w') as outfile:
        print('Writing metadata')
        dump(img.meta(), outfile, sort_keys=True, indent=4, ensure_ascii=False)
        file_names.append(metadata_fname)
    # save each band as a TIF
    for i, band in enumerate(img):
        fname = hdf.replace('.hdf', '') + '_B' + str(i).zfill(2) + '.TIF'
        print('Writing %s' % fname)
        a = gippy.GeoImage.create_from(img, fname, nb=1)
        a.add_band(img[i])
        a.save(fname)
        file_names.append(fname)

    return file_names

def get_product_name(filename):
    return filename.split('.')[0] + '.' + filename.split('.')[3]

def get_tile_id(filename):
    return filename.split('.')[2].replace('h', '').replace('v', '/')

def get_date(filename):
    return filename.split('.')[1].replace('A', '')

def get_s3_folder(filename):
    return '%s/%s/%s' % (get_product_name(filename), get_tile_id(filename),
                         get_date(filename))

def push_to_s3(filename, bucket, folder):
    """ Copy file to S3 """
    s3 = boto3.client(
        's3',
        aws_access_key_id=AWS_ACCESS_KEY_ID,
        aws_secret_access_key=AWS_SECRET_ACCESS_KEY
    )
    key = '%s/%s' % (folder, os.path.basename(filename))
    with open(filename, 'rb') as f:
        print('Uploading %s to: %s' % (key, bucket))
        resp = s3.put_object(Bucket=bucket, Key=key, Body=f, ACL='public-read')
    return 's3://%s/%s' % (bucket, key)

def make_index(thumb, product, files):
    html = template.render(thumb=thumb, product=product, files=files)
    index_fname = 'index.html'
    with open(index_fname, 'w') as outfile:
        print('Writing %s' % index_fname)
        outfile.write(html)

    return index_fname
<|endoftext|>