<commit_before>from django.http import HttpResponseRedirect, HttpResponse, Http404
from django.template import RequestContext
from django.contrib.auth.decorators import login_required
from django.shortcuts import render_to_response, get_object_or_404
from django.contrib.auth.models import User
from django.utils import simplejson
from django.contrib import messages
from django.utils.translation import ugettext, ugettext_lazy as _
from django.core.urlresolvers import reverse

from threadedcomments.models import ThreadedComment
from threadedcomments.forms import RichCommentForm

from feedback.models import Feedback
from feedback.forms import FeedbackForm, WidgetForm
from papers.models import Paper

def list(request, list="open", type="all", status="all", mine=False, template_name="feedback/list.html"):
    feedbacks = Feedback.objects.all().order_by('-created')

    if mine:
      feedbacks = feedbacks.filter(user=request.user)
      list_title = "My Feedback"

    if list == "all":
        list_title = "All Feedback"
    elif list == "open":
        list_title = "Open Feedback"
        feedbacks = feedbacks.filter(status__status='open')
    elif list == "closed":
        list_title = "Closed Feedback"
        feedbacks = feedbacks.filter(status__status='closed')
    elif list == "mine":
        list_title = "My Feedback"
        feedbacks = feedbacks.filter(user=request.user)

    if type != "all":
        feedbacks = feedbacks.filter(type__slug=type)

    if status != "all":
        feedbacks = feedbacks.filter(status__slug=status)

    if not request.user.is_staff:
        feedbacks = feedbacks.filter(private=False)

    return render_to_response(template_name, {
        'feedbacks': feedbacks, 'list': list, 'status': status, 'type': type,
        'list_title': list_title,
        }, context_instance=RequestContext(request))

def detail(request, object_id, template_name="feedback/detail.html"):
    feedback = get_object_or_404(Feedback, pk=object_id)

    if feedback.private == True:
        if not request.user.is_staff and request.user != feedback.user:
            raise Http404

    comment_form = RichCommentForm(auto_id='new_comment_form_%s')

    return render_to_response(template_name, {
        'feedback': feedback,
        'comment_form': comment_form,
        }, context_instance=RequestContext(request))

def submit(request, form_class=FeedbackForm, template_name="feedback/submit.html"):
    form = form_class(request.POST or None, auto_id='create_feedback_%s')

    if form.is_valid():
        feedback = form.save(commit=False)
        if request.user.is_authenticated:
            feedback.user = request.user
        feedback.save()

        feedback.register_action(request.user, 'create-feedback', feedback)

        return HttpResponseRedirect(feedback.get_absolute_url())

    return render_to_response(template_name, {
        "form": form,
    }, context_instance=RequestContext(request))

@login_required
def edit(request, object_id=None, form_class=FeedbackForm, template_name="feedback/edit.html"):
    feedback = Feedback.objects.get(id=object_id)

    if feedback.user != request.user and not request.user.is_staff:
        return render_to_response('feedback/forbidden.html', {}, context_instance=RequestContext(request))

    form = form_class(request.POST or None, instance=feedback, auto_id='edit_feedback_%s')

    if form.is_valid():
        feedback = form.save()

        feedback.register_action(request.user, 'edit', feedback)

        return HttpResponseRedirect(feedback.get_absolute_url())

    return render_to_response(template_name, {
        'feedback': feedback,
        'form': form,
    }, context_instance=RequestContext(request))

@login_required
def delete(request, object_id=None, template_name="feedback/delete.html"):
    feedback = Feedback.objects.get(id=object_id)
    redirect_url = feedback.get_absolute_url()

    if request.user == feedback.user:
        if not Paper.objects.get_for_object(feedback).exists():
            if not ThreadedComment.objects.get_for_object(feedback).exists():
                feedback.feed.delete()
                feedback.delete()
                messages.add_message(request, messages.SUCCESS,
                    _("Feedback %s deleted.") % feedback.title
                )
                redirect_url = reverse("feedback_list")
            else:
                messages.add_message(request, messages.ERROR,
                    _("Please delete comments before deleting the feedback.")
                )
        else:
            messages.add_message(request, messages.SUCCESS,
                _("Please delete papers before deleting the feedback.")
            )
    else:
        messages.add_message(request, messages.SUCCESS,
            _("You are not the creator of this feedback.")
        )

    return HttpResponseRedirect(redirect_url)

def widget(request, form_class=WidgetForm, template_name="feedback/widget.html"):
    form = form_class(request.POST or None)

    if form.is_valid():
        feedback = form.save(commit=False)
        if not form.cleaned_data['anonymous']:
            feedback.creator = u
        feedback.save()

        feedback.register_action(request.user, 'create', feedback)

        data = simplejson.dumps({'url':feedback.get_absolute_url(), 'errors': False})
    else:
        data = simplejson.dumps({'errors': True})

    return HttpResponse(data, mimetype='application/json')

def leave_feedback(request):
    if request.method == 'POST':
        form = FeedbackForm(request.POST)
        if form.is_valid():
            feedback = form.save()

            if feedback.user:
                request.user.message_set.create(message="Your feedback has been saved successfully.")
            return HttpResponseRedirect(request.POST.get('next', request.META.get('HTTP_REFERER', '/')))
    else:
        form = FeedbackForm(request.GET)
    return render_to_response('feedback/feedback_form.html', {'form': form}, context_instance=RequestContext(request))
<commit_msg>follow on feedback creation<commit_after>from django.http import HttpResponseRedirect, HttpResponse, Http404
from django.template import RequestContext
from django.contrib.auth.decorators import login_required
from django.shortcuts import render_to_response, get_object_or_404
from django.contrib.auth.models import User
from django.utils import simplejson
from django.contrib import messages
from django.utils.translation import ugettext, ugettext_lazy as _
from django.core.urlresolvers import reverse

from threadedcomments.models import ThreadedComment
from threadedcomments.forms import RichCommentForm

from feedback.models import Feedback
from feedback.forms import FeedbackForm, WidgetForm
from papers.models import Paper

def list(request, list="open", type="all", status="all", mine=False, template_name="feedback/list.html"):
    feedbacks = Feedback.objects.all().order_by('-created')

    if mine:
      feedbacks = feedbacks.filter(user=request.user)
      list_title = "My Feedback"

    if list == "all":
        list_title = "All Feedback"
    elif list == "open":
        list_title = "Open Feedback"
        feedbacks = feedbacks.filter(status__status='open')
    elif list == "closed":
        list_title = "Closed Feedback"
        feedbacks = feedbacks.filter(status__status='closed')
    elif list == "mine":
        list_title = "My Feedback"
        feedbacks = feedbacks.filter(user=request.user)

    if type != "all":
        feedbacks = feedbacks.filter(type__slug=type)

    if status != "all":
        feedbacks = feedbacks.filter(status__slug=status)

    if not request.user.is_staff:
        feedbacks = feedbacks.filter(private=False)

    return render_to_response(template_name, {
        'feedbacks': feedbacks, 'list': list, 'status': status, 'type': type,
        'list_title': list_title,
        }, context_instance=RequestContext(request))

def detail(request, object_id, template_name="feedback/detail.html"):
    feedback = get_object_or_404(Feedback, pk=object_id)

    if feedback.private == True:
        if not request.user.is_staff and request.user != feedback.user:
            raise Http404

    comment_form = RichCommentForm(auto_id='new_comment_form_%s')

    return render_to_response(template_name, {
        'feedback': feedback,
        'comment_form': comment_form,
        }, context_instance=RequestContext(request))

def submit(request, form_class=FeedbackForm, template_name="feedback/submit.html"):
    form = form_class(request.POST or None, auto_id='create_feedback_%s')

    if form.is_valid():
        feedback = form.save(commit=False)
        if request.user.is_authenticated:
            feedback.user = request.user
        feedback.save()

        feedback.register_action(request.user, 'create-feedback', feedback)

        feedback.feed.subscribe(request.user)

        return HttpResponseRedirect(feedback.get_absolute_url())

    return render_to_response(template_name, {
        "form": form,
    }, context_instance=RequestContext(request))

@login_required
def edit(request, object_id=None, form_class=FeedbackForm, template_name="feedback/edit.html"):
    feedback = Feedback.objects.get(id=object_id)

    if feedback.user != request.user and not request.user.is_staff:
        return render_to_response('feedback/forbidden.html', {}, context_instance=RequestContext(request))

    form = form_class(request.POST or None, instance=feedback, auto_id='edit_feedback_%s')

    if form.is_valid():
        feedback = form.save()

        feedback.register_action(request.user, 'edit', feedback)

        return HttpResponseRedirect(feedback.get_absolute_url())

    return render_to_response(template_name, {
        'feedback': feedback,
        'form': form,
    }, context_instance=RequestContext(request))

@login_required
def delete(request, object_id=None, template_name="feedback/delete.html"):
    feedback = Feedback.objects.get(id=object_id)
    redirect_url = feedback.get_absolute_url()

    if request.user == feedback.user:
        if not Paper.objects.get_for_object(feedback).exists():
            if not ThreadedComment.objects.get_for_object(feedback).exists():
                feedback.feed.delete()
                feedback.delete()
                messages.add_message(request, messages.SUCCESS,
                    _("Feedback %s deleted.") % feedback.title
                )
                redirect_url = reverse("feedback_list")
            else:
                messages.add_message(request, messages.ERROR,
                    _("Please delete comments before deleting the feedback.")
                )
        else:
            messages.add_message(request, messages.SUCCESS,
                _("Please delete papers before deleting the feedback.")
            )
    else:
        messages.add_message(request, messages.SUCCESS,
            _("You are not the creator of this feedback.")
        )

    return HttpResponseRedirect(redirect_url)

def widget(request, form_class=WidgetForm, template_name="feedback/widget.html"):
    form = form_class(request.POST or None)

    if form.is_valid():
        feedback = form.save(commit=False)
        if not form.cleaned_data['anonymous']:
            feedback.creator = u
        feedback.save()

        feedback.register_action(request.user, 'create', feedback)

        data = simplejson.dumps({'url':feedback.get_absolute_url(), 'errors': False})
    else:
        data = simplejson.dumps({'errors': True})

    return HttpResponse(data, mimetype='application/json')

def leave_feedback(request):
    if request.method == 'POST':
        form = FeedbackForm(request.POST)
        if form.is_valid():
            feedback = form.save()

            if feedback.user:
                request.user.message_set.create(message="Your feedback has been saved successfully.")
            return HttpResponseRedirect(request.POST.get('next', request.META.get('HTTP_REFERER', '/')))
    else:
        form = FeedbackForm(request.GET)
    return render_to_response('feedback/feedback_form.html', {'form': form}, context_instance=RequestContext(request))
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import subprocess
import json
import time

def getcontainerinfo(containeruids):
    cdetails = list ()
    for containers in containeruids:
        mycommand = "docker inspect %s" % containers
        containerproc = subprocess.Popen([mycommand], stdout=subprocess.PIPE, shell=True)
        cdetails.append(json.loads(containerproc.stdout.read())[0])
    return cdetails

def getsummary(containerinfo):
    #chostname = containerinfo['Config']['Hostname']
    cuid = containerinfo['Id']
    #ccmd = containerinfo['HostConfig']['Cmd']
    cimage = containerinfo['Config']['Image']
    if containerinfo['State']['Running'] == 1:
        crun = "Running"
    else:
        crun = "Not Running"
    return cuid[:8], cimage, crun

def isInt(mystr):
    try:
        int(mystr)
        return True
    except ValueError:
        return False

def str2list(inlist):
    """Converts input string into valid list

    Parses spaces, commas and range of values('-')"""
    delim = " "
    if ',' in inlist:
        delim = ","
    inlist = delim.join(inlist.split(delim))
    containerlist = inlist.split(delim)
    rangelist = [containerlist.pop(r[0]) for r in enumerate(containerlist) if '-' in containerlist[r[0]]]
    for rl in rangelist:
        start, end = rl.split('-')
        containerlist.extend(range(int(start), int(end)+1))
    return containerlist

def terminal2(cpid):
    nsenter = ('sudo nsenter -m -u -n -i -p -t {0} /bin/bash'.format(cpid))
    import pty
    import os
    if os.getenv('DISPLAY',"") == "":
        pty.spawn(nsenter.split())
    else:
        mycommand = "xterm -T {0} -e {1}".format(cpid, nsenter)
        containerproc = subprocess.Popen([mycommand], stdout=subprocess.PIPE, shell=True)

def getpid(containarray, mynum):
    return containarray[int(mynum)]['State']['Pid']

def isRunning(containarray, mynum):
    if (containarray[int(mynum)]['State']['Running']):
        return True
    else:
        return False

def returnuid(containarray, mynum):
    myuid = containarray[int(mynum)]['Id']
    return myuid[:8]

def containerinrange(cdetails, stopcontainers):
    for i in stopcontainers:
        foo = int(len(cdetails))
        foo = foo -1
        if (0 <= int(i) <= foo) == False:
            print " "
            print ("{0} isn't a valid container number...".format(i))
            time.sleep(2)
            return False
    return True

def getcontainer(cdetails):
    """returns valid list of container IDs"""
    print " "
    stopcontainers = raw_input("Which Container(s)?: ")
    stopcontainers = str2list(stopcontainers)
    if containerinrange(cdetails, stopcontainers) == False:
        return False
    for container in stopcontainers:
        # FIXME: check if it's valid integer
        if not(isInt(container)):
            print " "
            print ("{0} isn't a integer...".format(container))
            time.sleep(2)
            return False
        else:
            return stopcontainers

def printsummary():
    print " "
    proc = subprocess.Popen(["docker ps -qa"], stdout=subprocess.PIPE, shell=True)
    out = proc.stdout.read()
    containeruids = out.split()
    cdetails = getcontainerinfo(containeruids)
    print ('{0:2} {1:12} {2:15} {3:8}'.format("#", "ID","Image","Status"))
    for s in range(len(cdetails)):
        chostname, cimage, crun = getsummary(cdetails[s])
        print ('{0:2} {1:12} {2:15} {3:8}'.format(s, chostname, cimage, crun))

    print " "
    print "Command Reference: (q)uit (r)efresh (s)top (d)elete (p)eek"
    print " "
    containernum = raw_input("Command: ")
    if containernum.upper() == "R":
        printsummary()
    if containernum.upper() == "Q":
        quit()
    if containernum.upper() == "S":
        stopcontainer = getcontainer(cdetails)
        for container in stopcontainer:
            try:
                cid = returnuid(cdetails, container)
                print "Stopping %s" % cid
                myval = ("docker stop {0}".format(cid))
                stopprocess = subprocess.Popen([myval], stdout=subprocess.PIPE, shell=True)

            except:
                print "Unable to find that container..."
        printsummary()

    if containernum.upper() == "D":
       delcontainer = getcontainer(cdetails)
       for container in delcontainer:
           try:
               cid = returnuid(cdetails, container)
               print "Deleting %s" % cid
               myval = ("docker rm {0}".format(cid))
               delprocess = subprocess.Popen([myval], stdout=subprocess.PIPE, shell=True)
           except:
               print "Unable to find that container ..."
       printsummary()

    if containernum.upper() == "P":
        peekcontainer = getcontainer(cdetails)
        if peekcontainer != False:
            for container in peekcontainer:
                if not isRunning(cdetails, container):
                    print " "
                    print ("{0} is not a running container".format(returnuid(cdetails, container)))
                    time.sleep(2)
                    printsummary()
                cpid = getpid(cdetails, container)
                print "Entering container %s" % returnuid(cdetails, container)
                foo = terminal2(cpid)
        printsummary()
    else:
        printsummary()

if __name__ == '__main__':
    printsummary()
<commit_msg>Added threading to the docker stop, cleaned up some various syntax related bugs<commit_after>#!/usr/bin/env python

import os
import subprocess
import json
import time
import argparse
import threading

parser = argparse.ArgumentParser()
parser.add_argument("-a", "--all", help="Work with non-active containers too", action="store_true")
args = parser.parse_args()

if args.all:
    print "View all containers"

def getcontainerinfo(containeruids):
    """ This function takes an array of of container uids and
    returns an array of dicts with the inspect info
    """
    cdetails = list ()
    for containers in containeruids:
        mycommand = "docker inspect %s" % containers
        containerproc = subprocess.Popen([mycommand], stdout=subprocess.PIPE, shell=True)
        cdetails.append(json.loads(containerproc.stdout.read())[0])
    return cdetails

def getsummary(containerinfo):
    """
    This function takes a container and returns its run state
    """

    cuid = containerinfo['Id']
    cimage = containerinfo['Config']['Image']
    if containerinfo['State']['Running'] == 1:
        crun = "Running"
    else:
        crun = "Not Running"
    return cuid[:8], cimage, crun

def isInt(mystr):
    try:
        int(mystr)
        return True
    except ValueError:
        return False

def str2list(inlist):
    """Converts input string into valid list

    Parses spaces, commas and range of values('-')"""
    delim = " "
    if ',' in inlist:
        delim = ","
    inlist = delim.join(inlist.split(delim))
    containerlist = inlist.split(delim)
    rangelist = [containerlist.pop(r[0]) for r in enumerate(containerlist) if '-' in containerlist[r[0]]]
    for rl in rangelist:
        start, end = rl.split('-')
        containerlist.extend(range(int(start), int(end)+1))
    return containerlist

def terminal2(cpid):
    """
    This function takes a pid of a running container and opens it in
    xterm if its available
    """
    nsenter = ('sudo nsenter -m -u -n -i -p -t {0} /bin/bash'.format(cpid))
    import pty
    import os
    if os.getenv('DISPLAY',"") == "":
        pty.spawn(nsenter.split())
    else:
        mycommand = "xterm -T {0} -e {1}".format(cpid, nsenter)
        containerproc = subprocess.Popen([mycommand], stdout=subprocess.PIPE, shell=True)

def getpid(containarray, mynum):
    return containarray[int(mynum)]['State']['Pid']

def isRunning(containarray, mynum):
    if (containarray[int(mynum)]['State']['Running']):
        return True
    else:
        return False

def returnuid(containarray, mynum):
    myuid = containarray[int(mynum)]['Id']
    return myuid[:8]

def containerinrange(cdetails, stopcontainers):
    for i in stopcontainers:
        foo = int(len(cdetails))
        foo = foo -1
        if (0 <= int(i) <= foo) == False:
            print " "
            print ("{0} isn't a valid container number...".format(i))
            time.sleep(2)
            return False
    return True

def getcontainer(cdetails):
    """returns valid list of container IDs"""
    print " "
    stopcontainers = raw_input("Which Container(s)?: ")
    stopcontainers = str2list(stopcontainers)
    if containerinrange(cdetails, stopcontainers) == False:
        return False
    for container in stopcontainers:
        # FIXME: check if it's valid integer
        if not(isInt(container)):
            print " "
            print ("{0} isn't a integer...".format(container))
            time.sleep(2)
            return False
        else:
            return stopcontainers

def printsummary():
    print " "

    dockercmd = ["docker", "ps"]
    if args.all:
        dockall = "-qa"
    else:
        dockall = "-q"
    dockercmd.append(dockall)
    proc = subprocess.Popen(dockercmd, stdout=subprocess.PIPE )
    out = proc.stdout.read()
    containeruids = out.split()
    cdetails = getcontainerinfo(containeruids)
    if len(cdetails) != 0:
        print ('{0:2} {1:12} {2:15} {3:8}'.format("#", "ID","Image","Status"))
        for s in range(len(cdetails)):
            chostname, cimage, crun = getsummary(cdetails[s])
            print ('{0:2} {1:12} {2:15} {3:8}'.format(s, chostname, cimage, crun))
    else:
        print "No active containers ..."
    print " "
    print "Command Reference: (q)uit (r)efresh (s)top (d)elete (p)eek"
    print " "
    containernum = raw_input("Command: ")
    if containernum.upper() == "R":
        printsummary()
    if containernum.upper() == "Q":
        quit()
    if containernum.upper() == "S":
        stopcontainer = getcontainer(cdetails)
        stopthreads = []
        for container in stopcontainer:
            cid = returnuid(cdetails, container)
            if not isRunning(cdetails, container):
                print "%s is not running" % cid
                time.sleep(1)
                break
            cpid = getpid(cdetails, container)
            myval = ("docker stop {0}".format(cid))
            t = threading.Thread(target=stopcontainers, args=(myval, cpid,))
            stopthreads.append(t)
            t.start()
        print "Waiting for containers to stop"
        [x.join() for x in stopthreads]
        printsummary()

    if containernum.upper() == "D":
       delcontainer = getcontainer(cdetails)
       for container in delcontainer:
           try:
               cid = returnuid(cdetails, container)
               print "Deleting %s" % cid
               myval = ("docker rm {0}".format(cid))
               delprocess = subprocess.Popen([myval], stdout=subprocess.PIPE, shell=True)
           except:
               print "Unable to find that container ..."
       printsummary()

    if containernum.upper() == "P":
        peekcontainer = getcontainer(cdetails)
        if peekcontainer != False:
            for container in peekcontainer:
                if not isRunning(cdetails, container):
                    print " "
                    print ("{0} is not a running container".format(returnuid(cdetails, container)))
                    time.sleep(2)
                    printsummary()
                cpid = getpid(cdetails, container)
                print "Entering container %s" % returnuid(cdetails, container)
                foo = terminal2(cpid)
        printsummary()
    else:
        printsummary()

def stopcontainers(myval, cpid):
    stopprocess = subprocess.call([myval], stdout=subprocess.PIPE, shell=True)
    lambda: os.waitpid(cpid,0)

if __name__ == '__main__':
    printsummary()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import scrapy
import json

from locations.items import GeojsonPointItem

class SephoraSpider(scrapy.Spider):
    name = "sephora"
    allowed_domains = ["www.sephora.com"]
    download_delay = 1.5
    start_urls = (
        'https://www.sephora.com/storelist',
    )

    def store_hours(self, store_hours):
        day_groups = []
        this_day_group = None
        for day in ('monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'):
            day_open = store_hours[day]['open'].zfill(5)
            day_close = store_hours[day]['close'].zfill(5)

            hours = day_open + "-" + day_close

            if hours == '00:00-00:00':
                # They're closed on this day
                continue

            day_short = day.title()[:2]

            if not this_day_group:
                this_day_group = dict(from_day=day_short, to_day=day_short, hours=hours)
            elif this_day_group['hours'] == hours:
                this_day_group['to_day'] = day_short
            elif this_day_group['hours'] != hours:
                day_groups.append(this_day_group)
                this_day_group = dict(from_day=day_short, to_day=day_short, hours=hours)
        day_groups.append(this_day_group)

        if not day_groups:
            return None

        if len(day_groups) == 1:
            opening_hours = day_groups[0]['hours']
            if opening_hours == '07:00-07:00':
                opening_hours = '24/7'
        else:
            opening_hours = ''
            for day_group in day_groups:
                if day_group['from_day'] == day_group['to_day']:
                    opening_hours += '{from_day} {hours}; '.format(**day_group)
                else:
                    opening_hours += '{from_day}-{to_day} {hours}; '.format(**day_group)
            opening_hours = opening_hours[:-2]

        return opening_hours

    def parse(self, response):
        for path in response.xpath('//a[@class="u-hoverRed u-p1 u-db"]/@href'):
            yield scrapy.Request(
                response.urljoin(path.extract()),
                callback=self.parse_store,
            )

    def parse_store(self, response):
        data = json.loads(response.xpath('//script[@type="application/ld+json"]/text()').extract_first())

        properties = {
            'website': data['url'],
            'phone': data['address']['telephone'],
            'addr:full': data['address']['streetAddress'],
            'addr:postcode': data['address']['postalCode'],
            'addr:state': data['address']['addressRegion'],
            'addr:city': data['address']['addressLocality'],
            'ref': data['url'],
        }

        opening_hours = data.get('openingHours')
        if opening_hours:
            properties['opening_hours'] = '; '.join(opening_hours)

        lon_lat = [
            float(response.xpath('//div[@class="storehq-detail"]/@data-lng').extract_first()),
            float(response.xpath('//div[@class="storehq-detail"]/@data-lat').extract_first()),
        ]

        yield GeojsonPointItem(
            properties=properties,
            lon_lat=lon_lat,
        )
<commit_msg>Update Sephora spider<commit_after># -*- coding: utf-8 -*-
import scrapy
import json

from locations.items import GeojsonPointItem

class SephoraSpider(scrapy.Spider):
    name = "sephora"
    allowed_domains = ["www.sephora.com"]
    download_delay = 1.5
    start_urls = (
        'https://www.sephora.com/storelist',
    )

    def store_hours(self, store_hours):
        day_groups = []
        this_day_group = None
        for day in ('monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'):
            day_open = store_hours[day]['open'].zfill(5)
            day_close = store_hours[day]['close'].zfill(5)

            hours = day_open + "-" + day_close

            if hours == '00:00-00:00':
                # They're closed on this day
                continue

            day_short = day.title()[:2]

            if not this_day_group:
                this_day_group = dict(from_day=day_short, to_day=day_short, hours=hours)
            elif this_day_group['hours'] == hours:
                this_day_group['to_day'] = day_short
            elif this_day_group['hours'] != hours:
                day_groups.append(this_day_group)
                this_day_group = dict(from_day=day_short, to_day=day_short, hours=hours)
        day_groups.append(this_day_group)

        if not day_groups:
            return None

        if len(day_groups) == 1:
            opening_hours = day_groups[0]['hours']
            if opening_hours == '07:00-07:00':
                opening_hours = '24/7'
        else:
            opening_hours = ''
            for day_group in day_groups:
                if day_group['from_day'] == day_group['to_day']:
                    opening_hours += '{from_day} {hours}; '.format(**day_group)
                else:
                    opening_hours += '{from_day}-{to_day} {hours}; '.format(**day_group)
            opening_hours = opening_hours[:-2]

        return opening_hours

    def parse(self, response):
        for path in response.xpath('//a[@class="u-hoverRed u-p1 u-db"]/@href'):
            yield scrapy.Request(
                response.urljoin(path.extract()),
                callback=self.parse_store,
            )

    def parse_store(self, response):
        data = json.loads(response.xpath('//script[@type="application/ld+json"]/text()').extract_first())

        properties = {
            'website': data['url'],
            'phone': data['address']['telephone'],
            'addr_full': data['address']['streetAddress'],
            'postcode': data['address']['postalCode'],
            'state': data['address']['addressRegion'],
            'city': data['address']['addressLocality'],
            'ref': data['url'],
            'lon': float(response.xpath('//div[@class="storehq-detail"]/@data-lng').extract_first()),
            'lat': float(response.xpath('//div[@class="storehq-detail"]/@data-lat').extract_first()),
        }

        opening_hours = data.get('openingHours')
        if opening_hours:
            properties['opening_hours'] = '; '.join(opening_hours)

        yield GeojsonPointItem(**properties)
<|endoftext|>


<commit_before>import logging
import uuid

from django.conf import settings
from log_request_id import local, REQUEST_ID_HEADER_SETTING, LOG_REQUESTS_SETTING, NO_REQUEST_ID, \
    REQUEST_ID_RESPONSE_HEADER_SETTING

logger = logging.getLogger(__name__)

class RequestIDMiddleware(object):

    def process_request(self, request):
        request_id = self._get_request_id(request)
        local.request_id = request_id
        request.id = request_id

    def process_response(self, request, response):
        if getattr(settings, REQUEST_ID_RESPONSE_HEADER_SETTING, False) and getattr(request, 'id', None):
            response[getattr(settings, REQUEST_ID_RESPONSE_HEADER_SETTING)] = request.id

        if not getattr(settings, LOG_REQUESTS_SETTING, False):
            return response

        # Don't log favicon
        if 'favicon' in request.path:
            return response

        user = getattr(request, 'user', None)
        user_id = getattr(user, 'pk', None) or getattr(user, 'id', None)

        message = 'method=%s path=%s status=%s'
        args = (request.method, request.path, response.status_code)

        if user_id:
            message += ' user=%s'
            args += (user_id,)

        logger.info(message, *args)

        return response

    def _get_request_id(self, request):
        request_id_header = getattr(settings, REQUEST_ID_HEADER_SETTING, None)
        if request_id_header:
            return request.META.get(request_id_header, NO_REQUEST_ID)
        return self._generate_id()

    def _generate_id(self):
        return uuid.uuid4().hex
<commit_msg>Unset request ID at end of process_response<commit_after>import logging
import uuid

from django.conf import settings
from log_request_id import local, REQUEST_ID_HEADER_SETTING, LOG_REQUESTS_SETTING, NO_REQUEST_ID, \
    REQUEST_ID_RESPONSE_HEADER_SETTING

logger = logging.getLogger(__name__)

class RequestIDMiddleware(object):

    def process_request(self, request):
        request_id = self._get_request_id(request)
        local.request_id = request_id
        request.id = request_id

    def process_response(self, request, response):
        if getattr(settings, REQUEST_ID_RESPONSE_HEADER_SETTING, False) and getattr(request, 'id', None):
            response[getattr(settings, REQUEST_ID_RESPONSE_HEADER_SETTING)] = request.id

        if not getattr(settings, LOG_REQUESTS_SETTING, False):
            return response

        # Don't log favicon
        if 'favicon' in request.path:
            return response

        user = getattr(request, 'user', None)
        user_id = getattr(user, 'pk', None) or getattr(user, 'id', None)

        message = 'method=%s path=%s status=%s'
        args = (request.method, request.path, response.status_code)

        if user_id:
            message += ' user=%s'
            args += (user_id,)

        logger.info(message, *args)

        del local.request_id

        return response

    def _get_request_id(self, request):
        request_id_header = getattr(settings, REQUEST_ID_HEADER_SETTING, None)
        if request_id_header:
            return request.META.get(request_id_header, NO_REQUEST_ID)
        return self._generate_id()

    def _generate_id(self):
        return uuid.uuid4().hex
<|endoftext|>


<commit_before>from ._metakernel import MetaKernel, IPythonKernel, register_ipython_magics, get_metakernel
from . import pexpect
from .replwrap import REPLWrapper, u
from .process_metakernel import ProcessMetaKernel
from .magic import Magic, option
from .parser import Parser

__all__ = ['Magic', 'MetaKernel', 'option']

__version__ = '0.11.5'

del magic, _metakernel, parser, process_metakernel
<commit_msg>Bump version<commit_after>from ._metakernel import MetaKernel, IPythonKernel, register_ipython_magics, get_metakernel
from . import pexpect
from .replwrap import REPLWrapper, u
from .process_metakernel import ProcessMetaKernel
from .magic import Magic, option
from .parser import Parser

__all__ = ['Magic', 'MetaKernel', 'option']

__version__ = '0.11.6'

del magic, _metakernel, parser, process_metakernel
<|endoftext|>


<commit_before>import unittest

import gates

class Tests(unittest.TestCase):

    def test_AND(self):
        self.assertEqual(gates.AND(0, 0), 0)
        self.assertEqual(gates.AND(0, 1), 0)
        self.assertEqual(gates.AND(1, 0), 0)
        self.assertEqual(gates.AND(1, 1), 1)

    def test_AND3(self):
        self.assertEqual(gates.AND3(0, 0, 0), 0)
        self.assertEqual(gates.AND3(0, 0, 1), 0)
        self.assertEqual(gates.AND3(0, 1, 0), 0)
        self.assertEqual(gates.AND3(0, 1, 1), 0)
        self.assertEqual(gates.AND3(1, 0, 0), 0)
        self.assertEqual(gates.AND3(1, 0, 1), 0)
        self.assertEqual(gates.AND3(1, 1, 0), 0)
        self.assertEqual(gates.AND3(1, 1, 1), 1)

    def test_OR(self):
        self.assertEqual(gates.OR(0, 0), 0)
        self.assertEqual(gates.OR(0, 1), 1)
        self.assertEqual(gates.OR(1, 0), 1)
        self.assertEqual(gates.OR(1, 1), 1)

    def test_NOT(self):
        self.assertEqual(gates.NOT(0), 1)
        self.assertEqual(gates.NOT(1), 0)

    def test_NAND(self):
        self.assertEqual(gates.NAND(0, 0), 1)
        self.assertEqual(gates.NAND(0, 1), 1)
        self.assertEqual(gates.NAND(1, 0), 1)
        self.assertEqual(gates.NAND(1, 1), 0)

    def test_NOR(self):
        self.assertEqual(gates.NOR(0, 0), 1)
        self.assertEqual(gates.NOR(0, 1), 0)
        self.assertEqual(gates.NOR(1, 0), 0)
        self.assertEqual(gates.NOR(1, 1), 0)

    def test_XOR(self):
        self.assertEqual(gates.XOR(0, 0), 0)
        self.assertEqual(gates.XOR(0, 1), 1)
        self.assertEqual(gates.XOR(1, 0), 1)
        self.assertEqual(gates.XOR(1, 1), 0)

    def test_XNOR(self):
        self.assertEqual(gates.XNOR(0, 0), 1)
        self.assertEqual(gates.XNOR(0, 1), 0)
        self.assertEqual(gates.XNOR(1, 0), 0)
        self.assertEqual(gates.XNOR(1, 1), 1)

if __name__ == '__main__':
    unittest.main(verbosity=2)

<commit_msg>Added test for 4-input OR<commit_after>import unittest

import gates

class Tests(unittest.TestCase):

    def test_AND(self):
        self.assertEqual(gates.AND(0, 0), 0)
        self.assertEqual(gates.AND(0, 1), 0)
        self.assertEqual(gates.AND(1, 0), 0)
        self.assertEqual(gates.AND(1, 1), 1)

    def test_AND3(self):
        self.assertEqual(gates.AND3(0, 0, 0), 0)
        self.assertEqual(gates.AND3(0, 0, 1), 0)
        self.assertEqual(gates.AND3(0, 1, 0), 0)
        self.assertEqual(gates.AND3(0, 1, 1), 0)
        self.assertEqual(gates.AND3(1, 0, 0), 0)
        self.assertEqual(gates.AND3(1, 0, 1), 0)
        self.assertEqual(gates.AND3(1, 1, 0), 0)
        self.assertEqual(gates.AND3(1, 1, 1), 1)

    def test_OR(self):
        self.assertEqual(gates.OR(0, 0), 0)
        self.assertEqual(gates.OR(0, 1), 1)
        self.assertEqual(gates.OR(1, 0), 1)
        self.assertEqual(gates.OR(1, 1), 1)

    def test_OR4(self):
        self.assertEqual(gates.OR4(0, 0, 0, 0), 0)
        self.assertEqual(gates.OR4(0, 0, 0, 1), 1)
        self.assertEqual(gates.OR4(0, 0, 1, 0), 1)
        self.assertEqual(gates.OR4(0, 0, 1, 1), 1)
        self.assertEqual(gates.OR4(0, 1, 0, 0), 1)
        self.assertEqual(gates.OR4(0, 1, 0, 1), 1)
        self.assertEqual(gates.OR4(0, 1, 1, 0), 1)
        self.assertEqual(gates.OR4(0, 1, 1, 1), 1)
        self.assertEqual(gates.OR4(1, 0, 0, 0), 1)
        self.assertEqual(gates.OR4(1, 0, 0, 1), 1)
        self.assertEqual(gates.OR4(1, 0, 1, 0), 1)
        self.assertEqual(gates.OR4(1, 0, 1, 1), 1)
        self.assertEqual(gates.OR4(1, 1, 0, 0), 1)
        self.assertEqual(gates.OR4(1, 1, 0, 1), 1)
        self.assertEqual(gates.OR4(1, 1, 1, 0), 1)
        self.assertEqual(gates.OR4(1, 1, 1, 1), 1)

    def test_NOT(self):
        self.assertEqual(gates.NOT(0), 1)
        self.assertEqual(gates.NOT(1), 0)

    def test_NAND(self):
        self.assertEqual(gates.NAND(0, 0), 1)
        self.assertEqual(gates.NAND(0, 1), 1)
        self.assertEqual(gates.NAND(1, 0), 1)
        self.assertEqual(gates.NAND(1, 1), 0)

    def test_NOR(self):
        self.assertEqual(gates.NOR(0, 0), 1)
        self.assertEqual(gates.NOR(0, 1), 0)
        self.assertEqual(gates.NOR(1, 0), 0)
        self.assertEqual(gates.NOR(1, 1), 0)

    def test_XOR(self):
        self.assertEqual(gates.XOR(0, 0), 0)
        self.assertEqual(gates.XOR(0, 1), 1)
        self.assertEqual(gates.XOR(1, 0), 1)
        self.assertEqual(gates.XOR(1, 1), 0)

    def test_XNOR(self):
        self.assertEqual(gates.XNOR(0, 0), 1)
        self.assertEqual(gates.XNOR(0, 1), 0)
        self.assertEqual(gates.XNOR(1, 0), 0)
        self.assertEqual(gates.XNOR(1, 1), 1)

if __name__ == '__main__':
    unittest.main(verbosity=2)

<|endoftext|>


<commit_before>import logging
import hmac
from cryptography.hazmat.primitives.hashes import SHA1
from pyasn1.codec.der.encoder import encode as asn1_encode
from pyasn1.codec.der.decoder import decode as asn1_decode
from os import urandom
from datetime import datetime
from pyasn1.type import univ
import mflod.crypto.asn1_structures as asn1_dec
from mflod.crypto.constants import Constants as const
from mflod.crypto.log_strings import LogStrings as logstr
from cryptography.hazmat.primitives import padding
from cryptography.hazmat.primitives.asymmetric import padding as asym_padding
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from cryptography.hazmat.backends import default_backend

class Crypto(object):
    """ Class that handles assembly of FLOD protocol message packet

    The main purpose of this class is to provide cryptographic back-end
    for FLOD protocol client to prepare content user wishes to send to
    a transportation.

    The Steps involved are hybrid encryption, HMAC calculation, signing
    etc. The structure of a message packet and some implementation notes
    can be found in module README.md file: https://goo.gl/leWWa4

    Developers:
        - vsmysle (Kyrylo Voronchenko)
        - ddnomad (Artem Fliunt)
        - Sofiya Demchuk

    """

    def __init__(self):
        """ Initialization method """

        self.logger = logging.getLogger(__name__)
        self.logger.debug(logstr.CRYPTO_CLASS_INIT)

    def assemble_message_packet(msg_content, recipient_pk, sign=None):
        """ Assemble FLOD message packet

        @developer: ???

        Assembly involves a creation of DER-encoded ASN.1 structure that
        corresponds to a specification of FLOD message packet format. To
        include optional signing in header block pass a sender_sk named
        argument.

        Also this method handles the assembly of a HEADER block that involves
        putting all the keys and meta-information together.

        :param msg_content:         string message to include into FLOD
                                    message packet
        :param recipient_pk:        instance of cryptography.hazmat.
                                    primitives.asymmetric.rsa.RSAPublicKey
                                    that is a public key of a recipient.
                                    Used to encrypt a header block of the
                                    message packet.
        :param sign=None:      instance of cryptography.hazmat.
                                    primitives.asymmetric.rsa.RSAPrivateKey
                                    that is used to create a signature in the
                                    header block of FLOD message packet

        :return: string DER-encoded ASN.1 structure that is FLOD message packet
                 ready to be sent to a recipient

        :raise: ???

        """

        pass

    def disassemble_message_packet(msg_packet, get_sks_func, get_pk_byid_func):
        """ Attempt to disassemble FLOD message packet that was received

        @developer: ???

        Disassembly involves test decryption of header block with all available
        private keys of a user. If decryption was successful then the message
        was addressed to the user. This does not means though that the
        signature verification will succeed as well as HMAC integrity check.

        On successful decryption the method returns a recovered message
        together with a supplementary exit code which determines conditions
        that occur during disassembly process. The code can be one of the
        following integers:

            - 0: indicates that signature verification was successful with a
              known sender PGPKeyID
            - 1: indicates that signature verification was successful but the
              key used was not a PGP key (but it exists in a user key chain)
            - 2: indicates that decryption was successful but the message was
              not signed by a sender
            - 3: indicates that the signature authenticity cannot be
              established due to an absence of a corresponding public key

        :param msg_packet:          string DER-encoded ASN.1 structure of FLOD
                                    message packet to decrypt
        :param get_sks_func:        generator that yields all available secret
                                    keys of a user. The user keys have to be
                                    instances of cryptography.hazmat.primitives
                                    .assymetric.rsa.RSAPublicKey class.
        :param get_pk_byid_func:    function that returns an instance of
                                    cryptography.hazmat.primitives.assymetric.rsa
                                    .RSAPrivateKey that corresponds to a
                                    PGPKeyID that is passed to it. If there is
                                    no such key available the function should
                                    return a list of instances of the same
                                    class that are non-PGP keys without IDs.

        :return: one of the following lists (see supplementary exit codes
                 paragraph for details):
                    - [dec_msg, 0, pgp_key_id]
                    - [dec_msg, 1, sign_pk]
                    - [dec_msg, 2]
                    - [dec_msg, 3]
                The values in lists are the following:
                    - dec_msg:      string decryption of a message received
                    - pgp_key_id:   string PGPKeyID of a public key that
                                    verified a signature
                    - sign_pk:      an instance of cryptography.hazmat.
                                    primitives.rsa.RSAPublicKey that verified a
                                    signature

        :raise ???

        """

        pass

    def __assemble_content_block(self, content, key, iv):
        """ Create an ASN.1 DER-encoded structure of a content block

        @developer: ddnomad

        The corresponding ASN.1 structure from a documentation is
        MPContentContainer

        :param content: string content to encapsulate
        :param key:     string AES key to use for encryption
        :param iv:      string CBC mode initialization vector

        :return: string DER-encoding of MPContentContainer ASN.1 structure

        """

        # logger entry
        self.logger.debug(logstr.ASSEMBLE_CONTENT_BLOCK_CALL)

        # create an ASN.1 structure of MPContent and DER-encode it
        mp_content_pt = asn1_dec.MPContent()
        mp_content_pt['timestamp'] = datetime.utcnow(). \
            strftime(const.TIMESTAMP_FORMAT)
        mp_content_pt['content'] = content
        mp_content_pt_der = asn1_encode(mp_content_pt)

        # pad MPContent with PKCS#7

        # initialize necessary crypto backend instances
        backend = default_backend()
        aes = Cipher(algorithms.AES(key), modes.CBC(iv),
                backend=backend).encryptor()

    def __disassemble_content_block(content, key):
        """ Decrypt and decode content from a content block

        @developer: ???

        :param content: DER-encoded ASN.1 structure that encapsulates
                        encrypted content
        :param key:     string AES key to be used for decryption

        :return: string decrypted message

        """

        pass

    def __assemble_hmac_block(self, content, key):
        """ Produce HMAC block ASN.1 structure (MPHMACContainer)

        @developer: vsmysle

        :param content: string DER-encoded content generate HMAC of and
                        encapsulate into HMAC FLOD block
        :param key:     string key to use for HMAC generation

        :return: DER-encoded ASN.1 structure that encapsulates HMAC
                 block

        """
        # TODO: add exceptions

        self.logger.debug("Producing HMAC block with ASN.1 structure...")
        # calculating hmac digest of content
        digest = self.__generate_hmac(content, key)

        # oid for SHA1 hash function
        oid = '1.3.14.3.2.26'

        # creating instance of AlgorithmIdentifier class
        ai = asn1_dec.AlgorithmIdentifier()

        # setting corresponding parameters
        ai['algorithm'] = oid
        ai['parameters'] = univ.Null()

        # creating instance of AlgorithmIdentifier class
        hmac_block = asn1_dec.MPHMACContainer()

        # setting corresponding parameters
        hmac_block['digestAlgorithm'] = ai
        hmac_block['digest'] = digest

        return asn1_encode(hmac_block)

    def __verify_hmac(self, hmac_blk, key, content_blk):
        """ Verify content HMAC

        @developer: vsmysle

        :param hmac_blk:        string DER-encoded ASN.1 structure of HMAC
                                block (MPHMACContainer)
        :param key:             string HMAC secret key
        :param content_blk:     string DER-encoded ASN.1 structure of content
                                block

        :return: bool verification result

        """
        # TODO: add exceptions

        self.logger.debug("Verifying  HMAC...")
        2
        # calculation of the HMAC digest for received content block
        hmac_of_content_blk = self.__generate_hmac(content_blk, key)

        # get digest from the HMAC block
        # TODO: check index !
        decoded_hmac_block = asn1_decode(hmac_blk)[0][1]

        if decoded_hmac_block == hmac_of_content_blk:
            return True
        return False

    def __generate_hmac(self, content, key):
        """ Generate HMAC for in input content and key

        @developer: vsmysle

        :param content: string DER-encoded content to produce digest of
        :param key:     string key to use for HMAC

        :return: string HMAC of the input content

        """
        # TODO: add exceptions

        self.logger.debug("Generation HMAC for input content...")

        # generating instance of HMAC with sha1 hash function
        hmac_digest = hmac.new(key, None, sha1)

        # feed the content to generated HMAC instance
        hmac_digest.update(content)
        return hmac_digest.digest()

    def __encrypt_with_aes(content, key):
        """ Encrypt content with AES-128-CBC (with PCKS#7 padding)

        @developer: ???

        :param content: string DER-encoded MPContent ASN.1 structure to encrypt
        :param key:     string key to use for encryption

        :return: string encryption of an input content

        """

        pass

    def __decrypt_with_aes(content, key):
        """ Decrypt AES-128-CBC encrypted content (PCKS#7 padded)

        @developer: ???

        :param content: string ciphertext of MPContent ASN.1 structure
        :param key:     string AES secret key

        :return: string decrypted DER-encoded MPContent ASN.1 structure

        """

        pass

    def __encrypt_with_rsa(self, content, recipient_pk):
        """ Encrypt content with RSAES-OAEP scheme

        @developer: vsmysle

        This method handles an encryption of a *single* RSA block with a
        specified above scheme. It does not handle splitting of a header into
        several blocks. It has to be done by other method that would use this
        one only for single block encryption purpose.

        TODO: what is a maximum size of a content that can be padded and
        encrypted given a particular size of RSA key?

        :param content:         string content to encrypt (probably a part of
                                ASN.1 DER-encoded MPHeader block)
        :param recipient_pk:    instance of cryptography.hazmat.primitives.rsa
                                .RSAPublicKey to use for a content encryption

        :return: string encryption of an input content

        """
        self.logger.debug("RSA encryption ...")

        ciphertext = recipient_pk.encrypt(
            content, asym_padding.OAEP(
                mgf=asym_padding.MGF1(algorithm=SHA1()),
                algorithm=SHA1(),
                label=None
            )
        )
        return ciphertext

    def __decrypt_with_rsa(self, content, user_sk):
        """ Decrypt RSAES-OAEP encrypted content (single block)

        @developer: ???

        This method decrypts a single RSA ciphertext block only

        :param content: string content to decrypt
        :param user_sk: instance of cryptography.hazmat.primitives.rsa
                        .RSAPrivateKey to use for a decryption

        :return: string decryption of an input content

        """

    def __sign_content(self, content, user_sk):
        """ Produce a signature of an input content using RSASSA-PSS scheme

        @developer: ???

        :param content: string content to sign
        :param user_sk: instance of cryptography.hazmat.primitives.rsa.
                        RSAPrivateKey

        :return: string signature of the input content

        """

        pass

    def __verify_signature(self, signature, signer_pk, content):
        """ Verify RSASSA-PSS signature

        @developer: ???

        :param signature: string signature to verify
        :param signer_pk: instance of cryptography.hazmat.primitives.
                          rsa.RSAPublicKey that is a public key of a signer
        :param content:   content to verify a signature of

        :return: bool verification result

        """

        pass

    def __get_asn1_algorithm_identifier_der(self, oid_str):
        """ Generate ASN.1 structure for algorithm identifier

        @developer: vsmysle

        :param oid_str: string OID to encapsulate

        :return: pyasn1.type.univ.Sequence object

        """

        # TODO: add exceptions

        # create the instance of AlgorithmIdentifier
        self.logger.debug("Receiving ASN.1 AlgorithmIdentifier"
                          " structure with OID=%s") % oid_str
        ai = asn1_dec.AlgorithmIdentifier()

        # set corresponding parameters
        ai['algorithm'] = oid_str
        ai['parameters'] = univ.Null()
        return ai

    def __get_random_bytes(self, spec_lst):
        """ Generate random bytes

        @developer: vsmysle

        :param spec_lst: list of integers that is lengths of bytestings to
                         return

        :return: list of random bytestrings with lengths corresponding to the
                 ones from a spec list

        """

        # TODO: add exception for negative integers

        self.logger.debug("Generating random bytes...")
        return [urandom(i) for i in spec_lst]
<commit_msg>added RSA decryption, changed hashlib to cryptography hashes lib<commit_after>import logging
import hmac
from cryptography.hazmat.primitives.hashes import SHA1
from pyasn1.codec.der.encoder import encode as asn1_encode
from pyasn1.codec.der.decoder import decode as asn1_decode
from os import urandom
from datetime import datetime
from pyasn1.type import univ
import mflod.crypto.asn1_structures as asn1_dec
from mflod.crypto.constants import Constants as const
from mflod.crypto.log_strings import LogStrings as logstr
from cryptography.hazmat.primitives import padding
from cryptography.hazmat.primitives.asymmetric import padding as asym_padding
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from cryptography.hazmat.backends import default_backend

class Crypto(object):
    """ Class that handles assembly of FLOD protocol message packet

    The main purpose of this class is to provide cryptographic back-end
    for FLOD protocol client to prepare content user wishes to send to
    a transportation.

    The Steps involved are hybrid encryption, HMAC calculation, signing
    etc. The structure of a message packet and some implementation notes
    can be found in module README.md file: https://goo.gl/leWWa4

    Developers:
        - vsmysle (Kyrylo Voronchenko)
        - ddnomad (Artem Fliunt)
        - Sofiya Demchuk

    """

    def __init__(self):
        """ Initialization method """

        self.logger = logging.getLogger(__name__)
        self.logger.debug(logstr.CRYPTO_CLASS_INIT)

    def assemble_message_packet(msg_content, recipient_pk, sign=None):
        """ Assemble FLOD message packet

        @developer: ???

        Assembly involves a creation of DER-encoded ASN.1 structure that
        corresponds to a specification of FLOD message packet format. To
        include optional signing in header block pass a sender_sk named
        argument.

        Also this method handles the assembly of a HEADER block that involves
        putting all the keys and meta-information together.

        :param msg_content:         string message to include into FLOD
                                    message packet
        :param recipient_pk:        instance of cryptography.hazmat.
                                    primitives.asymmetric.rsa.RSAPublicKey
                                    that is a public key of a recipient.
                                    Used to encrypt a header block of the
                                    message packet.
        :param sign=None:      instance of cryptography.hazmat.
                                    primitives.asymmetric.rsa.RSAPrivateKey
                                    that is used to create a signature in the
                                    header block of FLOD message packet

        :return: string DER-encoded ASN.1 structure that is FLOD message packet
                 ready to be sent to a recipient

        :raise: ???

        """

        pass

    def disassemble_message_packet(msg_packet, get_sks_func, get_pk_byid_func):
        """ Attempt to disassemble FLOD message packet that was received

        @developer: ???

        Disassembly involves test decryption of header block with all available
        private keys of a user. If decryption was successful then the message
        was addressed to the user. This does not means though that the
        signature verification will succeed as well as HMAC integrity check.

        On successful decryption the method returns a recovered message
        together with a supplementary exit code which determines conditions
        that occur during disassembly process. The code can be one of the
        following integers:

            - 0: indicates that signature verification was successful with a
              known sender PGPKeyID
            - 1: indicates that signature verification was successful but the
              key used was not a PGP key (but it exists in a user key chain)
            - 2: indicates that decryption was successful but the message was
              not signed by a sender
            - 3: indicates that the signature authenticity cannot be
              established due to an absence of a corresponding public key

        :param msg_packet:          string DER-encoded ASN.1 structure of FLOD
                                    message packet to decrypt
        :param get_sks_func:        generator that yields all available secret
                                    keys of a user. The user keys have to be
                                    instances of cryptography.hazmat.primitives
                                    .assymetric.rsa.RSAPublicKey class.
        :param get_pk_byid_func:    function that returns an instance of
                                    cryptography.hazmat.primitives.assymetric.rsa
                                    .RSAPrivateKey that corresponds to a
                                    PGPKeyID that is passed to it. If there is
                                    no such key available the function should
                                    return a list of instances of the same
                                    class that are non-PGP keys without IDs.

        :return: one of the following lists (see supplementary exit codes
                 paragraph for details):
                    - [dec_msg, 0, pgp_key_id]
                    - [dec_msg, 1, sign_pk]
                    - [dec_msg, 2]
                    - [dec_msg, 3]
                The values in lists are the following:
                    - dec_msg:      string decryption of a message received
                    - pgp_key_id:   string PGPKeyID of a public key that
                                    verified a signature
                    - sign_pk:      an instance of cryptography.hazmat.
                                    primitives.rsa.RSAPublicKey that verified a
                                    signature

        :raise ???

        """

        pass

    def __assemble_content_block(self, content, key, iv):
        """ Create an ASN.1 DER-encoded structure of a content block

        @developer: ddnomad

        The corresponding ASN.1 structure from a documentation is
        MPContentContainer

        :param content: string content to encapsulate
        :param key:     string AES key to use for encryption
        :param iv:      string CBC mode initialization vector

        :return: string DER-encoding of MPContentContainer ASN.1 structure

        """

        # logger entry
        self.logger.debug(logstr.ASSEMBLE_CONTENT_BLOCK_CALL)

        # create an ASN.1 structure of MPContent and DER-encode it
        mp_content_pt = asn1_dec.MPContent()
        mp_content_pt['timestamp'] = datetime.utcnow(). \
            strftime(const.TIMESTAMP_FORMAT)
        mp_content_pt['content'] = content
        mp_content_pt_der = asn1_encode(mp_content_pt)

        # pad MPContent with PKCS#7

        # initialize necessary crypto backend instances
        backend = default_backend()
        aes = Cipher(algorithms.AES(key), modes.CBC(iv),
                backend=backend).encryptor()

    def __disassemble_content_block(content, key):
        """ Decrypt and decode content from a content block

        @developer: ???

        :param content: DER-encoded ASN.1 structure that encapsulates
                        encrypted content
        :param key:     string AES key to be used for decryption

        :return: string decrypted message

        """

        pass

    def __assemble_hmac_block(self, content, key):
        """ Produce HMAC block ASN.1 structure (MPHMACContainer)

        @developer: vsmysle

        :param content: string DER-encoded content generate HMAC of and
                        encapsulate into HMAC FLOD block
        :param key:     string key to use for HMAC generation

        :return: DER-encoded ASN.1 structure that encapsulates HMAC
                 block

        """
        # TODO: add exceptions

        self.logger.debug("Producing HMAC block with ASN.1 structure...")
        # calculating hmac digest of content
        digest = self.__generate_hmac(content, key)

        # oid for SHA1 hash function
        oid = '1.3.14.3.2.26'

        # creating instance of AlgorithmIdentifier class
        ai = asn1_dec.AlgorithmIdentifier()

        # setting corresponding parameters
        ai['algorithm'] = oid
        ai['parameters'] = univ.Null()

        # creating instance of AlgorithmIdentifier class
        hmac_block = asn1_dec.MPHMACContainer()

        # setting corresponding parameters
        hmac_block['digestAlgorithm'] = ai
        hmac_block['digest'] = digest

        return asn1_encode(hmac_block)

    def __verify_hmac(self, hmac_blk, key, content_blk):
        """ Verify content HMAC

        @developer: vsmysle

        :param hmac_blk:        string DER-encoded ASN.1 structure of HMAC
                                block (MPHMACContainer)
        :param key:             string HMAC secret key
        :param content_blk:     string DER-encoded ASN.1 structure of content
                                block

        :return: bool verification result

        """
        # TODO: add exceptions

        self.logger.debug("Verifying  HMAC...")
        2
        # calculation of the HMAC digest for received content block
        hmac_of_content_blk = self.__generate_hmac(content_blk, key)

        # get digest from the HMAC block
        # TODO: check index !
        decoded_hmac_block = asn1_decode(hmac_blk)[0][1]

        if decoded_hmac_block == hmac_of_content_blk:
            return True
        return False

    def __generate_hmac(self, content, key):
        """ Generate HMAC for in input content and key

        @developer: vsmysle

        :param content: string DER-encoded content to produce digest of
        :param key:     string key to use for HMAC

        :return: string HMAC of the input content

        """
        # TODO: add exceptions

        self.logger.debug("Generation HMAC for input content...")

        # generating instance of HMAC with sha1 hash function
        hmac_digest = hmac.new(key, None, SHA1())

        # feed the content to generated HMAC instance
        hmac_digest.update(content)
        return hmac_digest.digest()

    def __encrypt_with_aes(content, key):
        """ Encrypt content with AES-128-CBC (with PCKS#7 padding)

        @developer: ???

        :param content: string DER-encoded MPContent ASN.1 structure to encrypt
        :param key:     string key to use for encryption

        :return: string encryption of an input content

        """

        pass

    def __decrypt_with_aes(content, key):
        """ Decrypt AES-128-CBC encrypted content (PCKS#7 padded)

        @developer: ???

        :param content: string ciphertext of MPContent ASN.1 structure
        :param key:     string AES secret key

        :return: string decrypted DER-encoded MPContent ASN.1 structure

        """

        pass

    def __encrypt_with_rsa(self, content, recipient_pk):
        """ Encrypt content with RSAES-OAEP scheme

        @developer: vsmysle

        This method handles an encryption of a *single* RSA block with a
        specified above scheme. It does not handle splitting of a header into
        several blocks. It has to be done by other method that would use this
        one only for single block encryption purpose.

        TODO: what is a maximum size of a content that can be padded and
        encrypted given a particular size of RSA key?

        :param content:         string content to encrypt (probably a part of
                                ASN.1 DER-encoded MPHeader block)
        :param recipient_pk:    instance of cryptography.hazmat.primitives.rsa
                                .RSAPublicKey to use for a content encryption

        :return: string encryption of an input content

        """
        # TODO: add exceptions
        self.logger.debug("RSA encryption ...")

        ciphertext = recipient_pk.encrypt(
            content, asym_padding.OAEP(
                mgf=asym_padding.MGF1(algorithm=SHA1()),
                algorithm=SHA1(),
                label=None
            )
        )
        return ciphertext

    def __decrypt_with_rsa(self, content, user_sk):
        """ Decrypt RSAES-OAEP encrypted content (single block)

        @developer: ???

        This method decrypts a single RSA ciphertext block only

        :param content: string content to decrypt
        :param user_sk: instance of cryptography.hazmat.primitives.rsa
                        .RSAPrivateKey to use for a decryption

        :return: string decryption of an input content

        """
        # TODO: add exceptions

        self.logger.debug("RSA decryption ...")

        plaintext = user_sk.decrypt(
            content, asym_padding.OAEP(
                mgf=asym_padding.MGF1(algorithm=SHA1()),
                algorithm=SHA1(),
                label=None
            )
        )

        return plaintext

    def __sign_content(self, content, user_sk):
        """ Produce a signature of an input content using RSASSA-PSS scheme

        @developer: ???

        :param content: string content to sign
        :param user_sk: instance of cryptography.hazmat.primitives.rsa.
                        RSAPrivateKey

        :return: string signature of the input content

        """

        pass

    def __verify_signature(self, signature, signer_pk, content):
        """ Verify RSASSA-PSS signature

        @developer: ???

        :param signature: string signature to verify
        :param signer_pk: instance of cryptography.hazmat.primitives.
                          rsa.RSAPublicKey that is a public key of a signer
        :param content:   content to verify a signature of

        :return: bool verification result

        """

        pass

    def __get_asn1_algorithm_identifier_der(self, oid_str):
        """ Generate ASN.1 structure for algorithm identifier

        @developer: vsmysle

        :param oid_str: string OID to encapsulate

        :return: pyasn1.type.univ.Sequence object

        """

        # TODO: add exceptions

        # create the instance of AlgorithmIdentifier
        self.logger.debug("Receiving ASN.1 AlgorithmIdentifier"
                          " structure with OID=%s") % oid_str
        ai = asn1_dec.AlgorithmIdentifier()

        # set corresponding parameters
        ai['algorithm'] = oid_str
        ai['parameters'] = univ.Null()
        return ai

    def __get_random_bytes(self, spec_lst):
        """ Generate random bytes

        @developer: vsmysle

        :param spec_lst: list of integers that is lengths of bytestings to
                         return

        :return: list of random bytestrings with lengths corresponding to the
                 ones from a spec list

        """

        # TODO: add exception for negative integers

        self.logger.debug("Generating random bytes...")
        return [urandom(i) for i in spec_lst]
<|endoftext|>


<commit_before># Copyright 2018 The Lucid Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""High-level wrapper for paramaterizing images."""

import tensorflow as tf

from lucid.optvis.param.color import to_valid_rgb
from lucid.optvis.param.spatial import pixel_image, fft_image

def image(w, h=None, batch=None, sd=None, decorrelate=True, fft=True, alpha=False):
    h = h or w
    batch = batch or 1
    channels = 4 if alpha else 3
    shape = [batch, h, w, channels]
    param_f = fft_image if fft else pixel_image
    t = param_f(shape, sd=sd)
    rgb = to_valid_rgb(t[..., :3], decorrelate=decorrelate, sigmoid=True)
    if alpha:
        a = tf.nn.sigmoid(t[..., 3:])
        return tf.concat([rgb, a], -1)
    return rgb
<commit_msg>Updated image param to accept any number of channels<commit_after># Copyright 2018 The Lucid Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""High-level wrapper for paramaterizing images."""

import tensorflow as tf

from lucid.optvis.param.color import to_valid_rgb
from lucid.optvis.param.spatial import pixel_image, fft_image

def image(w, h=None, batch=None, sd=None, decorrelate=True, fft=True, alpha=False, channels=None):
    h = h or w
    batch = batch or 1
    ch = channels or (4 if alpha else 3)
    shape = [batch, h, w, ch]
    param_f = fft_image if fft else pixel_image
    t = param_f(shape, sd=sd)
    if channels:
        output = to_valid_rgb(t, decorrelate=False, sigmoid=True)
    else:
        output = to_valid_rgb(t[..., :3], decorrelate=decorrelate, sigmoid=True)
        if alpha:
            a = tf.nn.sigmoid(t[..., 3:])
            return tf.concat([rgb, a], -1)
    return output
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
Script to update Ska file archive aspect L1 products.  Module
also provides methods to retrieve the directory (or directories)
for an obsid.

This uses the obsid_archive module with a configuration specific
to the aspect L1 products.

"""
import os
import logging
import numpy as np
from astropy.table import Table
from Quaternion import Quat

from mica.archive import obsid_archive
from mica.archive import asp_l1_proc
from mica.common import MICA_ARCHIVE

# these columns are available in the headers of the fetched telemetry
# for this product (ASP L1) and will be included in the file lookup table
ARCHFILES_HDR_COLS = ('tstart', 'tstop', 'caldbver', 'content',
                      'ascdsver', 'revision', 'date')

#config = ConfigObj('asp1.conf')
CONFIG = dict(data_root=os.path.join(MICA_ARCHIVE, 'asp1'),
              temp_root=os.path.join(MICA_ARCHIVE, 'temp'),
              bad_obsids=os.path.join(MICA_ARCHIVE, 'asp1', 'asp_l1_bad_obsids.dat'),
              sql_def='archfiles_asp_l1_def.sql',
              apstat_table='aspect_1',
              apstat_id='aspect_1_id',
              label='asp_l1',
              small='asp1{fidprops}',
              small_glob='*fidpr*',
              small_ver_regex=r'pcadf\d+N(\d{3})_',
              delete=['kalm1', 'gdat', 'adat'],
              full='asp1',
              filecheck=False,
              cols=ARCHFILES_HDR_COLS,
              content_types=['ASPQUAL', 'ASPSOL', 'ACADATA', 'GSPROPS',
                             'GYRODATA', 'KALMAN', 'ACACAL', 'ACACENT',
                             'FIDPROPS', 'GYROCAL', 'ACA_BADPIX'])

def get_options():
    import argparse
    desc = \
"""
Run the update process to get new ASP L1 telemetry, save it in the Ska
file archive, and include it in the file lookup database.  This is intended
to be run as a cron task, and in regular processing, the update will fetch
and ingest all telemetry since the task's last run.  Options also provided
to fetch and ingest specific obsids and versions.

See the ``CONFIG`` in the asp_l1.py file and the config description in
obsid_archive for more information on the asp l1 default config if parameters
without command-line options need to be changed.
"""
    parser = argparse.ArgumentParser(description=desc)
    defaults = dict(CONFIG)
    parser.set_defaults(**defaults)
    parser.add_argument("--obsid",
                        type=int,
                        help="specific obsid to process")
    parser.add_argument("--version",
                        default='last',
                        help="specific processing version to retrieve")
    parser.add_argument("--firstrun",
                        action='store_true',
                        help="for archive init., ignore rev in aspect_1 table")
    parser.add_argument("--data-root",
                        help="parent directory for all data")
    parser.add_argument("--temp-root",
                        help="parent temp directory")
    parser.add_argument("--filecheck",
                        action="store_true",
                        help="for provisional data, download files and check"
                        + " that all are present.  If unset, proceed if dir"
                        + " exists")
    parser.add_argument("--rebuild",
                        action="store_true",
                        help="Allow update to rebuild archive from obsid 1")
    opt = parser.parse_args()
    return opt

# set up an archive object with default config for use by the other
# get_* methods
archive = obsid_archive.ObsArchive(CONFIG)

def get_dir(obsid):
    """
    Get ASP L1 directory for default/released products for an obsid.

      >>> from mica.archive import asp_l1
      >>> asp_l1.get_dir(2121)
      '/proj/sot/ska/data/mica/archive/asp1/02/02121'

    :param obsid: obsid
    :returns: directory
    :rtype: string
    """
    return archive.get_dir(obsid)

def get_obs_dirs(obsid):
    """
    Get all ASP L1 directories for an obsid in the Ska file archive.

      >>> from mica.archive import asp_l1
      >>> obsdirs = asp_l1.get_obs_dirs(6000)

    obsdirs will look something like::

      {'default': '/proj/sot/ska/data/mica/archive/asp1/06/06000',
      2: '/proj/sot/ska/data/mica/archive/asp1/06/06000_v02',
      3: '/proj/sot/ska/data/mica/archive/asp1/06/06000_v03',
      'last': '/proj/sot/ska/data/mica/archive/asp1/06/06000',
      'revisions': [2, 3]}

    :param obsid: obsid
    :returns: map of obsid version to directories
    :rtype: dictionary
    """
    return archive.get_obs_dirs(obsid)

def get_files(obsid=None, start=None, stop=None,
              revision=None, content=None):
    """
    List asp_l1 files for an obsid or a time range.

      >>> from mica.archive import asp_l1
      >>> obs_files = asp_l1.get_files(6000)
      >>> obs_gspr = asp_l1.get_files(6000, content=['GSPROPS'])
      >>> range_fidpr = asp_l1.get_files(start='2012:001',
      ...                                stop='2012:030',
      ...                                content=['FIDPROPS'])

    The available content types are: ASPQUAL, ASPSOL, ASPSOLOBI, ACACAL,
    ACA_BADPIX, FIDPROPS, GYROCAL, GSPROPS, and ACACENT.

    :param obsid: obsid
    :param start: time range start (Chandra.Time compatible)
    :param stop: time range stop (Chandra.Time compatible)
    :param revision: revision integer or 'last'
                     defaults to current released version
    :param content: archive CONTENT type
                    defaults to all available ASP1 types
    :returns: full path of files matching query
    """
    return archive.get_files(obsid=obsid, start=start, stop=stop,
                             revision=revision, content=content)

def get_atts(obsid=None, start=None, stop=None, revision=None, filter=True):
    """
    Get the ground aspect solution quaternions and times covering obsid or start to stop,
    in the ACA frame.

    :obsid: obsid
    :start: start time (DateTime compat)
    :stop: stop time (DateTime compat)
    :revision: aspect pipeline processing revision (integer version, None, or 'last')
    :filter: boolean, true means returned values will not include quaternions during times when asp_sol_status is non-zero

    :returns: Nx4 np.array of quaternions, np.array of N times, list of dict with header from each asol file.
    """
    if revision == 'all':
        raise ValueError("revision 'all' doesn't really make sense for this function")
    # These are in time order by default from get_files
    asol_files = get_files(obsid=obsid, start=start, stop=stop,
                           revision=revision, content=['ASPSOL'])
    acal_files = get_files(obsid=obsid, start=start, stop=stop,
                           revision=revision, content=['ACACAL'])
    aqual_files = get_files(obsid=obsid, start=start, stop=stop,
                            revision=revision, content=['ASPQUAL'])
    return get_atts_from_files(asol_files, acal_files, aqual_files, filter=filter)

def get_atts_from_files(asol_files, acal_files, aqual_files, filter=True):
    """
    From ASP1 source files (asol, acal, aqual) get the ground aspect solution quaternions and times covering
    the range of asol_files in the ACA frame.  The asol, acl, and aqual files are assumed to have one-to-one correspondence
    (though the asol to acal times are checked).

    :asol_files: list of aspect asol1 files
    :acal_files: list of acal1 files associated with asol_files
    :aqual_files: list of aqual files associated with asol_files
    :filter: boolean, true means returned values will not include quaternions during times when asp_sol_status is non-zero

    :returns: Nx4 np.array of quaternions, np.array of N times, list of dict with header from each asol file.
    """
    # There should be one asol and one acal file for each aspect interval in the range
    att_chunks = []
    time_chunks = []
    records = []
    for asol_f, acal_f, aqual_f in zip(asol_files, acal_files, aqual_files):
        asol = Table.read(asol_f, hdu=1)
        acal = Table.read(acal_f, hdu=1)
        aqual = Table.read(aqual_f, hdu=1)
        # Check that the time ranges match from the fits headers (meta in the table)
        if not np.allclose(np.array([asol.meta['TSTART'], asol.meta['TSTOP']]),
                           np.array([acal.meta['TSTART'], acal.meta['TSTOP']]),
                           atol=10):
            raise ValueError("ACAL and ASOL have mismatched time ranges")
        if filter and np.any(aqual['asp_sol_status'] != 0):
            # For each sample with bad status find the overlapping time range in asol
            # and remove from set
            for idx in np.flatnonzero(aqual['asp_sol_status']):
                nok = ((asol['time'] >= (aqual['time'][idx] - 1.025))
                       & (asol['time'] <= (aqual['time'][idx] + 1.025)))
                asol = asol[~nok]
        # Make a Nx4 list of the inv misalign quats
        q_mis_inv = np.repeat(Quat(transform=acal['aca_misalign'][0]).inv().q,
                              len(asol)).reshape((4, len(asol))).transpose()
        # Quaternion multiply the asol quats with that inv misalign and save
        # I could also do this with the transform matrix and then only need
        # one accessory quat function.
        q_att_name = 'q_att_raw' if 'q_att_raw' in asol.colnames else 'q_att'
        att_chunks.append((Quat(q=asol[q_att_name]) * Quat(q=q_mis_inv)).q)
        time_chunks.append(np.array(asol['time']))
        records.append(asol.meta)
    if len(att_chunks) > 0 and len(time_chunks) > 0:
        return np.vstack(att_chunks), np.hstack(time_chunks), records
    else:
        return np.array([]), np.array([]), []

def main():
    """
    Run the update process to get new ASP L1 telemetry, save it in the Ska
    file archive, and include it in the file lookup database.
    """
    opt = get_options()
    config = vars(opt)
    archive = obsid_archive.ObsArchive(config)
    archive.logger.setLevel(logging.INFO)
    archive.logger.addHandler(logging.StreamHandler())
    obsids = archive.update()

if __name__ == '__main__':
    main()
<commit_msg>Do transform in get_atts_from_files in more lines<commit_after>#!/usr/bin/env python
# Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
Script to update Ska file archive aspect L1 products.  Module
also provides methods to retrieve the directory (or directories)
for an obsid.

This uses the obsid_archive module with a configuration specific
to the aspect L1 products.

"""
import os
import logging
import numpy as np
from astropy.table import Table
from Quaternion import Quat

from mica.archive import obsid_archive
from mica.archive import asp_l1_proc
from mica.common import MICA_ARCHIVE

# these columns are available in the headers of the fetched telemetry
# for this product (ASP L1) and will be included in the file lookup table
ARCHFILES_HDR_COLS = ('tstart', 'tstop', 'caldbver', 'content',
                      'ascdsver', 'revision', 'date')

#config = ConfigObj('asp1.conf')
CONFIG = dict(data_root=os.path.join(MICA_ARCHIVE, 'asp1'),
              temp_root=os.path.join(MICA_ARCHIVE, 'temp'),
              bad_obsids=os.path.join(MICA_ARCHIVE, 'asp1', 'asp_l1_bad_obsids.dat'),
              sql_def='archfiles_asp_l1_def.sql',
              apstat_table='aspect_1',
              apstat_id='aspect_1_id',
              label='asp_l1',
              small='asp1{fidprops}',
              small_glob='*fidpr*',
              small_ver_regex=r'pcadf\d+N(\d{3})_',
              delete=['kalm1', 'gdat', 'adat'],
              full='asp1',
              filecheck=False,
              cols=ARCHFILES_HDR_COLS,
              content_types=['ASPQUAL', 'ASPSOL', 'ACADATA', 'GSPROPS',
                             'GYRODATA', 'KALMAN', 'ACACAL', 'ACACENT',
                             'FIDPROPS', 'GYROCAL', 'ACA_BADPIX'])

def get_options():
    import argparse
    desc = \
"""
Run the update process to get new ASP L1 telemetry, save it in the Ska
file archive, and include it in the file lookup database.  This is intended
to be run as a cron task, and in regular processing, the update will fetch
and ingest all telemetry since the task's last run.  Options also provided
to fetch and ingest specific obsids and versions.

See the ``CONFIG`` in the asp_l1.py file and the config description in
obsid_archive for more information on the asp l1 default config if parameters
without command-line options need to be changed.
"""
    parser = argparse.ArgumentParser(description=desc)
    defaults = dict(CONFIG)
    parser.set_defaults(**defaults)
    parser.add_argument("--obsid",
                        type=int,
                        help="specific obsid to process")
    parser.add_argument("--version",
                        default='last',
                        help="specific processing version to retrieve")
    parser.add_argument("--firstrun",
                        action='store_true',
                        help="for archive init., ignore rev in aspect_1 table")
    parser.add_argument("--data-root",
                        help="parent directory for all data")
    parser.add_argument("--temp-root",
                        help="parent temp directory")
    parser.add_argument("--filecheck",
                        action="store_true",
                        help="for provisional data, download files and check"
                        + " that all are present.  If unset, proceed if dir"
                        + " exists")
    parser.add_argument("--rebuild",
                        action="store_true",
                        help="Allow update to rebuild archive from obsid 1")
    opt = parser.parse_args()
    return opt

# set up an archive object with default config for use by the other
# get_* methods
archive = obsid_archive.ObsArchive(CONFIG)

def get_dir(obsid):
    """
    Get ASP L1 directory for default/released products for an obsid.

      >>> from mica.archive import asp_l1
      >>> asp_l1.get_dir(2121)
      '/proj/sot/ska/data/mica/archive/asp1/02/02121'

    :param obsid: obsid
    :returns: directory
    :rtype: string
    """
    return archive.get_dir(obsid)

def get_obs_dirs(obsid):
    """
    Get all ASP L1 directories for an obsid in the Ska file archive.

      >>> from mica.archive import asp_l1
      >>> obsdirs = asp_l1.get_obs_dirs(6000)

    obsdirs will look something like::

      {'default': '/proj/sot/ska/data/mica/archive/asp1/06/06000',
      2: '/proj/sot/ska/data/mica/archive/asp1/06/06000_v02',
      3: '/proj/sot/ska/data/mica/archive/asp1/06/06000_v03',
      'last': '/proj/sot/ska/data/mica/archive/asp1/06/06000',
      'revisions': [2, 3]}

    :param obsid: obsid
    :returns: map of obsid version to directories
    :rtype: dictionary
    """
    return archive.get_obs_dirs(obsid)

def get_files(obsid=None, start=None, stop=None,
              revision=None, content=None):
    """
    List asp_l1 files for an obsid or a time range.

      >>> from mica.archive import asp_l1
      >>> obs_files = asp_l1.get_files(6000)
      >>> obs_gspr = asp_l1.get_files(6000, content=['GSPROPS'])
      >>> range_fidpr = asp_l1.get_files(start='2012:001',
      ...                                stop='2012:030',
      ...                                content=['FIDPROPS'])

    The available content types are: ASPQUAL, ASPSOL, ASPSOLOBI, ACACAL,
    ACA_BADPIX, FIDPROPS, GYROCAL, GSPROPS, and ACACENT.

    :param obsid: obsid
    :param start: time range start (Chandra.Time compatible)
    :param stop: time range stop (Chandra.Time compatible)
    :param revision: revision integer or 'last'
                     defaults to current released version
    :param content: archive CONTENT type
                    defaults to all available ASP1 types
    :returns: full path of files matching query
    """
    return archive.get_files(obsid=obsid, start=start, stop=stop,
                             revision=revision, content=content)

def get_atts(obsid=None, start=None, stop=None, revision=None, filter=True):
    """
    Get the ground aspect solution quaternions and times covering obsid or start to stop,
    in the ACA frame.

    :obsid: obsid
    :start: start time (DateTime compat)
    :stop: stop time (DateTime compat)
    :revision: aspect pipeline processing revision (integer version, None, or 'last')
    :filter: boolean, true means returned values will not include quaternions during times when asp_sol_status is non-zero

    :returns: Nx4 np.array of quaternions, np.array of N times, list of dict with header from each asol file.
    """
    if revision == 'all':
        raise ValueError("revision 'all' doesn't really make sense for this function")
    # These are in time order by default from get_files
    asol_files = get_files(obsid=obsid, start=start, stop=stop,
                           revision=revision, content=['ASPSOL'])
    acal_files = get_files(obsid=obsid, start=start, stop=stop,
                           revision=revision, content=['ACACAL'])
    aqual_files = get_files(obsid=obsid, start=start, stop=stop,
                            revision=revision, content=['ASPQUAL'])
    return get_atts_from_files(asol_files, acal_files, aqual_files, filter=filter)

def get_atts_from_files(asol_files, acal_files, aqual_files, filter=True):
    """
    From ASP1 source files (asol, acal, aqual) get the ground aspect solution quaternions and times covering
    the range of asol_files in the ACA frame.  The asol, acl, and aqual files are assumed to have one-to-one correspondence
    (though the asol to acal times are checked).

    :asol_files: list of aspect asol1 files
    :acal_files: list of acal1 files associated with asol_files
    :aqual_files: list of aqual files associated with asol_files
    :filter: boolean, true means returned values will not include quaternions during times when asp_sol_status is non-zero

    :returns: Nx4 np.array of quaternions, np.array of N times, list of dict with header from each asol file.
    """
    # There should be one asol and one acal file for each aspect interval in the range
    att_chunks = []
    time_chunks = []
    records = []
    for asol_f, acal_f, aqual_f in zip(asol_files, acal_files, aqual_files):
        asol = Table.read(asol_f, hdu=1)
        acal = Table.read(acal_f, hdu=1)
        aqual = Table.read(aqual_f, hdu=1)
        # Check that the time ranges match from the fits headers (meta in the table)
        if not np.allclose(np.array([asol.meta['TSTART'], asol.meta['TSTOP']]),
                           np.array([acal.meta['TSTART'], acal.meta['TSTOP']]),
                           atol=10):
            raise ValueError("ACAL and ASOL have mismatched time ranges")
        if filter and np.any(aqual['asp_sol_status'] != 0):
            # For each sample with bad status find the overlapping time range in asol
            # and remove from set
            for idx in np.flatnonzero(aqual['asp_sol_status']):
                nok = ((asol['time'] >= (aqual['time'][idx] - 1.025))
                       & (asol['time'] <= (aqual['time'][idx] + 1.025)))
                asol = asol[~nok]

        # Transpose of transform/rotation matrix is the inverse
        aca_mis_inv = acal['aca_misalign'][0].transpose()

        # Repeat the transform to match asol
        aca_mis_inv = np.repeat(aca_mis_inv[np.newaxis, ...], repeats=len(asol), axis=0)
        q_mis_inv = Quat(transform=aca_mis_inv)

        # Quaternion multiply the asol quats with that inv misalign and save
        q_att_name = 'q_att_raw' if 'q_att_raw' in asol.colnames else 'q_att'
        att_chunks.append((Quat(q=asol[q_att_name]) * q_mis_inv).q)
        time_chunks.append(np.array(asol['time']))
        records.append(asol.meta)
    if len(att_chunks) > 0 and len(time_chunks) > 0:
        return np.vstack(att_chunks), np.hstack(time_chunks), records
    else:
        return np.array([]), np.array([]), []

def main():
    """
    Run the update process to get new ASP L1 telemetry, save it in the Ska
    file archive, and include it in the file lookup database.
    """
    opt = get_options()
    config = vars(opt)
    archive = obsid_archive.ObsArchive(config)
    archive.logger.setLevel(logging.INFO)
    archive.logger.addHandler(logging.StreamHandler())
    obsids = archive.update()

if __name__ == '__main__':
    main()
<|endoftext|>