<commit_before># -*- coding: UTF-8 -*-
from collections import defaultdict
from conference.models import Ticket, Speaker
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.db.models import Q, Count
from p3 import models

def _tickets(conf, ticket_type=None, fare_code=None):
    qs = Ticket.objects\
        .filter(
            orderitem__order___complete=True,
            fare__conference=conf,
        )
    if ticket_type:
        qs = qs.filter(fare__ticket_type=ticket_type)
    if fare_code:
        if fare_code.endswith('%'):
            qs = qs.filter(fare__code__startswith=fare_code[:-1])
        else:
            qs = qs.filter(fare__code=fare_code)
    return qs

def _compiled(conf, ticket_type='conference'):
    return _tickets(conf, ticket_type)\
        .exclude(p3_conference=None)\
        .exclude(name='')

def _not_compiled(conf, ticket_type='conference'):
    return _tickets(conf, ticket_type)\
        .filter(Q(p3_conference=None)|Q(name=''))

def shirt_sizes(conf):
    sizes = dict(models.TICKET_CONFERENCE_SHIRT_SIZES)
    qs = _compiled(conf)\
        .values('p3_conference__shirt_size')\
        .annotate(total=Count('id'))

    output = []
    for x in qs:
        output.append({
            'title': sizes.get(x['p3_conference__shirt_size']),
            'total': x['total'],
        })
    return output
shirt_sizes.short_description = "Taglie maglette (solo biglietti compilati)"

def diet_types(conf):
    diets = dict(models.TICKET_CONFERENCE_DIETS)
    qs = _compiled(conf)\
        .values('p3_conference__diet')\
        .annotate(total=Count('id'))

    output = []
    for x in qs:
        output.append({
            'title': diets.get(x['p3_conference__diet']),
            'total': x['total'],
        })
    return output
diet_types.short_description = "Dieta partecipanti (solo biglietti compilati)"

def presence_days(conf):
    totals = {
        'c': _compiled(conf).count(),
        'n': _not_compiled(conf).count(),
    }
    output = {
        'columns': (
            ('total', 'Totale'),
            ('total_nc', 'Proiezione NC'),
        ),
        'data': [],
    }

    days = defaultdict(lambda: 0)
    qs = _compiled(conf)\
        .values_list('p3_conference__days', flat=True)
    for x in qs:
        val = filter(None, map(lambda v: v.strip(), x.split(',')))
        if not val:
            days['x'] += 1
        else:
            for v in val:
                days[v] += 1

    for day, count in sorted(days.items()):
        if day != 'x':
            nc = float(count) / (totals['c'] - days['x']) * (totals['c'] + totals['n'])
        else:
            nc = 0.0
        output['data'].append({
            'title': day,
            'total': count,
            'total_nc': int(round(nc)),
        })
    return output
presence_days.short_description = "Affluenza per giorno (solo biglietti compilati)"

def tickets_status(conf, code=None):
    orphan_tickets = _tickets(conf, 'conference')\
        .filter(p3_conference__isnull=False)\
        .exclude(p3_conference__assigned_to='')\
        .exclude(p3_conference__assigned_to__in=User.objects.values('email'))
    multiple_assignments = _tickets(conf, 'conference')\
        .exclude(Q(p3_conference=None)|Q(p3_conference__assigned_to=''))\
        .values('p3_conference__assigned_to')\
        .annotate(total=Count('id'))\
        .filter(total__gt=1)
    if code is None:
        output = [
            {
                'id': 'ticket_sold',
                'title': 'Venduti',
                'total': _tickets(conf, 'conference').count(),
            },
            {
                'title': 'Compilati',
                'total': _compiled(conf).count(),
            },
            {
                'title': 'Non compilati',
                'total': _not_compiled(conf).count(),
            }
        ]
        qs = _tickets(conf, fare_code='SIM%')\
            .filter(Q(p3_conference_sim=None)|Q(name='')|Q(p3_conference_sim__document=''))\
            .select_related('p3_conference_sim')
        output.append({
            'title': 'SIM non compilati',
            'total': qs.count(),
        })

        qs = _tickets(conf, 'conference')\
            .exclude(Q(p3_conference=None)|Q(p3_conference__assigned_to=''))
        output.append({
            'title': 'Biglietti assegnati',
            'total': qs.count(),
        })

        output.append({
            'id': 'multiple_assignments',
            'title': 'Biglietti multipli assegnati alla stessa persona',
            'total': multiple_assignments.count(),
        })

        output.append({
            'id': 'orphan_tickets',
            'title': 'Biglietti assegnati orfani',
            'total': orphan_tickets.count(),
        })
    else:
        if code == 'ticket_sold':
            output = {
                'columns': (
                    ('name', 'Name'),
                    ('email', 'Email'),
                    ('buyer', 'Buyer'),
                    ('buyer_email', 'Buyer Email'),
                ),
                'data': [],
            }
            qs = _tickets(conf, 'conference')\
                .select_related('p3_conference', 'user')
            assignees = dict([
                (u.email, u) for u in User.objects\
                    .filter(email__in=qs\
                        .exclude(p3_conference__assigned_to='')\
                        .values('p3_conference__assigned_to'))])
            data = output['data']
            for x in qs:
                # p3_conference può essere None perché viene costruito lazy al
                # primo salvataggio del biglietto.
                if x.p3_conference and x.p3_conference.assigned_to:
                    email = x.p3_conference.assigned_to
                    u = assignees.get(email)
                else:
                    email = x.user.email
                    u = x.user
                if u:
                    name = '%s %s' % (
                        reverse('admin:auth_user_change', args=(u.id,)),
                        u.first_name,
                        u.last_name)
                    order = u.first_name + u.last_name
                else:
                    name = '%s Biglietto orfano' % x.name
                    order = x.name
                if x.user == u:
                    buyer = ''
                else:
                    buyer = '%s %s' % (
                        reverse('admin:auth_user_change', args=(x.user.id,)),
                        x.user.first_name,
                        x.user.last_name)
                    if not order:
                        order = x.user.first_name + x.user.last_name
                if x.user.email == email:
                    buyer_email = ''
                else:
                    buyer_email = x.user.email
                row = {
                    'name': name,
                    'email': email,
                    'buyer': buyer,
                    'buyer_email': buyer_email,
                    '_order': order,
                    'uid': (u or x.user).id,
                }
                data.append(row)
            data.sort(key=lambda x: x['_order'])
        elif code in ('orphan_tickets', 'multiple_assignments'):
            output = {
                'columns': (
                    ('name', 'Name'),
                    ('email', 'Email'),
                    ('fare', 'Fare code'),
                    ('buyer', 'Buyer'),
                    ('buyer_email', 'Buyer Email'),
                ),
                'data': [],
            }
            if code == 'orphan_tickets':
                qs = orphan_tickets
            else:
                qs = _tickets(conf, 'conference')\
                    .filter(p3_conference__assigned_to__in=multiple_assignments\
                        .values('p3_conference__assigned_to'))
            qs = qs.select_related('p3_conference', 'user', 'fare')
            data = output['data']
            for x in qs:
                buyer = '%s %s' % (
                    reverse('admin:auth_user_change', args=(x.user.id,)),
                    x.user.first_name,
                    x.user.last_name)
                data.append({
                    'name': x.name,
                    'email': x.p3_conference.assigned_to,
                    'fare': x.fare.code,
                    'buyer': buyer,
                    'buyer_email': x.user.email,
                })
    return output
tickets_status.short_description = 'Statistiche biglietti'

def speaker_status(conf, code=None):
    t = _tickets(conf, 'conference')
    spk_noticket = Speaker.objects.byConference(conf)\
        .exclude(user__in=t.values('user'))\
        .exclude(user__email__in=t.extra(where=["assigned_to!=''"]).values('p3_conference__assigned_to'))
    spk_nodata = Speaker.objects.byConference(conf)\
        .filter(Q(
                user__attendeeprofile__image='',
                user__attendeeprofile__p3_profile__image_gravatar=False,
                user__attendeeprofile__p3_profile__image_url='')
            | Q(user__attendeeprofile__bios__language=None)
            | Q(
                user__attendeeprofile__bios__language='en',
                user__attendeeprofile__bios__content='bios',
                user__attendeeprofile__bios__body='')
            )
    if code is None:
        output = {
            'columns': (
                ('total', 'Total'),
                ('note', 'Note'),
            ),
            'data': [
                {
                    'id': 'no_ticket',
                    'title': 'Senza biglietto',
                    'total': spk_noticket.count(),
                    'note': '',
                },
                {
                    'id': 'no_data',
                    'title': 'Senza avatar o biografia',
                    'total': spk_nodata.count(),
                    'note': 'Non vengono inclusi gli utenti con gravatar (ma l\'avatar potrebbe non essere significativo)',
                }
            ]
        }
    else:
        if code == 'no_ticket':
            qs = spk_noticket
        elif code == 'no_data':
            qs = spk_nodata
        output = {
            'columns': (
                ('name', 'Name'),
                ('email', 'Email'),
            ),
            'data': [],
        }
        data = output['data']
        qs = qs\
            .select_related('user')\
            .order_by('user__first_name', 'user__last_name')
        for x in qs:
            data.append({
                'name': '%s %s' % (
                    reverse('admin:auth_user_change', args=(x.user_id,)),
                    x.user.first_name,
                    x.user.last_name),
                'email': x.user.email,
                'uid': x.user_id,
            })
    return output
speaker_status.short_description = 'Statistiche speaker'
<commit_msg>dettagli per statistica sim incomplete<commit_after># -*- coding: UTF-8 -*-
from collections import defaultdict
from conference.models import Ticket, Speaker
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.db.models import Q, Count
from p3 import models

def _tickets(conf, ticket_type=None, fare_code=None):
    qs = Ticket.objects\
        .filter(
            orderitem__order___complete=True,
            fare__conference=conf,
        )
    if ticket_type:
        qs = qs.filter(fare__ticket_type=ticket_type)
    if fare_code:
        if fare_code.endswith('%'):
            qs = qs.filter(fare__code__startswith=fare_code[:-1])
        else:
            qs = qs.filter(fare__code=fare_code)
    return qs

def _compiled(conf, ticket_type='conference'):
    return _tickets(conf, ticket_type)\
        .exclude(p3_conference=None)\
        .exclude(name='')

def _not_compiled(conf, ticket_type='conference'):
    return _tickets(conf, ticket_type)\
        .filter(Q(p3_conference=None)|Q(name=''))

def shirt_sizes(conf):
    sizes = dict(models.TICKET_CONFERENCE_SHIRT_SIZES)
    qs = _compiled(conf)\
        .values('p3_conference__shirt_size')\
        .annotate(total=Count('id'))

    output = []
    for x in qs:
        output.append({
            'title': sizes.get(x['p3_conference__shirt_size']),
            'total': x['total'],
        })
    return output
shirt_sizes.short_description = "Taglie maglette (solo biglietti compilati)"

def diet_types(conf):
    diets = dict(models.TICKET_CONFERENCE_DIETS)
    qs = _compiled(conf)\
        .values('p3_conference__diet')\
        .annotate(total=Count('id'))

    output = []
    for x in qs:
        output.append({
            'title': diets.get(x['p3_conference__diet']),
            'total': x['total'],
        })
    return output
diet_types.short_description = "Dieta partecipanti (solo biglietti compilati)"

def presence_days(conf):
    totals = {
        'c': _compiled(conf).count(),
        'n': _not_compiled(conf).count(),
    }
    output = {
        'columns': (
            ('total', 'Totale'),
            ('total_nc', 'Proiezione NC'),
        ),
        'data': [],
    }

    days = defaultdict(lambda: 0)
    qs = _compiled(conf)\
        .values_list('p3_conference__days', flat=True)
    for x in qs:
        val = filter(None, map(lambda v: v.strip(), x.split(',')))
        if not val:
            days['x'] += 1
        else:
            for v in val:
                days[v] += 1

    for day, count in sorted(days.items()):
        if day != 'x':
            nc = float(count) / (totals['c'] - days['x']) * (totals['c'] + totals['n'])
        else:
            nc = 0.0
        output['data'].append({
            'title': day,
            'total': count,
            'total_nc': int(round(nc)),
        })
    return output
presence_days.short_description = "Affluenza per giorno (solo biglietti compilati)"

def tickets_status(conf, code=None):
    orphan_tickets = _tickets(conf, 'conference')\
        .filter(p3_conference__isnull=False)\
        .exclude(p3_conference__assigned_to='')\
        .exclude(p3_conference__assigned_to__in=User.objects.values('email'))
    multiple_assignments = _tickets(conf, 'conference')\
        .exclude(Q(p3_conference=None)|Q(p3_conference__assigned_to=''))\
        .values('p3_conference__assigned_to')\
        .annotate(total=Count('id'))\
        .filter(total__gt=1)
    sim_tickets = _tickets(conf, fare_code='SIM%')\
        .filter(Q(p3_conference_sim=None)|Q(name='')|Q(p3_conference_sim__document=''))\
        .select_related('p3_conference_sim')
    if code is None:
        output = [
            {
                'id': 'ticket_sold',
                'title': 'Venduti',
                'total': _tickets(conf, 'conference').count(),
            },
            {
                'title': 'Compilati',
                'total': _compiled(conf).count(),
            },
            {
                'title': 'Non compilati',
                'total': _not_compiled(conf).count(),
            },
            {
                'id': 'sim_tickets',
                'title': 'SIM non compilati',
                'total': sim_tickets.count(),
            },
        ]

        qs = _tickets(conf, 'conference')\
            .exclude(Q(p3_conference=None)|Q(p3_conference__assigned_to=''))
        output.append({
            'title': 'Biglietti assegnati',
            'total': qs.count(),
        })

        output.append({
            'id': 'multiple_assignments',
            'title': 'Biglietti multipli assegnati alla stessa persona',
            'total': multiple_assignments.count(),
        })

        output.append({
            'id': 'orphan_tickets',
            'title': 'Biglietti assegnati orfani',
            'total': orphan_tickets.count(),
        })
    else:
        if code == 'ticket_sold':
            output = {
                'columns': (
                    ('name', 'Name'),
                    ('email', 'Email'),
                    ('buyer', 'Buyer'),
                    ('buyer_email', 'Buyer Email'),
                ),
                'data': [],
            }
            qs = _tickets(conf, 'conference')\
                .select_related('p3_conference', 'user')
            assignees = dict([
                (u.email, u) for u in User.objects\
                    .filter(email__in=qs\
                        .exclude(p3_conference__assigned_to='')\
                        .values('p3_conference__assigned_to'))])
            data = output['data']
            for x in qs:
                # p3_conference può essere None perché viene costruito lazy al
                # primo salvataggio del biglietto.
                if x.p3_conference and x.p3_conference.assigned_to:
                    email = x.p3_conference.assigned_to
                    u = assignees.get(email)
                else:
                    email = x.user.email
                    u = x.user
                if u:
                    name = '%s %s' % (
                        reverse('admin:auth_user_change', args=(u.id,)),
                        u.first_name,
                        u.last_name)
                    order = u.first_name + u.last_name
                else:
                    name = '%s Biglietto orfano' % x.name
                    order = x.name
                if x.user == u:
                    buyer = ''
                else:
                    buyer = '%s %s' % (
                        reverse('admin:auth_user_change', args=(x.user.id,)),
                        x.user.first_name,
                        x.user.last_name)
                    if not order:
                        order = x.user.first_name + x.user.last_name
                if x.user.email == email:
                    buyer_email = ''
                else:
                    buyer_email = x.user.email
                row = {
                    'name': name,
                    'email': email,
                    'buyer': buyer,
                    'buyer_email': buyer_email,
                    '_order': order,
                    'uid': (u or x.user).id,
                }
                data.append(row)
            data.sort(key=lambda x: x['_order'])
        elif code in ('orphan_tickets', 'multiple_assignments'):
            output = {
                'columns': (
                    ('name', 'Name'),
                    ('email', 'Email'),
                    ('fare', 'Fare code'),
                    ('buyer', 'Buyer'),
                    ('buyer_email', 'Buyer Email'),
                ),
                'data': [],
            }
            if code == 'orphan_tickets':
                qs = orphan_tickets
            else:
                qs = _tickets(conf, 'conference')\
                    .filter(p3_conference__assigned_to__in=multiple_assignments\
                        .values('p3_conference__assigned_to'))
            qs = qs.select_related('p3_conference', 'user', 'fare')
            data = output['data']
            for x in qs:
                buyer = '%s %s' % (
                    reverse('admin:auth_user_change', args=(x.user.id,)),
                    x.user.first_name,
                    x.user.last_name)
                data.append({
                    'name': x.name,
                    'email': x.p3_conference.assigned_to,
                    'fare': x.fare.code,
                    'buyer': buyer,
                    'buyer_email': x.user.email,
                })
        elif code in ('sim_tickets',):
            output = {
                'columns': (
                    ('name', 'Name'),
                    ('email', 'Email'),
                    ('fare', 'Fare code'),
                ),
                'data': [],
            }
            if code == 'sim_tickets':
                qs = sim_tickets
            qs = qs.select_related('user', 'fare')
            data = output['data']
            for x in qs:
                buyer = '%s %s' % (
                    reverse('admin:auth_user_change', args=(x.user.id,)),
                    x.user.first_name,
                    x.user.last_name)
                data.append({
                    'name': buyer,
                    'email': x.user.email,
                    'fare': x.fare.code,
                    'uid': x.user.id,
                })
    return output
tickets_status.short_description = 'Statistiche biglietti'

def speaker_status(conf, code=None):
    t = _tickets(conf, 'conference')
    spk_noticket = Speaker.objects.byConference(conf)\
        .exclude(user__in=t.values('user'))\
        .exclude(user__email__in=t.extra(where=["assigned_to!=''"]).values('p3_conference__assigned_to'))
    spk_nodata = Speaker.objects.byConference(conf)\
        .filter(Q(
                user__attendeeprofile__image='',
                user__attendeeprofile__p3_profile__image_gravatar=False,
                user__attendeeprofile__p3_profile__image_url='')
            | Q(user__attendeeprofile__bios__language=None)
            | Q(
                user__attendeeprofile__bios__language='en',
                user__attendeeprofile__bios__content='bios',
                user__attendeeprofile__bios__body='')
            )
    if code is None:
        output = {
            'columns': (
                ('total', 'Total'),
                ('note', 'Note'),
            ),
            'data': [
                {
                    'id': 'no_ticket',
                    'title': 'Senza biglietto',
                    'total': spk_noticket.count(),
                    'note': '',
                },
                {
                    'id': 'no_data',
                    'title': 'Senza avatar o biografia',
                    'total': spk_nodata.count(),
                    'note': 'Non vengono inclusi gli utenti con gravatar (ma l\'avatar potrebbe non essere significativo)',
                }
            ]
        }
    else:
        if code == 'no_ticket':
            qs = spk_noticket
        elif code == 'no_data':
            qs = spk_nodata
        output = {
            'columns': (
                ('name', 'Name'),
                ('email', 'Email'),
            ),
            'data': [],
        }
        data = output['data']
        qs = qs\
            .select_related('user')\
            .order_by('user__first_name', 'user__last_name')
        for x in qs:
            data.append({
                'name': '%s %s' % (
                    reverse('admin:auth_user_change', args=(x.user_id,)),
                    x.user.first_name,
                    x.user.last_name),
                'email': x.user.email,
                'uid': x.user_id,
            })
    return output
speaker_status.short_description = 'Statistiche speaker'
<|endoftext|>


<commit_before>#!/usr/bin/env python

from distutils.spawn import find_executable
from kvirt.common import info, pprint, pwd_path, get_kubectl
from kvirt.defaults import UBUNTUS
import os
import sys

def scale(config, plandir, cluster, overrides):
    data = {'cluster': cluster}
    data.update(overrides)
    data['basedir'] = '/workdir' if os.path.exists("/i_am_a_container") else '.'
    cluster = data.get('cluster')
    client = config.client
    k = config.k
    pprint("Scaling on client %s" % client, color='blue')
    image = k.info("%s-master-0" % cluster).get('image')
    if image is None:
        pprint("Missing image...", color='red')
        sys.exit(1)
    else:
        pprint("Using image %s" % image, color='blue')
    data['image'] = image
    config.plan(cluster, inputfile='%s/workers.yml' % plandir, overrides=data)

def create(config, plandir, cluster, overrides):
    data = {'kubetype': 'generic'}
    data.update(overrides)
    data['cluster'] = overrides['cluster'] if 'cluster' in overrides else cluster
    data['kube'] = data['cluster']
    masters = data.get('masters', 1)
    if masters == 0:
        pprint("Invalid number of masters", color='red')
        os._exit(1)
    version = data.get('version')
    if version is not None and not version.startswith('1.'):
        pprint("Invalid version %s" % version, color='red')
        os._exit(1)
    data['basedir'] = '/workdir' if os.path.exists("/i_am_a_container") else '.'
    cluster = data.get('cluster')
    image = data.get('image', 'centos7')
    domain = data.get('domain', 'karmalabs.com')
    clusterdir = pwd_path("clusters/%s" % cluster)
    firstmaster = "%s-master-0" % cluster
    if os.path.exists(clusterdir):
        pprint("Please remove existing directory %s first..." % clusterdir, color='red')
        sys.exit(1)
    if find_executable('kubectl') is None:
        get_kubectl()
    if not os.path.exists(clusterdir):
        os.makedirs(clusterdir)
        os.mkdir("%s/auth" % clusterdir)
    k = config.k
    result = config.plan(cluster, inputfile='%s/masters.yml' % plandir, overrides=data, wait=True)
    if result['result'] != "success":
        os._exit(1)
    source, destination = "/root/join.sh", "%s/join.sh" % clusterdir
    scpcmd = k.scp(firstmaster, user='root', source=source, destination=destination, tunnel=config.tunnel,
                   tunnelhost=config.tunnelhost, tunnelport=config.tunnelport, tunneluser=config.tunneluser,
                   download=True, insecure=True)
    os.system(scpcmd)
    source, destination = "/etc/kubernetes/admin.conf", "%s/auth/kubeconfig" % clusterdir
    scpcmd = k.scp(firstmaster, user='root', source=source, destination=destination, tunnel=config.tunnel,
                   tunnelhost=config.tunnelhost, tunnelport=config.tunnelport, tunneluser=config.tunneluser,
                   download=True, insecure=True)
    os.system(scpcmd)
    workers = data.get('workers', 0)
    if workers > 0:
        pprint("Deploying workers", color='blue')
        if 'name' in data:
            del data['name']
        config.plan(cluster, inputfile='%s/workers.yml' % plandir, overrides=data)
    pprint("Kubernetes cluster %s deployed!!!" % cluster)
    masters = data.get('masters', 1)
    master_node = "%s-master" % cluster if masters > 1 else "%s-master-0" % cluster
    master_ip = k.info(master_node)['ip']
    info("Create the following /etc/hosts entry if needed")
    info("%s %s %s.%s" % (master_ip, master_node, master_node, domain))
    info("Use The following command to interact with this cluster")
    info("export KUBECONFIG=clusters/%s/auth/kubeconfig" % cluster)
    info("export PATH=$PWD:$PATH")
    prefile = 'pre_ubuntu.sh' if image in UBUNTUS else 'pre_el.sh'
    predata = config.process_inputfile(cluster, "%s/%s" % (plandir, prefile), overrides=data)
    with open("%s/pre.sh" % clusterdir, 'w') as f:
        f.write(predata)
<commit_msg>kubeadm: only indicate /etc/hosts entry when more than one master<commit_after>#!/usr/bin/env python

from distutils.spawn import find_executable
from kvirt.common import info, pprint, pwd_path, get_kubectl
from kvirt.defaults import UBUNTUS
import os
import sys

def scale(config, plandir, cluster, overrides):
    data = {'cluster': cluster}
    data.update(overrides)
    data['basedir'] = '/workdir' if os.path.exists("/i_am_a_container") else '.'
    cluster = data.get('cluster')
    client = config.client
    k = config.k
    pprint("Scaling on client %s" % client, color='blue')
    image = k.info("%s-master-0" % cluster).get('image')
    if image is None:
        pprint("Missing image...", color='red')
        sys.exit(1)
    else:
        pprint("Using image %s" % image, color='blue')
    data['image'] = image
    config.plan(cluster, inputfile='%s/workers.yml' % plandir, overrides=data)

def create(config, plandir, cluster, overrides):
    data = {'kubetype': 'generic'}
    data.update(overrides)
    data['cluster'] = overrides['cluster'] if 'cluster' in overrides else cluster
    data['kube'] = data['cluster']
    masters = data.get('masters', 1)
    if masters == 0:
        pprint("Invalid number of masters", color='red')
        os._exit(1)
    version = data.get('version')
    if version is not None and not version.startswith('1.'):
        pprint("Invalid version %s" % version, color='red')
        os._exit(1)
    data['basedir'] = '/workdir' if os.path.exists("/i_am_a_container") else '.'
    cluster = data.get('cluster')
    image = data.get('image', 'centos7')
    domain = data.get('domain', 'karmalabs.com')
    clusterdir = pwd_path("clusters/%s" % cluster)
    firstmaster = "%s-master-0" % cluster
    if os.path.exists(clusterdir):
        pprint("Please remove existing directory %s first..." % clusterdir, color='red')
        sys.exit(1)
    if find_executable('kubectl') is None:
        get_kubectl()
    if not os.path.exists(clusterdir):
        os.makedirs(clusterdir)
        os.mkdir("%s/auth" % clusterdir)
    k = config.k
    result = config.plan(cluster, inputfile='%s/masters.yml' % plandir, overrides=data, wait=True)
    if result['result'] != "success":
        os._exit(1)
    source, destination = "/root/join.sh", "%s/join.sh" % clusterdir
    scpcmd = k.scp(firstmaster, user='root', source=source, destination=destination, tunnel=config.tunnel,
                   tunnelhost=config.tunnelhost, tunnelport=config.tunnelport, tunneluser=config.tunneluser,
                   download=True, insecure=True)
    os.system(scpcmd)
    source, destination = "/etc/kubernetes/admin.conf", "%s/auth/kubeconfig" % clusterdir
    scpcmd = k.scp(firstmaster, user='root', source=source, destination=destination, tunnel=config.tunnel,
                   tunnelhost=config.tunnelhost, tunnelport=config.tunnelport, tunneluser=config.tunneluser,
                   download=True, insecure=True)
    os.system(scpcmd)
    workers = data.get('workers', 0)
    if workers > 0:
        pprint("Deploying workers", color='blue')
        if 'name' in data:
            del data['name']
        config.plan(cluster, inputfile='%s/workers.yml' % plandir, overrides=data)
    pprint("Kubernetes cluster %s deployed!!!" % cluster)
    masters = data.get('masters', 1)
    if masters > 1:
        master_node = "%s-master" % cluster if masters > 1 else "%s-master-0" % cluster
        master_ip = k.info(master_node)['ip']
        info("Create the following /etc/hosts entry")
        info("%s %s %s.%s" % (master_ip, master_node, master_node, domain))
        info("Use The following command to interact with this cluster")
    info("export KUBECONFIG=clusters/%s/auth/kubeconfig" % cluster)
    info("export PATH=$PWD:$PATH")
    prefile = 'pre_ubuntu.sh' if image in UBUNTUS else 'pre_el.sh'
    predata = config.process_inputfile(cluster, "%s/%s" % (plandir, prefile), overrides=data)
    with open("%s/pre.sh" % clusterdir, 'w') as f:
        f.write(predata)
<|endoftext|>


<commit_before>"""
WSGI config for imageboard project.

This module contains the WSGI application used by Django's development server
and any production WSGI deployments. It should expose a module-level variable
named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover
this application via the ``WSGI_APPLICATION`` setting.

Usually you will have the standard Django WSGI application here, but it also
might make sense to replace the whole Django WSGI application with a custom one
that later delegates to the Django one. For example, you could introduce WSGI
middleware here, or combine a Django application with an application of another
framework.
"""
import os
#from os.path import abspath, dirname
#from sys import path

SITE_ROOT = dirname(dirname(abspath(__file__)))
path.append(SITE_ROOT)

# We defer to a DJANGO_SETTINGS_MODULE already in the environment. This breaks
# if running multiple sites in the same mod_wsgi process. To fix this, use
# mod_wsgi daemon mode with each site in its own daemon process, or use
# os.environ["DJANGO_SETTINGS_MODULE"] = "jajaja.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "imageboard.settings.production")

# This application object is used by any WSGI server configured to use this
# file. This includes Django's development server, if the WSGI_APPLICATION
# setting points here.
from django.core.wsgi import get_wsgi_application
from dj_static import Cling
# source: https://devcenter.heroku.com/articles/getting-started-with-django#wsgi-py
application = Cling(get_wsgi_application())

# Apply WSGI middleware here.
# from helloworld.wsgi import HelloWorldApplication
# application = HelloWorldApplication(application)
<commit_msg>Changed base settings to fix a problem in production environment version 9<commit_after>"""
WSGI config for imageboard project.

This module contains the WSGI application used by Django's development server
and any production WSGI deployments. It should expose a module-level variable
named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover
this application via the ``WSGI_APPLICATION`` setting.

Usually you will have the standard Django WSGI application here, but it also
might make sense to replace the whole Django WSGI application with a custom one
that later delegates to the Django one. For example, you could introduce WSGI
middleware here, or combine a Django application with an application of another
framework.
"""
import os
from os.path import abspath, dirname
from sys import path

SITE_ROOT = dirname(dirname(abspath(__file__)))
path.append(SITE_ROOT)

# We defer to a DJANGO_SETTINGS_MODULE already in the environment. This breaks
# if running multiple sites in the same mod_wsgi process. To fix this, use
# mod_wsgi daemon mode with each site in its own daemon process, or use
# os.environ["DJANGO_SETTINGS_MODULE"] = "jajaja.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "imageboard.settings.production")

# This application object is used by any WSGI server configured to use this
# file. This includes Django's development server, if the WSGI_APPLICATION
# setting points here.
from django.core.wsgi import get_wsgi_application
from dj_static import Cling
# source: https://devcenter.heroku.com/articles/getting-started-with-django#wsgi-py
application = Cling(get_wsgi_application())

# Apply WSGI middleware here.
# from helloworld.wsgi import HelloWorldApplication
# application = HelloWorldApplication(application)
<|endoftext|>


<commit_before>import argparse
import json
import requests

parser = argparse.ArgumentParser(description='Blind SQL Helper')
parser.add_argument('--mode', choices=['integer', 'character'], default='character')
parser.add_argument('--min', default=0, type=int)
parser.add_argument('--max', default=255, type=int)
parser.add_argument('--debug', action='store_true')
parser.add_argument('--cookies', type=json.loads)
parser.add_argument('--url', required=True)
parser.add_argument('--detect', required=True)

args = parser.parse_args()

if args.mode == 'integer':
	now_min = args.min
	now_max = args.max

	while now_min <= now_max:
		t = (now_min + now_max) >> 1
		temp_url = args.url.replace('@@@', str(t))
		r = requests.get(temp_url, cookies=args.cookies)
		if args.debug:
			print(r.text)
		if args.detect in r.text:
			print('%d success' % t)
			now_min = t + 1
		else:
			print('%d fail' % t)
			now_max = t - 1
	print(now_min - 1)
elif args.mode == 'character':
	key = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
	for c in key:
		temp_url = args.url.replace('@@@', c)
		r = requests.get(temp_url, cookies=args.cookies)
		if args.debug:
			print(r.text)
		if args.detect in r.text:
			print(c)
			break
		else:
			print('%c fail' % c)
<commit_msg>change character order<commit_after>import argparse
import json
import requests

parser = argparse.ArgumentParser(description='Blind SQL Helper')
parser.add_argument('--mode', choices=['integer', 'character'], default='character')
parser.add_argument('--min', default=0, type=int)
parser.add_argument('--max', default=255, type=int)
parser.add_argument('--debug', action='store_true')
parser.add_argument('--cookies', type=json.loads)
parser.add_argument('--url', required=True)
parser.add_argument('--detect', required=True)

args = parser.parse_args()

if args.mode == 'integer':
	now_min = args.min
	now_max = args.max

	while now_min <= now_max:
		t = (now_min + now_max) >> 1
		temp_url = args.url.replace('@@@', str(t))
		r = requests.get(temp_url, cookies=args.cookies)
		if args.debug:
			print(r.text)
		if args.detect in r.text:
			print('%d success' % t)
			now_min = t + 1
		else:
			print('%d fail' % t)
			now_max = t - 1
	print(now_min - 1)
elif args.mode == 'character':
	key = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
	for c in key:
		temp_url = args.url.replace('@@@', c)
		r = requests.get(temp_url, cookies=args.cookies)
		if args.debug:
			print(r.text)
		if args.detect in r.text:
			print(c)
			break
		else:
			print('%c fail' % c)
<|endoftext|>


<commit_before># Copyright 2019-present Kensho Technologies, LLC.
from collections import namedtuple

from graphql import build_ast_schema, parse
from graphql.language.printer import print_ast
from graphql.language.visitor import Visitor, visit
import six

from .utils import (
    SchemaNameConflictError, check_ast_schema_is_valid, check_type_name_is_valid,
    get_query_type_name, get_scalar_names
)

RenamedSchemaDescriptor = namedtuple(
    'RenamedSchemaDescriptor', (
        'schema_ast',  # Document, AST representing the renamed schema
        'reverse_name_map',  # Dict[str, str], renamed type/query type field name to original name
        # reverse_name_map only contains names that were changed
    )
)

def rename_schema(ast, renamings):
    """Create a RenamedSchemaDescriptor; types and query type fields are renamed using renamings.

    Any type, interface, enum, or fields of the root type/query type whose name
    appears in renamings will be renamed to the corresponding value. Any such names that do not
    appear in renamings will be unchanged. Scalars, directives, enum values, and fields not
    belonging to the root/query type will never be renamed.

    Args:
        ast: Document, representing a valid schema that does not contain extensions, input
             object definitions, mutations, or subscriptions, whose fields of the query type share
             the same name as the types they query. Not modified by this function
        renamings: Dict[str, str], mapping original type/field names to renamed type/field names.
                   Type or query type field names that do not appear in the dict will be unchanged.
                   Any dict-like object that implements get(key, [default]) may also be used

    Returns:
        RenamedSchemaDescriptor, a namedtuple that contains the AST of the renamed schema, and the
        map of renamed type/field names to original names. Only renamed names will be included
        in the map.

    Raises:
        - InvalidTypeNameError if the schema contains an invalid type name, or if the user attempts
          to rename a type to an invalid name. A name is considered invalid if it does not consist
          of alphanumeric characters and underscores, if it starts with a numeric character, or
          if it starts with double underscores
        - SchemaStructureError if the schema does not have the expected form; in particular, if
          the AST does not represent a valid schema, if any query type field does not have the
          same name as the type that it queries, if the schema contains type extensions or
          input object definitions, or if the schema contains mutations or subscriptions
        - SchemaNameConflictError if there are conflicts between the renamed types or fields
    """
    # Prevent modifying input
    ast = parse(print_ast(ast))  # much faster than deepcopy

    # Check input schema satisfies various structural requirements
    check_ast_schema_is_valid(ast)

    schema = build_ast_schema(ast)
    query_type = get_query_type_name(schema)
    scalars = get_scalar_names(schema)

    # Rename types, interfaces, enums
    reverse_name_map = _rename_types(ast, renamings, query_type, scalars)
    reverse_name_map_changed_names_only = {
        renamed_name: original_name
        for renamed_name, original_name in six.iteritems(reverse_name_map)
        if renamed_name != original_name
    }

    # Rename query type fields
    _rename_query_type_fields(ast, renamings, query_type)

    return RenamedSchemaDescriptor(
        schema_ast=ast, reverse_name_map=reverse_name_map_changed_names_only
    )

def _rename_types(ast, renamings, query_type, scalars):
    """Rename types, enums, interfaces using renamings.

    The query type will not be renamed. Scalar types, field names, enum values will not be renamed.

    ast will be modified as a result.

    Args:
        ast: Document, the schema AST that we modify
        renamings: Dict[str, str], mapping original type/interface/enum name to renamed name. If
                   a name does not appear in the dict, it will be unchanged
        query_type: str, name of the query type, e.g. 'RootSchemaQuery'
        scalars: Set[str], the set of all scalars used in the schema, including user defined
                 scalars and and used builtin scalars, excluding unused builtins

    Returns:
        Dict[str, str], the renamed type name to original type name map. Map contains all types,
        including those that were not renamed.

    Raises:
        - InvalidTypeNameError if the schema contains an invalid type name, or if the user attempts
          to rename a type to an invalid name
        - SchemaNameConflictError if the rename causes name conflicts
    """
    visitor = RenameSchemaTypesVisitor(renamings, query_type, scalars)
    visit(ast, visitor)

    return visitor.reverse_name_map

def _rename_query_type_fields(ast, renamings, query_type):
    """Rename all fields of the query type.

    ast will be modified as a result.

    Args:
        ast: Document, the schema AST that we modify
        renamings: Dict[str, str], mapping original field name to renamed name. If a name
                   does not appear in the dict, it will be unchanged
        query_type: string, name of the query type, e.g. 'RootSchemaQuery'
    """
    visitor = RenameQueryTypeFieldsVisitor(renamings, query_type)
    visit(ast, visitor)

class RenameSchemaTypesVisitor(Visitor):
    """Traverse a Document AST, editing the names of nodes."""
    noop_types = frozenset({
        'Argument',
        'BooleanValue',
        'Directive',
        'DirectiveDefinition',
        'Document',
        'EnumValue',
        'EnumValueDefinition',
        'Field',
        'FieldDefinition',
        'FloatValue',
        'FragmentDefinition',
        'FragmentSpread',
        'InlineFragment',
        'InputObjectTypeDefinition',
        'InputValueDefinition',
        'IntValue',
        'ListType',
        'ListValue',
        'Name',
        'NonNullType',
        'ObjectField',
        'ObjectValue',
        'OperationDefinition',
        'OperationTypeDefinition',
        'ScalarTypeDefinition',
        'SchemaDefinition',
        'SelectionSet',
        'StringValue',
        'TypeExtensionDefinition',
        'Variable',
        'VariableDefinition',
    })
    rename_types = frozenset({
        'EnumTypeDefinition',
        'InterfaceTypeDefinition',
        'NamedType',
        'ObjectTypeDefinition',
        'UnionTypeDefinition',
    })

    def __init__(self, renamings, query_type, scalar_types):
        """Create a visitor for renaming types in a schema AST.

        Args:
            renamings: Dict[str, str], mapping from original type name to renamed type name.
                       Any name not in the dict will be unchanged
            query_type: str, name of the query type (e.g. RootSchemaQuery), which will not
                        be renamed
            scalar_types: Set[str], set of names of all scalars used in the schema, including
                          all user defined scalars and any builtin scalars that were used
        """
        self.renamings = renamings
        self.reverse_name_map = {}  # Dict[str, str], from renamed type name to original type name
        # reverse_name_map contains all types, including those that were unchanged
        self.query_type = query_type
        self.scalar_types = frozenset(scalar_types)
        self.builtin_types = frozenset({'String', 'Int', 'Float', 'Boolean', 'ID'})

    def _rename_name_and_add_to_record(self, node):
        """Rename the value of the node, and add the name pair to reverse_name_map.

        Don't rename if the type is the query type, a scalar type, or a builtin type.

        Modifies node and potentially modifies reverse_name_map.

        Args:
            node: type Name (see graphql/language/ast), an object describing the name of an AST
                  element such as type, interface, or scalar (user defined or builtin)

        Raises:
            - InvalidTypeNameError if either the node's current name or renamed name is invalid
            - SchemaNameConflictError if the newly renamed node causes name conflicts with
              existing types, scalars, or builtin types
        """
        name_string = node.value

        if name_string == self.query_type or name_string in self.scalar_types:
            return

        new_name_string = self.renamings.get(name_string, name_string)  # Default use original
        check_type_name_is_valid(new_name_string)

        if (
            new_name_string in self.reverse_name_map and
            self.reverse_name_map[new_name_string] != name_string
        ):
            raise SchemaNameConflictError(
                u'"{}" and "{}" are both renamed to "{}"'.format(
                    name_string, self.reverse_name_map[new_name_string], new_name_string
                )
            )
        if new_name_string in self.scalar_types or new_name_string in self.builtin_types:
            raise SchemaNameConflictError(
                u'"{}" was renamed to "{}", clashing with scalar "{}"'.format(
                    name_string, new_name_string, new_name_string
                )
            )

        node.value = new_name_string
        self.reverse_name_map[new_name_string] = name_string

    def enter(self, node, key, parent, path, ancestors):
        """Upon entering a node, operate depending on node type."""
        node_type = type(node).__name__
        if node_type in self.noop_types:
            # Do nothing, continue traversal
            return None
        elif node_type in self.rename_types:
            # Rename and put into record the name attribute of current node; continue traversal
            self._rename_name_and_add_to_record(node.name)
        else:
            # All Node types should've been taken care of, this line should never be reached
            raise AssertionError(u'Missed type: "{}"'.format(node_type))

class RenameQueryTypeFieldsVisitor(Visitor):
    def __init__(self, renamings, query_type):
        """Create a visitor for renaming fields of the query type in a schema AST.

        Args:
            renamings: Dict[str, str], from original field name to renamed field name. Any
                       name not in the dict will be unchanged
            query_type: str, name of the query type (e.g. RootSchemaQuery)
        """
        # Note that as field names and type names have been confirmed to match up, any renamed
        # field already has a corresponding renamed type. If no errors, due to either invalid
        # names or name conflicts, were raised when renaming type, no errors will occur when
        # renaming query type fields.
        self.in_query_type = False
        self.renamings = renamings
        self.query_type = query_type

    def enter_ObjectTypeDefinition(self, node, *args):
        """If the node's name matches the query type, record that we entered the query type."""
        if node.name.value == self.query_type:
            self.in_query_type = True

    def leave_ObjectTypeDefinition(self, node, key, parent, path, ancestors):
        """If the node's name matches the query type, record that we left the query type."""
        if node.name.value == self.query_type:
            self.in_query_type = False

    def enter_FieldDefinition(self, node, *args):
        """If inside the query type, rename field and add the name pair to reverse_field_map."""
        if self.in_query_type:
            field_name = node.name.value
            new_field_name = self.renamings.get(field_name, field_name)  # Default use original
            node.name.value = new_field_name
<commit_msg>Revert to deepcopy<commit_after># Copyright 2019-present Kensho Technologies, LLC.
from collections import namedtuple
from copy import deepcopy

from graphql import build_ast_schema
from graphql.language.visitor import Visitor, visit
import six

from .utils import (
    SchemaNameConflictError, check_ast_schema_is_valid, check_type_name_is_valid,
    get_query_type_name, get_scalar_names
)

RenamedSchemaDescriptor = namedtuple(
    'RenamedSchemaDescriptor', (
        'schema_ast',  # Document, AST representing the renamed schema
        'reverse_name_map',  # Dict[str, str], renamed type/query type field name to original name
        # reverse_name_map only contains names that were changed
    )
)

def rename_schema(ast, renamings):
    """Create a RenamedSchemaDescriptor; types and query type fields are renamed using renamings.

    Any type, interface, enum, or fields of the root type/query type whose name
    appears in renamings will be renamed to the corresponding value. Any such names that do not
    appear in renamings will be unchanged. Scalars, directives, enum values, and fields not
    belonging to the root/query type will never be renamed.

    Args:
        ast: Document, representing a valid schema that does not contain extensions, input
             object definitions, mutations, or subscriptions, whose fields of the query type share
             the same name as the types they query. Not modified by this function
        renamings: Dict[str, str], mapping original type/field names to renamed type/field names.
                   Type or query type field names that do not appear in the dict will be unchanged.
                   Any dict-like object that implements get(key, [default]) may also be used

    Returns:
        RenamedSchemaDescriptor, a namedtuple that contains the AST of the renamed schema, and the
        map of renamed type/field names to original names. Only renamed names will be included
        in the map.

    Raises:
        - InvalidTypeNameError if the schema contains an invalid type name, or if the user attempts
          to rename a type to an invalid name. A name is considered invalid if it does not consist
          of alphanumeric characters and underscores, if it starts with a numeric character, or
          if it starts with double underscores
        - SchemaStructureError if the schema does not have the expected form; in particular, if
          the AST does not represent a valid schema, if any query type field does not have the
          same name as the type that it queries, if the schema contains type extensions or
          input object definitions, or if the schema contains mutations or subscriptions
        - SchemaNameConflictError if there are conflicts between the renamed types or fields
    """
    # Prevent modifying input
    ast = deepcopy(ast)

    # Check input schema satisfies various structural requirements
    check_ast_schema_is_valid(ast)

    schema = build_ast_schema(ast)
    query_type = get_query_type_name(schema)
    scalars = get_scalar_names(schema)

    # Rename types, interfaces, enums
    reverse_name_map = _rename_types(ast, renamings, query_type, scalars)
    reverse_name_map_changed_names_only = {
        renamed_name: original_name
        for renamed_name, original_name in six.iteritems(reverse_name_map)
        if renamed_name != original_name
    }

    # Rename query type fields
    _rename_query_type_fields(ast, renamings, query_type)

    return RenamedSchemaDescriptor(
        schema_ast=ast, reverse_name_map=reverse_name_map_changed_names_only
    )

def _rename_types(ast, renamings, query_type, scalars):
    """Rename types, enums, interfaces using renamings.

    The query type will not be renamed. Scalar types, field names, enum values will not be renamed.

    ast will be modified as a result.

    Args:
        ast: Document, the schema AST that we modify
        renamings: Dict[str, str], mapping original type/interface/enum name to renamed name. If
                   a name does not appear in the dict, it will be unchanged
        query_type: str, name of the query type, e.g. 'RootSchemaQuery'
        scalars: Set[str], the set of all scalars used in the schema, including user defined
                 scalars and and used builtin scalars, excluding unused builtins

    Returns:
        Dict[str, str], the renamed type name to original type name map. Map contains all types,
        including those that were not renamed.

    Raises:
        - InvalidTypeNameError if the schema contains an invalid type name, or if the user attempts
          to rename a type to an invalid name
        - SchemaNameConflictError if the rename causes name conflicts
    """
    visitor = RenameSchemaTypesVisitor(renamings, query_type, scalars)
    visit(ast, visitor)

    return visitor.reverse_name_map

def _rename_query_type_fields(ast, renamings, query_type):
    """Rename all fields of the query type.

    ast will be modified as a result.

    Args:
        ast: Document, the schema AST that we modify
        renamings: Dict[str, str], mapping original field name to renamed name. If a name
                   does not appear in the dict, it will be unchanged
        query_type: string, name of the query type, e.g. 'RootSchemaQuery'
    """
    visitor = RenameQueryTypeFieldsVisitor(renamings, query_type)
    visit(ast, visitor)

class RenameSchemaTypesVisitor(Visitor):
    """Traverse a Document AST, editing the names of nodes."""
    noop_types = frozenset({
        'Argument',
        'BooleanValue',
        'Directive',
        'DirectiveDefinition',
        'Document',
        'EnumValue',
        'EnumValueDefinition',
        'Field',
        'FieldDefinition',
        'FloatValue',
        'FragmentDefinition',
        'FragmentSpread',
        'InlineFragment',
        'InputObjectTypeDefinition',
        'InputValueDefinition',
        'IntValue',
        'ListType',
        'ListValue',
        'Name',
        'NonNullType',
        'ObjectField',
        'ObjectValue',
        'OperationDefinition',
        'OperationTypeDefinition',
        'ScalarTypeDefinition',
        'SchemaDefinition',
        'SelectionSet',
        'StringValue',
        'TypeExtensionDefinition',
        'Variable',
        'VariableDefinition',
    })
    rename_types = frozenset({
        'EnumTypeDefinition',
        'InterfaceTypeDefinition',
        'NamedType',
        'ObjectTypeDefinition',
        'UnionTypeDefinition',
    })

    def __init__(self, renamings, query_type, scalar_types):
        """Create a visitor for renaming types in a schema AST.

        Args:
            renamings: Dict[str, str], mapping from original type name to renamed type name.
                       Any name not in the dict will be unchanged
            query_type: str, name of the query type (e.g. RootSchemaQuery), which will not
                        be renamed
            scalar_types: Set[str], set of names of all scalars used in the schema, including
                          all user defined scalars and any builtin scalars that were used
        """
        self.renamings = renamings
        self.reverse_name_map = {}  # Dict[str, str], from renamed type name to original type name
        # reverse_name_map contains all types, including those that were unchanged
        self.query_type = query_type
        self.scalar_types = frozenset(scalar_types)
        self.builtin_types = frozenset({'String', 'Int', 'Float', 'Boolean', 'ID'})

    def _rename_name_and_add_to_record(self, node):
        """Rename the value of the node, and add the name pair to reverse_name_map.

        Don't rename if the type is the query type, a scalar type, or a builtin type.

        Modifies node and potentially modifies reverse_name_map.

        Args:
            node: type Name (see graphql/language/ast), an object describing the name of an AST
                  element such as type, interface, or scalar (user defined or builtin)

        Raises:
            - InvalidTypeNameError if either the node's current name or renamed name is invalid
            - SchemaNameConflictError if the newly renamed node causes name conflicts with
              existing types, scalars, or builtin types
        """
        name_string = node.value

        if name_string == self.query_type or name_string in self.scalar_types:
            return

        new_name_string = self.renamings.get(name_string, name_string)  # Default use original
        check_type_name_is_valid(new_name_string)

        if (
            new_name_string in self.reverse_name_map and
            self.reverse_name_map[new_name_string] != name_string
        ):
            raise SchemaNameConflictError(
                u'"{}" and "{}" are both renamed to "{}"'.format(
                    name_string, self.reverse_name_map[new_name_string], new_name_string
                )
            )
        if new_name_string in self.scalar_types or new_name_string in self.builtin_types:
            raise SchemaNameConflictError(
                u'"{}" was renamed to "{}", clashing with scalar "{}"'.format(
                    name_string, new_name_string, new_name_string
                )
            )

        node.value = new_name_string
        self.reverse_name_map[new_name_string] = name_string

    def enter(self, node, key, parent, path, ancestors):
        """Upon entering a node, operate depending on node type."""
        node_type = type(node).__name__
        if node_type in self.noop_types:
            # Do nothing, continue traversal
            return None
        elif node_type in self.rename_types:
            # Rename and put into record the name attribute of current node; continue traversal
            self._rename_name_and_add_to_record(node.name)
        else:
            # All Node types should've been taken care of, this line should never be reached
            raise AssertionError(u'Missed type: "{}"'.format(node_type))

class RenameQueryTypeFieldsVisitor(Visitor):
    def __init__(self, renamings, query_type):
        """Create a visitor for renaming fields of the query type in a schema AST.

        Args:
            renamings: Dict[str, str], from original field name to renamed field name. Any
                       name not in the dict will be unchanged
            query_type: str, name of the query type (e.g. RootSchemaQuery)
        """
        # Note that as field names and type names have been confirmed to match up, any renamed
        # field already has a corresponding renamed type. If no errors, due to either invalid
        # names or name conflicts, were raised when renaming type, no errors will occur when
        # renaming query type fields.
        self.in_query_type = False
        self.renamings = renamings
        self.query_type = query_type

    def enter_ObjectTypeDefinition(self, node, *args):
        """If the node's name matches the query type, record that we entered the query type."""
        if node.name.value == self.query_type:
            self.in_query_type = True

    def leave_ObjectTypeDefinition(self, node, key, parent, path, ancestors):
        """If the node's name matches the query type, record that we left the query type."""
        if node.name.value == self.query_type:
            self.in_query_type = False

    def enter_FieldDefinition(self, node, *args):
        """If inside the query type, rename field and add the name pair to reverse_field_map."""
        if self.in_query_type:
            field_name = node.name.value
            new_field_name = self.renamings.get(field_name, field_name)  # Default use original
            node.name.value = new_field_name
<|endoftext|>


<commit_before>import os, subprocess, csv, glob
from imutils import face_utils
import numpy as np
import dlib
import cv2
import imutils
from imutils.face_utils import FaceAligner
from imutils.face_utils import rect_to_bb
from skimage.draw import polygon
from scipy.ndimage.measurements import center_of_mass
from scipy.ndimage import zoom

def get_video_frame(video, time):
    """
    Return the single frame closest to the given timepoint. Can then run detect_landmarks on the frame.
    Inputs: video - an MXF file; time in seconds - format d.ddd (sec.msec), rounded to three decimal places.
    Outputs: an ndarray image of the desired frame.
    """
    output_bmp = 'temp.bmp'
    try:
        os.remove(output_bmp)
    except OSError:
        pass
    frame_get_args = ['ffmpeg',
                      '-nostats',
                      '-loglevel', 'warning',
                      '-ss', str(time),
                      '-i', video,
                      '-vcodec', 'bmp',
                      '-vframes', '1',
                      '-an', '-f', 'rawvideo',
                       output_bmp]
    subprocess.check_call(frame_get_args)
    frame = cv2.imread(output_bmp)
    return frame

def detect_landmarks(my_ndarray, detector, predictor):
    """
    Inputs: an ndarray frame output from cv2.VideoCapture object,
            a detector of choice from dlib,
            and a dlib face landmark predictor trained on data of choice.
    Outputs:the portion of the image containing the detected face,
            and a (68,2) ndarray containing X,Y coordinates for the 68 face points dlib detects.
    """

    # read in image
    gray = cv2.cvtColor(my_ndarray, cv2.COLOR_BGR2GRAY)

    # run face detector to get bounding rectangle
    rect = detector(gray, 1)[0]

    # run landmark prediction on portion of image in face rectangle; output
    shape = predictor(gray, rect)
    shape_np = face_utils.shape_to_np(shape)

    return shape_np

def get_norm_face(my_ndarray, detector, predictor, aligner):
    """
    Inputs: an ndarray frame output from cv2.VideoCapture object,
            a detector of choice from dlib,
            a dlib face landmark predictor trained on data of choice,
            and an imutils FaceAligner instance.
    Outputs:an affine-transformed/rotated and rescaled face bounding box
            and a (68,2) ndarray containing X,Y coordinates for the 68 face points dlib detects.
    Inspired by code by Adrian Rosebrock.
    """
    # TODO! build in ability to manually define where eye is, for cases where eyes are consistently mis-detected
    # read in image
    gray = cv2.cvtColor(my_ndarray, cv2.COLOR_BGR2GRAY)

    # run face detector to get bounding rectangle
    rect = detector(gray,1)[0]

    # align face
    # TODO fork imutils, change this function to rotate and return the landmarks used for alignment;
    # replace current shape_np
    faceAligned = aligner.align(my_ndarray, gray, rect)

    return faceAligned

def draw_landmarks(my_ndarray, shape, anonymize = False, aperture_xy = False, line_width=2):
    """
    Inputs: an ndarray frame output from cv2.VideoCapture object, and a (68,2) ndarray of x,y coords that dlib detects.
    Outputs: an image with lines drawn over the detected landmarks; useful for testing and visualization.
    anonymize: if True, also apply a Gaussian blur to the top of the face to remove identifiable features.
    aperture_xy: if True, also draw (next to face) numerical values for x and y diameters of lip aperture.
    """

    out_image = my_ndarray.copy()
    lwd = int(line_width)

    if anonymize:
        # get bottom of box using nose
        nose_start,nose_end = face_utils.FACIAL_LANDMARKS_IDXS['nose']
        max_y = max([shape[i][1] for i in range(nose_start, nose_end)])

        # get top of box using eyebrows
        eyebrows_start = face_utils.FACIAL_LANDMARKS_IDXS['right_eyebrow'][0]
        eyebrows_end = face_utils.FACIAL_LANDMARKS_IDXS['left_eyebrow'][1]
        min_y = min([shape[i][1] for i in range(eyebrows_start, eyebrows_end)])

        # get sides of box using jaw
        jaw_start,jaw_end = face_utils.FACIAL_LANDMARKS_IDXS['jaw']
        max_x = max([shape[i][0] for i in range(jaw_start, jaw_end)])
        min_x = min([shape[i][0] for i in range(jaw_start, jaw_end)])

        # pad out the blurred area
        # TODO change to integer pixel values (will throw error in notebooks)
        width = max_x - min_x
        height = max_y - min_y
        min_y -= 0.4*height
        min_x -= 0.1*width
        max_y += 0.1*height
        max_x += 0.1*width

        # replace the area around the selected landmarks with a blurred version of the area
        upper_face = out_image[min_y:max_y, min_x:max_x]
        blur = cv2.GaussianBlur(upper_face,(101, 101), 30)
        out_image[min_y:min_y+upper_face.shape[0], min_x:min_x+upper_face.shape[1]] = blur

    for i,name in enumerate(face_utils.FACIAL_LANDMARKS_IDXS.keys()):
        if name == "mouth":
            continue
        j,k = face_utils.FACIAL_LANDMARKS_IDXS[name]
        pts = np.array(shape[j:k], dtype=np.uint32)
        for idx,pt in enumerate(pts):
            pt1 = pt
            try:
                pt2 = pts[idx+1]
            except IndexError:
                if name == "left_eye" or name == "right_eye":
                    pt2 = pts[0]
                else:
                    continue
            cv2.line(out_image, tuple(pt1), tuple(pt2), (255,255,255),lwd)

    # drawing the mouth with some more precision
    # draw most of the outer perimeter of lips
    jm,km = face_utils.FACIAL_LANDMARKS_IDXS['mouth']
    for idx in range(jm,jm+11):
        pt1 = shape[idx]
        pt2 = shape[idx+1]
        cv2.line(out_image, tuple(pt1), tuple(pt2), (255,255,255),lwd)

    # draw the last segment for the outer perimiter of lips
    cv2.line(out_image, tuple(shape[48]), tuple(shape[59]), (255,255,255),lwd)

    # draw the inner aperture of the lips
    for idx in range(jm+12,km):
        pt1 = shape[idx]
        try:
            pt2 = shape[idx+1]
        except IndexError:
            pt2 = shape[jm+12]
        cv2.line(out_image, tuple(pt1), tuple(pt2), (255,255,255),lwd)

    # add text indicating measured lip aperture in px
    if aperture_xy:
        x,y = get_lip_aperture(shape)
        add_string = "x={}, y={}".format(round(x,1),round(y,1))
        loc = tuple(np.subtract(shape[4], (200,0)))
        font = cv2.FONT_HERSHEY_SIMPLEX
        cv2.putText(out_image, add_string, loc, font, 0.8, (255,255,255), 2, cv2.LINE_AA)

    return out_image

def get_lip_aperture(shape):
    """
    Inputs: the typical 68,2 ndarray "shape" object output by detect_landmarks.
    Outputs: a 2-tuple of horizontal and vertical diameters of the lip aperture,
     treating the horizontal line like the major axis of an ellipse,
     and the vertical line like the minor axis.
    """
    horizontal_axis = np.linalg.norm(shape[60] - shape[64])
    vertical_axis = np.linalg.norm(shape[62] - shape[66])

    return horizontal_axis,vertical_axis

# TODO convert lip_mask into a class and add the other clean-up functions below as methods

def lip_mask(frame, shape):
    """
    Returns a simplified ndarray containing 0s/1s, with lips a filled polygon of 1s
    """
    # fetch indices of mouth landmark points
    jm,km = face_utils.FACIAL_LANDMARKS_IDXS['mouth']

    # initialize a blank background in the shape of the original image
    mask_dims = [frame.shape[0],frame.shape[1]]
    mask = np.zeros(mask_dims, dtype=np.uint8)

    # fill outer lip polygon
    mouth_outer = shape[jm:jm+11]
    mouth_outer_col = [p[0] for p in mouth_outer]
    mouth_outer_row = [p[1] for p in mouth_outer]
    # last point's coords need to be appended manually
    mouth_outer_col.append(shape[59][0])
    mouth_outer_row.append(shape[59][1])
    rr,cc = polygon(mouth_outer_row,mouth_outer_col)
    mask[rr,cc] = 1

    # then, cancel out inner polygon
    mouth_inner = shape[jm+12:km]
    mouth_inner_col = [p[0] for p in mouth_inner]
    mouth_inner_row = [p[1] for p in mouth_inner]
    rr,cc = polygon(mouth_inner_row,mouth_inner_col)
    mask[rr,cc] = 0

    return mask

def get_center(image):
    """ Returns the center location of an array as (x, y). """
    return np.array([image.shape[0] // 2, image.shape[1] // 2])

def centroid(image):
    """ Returns centroid. """
    if (image == 0).all():
        return get_center(image)
    centroid = center_of_mass(image)
    centroid = np.rint(np.array(centroid))
    return centroid.astype(int)

def cross(image, point):
    """ Point-marking (with cross shape) function, to mark off i.e. centroid. """
    image_w, image_h = image.shape
    x, y = point
    blank = np.zeros(image.shape)
    blank[max(0, x - 3):min(x + 4, image_w), y] = 1
    blank[x, max(0, y - 3):min(y + 4, image_h)] = 1
    blank[x,y] = 0
    return blank.astype(bool).astype(int)

#from scipy.ndimage.measurements import center_of_mass
#from scipy.ndimage import zoom

def crop_center(input_mask):
    """
    Returns mask (1s/0s) with centroid of 1s centered on center of a smaller ground of 0s.
    This normalizes for some aspects of head movement (relative size and absolute position),
      but does not normalize for left/right head tilt.
    """
    mask = input_mask.copy()
    # define a smaller ground of zeros
    # TODO make this scaled to the individual acquisition somehow, by e.g. lip width
    ground_shape = np.array([int(d/3) for d in mask.shape])
    ground = np.zeros(ground_shape)

    # get mask's centroid and ground's center point
    cg_x, cg_y = get_center(ground)
    mask_centroid = centroid(mask)
    ci_x, ci_y = mask_centroid
    # unpack dims of mask and ground
    image_w, image_h = mask.shape
    ground_w, ground_h = ground_shape

    # determine preliminary amounts to crop from left and top
    left_diff = ci_x - cg_x
    top_diff = ci_y - cg_y

    # check if left/top sides of image fit on ground; chop off if not
    if left_diff > 0:
        mask = mask[left_diff:,]
        ci_x -= left_diff
    if top_diff > 0:
        mask = mask[:,top_diff:]
        ci_y -= top_diff

    # get mask dims again
    image_w, image_h = mask.shape

    # get right and bottom dimensions
    ground_r = ground_w - cg_x
    ground_b = ground_h - cg_y
    image_r = image_w - ci_x
    image_b = image_h - ci_y

    # determine amounts to crop from right and bottom
    right_diff = ground_r - image_r
    bottom_diff = ground_b - image_b

    # check if right/bottom sides of image fit on ground; chop off if not
    if right_diff < 0:
        mask = mask[:image_w + right_diff]
    if bottom_diff < 0:
        mask = mask[:,:image_h + bottom_diff]

    image_w, image_h = mask.shape

    # copy modified mask onto ground shape
    left_start = cg_x - ci_x
    top_start = cg_y - ci_y
    ground[left_start:left_start + image_w, top_start:top_start + image_h] += mask

    return ground
<commit_msg>Update imgphon.py<commit_after>import os, subprocess, csv, glob
from imutils import face_utils
import numpy as np
import dlib
import cv2
import imutils
from imutils.face_utils import FaceAligner
from imutils.face_utils import rect_to_bb
from skimage.draw import polygon
from scipy.ndimage.measurements import center_of_mass
from scipy.ndimage import zoom

def get_video_frame(video, time):
    """
    Return the single frame closest to the given timepoint. Can then run detect_landmarks on the frame.
    Inputs: video - a video file (so far tested on MOV and MXF); time - format d.ddd (sec.msec), up to three decimal places.
    Outputs: an ndarray of the desired frame, in BGR color space.
    """
    output_bmp = 'temp.bmp'
    try:
        os.remove(output_bmp)
    except OSError:
        pass
    frame_get_args = ['ffmpeg',
                      '-nostats',
                      '-loglevel', 'warning',
                      '-ss', str(time),
                      '-i', video,
                      '-vcodec', 'bmp',
                      '-vframes', '1',
                      '-an', '-f', 'rawvideo',
                       output_bmp]
    subprocess.check_call(frame_get_args)
    frame = cv2.imread(output_bmp)
    return frame

def detect_landmarks(my_ndarray, detector, predictor):
    """
    Inputs: an ndarray frame output from cv2.VideoCapture object,
            a detector of choice from dlib,
            and a dlib face landmark predictor trained on data of choice.
    Outputs:the portion of the image containing the detected face,
            and a (68,2) ndarray containing X,Y coordinates for the 68 face points dlib detects.
    """

    # read in image
    gray = cv2.cvtColor(my_ndarray, cv2.COLOR_BGR2GRAY)

    # run face detector to get bounding rectangle
    rect = detector(gray, 1)[0]

    # run landmark prediction on portion of image in face rectangle; output
    shape = predictor(gray, rect)
    shape_np = face_utils.shape_to_np(shape)

    return shape_np

def get_norm_face(my_ndarray, detector, predictor, aligner):
    """
    Inputs: an ndarray frame output from cv2.VideoCapture object,
            a detector of choice from dlib,
            a dlib face landmark predictor trained on data of choice,
            and an imutils FaceAligner instance.
    Outputs:an affine-transformed/rotated and rescaled face bounding box
            and a (68,2) ndarray containing X,Y coordinates for the 68 face points dlib detects.
    Inspired by code by Adrian Rosebrock.
    """
    # TODO! build in ability to manually define where eye is, for cases where eyes are consistently mis-detected
    # read in image
    gray = cv2.cvtColor(my_ndarray, cv2.COLOR_BGR2GRAY)

    # run face detector to get bounding rectangle
    rect = detector(gray,1)[0]

    # align face
    # TODO fork imutils, change this function to rotate and return the landmarks used for alignment;
    # replace current shape_np
    faceAligned = aligner.align(my_ndarray, gray, rect)

    return faceAligned

def draw_landmarks(my_ndarray, shape, anonymize = False, aperture_xy = False, line_width=2):
    """
    Inputs: an ndarray frame output from cv2.VideoCapture object, and a (68,2) ndarray of x,y coords that dlib detects.
    Outputs: an image with lines drawn over the detected landmarks; useful for testing and visualization.
    anonymize: if True, also apply a Gaussian blur to the top of the face to remove identifiable features.
    aperture_xy: if True, also draw (next to face) numerical values for x and y diameters of lip aperture.
    """

    out_image = my_ndarray.copy()
    lwd = int(line_width)

    if anonymize:
        # get bottom of box using nose
        nose_start,nose_end = face_utils.FACIAL_LANDMARKS_IDXS['nose']
        max_y = max([shape[i][1] for i in range(nose_start, nose_end)])

        # get top of box using eyebrows
        eyebrows_start = face_utils.FACIAL_LANDMARKS_IDXS['right_eyebrow'][0]
        eyebrows_end = face_utils.FACIAL_LANDMARKS_IDXS['left_eyebrow'][1]
        min_y = min([shape[i][1] for i in range(eyebrows_start, eyebrows_end)])

        # get sides of box using jaw
        jaw_start,jaw_end = face_utils.FACIAL_LANDMARKS_IDXS['jaw']
        max_x = max([shape[i][0] for i in range(jaw_start, jaw_end)])
        min_x = min([shape[i][0] for i in range(jaw_start, jaw_end)])

        # pad out the blurred area
        # TODO change to integer pixel values (will throw error in notebooks)
        width = max_x - min_x
        height = max_y - min_y
        min_y -= 0.4*height
        min_x -= 0.1*width
        max_y += 0.1*height
        max_x += 0.1*width

        # replace the area around the selected landmarks with a blurred version of the area
        upper_face = out_image[min_y:max_y, min_x:max_x]
        blur = cv2.GaussianBlur(upper_face,(101, 101), 30)
        out_image[min_y:min_y+upper_face.shape[0], min_x:min_x+upper_face.shape[1]] = blur

    for i,name in enumerate(face_utils.FACIAL_LANDMARKS_IDXS.keys()):
        if name == "mouth":
            continue
        j,k = face_utils.FACIAL_LANDMARKS_IDXS[name]
        pts = np.array(shape[j:k], dtype=np.uint32)
        for idx,pt in enumerate(pts):
            pt1 = pt
            try:
                pt2 = pts[idx+1]
            except IndexError:
                if name == "left_eye" or name == "right_eye":
                    pt2 = pts[0]
                else:
                    continue
            cv2.line(out_image, tuple(pt1), tuple(pt2), (255,255,255),lwd)

    # drawing the mouth with some more precision
    # draw most of the outer perimeter of lips
    jm,km = face_utils.FACIAL_LANDMARKS_IDXS['mouth']
    for idx in range(jm,jm+11):
        pt1 = shape[idx]
        pt2 = shape[idx+1]
        cv2.line(out_image, tuple(pt1), tuple(pt2), (255,255,255),lwd)

    # draw the last segment for the outer perimiter of lips
    cv2.line(out_image, tuple(shape[48]), tuple(shape[59]), (255,255,255),lwd)

    # draw the inner aperture of the lips
    for idx in range(jm+12,km):
        pt1 = shape[idx]
        try:
            pt2 = shape[idx+1]
        except IndexError:
            pt2 = shape[jm+12]
        cv2.line(out_image, tuple(pt1), tuple(pt2), (255,255,255),lwd)

    # add text indicating measured lip aperture in px
    if aperture_xy:
        x,y = get_lip_aperture(shape)
        add_string = "x={}, y={}".format(round(x,1),round(y,1))
        loc = tuple(np.subtract(shape[4], (200,0)))
        font = cv2.FONT_HERSHEY_SIMPLEX
        cv2.putText(out_image, add_string, loc, font, 0.8, (255,255,255), 2, cv2.LINE_AA)

    return out_image

def get_lip_aperture(shape):
    """
    Inputs: the typical 68,2 ndarray "shape" object output by detect_landmarks.
    Outputs: a 2-tuple of horizontal and vertical diameters of the lip aperture,
     treating the horizontal line like the major axis of an ellipse,
     and the vertical line like the minor axis.
    """
    horizontal_axis = np.linalg.norm(shape[60] - shape[64])
    vertical_axis = np.linalg.norm(shape[62] - shape[66])

    return horizontal_axis,vertical_axis

# TODO convert lip_mask into a class and add the other clean-up functions below as methods

def lip_mask(frame, shape):
    """
    Returns a simplified ndarray containing 0s/1s, with lips a filled polygon of 1s
    """
    # fetch indices of mouth landmark points
    jm,km = face_utils.FACIAL_LANDMARKS_IDXS['mouth']

    # initialize a blank background in the shape of the original image
    mask_dims = [frame.shape[0],frame.shape[1]]
    mask = np.zeros(mask_dims, dtype=np.uint8)

    # fill outer lip polygon
    mouth_outer = shape[jm:jm+11]
    mouth_outer_col = [p[0] for p in mouth_outer]
    mouth_outer_row = [p[1] for p in mouth_outer]
    # last point's coords need to be appended manually
    mouth_outer_col.append(shape[59][0])
    mouth_outer_row.append(shape[59][1])
    rr,cc = polygon(mouth_outer_row,mouth_outer_col)
    mask[rr,cc] = 1

    # then, cancel out inner polygon
    mouth_inner = shape[jm+12:km]
    mouth_inner_col = [p[0] for p in mouth_inner]
    mouth_inner_row = [p[1] for p in mouth_inner]
    rr,cc = polygon(mouth_inner_row,mouth_inner_col)
    mask[rr,cc] = 0

    return mask

def get_center(image):
    """ Returns the center location of an array as (x, y). """
    return np.array([image.shape[0] // 2, image.shape[1] // 2])

def centroid(image):
    """ Returns centroid. """
    if (image == 0).all():
        return get_center(image)
    centroid = center_of_mass(image)
    centroid = np.rint(np.array(centroid))
    return centroid.astype(int)

def cross(image, point):
    """ Point-marking (with cross shape) function, to mark off i.e. centroid. """
    image_w, image_h = image.shape
    x, y = point
    blank = np.zeros(image.shape)
    blank[max(0, x - 3):min(x + 4, image_w), y] = 1
    blank[x, max(0, y - 3):min(y + 4, image_h)] = 1
    blank[x,y] = 0
    return blank.astype(bool).astype(int)

#from scipy.ndimage.measurements import center_of_mass
#from scipy.ndimage import zoom

def crop_center(input_mask):
    """
    Returns mask (1s/0s) with centroid of 1s centered on center of a smaller ground of 0s.
    This normalizes for some aspects of head movement (relative size and absolute position),
      but does not normalize for left/right head tilt.
    """
    mask = input_mask.copy()
    # define a smaller ground of zeros
    # TODO make this scaled to the individual acquisition somehow, by e.g. lip width
    ground_shape = np.array([int(d/3) for d in mask.shape])
    ground = np.zeros(ground_shape)

    # get mask's centroid and ground's center point
    cg_x, cg_y = get_center(ground)
    mask_centroid = centroid(mask)
    ci_x, ci_y = mask_centroid
    # unpack dims of mask and ground
    image_w, image_h = mask.shape
    ground_w, ground_h = ground_shape

    # determine preliminary amounts to crop from left and top
    left_diff = ci_x - cg_x
    top_diff = ci_y - cg_y

    # check if left/top sides of image fit on ground; chop off if not
    if left_diff > 0:
        mask = mask[left_diff:,]
        ci_x -= left_diff
    if top_diff > 0:
        mask = mask[:,top_diff:]
        ci_y -= top_diff

    # get mask dims again
    image_w, image_h = mask.shape

    # get right and bottom dimensions
    ground_r = ground_w - cg_x
    ground_b = ground_h - cg_y
    image_r = image_w - ci_x
    image_b = image_h - ci_y

    # determine amounts to crop from right and bottom
    right_diff = ground_r - image_r
    bottom_diff = ground_b - image_b

    # check if right/bottom sides of image fit on ground; chop off if not
    if right_diff < 0:
        mask = mask[:image_w + right_diff]
    if bottom_diff < 0:
        mask = mask[:,:image_h + bottom_diff]

    image_w, image_h = mask.shape

    # copy modified mask onto ground shape
    left_start = cg_x - ci_x
    top_start = cg_y - ci_y
    ground[left_start:left_start + image_w, top_start:top_start + image_h] += mask

    return ground
<|endoftext|>


<commit_before>'''
1) start by aggregating a new collection that includes the video interaction, page navigation and page close events for users sorted by username and time
db.tracking.aggregate([{ $limit : 30 },{$match:{$and:[{'event_source':'browser'},{$or:[{'event_type':'play_video'},{'event_type':'speed_change_video'},{'event_type':'seq_goto'}, {'event_type':'seq_next'}, {'event_type':'seq_prev'}, {'event_type':'page_close'}, {'event_type':'play_video'},{'event_type':'pause_video'}, {'event_type':'seek_video'}, {'event_type':'pause_video'}]}]}}, {$project:{"username":1, "event_type":1, 'time':1,"event":1}},{ $sort: {'username':1,'time':1}},{$out: "video_watching"}])
'''

import sys
import csv
import time
from datetime import datetime

from base_edx import EdXConnection
from generate_csv_report import CSV

db_name = sys.argv[1]

eventCollection = 'video_watching'
connection = EdXConnection(db_name, eventCollection)
collection = connection.get_access_to_collection()

students = collection[eventCollection].distinct('username')
print
print students

watch_durations =[]
start_event_time = {'blank'}
end_event_time = {'blank'}

for student in students[0:2]:
    print
    print student
    cursor = collection[eventCollection].find({'username':student})
    #if play_video event then count as start
    for index,item in enumerate(cursor):
#if both a start and end event have been found append them to the list
        if start_event_time !={'blank'} and end_event_time != {'blank'}:
            duration = end_event_time - start_event_time
#append all the values for a given watch_duration to the watch_durations list
            watch_durations.append([student, video_id, video_code, time_point, start_event_time,end_event_time,duration])
            start_event_time = {'blank'}
            end_event_time = {'blank'}
#if event is play_video then assign as start
        if item['event_type'] == 'play_video':
            start_event_time = datetime.strptime(item['time'].split('+')[0], "%Y-%m-%dT%H:%M:%S.%f")
#get values related to video identification
            video = item['event']
            video_code = video['code']
            video_id = video['id']
            time_point = video['currentTime']
            print 'start'
            print (start_event_time)
#when a start event is found set the end event to blank
            end_event_time = {'blank'}
            continue
#assign all other event types to end
#Note: currently seek_video events count as an end_event
        else:
            end_event_time = datetime.strptime(item['time'].split('+')[0], "%Y-%m-%dT%H:%M:%S.%f")
            print 'end'
            print (end_event_time)
            continue
print watch_durations

output = CSV(watch_durations, ['Username','video_id','youtube id (video_code)','time_point (seconds)','start_event_time','end_event_time', 'duration (minutes)'], output_file=db_name+'video_watch_duration.csv', row_limit=200000)
output.generate_csv()
<commit_msg>Update video_watch_duration.py<commit_after>1) start by aggregating a new collection that includes the video interaction, page navigation and page close events for users sorted by username and time
db.tracking.aggregate([{$match:{$and:[{'event_source':'browser'},{$or:[{'event_type':'play_video'},{'event_type':'speed_change_video'},{'event_type':'seq_goto'}, {'event_type':'seq_next'}, {'event_type':'seq_prev'}, {'event_type':'page_close'}, {'event_type':'play_video'},{'event_type':'pause_video'}, {'event_type':'seek_video'}, {'event_type':'pause_video'}]}]}}, {$project:{"username":1, "event_type":1, 'time':1,"event":1}},{ $sort: {'username':1,'time':1}},{$out: "video_watching"}],{allowDiskUse:true})
or
db.runCommand({aggregate:'tracking',pipeline:[{$match:{$and:[{'event_source':'browser'},{$or:[{'event_type':'play_video'},{'event_type':'speed_change_video'},{'event_type':'seq_goto'}, {'event_type':'seq_next'}, {'event_type':'seq_prev'}, {'event_type':'page_close'}, {'event_type':'play_video'},{'event_type':'pause_video'}, {'event_type':'seek_video'}, {'event_type':'pause_video'}]}]}}, {$project:{"username":1, "event_type":1, 'time':1,"event":1}},{ $sort: {'username':1,'time':1}},{$out: "video_watching"}],allowDiskUse:true})

import sys
import csv
import time
from datetime import datetime

from base_edx import EdXConnection
from generate_csv_report import CSV

db_name = sys.argv[1]

eventCollection = 'video_watching'
connection = EdXConnection(db_name, eventCollection)
collection = connection.get_access_to_collection()

students = collection[eventCollection].distinct('username')
print
print students
print len(students)

watch_durations =[]
start_event_time = {'blank'}
end_event_time = {'blank'}
errors = []
count_errors = 0

for student in students:
    cursor = collection[eventCollection].find({'username':student})
    #if play_video event then count as start
    for index,item in enumerate(cursor):
        try:
    #if both a start and end event have been found append them to the list
            if start_event_time !={'blank'} and end_event_time != {'blank'}:
                duration = end_event_time - start_event_time
    #append all the values for a given watch_duration to the watch_durations list
                watch_durations.append([student, video_id, video_code, time_point, start_event_time,end_event_time,duration])
                start_event_time = {'blank'}
                end_event_time = {'blank'}
    #if event is play_video then assign as start
            if item['event_type'] == 'play_video':
                start_event_time = datetime.strptime(item['time'].split('+')[0], "%Y-%m-%dT%H:%M:%S.%f")
    #get values related to video identification
                video = item['event']
                video_code = video['code']
                video_id = video['id']
                time_point = video['currentTime']
                print 'start'
                print (start_event_time)
    #when a start event is found set the end event to blank
                end_event_time = {'blank'}
                continue
    #assign all other event types to end
    #Note: currently seek_video events count as an end_event
            else:
                end_event_time = datetime.strptime(item['time'].split('+')[0], "%Y-%m-%dT%H:%M:%S.%f")
                print 'end'
                print (end_event_time)
                continue
        except:
            count_errors = count_errors+1
            errors.append([index,item])
#print watch_durations
print
print len(watch_durations)
print errors
output = CSV(watch_durations, ['Username','video_id','youtube id (video_code)','time_point (seconds)','start_event_time','end_event_time', 'duration (minutes)'], output_file=db_name+'video_watch_duration.csv', row_limit=200000)
output.generate_csv()
output_errors = CSV(errors, ['Errors'], output_file=db_name+'video_watch_duration_errors.csv', row_limit=200000)
output_errors.generate_csv()
<|endoftext|>