<commit_before>#!/usr/bin/env python3
"""
RDS Functions
"""

import sys
from datetime import datetime, timedelta
from exclusions import excluded_instances
import boto3
import botocore

def get_session(access_key_id, secret_access_key):
    " Establishes a session with AWS "
    return boto3.session.Session(
        aws_access_key_id=access_key_id,
        aws_secret_access_key=secret_access_key
    )

def get_rds_client(session):
    " Returns a RDS boto client "
    return session.client('rds')

def get_cloudwatch_client(session):
    " Returns a CloudWatch boto client "
    return session.client('cloudwatch')

def get_ec2_client(session):
    " Returns a EC2 boto client "
    return session.client('ec2')

def get_vpc_ids(client):
    " Returns a list of VPC IDs in the account "
    vpc_ids = []
    vpcs = client.describe_vpcs()['Vpcs']
    for vpc in vpcs:
        vpc_ids.append(vpc['VpcId'])
    return vpc_ids

def get_isolated_sgs(client):
    " Returns a dict of rds-isolate SG IDs for each VPC in account. "
    vpc_ids = get_vpc_ids(client)
    isolated_sgs = {}
    for vpc in vpc_ids:
        sec_groups = client.describe_security_groups(
            Filters=[
                {
                    "Name": "vpc-id",
                    "Values": [vpc]
                },
                {
                    "Name": "group-name",
                    "Values": ["rds-isolate"]
                }
            ]
        )['SecurityGroups']
        try:
            isolated_sgs[vpc] = sec_groups[0]['GroupId']
        except IndexError:
            print("No rds-isolate group found for VPC: {}".format(vpc))
    return isolated_sgs

def get_connections_statistics(client, rds_instances):
    " Returns a dict of all instances and their avg DB conns over all datapoints "
    rds_stats = {}
    for rds_instance in rds_instances:
        stats = client.get_metric_statistics(
            Namespace="AWS/RDS",
            MetricName="DatabaseConnections",
            Statistics=['Average'],
            Period=57600,
            StartTime=(datetime.today() - timedelta(days=3)),
            EndTime=datetime.today(),
            Dimensions=[
                {
                    'Name': 'DBInstanceIdentifier',
                    'Value': rds_instance['DBInstanceIdentifier']
                }
            ]
        )['Datapoints']
        datapoints = []
        for stat in stats:
            datapoints.append(stat['Average'])
        if len(datapoints) > 0:
            dp_conns = sum(datapoints)/float(len(datapoints))
            rds_stats[rds_instance['DBInstanceIdentifier']] = dp_conns
        else:
            print("Instance: %s has no datapoints." % rds_instance['DBInstanceIdentifier'])

    return rds_stats

def get_rds_instances(client, vpc_id=None):
    " Gets all RDS instances, per VPC, if specified. "

    rds_instances = []
    resp = client.describe_db_instances()
    while 'Marker' in resp:
        rds_instances.extend(resp['DBInstances'])
        resp = client.describe_db_instances(Marker=resp['Marker'])
    rds_instances.extend(resp['DBInstances'])
    if not vpc_id:
        return rds_instances
    else:
        return [r for r in rds_instances if r['DBSubnetGroup']['VpcId'] == vpc_id]

def set_no_multiaz(client, rds_instance):
    " Takes a rds instance obj and turns off MultiAZ "
    try:
        client.modify_db_instance(
            DBInstanceIdentifier=rds_instance['DBInstanceIdentifier'],
            MultiAZ=False,
            ApplyImmediately=True
        )
    except botocore.exceptions.ClientError:
        print("Error setting no-multiaz on instance %s" % rds_instance['DBInstanceIdentifier'])

def set_security_group(client, rds_instance, sg_id):
    " Sets the rds_instance Security Group to sg_id "
    try:
        client.modify_db_instance(
            DBInstanceIdentifier=rds_instance['DBInstanceIdentifier'],
            VpcSecurityGroupIds=[sg_id]
        )
    except botocore.exceptions.ClientError:
        print("Error setting SG on instance %s" % rds_instance['DBInstanceIdentifier'])

def set_instance_size(client, rds_instance, size=None):
    " Sets instance to the smallest available size "
    if not size:
        available_sizes = client.describe_orderable_db_instance_options(
            Engine=rds_instance['Engine']
        )['OrderableDBInstanceOptions']
        size = available_sizes[0]['DBInstanceClass']
    try:
        client.modify_db_instance(
            DBInstanceIdentifier=rds_instance['DBInstanceIdentifier'],
            DBInstanceClass=size,
            ApplyImmediately=True
        )
    except botocore.exceptions.ClientError:
        print("Error setting size on instance %s" % rds_instance['DBInstanceIdentifier'])

def get_instances_with_sg(client, sg_id, vpc_id=None):
    """ Gets all RDS instances that are using the sg_id """
    rds_instances = get_rds_instances(client, vpc_id)

    instances_with_sg = []
    for instance in rds_instances:
        sgs = instance['VpcSecurityGroups']
        for sg in sgs:
            if sg['VpcSecurityGroupId'] == sg_id:
                instances_with_sg.append(instance)

    return instances_with_sg

def take_snapshot(client, rds_instance):
    """ Takes a snapshot of an RDS instance """

    resp = client.create_db_snapshot(
        DBSnapshotIdentifier='%s-FINAL' % rds_instance['DBInstanceIdentifier'],
        DBInstanceIdentifier=rds_instance['DBInstanceIdentifier'],
    )
    print("Created final snapshot for %s, %s"
          % (rds_instance['DBInstanceIdentifier'], resp['DBSnapshot']['DBSnapshotIdentifier']))

def get_cfn_stacks(client):
    pass

def destroy_cfn_stack(client, cfn_stack):
    """ Destroys a Cloudformation stack """
    #TODO
    pass

def get_old_instances(ec2, rds, dry_run=True, debug=True):
    """ Gets RDS instances slated for decomm """
    isolated_sgs = get_isolated_sgs(ec2)
    old_instances = []
    for group in isolated_sgs.values():
        isolated_instances = get_instances_with_sg(rds, group)
        for instance in isolated_instances:
            old_instances.append(instance)

    if debug:
        for instance in old_instances:
            print(instance['DBInstanceIdentifier'])
        print("%s instances found." % len(old_instances))
    return old_instances

def snapshot_old_rds_instances(rds, old_instances, dry_run=True):
    """ Performs a final snapshot on old RDS instances. """
    for instance in old_instances:
        if dry_run:
            take_snapshot(rds, instance)
        else:
            print("DRYRUN: Would have taken a snapshot of %s" % instance['DBInstanceIdentifier'])

def prep_rds_instances_for_decomm(ec2, rds, cloudwatch, dry_run=True, debug=True):
    """
    Finds RDS instances with low connection counts and
    applies an isolated SG, sizes it down and sets to single AZ
    """

    isolated_sgs = get_isolated_sgs(ec2)
    all_rds_instances = get_rds_instances(rds)
    if debug:
        print("DEBUG: Number of RDS instances found: %s" % len(all_rds_instances))
    all_rds_stats = get_connections_statistics(cloudwatch, all_rds_instances)
    if debug:
        print("DEBUG: Isolated SGs {}".format(isolated_sgs))
        print("DEBUG: All RDS Instances: ")
        for instance in all_rds_instances:
            print(instance['DBInstanceIdentifier'])
    abandoned_instances = []
    if len(excluded_instances) > 0:
        print("\nThe following instances meet low connections criteria, but have been excluded.")
    for key in all_rds_stats:
        if all_rds_stats[key] == 0 and key not in excluded_instances:
            abandoned_instances.append(key)
        elif all_rds_stats[key] == 0 and key in excluded_instances:
            print(key)
        if debug:
            print("DEBUG: Instance: %s. Connections: %s" % (key, all_rds_stats[key]))
    if len(abandoned_instances) > 0:
        print("\nThe following instances appear to be abandoned. Please investigate.")
        for instance in abandoned_instances:
            print(instance)
    else:
        print("\nNo instances appear to be abandoned.")
        sys.exit(0)
    print("\nTaking action on the following instances: ")
    for rds_instance in all_rds_instances:
        if rds_instance['DBInstanceIdentifier'] in abandoned_instances:
            if dry_run:
                print("DRYRUN: %s would have been isolated and downsized."
                      % rds_instance['DBInstanceIdentifier'])
            else:
                print("Isolating and downsizing instance: %s"
                      % rds_instance['DBInstanceIdentifier'])
                set_security_group(rds,
                                   rds_instance,
                                   isolated_sgs[rds_instance['DBSubnetGroup']['VpcId']])

                set_instance_size(rds,
                                  rds_instance,
                                  'db.t2.micro')

                set_no_multiaz(rds, rds_instance)

def main():
    dry_run = True
    debug = True

    session = get_session('', '')
    ec2 = get_ec2_client(session)
    rds = get_rds_client(session)
    cfn = get_cloudwatch_client(session)

    # prep_rds_instances_for_decomm(ec2, rds, cfn, dry_run, debug)
    old_instances = get_old_instances(ec2, rds, dry_run, debug)
    snapshot_old_rds_instances(rds, old_instances, dry_run)

main()
<commit_msg>Group header comments<commit_after>#!/usr/bin/env python3
"""
RDS Functions
"""

import sys
from datetime import datetime, timedelta
from exclusions import excluded_instances
import boto3
import botocore

## Session/client setup operations
def get_session(access_key_id, secret_access_key):
    " Establishes a session with AWS "
    return boto3.session.Session(
        aws_access_key_id=access_key_id,
        aws_secret_access_key=secret_access_key
    )

def get_rds_client(session):
    " Returns a RDS boto client "
    return session.client('rds')

def get_cloudwatch_client(session):
    " Returns a CloudWatch boto client "
    return session.client('cloudwatch')

def get_ec2_client(session):
    " Returns a EC2 boto client "
    return session.client('ec2')

## EC2 operations
def get_vpc_ids(client):
    " Returns a list of VPC IDs in the account "
    vpc_ids = []
    vpcs = client.describe_vpcs()['Vpcs']
    for vpc in vpcs:
        vpc_ids.append(vpc['VpcId'])
    return vpc_ids

def get_isolated_sgs(client):
    " Returns a dict of rds-isolate SG IDs for each VPC in account. "
    vpc_ids = get_vpc_ids(client)
    isolated_sgs = {}
    for vpc in vpc_ids:
        sec_groups = client.describe_security_groups(
            Filters=[
                {
                    "Name": "vpc-id",
                    "Values": [vpc]
                },
                {
                    "Name": "group-name",
                    "Values": ["rds-isolate"]
                }
            ]
        )['SecurityGroups']
        try:
            isolated_sgs[vpc] = sec_groups[0]['GroupId']
        except IndexError:
            print("No rds-isolate group found for VPC: {}".format(vpc))
    return isolated_sgs

## Cloudwatch operations
def get_connections_statistics(client, rds_instances):
    " Returns a dict of all instances and their avg DB conns over all datapoints "
    rds_stats = {}
    for rds_instance in rds_instances:
        stats = client.get_metric_statistics(
            Namespace="AWS/RDS",
            MetricName="DatabaseConnections",
            Statistics=['Average'],
            Period=57600,
            StartTime=(datetime.today() - timedelta(days=3)),
            EndTime=datetime.today(),
            Dimensions=[
                {
                    'Name': 'DBInstanceIdentifier',
                    'Value': rds_instance['DBInstanceIdentifier']
                }
            ]
        )['Datapoints']
        datapoints = []
        for stat in stats:
            datapoints.append(stat['Average'])
        if len(datapoints) > 0:
            dp_conns = sum(datapoints)/float(len(datapoints))
            rds_stats[rds_instance['DBInstanceIdentifier']] = dp_conns
        else:
            print("Instance: %s has no datapoints." % rds_instance['DBInstanceIdentifier'])

    return rds_stats

## RDS operations
def get_rds_instances(client, vpc_id=None):
    " Gets all RDS instances, per VPC, if specified. "

    rds_instances = []
    resp = client.describe_db_instances()
    while 'Marker' in resp:
        rds_instances.extend(resp['DBInstances'])
        resp = client.describe_db_instances(Marker=resp['Marker'])
    rds_instances.extend(resp['DBInstances'])
    if not vpc_id:
        return rds_instances
    else:
        return [r for r in rds_instances if r['DBSubnetGroup']['VpcId'] == vpc_id]

def set_no_multiaz(client, rds_instance):
    " Takes a rds instance obj and turns off MultiAZ "
    try:
        client.modify_db_instance(
            DBInstanceIdentifier=rds_instance['DBInstanceIdentifier'],
            MultiAZ=False,
            ApplyImmediately=True
        )
    except botocore.exceptions.ClientError:
        print("Error setting no-multiaz on instance %s" % rds_instance['DBInstanceIdentifier'])

def set_security_group(client, rds_instance, sg_id):
    " Sets the rds_instance Security Group to sg_id "
    try:
        client.modify_db_instance(
            DBInstanceIdentifier=rds_instance['DBInstanceIdentifier'],
            VpcSecurityGroupIds=[sg_id]
        )
    except botocore.exceptions.ClientError:
        print("Error setting SG on instance %s" % rds_instance['DBInstanceIdentifier'])

def set_instance_size(client, rds_instance, size=None):
    " Sets instance to the smallest available size "
    if not size:
        available_sizes = client.describe_orderable_db_instance_options(
            Engine=rds_instance['Engine']
        )['OrderableDBInstanceOptions']
        size = available_sizes[0]['DBInstanceClass']
    try:
        client.modify_db_instance(
            DBInstanceIdentifier=rds_instance['DBInstanceIdentifier'],
            DBInstanceClass=size,
            ApplyImmediately=True
        )
    except botocore.exceptions.ClientError:
        print("Error setting size on instance %s" % rds_instance['DBInstanceIdentifier'])

def get_instances_with_sg(client, sg_id, vpc_id=None):
    """ Gets all RDS instances that are using the sg_id """
    rds_instances = get_rds_instances(client, vpc_id)

    instances_with_sg = []
    for instance in rds_instances:
        sgs = instance['VpcSecurityGroups']
        for sg in sgs:
            if sg['VpcSecurityGroupId'] == sg_id:
                instances_with_sg.append(instance)

    return instances_with_sg

def take_snapshot(client, rds_instance):
    """ Takes a snapshot of an RDS instance """

    resp = client.create_db_snapshot(
        DBSnapshotIdentifier='%s-FINAL' % rds_instance['DBInstanceIdentifier'],
        DBInstanceIdentifier=rds_instance['DBInstanceIdentifier'],
    )
    print("Created final snapshot for %s, %s"
          % (rds_instance['DBInstanceIdentifier'], resp['DBSnapshot']['DBSnapshotIdentifier']))

## CloudFormation operations
def get_cfn_stacks(client):
    pass

def destroy_cfn_stack(client, cfn_stack):
    """ Destroys a Cloudformation stack """
    #TODO
    pass

##
def get_old_instances(ec2, rds, dry_run=True, debug=True):
    """ Gets RDS instances slated for decomm """
    isolated_sgs = get_isolated_sgs(ec2)
    old_instances = []
    for group in isolated_sgs.values():
        isolated_instances = get_instances_with_sg(rds, group)
        for instance in isolated_instances:
            old_instances.append(instance)

    if debug:
        for instance in old_instances:
            print(instance['DBInstanceIdentifier'])
        print("%s instances found." % len(old_instances))
    return old_instances

def snapshot_old_rds_instances(rds, old_instances, dry_run=True):
    """ Performs a final snapshot on old RDS instances. """
    for instance in old_instances:
        if dry_run:
            take_snapshot(rds, instance)
        else:
            print("DRYRUN: Would have taken a snapshot of %s" % instance['DBInstanceIdentifier'])

def prep_rds_instances_for_decomm(ec2, rds, cloudwatch, dry_run=True, debug=True):
    """
    Finds RDS instances with low connection counts and
    applies an isolated SG, sizes it down and sets to single AZ
    """

    isolated_sgs = get_isolated_sgs(ec2)
    all_rds_instances = get_rds_instances(rds)
    if debug:
        print("DEBUG: Number of RDS instances found: %s" % len(all_rds_instances))
    all_rds_stats = get_connections_statistics(cloudwatch, all_rds_instances)
    if debug:
        print("DEBUG: Isolated SGs {}".format(isolated_sgs))
        print("DEBUG: All RDS Instances: ")
        for instance in all_rds_instances:
            print(instance['DBInstanceIdentifier'])
    abandoned_instances = []
    if len(excluded_instances) > 0:
        print("\nThe following instances meet low connections criteria, but have been excluded.")
    for key in all_rds_stats:
        if all_rds_stats[key] == 0 and key not in excluded_instances:
            abandoned_instances.append(key)
        elif all_rds_stats[key] == 0 and key in excluded_instances:
            print(key)
        if debug:
            print("DEBUG: Instance: %s. Connections: %s" % (key, all_rds_stats[key]))
    if len(abandoned_instances) > 0:
        print("\nThe following instances appear to be abandoned. Please investigate.")
        for instance in abandoned_instances:
            print(instance)
    else:
        print("\nNo instances appear to be abandoned.")
        sys.exit(0)
    print("\nTaking action on the following instances: ")
    for rds_instance in all_rds_instances:
        if rds_instance['DBInstanceIdentifier'] in abandoned_instances:
            if dry_run:
                print("DRYRUN: %s would have been isolated and downsized."
                      % rds_instance['DBInstanceIdentifier'])
            else:
                print("Isolating and downsizing instance: %s"
                      % rds_instance['DBInstanceIdentifier'])
                set_security_group(rds,
                                   rds_instance,
                                   isolated_sgs[rds_instance['DBSubnetGroup']['VpcId']])

                set_instance_size(rds,
                                  rds_instance,
                                  'db.t2.micro')

                set_no_multiaz(rds, rds_instance)

def main():
    dry_run = True
    debug = True

    session = get_session('', '')
    ec2 = get_ec2_client(session)
    rds = get_rds_client(session)
    cfn = get_cloudwatch_client(session)

    # prep_rds_instances_for_decomm(ec2, rds, cfn, dry_run, debug)
    old_instances = get_old_instances(ec2, rds, dry_run, debug)
    snapshot_old_rds_instances(rds, old_instances, dry_run)

main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
import subprocess
import sys
import os
import hashlib
import shutil
from glob import glob
from premis import make_premis
from premis import write_premis
from premis import make_agent
from premis import make_event
import uuid

'''
1. Accepts the wav as input
2. in /audio subfolder, the only files should be the audio.wav and the log.txt
3. workhorse copy sent to desktop
4. mediainfo in /xml_files
5. framemd5 in /md5
6. manifest in audio directory
7. log gets moved to /log
8. audio gets moved to aeo_raw_extract_wav
9. To DO - Premis/revtmd.

usage = python rawaudio.py audio.wav

'''
def set_environment(logfile):
    env_dict = os.environ.copy()
    # https://github.com/imdn/scripts/blob/0dd89a002d38d1ff6c938d6f70764e6dd8815fdd/ffmpy.py#L272
    env_dict['FFREPORT'] = 'file={}:level=48'.format(logfile)
    return env_dict

input = sys.argv[1]
desktop_dir = os.path.expanduser("~/Desktop/%s") % os.path.basename(input)
parent_dir = os.path.dirname(input)
md5_file = input + '.md5'
metadata_dir = parent_dir + '/xml_files'
md5_dir = parent_dir + '/md5'
logs_dir = parent_dir + '/log'
aeo_raw_extract_wav_dir = parent_dir + '/aeo_raw_extract_wav'
framemd5 = md5_dir + '/' + os.path.basename(input) +'.framemd5'
logfile = logs_dir + '/' + os.path.basename(input) + '_framemd5.log'
normpath = os.path.normpath(parent_dir)
relative_path = normpath.split(os.sep)[-1]
manifest =  '%s_manifest.md5' % (relative_path)

filenoext = os.path.splitext(input)[0]
os.makedirs(metadata_dir)
os.makedirs(md5_dir)
os.makedirs(aeo_raw_extract_wav_dir)
os.makedirs(logs_dir)

inputxml =  inputxml  = "%s/%s_mediainfo.xml" % (metadata_dir,os.path.basename(input) )
print inputxml
def make_mediainfo(xmlfilename, xmlvariable, inputfilename):
    with open(xmlfilename, "w+") as fo:
        xmlvariable = subprocess.check_output(['mediainfo',
                        '-f',
                        '--language=raw', # Use verbose output.
                        '--output=XML',
                        inputfilename])       #input filename
        fo.write(xmlvariable)

make_mediainfo(inputxml,'mediaxmlinput',input)

env_dict = set_environment(logfile)
ffmpegcmd = ['ffmpeg',    # Create decoded md5 checksums for every frame of the ffv1 output
                        '-i',input,
                        '-report',
                        '-f','framemd5',
                        framemd5 ]
subprocess.call(ffmpegcmd,env=env_dict)
shutil.copy(input, desktop_dir)

def remove_bad_files(root_dir):
    rm_these = ['.DS_Store', 'Thumbs.db', 'desktop.ini']
    for root, dirs, files in os.walk(root_dir):
        for name in files:
            path = os.path.join(root, name)
            for i in rm_these:
                if name == i:
                    print '***********************' + 'removing: ' + path
                    os.remove(path)

remove_bad_files(parent_dir)
shutil.move(input,aeo_raw_extract_wav_dir + '/' + os.path.basename(input))
os.chdir(parent_dir)
log_files =  glob('*.txt')
for i in log_files:

        shutil.move(i, '%s/%s' % (logs_dir,i))
def make_manifest(relative_manifest_path, manifest_textfile):
    print relative_manifest_path
    os.chdir(relative_manifest_path)
    manifest_generator = subprocess.check_output(['md5deep', '-ler', '.'])
    manifest_list = manifest_generator.splitlines()
    # http://stackoverflow.com/a/31306961/2188572
    manifest_list = sorted(manifest_list,  key=lambda x:(x[34:]))
    with open(manifest_textfile,"wb") as fo:
        for i in manifest_list:
            fo.write(i + '\n')
make_manifest(parent_dir,manifest)
split_list = os.path.basename(os.path.dirname(os.path.dirname(sys.argv[1]))).split('_')
items = {"workflow":"raw audio","oe":split_list[0], "filmographic":split_list[1], "sourceAccession":split_list[2], "interventions":['placeholder'], "prepList":['placeholder'], "user":'Brian Cash'}
xml_info    = make_premis(aeo_raw_extract_wav_dir, items)
doc         = xml_info[0]
premisxml   = xml_info[1]
premis = doc.getroot()
capture_uuid                                = str(uuid.uuid4())
capture_received_uuid                       = str(uuid.uuid4())
premis_checksum_uuid                        = str(uuid.uuid4())
framemd5_uuid                               = str(uuid.uuid4())
aeolightAgent                               = make_agent(premis,capture_uuid, 'agentaa00005')
hashlibAgent                                = make_agent(premis,capture_uuid, 'agentaa00008')
operatorAgent                               = make_agent(premis,capture_uuid,items['user'])
macMiniTelecineMachineAgent                 = make_agent(premis,premis_checksum_uuid, 'agentaa00022')
macMiniTelecineOSAgent                      = make_agent(premis,premis_checksum_uuid, 'agentaa00011')
ffmpegAgent                                 = make_agent(premis,framemd5_uuid , 'agentaa00006')

make_event(premis, 'creation', 'PCM WAV file extracted from overscanned image area of source TIFF files', [aeolightAgent, operatorAgent, macMiniTelecineMachineAgent, macMiniTelecineOSAgent], capture_uuid,xml_info[2])
make_event(premis, 'message digest calculation', '', [hashlibAgent, operatorAgent,macMiniTelecineMachineAgent, macMiniTelecineOSAgent], premis_checksum_uuid,xml_info[2])
make_event(premis, 'message digest calculation', 'Frame level checksums', [ffmpegAgent, operatorAgent,macMiniTelecineMachineAgent, macMiniTelecineOSAgent], framemd5_uuid,xml_info[2] )
write_premis(doc, premisxml)
print premisxml<commit_msg>rawaudio - cleans up functions<commit_after>#!/usr/bin/env python
import subprocess
import sys
import os
import hashlib
import shutil
from glob import glob
from premis import make_premis
from premis import write_premis
from premis import make_agent
from premis import make_event
import uuid

'''
1. Accepts the wav as input
2. in /audio subfolder, the only files should be the audio.wav and the log.txt
3. workhorse copy sent to desktop
4. mediainfo in /xml_files
5. framemd5 in /md5
6. manifest in audio directory
7. log gets moved to /log
8. audio gets moved to aeo_raw_extract_wav
9. To DO - Premis/revtmd.

usage = python rawaudio.py audio.wav

'''
def set_environment(logfile):
    env_dict = os.environ.copy()
    # https://github.com/imdn/scripts/blob/0dd89a002d38d1ff6c938d6f70764e6dd8815fdd/ffmpy.py#L272
    env_dict['FFREPORT'] = 'file={}:level=48'.format(logfile)
    return env_dict

def make_mediainfo(xmlfilename, xmlvariable, inputfilename):
    with open(xmlfilename, "w+") as fo:
        xmlvariable = subprocess.check_output(['mediainfo',
                        '-f',
                        '--language=raw', # Use verbose output.
                        '--output=XML',
                        inputfilename])       #input filename
        fo.write(xmlvariable)

def remove_bad_files(root_dir):
    rm_these = ['.DS_Store', 'Thumbs.db', 'desktop.ini']
    for root, dirs, files in os.walk(root_dir):
        for name in files:
            path = os.path.join(root, name)
            for i in rm_these:
                if name == i:
                    print '***********************' + 'removing: ' + path
                    os.remove(path)

def make_manifest(relative_manifest_path, manifest_textfile):
    print relative_manifest_path
    os.chdir(relative_manifest_path)
    manifest_generator = subprocess.check_output(['md5deep', '-ler', '.'])
    manifest_list = manifest_generator.splitlines()
    # http://stackoverflow.com/a/31306961/2188572
    manifest_list = sorted(manifest_list,  key=lambda x:(x[34:]))
    with open(manifest_textfile,"wb") as fo:
        for i in manifest_list:
            fo.write(i + '\n')

input = sys.argv[1]
desktop_dir = os.path.expanduser("~/Desktop/%s") % os.path.basename(input)
parent_dir = os.path.dirname(input)
md5_file = input + '.md5'
metadata_dir = parent_dir + '/xml_files'
md5_dir = parent_dir + '/md5'
logs_dir = parent_dir + '/log'
aeo_raw_extract_wav_dir = parent_dir + '/aeo_raw_extract_wav'
framemd5 = md5_dir + '/' + os.path.basename(input) +'.framemd5'
logfile = logs_dir + '/' + os.path.basename(input) + '_framemd5.log'
normpath = os.path.normpath(parent_dir)
relative_path = normpath.split(os.sep)[-1]
manifest =  '%s_manifest.md5' % (relative_path)

filenoext = os.path.splitext(input)[0]
os.makedirs(metadata_dir)
os.makedirs(md5_dir)
os.makedirs(aeo_raw_extract_wav_dir)
os.makedirs(logs_dir)

inputxml =  inputxml  = "%s/%s_mediainfo.xml" % (metadata_dir,os.path.basename(input) )
print inputxml

make_mediainfo(inputxml,'mediaxmlinput',input)

env_dict = set_environment(logfile)
ffmpegcmd = ['ffmpeg',    # Create decoded md5 checksums for every frame of the ffv1 output
                        '-i',input,
                        '-report',
                        '-f','framemd5',
                        framemd5 ]
subprocess.call(ffmpegcmd,env=env_dict)
shutil.copy(input, desktop_dir)
remove_bad_files(parent_dir)
shutil.move(input,aeo_raw_extract_wav_dir + '/' + os.path.basename(input))
os.chdir(parent_dir)
log_files =  glob('*.txt')
for i in log_files:

        shutil.move(i, '%s/%s' % (logs_dir,i))

make_manifest(parent_dir,manifest)
split_list = os.path.basename(os.path.dirname(os.path.dirname(sys.argv[1]))).split('_')
items = {"workflow":"raw audio","oe":split_list[0], "filmographic":split_list[1], "sourceAccession":split_list[2], "interventions":['placeholder'], "prepList":['placeholder'], "user":'Brian Cash'}
xml_info    = make_premis(aeo_raw_extract_wav_dir, items)
doc         = xml_info[0]
premisxml   = xml_info[1]
premis = doc.getroot()
capture_uuid                                = str(uuid.uuid4())
capture_received_uuid                       = str(uuid.uuid4())
premis_checksum_uuid                        = str(uuid.uuid4())
framemd5_uuid                               = str(uuid.uuid4())
aeolightAgent                               = make_agent(premis,capture_uuid, 'agentaa00005')
hashlibAgent                                = make_agent(premis,capture_uuid, 'agentaa00008')
operatorAgent                               = make_agent(premis,capture_uuid,items['user'])
macMiniTelecineMachineAgent                 = make_agent(premis,premis_checksum_uuid, 'agentaa00022')
macMiniTelecineOSAgent                      = make_agent(premis,premis_checksum_uuid, 'agentaa00011')
ffmpegAgent                                 = make_agent(premis,framemd5_uuid , 'agentaa00006')

make_event(premis, 'creation', 'PCM WAV file extracted from overscanned image area of source TIFF files', [aeolightAgent, operatorAgent, macMiniTelecineMachineAgent, macMiniTelecineOSAgent], capture_uuid,xml_info[2])
make_event(premis, 'message digest calculation', '', [hashlibAgent, operatorAgent,macMiniTelecineMachineAgent, macMiniTelecineOSAgent], premis_checksum_uuid,xml_info[2])
make_event(premis, 'message digest calculation', 'Frame level checksums', [ffmpegAgent, operatorAgent,macMiniTelecineMachineAgent, macMiniTelecineOSAgent], framemd5_uuid,xml_info[2] )
write_premis(doc, premisxml)
print premisxml<|endoftext|>


<commit_before>import re
import sys
import os
from subprocess import run, check_output
import atexit
import requests

run(['bash', 'make_readme.sh'], check=True)

if not os.path.exists('README.rst'):
    import pypandoc
    long_description = pypandoc.convert_file('README.md', 'rst', outputfile='README.rst', extra_args=['--no-wrap'])
    atexit.register(lambda: os.remove('README.rst'))
run(['python', 'setup.py', 'check', '-rms'], check=True)

version_pattern = '(\d+(?:\.\d+)+)'
last_version = re.search(version_pattern, open('CHANGES.md').read()).group(1)
print('The last version was: {}'.format(last_version))
new_version = input('Enter new version: ') if len(sys.argv) == 1 else sys.argv[1]
assert re.fullmatch(version_pattern, new_version)

commits = check_output(['git', 'log', 'v{}..HEAD'.format(last_version), '--oneline'], universal_newlines=True)
with open('message.txt', 'w') as message_file:
    atexit.register(lambda: os.remove('message.txt'))

    message_file.write('\n\n\n')
    message_file.write('# Enter changes one per line like this:\n')
    message_file.write('# - Added `foobar`.\n\n\n')
    message_file.write('# As a reminder, here\'s the last commits since version {}:\n\n'.format(last_version))
    for line in commits.strip().split('\n'):
        message_file.write('# {}\n'.format(line))

run(['vim', 'message.txt'])
with open('message.txt') as message_file:
    lines = [line for line in message_file.readlines() if not line.startswith('#')]
message = '\n'.join(lines).strip()
if not message:
    print('Aborting release due to empty message.')
    exit()
with open('message.txt', 'w') as message_file:
    message_file.write(message)

with open('CHANGES.md') as changes_file:
    old_changes = changes_file.read()
with open('CHANGES.md', 'w') as changes_file:
    changes_file.write('# {}\n\n{}\n\n\n{}'.format(new_version, message, old_changes))

tag_name = 'v' + new_version
if input('Commit CHANGES.md file? ').lower().startswith('y'):
    run(['git', 'add', 'CHANGES.md'])
    run(['git', 'commit', '-m', 'Update changes for {}'.format(tag_name)])
run(['git', 'tag', '-a', tag_name, '--file', 'message.txt'], check=True)
run(['git', 'push', 'origin', tag_name], check=True)

token = input('To make a release enter your GitHub repo authorization token: ').strip()
if token:
    git_remotes = check_output(['git', 'remote', '-v']).decode('utf-8')
    repo_path = re.search(r'([^/]+/[^/. ]+)(?:\.git)? \(push\)', git_remotes).group(1)
    releases_url = 'https://api.github.com/repos/{}/releases'.format(repo_path)
    print(releases_url)
    release = {
        "tag_name": tag_name,
        "target_commitish": "master",
        "name": tag_name,
        "body": message,
        "draft": False,
        "prerelease": False,
    }
    response = requests.post(releases_url, json=release, headers={'Authorization': 'token ' + token})
    print(response.status_code, response.text)

run(['python', 'setup.py', 'sdist', '--format=zip', 'bdist', '--format=zip', 'bdist_wheel', '--universal', 'bdist_wininst', 'register', 'upload'], check=True)<commit_msg>Avoid unnecessary blank lines in CHANGES.md<commit_after>import re
import sys
import os
from subprocess import run, check_output
import atexit
import requests

run(['bash', 'make_readme.sh'], check=True)

if not os.path.exists('README.rst'):
    import pypandoc
    long_description = pypandoc.convert_file('README.md', 'rst', outputfile='README.rst', extra_args=['--no-wrap'])
    atexit.register(lambda: os.remove('README.rst'))
run(['python', 'setup.py', 'check', '-rms'], check=True)

version_pattern = '(\d+(?:\.\d+)+)'
last_version = re.search(version_pattern, open('CHANGES.md').read()).group(1)
print('The last version was: {}'.format(last_version))
new_version = input('Enter new version: ') if len(sys.argv) == 1 else sys.argv[1]
assert re.fullmatch(version_pattern, new_version)

commits = check_output(['git', 'log', 'v{}..HEAD'.format(last_version), '--oneline'], universal_newlines=True)
with open('message.txt', 'w') as message_file:
    atexit.register(lambda: os.remove('message.txt'))

    message_file.write('\n\n\n')
    message_file.write('# Enter changes one per line like this:\n')
    message_file.write('# - Added `foobar`.\n\n\n')
    message_file.write('# As a reminder, here\'s the last commits since version {}:\n\n'.format(last_version))
    for line in commits.strip().split('\n'):
        message_file.write('# {}\n'.format(line))

run(['vim', 'message.txt'])
with open('message.txt') as message_file:
    lines = [line for line in message_file.readlines() if not line.startswith('#')]
message = ''.join(lines).strip()
if not message:
    print('Aborting release due to empty message.')
    exit()
with open('message.txt', 'w') as message_file:
    message_file.write(message)

with open('CHANGES.md') as changes_file:
    old_changes = changes_file.read()
with open('CHANGES.md', 'w') as changes_file:
    changes_file.write('# {}\n\n{}\n\n\n{}'.format(new_version, message, old_changes))

tag_name = 'v' + new_version
if input('Commit CHANGES.md file? ').lower().startswith('y'):
    run(['git', 'add', 'CHANGES.md'])
    run(['git', 'commit', '-m', 'Update changes for {}'.format(tag_name)])
run(['git', 'tag', '-a', tag_name, '--file', 'message.txt'], check=True)
run(['git', 'push', 'origin', tag_name], check=True)

token = input('To make a release enter your GitHub repo authorization token: ').strip()
if token:
    git_remotes = check_output(['git', 'remote', '-v']).decode('utf-8')
    repo_path = re.search(r'([^/]+/[^/. ]+)(?:\.git)? \(push\)', git_remotes).group(1)
    releases_url = 'https://api.github.com/repos/{}/releases'.format(repo_path)
    print(releases_url)
    release = {
        "tag_name": tag_name,
        "target_commitish": "master",
        "name": tag_name,
        "body": message,
        "draft": False,
        "prerelease": False,
    }
    response = requests.post(releases_url, json=release, headers={'Authorization': 'token ' + token})
    print(response.status_code, response.text)

run(['python', 'setup.py', 'sdist', '--format=zip', 'bdist', '--format=zip', 'bdist_wheel', '--universal', 'bdist_wininst', 'register', 'upload'], check=True)<|endoftext|>


<commit_before><commit_msg>Revert "SKINSTXTRCOM-4537 | Using `getNewPreviewBasket` call for cardcode - adding/removing voucher."<commit_after><|endoftext|>


<commit_before>from contextlib import contextmanager
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

from db import model
from constants import PALINDROME_DB_URI

def get_session():
    # TODO docs
    engine = create_engine(PALINDROME_DB_URI)
    session_maker = sessionmaker(bind=engine)

    return session_maker()

@contextmanager
def connection():
    # TODO docs
    session = get_session()
    try:
        yield session
        session.commit()
    except:
        session.rollback()
        raise
    finally:
        session.close()

def get_messages(session):
    messages = session.query(model.Messages).all()
    if messages:
        messages = messages.order_by(model.Messages.created_on)
    return messages

def get_message(session, message_id):
    return session.query(model.Messages).filter(
        model.Messages.id == message_id
    ).first()

def add_message(session, msg, username, is_palindrome):
    message = model.Messages(msg, username, is_palindrome)
    session.add(message)
    session.flush()

    return message

def delete_message(session, message_id):
    message = session.query(model.Messages).filter(
        model.Messages.id == message_id
    )
    if message:
        session.delete(message)
        session.flush()
    return message
<commit_msg>PEP8 and formatting<commit_after>from contextlib import contextmanager
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

from db import model
from constants import PALINDROME_DB_URI

def get_session():
    # TODO docs
    engine = create_engine(PALINDROME_DB_URI)
    session_maker = sessionmaker(bind=engine)

    return session_maker()

@contextmanager
def connection():
    # TODO docs
    session = get_session()
    try:
        yield session
        session.commit()
    except:
        session.rollback()
        raise
    finally:
        session.close()

def get_messages(session):
    messages = session.query(model.Messages)
    if messages:
        messages = messages.order_by(model.Messages.created_on)
    return messages.all()

def get_message(session, message_id):
    return session.query(model.Messages).filter(
        model.Messages.id == message_id
    ).first()

def add_message(session, msg, username, is_palindrome):
    message = model.Messages(
        message=msg,
        username=username,
        is_palindrome=is_palindrome
    )
    session.add(message)
    session.flush()

    return message

def delete_message(session, message_id):
    message = session.query(model.Messages).filter(
        model.Messages.id == message_id
    )
    if message:
        session.delete(message)
        session.flush()
    return message
<|endoftext|>


<commit_before>from django.db import models
from rapidsms.models import Contact
from unregister.models import Blacklist

import logging
logger = logging.getLogger(__name__)

from .school import School

class EmisReporterManager(models.Manager):
    def get_query_set(self):
        return super(EmisReporterManager, self).get_query_set().exclude(
            connection__in=Blacklist.objects.values_list('connection', flat=True))

class EmisReporter(Contact):
    CLASS_CHOICES = (
        ('P3', 'P3'),
        ('P6', 'P6'))

    grade = models.CharField(max_length=2, choices=CLASS_CHOICES, null=True)
    schools = models.ManyToManyField(School, null=True)
    has_exact_matched_school = models.BooleanField(default=True)
    objects = EmisReporterManager()
    last_reporting_date = models.DateTimeField(null=True)

    class Meta:
        ordering = ["name"]
        app_label = 'education'

    def __unicode__(self):
        return self.name

    def is_member_of(self, group):
        grps = self.groups.objects.values_list('name', flat=True)
        return group.lower() in [grp.lower for grp in grps]

    def schools_list(self):
        return self.schools.values_list('name', flat=True)

def update_last_reporting_date(sender, instance, **kwargs):
    created = kwargs.get('created', False)
    if created:
        if instance.direction == 'I':
            logger.info('Updating last reporting date for reporter - New Message')
            reporter = instance.connection.contact.emisreporter
            reporter.last_reporting_date = instance.date
            reporter.save()
    else:
        logger.info("This is not a new message - Don't update last reporting date")<commit_msg>bug that stops reporters from joining<commit_after>from django.db import models
from rapidsms.models import Contact
from unregister.models import Blacklist

import logging
logger = logging.getLogger(__name__)

from .school import School

class EmisReporterManager(models.Manager):
    def get_query_set(self):
        return super(EmisReporterManager, self).get_query_set().exclude(
            connection__in=Blacklist.objects.values_list('connection', flat=True))

class EmisReporter(Contact):
    CLASS_CHOICES = (
        ('P3', 'P3'),
        ('P6', 'P6'))

    grade = models.CharField(max_length=2, choices=CLASS_CHOICES, null=True)
    schools = models.ManyToManyField(School, null=True)
    has_exact_matched_school = models.BooleanField(default=True)
    objects = EmisReporterManager()
    last_reporting_date = models.DateTimeField(null=True)

    class Meta:
        ordering = ["name"]
        app_label = 'education'

    def __unicode__(self):
        return self.name

    def is_member_of(self, group):
        grps = self.groups.objects.values_list('name', flat=True)
        return group.lower() in [grp.lower for grp in grps]

    def schools_list(self):
        return self.schools.values_list('name', flat=True)

def update_last_reporting_date(sender, instance, **kwargs):
    created = kwargs.get('created', False)
    if created:
        if instance.direction == 'I':
            logger.info('Updating last reporting date for reporter - New Message')
            reporter = instance.connection
            if reporter.contact:
                reporter = reporter.contact.emisreporter
                reporter.last_reporting_date = instance.date
                reporter.save()
    else:
        logger.info("This is not a new message - Don't update last reporting date")<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import unicode_literals

import os
import cgi
import time
from io import BytesIO

import pytest

import tabun_api as api
from tabun_api.compat import urequest, text, text_types, binary, PY2

if PY2:
    from httplib import HTTPMessage
    def parse_headers(fp):
        return HTTPMessage(fp)
    import urlparse
else:
    from http.client import parse_headers
    from urllib import parse as urlparse

# для выбора загружаемых страниц из каталога data
guest_mode = False

data_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data')
file_cache = {}

templates = {
    'HEADER1': 'header1.html',
    'HEADER2': 'header2.html',
    'BODY': 'body.html',
    'AUTH1_GUEST': 'auth1_guest.html',
    'AUTH2_GUEST': 'auth2_guest.html',
    'AUTH1_AUTHORIZED': 'auth1_authorized.html',
    'AUTH2_AUTHORIZED': 'auth2_authorized.html',
    'NO_SIDEBAR': 'no_sidebar.html',
    'SIDEBAR': 'sidebar.html',
    'SIDEBAR_PROFILE': 'sidebar_profile.html',
    'SIDEBAR_PROFILE_TOPICS': 'sidebar_profile_topics.html',
    'FOOTER': 'footer.html',
    'ACTIVITY_ITEMS': 'activity_items.html',
}

context_templates = ['AUTH1', 'AUTH2']

interceptors = {}
form_interceptors = {}
current_mocks = {}

mocks = {
    '/': ('index.html', None),
    '/login/': ('login.html', None),
    '/blog/132085.html': ('132085.html', None),
    '/blog/borderline/138982.html': ('138982.html', None),
    '/blog/138983.html': ('138983.html', None),
    '/comments/': ('comments.html', None),
    '/stream/all/': ('activity.html', None),
}

@pytest.yield_fixture(scope='function')
def set_mock():
    try:
        yield load_mocks
    finally:
        clear_mock()

@pytest.yield_fixture(scope='function')
def intercept():
    def dfunc(url):
        def decorator(f):
            interceptors[url] = f
            return f
        return decorator
    try:
        yield dfunc
    finally:
        interceptors.clear()

@pytest.yield_fixture(scope='function')
def form_intercept():
    def dfunc(url):
        def decorator(f):
            form_interceptors[url] = f
            return f
        return decorator
    try:
        yield dfunc
    finally:
        form_interceptors.clear()

@pytest.yield_fixture(scope='function')
def as_guest():
    global guest_mode
    guest_mode = True
    try:
        yield
    finally:
        guest_mode = False

@pytest.fixture
def user():
    return UserTest()

def load_mocks(new_mocks):
    global current_mocks
    for key, value in new_mocks.items():
        current_mocks[key] = ((value, None) if isinstance(value, text_types) else value)

def clear_mock():
    current_mocks.clear()

def load_file(name, ignorekeys=(), template=True):
    # Загружаем файл или достаём из кэша
    path = os.path.join(data_dir, name)
    if name in file_cache:
        data = file_cache[name]
    else:
        data = open(path, 'rb').read()
        file_cache[name] = data

    if not template:
        return data

    # Для этих шаблонов имеется два режима — неавторизованного и авторизованного пользователя
    for metakey in context_templates:
        bmetakey = b'%' + metakey.encode('utf-8') + b'%'
        if metakey not in ignorekeys and bmetakey in data:
            key = (metakey + '_GUEST') if guest_mode else (metakey + '_AUTHORIZED')
            tname = templates[key]
            data = data.replace(bmetakey, load_file(tname, ignorekeys + (key,)))

    # Пародируем шаблонизатор и включаем другие файлы в шаблон
    for key, tname in templates.items():
        bkey = b'%' + key.encode('utf-8') + b'%'
        if key not in ignorekeys and bkey in data:
            data = data.replace(bkey, load_file(tname, ignorekeys + (key,)))

    return data

def assert_data(obj, data, exclude=('post_id', 'comment_id')):
    for key, value in data.items():
        if key == 'time' and value is not None:
            assert time.strftime("%Y-%m-%d %H:%M:%S", obj.time) == value

        elif key == 'utctime' and value is not None:
            assert obj.utctime.strftime('%Y-%m-%d %H:%M:%S') == value

        elif key.startswith('context:'):
            ov = obj.context[key[8:]]
            if value is True or value is False or value is None:  # 1 == True, but 1 is not True
                assert ov is value
            else:
                assert ov is not True and ov is not False and ov is not None
                assert ov == value

        elif key not in exclude:
            ov = getattr(obj, key)
            if value is True or value is False or value is None:
                assert ov is value
            else:
                assert ov is not True and ov is not False and ov is not None
                assert ov == value

def build_response(req_url, result_path, optparams=None):
    # Параметры ответа по умолчанию
    params = {
        'status': 200,
        'status_msg': 'OK',
        'headers': {
            'Content-Type': 'text/html; charset=utf-8',
            'Set-Cookie': [
                'TABUNSESSIONID=abcdef9876543210abcdef9876543210; path=/',
            ]},
        'url': req_url
    }
    # Параметры, переданные тестом
    if optparams:
        optparams = optparams.copy()
        params['headers'].update(optparams.pop('headers', {}))
        params.update(optparams)

    # Само содержимое подделываемого ответа на HTTP-запрос
    fp = BytesIO(load_file(result_path) if result_path else params.get('data', b''))

    # Собираем HTTP-заголовки
    raw_headers = ''
    for header, value in params['headers'].items():
        raw_headers += header + ': '
        if isinstance(value, (tuple, list)):
            raw_headers += ('\r\n' + header + ': ').join(value)
        else:
            raw_headers += value
        raw_headers += '\r\n'
    headers = parse_headers(BytesIO(raw_headers.encode('utf-8')))

    # Для некоторых ошибок нужно сгенерировать исключение
    if (params['status'] >= 500 or params['status'] in (404,)) and not params.get('noexc'):
        raise urequest.HTTPError(params['url'], params['status'], params['status_msg'], headers, fp)

    # Собираем ответ на HTTP-запрос
    resp = urequest.addinfourl(fp, headers, params['url'])
    resp.code = params['status']
    resp.msg = params['status_msg']
    return resp

class UserTest(api.User):
    def __init__(self, *args, **kwargs):
        if 'avoid_cf' not in kwargs:
            kwargs['avoid_cf'] = False  # Чтобы не тратить время на импорт js2py
        self.sleeps = [0, 0.0]
        super(UserTest, self).__init__(*args, **kwargs)

    def sleep_func(self, tm):
        self.sleeps[0] += 1
        self.sleeps[1] += tm

    def test_open(self, fullurl, data=None, timeout=None):
        if isinstance(fullurl, (text, binary)):
            req_url = fullurl
        else:
            assert data is None
            data = fullurl.data
            req_url = fullurl.get_full_url()

        # Тестируем совместимость с разными питонами и попутно собираем заголовки
        headers = {}
        if PY2:
            assert isinstance(req_url, binary)
            if req_url is not fullurl:
                for h, v in fullurl.headers.items():
                    assert isinstance(h, binary)
                    assert isinstance(v, binary)
                    headers[h.decode('utf-8').lower()] = v
        else:
            assert isinstance(req_url, text)
            if req_url is not fullurl:
                for h, v in fullurl.headers.items():
                    assert isinstance(h, text)
                    assert isinstance(v, binary)  # значение всегда binary
                    headers[h.lower()] = v

        assert data is None or isinstance(data, binary)
        assert timeout is None or isinstance(timeout, (int, float))

        # Потестировали, а дальше проще с юникодом работать
        if PY2:
            req_url = req_url.decode('utf-8')

        # Перехват запроса при необходимости, можно его параметры assert'ить здесь
        for url, func in form_interceptors.items():
            if req_url != url and req_url != api.http_host + url:
                continue
            ctype = headers['content-type'].decode('utf-8')
            if ctype.startswith('multipart/form-data;'):
                pdict = cgi.parse_header(headers['content-type'] if PY2 else ctype)[1]
                data = cgi.parse_multipart(BytesIO(data), {'boundary': pdict['boundary'].encode('utf-8')})
            else:
                data = urlparse.parse_qs(data.decode('utf-8'))
            func(data, headers)
            break

        for url, func in interceptors.items():
            if req_url == url or req_url == api.http_host + url:
                func(data, headers)
                break

        # Ищем ответ на запрос
        for url, x in list(current_mocks.items()) + list(mocks.items()):
            result_path, optparams = x
            if url.startswith('/'):
                url = api.http_host + url
            if req_url == url:
                # Отвечаем, если нашёлся
                return build_response(req_url, result_path, optparams)

        # Ответ не нашёлся, отвечаем 404
        params = {'status': 404, 'status_msg': 'Not Found'}
        if '404' in mocks:
            if mocks['404'][1]:
                params.update(mocks['404'][1])
            return build_response(req_url, mocks['404'][0], params)
        else:
            return build_response(req_url, '404.html', params)

    def send_request(self, *args, **kwargs):
        self.opener.open = self.test_open
        self.noredir.open = self.test_open
        return super(UserTest, self).send_request(*args, **kwargs)
<commit_msg>Add error messages for some asserts<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import unicode_literals

import os
import cgi
import time
from io import BytesIO

import pytest

import tabun_api as api
from tabun_api.compat import urequest, text, text_types, binary, PY2

if PY2:
    from httplib import HTTPMessage
    def parse_headers(fp):
        return HTTPMessage(fp)
    import urlparse
else:
    from http.client import parse_headers
    from urllib import parse as urlparse

# для выбора загружаемых страниц из каталога data
guest_mode = False

data_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data')
file_cache = {}

templates = {
    'HEADER1': 'header1.html',
    'HEADER2': 'header2.html',
    'BODY': 'body.html',
    'AUTH1_GUEST': 'auth1_guest.html',
    'AUTH2_GUEST': 'auth2_guest.html',
    'AUTH1_AUTHORIZED': 'auth1_authorized.html',
    'AUTH2_AUTHORIZED': 'auth2_authorized.html',
    'NO_SIDEBAR': 'no_sidebar.html',
    'SIDEBAR': 'sidebar.html',
    'SIDEBAR_PROFILE': 'sidebar_profile.html',
    'SIDEBAR_PROFILE_TOPICS': 'sidebar_profile_topics.html',
    'FOOTER': 'footer.html',
    'ACTIVITY_ITEMS': 'activity_items.html',
}

context_templates = ['AUTH1', 'AUTH2']

interceptors = {}
form_interceptors = {}
current_mocks = {}

mocks = {
    '/': ('index.html', None),
    '/login/': ('login.html', None),
    '/blog/132085.html': ('132085.html', None),
    '/blog/borderline/138982.html': ('138982.html', None),
    '/blog/138983.html': ('138983.html', None),
    '/comments/': ('comments.html', None),
    '/stream/all/': ('activity.html', None),
}

@pytest.yield_fixture(scope='function')
def set_mock():
    try:
        yield load_mocks
    finally:
        clear_mock()

@pytest.yield_fixture(scope='function')
def intercept():
    def dfunc(url):
        def decorator(f):
            interceptors[url] = f
            return f
        return decorator
    try:
        yield dfunc
    finally:
        interceptors.clear()

@pytest.yield_fixture(scope='function')
def form_intercept():
    def dfunc(url):
        def decorator(f):
            form_interceptors[url] = f
            return f
        return decorator
    try:
        yield dfunc
    finally:
        form_interceptors.clear()

@pytest.yield_fixture(scope='function')
def as_guest():
    global guest_mode
    guest_mode = True
    try:
        yield
    finally:
        guest_mode = False

@pytest.fixture
def user():
    return UserTest()

def load_mocks(new_mocks):
    global current_mocks
    for key, value in new_mocks.items():
        current_mocks[key] = ((value, None) if isinstance(value, text_types) else value)

def clear_mock():
    current_mocks.clear()

def load_file(name, ignorekeys=(), template=True):
    # Загружаем файл или достаём из кэша
    path = os.path.join(data_dir, name)
    if name in file_cache:
        data = file_cache[name]
    else:
        data = open(path, 'rb').read()
        file_cache[name] = data

    if not template:
        return data

    # Для этих шаблонов имеется два режима — неавторизованного и авторизованного пользователя
    for metakey in context_templates:
        bmetakey = b'%' + metakey.encode('utf-8') + b'%'
        if metakey not in ignorekeys and bmetakey in data:
            key = (metakey + '_GUEST') if guest_mode else (metakey + '_AUTHORIZED')
            tname = templates[key]
            data = data.replace(bmetakey, load_file(tname, ignorekeys + (key,)))

    # Пародируем шаблонизатор и включаем другие файлы в шаблон
    for key, tname in templates.items():
        bkey = b'%' + key.encode('utf-8') + b'%'
        if key not in ignorekeys and bkey in data:
            data = data.replace(bkey, load_file(tname, ignorekeys + (key,)))

    return data

def assert_data(obj, data, exclude=('post_id', 'comment_id')):
    for key, value in data.items():
        if key == 'time' and value is not None:
            t = time.strftime("%Y-%m-%d %H:%M:%S", obj.time)
            assert t == value, '%r: expected date %r, got %r' % (key, value, t)

        elif key == 'utctime' and value is not None:
            t = obj.utctime.strftime('%Y-%m-%d %H:%M:%S')
            assert t == value, '%r: expected date %r, got %r' % (key, value, t)

        elif key.startswith('context:'):
            ov = obj.context[key[8:]]
            if value is True or value is False or value is None:  # 1 == True, but 1 is not True
                assert ov is value, '%r: expected %r, got %r' % (key, value, ov)
            else:
                assert ov is not True and ov is not False and ov is not None, '%r: expected not None and not bool' % key
                assert ov == value, '%r: expected %r, got %r' % (key, value, ov)

        elif key not in exclude:
            ov = getattr(obj, key)
            if value is True or value is False or value is None:
                assert ov is value, '%r: expected %r, got %r' % (key, value, ov)
            else:
                assert ov is not True and ov is not False and ov is not None, '%r: expected not None and not bool' % key
                assert ov == value, '%r: expected %r, got %r' % (key, value, ov)

def build_response(req_url, result_path, optparams=None):
    # Параметры ответа по умолчанию
    params = {
        'status': 200,
        'status_msg': 'OK',
        'headers': {
            'Content-Type': 'text/html; charset=utf-8',
            'Set-Cookie': [
                'TABUNSESSIONID=abcdef9876543210abcdef9876543210; path=/',
            ]},
        'url': req_url
    }
    # Параметры, переданные тестом
    if optparams:
        optparams = optparams.copy()
        params['headers'].update(optparams.pop('headers', {}))
        params.update(optparams)

    # Само содержимое подделываемого ответа на HTTP-запрос
    fp = BytesIO(load_file(result_path) if result_path else params.get('data', b''))

    # Собираем HTTP-заголовки
    raw_headers = ''
    for header, value in params['headers'].items():
        raw_headers += header + ': '
        if isinstance(value, (tuple, list)):
            raw_headers += ('\r\n' + header + ': ').join(value)
        else:
            raw_headers += value
        raw_headers += '\r\n'
    headers = parse_headers(BytesIO(raw_headers.encode('utf-8')))

    # Для некоторых ошибок нужно сгенерировать исключение
    if (params['status'] >= 500 or params['status'] in (404,)) and not params.get('noexc'):
        raise urequest.HTTPError(params['url'], params['status'], params['status_msg'], headers, fp)

    # Собираем ответ на HTTP-запрос
    resp = urequest.addinfourl(fp, headers, params['url'])
    resp.code = params['status']
    resp.msg = params['status_msg']
    return resp

class UserTest(api.User):
    def __init__(self, *args, **kwargs):
        if 'avoid_cf' not in kwargs:
            kwargs['avoid_cf'] = False  # Чтобы не тратить время на импорт js2py
        self.sleeps = [0, 0.0]
        super(UserTest, self).__init__(*args, **kwargs)

    def sleep_func(self, tm):
        self.sleeps[0] += 1
        self.sleeps[1] += tm

    def test_open(self, fullurl, data=None, timeout=None):
        if isinstance(fullurl, (text, binary)):
            req_url = fullurl
        else:
            assert data is None
            data = fullurl.data
            req_url = fullurl.get_full_url()

        # Тестируем совместимость с разными питонами и попутно собираем заголовки
        headers = {}
        if PY2:
            assert isinstance(req_url, binary)
            if req_url is not fullurl:
                for h, v in fullurl.headers.items():
                    assert isinstance(h, binary)
                    assert isinstance(v, binary)
                    headers[h.decode('utf-8').lower()] = v
        else:
            assert isinstance(req_url, text)
            if req_url is not fullurl:
                for h, v in fullurl.headers.items():
                    assert isinstance(h, text)
                    assert isinstance(v, binary)  # значение всегда binary
                    headers[h.lower()] = v

        assert data is None or isinstance(data, binary)
        assert timeout is None or isinstance(timeout, (int, float))

        # Потестировали, а дальше проще с юникодом работать
        if PY2:
            req_url = req_url.decode('utf-8')

        # Перехват запроса при необходимости, можно его параметры assert'ить здесь
        for url, func in form_interceptors.items():
            if req_url != url and req_url != api.http_host + url:
                continue
            ctype = headers['content-type'].decode('utf-8')
            if ctype.startswith('multipart/form-data;'):
                pdict = cgi.parse_header(headers['content-type'] if PY2 else ctype)[1]
                data = cgi.parse_multipart(BytesIO(data), {'boundary': pdict['boundary'].encode('utf-8')})
            else:
                data = urlparse.parse_qs(data.decode('utf-8'))
            func(data, headers)
            break

        for url, func in interceptors.items():
            if req_url == url or req_url == api.http_host + url:
                func(data, headers)
                break

        # Ищем ответ на запрос
        for url, x in list(current_mocks.items()) + list(mocks.items()):
            result_path, optparams = x
            if url.startswith('/'):
                url = api.http_host + url
            if req_url == url:
                # Отвечаем, если нашёлся
                return build_response(req_url, result_path, optparams)

        # Ответ не нашёлся, отвечаем 404
        params = {'status': 404, 'status_msg': 'Not Found'}
        if '404' in mocks:
            if mocks['404'][1]:
                params.update(mocks['404'][1])
            return build_response(req_url, mocks['404'][0], params)
        else:
            return build_response(req_url, '404.html', params)

    def send_request(self, *args, **kwargs):
        self.opener.open = self.test_open
        self.noredir.open = self.test_open
        return super(UserTest, self).send_request(*args, **kwargs)
<|endoftext|>


<commit_before>'''
Dummy SCO class
---------------

WARNING! Only for testing purposes.
         Considers the persitence within the /tmp folder of the localhost.
'''
import os
import uuid
from cPickle import dump

storage_path = '/tmp/'

class SCO(object):

    id = None

    def getID(self):
        return self.id

    def makePersistent(self):
        if self.id is None:
            uid = uuid.uuid4()
            self.id = str(uid)
            file_name = str(uid) + '.PSCO'
            file_path = storage_path + file_name

            # Serialize object and write to disk
            dump(self, open(file_path, "wb"))

    def delete(self):
        if self.id is None:
            # Remove file from /tmp
            file_name = str(self.id) + '.PSCO'
            file_path = storage_path + file_name
            try:
                os.remove(file_path)
            except:
                print "PSCO: " + file_path + " Does not exist!"

<commit_msg>[PyCOMPSs] Added dummy constructor with alias to dummy SCO.<commit_after>'''
Dummy SCO class
---------------

WARNING! Only for testing purposes.
         Considers the persitence within the /tmp folder of the localhost.
'''
import os
import uuid
from cPickle import dump

storage_path = '/tmp/'

class SCO(object):

    id = None
    alias = None

    def __init__(self):
        pass

    def __init__(self, alias):
        self.alias = alias

    def getID(self):
        return self.id

    def makePersistent(self):
        if self.id is None:
            uid = uuid.uuid4()
            self.id = str(uid)
            file_name = str(uid) + '.PSCO'
            file_path = storage_path + file_name

            # Serialize object and write to disk
            dump(self, open(file_path, "wb"))

    def delete(self):
        if self.id is None:
            # Remove file from /tmp
            file_name = str(self.id) + '.PSCO'
            file_path = storage_path + file_name
            try:
                os.remove(file_path)
            except:
                print "PSCO: " + file_path + " Does not exist!"

<|endoftext|>


<commit_before>import requests
import json
import os
from socket import TagoRealTime

API_TAGO = os.environ.get('TAGO_SERVER') or 'https://api.tago.io'
REALTIME = os.environ.get('TAGO_REALTIME') or 'https://realtime.tago.io'

class Device:
    def __init__(self, token):
        self.token = token
        self.default_headers = { 'content-type': 'application/json', 'Device-Token': token }

    def handle_url(self, id):
        if id:
            url = '{api_endpoint}/data/{id}'.format(api_endpoint=API_TAGO,id=id)
        else:
            url = '{api_endpoint}/data'.format(api_endpoint=API_TAGO,id=id)
        return url

    def api_data_post(self, data):
        return requests.post(self.handle_url(id=False), data=json.dumps(data), headers=self.default_headers).json()

    def api_data_get(self, query):
        return requests.get(self.handle_url(id=False), params=query, headers=self.default_headers).json()

    def api_data_delete(self, id):
        return requests.delete(self.handle_url(id=id), headers=self.default_headers).json()

    def info(self, data):
        return requests.get('{api_endpoint}/info'.format(api_endpoint=API_TAGO), headers=self.default_headers).json()

    def insert(self, data):
        return self.api_data_post(data)

    def find(self, query):
        return self.api_data_get(query)

    def remove(self, id=False):
        return self.api_data_delete(id)

    def listening(self, callback, wait=False):
        return TagoRealTime(REALTIME, self.token, callback).listening(wait)
<commit_msg>adding getparams and markparams method<commit_after>import requests
import json
import os
from socket import TagoRealTime

API_TAGO = os.environ.get('TAGO_SERVER') or 'https://api.tago.io'
REALTIME = os.environ.get('TAGO_REALTIME') or 'https://realtime.tago.io'

class Device:
    def __init__(self, token):
        self.token = token
        self.default_headers = { 'content-type': 'application/json', 'Device-Token': token }

    def handle_url(self, id):
        if id:
            url = '{api_endpoint}/data/{id}'.format(api_endpoint=API_TAGO,id=id)
        else:
            url = '{api_endpoint}/data'.format(api_endpoint=API_TAGO,id=id)
        return url

    def api_data_post(self, data):
        return requests.post(self.handle_url(id=False), data=json.dumps(data), headers=self.default_headers).json()

    def api_data_get(self, query):
        return requests.get(self.handle_url(id=False), params=query, headers=self.default_headers).json()

    def api_data_delete(self, id):
        return requests.delete(self.handle_url(id=id), headers=self.default_headers).json()

    def info(self, data):
        return requests.get('{api_endpoint}/info'.format(api_endpoint=API_TAGO), headers=self.default_headers).json()

    def insert(self, data):
        return self.api_data_post(data)

    def find(self, query):
        return self.api_data_get(query)

    def remove(self, id=False):
        return self.api_data_delete(id)

    def getparams(self, sent_status)
        return requests.get('{api_endpoint}/device/params'.format(api_endpoint=API_TAGO), params=sent_status, headers=self.default_headers).json()

    def markparams(self, param_id)
        return requests.put(self.'{api_endpoint}/device/params/{id}'.format(api_endpoint=API_TAGO,id=param_id), headers=self.default_headers).json()

    def listening(self, callback, wait=False):
        return TagoRealTime(REALTIME, self.token, callback).listening(wait)
<|endoftext|>


<commit_before>##
# copyright 2009, James William Pye
# http://python.projects.postgresql.org
##
"""
Python command with a PG-API connection(``db``).
"""
import os
import sys
import re
import code
import optparse
import contextlib
from .. import clientparameters
from ..python import command as pycmd
from .. import __version__

from ..driver import default as pg_driver
from .. import exceptions as pg_exc
from .. import sys as pg_sys
from .. import lib as pg_lib

pq_trace = optparse.make_option(
	'--pq-trace',
	dest = 'pq_trace',
	help = 'trace PQ protocol transmissions',
	default = None,
)
default_options = [
	pq_trace,
	clientparameters.option_lib,
	clientparameters.option_libpath,
] + pycmd.default_optparse_options

def command(argv = sys.argv):
	p = clientparameters.DefaultParser(
		"%prog [connection options] [script] ...",
		version = __version__,
		option_list = default_options
	)
	p.disable_interspersed_args()
	co, ca = p.parse_args(argv[1:])
	rv = 1

	# Resolve the category.
	pg_sys.libpath.insert(0, os.path.curdir)
	pg_sys.libpath.extend(co.libpath or [])
	if co.lib:
		cat = pg_lib.Category(*map(pg_lib.load, co.lib))
	else:
		cat = None

	try:
		need_prompt = False
		cond = None
		connector = None
		connection = None
		while connection is None:
			try:
				cond = clientparameters.collect(parsed_options = co, prompt_title = None)
				if need_prompt:
					# authspec error thrown last time, so force prompt.
					cond['prompt_password'] = True
				try:
					clientparameters.resolve_password(cond, prompt_title = 'pg_python')
				except EOFError:
					raise SystemExit(1)
				connector = pg_driver.fit(category = cat, **cond)
				connection = connector()
				connection.connect()
			except pg_exc.ClientCannotConnectError as err:
				for att in connection.failures:
					exc = att.error
					if isinstance(exc, pg_exc.AuthenticationSpecificationError):
						sys.stderr.write(os.linesep + exc.message + (os.linesep*2))
						# keep prompting the user
						need_prompt = True
						connection = None
						break
				else:
					# no invalid password failures..
					raise

		pythonexec = pycmd.Execution(ca,
			context = getattr(co, 'python_context', None),
			loader = getattr(co, 'python_main', None),
		)

		builtin_overload = {
		# New built-ins
			'connector' : connector,
			'db' : connection,
			'prepare' : connection.prepare,
			'execute' : connection.execute,
			'settings' : connection.settings,
			'proc' : connection.proc,
			'xact' : connection.xact,
		}
		if not isinstance(__builtins__, dict):
			builtins_d = __builtins__.__dict__
		else:
			builtins_d = __builtins__
		restore = {k : builtins_d.get(k) for k in builtin_overload}

		trace_file = None
		if co.pq_trace is not None:
			trace_file = open(co.pq_trace, 'a')
		builtins_d.update(builtin_overload)
		try:
			if trace_file is not None:
				connection.tracer = trace_file.write
			with connection:
				rv = pythonexec(
					context = pycmd.postmortem(os.environ.get('PYTHON_POSTMORTEM'))
				)
		finally:
			# restore __builtins__
			builtins_d.update(restore)
			for k, v in builtin_overload.items():
				if v is None:
					del builtins_d[x]
			if trace_file is not None:
				trace_file.close()
	except:
		pg_sys.libpath.remove(os.path.curdir)
		raise
	return rv

if __name__ == '__main__':
	sys.exit(command(sys.argv))
##
# vim: ts=3:sw=3:noet:
<commit_msg>Bind sqlexec as well.<commit_after>##
# copyright 2009, James William Pye
# http://python.projects.postgresql.org
##
"""
Python command with a PG-API connection(``db``).
"""
import os
import sys
import re
import code
import optparse
import contextlib
from .. import clientparameters
from ..python import command as pycmd
from .. import __version__

from ..driver import default as pg_driver
from .. import exceptions as pg_exc
from .. import sys as pg_sys
from .. import lib as pg_lib

pq_trace = optparse.make_option(
	'--pq-trace',
	dest = 'pq_trace',
	help = 'trace PQ protocol transmissions',
	default = None,
)
default_options = [
	pq_trace,
	clientparameters.option_lib,
	clientparameters.option_libpath,
] + pycmd.default_optparse_options

def command(argv = sys.argv):
	p = clientparameters.DefaultParser(
		"%prog [connection options] [script] ...",
		version = __version__,
		option_list = default_options
	)
	p.disable_interspersed_args()
	co, ca = p.parse_args(argv[1:])
	rv = 1

	# Resolve the category.
	pg_sys.libpath.insert(0, os.path.curdir)
	pg_sys.libpath.extend(co.libpath or [])
	if co.lib:
		cat = pg_lib.Category(*map(pg_lib.load, co.lib))
	else:
		cat = None

	try:
		need_prompt = False
		cond = None
		connector = None
		connection = None
		while connection is None:
			try:
				cond = clientparameters.collect(parsed_options = co, prompt_title = None)
				if need_prompt:
					# authspec error thrown last time, so force prompt.
					cond['prompt_password'] = True
				try:
					clientparameters.resolve_password(cond, prompt_title = 'pg_python')
				except EOFError:
					raise SystemExit(1)
				connector = pg_driver.fit(category = cat, **cond)
				connection = connector()
				connection.connect()
			except pg_exc.ClientCannotConnectError as err:
				for att in connection.failures:
					exc = att.error
					if isinstance(exc, pg_exc.AuthenticationSpecificationError):
						sys.stderr.write(os.linesep + exc.message + (os.linesep*2))
						# keep prompting the user
						need_prompt = True
						connection = None
						break
				else:
					# no invalid password failures..
					raise

		pythonexec = pycmd.Execution(ca,
			context = getattr(co, 'python_context', None),
			loader = getattr(co, 'python_main', None),
		)

		builtin_overload = {
		# New built-ins
			'connector' : connector,
			'db' : connection,
			'prepare' : connection.prepare,
			# XXX: being replaced by sqlexec.
			'execute' : connection.execute,

			'sqlexec' : connection.execute,
			'settings' : connection.settings,
			'proc' : connection.proc,
			'xact' : connection.xact,
		}
		if not isinstance(__builtins__, dict):
			builtins_d = __builtins__.__dict__
		else:
			builtins_d = __builtins__
		restore = {k : builtins_d.get(k) for k in builtin_overload}

		trace_file = None
		if co.pq_trace is not None:
			trace_file = open(co.pq_trace, 'a')
		builtins_d.update(builtin_overload)
		try:
			if trace_file is not None:
				connection.tracer = trace_file.write
			with connection:
				rv = pythonexec(
					context = pycmd.postmortem(os.environ.get('PYTHON_POSTMORTEM'))
				)
		finally:
			# restore __builtins__
			builtins_d.update(restore)
			for k, v in builtin_overload.items():
				if v is None:
					del builtins_d[x]
			if trace_file is not None:
				trace_file.close()
	except:
		pg_sys.libpath.remove(os.path.curdir)
		raise
	return rv

if __name__ == '__main__':
	sys.exit(command(sys.argv))
##
# vim: ts=3:sw=3:noet:
<|endoftext|>


<commit_before>from ..PulsePrimitives import *
from ..Compiler import compile_to_hardware
from ..PulseSequencePlotter import plot_pulse_files
from .helpers import create_cal_seqs
from itertools import product
import operator
from ..ControlFlow import *
from ..TdmInstructions import *
from functools import reduce

@qfunction
def qreset(qubits, signVec, measDelay, buf, reg_size=None, TDM_map=None):
    """
    for each qubit, build the set of feedback actions to perform when
    receiving a zero or one in the comparison register
    reg_size, TDM_map: optional arguments to reset a subset of the qubit register (see Reset)
    """
    if not reg_size:
        reg_size = len(qubits)
        TDM_map = np.arange(reg_size,0,-1)

    FbGates = []
    for ct, q in enumerate(qubits):
        if signVec[ct] == 0:
            FbGates.append([gate(q) for gate in [Id, X]])
        else:  # inverted logic
            FbGates.append([gate(q) for gate in [X, Id]])
    FbSeq = [reduce(operator.mul, x) for x in product(*FbGates)]

    # load register
    seq = [Id(qubits[0], measDelay), qwait(kind='CMP'), Id(qubits[0], buf)]
    # create a branch for each possible comparison value
    for ct in range(2**reg_size):
        # duplicate branches for the irrelevant results if reg_size > len(TDM_map)
        meas_result = [(ct & TDM_bit)>0 for TDM_bit in 2**(np.array(TDM_map)-1)]
        branch_idx = sum([t*2**(len(qubits)-ind-1) for ind,t in enumerate((meas_result))])
        seq += qif(ct, [FbSeq[branch_idx]])

    return seq

def Reset(qubits,
          measDelay=1e-6,
          signVec=None,
          doubleRound=True,
          buf=20e-9,
          showPlot=False,
          measChans=None,
          docals=True,
          calRepeats=2,
          reg_size=None,
          TDM_map=None):
    """

    Preparation, simultanoeus reset, and measurement of an arbitrary number of qubits

    Parameters
    ----------
    qubits : tuple of logical channels to implement sequence (LogicalChannel)
    measDelay : delay between end of measuerement and LOADCMP
    signVec : conditions for feedback. Tuple of 0 (flip if signal is above threshold) and 1 (flip if below) for each qubit. Default = 0 for all qubits
    doubleRound : if true, double round of feedback
    showPlot : whether to plot (boolean)
    measChans : tuble of qubits to be measured (LogicalChannel)
    docals, calRepeats: enable calibration sequences, repeated calRepeats times
    reg_size: total number of qubits, including those that are not reset. Default set to len(qubits)
    TDM_map: map each qubit to a TDM digital input. Default: np.array(qN, qN-1, ..., q1) from MSB to LSB.

    Returns
    -------
    plotHandle : handle to plot window to prevent destruction
    """
    if measChans is None:
        measChans = qubits

    if signVec == None:
        signVec = (0, ) * len(qubits)

    seqs = [prep + [qreset(qubits, signVec, measDelay, buf, reg_size=reg_size, TDM_map=TDM_map)]
            for prep in create_cal_seqs(qubits, 1)]
    measBlock = reduce(operator.mul, [MEAS(q) for q in qubits])
    if doubleRound:
        for seq in seqs:
            seq += [measBlock]
            seq.append(qreset(qubits, signVec, measDelay, buf, reg_size=reg_size, TDM_map=TDM_map))

    # add final measurement
    for seq in seqs:
        seq += [measBlock, Id(qubits[0], measDelay), qwait(kind='CMP')]

    if docals:
        seqs += create_cal_seqs(qubits,
                                calRepeats,
                                measChans=measChans,
                                waitcmp=True)

    metafile = compile_to_hardware(seqs, 'Reset/Reset')

    if showPlot:
        plot_pulse_files(metafile)
    return metafile

# do not make it a subroutine for now
def BitFlip3(data_qs, ancilla_qs, theta=None, phi=None, nrounds=1, meas_delay=1e-6):
    """

    Encoding on 3-qubit bit-flip code, followed by n rounds of syndrome detection, and final correction using the n results.

    Parameters
    ----------
    data_qs : tuple of logical channels for the 3 code qubits
    ancilla_qs: tuple of logical channels for the 2 syndrome qubits
    theta, phi: longitudinal and azimuthal rotation angles for encoded state (default = no encoding)
    meas_delay : delay between syndrome check rounds
    docals, calRepeats: enable calibration sequences, repeated calRepeats times

    Returns
    -------
    plotHandle : handle to plot window to prevent destruction
    """
    if len(data_qs) != 3 or len(ancilla_qs) != 2:
        raise Exception("Wrong number of qubits")
    seqs =  [
    DecodeSetRounds(1,0,nrounds),
    Invalidate(addr=10, mask=4**nrounds-1),
    Invalidate(addr=11, mask=0x1)]

    # encode single-qubit state into 3 qubits
    if theta and phi:
        seqs+=[Utheta(data_qs[1], theta, phi), CNOT(data_qs[1], data_qs[0]), CNOT(data_qs[1], data_qs[2])]

    # multiple rounds of syndrome measurements
    for k in range(nrounds):
        seqs+=[CNOT(data_qs[0],ancilla_qs[0])*CNOT(data_qs[1],ancilla_qs[1])],
        seqs+=[CNOT(data_qs[1], ancilla_qs[0])*CNOT(data_qs[2],ancilla_qs[1])],
        seqs+= [MEASA(ancilla_qs[0], maddr=(10,2*k+1))*MEASA(ancilla_qs[1], maddr=(10,2*k+2)),
        Id(ancilla_qs[0], meas_delay),
        MEAS(data_qs[0],amp=0)*MEAS(data_qs[1],amp=0)*MEAS(data_qs[2],amp=0)] # virtual msmt's just to keep the number of segments uniform across digitizer channels
    seqs+=Decode(10, 11, 4**nrounds-1)
    seqs+=qwait("RAM",11, 1)
    seqs+=[MEAS(data_qs[0])*MEAS(data_qs[1])*MEAS(data_qs[2])
    *MEAS(ancilla_qs[0],amp=0)*MEAS(ancilla_qs[1],amp=0)] # virtual msmt's just to keep the number

    # apply corrective pulses depending on the decoder result
    FbGates = []
    for q in data_qs:
        FbGates.append([gate(q) for gate in [Id, X]])
    FbSeq = [reduce(operator.mul, x) for x in product(*FbGates)]
    for k in range(8):
        seqs += qif(k, [FbSeq[k]])
    if docals:
        seqs += create_cal_seqs(qubits,
        calRepeats,
        measChans=measChans)
    return seqs
<commit_msg>Fix maddr in qec3q<commit_after>from ..PulsePrimitives import *
from ..Compiler import compile_to_hardware
from ..PulseSequencePlotter import plot_pulse_files
from .helpers import create_cal_seqs
from itertools import product
import operator
from ..ControlFlow import *
from ..TdmInstructions import *
from functools import reduce

@qfunction
def qreset(qubits, signVec, measDelay, buf, reg_size=None, TDM_map=None):
    """
    for each qubit, build the set of feedback actions to perform when
    receiving a zero or one in the comparison register
    reg_size, TDM_map: optional arguments to reset a subset of the qubit register (see Reset)
    """
    if not reg_size:
        reg_size = len(qubits)
        TDM_map = np.arange(reg_size,0,-1)

    FbGates = []
    for ct, q in enumerate(qubits):
        if signVec[ct] == 0:
            FbGates.append([gate(q) for gate in [Id, X]])
        else:  # inverted logic
            FbGates.append([gate(q) for gate in [X, Id]])
    FbSeq = [reduce(operator.mul, x) for x in product(*FbGates)]

    # load register
    seq = [Id(qubits[0], measDelay), qwait(kind='CMP'), Id(qubits[0], buf)]
    # create a branch for each possible comparison value
    for ct in range(2**reg_size):
        # duplicate branches for the irrelevant results if reg_size > len(TDM_map)
        meas_result = [(ct & TDM_bit)>0 for TDM_bit in 2**(np.array(TDM_map)-1)]
        branch_idx = sum([t*2**(len(qubits)-ind-1) for ind,t in enumerate((meas_result))])
        seq += qif(ct, [FbSeq[branch_idx]])

    return seq

def Reset(qubits,
          measDelay=1e-6,
          signVec=None,
          doubleRound=True,
          buf=20e-9,
          showPlot=False,
          measChans=None,
          docals=True,
          calRepeats=2,
          reg_size=None,
          TDM_map=None):
    """

    Preparation, simultanoeus reset, and measurement of an arbitrary number of qubits

    Parameters
    ----------
    qubits : tuple of logical channels to implement sequence (LogicalChannel)
    measDelay : delay between end of measuerement and LOADCMP
    signVec : conditions for feedback. Tuple of 0 (flip if signal is above threshold) and 1 (flip if below) for each qubit. Default = 0 for all qubits
    doubleRound : if true, double round of feedback
    showPlot : whether to plot (boolean)
    measChans : tuble of qubits to be measured (LogicalChannel)
    docals, calRepeats: enable calibration sequences, repeated calRepeats times
    reg_size: total number of qubits, including those that are not reset. Default set to len(qubits)
    TDM_map: map each qubit to a TDM digital input. Default: np.array(qN, qN-1, ..., q1) from MSB to LSB.

    Returns
    -------
    plotHandle : handle to plot window to prevent destruction
    """
    if measChans is None:
        measChans = qubits

    if signVec == None:
        signVec = (0, ) * len(qubits)

    seqs = [prep + [qreset(qubits, signVec, measDelay, buf, reg_size=reg_size, TDM_map=TDM_map)]
            for prep in create_cal_seqs(qubits, 1)]
    measBlock = reduce(operator.mul, [MEAS(q) for q in qubits])
    if doubleRound:
        for seq in seqs:
            seq += [measBlock]
            seq.append(qreset(qubits, signVec, measDelay, buf, reg_size=reg_size, TDM_map=TDM_map))

    # add final measurement
    for seq in seqs:
        seq += [measBlock, Id(qubits[0], measDelay), qwait(kind='CMP')]

    if docals:
        seqs += create_cal_seqs(qubits,
                                calRepeats,
                                measChans=measChans,
                                waitcmp=True)

    metafile = compile_to_hardware(seqs, 'Reset/Reset')

    if showPlot:
        plot_pulse_files(metafile)
    return metafile

# do not make it a subroutine for now
def BitFlip3(data_qs, ancilla_qs, theta=None, phi=None, nrounds=1, meas_delay=1e-6):
    """

    Encoding on 3-qubit bit-flip code, followed by n rounds of syndrome detection, and final correction using the n results.

    Parameters
    ----------
    data_qs : tuple of logical channels for the 3 code qubits
    ancilla_qs: tuple of logical channels for the 2 syndrome qubits
    theta, phi: longitudinal and azimuthal rotation angles for encoded state (default = no encoding)
    meas_delay : delay between syndrome check rounds
    docals, calRepeats: enable calibration sequences, repeated calRepeats times

    Returns
    -------
    plotHandle : handle to plot window to prevent destruction
    """
    if len(data_qs) != 3 or len(ancilla_qs) != 2:
        raise Exception("Wrong number of qubits")
    seqs =  [
    DecodeSetRounds(1,0,nrounds),
    Invalidate(addr=10, mask=4**nrounds-1),
    Invalidate(addr=11, mask=0x1)]

    # encode single-qubit state into 3 qubits
    if theta and phi:
        seqs+=[Utheta(data_qs[1], theta, phi), CNOT(data_qs[1], data_qs[0]), CNOT(data_qs[1], data_qs[2])]

    # multiple rounds of syndrome measurements
    for k in range(nrounds):
        seqs+=[CNOT(data_qs[0],ancilla_qs[0])*CNOT(data_qs[1],ancilla_qs[1])],
        seqs+=[CNOT(data_qs[1], ancilla_qs[0])*CNOT(data_qs[2],ancilla_qs[1])],
        seqs+= [MEASA(ancilla_qs[0], maddr=(10,2**(2*n))*MEASA(ancilla_qs[1], maddr=(10,2**(2*n+1))),
        Id(ancilla_qs[0], meas_delay),
        MEAS(data_qs[0],amp=0)*MEAS(data_qs[1],amp=0)*MEAS(data_qs[2],amp=0)] # virtual msmt's just to keep the number of segments uniform across digitizer channels
    seqs+=Decode(10, 11, 4**nrounds-1)
    seqs+=qwait("RAM",11, 1)
    seqs+=[MEAS(data_qs[0])*MEAS(data_qs[1])*MEAS(data_qs[2])
    *MEAS(ancilla_qs[0],amp=0)*MEAS(ancilla_qs[1],amp=0)] # virtual msmt's just to keep the number

    # apply corrective pulses depending on the decoder result
    FbGates = []
    for q in data_qs:
        FbGates.append([gate(q) for gate in [Id, X]])
    FbSeq = [reduce(operator.mul, x) for x in product(*FbGates)]
    for k in range(8):
        seqs += qif(k, [FbSeq[k]])
    if docals:
        seqs += create_cal_seqs(qubits,
        calRepeats,
        measChans=measChans)
    return seqs
<|endoftext|>


<commit_before>"""Utilities for asyncio-friendly file handling."""
from .threadpool import open

__version__ = '0.2'

__all__ = (open, )
<commit_msg>Version bump to 0.2.1.<commit_after>"""Utilities for asyncio-friendly file handling."""
from .threadpool import open

__version__ = '0.2.1'

__all__ = (open, )
<|endoftext|>


<commit_before>from mordred import Calculator, all_descriptors, Error
from rdkit.Chem import AllChem as Chem
from numpy.testing import assert_almost_equal
from nose.tools import eq_
import pickle

def test_parallel():
    calc = Calculator(all_descriptors())
    mols = list(map(Chem.AddHs, [
        Chem.MolFromSmiles('c1ccccc1'),
        Chem.MolFromSmiles('C1=CC(=C(C=C1C2=C(C=C3C(=CC(=CC3=[O+]2)O)O)O)O)O'),
        Chem.MolFromSmiles('CCCCCC'),
    ]))

    for mol in mols:
        Chem.EmbedMolecule(mol)

    for (_, serial), (_, parallel) in zip(calc.map(mols, nproc=1, quiet=True),
                                          calc.map(mols, quiet=True)):
        for d, s, p in zip(calc.descriptors, serial, parallel):
            if isinstance(s, Error):
                yield eq_, pickle.dumps(s), pickle.dumps(p), str(d)
                return

            yield assert_almost_equal, s, p, 7, str(d)
<commit_msg>fix test_parallel<commit_after>from mordred import Calculator, all_descriptors, Error
from rdkit.Chem import AllChem as Chem
from numpy.testing import assert_almost_equal
from nose.tools import eq_
import pickle

def test_parallel():
    calc = Calculator(all_descriptors())
    mols = list(map(Chem.AddHs, [
        Chem.MolFromSmiles('c1ccccc1'),
        Chem.MolFromSmiles('C1=CC(=C(C=C1C2=C(C=C3C(=CC(=CC3=[O+]2)O)O)O)O)O'),
        Chem.MolFromSmiles('CCCCCC'),
    ]))

    for mol in mols:
        Chem.EmbedMolecule(mol)

    for (_, serial), (_, parallel) in zip(calc.map(mols, nproc=1, quiet=True),
                                          calc.map(mols, quiet=True)):
        for d, s, p in zip(calc.descriptors, serial, parallel):
            if isinstance(s, Error):
                yield eq_, pickle.dumps(s), pickle.dumps(p), str(d)
            else:
                yield assert_almost_equal, s, p, 7, str(d)
<|endoftext|>