<commit_before>#!/usr/bin/env python
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

import os
import sys
from src.fasta_reader import FastaReader
from src.gff_reader import GFFReader
from src.annotator import Annotator
from src.filter_manager import FilterManager
from src.stats_manager import StatsManager

class Controller:

    def __init__(self):
        self.seqs = []
        self.removed_features = []
        self.annot = Annotator()
        self.filter_mgr = FilterManager()
        self.stats_mgr = StatsManager()

    def execute(self, args_dict):
        """At a minimum, write a fasta, gff and tbl to output directory. Optionally do more."""
        # Verify and read fasta file
        fastapath = args_dict["fasta"]
        if not os.path.isfile(fastapath):
            sys.stderr.write("Failed to find " + fastapath + ". No genome was loaded.\n")
            sys.exit()
        sys.stderr.write("Reading fasta...\n")
        self.read_fasta(fastapath)
        sys.stderr.write("Done.\n")

        # Verify and read gff file
        gffpath = args_dict["gff"]
        if not os.path.isfile(gffpath):
            sys.stderr.write("Failed to find " + gffpath + ". No genome was loaded.")
            return
        sys.stderr.write("Reading gff...\n")
        self.read_gff(gffpath)
        sys.stderr.write("Done.\n")

        # Optional annotation step
        if "anno" in args_dict:
            anno_filename = args_dict["anno"]
            self.annotate_from_file(anno_filename)

        # Write fasta, gff and tbl file to gag_output/ folder
        # TODO look for 'out' arg
        out_dir = "gag_output"
        # Create directory, open files
        os.system('mkdir ' + out_dir)
        # Open files
        fasta = open(out_dir+'/genome.fasta', 'w')
        gff = open(out_dir+'/genome.gff', 'w')
        tbl = open(out_dir+'/genome.tbl', 'w')
        sys.stderr.write("Writing gff, tbl and fasta...\n")
        # TODO track # of gagflags?
        # TODO stats file
        for seq in self.seqs:
            fasta.write(seq.to_fasta())
            gff.write(seq.to_gff())
            tbl.write(seq.to_tbl())
        # Close files
        gff.close()
        tbl.close()
        fasta.close()

    def barf_folder(self, line):
        # Create directory, open files
        os.system('mkdir '+line)
        gff = open(line+'/genome.gff', 'w')
        removed_gff = open(line+'/genome.removed.gff', 'w')
        tbl = open(line+'/genome.tbl', 'w')
        fasta = open(line+'/genome.fasta', 'w')
        mrna_fasta = open(line+'/genome.mrna.fasta', 'w')
        cds_fasta = open(line+'/genome.cds.fasta', 'w')
        protein_fasta = open(line+'/genome.proteins.fasta', 'w')
        stats_file = open(line+'/genome.stats', 'w')

        # Now write stuff
        sys.stderr.write("Writing gff, tbl and fasta...\n")
        number_of_gagflags = 0
        first_line = "Number of sequences:   " + str(len(self.seqs)) + "\n"
        update_alt = False
        self.stats_mgr.clear_alt()
        removed_gff.write("##gff-version 3\n")
        gff.write("##gff-version 3\n")
        for feature in self.removed_features:
            removed_gff.write(feature.to_gff())
        for seq in self.seqs:
            gff.write(seq.to_gff())
            removed_gff.write(seq.removed_to_gff())
            tbl.write(seq.to_tbl())
            mrna_fasta.write(seq.to_mrna_fasta())
            cds_fasta.write(seq.to_cds_fasta())
            protein_fasta.write(seq.to_protein_fasta())
            fasta.write(seq.to_fasta())
            self.stats_mgr.update_alt(seq.stats())
            number_of_gagflags += seq.number_of_gagflags()

        last_line = "(" + str(number_of_gagflags) + " features flagged)\n"
        stats_file.write(first_line + self.stats_mgr.summary() + last_line)

        # Close files
        gff.close()
        tbl.close()
        fasta.close()
        mrna_fasta.close()
        cds_fasta.close()
        protein_fasta.close()
        stats_file.close()
        sys.stderr.write( "Genome written to " + line + "\n")

    def load_folder(self, line):
        if not line:
            line = "."
        fastapath = line + '/genome.fasta'
        gffpath = line + '/genome.gff'

        # Verify files
        if not os.path.isfile(fastapath):
            sys.stderr.write("Failed to find " + fastapath + ". No genome was loaded.")
            return
        if not os.path.isfile(gffpath):
            sys.stderr.write("Failed to find " + gffpath + ". No genome was loaded.")
            return

        # Read the fasta
        sys.stderr.write("Reading fasta...\n")
        self.read_fasta(fastapath)
        sys.stderr.write("Done.\n")

        # Read the gff
        sys.stderr.write("Reading gff...\n")
        self.read_gff(gffpath)
        sys.stderr.write("Done.\n")

        # Remove empty features
        for seq in self.seqs:
            self.remove_empty_features(seq)

        # Clear stats; read in new stats
        sys.stderr.write("Calculating stats...\n")
        self.stats_mgr.clear_all()
        for seq in self.seqs:
            self.stats_mgr.update_ref(seq.stats())
        sys.stderr.write("Done.\n")

    def add_annotations_from_list(self, anno_list):
        for seq in self.seqs:
            seq.add_annotations_from_list(anno_list)

    def remove_from_file(self, filename):
        if not os.path.isfile(filename):
            sys.stderr.write("Error: " + filename + " is not a file. Nothing removed.\n")
            return
        remove_list = self.read_remove_list(filename)
        self.remove_from_list(remove_list)

    def trim_from_file(self, filename):
        if not os.path.isfile(filename):
            sys.stderr.write("Error: " + filename + " is not a file. Nothing trimmed.\n")
            return
        trimlist = self.read_bed_file(open(filename, 'rb'))
        if not trimlist:
            sys.stderr.write("Failed to read .bed file; nothing trimmed.\n")
            return
        else:
            self.trim_from_list(trimlist)

    def annotate_from_file(self, filename):
        if not os.path.isfile(filename):
            sys.stderr.write("Error: " + filename + " is not a file. Nothing annotated.\n")
            return
        annos = self.read_annotation_file(open(filename, 'rb'))
        if not annos:
            sys.stderr.write("Failed to read annotations from " + filename + "; no annotations added.\n")
            return
        else:
            sys.stderr.write("Adding annotations to genome ...\n")
            self.add_annotations_from_list(annos)
            sys.stderr.write("...done\n")

    def trim_from_list(self, trimlist):
        for seq in self.seqs:
            # Trim the ends first
            for entry in trimlist:
                if entry[0] == seq.header and entry[2] == len(seq.bases):
                    seq.trim_region(entry[1], entry[2])
                    sys.stderr.write("Trimmed " + entry[0] + " from ")
                    sys.stderr.write(str(entry[1]) + " to " + str(entry[2]) + "\n")
            # Now trim the beginnings
            for entry in trimlist:
                if entry[0] == seq.header and entry[1] == 1:
                    seq.trim_region(entry[1], entry[2])
                    sys.stderr.write("Trimmed " + entry[0] + " from ")
                    sys.stderr.write(str(entry[1]) + " to " + str(entry[2]) + "\n")
            self.remove_empty_features(seq)

    def get_filter_arg(self, filter_name):
        return self.filter_mgr.get_filter_arg(filter_name)

    def apply_filter(self, filter_name, val, filter_mode):
        for seq in self.seqs:
            self.filter_mgr.apply_filter(filter_name, val, filter_mode, seq)
            self.remove_empty_features(seq)

    def fix_terminal_ns(self):
        for seq in self.seqs:
            seq.remove_terminal_ns()
            self.remove_empty_features(seq)
        return "Terminal Ns fixed."

    def fix_start_stop_codons(self):
        for seq in self.seqs:
            seq.create_starts_and_stops()
        return "Verified and created start/stop codons."

## Reading in files

    def read_fasta(self, line):
        reader = FastaReader()
        self.seqs = reader.read(open(line, 'r'))

    def read_gff(self, line):
        gffreader = GFFReader()
        reader = open(line, 'rb')
        genes = gffreader.read_file(reader)
        for gene in genes:
            self.add_gene(gene)

    def read_remove_list(self, line):
        remove_list = []
        with open(line, 'rb') as infile:
            for line in infile:
                remove_list.append(line.strip())
        return remove_list

    def read_bed_file(self, io_buffer):
        trimlist = []
        for line in io_buffer:
            splitline = line.strip().split('\t')
            if len(splitline) != 3:
                return []
            else:
                try:
                    entry = [splitline[0], int(splitline[1]), int(splitline[2])]
                except ValueError:
                    sys.stderr.write("Error reading .bed file. Non-integer value ")
                    sys.sdterr.write("in column 2 or 3. Here is the line:\n")
                    sys.stderr.write(line)
                    return []
                trimlist.append(entry)
        return trimlist

    def read_annotation_file(self, io_buffer):
        annos = []
        for line in io_buffer:
            splitline = line.strip().split('\t')
            if len(splitline) != 3:
                return []
            else:
                annos.append(splitline)
        return annos

## Clean up

    def remove_empty_features(self, seq):
        """Removes any empty mRNAs or genes from a seq and adds them to self.removed_features."""
        self.removed_features.extend(seq.remove_empty_mrnas())
        self.removed_features.extend(seq.remove_empty_genes())

    def stats(self):
        if not self.seqs:
            return self.no_genome_message
        else:
            number_of_gagflags = 0
            # TODO have stats mgr handle "number of sequences"
            first_line = "Number of sequences:   " + str(len(self.seqs)) + "\n"
            sys.stderr.write("Calculating statistics on genome...\n")
            self.stats_mgr.clear_alt()
            for seq in self.seqs:
                self.stats_mgr.update_alt(seq.stats())
                number_of_gagflags += seq.number_of_gagflags()
            last_line = "(" + str(number_of_gagflags) + " features flagged)\n"
            return first_line + self.stats_mgr.summary() + last_line

## Utility methods

    def add_gene(self, gene):
        for seq in self.seqs:
            if seq.header == gene.seq_name:
                seq.add_gene(gene)

    def get_locus_tag(self):
        locus_tag = ""
        for seq in self.seqs:
            if locus_tag:
                break
            else:
                locus_tag = seq.get_locus_tag()
        return locus_tag

    def clear_seqs(self):
        self.seqs[:] = []

    def remove_from_list(self, bad_list):
        # First remove any seqs on the list
        to_remove = []
        for seq in self.seqs:
            if seq.header in bad_list:
                to_remove.append(seq)
        if to_remove:
            for seq in to_remove:
                self.seqs.remove(seq)
                sys.stderr.write("Warning: removing seq " + seq.header + ".\n")
                sys.stderr.write("You must reload genome to get this sequence back.\n")
            self.removed_features.extend(to_remove)
        # Now pass the list down to each seq
        for seq in self.seqs:
            removed_from_seq = seq.remove_from_list(bad_list)
            self.removed_features.extend(removed_from_seq)

    def contains_mrna(self, mrna_id):
        for seq in self.seqs:
            if seq.contains_mrna(mrna_id):
                return True
        return False

    def contains_gene(self, gene_id):
        for seq in self.seqs:
            if seq.contains_gene(gene_id):
                return True
        return False

    def contains_seq(self, seq_id):
        for seq in self.seqs:
            if seq.header == seq_id:
                return True
        return False

    def can_write_to_path(self, path):
        if len(path.split()) > 1:
            return False
        else:
            return not os.path.exists(path)

## Utility functions
def format_list_with_strings(entries):
    if len(entries) == 0:
        return ""
    result = entries[0]
    if len(entries) > 2:
        for entry in entries[1:-1]:
            result += ", " + entry
    if len(entries) > 1:
        result += ", " + entries[-1]
    result += "\n"
    return result
<commit_msg>added support for 'out' flag on command line<commit_after>#!/usr/bin/env python
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

import os
import sys
from src.fasta_reader import FastaReader
from src.gff_reader import GFFReader
from src.annotator import Annotator
from src.filter_manager import FilterManager
from src.stats_manager import StatsManager

class Controller:

    def __init__(self):
        self.seqs = []
        self.removed_features = []
        self.annot = Annotator()
        self.filter_mgr = FilterManager()
        self.stats_mgr = StatsManager()

    def execute(self, args_dict):
        """At a minimum, write a fasta, gff and tbl to output directory. Optionally do more."""
        # Verify and read fasta file
        fastapath = args_dict["fasta"]
        if not os.path.isfile(fastapath):
            sys.stderr.write("Failed to find " + fastapath + ". No genome was loaded.\n")
            sys.exit()
        sys.stderr.write("Reading fasta...\n")
        self.read_fasta(fastapath)
        sys.stderr.write("Done.\n")

        # Verify and read gff file
        gffpath = args_dict["gff"]
        if not os.path.isfile(gffpath):
            sys.stderr.write("Failed to find " + gffpath + ". No genome was loaded.")
            return
        sys.stderr.write("Reading gff...\n")
        self.read_gff(gffpath)
        sys.stderr.write("Done.\n")

        # Optional annotation step
        if "anno" in args_dict:
            anno_filename = args_dict["anno"]
            self.annotate_from_file(anno_filename)

        # Write fasta, gff and tbl file to gag_output/ folder
        out_dir = "gag_output"
        if "out" in args_dict:
            out_dir = args_dict["out"]
        # Create directory, open files
        os.system('mkdir ' + out_dir)
        # Open files
        fasta = open(out_dir+'/genome.fasta', 'w')
        gff = open(out_dir+'/genome.gff', 'w')
        tbl = open(out_dir+'/genome.tbl', 'w')
        sys.stderr.write("Writing gff, tbl and fasta...\n")
        # TODO track # of gagflags?
        # TODO stats file
        for seq in self.seqs:
            fasta.write(seq.to_fasta())
            gff.write(seq.to_gff())
            tbl.write(seq.to_tbl())
        # Close files
        gff.close()
        tbl.close()
        fasta.close()

    def barf_folder(self, line):
        # Create directory, open files
        os.system('mkdir '+line)
        gff = open(line+'/genome.gff', 'w')
        removed_gff = open(line+'/genome.removed.gff', 'w')
        tbl = open(line+'/genome.tbl', 'w')
        fasta = open(line+'/genome.fasta', 'w')
        mrna_fasta = open(line+'/genome.mrna.fasta', 'w')
        cds_fasta = open(line+'/genome.cds.fasta', 'w')
        protein_fasta = open(line+'/genome.proteins.fasta', 'w')
        stats_file = open(line+'/genome.stats', 'w')

        # Now write stuff
        sys.stderr.write("Writing gff, tbl and fasta...\n")
        number_of_gagflags = 0
        first_line = "Number of sequences:   " + str(len(self.seqs)) + "\n"
        update_alt = False
        self.stats_mgr.clear_alt()
        removed_gff.write("##gff-version 3\n")
        gff.write("##gff-version 3\n")
        for feature in self.removed_features:
            removed_gff.write(feature.to_gff())
        for seq in self.seqs:
            gff.write(seq.to_gff())
            removed_gff.write(seq.removed_to_gff())
            tbl.write(seq.to_tbl())
            mrna_fasta.write(seq.to_mrna_fasta())
            cds_fasta.write(seq.to_cds_fasta())
            protein_fasta.write(seq.to_protein_fasta())
            fasta.write(seq.to_fasta())
            self.stats_mgr.update_alt(seq.stats())
            number_of_gagflags += seq.number_of_gagflags()

        last_line = "(" + str(number_of_gagflags) + " features flagged)\n"
        stats_file.write(first_line + self.stats_mgr.summary() + last_line)

        # Close files
        gff.close()
        tbl.close()
        fasta.close()
        mrna_fasta.close()
        cds_fasta.close()
        protein_fasta.close()
        stats_file.close()
        sys.stderr.write( "Genome written to " + line + "\n")

    def load_folder(self, line):
        if not line:
            line = "."
        fastapath = line + '/genome.fasta'
        gffpath = line + '/genome.gff'

        # Verify files
        if not os.path.isfile(fastapath):
            sys.stderr.write("Failed to find " + fastapath + ". No genome was loaded.")
            return
        if not os.path.isfile(gffpath):
            sys.stderr.write("Failed to find " + gffpath + ". No genome was loaded.")
            return

        # Read the fasta
        sys.stderr.write("Reading fasta...\n")
        self.read_fasta(fastapath)
        sys.stderr.write("Done.\n")

        # Read the gff
        sys.stderr.write("Reading gff...\n")
        self.read_gff(gffpath)
        sys.stderr.write("Done.\n")

        # Remove empty features
        for seq in self.seqs:
            self.remove_empty_features(seq)

        # Clear stats; read in new stats
        sys.stderr.write("Calculating stats...\n")
        self.stats_mgr.clear_all()
        for seq in self.seqs:
            self.stats_mgr.update_ref(seq.stats())
        sys.stderr.write("Done.\n")

    def add_annotations_from_list(self, anno_list):
        for seq in self.seqs:
            seq.add_annotations_from_list(anno_list)

    def remove_from_file(self, filename):
        if not os.path.isfile(filename):
            sys.stderr.write("Error: " + filename + " is not a file. Nothing removed.\n")
            return
        remove_list = self.read_remove_list(filename)
        self.remove_from_list(remove_list)

    def trim_from_file(self, filename):
        if not os.path.isfile(filename):
            sys.stderr.write("Error: " + filename + " is not a file. Nothing trimmed.\n")
            return
        trimlist = self.read_bed_file(open(filename, 'rb'))
        if not trimlist:
            sys.stderr.write("Failed to read .bed file; nothing trimmed.\n")
            return
        else:
            self.trim_from_list(trimlist)

    def annotate_from_file(self, filename):
        if not os.path.isfile(filename):
            sys.stderr.write("Error: " + filename + " is not a file. Nothing annotated.\n")
            return
        annos = self.read_annotation_file(open(filename, 'rb'))
        if not annos:
            sys.stderr.write("Failed to read annotations from " + filename + "; no annotations added.\n")
            return
        else:
            sys.stderr.write("Adding annotations to genome ...\n")
            self.add_annotations_from_list(annos)
            sys.stderr.write("...done\n")

    def trim_from_list(self, trimlist):
        for seq in self.seqs:
            # Trim the ends first
            for entry in trimlist:
                if entry[0] == seq.header and entry[2] == len(seq.bases):
                    seq.trim_region(entry[1], entry[2])
                    sys.stderr.write("Trimmed " + entry[0] + " from ")
                    sys.stderr.write(str(entry[1]) + " to " + str(entry[2]) + "\n")
            # Now trim the beginnings
            for entry in trimlist:
                if entry[0] == seq.header and entry[1] == 1:
                    seq.trim_region(entry[1], entry[2])
                    sys.stderr.write("Trimmed " + entry[0] + " from ")
                    sys.stderr.write(str(entry[1]) + " to " + str(entry[2]) + "\n")
            self.remove_empty_features(seq)

    def get_filter_arg(self, filter_name):
        return self.filter_mgr.get_filter_arg(filter_name)

    def apply_filter(self, filter_name, val, filter_mode):
        for seq in self.seqs:
            self.filter_mgr.apply_filter(filter_name, val, filter_mode, seq)
            self.remove_empty_features(seq)

    def fix_terminal_ns(self):
        for seq in self.seqs:
            seq.remove_terminal_ns()
            self.remove_empty_features(seq)
        return "Terminal Ns fixed."

    def fix_start_stop_codons(self):
        for seq in self.seqs:
            seq.create_starts_and_stops()
        return "Verified and created start/stop codons."

## Reading in files

    def read_fasta(self, line):
        reader = FastaReader()
        self.seqs = reader.read(open(line, 'r'))

    def read_gff(self, line):
        gffreader = GFFReader()
        reader = open(line, 'rb')
        genes = gffreader.read_file(reader)
        for gene in genes:
            self.add_gene(gene)

    def read_remove_list(self, line):
        remove_list = []
        with open(line, 'rb') as infile:
            for line in infile:
                remove_list.append(line.strip())
        return remove_list

    def read_bed_file(self, io_buffer):
        trimlist = []
        for line in io_buffer:
            splitline = line.strip().split('\t')
            if len(splitline) != 3:
                return []
            else:
                try:
                    entry = [splitline[0], int(splitline[1]), int(splitline[2])]
                except ValueError:
                    sys.stderr.write("Error reading .bed file. Non-integer value ")
                    sys.sdterr.write("in column 2 or 3. Here is the line:\n")
                    sys.stderr.write(line)
                    return []
                trimlist.append(entry)
        return trimlist

    def read_annotation_file(self, io_buffer):
        annos = []
        for line in io_buffer:
            splitline = line.strip().split('\t')
            if len(splitline) != 3:
                return []
            else:
                annos.append(splitline)
        return annos

## Clean up

    def remove_empty_features(self, seq):
        """Removes any empty mRNAs or genes from a seq and adds them to self.removed_features."""
        self.removed_features.extend(seq.remove_empty_mrnas())
        self.removed_features.extend(seq.remove_empty_genes())

    def stats(self):
        if not self.seqs:
            return self.no_genome_message
        else:
            number_of_gagflags = 0
            # TODO have stats mgr handle "number of sequences"
            first_line = "Number of sequences:   " + str(len(self.seqs)) + "\n"
            sys.stderr.write("Calculating statistics on genome...\n")
            self.stats_mgr.clear_alt()
            for seq in self.seqs:
                self.stats_mgr.update_alt(seq.stats())
                number_of_gagflags += seq.number_of_gagflags()
            last_line = "(" + str(number_of_gagflags) + " features flagged)\n"
            return first_line + self.stats_mgr.summary() + last_line

## Utility methods

    def add_gene(self, gene):
        for seq in self.seqs:
            if seq.header == gene.seq_name:
                seq.add_gene(gene)

    def get_locus_tag(self):
        locus_tag = ""
        for seq in self.seqs:
            if locus_tag:
                break
            else:
                locus_tag = seq.get_locus_tag()
        return locus_tag

    def clear_seqs(self):
        self.seqs[:] = []

    def remove_from_list(self, bad_list):
        # First remove any seqs on the list
        to_remove = []
        for seq in self.seqs:
            if seq.header in bad_list:
                to_remove.append(seq)
        if to_remove:
            for seq in to_remove:
                self.seqs.remove(seq)
                sys.stderr.write("Warning: removing seq " + seq.header + ".\n")
                sys.stderr.write("You must reload genome to get this sequence back.\n")
            self.removed_features.extend(to_remove)
        # Now pass the list down to each seq
        for seq in self.seqs:
            removed_from_seq = seq.remove_from_list(bad_list)
            self.removed_features.extend(removed_from_seq)

    def contains_mrna(self, mrna_id):
        for seq in self.seqs:
            if seq.contains_mrna(mrna_id):
                return True
        return False

    def contains_gene(self, gene_id):
        for seq in self.seqs:
            if seq.contains_gene(gene_id):
                return True
        return False

    def contains_seq(self, seq_id):
        for seq in self.seqs:
            if seq.header == seq_id:
                return True
        return False

    def can_write_to_path(self, path):
        if len(path.split()) > 1:
            return False
        else:
            return not os.path.exists(path)

## Utility functions
def format_list_with_strings(entries):
    if len(entries) == 0:
        return ""
    result = entries[0]
    if len(entries) > 2:
        for entry in entries[1:-1]:
            result += ", " + entry
    if len(entries) > 1:
        result += ", " + entries[-1]
    result += "\n"
    return result
<|endoftext|>


<commit_before>import json
import requests
import socket

from nautilus.conventions.services import api_gateway_name
from nautilus.network.registry import service_location_by_name

def query_graphql_service(url, name, fields, filters=None):
    """ A graphql query wrapper factory"""

    # by default there are no args to add to the query
    args = None
    # if there are filters defined for the query
    if filters:
        # construct the argument string out of the given dictionary
        arg_string = ', '.join(['{}: {}'.format(key, json.dumps(value)) \
                                            for key, value in filters.items()])
        args = "(%s)" % arg_string if len(arg_string) > 0  else ''

    # construct the field string
    field_list = ', '.join(fields)

    # build the query out of the given paramters
    query = """
        query {
            %s %s {
                %s
            }
        }
    """ % (name, args or '', field_list)

    # query the service to retrieve the data
    data_request = requests.get(url + '?query='   + query).json()
    # if there is an error
    if 'errors' in data_request:
        raise RuntimeError(data_request['errors'])
    # otherwise there is no error
    else:
        # return the data
        return data_request['data'][name]

def query_service(service, fields, filters = {}):
    '''
        Apply the given filters to a query of a model service given its name
        and the desired fields.
    '''
    # necessary imports
    from nautilus.conventions import root_query

    # query the target using model service conventions
    return query_graphql_service(
        url = 'http://{}'.format(service_location_by_name(service)),
        name = root_query(service),
        fields = fields,
        filters = filters
    )

def query_api(query, mutation = None):
    '''
        Perform the given query on the api gateway and turn the results.
        Use this function to avoid hard coding the name of the api gateway.
    '''
    # grab the location of the api service from the registry
    api_location = service_location_by_name(api_gateway_name())
    # construct the url out of the location
    url = 'http://{}'.format(api_location)
    print(url + '?query='   + query)
    # query the service to retrieve the data
    dataRequest = requests.get(url + '?query='   + query).json()

    return query_service(api_gateway_name(), query)

def combine_action_handlers(*args):
    """
        This function combines the given action handlers into a single function
        which will call all of them.
    """
    # the combined action handler
    def combinedActionHandler(action_type, payload):
        # goes over every given handler
        for handler in args:
            # call the handler
            handler(action_type, payload)

    # return the combined action handler
    return combinedActionHandler

<commit_msg>removed unsafe function argument<commit_after>import json
import requests
import socket

from nautilus.conventions.services import api_gateway_name
from nautilus.network.registry import service_location_by_name

def query_graphql_service(url, name, fields, filters=None):
    """ A graphql query wrapper factory"""

    # by default there are no args to add to the query
    args = None
    # if there are filters defined for the query
    if filters:
        # construct the argument string out of the given dictionary
        arg_string = ', '.join(['{}: {}'.format(key, json.dumps(value)) \
                                            for key, value in filters.items()])
        args = "(%s)" % arg_string if len(arg_string) > 0  else ''

    # construct the field string
    field_list = ', '.join(fields)

    # build the query out of the given paramters
    query = """
        query {
            %s %s {
                %s
            }
        }
    """ % (name, args or '', field_list)

    # query the service to retrieve the data
    data_request = requests.get(url + '?query='   + query).json()
    # if there is an error
    if 'errors' in data_request:
        raise RuntimeError(data_request['errors'])
    # otherwise there is no error
    else:
        # return the data
        return data_request['data'][name]

def query_service(service, fields, filters = None):
    '''
        Apply the given filters to a query of a model service given its name
        and the desired fields.
    '''
    # necessary imports
    from nautilus.conventions import root_query

    # query the target using model service conventions
    return query_graphql_service(
        url='http://{}'.format(service_location_by_name(service)),
        name=root_query(service),
        fields=fields,
        filters=filters or {}
    )

def query_api(query, mutation = None):
    '''
        Perform the given query on the api gateway and turn the results.
        Use this function to avoid hard coding the name of the api gateway.
    '''
    # grab the location of the api service from the registry
    api_location = service_location_by_name(api_gateway_name())
    # construct the url out of the location
    url = 'http://{}'.format(api_location)
    print(url + '?query='   + query)
    # query the service to retrieve the data
    dataRequest = requests.get(url + '?query='   + query).json()

    return query_service(api_gateway_name(), query)

def combine_action_handlers(*args):
    """
        This function combines the given action handlers into a single function
        which will call all of them.
    """
    # the combined action handler
    def combinedActionHandler(action_type, payload):
        # goes over every given handler
        for handler in args:
            # call the handler
            handler(action_type, payload)

    # return the combined action handler
    return combinedActionHandler
<|endoftext|>


<commit_before>#!/usr/bin/env python

from itertools import product
from collections import defaultdict
from redis import Redis
import sys
import json

from Bio import SeqIO

from nearpy import Engine
from nearpy import hashes
from nearpy.storage import RedisStorage
from nearpy.filters import DistanceThresholdFilter

import numpy as np

# The distance ratio of an ANN y is it's distance to the minimal hypersphere
# around the query vector x, that contains all exact nearest neighbours n,
# clamped to zero and normalized with this hypersphere's radius

class FeatureGenerator:

    def __init__(self, **kwargs):
        ''' '''

        self.alphabet = [ 'A', 'R', 'N', 'D', 'C', 'E', 'Q', 'G', 'H', 'I', 'L', 'K',
                     'M', 'F', 'P', 'S', 'T', 'W', 'Y', 'V', ]

        assert len(self.alphabet) == 20
        self.k = kwargs['K']

        self.feature_space = list(''.join(i) for i in product(self.alphabet,
            repeat=self.k))

        self.n_features = len(self.feature_space)

    def shingles(self, s, k):
        ''' return shingles of a given string given a k-mer size k '''
        return [ s[i : i + k ] for i in range(0, len(s) - k + 1) ]

    def vectorize(self, s):
        ''' convert shingles to features vector '''

        d = defaultdict(lambda: 0)

        for i in s:
            d[i] += 1

        return np.array([ d[i] for i in self.feature_space ])

    def transform(self, str):
        return self.vectorize(self.shingles(str, self.k))

class SimilarStringStore:

    def __init__(self):

        self.config = {}
        self.config['K'] = 2 # k-mer size for shingles
        self.config['P'] = 100 # number of Random Binary Projections
        self.config['MAX_DIST'] = 20 # maximum distance threshold (euclidean)

        self.transformer = FeatureGenerator(K = self.config['K'])

        self.hasher = hashes.RandomBinaryProjections('rbp', self.config['P'], rand_seed=42)

        self.backend = RedisStorage(Redis(host='localhost', port=6379, db=0))

        self.filters = [ DistanceThresholdFilter(self.config['MAX_DIST']) ]

        self.engine = Engine(self.transformer.n_features,
                     lshashes=[self.hasher],
                     vector_filters = [], #self.filters,
                     storage=self.backend
                     )

    def add(self, s, id):
        ''' add a string to index '''
        vector = self.transformer.transform(s)
        self.engine.store_vector(vector, str(id))
        return vector

    def query(self, s):
        ''' query index '''
        vector = self.transformer.transform(s)
        neighbours = self.engine.neighbours(vector)
        return neighbours
<commit_msg>filters<commit_after>#!/usr/bin/env python

from itertools import product
from collections import defaultdict
from redis import Redis
import sys
import json

from Bio import SeqIO

from nearpy import Engine
from nearpy import hashes
from nearpy.storage import RedisStorage
from nearpy.filters import DistanceThresholdFilter

import numpy as np

# The distance ratio of an ANN y is it's distance to the minimal hypersphere
# around the query vector x, that contains all exact nearest neighbours n,
# clamped to zero and normalized with this hypersphere's radius

class FeatureGenerator:

    def __init__(self, **kwargs):
        ''' '''

        self.alphabet = [ 'A', 'R', 'N', 'D', 'C', 'E', 'Q', 'G', 'H', 'I', 'L', 'K',
                     'M', 'F', 'P', 'S', 'T', 'W', 'Y', 'V', ]

        assert len(self.alphabet) == 20
        self.k = kwargs['K']

        self.feature_space = list(''.join(i) for i in product(self.alphabet,
            repeat=self.k))

        self.n_features = len(self.feature_space)

    def shingles(self, s, k):
        ''' return shingles of a given string given a k-mer size k '''
        return [ s[i : i + k ] for i in range(0, len(s) - k + 1) ]

    def vectorize(self, s):
        ''' convert shingles to features vector '''

        d = defaultdict(lambda: 0)

        for i in s:
            d[i] += 1

        return np.array([ d[i] for i in self.feature_space ])

    def transform(self, str):
        return self.vectorize(self.shingles(str, self.k))

class SimilarStringStore:

    def __init__(self):

        self.config = {}
        self.config['K'] = 2 # k-mer size for shingles
        self.config['P'] = 100 # number of Random Binary Projections
        self.config['MAX_DIST'] = 20 # maximum distance threshold (euclidean)

        self.transformer = FeatureGenerator(K = self.config['K'])

        self.hasher = hashes.RandomBinaryProjections('rbp', self.config['P'], rand_seed=42)

        self.backend = RedisStorage(Redis(host='localhost', port=6379, db=0))

        self.filters = [ DistanceThresholdFilter(self.config['MAX_DIST']) ]

        self.engine = Engine(self.transformer.n_features,
                     lshashes=[self.hasher],
                     vector_filters = self.filters,
                     storage=self.backend
                     )

    def add(self, s, id):
        ''' add a string to index '''
        vector = self.transformer.transform(s)
        self.engine.store_vector(vector, str(id))
        return vector

    def query(self, s):
        ''' query index '''
        vector = self.transformer.transform(s)
        neighbours = self.engine.neighbours(vector)
        return neighbours
<|endoftext|>


<commit_before>courselist = [{'department': 'MATH', 'number': 417, 'crn': 65075},
              {'department': 'MATH', 'number': 484, 'crn': 30809},
              {'department': 'MATH', 'number': 347, 'crn': 34395},
             ]

<commit_msg>Update courselist.py<commit_after>courselist = [{'department': 'MATH', 'number': 417, 'crn': 65075},
              {'department': 'MATH', 'number': 484, 'crn': 30809},
              {'department': 'MATH', 'number': 347, 'crn': 34395},
              {'department': 'MATH', 'number': 124, 'crn': 45900},
             ]

<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright (c) 2010 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""A tool to archive layout test results generated by buildbots.

Actual result files (*-actual.txt), but not results from simplified diff
tests (*-simp-actual.txt) or JS-filtered diff tests (*-jsfilt.txt), will
be included in the archive.

When this is run, the current directory (cwd) should be the outer build
directory (e.g., chrome-release/build/).

For a list of command-line options, call this script with '--help'.
"""

import logging
import optparse
import os
import socket
import sys

from common import chromium_utils
from slave import slave_utils
import config

# Directory name, above the build directory, in which test results can be
# found if no --results-dir option is given.
RESULT_DIR = 'layout-test-results'

def _CollectArchiveFiles(output_dir):
  """Returns a tuple containing two lists list of file paths to archive,
  relative to the output_dir. The first list is all the actual results from the
  test run. The second list is the diffs from the expected results.

  Files in the output_dir or one of its subdirectories, whose names end with
  '-actual.txt' but not '-simp-actual.txt' or '-jsfilt-actual.txt',
  will be included in the list.
  """
  actual_file_list = []
  diff_file_list = []
  for path, _, files in os.walk(output_dir):
    rel_path = path[len(output_dir + '\\'):]
    for name in files:
      if ('-stack.' in name or
          ('-actual.' in name and
           (name.endswith('.txt') or
            name.endswith('.png') or
            name.endswith('.checksum')) and
           '-simp-actual.' not in name and
           '-jsfilt-actual.' not in name)):
        actual_file_list.append(os.path.join(rel_path, name))
      elif ('-wdiff.' in name or
            name.endswith('-diff.txt') or
            name.endswith('-diff.png')):
        diff_file_list.append(os.path.join(rel_path, name))
  if os.path.exists(os.path.join(output_dir, 'results.html')):
    actual_file_list.append('results.html')
  return (actual_file_list, diff_file_list)

def _ArchiveFullLayoutTestResults(staging_dir, dest_dir, diff_file_list,
    options):
  # Copy the actual and diff files to the web server.
  # Don't clobber the staging_dir in the MakeZip call so that it keeps the
  # files from the previous MakeZip call on diff_file_list.
  print "archiving results + diffs"
  full_zip_file = chromium_utils.MakeZip(staging_dir,
      'layout-test-results', diff_file_list, options.results_dir,
      remove_archive_directory=False)[1]
  slave_utils.CopyFileToArchiveHost(full_zip_file, dest_dir)

  # Extract the files on the web server.
  extract_dir = os.path.join(dest_dir, 'results')
  print 'extracting zip file to %s' % extract_dir

  if chromium_utils.IsWindows():
    chromium_utils.ExtractZip(full_zip_file, extract_dir)
  elif chromium_utils.IsLinux() or chromium_utils.IsMac():
    remote_zip_file = os.path.join(dest_dir, os.path.basename(full_zip_file))
    chromium_utils.SshExtractZip(config.Archive.archive_host, remote_zip_file,
                                 extract_dir)

def archive_layout(options, args):
  logging.basicConfig(level=logging.INFO,
                      format='%(asctime)s %(filename)s:%(lineno)-3d'
                             ' %(levelname)s %(message)s',
                      datefmt='%y%m%d %H:%M:%S')
  chrome_dir = os.path.abspath(options.build_dir)
  if options.results_dir is not None:
    options.results_dir = os.path.abspath(os.path.join(options.build_dir,
                                                       options.results_dir))
  else:
    options.results_dir = chromium_utils.FindUpward(chrome_dir, RESULT_DIR)
  print 'Archiving results from %s' % options.results_dir
  staging_dir = slave_utils.GetStagingDir(chrome_dir)
  print 'Staging in %s' % staging_dir

  (actual_file_list, diff_file_list) = _CollectArchiveFiles(options.results_dir)
  zip_file = chromium_utils.MakeZip(staging_dir,
                                    'layout-test-results',
                                    actual_file_list,
                                    options.results_dir)[1]

  # Extract the build name of this slave (e.g., 'chrome-release') from its
  # configuration file.
  build_name = slave_utils.SlaveBuildName(chrome_dir)

  last_change = str(slave_utils.SubversionRevision(chrome_dir))
  print 'last change: %s' % last_change
  print 'build name: %s' % build_name
  print 'host name: %s' % socket.gethostname()

  # Where to save layout test results.
  dest_parent_dir = os.path.join(config.Archive.www_dir_base,
      'layout_test_results', build_name)
  dest_dir = os.path.join(dest_parent_dir, last_change)
  slave_utils.MaybeMakeDirectoryOnArchiveHost(dest_dir)
  slave_utils.CopyFileToArchiveHost(zip_file, dest_dir)

  _ArchiveFullLayoutTestResults(staging_dir, dest_parent_dir, diff_file_list,
      options)
  return 0

def main():
  option_parser = optparse.OptionParser()
  option_parser.add_option('', '--build-dir', default='webkit',
                           help='path to main build directory (the parent of '
                                'the Release or Debug directory)')
  option_parser.add_option('', '--results-dir',
                           help='path to layout test results, relative to '
                                'the build_dir')
  option_parser.add_option('', '--builder-name',
                           default=None,
                           help='The name of the builder running this script.')
  option_parser.add_option('', '--build-number',
                           default=None,
                           help=('The build number of the builder running'
                                 'this script.'))
  options, args = option_parser.parse_args()
  return archive_layout(options, args)

if '__main__' == __name__:
  sys.exit(main())
<commit_msg>Fix the layout test archiving tool to not hard-code 'layout_test_results', so that the tool will work properly for both regular webkit_tests and the webkit_gpu tests without them clobbering each other's baselines.<commit_after>#!/usr/bin/python
# Copyright (c) 2010 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""A tool to archive layout test results generated by buildbots.

Actual result files (*-actual.txt), but not results from simplified diff
tests (*-simp-actual.txt) or JS-filtered diff tests (*-jsfilt.txt), will
be included in the archive.

When this is run, the current directory (cwd) should be the outer build
directory (e.g., chrome-release/build/).

For a list of command-line options, call this script with '--help'.
"""

import logging
import optparse
import os
import socket
import sys

from common import chromium_utils
from slave import slave_utils
import config

# Directory name, above the build directory, in which test results can be
# found if no --results-dir option is given.
RESULT_DIR = 'layout-test-results'

def _CollectArchiveFiles(output_dir):
  """Returns a tuple containing two lists list of file paths to archive,
  relative to the output_dir. The first list is all the actual results from the
  test run. The second list is the diffs from the expected results.

  Files in the output_dir or one of its subdirectories, whose names end with
  '-actual.txt' but not '-simp-actual.txt' or '-jsfilt-actual.txt',
  will be included in the list.
  """
  actual_file_list = []
  diff_file_list = []
  for path, _, files in os.walk(output_dir):
    rel_path = path[len(output_dir + '\\'):]
    for name in files:
      if ('-stack.' in name or
          ('-actual.' in name and
           (name.endswith('.txt') or
            name.endswith('.png') or
            name.endswith('.checksum')) and
           '-simp-actual.' not in name and
           '-jsfilt-actual.' not in name)):
        actual_file_list.append(os.path.join(rel_path, name))
      elif ('-wdiff.' in name or
            name.endswith('-diff.txt') or
            name.endswith('-diff.png')):
        diff_file_list.append(os.path.join(rel_path, name))
  if os.path.exists(os.path.join(output_dir, 'results.html')):
    actual_file_list.append('results.html')
  return (actual_file_list, diff_file_list)

def _ArchiveFullLayoutTestResults(staging_dir, dest_dir, diff_file_list,
    options):
  # Copy the actual and diff files to the web server.
  # Don't clobber the staging_dir in the MakeZip call so that it keeps the
  # files from the previous MakeZip call on diff_file_list.
  print "archiving results + diffs"
  full_zip_file = chromium_utils.MakeZip(staging_dir,
      'layout-test-results', diff_file_list, options.results_dir,
      remove_archive_directory=False)[1]
  slave_utils.CopyFileToArchiveHost(full_zip_file, dest_dir)

  # Extract the files on the web server.
  extract_dir = os.path.join(dest_dir, 'results')
  print 'extracting zip file to %s' % extract_dir

  if chromium_utils.IsWindows():
    chromium_utils.ExtractZip(full_zip_file, extract_dir)
  elif chromium_utils.IsLinux() or chromium_utils.IsMac():
    remote_zip_file = os.path.join(dest_dir, os.path.basename(full_zip_file))
    chromium_utils.SshExtractZip(config.Archive.archive_host, remote_zip_file,
                                 extract_dir)

def archive_layout(options, args):
  logging.basicConfig(level=logging.INFO,
                      format='%(asctime)s %(filename)s:%(lineno)-3d'
                             ' %(levelname)s %(message)s',
                      datefmt='%y%m%d %H:%M:%S')
  chrome_dir = os.path.abspath(options.build_dir)
  results_dir_basename = os.path.basename(options.results_dir)
  if options.results_dir is not None:
    options.results_dir = os.path.abspath(os.path.join(options.build_dir,
                                                       options.results_dir))
  else:
    options.results_dir = chromium_utils.FindUpward(chrome_dir, RESULT_DIR)
  print 'Archiving results from %s' % options.results_dir
  staging_dir = slave_utils.GetStagingDir(chrome_dir)
  print 'Staging in %s' % staging_dir

  (actual_file_list, diff_file_list) = _CollectArchiveFiles(options.results_dir)
  zip_file = chromium_utils.MakeZip(staging_dir,
                                    results_dir_basename,
                                    actual_file_list,
                                    options.results_dir)[1]

  # Extract the build name of this slave (e.g., 'chrome-release') from its
  # configuration file.
  build_name = slave_utils.SlaveBuildName(chrome_dir)

  last_change = str(slave_utils.SubversionRevision(chrome_dir))
  print 'last change: %s' % last_change
  print 'build name: %s' % build_name
  print 'host name: %s' % socket.gethostname()

  # Where to save layout test results.
  dest_parent_dir = os.path.join(config.Archive.www_dir_base,
      results_dir_basename, build_name)
  dest_dir = os.path.join(dest_parent_dir, last_change)
  slave_utils.MaybeMakeDirectoryOnArchiveHost(dest_dir)
  slave_utils.CopyFileToArchiveHost(zip_file, dest_dir)

  _ArchiveFullLayoutTestResults(staging_dir, dest_parent_dir, diff_file_list,
      options)
  return 0

def main():
  option_parser = optparse.OptionParser()
  option_parser.add_option('', '--build-dir', default='webkit',
                           help='path to main build directory (the parent of '
                                'the Release or Debug directory)')
  option_parser.add_option('', '--results-dir',
                           help='path to layout test results, relative to '
                                'the build_dir')
  option_parser.add_option('', '--builder-name',
                           default=None,
                           help='The name of the builder running this script.')
  option_parser.add_option('', '--build-number',
                           default=None,
                           help=('The build number of the builder running'
                                 'this script.'))
  options, args = option_parser.parse_args()
  return archive_layout(options, args)

if '__main__' == __name__:
  sys.exit(main())
<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""A tool to archive layout test results generated by buildbots.

Actual result files (*-actual.txt), but not results from simplified diff
tests (*-simp-actual.txt) or JS-filtered diff tests (*-jsfilt.txt), will
be included in the archive.

When this is run, the current directory (cwd) should be the outer build
directory (e.g., chrome-release/build/).

For a list of command-line options, call this script with '--help'.
"""

import logging
import optparse
import os
import re
import socket
import sys

from common import chromium_utils
from slave import slave_utils
import config

# Directory name, above the build directory, in which test results can be
# found if no --results-dir option is given.
RESULT_DIR = 'layout-test-results'

def _CollectArchiveFiles(output_dir):
  """Returns a tuple containing two lists list of file paths to archive,
  relative to the output_dir. The first list is all the actual results from the
  test run. The second list is the diffs from the expected results.

  Files in the output_dir or one of its subdirectories, whose names end with
  '-actual.txt' but not '-simp-actual.txt' or '-jsfilt-actual.txt',
  will be included in the list.
  """
  actual_file_list = []
  diff_file_list = []
  for path, _, files in os.walk(output_dir):
    rel_path = path[len(output_dir + '\\'):]
    for name in files:
      if ('-stack.' in name or
          ('-actual.' in name and
           (name.endswith('.txt') or
            name.endswith('.png') or
            name.endswith('.checksum')) and
           '-simp-actual.' not in name and
           '-jsfilt-actual.' not in name)):
        actual_file_list.append(os.path.join(rel_path, name))
      elif ('-wdiff.' in name or
            '-expected.' in name or
            name.endswith('-diff.txt') or
            name.endswith('-diff.png')):
        diff_file_list.append(os.path.join(rel_path, name))
      elif name.endswith('.json'):
        actual_file_list.append(os.path.join(rel_path, name))
  if os.path.exists(os.path.join(output_dir, 'results.html')):
    actual_file_list.append('results.html')
  return (actual_file_list, diff_file_list)

def _ArchiveFullLayoutTestResults(staging_dir, dest_dir, diff_file_list,
    options):
  # Copy the actual and diff files to the web server.
  # Don't clobber the staging_dir in the MakeZip call so that it keeps the
  # files from the previous MakeZip call on diff_file_list.
  print "archiving results + diffs"
  full_zip_file = chromium_utils.MakeZip(staging_dir,
      'layout-test-results', diff_file_list, options.results_dir,
      remove_archive_directory=False)[1]
  slave_utils.CopyFileToArchiveHost(full_zip_file, dest_dir)

  # Extract the files on the web server.
  extract_dir = os.path.join(dest_dir, 'results')
  print 'extracting zip file to %s' % extract_dir

  if chromium_utils.IsWindows():
    chromium_utils.ExtractZip(full_zip_file, extract_dir)
  elif chromium_utils.IsLinux() or chromium_utils.IsMac():
    remote_zip_file = os.path.join(dest_dir, os.path.basename(full_zip_file))
    chromium_utils.SshExtractZip(config.Archive.archive_host, remote_zip_file,
                                 extract_dir)

def archive_layout(options, args):
  logging.basicConfig(level=logging.INFO,
                      format='%(asctime)s %(filename)s:%(lineno)-3d'
                             ' %(levelname)s %(message)s',
                      datefmt='%y%m%d %H:%M:%S')
  chrome_dir = os.path.abspath(options.build_dir)
  results_dir_basename = os.path.basename(options.results_dir)
  if options.results_dir is not None:
    options.results_dir = os.path.abspath(os.path.join(options.build_dir,
                                                       options.results_dir))
  else:
    options.results_dir = chromium_utils.FindUpward(chrome_dir, RESULT_DIR)
  print 'Archiving results from %s' % options.results_dir
  staging_dir = slave_utils.GetStagingDir(chrome_dir)
  print 'Staging in %s' % staging_dir

  (actual_file_list, diff_file_list) = _CollectArchiveFiles(options.results_dir)
  zip_file = chromium_utils.MakeZip(staging_dir,
                                    results_dir_basename,
                                    actual_file_list,
                                    options.results_dir)[1]

  # Extract the build name of this slave (e.g., 'chrome-release') from its
  # configuration file if not provided as a param.
  build_name = options.builder_name or slave_utils.SlaveBuildName(chrome_dir)
  build_name = re.sub('[ .()]', '_', build_name)

  last_change = str(slave_utils.SubversionRevision(chrome_dir))
  print 'last change: %s' % last_change
  print 'build name: %s' % build_name
  print 'host name: %s' % socket.gethostname()

  # Where to save layout test results.
  dest_parent_dir = os.path.join(config.Archive.www_dir_base,
      results_dir_basename.replace('-','_'), build_name)
  dest_dir = os.path.join(dest_parent_dir, last_change)

  gs_bucket = options.factory_properties.get('gs_bucket', None)
  if gs_bucket:
    gs_base = '/'.join([gs_bucket, build_name, last_change])
    slave_utils.GSUtilCopyFile(zip_file, gs_base)
  else:
    slave_utils.MaybeMakeDirectoryOnArchiveHost(dest_dir)
    slave_utils.CopyFileToArchiveHost(zip_file, dest_dir)
    # Not supported on Google Storage yet.
    _ArchiveFullLayoutTestResults(staging_dir, dest_parent_dir, diff_file_list,
                                  options)
  return 0

def main():
  option_parser = optparse.OptionParser()
  option_parser.add_option('', '--build-dir', default='webkit',
                           help='path to main build directory (the parent of '
                                'the Release or Debug directory)')
  option_parser.add_option('', '--results-dir',
                           help='path to layout test results, relative to '
                                'the build_dir')
  option_parser.add_option('', '--builder-name',
                           default=None,
                           help='The name of the builder running this script.')
  option_parser.add_option('', '--build-number',
                           default=None,
                           help=('The build number of the builder running'
                                 'this script.'))
  option_parser.add_option('--build-properties', action='callback',
                           callback=chromium_utils.convert_json, type='string',
                           nargs=1, default={},
                           help='build properties in JSON format')
  option_parser.add_option('--factory-properties', action='callback',
                           callback=chromium_utils.convert_json, type='string',
                           nargs=1, default={},
                           help='factory properties in JSON format')
  options, args = option_parser.parse_args()
  return archive_layout(options, args)

if '__main__' == __name__:
  sys.exit(main())
<commit_msg>Include NRWT crash logs in archived results (per FILENAME_SUFFIX_CRASH_LOG from WebKit/Tools/Scripts/webkitpy/layout_tests/layout_package/test_result_writer.py they get a -crash-log.txt suffix). Review URL: http://codereview.chromium.org/7310014<commit_after>#!/usr/bin/python
# Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""A tool to archive layout test results generated by buildbots.

Actual result files (*-actual.txt), but not results from simplified diff
tests (*-simp-actual.txt) or JS-filtered diff tests (*-jsfilt.txt), will
be included in the archive.

When this is run, the current directory (cwd) should be the outer build
directory (e.g., chrome-release/build/).

For a list of command-line options, call this script with '--help'.
"""

import logging
import optparse
import os
import re
import socket
import sys

from common import chromium_utils
from slave import slave_utils
import config

# Directory name, above the build directory, in which test results can be
# found if no --results-dir option is given.
RESULT_DIR = 'layout-test-results'

def _CollectArchiveFiles(output_dir):
  """Returns a tuple containing two lists list of file paths to archive,
  relative to the output_dir. The first list is all the actual results from the
  test run. The second list is the diffs from the expected results.

  Files in the output_dir or one of its subdirectories, whose names end with
  '-actual.txt' but not '-simp-actual.txt' or '-jsfilt-actual.txt',
  will be included in the list.
  """
  actual_file_list = []
  diff_file_list = []
  for path, _, files in os.walk(output_dir):
    rel_path = path[len(output_dir + '\\'):]
    for name in files:
      if ('-stack.' in name or
          '-crash-log.' in name or
          ('-actual.' in name and
           (name.endswith('.txt') or
            name.endswith('.png') or
            name.endswith('.checksum')) and
           '-simp-actual.' not in name and
           '-jsfilt-actual.' not in name)):
        actual_file_list.append(os.path.join(rel_path, name))
      elif ('-wdiff.' in name or
            '-expected.' in name or
            name.endswith('-diff.txt') or
            name.endswith('-diff.png')):
        diff_file_list.append(os.path.join(rel_path, name))
      elif name.endswith('.json'):
        actual_file_list.append(os.path.join(rel_path, name))
  if os.path.exists(os.path.join(output_dir, 'results.html')):
    actual_file_list.append('results.html')
  return (actual_file_list, diff_file_list)

def _ArchiveFullLayoutTestResults(staging_dir, dest_dir, diff_file_list,
    options):
  # Copy the actual and diff files to the web server.
  # Don't clobber the staging_dir in the MakeZip call so that it keeps the
  # files from the previous MakeZip call on diff_file_list.
  print "archiving results + diffs"
  full_zip_file = chromium_utils.MakeZip(staging_dir,
      'layout-test-results', diff_file_list, options.results_dir,
      remove_archive_directory=False)[1]
  slave_utils.CopyFileToArchiveHost(full_zip_file, dest_dir)

  # Extract the files on the web server.
  extract_dir = os.path.join(dest_dir, 'results')
  print 'extracting zip file to %s' % extract_dir

  if chromium_utils.IsWindows():
    chromium_utils.ExtractZip(full_zip_file, extract_dir)
  elif chromium_utils.IsLinux() or chromium_utils.IsMac():
    remote_zip_file = os.path.join(dest_dir, os.path.basename(full_zip_file))
    chromium_utils.SshExtractZip(config.Archive.archive_host, remote_zip_file,
                                 extract_dir)

def archive_layout(options, args):
  logging.basicConfig(level=logging.INFO,
                      format='%(asctime)s %(filename)s:%(lineno)-3d'
                             ' %(levelname)s %(message)s',
                      datefmt='%y%m%d %H:%M:%S')
  chrome_dir = os.path.abspath(options.build_dir)
  results_dir_basename = os.path.basename(options.results_dir)
  if options.results_dir is not None:
    options.results_dir = os.path.abspath(os.path.join(options.build_dir,
                                                       options.results_dir))
  else:
    options.results_dir = chromium_utils.FindUpward(chrome_dir, RESULT_DIR)
  print 'Archiving results from %s' % options.results_dir
  staging_dir = slave_utils.GetStagingDir(chrome_dir)
  print 'Staging in %s' % staging_dir

  (actual_file_list, diff_file_list) = _CollectArchiveFiles(options.results_dir)
  zip_file = chromium_utils.MakeZip(staging_dir,
                                    results_dir_basename,
                                    actual_file_list,
                                    options.results_dir)[1]

  # Extract the build name of this slave (e.g., 'chrome-release') from its
  # configuration file if not provided as a param.
  build_name = options.builder_name or slave_utils.SlaveBuildName(chrome_dir)
  build_name = re.sub('[ .()]', '_', build_name)

  last_change = str(slave_utils.SubversionRevision(chrome_dir))
  print 'last change: %s' % last_change
  print 'build name: %s' % build_name
  print 'host name: %s' % socket.gethostname()

  # Where to save layout test results.
  dest_parent_dir = os.path.join(config.Archive.www_dir_base,
      results_dir_basename.replace('-','_'), build_name)
  dest_dir = os.path.join(dest_parent_dir, last_change)

  gs_bucket = options.factory_properties.get('gs_bucket', None)
  if gs_bucket:
    gs_base = '/'.join([gs_bucket, build_name, last_change])
    slave_utils.GSUtilCopyFile(zip_file, gs_base)
  else:
    slave_utils.MaybeMakeDirectoryOnArchiveHost(dest_dir)
    slave_utils.CopyFileToArchiveHost(zip_file, dest_dir)
    # Not supported on Google Storage yet.
    _ArchiveFullLayoutTestResults(staging_dir, dest_parent_dir, diff_file_list,
                                  options)
  return 0

def main():
  option_parser = optparse.OptionParser()
  option_parser.add_option('', '--build-dir', default='webkit',
                           help='path to main build directory (the parent of '
                                'the Release or Debug directory)')
  option_parser.add_option('', '--results-dir',
                           help='path to layout test results, relative to '
                                'the build_dir')
  option_parser.add_option('', '--builder-name',
                           default=None,
                           help='The name of the builder running this script.')
  option_parser.add_option('', '--build-number',
                           default=None,
                           help=('The build number of the builder running'
                                 'this script.'))
  option_parser.add_option('--build-properties', action='callback',
                           callback=chromium_utils.convert_json, type='string',
                           nargs=1, default={},
                           help='build properties in JSON format')
  option_parser.add_option('--factory-properties', action='callback',
                           callback=chromium_utils.convert_json, type='string',
                           nargs=1, default={},
                           help='factory properties in JSON format')
  options, args = option_parser.parse_args()
  return archive_layout(options, args)

if '__main__' == __name__:
  sys.exit(main())
<|endoftext|>


<commit_before># copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of astroid.
#
# astroid is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by the
# Free Software Foundation, either version 2.1 of the License, or (at your
# option) any later version.
#
# astroid is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
# for more details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with astroid. If not, see <http://www.gnu.org/licenses/>.
from textwrap import dedent
import unittest

from astroid import nodes
from astroid.node_classes import Assign, Expr, YieldFrom, Name, Const
from astroid.builder import AstroidBuilder
from astroid.scoped_nodes import ClassDef, FunctionDef
from astroid.test_utils import require_version, extract_node

class Python3TC(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.builder = AstroidBuilder()

    @require_version('3.0')
    def test_starred_notation(self):
        astroid = self.builder.string_build("*a, b = [1, 2, 3]", 'test', 'test')

        # Get the star node
        node = next(next(next(astroid.get_children()).get_children()).get_children())

        self.assertTrue(isinstance(node.assign_type(), Assign))

    @require_version('3.3')
    def test_yield_from(self):
        body = dedent("""
        def func():
            yield from iter([1, 2])
        """)
        astroid = self.builder.string_build(body)
        func = astroid.body[0]
        self.assertIsInstance(func, FunctionDef)
        yieldfrom_stmt = func.body[0]

        self.assertIsInstance(yieldfrom_stmt, Expr)
        self.assertIsInstance(yieldfrom_stmt.value, YieldFrom)
        self.assertEqual(yieldfrom_stmt.as_string(),
                         'yield from iter([1, 2])')

    @require_version('3.3')
    def test_yield_from_is_generator(self):
        body = dedent("""
        def func():
            yield from iter([1, 2])
        """)
        astroid = self.builder.string_build(body)
        func = astroid.body[0]
        self.assertIsInstance(func, FunctionDef)
        self.assertTrue(func.is_generator())

    @require_version('3.3')
    def test_yield_from_as_string(self):
        body = dedent("""
        def func():
            yield from iter([1, 2])
            value = yield from other()
        """)
        astroid = self.builder.string_build(body)
        func = astroid.body[0]
        self.assertEqual(func.as_string().strip(), body.strip())

    # metaclass tests

    @require_version('3.0')
    def test_simple_metaclass(self):
        astroid = self.builder.string_build("class Test(metaclass=type): pass")
        klass = astroid.body[0]

        metaclass = klass.metaclass()
        self.assertIsInstance(metaclass, ClassDef)
        self.assertEqual(metaclass.name, 'type')

    @require_version('3.0')
    def test_metaclass_error(self):
        astroid = self.builder.string_build("class Test(metaclass=typ): pass")
        klass = astroid.body[0]
        self.assertFalse(klass.metaclass())

    @require_version('3.0')
    def test_metaclass_imported(self):
        astroid = self.builder.string_build(dedent("""
        from abc import ABCMeta
        class Test(metaclass=ABCMeta): pass"""))
        klass = astroid.body[1]

        metaclass = klass.metaclass()
        self.assertIsInstance(metaclass, ClassDef)
        self.assertEqual(metaclass.name, 'ABCMeta')

    @require_version('3.0')
    def test_as_string(self):
        body = dedent("""
        from abc import ABCMeta
        class Test(metaclass=ABCMeta): pass""")
        astroid = self.builder.string_build(body)
        klass = astroid.body[1]

        self.assertEqual(klass.as_string(),
                         '\n\nclass Test(metaclass=ABCMeta):\n    pass\n')

    @require_version('3.0')
    def test_old_syntax_works(self):
        astroid = self.builder.string_build(dedent("""
        class Test:
            __metaclass__ = type
        class SubTest(Test): pass
        """))
        klass = astroid['SubTest']
        metaclass = klass.metaclass()
        self.assertIsNone(metaclass)

    @require_version('3.0')
    def test_metaclass_yes_leak(self):
        astroid = self.builder.string_build(dedent("""
        # notice `ab` instead of `abc`
        from ab import ABCMeta

        class Meta(metaclass=ABCMeta): pass
        """))
        klass = astroid['Meta']
        self.assertIsNone(klass.metaclass())

    @require_version('3.0')
    def test_parent_metaclass(self):
        astroid = self.builder.string_build(dedent("""
        from abc import ABCMeta
        class Test(metaclass=ABCMeta): pass
        class SubTest(Test): pass
        """))
        klass = astroid['SubTest']
        self.assertTrue(klass.newstyle)
        metaclass = klass.metaclass()
        self.assertIsInstance(metaclass, ClassDef)
        self.assertEqual(metaclass.name, 'ABCMeta')

    @require_version('3.0')
    def test_metaclass_ancestors(self):
        astroid = self.builder.string_build(dedent("""
        from abc import ABCMeta

        class FirstMeta(metaclass=ABCMeta): pass
        class SecondMeta(metaclass=type):
            pass

        class Simple:
            pass

        class FirstImpl(FirstMeta): pass
        class SecondImpl(FirstImpl): pass
        class ThirdImpl(Simple, SecondMeta):
            pass
        """))
        classes = {
            'ABCMeta': ('FirstImpl', 'SecondImpl'),
            'type': ('ThirdImpl', )
        }
        for metaclass, names in classes.items():
            for name in names:
                impl = astroid[name]
                meta = impl.metaclass()
                self.assertIsInstance(meta, ClassDef)
                self.assertEqual(meta.name, metaclass)

    @require_version('3.0')
    def test_annotation_support(self):
        astroid = self.builder.string_build(dedent("""
        def test(a: int, b: str, c: None, d, e,
                 *args: float, **kwargs: int)->int:
            pass
        """))
        func = astroid['test']
        self.assertIsInstance(func.args.varargannotation, Name)
        self.assertEqual(func.args.varargannotation.name, 'float')
        self.assertIsInstance(func.args.kwargannotation, Name)
        self.assertEqual(func.args.kwargannotation.name, 'int')
        self.assertIsInstance(func.returns, Name)
        self.assertEqual(func.returns.name, 'int')
        arguments = func.args
        self.assertIsInstance(arguments.annotations[0], Name)
        self.assertEqual(arguments.annotations[0].name, 'int')
        self.assertIsInstance(arguments.annotations[1], Name)
        self.assertEqual(arguments.annotations[1].name, 'str')
        self.assertIsInstance(arguments.annotations[2], Const)
        self.assertIsNone(arguments.annotations[2].value)
        self.assertIsNone(arguments.annotations[3])
        self.assertIsNone(arguments.annotations[4])

        astroid = self.builder.string_build(dedent("""
        def test(a: int=1, b: str=2):
            pass
        """))
        func = astroid['test']
        self.assertIsInstance(func.args.annotations[0], Name)
        self.assertEqual(func.args.annotations[0].name, 'int')
        self.assertIsInstance(func.args.annotations[1], Name)
        self.assertEqual(func.args.annotations[1].name, 'str')
        self.assertIsNone(func.returns)

    @require_version('3.0')
    def test_annotation_as_string(self):
        code1 = dedent('''
        def test(a, b:int=4, c=2, f:'lala'=4)->2:
            pass''')
        code2 = dedent('''
        def test(a:typing.Generic[T], c:typing.Any=24)->typing.Iterable:
            pass''')
        for code in (code1, code2):
            func = extract_node(code)
            self.assertEqual(func.as_string(), code)

    @require_version('3.5')
    def test_unpacking_in_dicts(self):
        code = "{'x': 1, **{'y': 2}}"
        node = extract_node(code)
        self.assertEqual(node.as_string(), code)
        keys = [key for (key, _) in node.items]
        self.assertIsInstance(keys[0], nodes.Const)
        self.assertIsInstance(keys[1], nodes.DictUnpack)

    @require_version('3.5')
    def test_nested_unpacking_in_dicts(self):
        code = "{'x': 1, **{'y': 2, **{'z': 3}}}"
        node = extract_node(code)
        self.assertEqual(node.as_string(), code)

    @require_version('3.5')
    def test_unpacking_in_dict_getitem(self):
        node = extract_node('{1:2, **{2:3, 3:4}, **{5: 6}}')
        for key, expected in ((1, 2), (2, 3), (3, 4), (5, 6)):
            value = node.getitem(key)
            self.assertIsInstance(value, nodes.Const)
            self.assertEqual(value.value, expected)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Remove trailing whitespace.<commit_after># copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of astroid.
#
# astroid is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by the
# Free Software Foundation, either version 2.1 of the License, or (at your
# option) any later version.
#
# astroid is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
# for more details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with astroid. If not, see <http://www.gnu.org/licenses/>.
from textwrap import dedent
import unittest

from astroid import nodes
from astroid.node_classes import Assign, Expr, YieldFrom, Name, Const
from astroid.builder import AstroidBuilder
from astroid.scoped_nodes import ClassDef, FunctionDef
from astroid.test_utils import require_version, extract_node

class Python3TC(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.builder = AstroidBuilder()

    @require_version('3.0')
    def test_starred_notation(self):
        astroid = self.builder.string_build("*a, b = [1, 2, 3]", 'test', 'test')

        # Get the star node
        node = next(next(next(astroid.get_children()).get_children()).get_children())

        self.assertTrue(isinstance(node.assign_type(), Assign))

    @require_version('3.3')
    def test_yield_from(self):
        body = dedent("""
        def func():
            yield from iter([1, 2])
        """)
        astroid = self.builder.string_build(body)
        func = astroid.body[0]
        self.assertIsInstance(func, FunctionDef)
        yieldfrom_stmt = func.body[0]

        self.assertIsInstance(yieldfrom_stmt, Expr)
        self.assertIsInstance(yieldfrom_stmt.value, YieldFrom)
        self.assertEqual(yieldfrom_stmt.as_string(),
                         'yield from iter([1, 2])')

    @require_version('3.3')
    def test_yield_from_is_generator(self):
        body = dedent("""
        def func():
            yield from iter([1, 2])
        """)
        astroid = self.builder.string_build(body)
        func = astroid.body[0]
        self.assertIsInstance(func, FunctionDef)
        self.assertTrue(func.is_generator())

    @require_version('3.3')
    def test_yield_from_as_string(self):
        body = dedent("""
        def func():
            yield from iter([1, 2])
            value = yield from other()
        """)
        astroid = self.builder.string_build(body)
        func = astroid.body[0]
        self.assertEqual(func.as_string().strip(), body.strip())

    # metaclass tests

    @require_version('3.0')
    def test_simple_metaclass(self):
        astroid = self.builder.string_build("class Test(metaclass=type): pass")
        klass = astroid.body[0]

        metaclass = klass.metaclass()
        self.assertIsInstance(metaclass, ClassDef)
        self.assertEqual(metaclass.name, 'type')

    @require_version('3.0')
    def test_metaclass_error(self):
        astroid = self.builder.string_build("class Test(metaclass=typ): pass")
        klass = astroid.body[0]
        self.assertFalse(klass.metaclass())

    @require_version('3.0')
    def test_metaclass_imported(self):
        astroid = self.builder.string_build(dedent("""
        from abc import ABCMeta
        class Test(metaclass=ABCMeta): pass"""))
        klass = astroid.body[1]

        metaclass = klass.metaclass()
        self.assertIsInstance(metaclass, ClassDef)
        self.assertEqual(metaclass.name, 'ABCMeta')

    @require_version('3.0')
    def test_as_string(self):
        body = dedent("""
        from abc import ABCMeta
        class Test(metaclass=ABCMeta): pass""")
        astroid = self.builder.string_build(body)
        klass = astroid.body[1]

        self.assertEqual(klass.as_string(),
                         '\n\nclass Test(metaclass=ABCMeta):\n    pass\n')

    @require_version('3.0')
    def test_old_syntax_works(self):
        astroid = self.builder.string_build(dedent("""
        class Test:
            __metaclass__ = type
        class SubTest(Test): pass
        """))
        klass = astroid['SubTest']
        metaclass = klass.metaclass()
        self.assertIsNone(metaclass)

    @require_version('3.0')
    def test_metaclass_yes_leak(self):
        astroid = self.builder.string_build(dedent("""
        # notice `ab` instead of `abc`
        from ab import ABCMeta

        class Meta(metaclass=ABCMeta): pass
        """))
        klass = astroid['Meta']
        self.assertIsNone(klass.metaclass())

    @require_version('3.0')
    def test_parent_metaclass(self):
        astroid = self.builder.string_build(dedent("""
        from abc import ABCMeta
        class Test(metaclass=ABCMeta): pass
        class SubTest(Test): pass
        """))
        klass = astroid['SubTest']
        self.assertTrue(klass.newstyle)
        metaclass = klass.metaclass()
        self.assertIsInstance(metaclass, ClassDef)
        self.assertEqual(metaclass.name, 'ABCMeta')

    @require_version('3.0')
    def test_metaclass_ancestors(self):
        astroid = self.builder.string_build(dedent("""
        from abc import ABCMeta

        class FirstMeta(metaclass=ABCMeta): pass
        class SecondMeta(metaclass=type):
            pass

        class Simple:
            pass

        class FirstImpl(FirstMeta): pass
        class SecondImpl(FirstImpl): pass
        class ThirdImpl(Simple, SecondMeta):
            pass
        """))
        classes = {
            'ABCMeta': ('FirstImpl', 'SecondImpl'),
            'type': ('ThirdImpl', )
        }
        for metaclass, names in classes.items():
            for name in names:
                impl = astroid[name]
                meta = impl.metaclass()
                self.assertIsInstance(meta, ClassDef)
                self.assertEqual(meta.name, metaclass)

    @require_version('3.0')
    def test_annotation_support(self):
        astroid = self.builder.string_build(dedent("""
        def test(a: int, b: str, c: None, d, e,
                 *args: float, **kwargs: int)->int:
            pass
        """))
        func = astroid['test']
        self.assertIsInstance(func.args.varargannotation, Name)
        self.assertEqual(func.args.varargannotation.name, 'float')
        self.assertIsInstance(func.args.kwargannotation, Name)
        self.assertEqual(func.args.kwargannotation.name, 'int')
        self.assertIsInstance(func.returns, Name)
        self.assertEqual(func.returns.name, 'int')
        arguments = func.args
        self.assertIsInstance(arguments.annotations[0], Name)
        self.assertEqual(arguments.annotations[0].name, 'int')
        self.assertIsInstance(arguments.annotations[1], Name)
        self.assertEqual(arguments.annotations[1].name, 'str')
        self.assertIsInstance(arguments.annotations[2], Const)
        self.assertIsNone(arguments.annotations[2].value)
        self.assertIsNone(arguments.annotations[3])
        self.assertIsNone(arguments.annotations[4])

        astroid = self.builder.string_build(dedent("""
        def test(a: int=1, b: str=2):
            pass
        """))
        func = astroid['test']
        self.assertIsInstance(func.args.annotations[0], Name)
        self.assertEqual(func.args.annotations[0].name, 'int')
        self.assertIsInstance(func.args.annotations[1], Name)
        self.assertEqual(func.args.annotations[1].name, 'str')
        self.assertIsNone(func.returns)

    @require_version('3.0')
    def test_annotation_as_string(self):
        code1 = dedent('''
        def test(a, b:int=4, c=2, f:'lala'=4)->2:
            pass''')
        code2 = dedent('''
        def test(a:typing.Generic[T], c:typing.Any=24)->typing.Iterable:
            pass''')
        for code in (code1, code2):
            func = extract_node(code)
            self.assertEqual(func.as_string(), code)

    @require_version('3.5')
    def test_unpacking_in_dicts(self):
        code = "{'x': 1, **{'y': 2}}"
        node = extract_node(code)
        self.assertEqual(node.as_string(), code)
        keys = [key for (key, _) in node.items]
        self.assertIsInstance(keys[0], nodes.Const)
        self.assertIsInstance(keys[1], nodes.DictUnpack)

    @require_version('3.5')
    def test_nested_unpacking_in_dicts(self):
        code = "{'x': 1, **{'y': 2, **{'z': 3}}}"
        node = extract_node(code)
        self.assertEqual(node.as_string(), code)

    @require_version('3.5')
    def test_unpacking_in_dict_getitem(self):
        node = extract_node('{1:2, **{2:3, 3:4}, **{5: 6}}')
        for key, expected in ((1, 2), (2, 3), (3, 4), (5, 6)):
            value = node.getitem(key)
            self.assertIsInstance(value, nodes.Const)
            self.assertEqual(value.value, expected)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import numpy

from ...import describe
from ...describe import Dimension, Synapses, Gradient
from .._lsuv import LSUVinit
from ..ops import NumpyOps
from ...api import layerize
from .model import Model
from ...extra.load_nlp import get_vectors

try:
    import cupy
except ImportError:
    cupy = None

@layerize
def get_word_ids(docs, drop=0.):
    '''Get word forms.'''
    seqs = []
    ops = Model.ops
    for doc in docs:
        arr = numpy.zeros((len(doc)+1,), dtype='uint64')
        for i, token in enumerate(doc):
            arr[i] = token.lex_id or token.orth
        arr[len(doc)] = 0
        seqs.append(ops.asarray(arr))
    return seqs, None

@describe.on_data(LSUVinit)
@describe.attributes(
        nM=Dimension("Vector dimensions"),
        nO=Dimension("Size of output"),
        W=Synapses(
            "A projection matrix, to change vector dimensionality",
            lambda obj: (obj.nO, obj.nM),
            lambda W, ops: ops.xavier_uniform_init(W)),
        d_W=Gradient("W"),
)
class StaticVectors(Model):
    '''Load a static embedding table, and learn a linear projection from it.

    Out-of-vocabulary items are modded into the table, receiving an arbitrary
    vector (but the same word will always receive the same vector).
    '''
    name = 'static-vectors'
    def __init__(self, lang, nO):
        Model.__init__(self)
        self.nO = nO
        # This doesn't seem the cleverest solution,
        # but it ensures multiple models load the
        # same copy of spaCy if they're deserialised.
        self.lang = lang
        vectors = self.get_vectors()
        self.nM = vectors.shape[1]
        if self.nM == 0:
            raise ValueError(
                "Cannot create vectors table with dimension 0.\n"
                "If you're using pre-trained vectors, are the vectors loaded?")
        self.nV = vectors.shape[0]

    def get_vectors(self):
        return get_vectors(self.ops, self.lang)

    def begin_update(self, ids, drop=0.):
        vector_table = self.get_vectors()
        vectors = vector_table[ids % vector_table.shape[0]]
        def finish_update(gradients, sgd=None):
            self.d_W += self.ops.batch_outer(gradients, vectors)
            if sgd is not None:
                sgd(self._mem.weights, self._mem.gradient, key=self.id)
            return None
        dotted = self.ops.batch_dot(vectors, self.W)
        return dotted, finish_update
<commit_msg>Support token dropout in get_word_ids, with POS tag backoff<commit_after>import numpy

from ...import describe
from ...describe import Dimension, Synapses, Gradient
from .._lsuv import LSUVinit
from ..ops import NumpyOps
from ...api import layerize
from .model import Model
from ...extra.load_nlp import get_vectors

try:
    import cupy
except ImportError:
    cupy = None

def get_word_ids(ops, pad=1, token_drop=0.):
    def get_word_ids(docs, drop=0.):
        '''Get word forms.'''
        seqs = []
        ops = Model.ops
        for doc in docs:
            arr = ops.allocate((len(doc)+pad,), dtype='uint64')
            mask = ops.get_dropout_mask(arr.shape, token_drop)
            for i, token in enumerate(doc):
                if mask:
                    arr[i] = token.lex_id or token.orth
                else:
                    arr[i] = token.tag
            for i in range(len(doc), len(doc)+pad):
                arr[i] = 0
            seqs.append(ops.asarray(arr))
        return seqs, None
    return layerize(get_word_ids)

@describe.on_data(LSUVinit)
@describe.attributes(
        nM=Dimension("Vector dimensions"),
        nO=Dimension("Size of output"),
        W=Synapses(
            "A projection matrix, to change vector dimensionality",
            lambda obj: (obj.nO, obj.nM),
            lambda W, ops: ops.xavier_uniform_init(W)),
        d_W=Gradient("W"),
)
class StaticVectors(Model):
    '''Load a static embedding table, and learn a linear projection from it.

    Out-of-vocabulary items are modded into the table, receiving an arbitrary
    vector (but the same word will always receive the same vector).
    '''
    name = 'static-vectors'
    def __init__(self, lang, nO):
        Model.__init__(self)
        self.nO = nO
        # This doesn't seem the cleverest solution,
        # but it ensures multiple models load the
        # same copy of spaCy if they're deserialised.
        self.lang = lang
        vectors = self.get_vectors()
        self.nM = vectors.shape[1]
        if self.nM == 0:
            raise ValueError(
                "Cannot create vectors table with dimension 0.\n"
                "If you're using pre-trained vectors, are the vectors loaded?")
        self.nV = vectors.shape[0]

    def get_vectors(self):
        return get_vectors(self.ops, self.lang)

    def begin_update(self, ids, drop=0.):
        vector_table = self.get_vectors()
        vectors = vector_table[ids % vector_table.shape[0]]
        def finish_update(gradients, sgd=None):
            self.d_W += self.ops.batch_outer(gradients, vectors)
            if sgd is not None:
                sgd(self._mem.weights, self._mem.gradient, key=self.id)
            return None
        dotted = self.ops.batch_dot(vectors, self.W)
        return dotted, finish_update
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2012 Dominic May
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
contains some functions
required by both main.py and humans.py
"""
# this could be dangerous D:
from __future__ import (
    division,
    absolute_import,
    division,
    generators,
    nested_scopes,
    print_function,
    with_statement
)

import os
import json
import urllib
import base64
import hashlib
import logging
from operator import itemgetter

# google appengine imports
import webapp2
from google.appengine.api import mail
from google.appengine.api import users
from google.appengine.api import memcache
from google.appengine.api import urlfetch
from google.appengine.ext.webapp import template

# for debugging exceptions
import sys
import traceback

# application specific
from module_utils import get_hardware_data, get_module_data

# authentication data
client_auth_data = memcache.get('client_auth_data')
if not client_auth_data:
    with open('auth_data.json', 'r') as fh:
        auth_data = json.load(fh)
        client_auth_data = auth_data["client_auth_data"]

def _get_live_data(handler, fragment):
    """
    get_url_content's fragment['url'], b64decode's module['content']
    """
    module = get_url_content(handler, fragment['url'])
    assert 'content' in module

    return base64.b64decode(module['content'])

def get_live_hardware_data(handler, fragment):
    """
    Given a get_tree fragment,
    returns hardware data in a python dict
    """
    return get_hardware_data(_get_live_data(handler, fragment))

def get_live_module_data(handler, fragment):
    """
    Given a get_tree fragment,
    returns module data in a python dict
    """
    return get_module_data(_get_live_data(handler, fragment))

def get_tree(handler=None):
    """
    Returns the file hierarchy/tree
    """

    result = get_url_content(handler, 'https://api.github.com/repos/DCPUTeam/DCPUModules/git/trees/master')

    assert result['tree'], result
    return result['tree']

def get_modules(handler=None):
    """
    Returns the file hierarchy/tree, filtered by a .lua extension
    """
    tree = get_tree(handler)

    return [
        fragment
        for fragment in tree
        if fragment['path'].endswith('.lua')
    ]

def get_module_names(handler):
    """
    Returns list containing the path attributes of all modules
    """

    modules = get_modules(handler)
    modules = map(itemgetter('path'), modules)
    return map(rpart, modules)

def get_url_content(handler, url):
    """
    A wrapper around authed_fetch_json, caches results to help keep wait time short
    """

    url_hash = hashlib.md5(url).hexdigest()
    result = memcache.get(url_hash)

    if result is None:
        logging.info('Getting the result from the GitHub API')

        try:
            result = authed_fetch_json(url)
        except urlfetch.DownloadError as e:
            logging.info('Fetching "{}" failed with a {}'.format(url, e))
            handler.error(408)
            return
        else:
            memcache.set(url_hash, result)

    else:
        logging.info('Memcache get successful; %.40s' % result)
    # check if the api limit has been reached
    assert not result.get('message', '').startswith(
        'API Rate Limit Exceeded for'), 'API Limit reached'
    return result

def authed_fetch(url, headers=None):
    # add admin contact, auth_data
    headers = headers or {}
    headers.update({'X-Admin-Contact': 'admin@lysdev.com'})

    # build the url
    url += '&' if '?' in url else '?'
    url += urllib.urlencode(client_auth_data)

    r = urlfetch.fetch(url=url, headers=headers)

    remaining = r.headers.get('x-ratelimit-remaining')
    if remaining:
        logging.info('{} requests remaining for this hour.'.format(remaining))
    else:
        logging.info(
            'Could not determine number of requests remaining for this hour')
        logging.info(r.content)

    return r

def authed_fetch_json(*args, **kwargs):
    """
    parse json output from proxied authed_fetch
    """
    return json.load(authed_fetch(*args, **kwargs).raw)

class BaseRequestHandler(webapp2.RequestHandler):
    def handle_exception(self, exception, debug_mode):
        if development():
            return super(BaseRequestHandler, self).handle_exception(exception, debug_mode)

        lines = ''.join(traceback.format_exception(*sys.exc_info()))
        logging.error(lines)
        template_values = {
            'traceback': lines.replace('\n', '
')
        }
        html = self.dorender('error.html', template_values, write=False)
        mail.send_mail(
            sender='debugging@dcputoolchain-module-site.appspotmail.com',
            to="jack.thatch@gmail.com",
            subject='Caught Exception',
            body=lines,
            html=html)
        if users.is_current_user_admin():
            raise exception
        else:
            self.error(500)
            if isinstance(exception, AssertionError):
                self.dorender('unexpected_result.html', {})

    def dorender(self, tname='base.html', values=None, write=True):
        """
        automates some stuff so we dont have to type
        it in everytime we want to use a template
        """

        self.response.headers['content-type'] = 'text/html'
        path = os.path.join(os.path.dirname(__file__), 'templates/' + tname)

        data = template.render(path, values or {})

        if write:
            self.response.out.write(data)

def development():
    return os.environ['SERVER_SOFTWARE'].find('Development') == 0

def rpart(path):
    return path.rpartition('/')[-1]
<commit_msg>Corrected some oversights<commit_after>#!/usr/bin/env python
#
# Copyright 2012 Dominic May
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
contains some functions
required by both main.py and humans.py
"""
# this could be dangerous D:
from __future__ import (
    division,
    absolute_import,
    division,
    generators,
    nested_scopes,
    print_function,
    with_statement
)

import os
import json
import urllib
import base64
import hashlib
import logging
from operator import itemgetter

# google appengine imports
import webapp2
from google.appengine.api import mail
from google.appengine.api import users
from google.appengine.api import memcache
from google.appengine.api import urlfetch
from google.appengine.ext.webapp import template

# for debugging exceptions
import sys
import traceback

# application specific
from module_utils import get_hardware_data, get_module_data

# authentication data
client_auth_data = memcache.get('client_auth_data')
if not client_auth_data:
    with open('auth_data.json', 'r') as fh:
        auth_data = json.load(fh)
        client_auth_data = auth_data["client_auth_data"]

def _get_live_data(handler, fragment):
    """
    get_url_content's fragment['url'], b64decode's module['content']
    """
    module = get_url_content(handler, fragment['url'])
    assert 'content' in module

    return base64.b64decode(module['content'])

def get_live_hardware_data(handler, fragment):
    """
    Given a get_tree fragment,
    returns hardware data in a python dict
    """
    return get_hardware_data(_get_live_data(handler, fragment))

def get_live_module_data(handler, fragment):
    """
    Given a get_tree fragment,
    returns module data in a python dict
    """
    return get_module_data(_get_live_data(handler, fragment))

def get_tree(handler=None):
    """
    Returns the file hierarchy/tree
    """

    result = get_url_content(handler, 'https://api.github.com/repos/DCPUTeam/DCPUModules/git/trees/master')

    assert result['tree'], result
    return result['tree']

def get_modules(handler=None):
    """
    Returns the file hierarchy/tree, filtered by a .lua extension
    """
    tree = get_tree(handler)

    return [
        fragment
        for fragment in tree
        if fragment['path'].endswith('.lua')
    ]

def get_module_names(handler):
    """
    Returns list containing the path attributes of all modules
    """

    modules = get_modules(handler)
    modules = map(itemgetter('path'), modules)
    return map(rpart, modules)

def get_url_content(handler, url):
    """
    A wrapper around authed_fetch_json, caches results to help keep wait time short
    """

    url_hash = hashlib.md5(url).hexdigest()
    result = memcache.get(url_hash)

    if result is None:
        logging.info('Getting the result from the GitHub API')

        try:
            result = authed_fetch_json(url)
        except urlfetch.DownloadError as e:
            logging.info('Fetching "{}" failed with a {}'.format(url, e))
            handler.error(408)
            return
        else:
            memcache.set(url_hash, result)

    else:
        logging.info('Memcache get successful; %.40s' % result)
    # check if the api limit has been reached
    assert not result.get('message', '').startswith(
        'API Rate Limit Exceeded for'), 'API Limit reached'
    return result

def authed_fetch(url, headers=None):
    # add admin contact, auth_data
    headers = headers or {}
    headers.update({'X-Admin-Contact': 'admin@lysdev.com'})

    # build the url
    url += '&' if '?' in url else '?'
    url += urllib.urlencode(client_auth_data)

    r = urlfetch.fetch(url=url, headers=headers)

    remaining = r.headers.get('x-ratelimit-remaining')
    if remaining:
        logging.info('{} requests remaining for this hour.'.format(remaining))
    else:
        logging.info(
            'Could not determine number of requests remaining for this hour')
        logging.info(r.content)

    return r

def authed_fetch_json(*args, **kwargs):
    """
    parse json output from proxied authed_fetch
    """
    return json.loads(authed_fetch(*args, **kwargs).content)

class BaseRequestHandler(webapp2.RequestHandler):
    def handle_exception(self, exception, debug_mode):
        if development():
            return super(BaseRequestHandler, self).handle_exception(exception, debug_mode)

        lines = ''.join(traceback.format_exception(*sys.exc_info()))
        logging.error(lines)
        template_values = {
            'traceback': lines.replace('\n', '
')
        }
        html = self.dorender('error.html', template_values, write=False)
        mail.send_mail(
            sender='debugging@dcputoolchain-module-site.appspotmail.com',
            to="jack.thatch@gmail.com",
            subject='Caught Exception',
            body=lines,
            html=html)
        if users.is_current_user_admin():
            raise exception
        else:
            self.error(500)
            if isinstance(exception, AssertionError):
                self.dorender('unexpected_result.html', {})

    def dorender(self, tname='base.html', values=None, write=True):
        """
        automates some stuff so we dont have to type
        it in everytime we want to use a template
        """

        self.response.headers['Content-Type'] = 'text/html'
        path = os.path.join(os.path.dirname(__file__), 'templates/' + tname)

        data = template.render(path, values or {})

        if write:
            self.response.out.write(data)
        else:
            return data

def development():
    return os.environ['SERVER_SOFTWARE'].find('Development') == 0

def rpart(path):
    return path.rpartition('/')[-1]
<|endoftext|>