<commit_before>from app import db
from datetime import datetime

# Declare Models - Dataset, User, Discovery
# each class becomes a table
class Dataset(db.Model):
    # each attribute on a "Model" inherited class becomes a Column
    #id = db.Column(db.Integer, primary_key=True)

    db_source_uid = db.Column(db.String(50),primary_key=True)
    db_source = db.Column(db.String(20))
    expt_link = db.Column(db.Text)
    expt_id = db.Column(db.String(30))
    expt_title = db.Column(db.Text)
    expt_design_description = db.Column(db.Text)
    library_name = db.Column(db.Text)
    library_strategy = db.Column(db.Text)
    library_source = db.Column(db.Text)
    library_screening_strategy = db.Column(db.Text)
    library_construction_method = db.Column(db.Text)
    library_construction_protocol = db.Column(db.Text)
    sequencing_method = db.Column(db.Text)
    instrument_model = db.Column(db.Text)
    submission_id = db.Column(db.String(30))
    organization_name = db.Column(db.Text)
    organization_address = db.Column(db.Text)
    organization_contacts = db.Column(db.Text)
    study_id = db.Column(db.String(30))
    bioproject_id = db.Column(db.String(40))
    study_title = db.Column(db.Text)
    study_type = db.Column(db.Text)
    study_type_other = db.Column(db.Text)
    study_abstract = db.Column(db.Text)
    study_links = db.Column(db.PickleType)
    study_attributes = db.Column(db.PickleType)
    sample_id = db.Column(db.String(30))
    biosample_id = db.Column(db.String(40))
    sample_title = db.Column(db.Text)
    ncbi_taxon_id = db.Column(db.String(50))
    taxon_scientific_name = db.Column(db.Text)
    taxon_common_name = db.Column(db.Text)
    sample_description = db.Column(db.Text)
    num_runs_in_accession = db.Column(db.Integer)
    run_ids = db.Column(db.PickleType)
    run_ids_maxrun = db.Column(db.String(30))
    library_reads_sequenced = db.Column(db.PickleType)
    library_reads_sequenced_maxrun = = db.Column(db.BIGINT)
    total_num_bases = db.Column(db.PickleType)
    total_num_bases_maxrun = = db.Column(db.BIGINT)
    download_size = db.Column(db.PickleType)
    download_size_maxrun = = db.Column(db.BIGINT)
    avg_read_length = db.Column(db.PickleType)
    avg_read_length_maxrun = db.Column(db.Float)
    baseA_count = db.Column(db.PickleType)
    baseA_count_maxrun = = db.Column(db.BIGINT)
    baseC_count = db.Column(db.PickleType)
    baseC_count_maxrun = = db.Column(db.BIGINT)
    baseG_count = db.Column(db.PickleType)
    baseG_count_maxrun = = db.Column(db.BIGINT)
    baseT_count = db.Column(db.PickleType)
    baseT_count_maxrun = = db.Column(db.BIGINT)
    baseN_count = db.Column(db.PickleType)
    baseN_count_maxrun = = db.Column(db.BIGINT)
    gc_percent = db.Column(db.PickleType)
    gc_percent_maxrun = db.Column(db.Float)
    run_quality_counts = db.Column(db.PickleType)
    run_quality_counts_maxrun = db.Column(db.PickleType)
    biosample_uid = db.Column(db.String(30))
    pubmed_uids = db.Column(db.PickleType)
    nuccore_uids = db.Column(db.PickleType)
    biosample_link = db.Column(db.Text)
    metadata_publication_date = db.Column(db.DateTime)
    biosample_package = db.Column(db.Text)
    biosample_models = db.Column(db.PickleType)
    sample_attributes = db.Column(db.PickleType)
    pubmed_link = db.Column(db.PickleType)
    pub_publication_date = db.Column(db.PickleType)
    pub_authors = db.Column(db.PickleType)
    pub_title = db.Column(db.PickleType)
    pub_volume = db.Column(db.PickleType)
    pub_issue = db.Column(db.PickleType)
    pub_pages = db.Column(db.PickleType)
    pub_journal = db.Column(db.PickleType)
    pub_doi = db.Column(db.PickleType)
    nuccore_link = db.Column(db.PickleType)
    investigation_type = db.Column(db.Text)
    env_package = db.Column(db.Text)
    project_name = db.Column(db.Text)
    lat_lon = db.Column(db.Text)
    latitude = db.Column(db.Text)
    longitude = db.Column(db.Text)
    geo_loc_name = db.Column(db.Text)
    collection_date = db.Column(db.Text)
    collection_time = db.Column(db.Text)
    env_biome = db.Column(db.Text)
    env_feature = db.Column(db.Text)
    env_material = db.Column(db.Text)
    depth = db.Column(db.Text)
    elevation = db.Column(db.Text)
    altitude = db.Column(db.Text)
    target_gene = db.Column(db.Text)
    target_subfragment = db.Column(db.Text)
    ploidy = db.Column(db.Text)
    num_replicons = db.Column(db.Text)
    estimated_size = db.Column(db.Text)
    ref_biomaterial = db.Column(db.Text)
    propagation = db.Column(db.Text)
    assembly = db.Column(db.Text)
    finishing_strategy = db.Column(db.Text)
    isol_growth_condt = db.Column(db.Text)
    experimental_factor = db.Column(db.Text)
    specific_host = db.Column(db.Text)
    subspecific_genetic_lineage = db.Column(db.Text)
    tissue = db.Column(db.Text)
    sex = db.Column(db.Text)
    sample_type = db.Column(db.Text)
    age = db.Column(db.Text)
    dev_stage = db.Column(db.Text)
    biomaterial_provider = db.Column(db.Text)
    host_disease = db.Column(db.Text)

    #etc = db.Column(db.PickleType)

    # Each class must have an init function
    def __init__(self, db_source_uid=None,db_source=None,expt_link=None,expt_id=None,expt_title=None,expt_design_description=None,library_name=None,library_strategy=None,library_source=None,library_screening_strategy=None,library_construction_method=None,library_construction_protocol=None,sequencing_method=None,instrument_model=None,
    submission_id=None,organization_name=None,organization_address=None,organization_contacts=None,
    study_id=None,bioproject_id=None,study_title=None,study_type=None,study_type_other=None,study_abstract=None,study_links=None,study_attributes=None,
    sample_id=None,biosample_id=None,sample_title=None,ncbi_taxon_id=None,taxon_scientific_name=None,taxon_common_name=None,sample_description=None,
    num_runs_in_accession=None,run_ids=None,run_ids_maxrun=None,library_reads_sequenced=None,library_reads_sequenced_maxrun=None,total_num_bases=None,total_num_bases_maxrun=None,download_size=None,download_size_maxrun=None,avg_read_length=None,avg_read_length_maxrun=None,
    baseA_count=None,baseA_count_maxrun=None,baseC_count=None,baseC_count_maxrun=None,baseG_count=None,baseG_count_maxrun=None,baseT_count=None,baseT_count_maxrun=None,baseN_count=None,baseN_count_maxrun=None,gc_percent=None,gc_percent_maxrun=None,run_quality_counts=None,run_quality_counts_maxrun=None,
    biosample_uid=None,pubmed_uids=None,nuccore_uids=None,biosample_link=None,metadata_publication_date=None,biosample_package=None,biosample_models=None,sample_attributes=None,
    pubmed_link=None,pub_publication_date=None,pub_authors=None,pub_title=None,pub_volume=None,pub_issue=None,pub_pages=None,pub_journal=None,pub_doi=None,nuccore_link=None,
    investigation_type=None,env_package=None,project_name=None,lat_lon=None,latitude=None,longitude=None,geo_loc_name=None,collection_date=None,collection_time=None,env_biome=None,env_feature=None,env_material=None,depth=None,elevation=None,altitude=None,target_gene=None,target_subfragment=None,
    ploidy=None,num_replicons=None,estimated_size=None,ref_biomaterial=None,propagation=None,assembly=None,finishing_strategy=None,isol_growth_condt=None,experimental_factor=None,specific_host=None,subspecific_genetic_lineage=None,tissue=None,sex=None,sample_type=None,age=None,dev_stage=None,biomaterial_provider=None,host_disease=None):

        self.db_source_uid = db_source_uid
        self.db_source = db_source
        self.expt_link = expt_link
        self.expt_id = expt_id
        self.expt_title = expt_title
        self.expt_design_description = expt_design_description
        self.library_name = library_name
        self.library_strategy = library_strategy
        self.library_source = library_source
        self.library_screening_strategy = library_screening_strategy
        self.library_construction_method = library_construction_method
        self.library_construction_protocol = library_construction_protocol
        self.sequencing_method = sequencing_method
        self.instrument_model = instrument_model
        self.submission_id = submission_id
        self.organization_name = organization_name
        self.organization_address = organization_address
        self.organization_contacts = organization_contacts
        self.study_id = study_id
        self.bioproject_id = bioproject_id
        self.study_title = study_title
        self.study_type = study_type
        self.study_type_other = study_type_other
        self.study_abstract = study_abstract
        self.study_links = study_links
        self.study_attributes = study_attributes
        self.sample_id = sample_id
        self.biosample_id = biosample_id
        self.sample_title = sample_title
        self.ncbi_taxon_id = ncbi_taxon_id
        self.taxon_scientific_name = taxon_scientific_name
        self.taxon_common_name = taxon_common_name
        self.sample_description = sample_description
        self.num_runs_in_accession = num_runs_in_accession
        self.run_ids = run_ids
        self.run_ids_maxrun = run_ids_maxrun
        self.library_reads_sequenced = library_reads_sequenced
        self.library_reads_sequenced_maxrun = library_reads_sequenced_maxrun
        self.total_num_bases = total_num_bases
        self.total_num_bases_maxrun = total_num_bases_maxrun
        self.download_size = download_size
        self.download_size_maxrun = download_size_maxrun
        self.avg_read_length = avg_read_length
        self.avg_read_length_maxrun = avg_read_length_maxrun
        self.baseA_count = baseA_count
        self.baseA_count_maxrun = baseA_count_maxrun
        self.baseC_count = baseC_count
        self.baseC_count_maxrun = baseC_count_maxrun
        self.baseG_count = baseG_count
        self.baseG_count_maxrun = baseG_count_maxrun
        self.baseT_count = baseT_count
        self.baseT_count_maxrun = baseT_count_maxrun
        self.baseN_count = baseN_count
        self.baseN_count_maxrun = baseN_count_maxrun
        self.gc_percent = gc_percent
        self.gc_percent_maxrun = gc_percent_maxrun
        self.run_quality_counts = run_quality_counts
        self.run_quality_counts_maxrun = run_quality_counts_maxrun
        self.biosample_uid = biosample_uid
        self.pubmed_uids = pubmed_uids
        self.nuccore_uids = nuccore_uids
        self.biosample_link = biosample_link
        self.metadata_publication_date = metadata_publication_date
        self.biosample_package = biosample_package
        self.biosample_models = biosample_models
        self.sample_attributes = sample_attributes
        self.pubmed_link = pubmed_link
        self.pub_publication_date = pub_publication_date
        self.pub_authors = pub_authors
        self.pub_title = pub_title
        self.pub_volume = pub_volume
        self.pub_issue = pub_issue
        self.pub_pages = pub_pages
        self.pub_journal = pub_journal
        self.pub_doi = pub_doi
        self.nuccore_link = nuccore_link
        self.investigation_type = investigation_type
        self.env_package = env_package
        self.project_name = project_name
        self.lat_lon = lat_lon
        self.latitude = latitude
        self.longitude = longitude
        self.geo_loc_name = geo_loc_name
        self.collection_date = collection_date
        self.collection_time = collection_time
        self.env_biome = env_biome
        self.env_feature = env_feature
        self.env_material = env_material
        self.depth = depth
        self.elevation = elevation
        self.altitude = altitude
        self.target_gene = target_gene
        self.target_subfragment = target_subfragment
        self.ploidy = ploidy
        self.num_replicons = num_replicons
        self.estimated_size = estimated_size
        self.ref_biomaterial = ref_biomaterial
        self.propagation = propagation
        self.assembly = assembly
        self.finishing_strategy = finishing_strategy
        self.isol_growth_condt = isol_growth_condt
        self.experimental_factor = experimental_factor
        self.specific_host = specific_host
        self.subspecific_genetic_lineage = subspecific_genetic_lineage
        self.tissue = tissue
        self.sex = sex
        self.sample_type = sample_type
        self.age = age
        self.dev_stage = dev_stage
        self.biomaterial_provider = biomaterial_provider
        self.host_disease = host_disease

    # Friendly string representation
    def __repr__(self):
        return '<Dataset %r>' % self.expt_id

# For a many to many database relationship, use a mapping table (no class definition directly)
# Eg. each discovery will have many datasets, and each dataset may belong to many discoveries
# Each row in this table is one "dataset in discovery" membership
dataset_to_discovery = db.Table('dataset_to_discovery',
    db.Column('dataset_id', db.Integer, db.ForeignKey('dataset.db_source_id')),
    db.Column('discovery_id', db.Integer, db.ForeignKey('discovery.id'))
)

class Discovery(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    filter_params = db.Column(db.Text)
    timestamp = db.Column(db.DateTime)
    datasets = db.relationship('Dataset', secondary=dataset_to_discovery, backref=db.backref('discoveries', lazy='dynamic'))
    owner_id = db.Column(db.Integer, db.ForeignKey('user.id'))
    owner = db.relationship('User',backref=db.backref('myDiscoveries',lazy='dynamic'))

    def __init__(self, owner_id, filter_params, datasets, timestamp=None, ):
        self.owner_id = owner_id
        self.filter_params = filter_params
        self.datasets = datasets
        if timestamp is None:
            timestamp = datetime.utcnow()
        self.timestamp = timestamp

    def __repr__(self):
        return '<Discovery %r>' % self.filter_params

class User(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    firebase_id = db.Column(db.String(28), unique=True)
    admin = db.Column(db.Boolean)
    discoveries = db.relationship('Discovery', backref='user', lazy='dynamic')

    def __init__(self, firebase_id, admin=False):
        self.firebase_id = firebase_id
        self.admin = admin

    def __repr__(self):
        return '<User %r>' % self.firebase_id
<commit_msg>update Datasets model with final fields<commit_after>from app import db
from datetime import datetime

# Declare Models - Dataset, User, Discovery
# each class becomes a table
class Dataset(db.Model):
    # each attribute on a "Model" inherited class becomes a Column
    #id = db.Column(db.Integer, primary_key=True)

    db_source_uid = db.Column(db.String(50),primary_key=True)
    db_source = db.Column(db.String(20))
    expt_link = db.Column(db.Text)
    expt_id = db.Column(db.String(30))
    expt_title = db.Column(db.Text)
    expt_design_description = db.Column(db.Text)
    library_name = db.Column(db.Text)
    library_strategy = db.Column(db.Text)
    library_source = db.Column(db.Text)
    library_screening_strategy = db.Column(db.Text)
    library_construction_method = db.Column(db.Text)
    library_construction_protocol = db.Column(db.Text)
    sequencing_method = db.Column(db.Text)
    instrument_model = db.Column(db.Text)
    submission_id = db.Column(db.String(30))
    organization_name = db.Column(db.Text)
    organization_address = db.Column(db.Text)
    organization_contacts = db.Column(db.Text)
    study_id = db.Column(db.String(30))
    bioproject_id = db.Column(db.String(40))
    study_title = db.Column(db.Text)
    study_type = db.Column(db.Text)
    study_type_other = db.Column(db.Text)
    study_abstract = db.Column(db.Text)
    study_links = db.Column(db.Text)
    study_attributes = db.Column(db.Text)
    sample_id = db.Column(db.String(30))
    biosample_id = db.Column(db.String(40))
    sample_title = db.Column(db.Text)
    ncbi_taxon_id = db.Column(db.String(50))
    taxon_scientific_name = db.Column(db.Text)
    taxon_common_name = db.Column(db.Text)
    sample_description = db.Column(db.Text)
    num_runs_in_accession = db.Column(db.Integer)
    run_ids_maxrun = db.Column(db.String(30))
    library_reads_sequenced_maxrun = = db.Column(db.BIGINT)
    total_num_bases_maxrun = = db.Column(db.BIGINT)
    download_size_maxrun = = db.Column(db.BIGINT)
    avg_read_length_maxrun = db.Column(db.Float)
    baseA_count_maxrun = = db.Column(db.BIGINT)
    baseC_count_maxrun = = db.Column(db.BIGINT)
    baseG_count_maxrun = = db.Column(db.BIGINT)
    baseT_count_maxrun = = db.Column(db.BIGINT)
    baseN_count_maxrun = = db.Column(db.BIGINT)
    gc_percent_maxrun = db.Column(db.Float)
    run_quality_counts_maxrun = db.Column(db.Text)
    biosample_uid = db.Column(db.String(30))
    nuccore_uids = db.Column(db.Text)
    biosample_link = db.Column(db.Text)
    metadata_publication_date = db.Column(db.DateTime)
    biosample_package = db.Column(db.Text)
    biosample_models = db.Column(db.Text)
    sample_attributes = db.Column(db.Text)
    nuccore_link = db.Column(db.Text)
    investigation_type = db.Column(db.Text)
    env_package = db.Column(db.Text)
    project_name = db.Column(db.Text)
    lat_lon = db.Column(db.Text)
    latitude = db.Column(db.Text)
    longitude = db.Column(db.Text)
    geo_loc_name = db.Column(db.Text)
    collection_date = db.Column(db.Text)
    collection_time = db.Column(db.Text)
    env_biome = db.Column(db.Text)
    env_feature = db.Column(db.Text)
    env_material = db.Column(db.Text)
    depth = db.Column(db.Text)
    elevation = db.Column(db.Text)
    altitude = db.Column(db.Text)
    target_gene = db.Column(db.Text)
    target_subfragment = db.Column(db.Text)
    ploidy = db.Column(db.Text)
    num_replicons = db.Column(db.Text)
    estimated_size = db.Column(db.Text)
    ref_biomaterial = db.Column(db.Text)
    propagation = db.Column(db.Text)
    assembly = db.Column(db.Text)
    finishing_strategy = db.Column(db.Text)
    isol_growth_condt = db.Column(db.Text)
    experimental_factor = db.Column(db.Text)
    specific_host = db.Column(db.Text)
    subspecific_genetic_lineage = db.Column(db.Text)
    tissue = db.Column(db.Text)
    sex = db.Column(db.Text)
    sample_type = db.Column(db.Text)
    age = db.Column(db.Text)
    dev_stage = db.Column(db.Text)
    biomaterial_provider = db.Column(db.Text)
    host_disease = db.Column(db.Text)

    #etc = db.Column(db.PickleType)

    # Each class must have an init function
    def __init__(self, db_source_uid=None,db_source=None,expt_link=None,expt_id=None,expt_title=None,expt_design_description=None,library_name=None,library_strategy=None,library_source=None,library_screening_strategy=None,library_construction_method=None,library_construction_protocol=None,sequencing_method=None,instrument_model=None,
    submission_id=None,organization_name=None,organization_address=None,organization_contacts=None,
    study_id=None,bioproject_id=None,study_title=None,study_type=None,study_type_other=None,study_abstract=None,study_links=None,study_attributes=None,
    sample_id=None,biosample_id=None,sample_title=None,ncbi_taxon_id=None,taxon_scientific_name=None,taxon_common_name=None,sample_description=None,
    num_runs_in_accession=None,run_ids_maxrun=None,library_reads_sequenced_maxrun=None,total_num_bases_maxrun=None,download_size_maxrun=None,avg_read_length_maxrun=None,
    baseA_count_maxrun=None,baseC_count_maxrun=None,baseG_count_maxrun=None,baseT_count_maxrun=None,baseN_count_maxrun=None,gc_percent_maxrun=None,run_quality_counts_maxrun=None,
    biosample_uid=None,nuccore_uids=None,biosample_link=None,metadata_publication_date=None,biosample_package=None,biosample_models=None,sample_attributes=None,nuccore_link=None,
    investigation_type=None,env_package=None,project_name=None,lat_lon=None,latitude=None,longitude=None,geo_loc_name=None,collection_date=None,collection_time=None,env_biome=None,env_feature=None,env_material=None,depth=None,elevation=None,altitude=None,target_gene=None,target_subfragment=None,
    ploidy=None,num_replicons=None,estimated_size=None,ref_biomaterial=None,propagation=None,assembly=None,finishing_strategy=None,isol_growth_condt=None,experimental_factor=None,specific_host=None,subspecific_genetic_lineage=None,tissue=None,sex=None,sample_type=None,age=None,dev_stage=None,biomaterial_provider=None,host_disease=None):

        self.db_source_uid = db_source_uid
        self.db_source = db_source
        self.expt_link = expt_link
        self.expt_id = expt_id
        self.expt_title = expt_title
        self.expt_design_description = expt_design_description
        self.library_name = library_name
        self.library_strategy = library_strategy
        self.library_source = library_source
        self.library_screening_strategy = library_screening_strategy
        self.library_construction_method = library_construction_method
        self.library_construction_protocol = library_construction_protocol
        self.sequencing_method = sequencing_method
        self.instrument_model = instrument_model
        self.submission_id = submission_id
        self.organization_name = organization_name
        self.organization_address = organization_address
        self.organization_contacts = organization_contacts
        self.study_id = study_id
        self.bioproject_id = bioproject_id
        self.study_title = study_title
        self.study_type = study_type
        self.study_type_other = study_type_other
        self.study_abstract = study_abstract
        self.study_links = study_links
        self.study_attributes = study_attributes
        self.sample_id = sample_id
        self.biosample_id = biosample_id
        self.sample_title = sample_title
        self.ncbi_taxon_id = ncbi_taxon_id
        self.taxon_scientific_name = taxon_scientific_name
        self.taxon_common_name = taxon_common_name
        self.sample_description = sample_description
        self.num_runs_in_accession = num_runs_in_accession
        self.run_ids_maxrun = run_ids_maxrun
        self.library_reads_sequenced_maxrun = library_reads_sequenced_maxrun
        self.total_num_bases_maxrun = total_num_bases_maxrun
        self.download_size_maxrun = download_size_maxrun
        self.avg_read_length_maxrun = avg_read_length_maxrun
        self.baseA_count_maxrun = baseA_count_maxrun
        self.baseC_count_maxrun = baseC_count_maxrun
        self.baseG_count_maxrun = baseG_count_maxrun
        self.baseT_count_maxrun = baseT_count_maxrun
        self.baseN_count_maxrun = baseN_count_maxrun
        self.gc_percent_maxrun = gc_percent_maxrun
        self.run_quality_counts_maxrun = run_quality_counts_maxrun
        self.biosample_uid = biosample_uid
        self.nuccore_uids = nuccore_uids
        self.biosample_link = biosample_link
        self.metadata_publication_date = metadata_publication_date
        self.biosample_package = biosample_package
        self.biosample_models = biosample_models
        self.sample_attributes = sample_attributes
        self.nuccore_link = nuccore_link
        self.investigation_type = investigation_type
        self.env_package = env_package
        self.project_name = project_name
        self.lat_lon = lat_lon
        self.latitude = latitude
        self.longitude = longitude
        self.geo_loc_name = geo_loc_name
        self.collection_date = collection_date
        self.collection_time = collection_time
        self.env_biome = env_biome
        self.env_feature = env_feature
        self.env_material = env_material
        self.depth = depth
        self.elevation = elevation
        self.altitude = altitude
        self.target_gene = target_gene
        self.target_subfragment = target_subfragment
        self.ploidy = ploidy
        self.num_replicons = num_replicons
        self.estimated_size = estimated_size
        self.ref_biomaterial = ref_biomaterial
        self.propagation = propagation
        self.assembly = assembly
        self.finishing_strategy = finishing_strategy
        self.isol_growth_condt = isol_growth_condt
        self.experimental_factor = experimental_factor
        self.specific_host = specific_host
        self.subspecific_genetic_lineage = subspecific_genetic_lineage
        self.tissue = tissue
        self.sex = sex
        self.sample_type = sample_type
        self.age = age
        self.dev_stage = dev_stage
        self.biomaterial_provider = biomaterial_provider
        self.host_disease = host_disease

    # Friendly string representation
    def __repr__(self):
        return '<Dataset %r>' % self.expt_id

# For a many to many database relationship, use a mapping table (no class definition directly)
# Eg. each discovery will have many datasets, and each dataset may belong to many discoveries
# Each row in this table is one "dataset in discovery" membership
dataset_to_discovery = db.Table('dataset_to_discovery',
    db.Column('dataset_id', db.Integer, db.ForeignKey('dataset.db_source_id')),
    db.Column('discovery_id', db.Integer, db.ForeignKey('discovery.id'))
)

class Discovery(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    filter_params = db.Column(db.Text)
    timestamp = db.Column(db.DateTime)
    datasets = db.relationship('Dataset', secondary=dataset_to_discovery, backref=db.backref('discoveries', lazy='dynamic'))
    owner_id = db.Column(db.Integer, db.ForeignKey('user.id'))
    owner = db.relationship('User',backref=db.backref('myDiscoveries',lazy='dynamic'))

    def __init__(self, owner_id, filter_params, datasets, timestamp=None, ):
        self.owner_id = owner_id
        self.filter_params = filter_params
        self.datasets = datasets
        if timestamp is None:
            timestamp = datetime.utcnow()
        self.timestamp = timestamp

    def __repr__(self):
        return '<Discovery %r>' % self.filter_params

class User(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    firebase_id = db.Column(db.String(28), unique=True)
    admin = db.Column(db.Boolean)
    discoveries = db.relationship('Discovery', backref='user', lazy='dynamic')

    def __init__(self, firebase_id, admin=False):
        self.firebase_id = firebase_id
        self.admin = admin

    def __repr__(self):
        return '<User %r>' % self.firebase_id
<|endoftext|>


<commit_before>import os, datetime, codecs, random, string, json, re, time, sqlite3
from flask import Flask, render_template, request, url_for, abort, redirect, send_from_directory, g
from werkzeug import secure_filename

fp = open("claim_secret") # this needs to exist. Put a long random string in it.
claim_secret = fp.read()
fp.close()

####################### config
UPLOAD_FOLDER = os.path.abspath(os.path.join(os.path.split(__file__)[0], "uploads"))
ALLOWED_EXTENSIONS = set(['click'])
KNOWN_DEVICES = []

####################### app config
app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER

DATABASE = os.path.join(os.path.split(__file__)[0], "requests.db")

def get_db():
    db = getattr(g, '_database', None)
    if db is None:
        db = g._database = sqlite3.connect(DATABASE)
    return db, db.cursor()

with app.app_context():
    db, crs = get_db()
    crs.execute(("create table if not exists requests ("
        "id integer primary key, ip varchar, click_filename varchar time t TIMESTAMP DEFAULT CURRENT_TIMESTAMP"
        ")"))
    crs.execute("create table if not exists devices (id integer primary key, printable_name varchar unique)")
    crs.execute("create table if not exists request2device (deviceid integer, requestid integer)")

####################### utility functions
def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS

def randomstring(N):
    return ''.join(
        random.SystemRandom().choice(
            string.ascii_uppercase + string.digits
        ) for _ in range(N)
    )

def slugify(s):
    return re.sub(r"[^A-Za-z0-9]", "_", s)

def prune_known_devices():
    global KNOWN_DEVICES
    now = time.time()
    binned = []
    keep = []
    for k in KNOWN_DEVICES:
        if now - k["seen"] > 300:
            binned.append(k)
        else:
            keep.append(k)
    KNOWN_DEVICES = keep
    if binned:
        print "Removed devices for age:", ",".join(['"%s"' % x["printable"] for x in binned])

####################### routes
@app.route("/")
def frontpage():
    prune_known_devices()
    return render_template("upload.html", devices=KNOWN_DEVICES)

@app.route("/upload", methods=["POST"])
def upload():
    file = request.files["click"]
    if file and allowed_file(file.filename):
        filename = secure_filename(file.filename)
        metadata = {
            "email": request.form['email'],
            "filename": filename,
            "devices": []
        }
        for device in KNOWN_DEVICES:
            if request.form.get("device_%s" % device["code"]) == "on":
                metadata["devices"].append({
                    "printable": device["printable"],
                    "status": "pending"
                })
        if not metadata["devices"]:
            return "You have to specify at least one device."
        ndir = "%s-%s" % (datetime.datetime.now().strftime("%Y%m%d%H%M%S"), randomstring(10))
        ndirpath = os.path.join(app.config['UPLOAD_FOLDER'], ndir)
        os.mkdir(ndirpath) # should not fail!
        ofile = os.path.join(ndirpath, filename)
        ometa = os.path.join(ndirpath, "metadata.json")
        file.save(ofile)
        fp = codecs.open(ometa, mode="w", encoding="utf8")
        json.dump(metadata, fp)
        fp.close()
        db, crs = get_db()
        crs.execute("insert into requests (ip, click_filename) values (?,?)", (request.remote_addr, file.filename))
        requestid = crs.lastrowid
        for d in metadata["devices"]:
            crs.execute("select id from devices where printable_name = ?", (d["printable"],))
            res = crs.fetchone()
            if res:
                deviceid = res[0]
            else:
                crs.execute("insert into devices (printable_name) values (?)", (d["printable"],))
                deviceid = crs.lastrowid
            crs.execute("insert into request2device (requestid, deviceid) values (?,?)", (requestid, deviceid))
        db.commit()
        return redirect(url_for('status', uid=ndir))
    else:
        return "failure.", 400

@app.route("/status/<uid>")
def status(uid):
    safe_uid = secure_filename(uid)
    folder = os.path.join(app.config["UPLOAD_FOLDER"], safe_uid)
    ometa = os.path.join(folder, "metadata.json")
    if not os.path.exists(ometa):
        return "No such pending test", 404
    fp = codecs.open(ometa, encoding="utf8")
    metadata = fp.read()
    fp.close()
    metadata = json.loads(metadata)
    return render_template("status.html", metadata=metadata)

@app.route("/claim")
def claim():
    device = request.args.get('device')
    if not device:
        return json.dumps({"error": "No device specified"}), 400, {'Content-Type': 'application/json'}
    if request.args.get("claim_secret") != claim_secret:
        return json.dumps({"error": "Bad claim secret"}), 400, {'Content-Type': 'application/json'}

    prune_known_devices()

    if device not in [x["printable"] for x in KNOWN_DEVICES]:
        KNOWN_DEVICES.append({"printable": device, "code": slugify(device)})

    for d in KNOWN_DEVICES:
        if d["printable"] == device:
            d["seen"] = time.time()

    device_code = [x["code"] for x in KNOWN_DEVICES if x["printable"] == device][0]
    # find the next unclaimed item which wants this device
    # this is a bit racy, but shouldn't be a problem in practice
    for fol in sorted(os.listdir(app.config["UPLOAD_FOLDER"])):
        ometa = os.path.join(app.config["UPLOAD_FOLDER"], fol, "metadata.json")
        if os.path.exists(ometa):
            fp = codecs.open(ometa, encoding="utf8")
            metadata = json.load(fp)
            fp.close()
            device_status = metadata.get("devices", [])
            for ds in device_status:
                if ds["printable"] == device:
                    if ds["status"] == "pending":
                        ds["status"] = "claimed"
                        metadata["devices"] = device_status
                        fp = codecs.open(ometa, mode="w", encoding="utf8")
                        json.dump(metadata, fp)
                        fp.close()
                        return json.dumps({
                            "job": fol,
                            "click": url_for("click", uid=fol),
                            "finished": url_for("finished", uid=fol, device_code=device_code),
                            "metadata": metadata
                        }), 200, {'Content-Type': 'application/json'}
    return json.dumps({"job": None}), 200, {'Content-Type': 'application/json'}

@app.route("/click/<uid>")
def click(uid):
    safe_uid = secure_filename(uid)
    folder = os.path.join(app.config["UPLOAD_FOLDER"], safe_uid)
    ometa = os.path.join(folder, "metadata.json")
    if not os.path.exists(ometa):
        return "No such pending test", 404
    fp = codecs.open(ometa, encoding="utf8")
    metadata = fp.read()
    fp.close()
    metadata = json.loads(metadata)
    if not os.path.exists(os.path.join(folder, metadata["filename"])):
        return "No such click", 404
    return send_from_directory(folder, metadata["filename"], as_attachment=True)

@app.route("/finished/<uid>/<device_code>")
def finished(uid, device_code):
    device_printable = [x["printable"] for x in KNOWN_DEVICES if x["code"] == device_code]
    if not device_printable:
        return json.dumps({"error": "Bad device code"}), 400, {'Content-Type': 'application/json'}
    device = device_printable[0]
    safe_uid = secure_filename(uid)
    folder = os.path.join(app.config["UPLOAD_FOLDER"], safe_uid)
    ometa = os.path.join(folder, "metadata.json")
    if not os.path.exists(ometa):
        return json.dumps({"error": "No such pending test"}), 400, {'Content-Type': 'application/json'}
    fp = codecs.open(ometa, encoding="utf8")
    metadata = json.load(fp)
    fp.close()
    device_status = metadata.get("devices", [])
    for ds in device_status:
        if ds["printable"] == device:
            if ds["status"] == "claimed":
                ds["status"] = "finished"
                metadata["devices"] = device_status
                fp = codecs.open(ometa, mode="w", encoding="utf8")
                json.dump(metadata, fp)
                fp.close()
                return json.dumps({"status": "finished"}), 200, {'Content-Type': 'application/json'}
            else:
                return json.dumps({"error": "Job not in state 'claimed' (in state '%s')" % ds["status"]}), 400, {'Content-Type': 'application/json'}
    return json.dumps({"error": "No such job"}), 400, {'Content-Type': 'application/json'}

if __name__ == "__main__":
    app.run(port=12346, debug=True)
<commit_msg>Add email to requests table<commit_after>import os, datetime, codecs, random, string, json, re, time, sqlite3
from flask import Flask, render_template, request, url_for, abort, redirect, send_from_directory, g
from werkzeug import secure_filename

fp = open("claim_secret") # this needs to exist. Put a long random string in it.
claim_secret = fp.read()
fp.close()

####################### config
UPLOAD_FOLDER = os.path.abspath(os.path.join(os.path.split(__file__)[0], "uploads"))
ALLOWED_EXTENSIONS = set(['click'])
KNOWN_DEVICES = []

####################### app config
app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER

DATABASE = os.path.join(os.path.split(__file__)[0], "requests.db")

def get_db():
    db = getattr(g, '_database', None)
    if db is None:
        db = g._database = sqlite3.connect(DATABASE)
    return db, db.cursor()

with app.app_context():
    db, crs = get_db()
    crs.execute(("create table if not exists requests ("
        "id integer primary key, ip varchar, click_filename varchar time t TIMESTAMP DEFAULT CURRENT_TIMESTAMP"
        ")"))
    crs.execute("create table if not exists devices (id integer primary key, printable_name varchar unique)")
    crs.execute("create table if not exists request2device (deviceid integer, requestid integer)")
    try:
        crs.execute("alter table requests add column email varchar")
    except sqlite3.OperationalError as e:
        if "duplicate column name: email" in e.message:
            pass
        else:
            raise e

####################### utility functions
def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS

def randomstring(N):
    return ''.join(
        random.SystemRandom().choice(
            string.ascii_uppercase + string.digits
        ) for _ in range(N)
    )

def slugify(s):
    return re.sub(r"[^A-Za-z0-9]", "_", s)

def prune_known_devices():
    global KNOWN_DEVICES
    now = time.time()
    binned = []
    keep = []
    for k in KNOWN_DEVICES:
        if now - k["seen"] > 300:
            binned.append(k)
        else:
            keep.append(k)
    KNOWN_DEVICES = keep
    if binned:
        print "Removed devices for age:", ",".join(['"%s"' % x["printable"] for x in binned])

####################### routes
@app.route("/")
def frontpage():
    prune_known_devices()
    return render_template("upload.html", devices=KNOWN_DEVICES)

@app.route("/upload", methods=["POST"])
def upload():
    file = request.files["click"]
    if file and allowed_file(file.filename):
        filename = secure_filename(file.filename)
        metadata = {
            "email": request.form['email'],
            "filename": filename,
            "devices": []
        }
        for device in KNOWN_DEVICES:
            if request.form.get("device_%s" % device["code"]) == "on":
                metadata["devices"].append({
                    "printable": device["printable"],
                    "status": "pending"
                })
        if not metadata["devices"]:
            return "You have to specify at least one device."
        ndir = "%s-%s" % (datetime.datetime.now().strftime("%Y%m%d%H%M%S"), randomstring(10))
        ndirpath = os.path.join(app.config['UPLOAD_FOLDER'], ndir)
        os.mkdir(ndirpath) # should not fail!
        ofile = os.path.join(ndirpath, filename)
        ometa = os.path.join(ndirpath, "metadata.json")
        file.save(ofile)
        fp = codecs.open(ometa, mode="w", encoding="utf8")
        json.dump(metadata, fp)
        fp.close()
        db, crs = get_db()
        crs.execute("insert into requests (ip, click_filename, email) values (?,?,?)",
            (request.remote_addr, file.filename, metadata["email"]))
        requestid = crs.lastrowid
        for d in metadata["devices"]:
            crs.execute("select id from devices where printable_name = ?", (d["printable"],))
            res = crs.fetchone()
            if res:
                deviceid = res[0]
            else:
                crs.execute("insert into devices (printable_name) values (?)", (d["printable"],))
                deviceid = crs.lastrowid
            crs.execute("insert into request2device (requestid, deviceid) values (?,?)", (requestid, deviceid))
        db.commit()
        return redirect(url_for('status', uid=ndir))
    else:
        return "failure.", 400

@app.route("/status/<uid>")
def status(uid):
    safe_uid = secure_filename(uid)
    folder = os.path.join(app.config["UPLOAD_FOLDER"], safe_uid)
    ometa = os.path.join(folder, "metadata.json")
    if not os.path.exists(ometa):
        return "No such pending test", 404
    fp = codecs.open(ometa, encoding="utf8")
    metadata = fp.read()
    fp.close()
    metadata = json.loads(metadata)
    return render_template("status.html", metadata=metadata)

@app.route("/claim")
def claim():
    device = request.args.get('device')
    if not device:
        return json.dumps({"error": "No device specified"}), 400, {'Content-Type': 'application/json'}
    if request.args.get("claim_secret") != claim_secret:
        return json.dumps({"error": "Bad claim secret"}), 400, {'Content-Type': 'application/json'}

    prune_known_devices()

    if device not in [x["printable"] for x in KNOWN_DEVICES]:
        KNOWN_DEVICES.append({"printable": device, "code": slugify(device)})

    for d in KNOWN_DEVICES:
        if d["printable"] == device:
            d["seen"] = time.time()

    device_code = [x["code"] for x in KNOWN_DEVICES if x["printable"] == device][0]
    # find the next unclaimed item which wants this device
    # this is a bit racy, but shouldn't be a problem in practice
    for fol in sorted(os.listdir(app.config["UPLOAD_FOLDER"])):
        ometa = os.path.join(app.config["UPLOAD_FOLDER"], fol, "metadata.json")
        if os.path.exists(ometa):
            fp = codecs.open(ometa, encoding="utf8")
            metadata = json.load(fp)
            fp.close()
            device_status = metadata.get("devices", [])
            for ds in device_status:
                if ds["printable"] == device:
                    if ds["status"] == "pending":
                        ds["status"] = "claimed"
                        metadata["devices"] = device_status
                        fp = codecs.open(ometa, mode="w", encoding="utf8")
                        json.dump(metadata, fp)
                        fp.close()
                        return json.dumps({
                            "job": fol,
                            "click": url_for("click", uid=fol),
                            "finished": url_for("finished", uid=fol, device_code=device_code),
                            "metadata": metadata
                        }), 200, {'Content-Type': 'application/json'}
    return json.dumps({"job": None}), 200, {'Content-Type': 'application/json'}

@app.route("/click/<uid>")
def click(uid):
    safe_uid = secure_filename(uid)
    folder = os.path.join(app.config["UPLOAD_FOLDER"], safe_uid)
    ometa = os.path.join(folder, "metadata.json")
    if not os.path.exists(ometa):
        return "No such pending test", 404
    fp = codecs.open(ometa, encoding="utf8")
    metadata = fp.read()
    fp.close()
    metadata = json.loads(metadata)
    if not os.path.exists(os.path.join(folder, metadata["filename"])):
        return "No such click", 404
    return send_from_directory(folder, metadata["filename"], as_attachment=True)

@app.route("/finished/<uid>/<device_code>")
def finished(uid, device_code):
    device_printable = [x["printable"] for x in KNOWN_DEVICES if x["code"] == device_code]
    if not device_printable:
        return json.dumps({"error": "Bad device code"}), 400, {'Content-Type': 'application/json'}
    device = device_printable[0]
    safe_uid = secure_filename(uid)
    folder = os.path.join(app.config["UPLOAD_FOLDER"], safe_uid)
    ometa = os.path.join(folder, "metadata.json")
    if not os.path.exists(ometa):
        return json.dumps({"error": "No such pending test"}), 400, {'Content-Type': 'application/json'}
    fp = codecs.open(ometa, encoding="utf8")
    metadata = json.load(fp)
    fp.close()
    device_status = metadata.get("devices", [])
    for ds in device_status:
        if ds["printable"] == device:
            if ds["status"] == "claimed":
                ds["status"] = "finished"
                metadata["devices"] = device_status
                fp = codecs.open(ometa, mode="w", encoding="utf8")
                json.dump(metadata, fp)
                fp.close()
                return json.dumps({"status": "finished"}), 200, {'Content-Type': 'application/json'}
            else:
                return json.dumps({"error": "Job not in state 'claimed' (in state '%s')" % ds["status"]}), 400, {'Content-Type': 'application/json'}
    return json.dumps({"error": "No such job"}), 400, {'Content-Type': 'application/json'}

if __name__ == "__main__":
    app.run(port=12346, debug=True)
<|endoftext|>


<commit_before>#!/usr/bin/python2.3
# -*- mode: python -*-
#
# Copyright (c) 2004 Specifix, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

import base64
import cgi
import os
import posixpath
import select
import sys
import tempfile
import xmlrpclib
import urllib
import zlib
from BaseHTTPServer import HTTPServer
from SimpleHTTPServer import SimpleHTTPRequestHandler

thisFile = sys.modules[__name__].__file__
thisPath = os.path.dirname(thisFile)
if thisPath:
    mainPath = thisPath + "/.."
else:
    mainPath = ".."
mainPath = os.path.realpath(mainPath)

sys.path.append(mainPath)

from repository.netrepos import netserver
from repository.netrepos.netserver import NetworkRepositoryServer
from conarycfg import ConfigFile
from conarycfg import STRINGDICT
from lib import options
from lib import util
from http import HttpHandler
from htmlengine import HtmlEngine

DEFAULT_FILE_PATH="/tmp/conary-server"

class HttpRequests(SimpleHTTPRequestHandler):

    outFiles = {}
    inFiles = {}

    def translate_path(self, path):
        """Translate a /-separated PATH to the local filename syntax.

        Components that mean special things to the local file system
        (e.g. drive or directory names) are ignored.  (XXX They should
        probably be diagnosed.)

        """
        path = posixpath.normpath(urllib.unquote(path))
	path = path.split("?", 1)[1]
        words = path.split('/')
        words = filter(None, words)
        path = FILE_PATH
        for word in words:
            drive, word = os.path.splitdrive(word)
            head, word = os.path.split(word)
            if word in (os.curdir, os.pardir): continue
            path = os.path.join(path, word)

	path += "-out"

	self.cleanup = path
        return path

    def do_GET(self):
        if self.path.endswith('/'):
            self.path = self.path[:-1]
        base = os.path.basename(self.path)
        if "?" in base:
            base, queryString = base.split("?")
        else:
            queryString = ""

        if base != 'changeset':
            if httpHandler.requiresAuth(base):
                authToken = self.checkAuth()
                if not authToken:
                    return
            else:
                authToken = (None, None)

            self.send_response(200)
            self.send_header("Content-type", "text/html")
            self.end_headers()
            fields = cgi.parse_qs(queryString)
            try:
                httpHandler.handleCmd(self.wfile.write, base, authToken, fields)
            except:
                self.traceback()
        else:
            urlPath = posixpath.normpath(urllib.unquote(self.path))
            localName = FILE_PATH + "/" + urlPath.split('?', 1)[1] + "-out"

            if localName.endswith(".cf-out"):
                try:
                    f = open(localName, "r")
                except IOError:
                    self.send_error(404, "File not found")
                    return None

                os.unlink(localName)

                items = []
                totalSize = 0
                for l in f.readlines():
                    (path, size) = l.split()
                    size = int(size)
                    totalSize += size
                    items.append((path, size))
                del f
            else:
                size = os.stat(localName).st_size;
                items = [ (localName, size) ]
                totalSize = size

            self.send_response(200)
            self.send_header("Content-type", "application/octet-stream")
            self.send_header("Content-Length", str(totalSize))
            self.end_headers()

            f = open(items[0][0], "r")
            util.copyfileobj(f, self.wfile)

            if not localName.endswith(".cf-out"):
                os.unlink(items[0][0])

    def do_POST(self):
        if self.headers.get('Content-Type', '') == 'text/xml':
            authToken = self.getAuth()
            if authToken is None:
                return

            return self.handleXml(authToken)

        cmd = os.path.basename(self.path)
        if httpHandler.requiresAuth(cmd):
            authToken = self.checkAuth()
            if not authToken:
                return
        else:
            authToken = (None, None)

        try:
            self.send_response(200)
            self.send_header("Content-type", "text/html")
            self.end_headers()
            c = cgi.FieldStorage(fp = self.rfile, headers = self.headers,
                                 environ = { 'REQUEST_METHOD' : 'POST' })
            httpHandler.handleCmd(self.wfile.write, cmd, authToken, c)
        except:
            self.traceback()

    def getAuth(self):
        info = self.headers.get('Authorization', None)
        if info is None:
            return (None, None)
        info = info.split()

        try:
            authString = base64.decodestring(info[1])
        except:
            self.send_response(400)
            return None

        if authString.count(":") != 1:
            self.send_response(400)
            return None

        authToken = authString.split(":")

        return authToken

    def checkAuth(self):
 	if not self.headers.has_key('Authorization'):
            self.requestAuth()
            return None
	else:
            authToken = self.getAuth()
            if authToken is None:
                return

            # verify that the user/password actually exists in the database
            if not netRepos.auth.checkUserPass(authToken):
               self.requestAuth()
               return None

	return authToken

    def requestAuth(self):
        self.send_response(401)
        self.send_header("WWW-Authenticate", 'Basic realm="Conary Repository"')
        self.end_headers()
        return None

    def traceback(self):
        htmlengine = HtmlEngine()
        htmlengine.setWriter(self.wfile)
        htmlengine.stackTrace(self.wfile)

    def handleXml(self, authToken):
	contentLength = int(self.headers['Content-Length'])
	(params, method) = xmlrpclib.loads(self.rfile.read(contentLength))

	try:
	    result = netRepos.callWrapper(method, authToken, params)
	except netserver.InsufficientPermission:
	    self.send_response(403)
	    return

	resp = xmlrpclib.dumps((result,), methodresponse=1)

	self.send_response(200)
        encoding = self.headers.get('Accept-encoding', '')
        if len(resp) > 200 and 'zlib' in encoding:
            resp = zlib.compress(resp, 5)
            self.send_header('Content-encoding', 'zlib')
	self.send_header("Content-type", "text/xml")
	self.send_header("Content-length", str(len(resp)))
	self.end_headers()
	self.wfile.write(resp)

	return resp

    def do_PUT(self):
	path = self.path.split("?")[-1]
	path = FILE_PATH + '/' + path + "-in"

	size = os.stat(path).st_size
	if size != 0:
	    self.send_response(410, "Gone")
	    return

	out = open(path, "w")

	contentLength = int(self.headers['Content-Length'])
	while contentLength:
	    s = self.rfile.read(contentLength)
	    contentLength -= len(s)
	    out.write(s)

	self.send_response(200, 'OK')

class ResetableNetworkRepositoryServer(NetworkRepositoryServer):

    def reset(self, authToken, clientVersion):
        import shutil
        from repository.netrepos import fsrepos
	del self.repos
	shutil.rmtree(self.repPath + '/contents')
        os.unlink(self.repPath + '/sqldb')
	self.repos = fsrepos.FilesystemRepository(self.name, self.repPath,
						  self.map)
        return 0

class ServerConfig(ConfigFile):

    defaults = {
	'port'			:   '8000',
	'repositoryMap'         : [ STRINGDICT, {} ],
	'tmpFilePath'           : DEFAULT_FILE_PATH,
    }

    def __init__(self, path="serverrc"):
	ConfigFile.__init__(self)
	self.read(path)

def usage():
    print "usage: %s repospath authdb reposname [config file]" %sys.argv[0]
    sys.exit(1)

if __name__ == '__main__':
    argDef = {}
    cfgMap = {
	'port'	: 'port',
	'map'	: 'repositoryMap',
	'tmp-file-path' : 'tmpFilePath',
    }

    cfg = ServerConfig()

    argSet, otherArgs = options.processArgs(argDef, cfgMap, cfg, usage)

    FILE_PATH = cfg.tmpFilePath

    if not os.path.isdir(FILE_PATH):
	print FILE_PATH + " needs to be a directory"
	sys.exit(1)

    if len(otherArgs) == 5:
        cfg.read(otherArgs.pop())

    if len(otherArgs) != 4:
	usage()

    profile = 0
    if profile:
        import hotshot
        prof = hotshot.Profile('server.prof')
        prof.start()

    baseUrl="http://%s:%s/" % (os.uname()[1], cfg.port)

    netRepos = ResetableNetworkRepositoryServer(otherArgs[1], FILE_PATH,
			baseUrl, otherArgs[2], otherArgs[3], cfg.repositoryMap)
    httpHandler = HttpHandler(netRepos)

    port = int(cfg.port)
    httpServer = HTTPServer(("", port), HttpRequests)

    fds = {}
    fds[httpServer.fileno()] = httpServer

    p = select.poll()
    for fd in fds.iterkeys():
        p.register(fd, select.POLLIN)

    while True:
        try:
            events = p.poll()
            for (fd, event) in events:
                fds[fd].handle_request()
        except select.error:
            pass
        except:
            if profile:
                prof.stop()
                print "exception happened, exiting"
                sys.exit(1)
            else:
                raise
<commit_msg>create a FieldStorage object for GET requests<commit_after>#!/usr/bin/python2.3
# -*- mode: python -*-
#
# Copyright (c) 2004 Specifix, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

import base64
import cgi
import os
import posixpath
import select
import sys
import tempfile
import xmlrpclib
import urllib
import zlib
from BaseHTTPServer import HTTPServer
from SimpleHTTPServer import SimpleHTTPRequestHandler

thisFile = sys.modules[__name__].__file__
thisPath = os.path.dirname(thisFile)
if thisPath:
    mainPath = thisPath + "/.."
else:
    mainPath = ".."
mainPath = os.path.realpath(mainPath)

sys.path.append(mainPath)

from repository.netrepos import netserver
from repository.netrepos.netserver import NetworkRepositoryServer
from conarycfg import ConfigFile
from conarycfg import STRINGDICT
from lib import options
from lib import util
from http import HttpHandler
from htmlengine import HtmlEngine

DEFAULT_FILE_PATH="/tmp/conary-server"

class HttpRequests(SimpleHTTPRequestHandler):

    outFiles = {}
    inFiles = {}

    def translate_path(self, path):
        """Translate a /-separated PATH to the local filename syntax.

        Components that mean special things to the local file system
        (e.g. drive or directory names) are ignored.  (XXX They should
        probably be diagnosed.)

        """
        path = posixpath.normpath(urllib.unquote(path))
	path = path.split("?", 1)[1]
        words = path.split('/')
        words = filter(None, words)
        path = FILE_PATH
        for word in words:
            drive, word = os.path.splitdrive(word)
            head, word = os.path.split(word)
            if word in (os.curdir, os.pardir): continue
            path = os.path.join(path, word)

	path += "-out"

	self.cleanup = path
        return path

    def do_GET(self):
        if self.path.endswith('/'):
            self.path = self.path[:-1]
        base = os.path.basename(self.path)
        if "?" in base:
            base, queryString = base.split("?")
        else:
            queryString = ""

        if base != 'changeset':
            if httpHandler.requiresAuth(base):
                authToken = self.checkAuth()
                if not authToken:
                    return
            else:
                authToken = (None, None)

            self.send_response(200)
            self.send_header("Content-type", "text/html")
            self.end_headers()

            fields = cgi.FieldStorage(environ = { 'QUERY_STRING' : queryString })
            try:
                httpHandler.handleCmd(self.wfile.write, base, authToken, fields)
            except:
                self.traceback()
        else:
            urlPath = posixpath.normpath(urllib.unquote(self.path))
            localName = FILE_PATH + "/" + urlPath.split('?', 1)[1] + "-out"

            if localName.endswith(".cf-out"):
                try:
                    f = open(localName, "r")
                except IOError:
                    self.send_error(404, "File not found")
                    return None

                os.unlink(localName)

                items = []
                totalSize = 0
                for l in f.readlines():
                    (path, size) = l.split()
                    size = int(size)
                    totalSize += size
                    items.append((path, size))
                del f
            else:
                size = os.stat(localName).st_size;
                items = [ (localName, size) ]
                totalSize = size

            self.send_response(200)
            self.send_header("Content-type", "application/octet-stream")
            self.send_header("Content-Length", str(totalSize))
            self.end_headers()

            f = open(items[0][0], "r")
            util.copyfileobj(f, self.wfile)

            if not localName.endswith(".cf-out"):
                os.unlink(items[0][0])

    def do_POST(self):
        if self.headers.get('Content-Type', '') == 'text/xml':
            authToken = self.getAuth()
            if authToken is None:
                return

            return self.handleXml(authToken)

        cmd = os.path.basename(self.path)
        if httpHandler.requiresAuth(cmd):
            authToken = self.checkAuth()
            if not authToken:
                return
        else:
            authToken = (None, None)

        try:
            self.send_response(200)
            self.send_header("Content-type", "text/html")
            self.end_headers()
            c = cgi.FieldStorage(fp = self.rfile, headers = self.headers,
                                 environ = { 'REQUEST_METHOD' : 'POST' })
            httpHandler.handleCmd(self.wfile.write, cmd, authToken, c)
        except:
            self.traceback()

    def getAuth(self):
        info = self.headers.get('Authorization', None)
        if info is None:
            return (None, None)
        info = info.split()

        try:
            authString = base64.decodestring(info[1])
        except:
            self.send_response(400)
            return None

        if authString.count(":") != 1:
            self.send_response(400)
            return None

        authToken = authString.split(":")

        return authToken

    def checkAuth(self):
 	if not self.headers.has_key('Authorization'):
            self.requestAuth()
            return None
	else:
            authToken = self.getAuth()
            if authToken is None:
                return

            # verify that the user/password actually exists in the database
            if not netRepos.auth.checkUserPass(authToken):
               self.requestAuth()
               return None

	return authToken

    def requestAuth(self):
        self.send_response(401)
        self.send_header("WWW-Authenticate", 'Basic realm="Conary Repository"')
        self.end_headers()
        return None

    def traceback(self):
        htmlengine = HtmlEngine()
        htmlengine.setWriter(self.wfile)
        htmlengine.stackTrace(self.wfile)

    def handleXml(self, authToken):
	contentLength = int(self.headers['Content-Length'])
	(params, method) = xmlrpclib.loads(self.rfile.read(contentLength))

	try:
	    result = netRepos.callWrapper(method, authToken, params)
	except netserver.InsufficientPermission:
	    self.send_response(403)
	    return

	resp = xmlrpclib.dumps((result,), methodresponse=1)

	self.send_response(200)
        encoding = self.headers.get('Accept-encoding', '')
        if len(resp) > 200 and 'zlib' in encoding:
            resp = zlib.compress(resp, 5)
            self.send_header('Content-encoding', 'zlib')
	self.send_header("Content-type", "text/xml")
	self.send_header("Content-length", str(len(resp)))
	self.end_headers()
	self.wfile.write(resp)

	return resp

    def do_PUT(self):
	path = self.path.split("?")[-1]
	path = FILE_PATH + '/' + path + "-in"

	size = os.stat(path).st_size
	if size != 0:
	    self.send_response(410, "Gone")
	    return

	out = open(path, "w")

	contentLength = int(self.headers['Content-Length'])
	while contentLength:
	    s = self.rfile.read(contentLength)
	    contentLength -= len(s)
	    out.write(s)

	self.send_response(200, 'OK')

class ResetableNetworkRepositoryServer(NetworkRepositoryServer):

    def reset(self, authToken, clientVersion):
        import shutil
        from repository.netrepos import fsrepos
	del self.repos
	shutil.rmtree(self.repPath + '/contents')
        os.unlink(self.repPath + '/sqldb')
	self.repos = fsrepos.FilesystemRepository(self.name, self.repPath,
						  self.map)
        return 0

class ServerConfig(ConfigFile):

    defaults = {
	'port'			:   '8000',
	'repositoryMap'         : [ STRINGDICT, {} ],
	'tmpFilePath'           : DEFAULT_FILE_PATH,
    }

    def __init__(self, path="serverrc"):
	ConfigFile.__init__(self)
	self.read(path)

def usage():
    print "usage: %s repospath authdb reposname [config file]" %sys.argv[0]
    sys.exit(1)

if __name__ == '__main__':
    argDef = {}
    cfgMap = {
	'port'	: 'port',
	'map'	: 'repositoryMap',
	'tmp-file-path' : 'tmpFilePath',
    }

    cfg = ServerConfig()

    argSet, otherArgs = options.processArgs(argDef, cfgMap, cfg, usage)

    FILE_PATH = cfg.tmpFilePath

    if not os.path.isdir(FILE_PATH):
	print FILE_PATH + " needs to be a directory"
	sys.exit(1)

    if len(otherArgs) == 5:
        cfg.read(otherArgs.pop())

    if len(otherArgs) != 4:
	usage()

    profile = 0
    if profile:
        import hotshot
        prof = hotshot.Profile('server.prof')
        prof.start()

    baseUrl="http://%s:%s/" % (os.uname()[1], cfg.port)

    netRepos = ResetableNetworkRepositoryServer(otherArgs[1], FILE_PATH,
			baseUrl, otherArgs[2], otherArgs[3], cfg.repositoryMap)
    httpHandler = HttpHandler(netRepos)

    port = int(cfg.port)
    httpServer = HTTPServer(("", port), HttpRequests)

    fds = {}
    fds[httpServer.fileno()] = httpServer

    p = select.poll()
    for fd in fds.iterkeys():
        p.register(fd, select.POLLIN)

    while True:
        try:
            events = p.poll()
            for (fd, event) in events:
                fds[fd].handle_request()
        except select.error:
            pass
        except:
            if profile:
                prof.stop()
                print "exception happened, exiting"
                sys.exit(1)
            else:
                raise
<|endoftext|>


<commit_before>import discord
import asyncio
import logging
import sys
import time
import threading

import uniirc
import uniformatter

print(sys.version)
print(discord.__version__)

logger = logging.getLogger('discord')
logger.setLevel(logging.DEBUG)
handler = logging.FileHandler(filename='discord.log', encoding='utf-8', mode='w')
handler.setFormatter(logging.Formatter('%(asctime)s:%(levelname)s:%(name)s: %(message)s'))
logger.addHandler(handler)

fp_logininfo = open("logininfo.txt", "r")
logintoken = fp_logininfo.read()				#reading login token from logininfo.txt as of OAuth2 update with BOT accounts
fp_logininfo.close()

chan_pairs = [	#(irc channel name, discord channel id)
	("#dota2mods", "250160069549883392"),
	("#dota2modhelpdesk", "259577266290294786"),
	("#dota2ai", "259185445131386881"),
]

voice_list = [
	"111342297999802368",	# @Syrin
]

client = discord.Client()

irc_client = uniirc.IRCClient(chan_pairs=chan_pairs, discord_client=client)

@client.event
@asyncio.coroutine								#notifying console that bot is logged in
def on_ready():
	print("Logged into discord as user: 「{}」".format(client.user.name))
	default_status = "with your messages"
	print("Setting default status: {}".format(default_status))
	yield from client.change_presence(game=discord.Game(name=default_status))
	return

@client.event
@asyncio.coroutine								#on message recieved, execute this block
def on_message(msg):
	for chan in chan_pairs:
		if msg.channel.id == chan[1] and msg.author.id != "263688414296145920":		#is bridge channel and not uni herself
			yield from msg_process(msg, chan[0])
	return

@asyncio.coroutine
def msg_process(msg, chan):
	#Nickname check
	if msg.author.nick:
		author = msg.author.nick
	else:
		author = msg.author.name

	#Formatting
	clean_msg = uniformatter.discordToIrc(msg.clean_content)

	#Sending
	author = "\x81".join(list(author))
	if len(author) > 1:
		author = author[:1] + "\x81" + author[1:]
	colour = str((sum([ ord(x) for x in author ]) % 12) + 2)		#get seeded random num between 2-13
	if len(colour) == 1:
		colour = "0" + colour

	irc_client.send_message(chan, "<\x03{}{}\x03> {}".format(colour, author, clean_msg))
	return

@asyncio.coroutine			#if irc thread has died then main program exits too
def irc_checker():
	yield from client.wait_until_ready()
	while not client.is_closed:
		if not irc_thread.is_alive():
			exit("IRC client disconnected. Exiting...")
		yield from asyncio.sleep(10)
	return

print("Starting Discord...")

loop = asyncio.get_event_loop()

print("Starting IRC...")

irc_thread = threading.Thread(target=irc_client.irc_run, daemon=True)
irc_thread.start()

try:
	loop.create_task(irc_checker())
	loop.run_until_complete(client.login(logintoken))
	loop.run_until_complete(client.connect())
except Exception:
	loop.run_until_complete(client.close())
finally:
	loop.close()
<commit_msg>Fixed extra HOP character at index 2 in discord relayed messages<commit_after>import discord
import asyncio
import logging
import sys
import time
import threading

import uniirc
import uniformatter

print(sys.version)
print(discord.__version__)

logger = logging.getLogger('discord')
logger.setLevel(logging.DEBUG)
handler = logging.FileHandler(filename='discord.log', encoding='utf-8', mode='w')
handler.setFormatter(logging.Formatter('%(asctime)s:%(levelname)s:%(name)s: %(message)s'))
logger.addHandler(handler)

fp_logininfo = open("logininfo.txt", "r")
logintoken = fp_logininfo.read()				#reading login token from logininfo.txt as of OAuth2 update with BOT accounts
fp_logininfo.close()

chan_pairs = [	#(irc channel name, discord channel id)
	("#dota2mods", "250160069549883392"),
	("#dota2modhelpdesk", "259577266290294786"),
	("#dota2ai", "259185445131386881"),
]

voice_list = [
	"111342297999802368",	# @Syrin
]

client = discord.Client()

irc_client = uniirc.IRCClient(chan_pairs=chan_pairs, discord_client=client)

@client.event
@asyncio.coroutine								#notifying console that bot is logged in
def on_ready():
	print("Logged into discord as user: 「{}」".format(client.user.name))
	default_status = "with your messages"
	print("Setting default status: {}".format(default_status))
	yield from client.change_presence(game=discord.Game(name=default_status))
	return

@client.event
@asyncio.coroutine								#on message recieved, execute this block
def on_message(msg):
	for chan in chan_pairs:
		if msg.channel.id == chan[1] and msg.author.id != "263688414296145920":		#is bridge channel and not uni herself
			yield from msg_process(msg, chan[0])
	return

@asyncio.coroutine
def msg_process(msg, chan):
	#Nickname check
	if msg.author.nick:
		author = msg.author.nick
	else:
		author = msg.author.name

	#Formatting
	clean_msg = uniformatter.discordToIrc(msg.clean_content)

	#Sending
	author = "\x81".join(list(author))
	colour = str((sum([ ord(x) for x in author ]) % 12) + 2)		#get seeded random num between 2-13
	if len(colour) == 1:
		colour = "0" + colour

	irc_client.send_message(chan, "<\x03{}{}\x03> {}".format(colour, author, clean_msg))
	return

@asyncio.coroutine			#if irc thread has died then main program exits too
def irc_checker():
	yield from client.wait_until_ready()
	while not client.is_closed:
		if not irc_thread.is_alive():
			exit("IRC client disconnected. Exiting...")
		yield from asyncio.sleep(10)
	return

print("Starting Discord...")

loop = asyncio.get_event_loop()

print("Starting IRC...")

irc_thread = threading.Thread(target=irc_client.irc_run, daemon=True)
irc_thread.start()

try:
	loop.create_task(irc_checker())
	loop.run_until_complete(client.login(logintoken))
	loop.run_until_complete(client.connect())
except Exception:
	loop.run_until_complete(client.close())
finally:
	loop.close()
<|endoftext|>


<commit_before>'''
Informations Retrieval Library
===============================
Analysis has some text analysis tools
'''

# Author: Tarek Amr <@gr33ndata>

class Freq:

    def __init__(self):
        self.freq_dict = {}
        self.freq_list = []

    def _to_array(self):
        ''' Converts self.freq_dict to self.freq_list
        '''
        for tok in self.freq_dict:
            self.freq_list.append({'tok': tok, 'freq': self.freq_dict[tok]})

    def _freq_cmp(self, a, b):
        ''' Helper functions used in sorting self.freq_list
        '''
        if a['freq'] > b['freq']:
            return  1
        else:
            return -1

    def add(self, tokenz):
        ''' Add more tokenized text to self.freq_dict
        '''
        for token in tokenz:
            if token in self.freq_dict:
                self.freq_dict[token] += 1
            else:
                self.freq_dict[token] = 1

    def display(self):
        for key in self.freqz:
            print key, self.freqz[key]

    def topn(self, n=1):
        self._to_array()
        self.freq_list.sort(self._freq_cmp, reverse=True)
        return self.freq_list

    def print_topn(self, n=1):
        for item in self.topn(n):
            print item['tok'], item['freq']

if __name__ == '__main__':

    f = Freq()
    f.add(['apple', 'juice'])
    f.add(['apple', 'pie'])
    f.add(['orange', 'juice'])
    f.add(['banana', 'split'])
    #f.display()
    f.print_topn(10)<commit_msg>Addig some comments to Analysis.py<commit_after>'''
Informations Retrieval Library
===============================
Analysis has some text analysis tools
'''

# Author: Tarek Amr <@gr33ndata>

class Freq:

    def __init__(self):
        self.freq_dict = {}
        self.freq_list = []

    def _to_array(self):
        ''' Converts self.freq_dict to self.freq_list
        '''
        for tok in self.freq_dict:
            self.freq_list.append({'tok': tok, 'freq': self.freq_dict[tok]})

    def _freq_cmp(self, a, b):
        ''' Helper functions used in sorting self.freq_list
        '''
        if a['freq'] > b['freq']:
            return  1
        else:
            return -1

    def add(self, tokenz):
        ''' Add more tokenized text to self.freq_dict
        '''
        for token in tokenz:
            if token in self.freq_dict:
                self.freq_dict[token] += 1
            else:
                self.freq_dict[token] = 1

    def display(self):
        for key in self.freqz:
            print key, self.freqz[key]

    def topn(self, n=1):
        ''' Returns top n tokens by their frequencies
            Defaul value for n is 1, i.e. most frequent token
        '''
        self._to_array()
        self.freq_list.sort(self._freq_cmp, reverse=True)
        return self.freq_list

    def print_topn(self, n=1):
        for item in self.topn(n):
            print item['tok'], item['freq']

if __name__ == '__main__':

    f = Freq()
    f.add(['apple', 'juice'])
    f.add(['apple', 'pie'])
    f.add(['orange', 'juice'])
    f.add(['banana', 'split'])
    #f.display()
    f.print_topn(10)<|endoftext|>


<commit_before>'''
Informations Retrieval Library
===============================
Analysis has some text analysis tools
'''

# Author: Tarek Amr <@gr33ndata>

class Freq:

    def __init__(self):
        self.freq_dict = {}
        self.freq_list = []

    def _to_array(self):
        ''' Converts self.freq_dict to self.freq_list
        '''
        for tok in self.freq_dict:
            self.freq_list.append({'tok': tok, 'freq': self.freq_dict[tok]})

    def _freq_cmp(self, a, b):
        ''' Helper functions used in sorting self.freq_list
        '''
        if a['freq'] > b['freq']:
            return  1
        else:
            return -1

    def add(self, tokenz):
        ''' Add more tokenized text to self.freq_dict
        '''
        for token in tokenz:
            if token in self.freq_dict:
                self.freq_dict[token] += 1
            else:
                self.freq_dict[token] = 1

    def display(self):
        for key in self.freqz:
            print key, self.freqz[key]

    def topn(self, n=1):
        ''' Returns top n tokens by their frequencies
            Defaul value for n is 1, i.e. most frequent token
        '''
        self._to_array()
        self.freq_list.sort(self._freq_cmp, reverse=True)
        return self.freq_list

    def print_topn(self, n=1):
        for item in self.topn(n):
            print item['tok'], item['freq']

if __name__ == '__main__':

    f = Freq()
    f.add(['apple', 'juice'])
    f.add(['apple', 'pie'])
    f.add(['orange', 'juice'])
    f.add(['banana', 'split'])
    #f.display()
    f.print_topn(10)<commit_msg>Refactoring analysis.py:add()<commit_after>'''
Informations Retrieval Library
===============================
Analysis has some text analysis tools
'''

# Author: Tarek Amr <@gr33ndata>

class Freq:

    def __init__(self):
        self.freq_dict = {}
        self.freq_list = []

    def _to_array(self):
        ''' Converts self.freq_dict to self.freq_list
        '''
        for tok in self.freq_dict:
            self.freq_list.append({'tok': tok, 'freq': self.freq_dict[tok]})

    def _freq_cmp(self, a, b):
        ''' Helper functions used in sorting self.freq_list
        '''
        if a['freq'] > b['freq']:
            return  1
        else:
            return -1

    def add(self, tokenz):
        ''' Add more tokenized text to self.freq_dict
        '''
        for token in tokenz:
            self.freq_dict[token] = self.freq_dict.get(token,0) + 1

    def display(self):
        for key in self.freqz:
            print key, self.freqz[key]

    def topn(self, n=1):
        ''' Returns top n tokens by their frequencies
            Defaul value for n is 1, i.e. most frequent token
        '''
        self._to_array()
        self.freq_list.sort(self._freq_cmp, reverse=True)
        return self.freq_list

    def print_topn(self, n=1):
        for item in self.topn(n):
            print item['tok'], item['freq']

if __name__ == '__main__':

    f = Freq()
    f.add(['apple', 'juice'])
    f.add(['apple', 'pie'])
    f.add(['orange', 'juice'])
    f.add(['banana', 'split'])
    #f.display()
    f.print_topn(10)<|endoftext|>


<commit_before>
def collocates(data, nbest = 30, window = 5):
    """Feed this a csv file generated with conc() and get its collocations"""
    import nltk
    from nltk import collocations
    from nltk.collocations import BigramCollocationFinder
    import re
    import os
    import time
    from time import localtime, strftime
    from corpkit.edit import datareader
    from corpkit.query import check_dit
    try:
        from IPython.display import display, clear_output
    except ImportError:
        pass

    on_cloud = check_dit()
    # turn all sentences into long string
    time = strftime("%H:%M:%S", localtime())
    #if noprint is False:
    print "\n%s: Generating %d collocates ... \n" % (time, nbest)
    good = datareader(data, on_cloud = on_cloud)
    good = unicode(good.lower(), 'utf-8', errors = 'ignore')
    sent_tokenizer=nltk.data.load('tokenizers/punkt/english.pickle')
    sents = sent_tokenizer.tokenize(good)
    tokenized_sents = [nltk.word_tokenize(i) for i in sents]
    allwords = []
    # for each sentence,
    for sent in tokenized_sents:
    # for each word,
        for word in sent:
        # make a list of all words
            allwords.append(word)
    bigram_measures = nltk.collocations.BigramAssocMeasures()
    finder = BigramCollocationFinder.from_words(allwords, window_size=window)
    ignored_words = nltk.corpus.stopwords.words('english')
    # anything containing letter or number
    regex = r'[A-Za-z0-9]'
    # the n't token
    nonot = r'n\'t'
    # lots of conditions!
    finder.apply_word_filter(lambda w: len(w) < 2 or w.lower() \
        in ignored_words or not re.match(regex, w) or re.match(nonot, w))
    finder.apply_freq_filter(2)
    results = sorted(finder.nbest(bigram_measures.raw_freq, nbest))
    listversion = []
    for index, thecollocation in enumerate(results):
        aslist = [index, thecollocation[0], thecollocation[1]]
        listversion.append(aslist)
    clear_output()
    return listversion
<commit_msg>unicode handling<commit_after>
def collocates(data, nbest = 30, window = 5):
    """Feed this data and get its collocations"""
    import nltk
    from nltk import collocations
    from nltk.collocations import BigramCollocationFinder
    import os
    import time
    from time import localtime, strftime
    from corpkit.edit import datareader
    from corpkit.query import check_dit
    try:
        from IPython.display import display, clear_output
    except ImportError:
        pass
    on_cloud = check_dit()
    # turn all sentences into long string
    time = strftime("%H:%M:%S", localtime())
    #if noprint is False:
    print "\n%s: Generating %d collocates ... \n" % (time, nbest)
    good = datareader(data, on_cloud = on_cloud)
    if type(good) != unicode:
        good = unicode(good.lower(), 'utf-8', errors = 'ignore')
    else:
        good = good.lower()
    sent_tokenizer=nltk.data.load('tokenizers/punkt/english.pickle')
    sents = sent_tokenizer.tokenize(good)
    tokenized_sents = [nltk.word_tokenize(i) for i in sents]
    allwords = []
    # for each sentence,
    for sent in tokenized_sents:
    # for each word,
        for word in sent:
        # make a list of all words
            allwords.append(word)
    bigram_measures = nltk.collocations.BigramAssocMeasures()
    finder = BigramCollocationFinder.from_words(allwords, window_size=window)
    # should be consistent in stopwords
    ignored_words = nltk.corpus.stopwords.words('english')
    finder.apply_word_filter(lambda w: len(w) < 2 or w.lower() \
        in ignored_words or not w.isalnum())
    #unncessary?:
    finder.apply_freq_filter(2)
    results = sorted(finder.nbest(bigram_measures.raw_freq, nbest))
    listversion = []
    for index, thecollocation in enumerate(results):
        aslist = [index, thecollocation[0], thecollocation[1]]
        listversion.append(aslist)
    clear_output()
    return listversion
<|endoftext|>


<commit_before>import json
import optparse

refTrancript = None
refGene = None
refProtein = None
gene_dict = None
version = "0.0.1"

def write_json(aequatus_dict, outfile=None, sort_keys=False):
    if outfile:
        with open(outfile, 'w') as f:
            json.dump(aequatus_dict, f, sort_keys=sort_keys)
    else:
        print json.dumps(aequatus_dict, indent=4, sort_keys=sort_keys)

def cigar_to_json(fname):
    global refProtein
    global gene_dict

    cigar_dict = dict()
    with open(fname) as f:
        for element in f.readlines():
            seq_id, cigar = element.rstrip('\n').split('\t')
            cigar_dict[getProteinIDfromGeneJSON(seq_id)] = cigar

    cd = list(cigar_dict)

    if refProtein is None:
        refProtein = cd[0]
        for each in gene_dict:
            parseGeneJSON(gene_dict[each], refProtein)
    return cigar_dict

def newicktree_to_json(fname):
    with open(fname) as f:
        return f.read().replace('\n', '')

def jsontree_to_json(fname):
    with open(fname) as f:
        return json.load(f)['tree']

def gene_to_json(fname):
    with open(fname) as f:
        return json.load(f)

def join_json(tree_dict, gene_dict, cigar_dict):
    aequatus_dict = {'tree': tree_dict,
                     'member': gene_dict}

    if cigar_dict:
        aequatus_dict['cigar'] = cigar_dict

    aequatus_dict['ref'] = refGene
    aequatus_dict['protein_id'] = refProtein
    aequatus_dict['transcript_id'] = refTrancript
    aequatus_dict['version'] = version

    return aequatus_dict

def parseGeneJSON(obj, id):
    global refGene
    global refTrancript

    if "Transcript" in obj:
        for each in obj["Transcript"]:
            if "Translation" in each:
                if each["Translation"]["id"] == id:
                    refGene = obj["id"]
                    refTrancript = each["id"]

def getProteinIDfromGeneJSON(id):
    global gene_dict
    for obj in gene_dict:
        if "Transcript" in gene_dict[obj]:
            for each in gene_dict[obj]["Transcript"]:
                if each["id"] == id:
                    return each["Translation"]["id"]

def parseTree(obj):
    global refProtein
    if "children" not in obj:
        if refProtein is None:
            refProtein = obj["sequence"]["id"][0]["accession"]
            for each in gene_dict:
                parseGeneJSON(gene_dict[each], refProtein)
    else:
        for child in obj["children"]:
            parseTree(child)

def __main__():
    global gene_dict
    parser = optparse.OptionParser()
    parser.add_option('-t', '--tree', help='Tree file')
    parser.add_option('-f', '--format', type='choice', choices=['newick', 'json'], default='json', help='Tree Format')
    parser.add_option('-c', '--cigar', help='CIGAR file in table format (only if tree is in newick format)')
    parser.add_option('-g', '--gene', help='Gene file in JSON format')
    parser.add_option('-s', '--sort', action='store_true', help='Sort the keys in the JSON output')
    parser.add_option('-o', '--output', help='Path of the output file. If not specified, will print on the standard output')
    options, args = parser.parse_args()
    if args:
        raise Exception('Use options to provide inputs')

    gene_dict = gene_to_json(options.gene)

    if options.format == "newick":
        cigar_dict = cigar_to_json(options.cigar)
        tree_dict = newicktree_to_json(options.tree)
    else:
        cigar_dict = dict()
        tree_dict = jsontree_to_json(options.tree)
        parseTree(tree_dict)

    aequatus_dict = join_json(tree_dict, gene_dict, cigar_dict)

    write_json(aequatus_dict, options.output, options.sort)

if __name__ == '__main__':
    __main__()
<commit_msg>Rewrite without unnecessary global variables. Better function names. Increase performances.<commit_after>import json
import optparse

version = "0.0.1"

def write_json(aequatus_dict, outfile=None, sort_keys=False):
    if outfile:
        with open(outfile, 'w') as f:
            json.dump(aequatus_dict, f, sort_keys=sort_keys)
    else:
        print json.dumps(aequatus_dict, indent=4, sort_keys=sort_keys)

def cigar_to_dict(fname, gene_dict):
    cigar_dict = dict()
    with open(fname) as f:
        for element in f.readlines():
            seq_id, cigar = element.rstrip('\n').split('\t')
            cigar_dict[get_protein_id_from_transcript_id(gene_dict, seq_id)] = cigar

    return cigar_dict

def newicktree_to_string(fname):
    with open(fname) as f:
        return f.read().replace('\n', '')

def jsontree_to_dict(fname):
    with open(fname) as f:
        return json.load(f)['tree']

def gene_json_to_dict(fname):
    with open(fname) as f:
        return json.load(f)

def join_json(tree, gene_dict, cigar_dict, ref_gene, ref_protein, ref_transcript):
    aequatus_dict = {'tree': tree,
                     'member': gene_dict}

    if cigar_dict:
        aequatus_dict['cigar'] = cigar_dict

    aequatus_dict['ref'] = ref_gene
    aequatus_dict['protein_id'] = ref_protein
    aequatus_dict['transcript_id'] = ref_transcript
    aequatus_dict['version'] = version

    return aequatus_dict

def get_gene_and_transcript_ids_from_protein_id(gene_dict, protein_id):
    for gene in gene_dict.values():
        if "Transcript" in gene:
            for transcript in gene["Transcript"]:
                if 'Translation' in transcript and 'id' in transcript["Translation"]:
                    if transcript["Translation"]["id"] == protein_id:
                        return (gene["id"], transcript["id"])

def get_protein_id_from_transcript_id(gene_dict, transcript_id):
    for gene in gene_dict.values():
        if "Transcript" in gene:
            for transcript in gene["Transcript"]:
                if transcript['id'] == transcript_id:
                    if 'Translation' in transcript and 'id' in transcript['Translation']:
                        return transcript["Translation"]["id"]
                    else:
                        break

def get_first_protein_id_from_tree(tree_el):
    if "children" not in tree_el:
        return tree_el["sequence"]["id"][0]["accession"]
    else:
        # Depth-first search
        first_child = tree_el["children"][0]
        return get_first_protein_id_from_tree(first_child)

def __main__():
    parser = optparse.OptionParser()
    parser.add_option('-t', '--tree', help='Gene tree file')
    parser.add_option('-f', '--format', type='choice', choices=['newick', 'json'], default='json', help='Gene tree format')
    parser.add_option('-c', '--cigar', help='CIGAR alignments of CDS file in tabular format (only if tree is in newick format)')
    parser.add_option('-g', '--gene', help='Gene features file in JSON format')
    parser.add_option('-s', '--sort', action='store_true', help='Sort the keys in the JSON output')
    parser.add_option('-o', '--output', help='Path of the output file. If not specified, will print on the standard output')
    options, args = parser.parse_args()
    if args:
        raise Exception('Use options to provide inputs')

    gene_dict = gene_json_to_dict(options.gene)

    if options.format == "newick":
        cigar_dict = cigar_to_dict(options.cigar, gene_dict)
        tree = newicktree_to_string(options.tree)
        # Pick a random protein as reference
        ref_protein = next(iter(cigar_dict))
    else:
        cigar_dict = dict()
        tree = jsontree_to_dict(options.tree)
        ref_protein = get_first_protein_id_from_tree(tree)

    ref_gene, ref_transcript = get_gene_and_transcript_ids_from_protein_id(gene_dict, ref_protein)
    aequatus_dict = join_json(tree, gene_dict, cigar_dict, ref_gene, ref_protein, ref_transcript)

    write_json(aequatus_dict, options.output, options.sort)

if __name__ == '__main__':
    __main__()
<|endoftext|>


<commit_before>from nose.tools import *
from openpassword import Keychain
from openpassword.keychain_item import KeychainItem
from openpassword.exceptions import InvalidPasswordException
from openpassword.exceptions import KeychainLockedException

import fudge

class KeychainSpec:

    def it_unlocks_the_keychain_with_the_right_password(self):
        key_repository = self._unlockable_key_repository()
        item_repository = fudge.Fake('keychain_item_repository')

        keychain = Keychain(key_repository, item_repository)
        keychain.unlock('rightpassword')

        eq_(keychain.is_locked(), False)

    @raises(InvalidPasswordException)
    def it_raises_invalidpasswordexception_with_wrong_password(self):
        key = fudge.Fake('encryption_key')
        key.provides("decrypt").raises(InvalidPasswordException)

        key_repository = fudge.Fake('encryption_key_repository')
        key_repository.provides("key_for_security_level").with_args("SL5").returns(key)

        item_repository = fudge.Fake('keychain_item_repository')

        keychain = Keychain(key_repository, item_repository)
        keychain.unlock('wrongpassword')

    def it_fails_to_unlock_with_wrong_password(self):
        key = fudge.Fake('encryption_key')
        key.provides("decrypt").raises(InvalidPasswordException)

        key_repository = fudge.Fake('encryption_key_repository')
        key_repository.provides("key_for_security_level").with_args("SL5").returns(key)

        item_repository = fudge.Fake('keychain_item_repository')

        keychain = Keychain(key_repository, item_repository)
        try:
            keychain.unlock('wrongpassword')
        except InvalidPasswordException:
            pass

        eq_(keychain.is_locked(), True)

    def it_locks_when_lock_is_called(self):
        key_repository = self._unlockable_key_repository()
        item_repository = fudge.Fake('keychain_item_repository')

        keychain = Keychain(key_repository, item_repository)

        keychain.unlock('rightpassword')
        eq_(keychain.is_locked(), False)
        keychain.lock()
        eq_(keychain.is_locked(), True)

    def it_returns_an_item_by_unique_id(self):
        key_repository = self._unlockable_key_repository()

        item_repository = fudge.Fake('keychain_item_repository')
        item_repository.provides('get_item_by_unique_id').with_args('random_unique_id').returns(KeychainItem({}))

        keychain = Keychain(key_repository, item_repository)
        keychain.unlock("password")

        eq_(KeychainItem, type(keychain.get_item_by_unique_id('random_unique_id')))

    @raises(KeychainLockedException)
    def it_raises_keychainlocked_exception_when_trying_to_get_item_from_locked_keychain(self):
        key_repository = fudge.Fake('encryption_key_repository')
        item_repository = fudge.Fake('keychain_item_repository')

        keychain = Keychain(key_repository, item_repository)
        keychain.get_item_by_unique_id('some_random_item')

    def _unlockable_key_repository(self):
        key = fudge.Fake('encryption_key')
        key.provides("decrypt")

        key_repository = fudge.Fake('encryption_key_repository')
        key_repository.provides("key_for_security_level").with_args("SL5").returns(key)

        return key_repository
<commit_msg>Change test to check for the actual object rather than type<commit_after>from nose.tools import *
from openpassword import Keychain
from openpassword.keychain_item import KeychainItem
from openpassword.exceptions import InvalidPasswordException
from openpassword.exceptions import KeychainLockedException

import fudge

class KeychainSpec:

    def it_unlocks_the_keychain_with_the_right_password(self):
        key_repository = self._unlockable_key_repository()
        item_repository = fudge.Fake('keychain_item_repository')

        keychain = Keychain(key_repository, item_repository)
        keychain.unlock('rightpassword')

        eq_(keychain.is_locked(), False)

    @raises(InvalidPasswordException)
    def it_raises_invalidpasswordexception_with_wrong_password(self):
        key = fudge.Fake('encryption_key')
        key.provides("decrypt").raises(InvalidPasswordException)

        key_repository = fudge.Fake('encryption_key_repository')
        key_repository.provides("key_for_security_level").with_args("SL5").returns(key)

        item_repository = fudge.Fake('keychain_item_repository')

        keychain = Keychain(key_repository, item_repository)
        keychain.unlock('wrongpassword')

    def it_fails_to_unlock_with_wrong_password(self):
        key = fudge.Fake('encryption_key')
        key.provides("decrypt").raises(InvalidPasswordException)

        key_repository = fudge.Fake('encryption_key_repository')
        key_repository.provides("key_for_security_level").with_args("SL5").returns(key)

        item_repository = fudge.Fake('keychain_item_repository')

        keychain = Keychain(key_repository, item_repository)
        try:
            keychain.unlock('wrongpassword')
        except InvalidPasswordException:
            pass

        eq_(keychain.is_locked(), True)

    def it_locks_when_lock_is_called(self):
        key_repository = self._unlockable_key_repository()
        item_repository = fudge.Fake('keychain_item_repository')

        keychain = Keychain(key_repository, item_repository)

        keychain.unlock('rightpassword')
        eq_(keychain.is_locked(), False)
        keychain.lock()
        eq_(keychain.is_locked(), True)

    def it_returns_an_item_by_unique_id(self):
        key_repository = self._unlockable_key_repository()

        keychain_item = fudge.Fake('keychain_item')
        item_repository = fudge.Fake('keychain_item_repository')
        item_repository.provides('get_item_by_unique_id').with_args('random_unique_id').returns(keychain_item)

        keychain = Keychain(key_repository, item_repository)
        keychain.unlock("password")

        eq_(keychain_item, keychain.get_item_by_unique_id('random_unique_id'))

    @raises(KeychainLockedException)
    def it_raises_keychainlocked_exception_when_trying_to_get_item_from_locked_keychain(self):
        key_repository = fudge.Fake('encryption_key_repository')
        item_repository = fudge.Fake('keychain_item_repository')

        keychain = Keychain(key_repository, item_repository)
        keychain.get_item_by_unique_id('some_random_item')

    def _unlockable_key_repository(self):
        key = fudge.Fake('encryption_key')
        key.provides("decrypt")

        key_repository = fudge.Fake('encryption_key_repository')
        key_repository.provides("key_for_security_level").with_args("SL5").returns(key)

        return key_repository
<|endoftext|>


<commit_before>#!/usr/bin/env python
import argparse
from collections import defaultdict

import pysam

def Parser():
    the_parser = argparse.ArgumentParser()
    the_parser.add_argument('--output', nargs='+', action='store', type=str,
                            help='Count tables')
    the_parser.add_argument('--alignments', nargs='+',
                            help="bam alignments files")
    the_parser.add_argument('--labels', nargs='+', help="Alignments labels")
    the_parser.add_argument('--number',
                            choices=["unique", "multiple"],
                            help="output is a single table or multiple tables")
    args = the_parser.parse_args()
    return args

def get_counts(bamfile):
    """
    Takes an AlignmentFile object and returns a dictionary of counts for sense,
    antisense, or both sense and antisense reads aligning to the bam references,
    depending on the pre-treatment performed by sambamba in the xml wrapper
    """
    counts = defaultdict(int)
    for ref_name in bamfile.references:
        counts[ref_name] = 0
    for ref_name in bamfile.references:
        counts[ref_name] = bamfile.count(reference=ref_name)
    return counts

def writetable(diclist, labels, output, number):
    ''' diclist is a list of count dictionnaries '''
    countlists = []
    for dic in diclist:
        counts = sorted(dic.items())
        counts = [j for (i, j) in counts]
        countlists.append(counts)
    if number == "unique":
        out = open("outputdir/table.tabular", "w")
        out.write("gene\t%s\n" % "\t".join(labels))
        for countline in zip(sorted(diclist[0]), *countlists):
            line = [str(i) for i in countline]
            out.write("%s\n" % "\t".join(line))
        out.close()
    else:
        for i, (dic, label) in enumerate(zip(diclist, labels)):
            out = open("outputdir/table" + str(i) + ".tabular", "w")
            out.write("gene\t%s\n" % label)
            for gene in sorted(dic):
                out.write("%s\t%s\n" % (gene, dic[gene]))
            out.close()

def main(alignments, labels, output, number):
    diclist = []
    for file in alignments:
        bam_object = pysam.AlignmentFile(file, 'rb')
        diclist.append(get_counts(bam_object))
    writetable(diclist, labels, output, number)

if __name__ == "__main__":
    args = Parser()
    main(args.alignments, args.labels, args.output, args.number)
<commit_msg>lint python code<commit_after>#!/usr/bin/env python
import argparse
from collections import defaultdict

import pysam

def Parser():
    the_parser = argparse.ArgumentParser()
    the_parser.add_argument('--output', nargs='+', action='store', type=str,
                            help='Count tables')
    the_parser.add_argument('--alignments', nargs='+',
                            help="bam alignments files")
    the_parser.add_argument('--labels', nargs='+', help="Alignments labels")
    the_parser.add_argument('--number',
                            choices=["unique", "multiple"],
                            help="output is a single table or multiple tables")
    args = the_parser.parse_args()
    return args

def get_counts(bamfile):
    """
    Takes an AlignmentFile object and returns a dictionary of counts for sense,
    antisense, or both sense and antisense bam alignments to the references,
    depending on the pre-treatment performed by sambamba in the xml wrapper
    """
    counts = defaultdict(int)
    for ref_name in bamfile.references:
        counts[ref_name] = 0
    for ref_name in bamfile.references:
        counts[ref_name] = bamfile.count(reference=ref_name)
    return counts

def writetable(diclist, labels, output, number):
    ''' diclist is a list of count dictionnaries '''
    countlists = []
    for dic in diclist:
        counts = sorted(dic.items())
        counts = [j for (i, j) in counts]
        countlists.append(counts)
    if number == "unique":
        out = open("outputdir/table.tabular", "w")
        out.write("gene\t%s\n" % "\t".join(labels))
        for countline in zip(sorted(diclist[0]), *countlists):
            line = [str(i) for i in countline]
            out.write("%s\n" % "\t".join(line))
        out.close()
    else:
        for i, (dic, label) in enumerate(zip(diclist, labels)):
            out = open("outputdir/table" + str(i) + ".tabular", "w")
            out.write("gene\t%s\n" % label)
            for gene in sorted(dic):
                out.write("%s\t%s\n" % (gene, dic[gene]))
            out.close()

def main(alignments, labels, output, number):
    diclist = []
    for file in alignments:
        bam_object = pysam.AlignmentFile(file, 'rb')
        diclist.append(get_counts(bam_object))
    writetable(diclist, labels, output, number)

if __name__ == "__main__":
    args = Parser()
    main(args.alignments, args.labels, args.output, args.number)
<|endoftext|>