<commit_before># -*- coding: utf-8 -*-
"""

"main concept of MongoDB is embed whenever possible"
Ref: http://stackoverflow.com/questions/4655610#comment5129510_4656431
"""
from __future__ import (absolute_import, unicode_literals, division)

from mongoengine import (Document, EmbeddedDocument, EmbeddedDocumentField,
                         FloatField, IntField, ListField, StringField,
                         ReferenceField)

from .event import Event
from .case import Case

######## These are defined terms for different categories ########

CONSERVATION = ('NotConserved', 'Conserved')

CONSEQUENCE = ('deleterious', 'probably_damaging', 'possibly_damaging',
               'tolerated', 'benign', 'unknown')

SO_TERMS = (
  'transcript_ablation',
  'splice_donor_variant',
  'splice_acceptor_variant',
  'stop_gained',
  'frameshift_variant',
  'stop_lost',
  'initiator_codon_variant',
  'transcript_amplification',
  'inframe_insertion',
  'inframe_deletion',
  'missense_variant',
  'splice_region_variant',
  'incomplete_terminal_codon_variant',
  'stop_retained_variant',
  'synonymous_variant',
  'coding_sequence_variant',
  'mature_miRNA_variant',
  '5_prime_UTR_variant',
  '3_prime_UTR_variant',
  'non_coding_exon_variant',
  'non_coding_transcript_exon_variant',
  'non_coding_transcript_variant',
  'nc_transcript_variant',
  'intron_variant',
  'NMD_transcript_variant',
  'non_coding_transcript_variant',
  'upstream_gene_variant',
  'downstream_gene_variant',
  'TFBS_ablation',
  'TFBS_amplification',
  'TF_binding_site_variant',
  'regulatory_region_ablation',
  'regulatory_region_amplification',
  'regulatory_region_variant',
  'feature_elongation',
  'feature_truncation',
  'intergenic_variant'
)

FEATURE_TYPES = (
  'exonic',
  'splicing',
  'ncRNA_exonic',
  'intronic',
  'ncRNA',
  'upstream',
  '5UTR',
  '3UTR',
  'downstream',
  'TFBS',
  'regulatory_region',
  'genomic_feature',
  'intergenic_variant'
)

GENETIC_MODELS = (
  ('AR_hom', 'Autosomal Recessive Homozygote'),
  ('AR_hom_dn', 'Autosomal Recessive Homozygote De Novo'),
  ('AR_comp', 'Autosomal Recessive Compound'),
  ('AR_comp_dn', 'Autosomal Recessive Compound De Novo'),
  ('AD', 'Autosomal Dominant'),
  ('AD_dn', 'Autosomal Dominant De Novo'),
  ('XR', 'X Linked Recessive'),
  ('XR_dn', 'X Linked Recessive De Novo'),
  ('XD', 'X Linked Dominant'),
  ('XD_dn', 'X Linked Dominant De Novo'),
)

class Transcript(EmbeddedDocument):
  transcript_id = StringField(required=True)
  hgnc_symbol = StringField()
  sift_prediction = StringField(choices=CONSEQUENCE)
  polyphen_prediction = StringField(choices=CONSEQUENCE)
  functional_annotations = ListField(StringField(choices=SO_TERMS))
  region_annotations = ListField(StringField(choices=FEATURE_TYPES))
  exon = StringField()
  intron = StringField()
  coding_sequence_name = StringField()
  protein_sequence_name = StringField()

class DiseaseModel(EmbeddedDocument):
  omim_id = IntField(required=True)
  disease_models = ListField()

class Gene(EmbeddedDocument):
  hgnc_symbol = StringField(required=True)
  transcripts = ListField(EmbeddedDocumentField(Transcript))
  functional_annotation = StringField(choices=SO_TERMS)
  region_annotation = StringField(choices=FEATURE_TYPES)
  sift_prediction = StringField(choices=CONSEQUENCE)
  polyphen_prediction = StringField(choices=CONSEQUENCE)
  omim_terms = ListField(IntField())
  expected_inheritance = ListField(EmbeddedDocumentField(DiseaseModel))

class Compound(EmbeddedDocument):
  # This must be the document_id for this variant
  variant = ReferenceField('Variant')
  # This is the variant id
  display_name = StringField(required=True)
  combined_score = FloatField(required=True)
  region_annotations = ListField(StringField())
  functional_annotations = ListField(StringField())

  @property
  def rank_score(self):
    """Return the individual rank score for this variant."""
    return self.variant.rank_score

class GTCall(EmbeddedDocument):
  sample = StringField()
  genotype_call = StringField()
  allele_depths = ListField(IntField())
  read_depth = IntField()
  genotype_quality = IntField()

  def __unicode__(self):
    return self.sample

class Variant(Document):
  # document_id is a md5 string created by institute_genelist_caseid_variantid:
  document_id = StringField(primary_key=True)
  # variant_id is a md5 string created by variant_id
  variant_id = StringField(required=True)
  # display name in variant_id (no md5)
  display_name = StringField(required=True)
  # the variant can be either a reserchvariant or a clinical variant.
  # for research variants we display all the available information while
  # the clinical variants hae limited annotation fields.
  variant_type = StringField(required=True, choices=(
                                                  'research',
                                                  'clinical'
                                                  )
                            )
  # case_id is a string like institute_caseid
  case_id = StringField(required=True)
  chromosome = StringField(required=True)
  position = IntField(required=True)
  reference = StringField(required=True)
  alternative = StringField(required=True)
  rank_score = FloatField(required=True)
  variant_rank = IntField(required=True)
  quality = FloatField()
  filters = ListField(StringField())
  samples = ListField(EmbeddedDocumentField(GTCall))
  genetic_models = ListField(StringField(choices=GENETIC_MODELS))
  compounds = ListField(EmbeddedDocumentField(Compound))
  events = ListField(EmbeddedDocumentField(Event))
  genes = ListField(EmbeddedDocumentField(Gene))
  db_snp_ids = ListField(StringField())
  # Gene ids:
  hgnc_symbols = ListField(StringField())
  ensemble_gene_ids = ListField(StringField())
  # Frequencies:
  thousand_genomes_frequency = FloatField()
  exac_frequency = FloatField()
  local_frequency = FloatField()
  # Predicted deleteriousness:
  cadd_score = FloatField()
  # Conservation:
  phast_conservation = ListField(StringField(choices=CONSERVATION))
  gerp_conservation = ListField(StringField(choices=CONSERVATION))
  phylop_conservation = ListField(StringField(choices=CONSERVATION))
  # Database options:
  gene_lists = ListField(StringField())
  expected_inheritance = ListField(StringField())

  @property
  def local_requency(self):
    """Returns a float with the local freauency for this position."""
    return (Variant.objects(variant_id=self.variant_id).count /
              Case.objects.count())

  @property
  def expected_inheritance(self):
    """Returns a list with expected inheritance model(s)."""
    expected_inheritance = set([])
    for gene in self.genes:
      for entry in gene.expected_inheritance:
        for gene_model in entry.disease_models:
          expected_inheritance.add(gene_model)

    return list(expected_inheritance)

  @property
  def omim_annotations(self):
    """Returns a list with omim id(s)."""
    omim_annotations = []
    if len(self.genes) == 1:
      omim_annotations = [gene.omim_terms for gene in self.genes]
    else:
      for gene in self.genes:
        omim_annotations.append(':'.join([gene.hgnc_symbol, gene.omim_terms]))
    return region_annotations

  @property
  def region_annotations(self):
    """Returns a list with region annotation(s)."""
    region_annotations = []
    if len(self.genes) == 1:
      return [gene.region_annotation for gene in self.genes]
    else:
      for gene in self.genes:
        region_annotations.append(':'.join([gene.hgnc_symbol, gene.region_annotation]))
    return region_annotations

  @property
  def sift_predictions(self):
    """Return a list with the sift prediction(s) for this variant. The most severe for each gene."""
    sift_predictions = []
    if len(self.genes) == 1:
      sift_predictions = [gene.sift_prediction for gene in self.genes]
    else:
      for gene in self.genes:
        sift_predictions.append(':'.join([gene.hgnc_symbol, gene.sift_prediction or '-']))
    return sift_predictions

  @property
  def polyphen_predictions(self):
    """Return a list with the polyphen prediction(s) for this variant. The most severe for each gene."""
    polyphen_predictions = []
    if len(self.genes) == 1:
      polyphen_predictions = [gene.polyphen_prediction for gene in self.genes]
    else:
      for gene in self.genes:
        polyphen_predictions.append(':'.join([gene.hgnc_symbol, gene.polyphen_prediction or '-']))
    return polyphen_predictions

  @property
  def functional_annotations(self):
    """Return a list with the functional annotation(s) for this variant. The most severe for each gene."""
    functional_annotations = []
    if len(self.genes) == 1:
      functional_annotations = [gene.functional_annotation for gene in self.genes]
    else:
      for gene in self.genes:
        functional_annotations.append(':'.join([gene.hgnc_symbol, gene.functional_annotation or '']))
    return functional_annotations

  @property
  def region_annotations(self):
    """Returns a list with region annotation(s)."""
    region_annotations = []
    if len(self.genes) == 1:
      return [gene.region_annotation for gene in self.genes]
    else:
      for gene in self.genes:
        region_annotations.append(':'.join([gene.hgnc_symbol, gene.region_annotation or '']))
    return region_annotations

  @property
  def transcripts(self):
    """Yield all transcripts as a flat iterator.

    For each transcript both the parent gene object as well as the
    transcript is yielded.

    Yields:
      class, class: Gene and Transcript ODM
    """
    # loop over each gene in order
    for gene in self.genes:
      # loop over each child transcript for the gene
      for transcript in gene.transcripts:
        # yield the parent gene, child transcript combo
        yield transcript

  @property
  def end_position(self):
    # which alternative allele contains most bases
    alt_bases = max([len(alt) for alt in self.alternatives])
    # vs. reference allele
    bases = max(len(self.reference), alt_bases)

    return self.position + (bases - 1)

  @property
  def frequency(self):
    """Returns a judgement on the overall frequency of the variant.

    Combines multiple metrics into a single call.
    """
    most_common_frequency = max(self.thousand_genomes_frequency,
                                self.exac_frequency)

    if most_common_frequency > .05:
      return 'common'

    elif most_common_frequency > .01:
      return 'uncommon'

    else:
      return 'rare'

  def __unicode__(self):
    return self.display_name
<commit_msg>fix bug in variant model<commit_after># -*- coding: utf-8 -*-
"""

"main concept of MongoDB is embed whenever possible"
Ref: http://stackoverflow.com/questions/4655610#comment5129510_4656431
"""
from __future__ import (absolute_import, unicode_literals, division)

from mongoengine import (Document, EmbeddedDocument, EmbeddedDocumentField,
                         FloatField, IntField, ListField, StringField,
                         ReferenceField)

from .event import Event
from .case import Case

######## These are defined terms for different categories ########

CONSERVATION = ('NotConserved', 'Conserved')

CONSEQUENCE = ('deleterious', 'probably_damaging', 'possibly_damaging',
               'tolerated', 'benign', 'unknown')

SO_TERMS = (
  'transcript_ablation',
  'splice_donor_variant',
  'splice_acceptor_variant',
  'stop_gained',
  'frameshift_variant',
  'stop_lost',
  'initiator_codon_variant',
  'transcript_amplification',
  'inframe_insertion',
  'inframe_deletion',
  'missense_variant',
  'splice_region_variant',
  'incomplete_terminal_codon_variant',
  'stop_retained_variant',
  'synonymous_variant',
  'coding_sequence_variant',
  'mature_miRNA_variant',
  '5_prime_UTR_variant',
  '3_prime_UTR_variant',
  'non_coding_exon_variant',
  'non_coding_transcript_exon_variant',
  'non_coding_transcript_variant',
  'nc_transcript_variant',
  'intron_variant',
  'NMD_transcript_variant',
  'non_coding_transcript_variant',
  'upstream_gene_variant',
  'downstream_gene_variant',
  'TFBS_ablation',
  'TFBS_amplification',
  'TF_binding_site_variant',
  'regulatory_region_ablation',
  'regulatory_region_amplification',
  'regulatory_region_variant',
  'feature_elongation',
  'feature_truncation',
  'intergenic_variant'
)

FEATURE_TYPES = (
  'exonic',
  'splicing',
  'ncRNA_exonic',
  'intronic',
  'ncRNA',
  'upstream',
  '5UTR',
  '3UTR',
  'downstream',
  'TFBS',
  'regulatory_region',
  'genomic_feature',
  'intergenic_variant'
)

GENETIC_MODELS = (
  ('AR_hom', 'Autosomal Recessive Homozygote'),
  ('AR_hom_dn', 'Autosomal Recessive Homozygote De Novo'),
  ('AR_comp', 'Autosomal Recessive Compound'),
  ('AR_comp_dn', 'Autosomal Recessive Compound De Novo'),
  ('AD', 'Autosomal Dominant'),
  ('AD_dn', 'Autosomal Dominant De Novo'),
  ('XR', 'X Linked Recessive'),
  ('XR_dn', 'X Linked Recessive De Novo'),
  ('XD', 'X Linked Dominant'),
  ('XD_dn', 'X Linked Dominant De Novo'),
)

class Transcript(EmbeddedDocument):
  transcript_id = StringField(required=True)
  hgnc_symbol = StringField()
  sift_prediction = StringField(choices=CONSEQUENCE)
  polyphen_prediction = StringField(choices=CONSEQUENCE)
  functional_annotations = ListField(StringField(choices=SO_TERMS))
  region_annotations = ListField(StringField(choices=FEATURE_TYPES))
  exon = StringField()
  intron = StringField()
  coding_sequence_name = StringField()
  protein_sequence_name = StringField()

class DiseaseModel(EmbeddedDocument):
  omim_id = IntField(required=True)
  disease_models = ListField(StringField())

class Gene(EmbeddedDocument):
  hgnc_symbol = StringField(required=True)
  transcripts = ListField(EmbeddedDocumentField(Transcript))
  functional_annotation = StringField(choices=SO_TERMS)
  region_annotation = StringField(choices=FEATURE_TYPES)
  sift_prediction = StringField(choices=CONSEQUENCE)
  polyphen_prediction = StringField(choices=CONSEQUENCE)
  omim_terms = ListField(IntField())
  expected_inheritance = ListField(EmbeddedDocumentField(DiseaseModel))

class Compound(EmbeddedDocument):
  # This must be the document_id for this variant
  variant = ReferenceField('Variant')
  # This is the variant id
  display_name = StringField(required=True)
  combined_score = FloatField(required=True)
  region_annotations = ListField(StringField())
  functional_annotations = ListField(StringField())

  @property
  def rank_score(self):
    """Return the individual rank score for this variant."""
    return self.variant.rank_score

class GTCall(EmbeddedDocument):
  sample = StringField()
  genotype_call = StringField()
  allele_depths = ListField(IntField())
  read_depth = IntField()
  genotype_quality = IntField()

  def __unicode__(self):
    return self.sample

class Variant(Document):
  # document_id is a md5 string created by institute_genelist_caseid_variantid:
  document_id = StringField(primary_key=True)
  # variant_id is a md5 string created by variant_id
  variant_id = StringField(required=True)
  # display name in variant_id (no md5)
  display_name = StringField(required=True)
  # the variant can be either a reserchvariant or a clinical variant.
  # for research variants we display all the available information while
  # the clinical variants hae limited annotation fields.
  variant_type = StringField(required=True, choices=(
                                                  'research',
                                                  'clinical'
                                                  )
                            )
  # case_id is a string like institute_caseid
  case_id = StringField(required=True)
  chromosome = StringField(required=True)
  position = IntField(required=True)
  reference = StringField(required=True)
  alternative = StringField(required=True)
  rank_score = FloatField(required=True)
  variant_rank = IntField(required=True)
  quality = FloatField()
  filters = ListField(StringField())
  samples = ListField(EmbeddedDocumentField(GTCall))
  genetic_models = ListField(StringField(choices=GENETIC_MODELS))
  compounds = ListField(EmbeddedDocumentField(Compound))
  events = ListField(EmbeddedDocumentField(Event))
  genes = ListField(EmbeddedDocumentField(Gene))
  db_snp_ids = ListField(StringField())
  # Gene ids:
  hgnc_symbols = ListField(StringField())
  ensemble_gene_ids = ListField(StringField())
  # Frequencies:
  thousand_genomes_frequency = FloatField()
  exac_frequency = FloatField()
  local_frequency = FloatField()
  # Predicted deleteriousness:
  cadd_score = FloatField()
  # Conservation:
  phast_conservation = ListField(StringField(choices=CONSERVATION))
  gerp_conservation = ListField(StringField(choices=CONSERVATION))
  phylop_conservation = ListField(StringField(choices=CONSERVATION))
  # Database options:
  gene_lists = ListField(StringField())
  expected_inheritance = ListField(StringField())

  @property
  def local_requency(self):
    """Returns a float with the local freauency for this position."""
    return (Variant.objects(variant_id=self.variant_id).count /
              Case.objects.count())

  @property
  def expected_inheritance(self):
    """Returns a list with expected inheritance model(s)."""
    expected_inheritance = set([])
    for gene in self.genes:
      for entry in gene.expected_inheritance:
        for gene_model in entry.disease_models:
          expected_inheritance.add(gene_model)

    return list(expected_inheritance)

  @property
  def omim_annotations(self):
    """Returns a list with omim id(s)."""
    omim_annotations = []
    if len(self.genes) == 1:
      omim_annotations = [gene.omim_terms for gene in self.genes]
    else:
      for gene in self.genes:
        omim_annotations.append(':'.join([gene.hgnc_symbol, gene.omim_terms]))
    return region_annotations

  @property
  def region_annotations(self):
    """Returns a list with region annotation(s)."""
    region_annotations = []
    if len(self.genes) == 1:
      return [gene.region_annotation for gene in self.genes]
    else:
      for gene in self.genes:
        region_annotations.append(':'.join([gene.hgnc_symbol, gene.region_annotation]))
    return region_annotations

  @property
  def sift_predictions(self):
    """Return a list with the sift prediction(s) for this variant. The most severe for each gene."""
    sift_predictions = []
    if len(self.genes) == 1:
      sift_predictions = [gene.sift_prediction for gene in self.genes]
    else:
      for gene in self.genes:
        sift_predictions.append(':'.join([gene.hgnc_symbol, gene.sift_prediction or '-']))
    return sift_predictions

  @property
  def polyphen_predictions(self):
    """Return a list with the polyphen prediction(s) for this variant. The most severe for each gene."""
    polyphen_predictions = []
    if len(self.genes) == 1:
      polyphen_predictions = [gene.polyphen_prediction for gene in self.genes]
    else:
      for gene in self.genes:
        polyphen_predictions.append(':'.join([gene.hgnc_symbol, gene.polyphen_prediction or '-']))
    return polyphen_predictions

  @property
  def functional_annotations(self):
    """Return a list with the functional annotation(s) for this variant. The most severe for each gene."""
    functional_annotations = []
    if len(self.genes) == 1:
      functional_annotations = [gene.functional_annotation for gene in self.genes]
    else:
      for gene in self.genes:
        functional_annotations.append(':'.join([gene.hgnc_symbol, gene.functional_annotation or '']))
    return functional_annotations

  @property
  def region_annotations(self):
    """Returns a list with region annotation(s)."""
    region_annotations = []
    if len(self.genes) == 1:
      return [gene.region_annotation for gene in self.genes]
    else:
      for gene in self.genes:
        region_annotations.append(':'.join([gene.hgnc_symbol, gene.region_annotation or '']))
    return region_annotations

  @property
  def transcripts(self):
    """Yield all transcripts as a flat iterator.

    For each transcript both the parent gene object as well as the
    transcript is yielded.

    Yields:
      class, class: Gene and Transcript ODM
    """
    # loop over each gene in order
    for gene in self.genes:
      # loop over each child transcript for the gene
      for transcript in gene.transcripts:
        # yield the parent gene, child transcript combo
        yield transcript

  @property
  def end_position(self):
    # which alternative allele contains most bases
    alt_bases = max([len(alt) for alt in self.alternatives])
    # vs. reference allele
    bases = max(len(self.reference), alt_bases)

    return self.position + (bases - 1)

  @property
  def frequency(self):
    """Returns a judgement on the overall frequency of the variant.

    Combines multiple metrics into a single call.
    """
    most_common_frequency = max(self.thousand_genomes_frequency,
                                self.exac_frequency)

    if most_common_frequency > .05:
      return 'common'

    elif most_common_frequency > .01:
      return 'uncommon'

    else:
      return 'rare'

  def __unicode__(self):
    return self.display_name
<|endoftext|>


<commit_before># Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module containing classes related to CloudStack disks.

Disks can be created, deleted, attached to VMs, and detached from VMs.
"""

from perfkitbenchmarker import disk
from perfkitbenchmarker import vm_util
from perfkitbenchmarker import flags
from perfkitbenchmarker.cloudstack import util

import string
import logging

FLAGS = flags.FLAGS

class CloudStackDisk(disk.BaseDisk):
  """Object representing a Cloudstack Disk."""

  def __init__(self, disk_spec, name, zone_id, project_id=None):
    super(CloudStackDisk, self).__init__(disk_spec)

    self.cs = util.CsClient(
        FLAGS.CS_API_URL,
        FLAGS.CS_API_KEY,
        FLAGS.CS_API_SECRET
    )

    self.attached_vm_name = None
    self.attached_vm_id = None
    self.name = name

    self.zone_id = zone_id
    self.project_id = project_id

    self.disk_offering_id = self._GetBestOfferingId(self.disk_size)
    assert self.disk_offering_id, "Unable get disk offering of given size"

    if self.disk_spec.disk_type:
        logging.warn("Cloudstack does not support disk types")

  @vm_util.Retry(max_retries=3)
  def _Create(self):
    """Creates the disk."""

    volume = self.cs.create_volume(self.name,
                                   self.disk_offering_id,
                                   self.zone_id,
                                   self.project_id)

    assert volume, "Unable to create volume"

    self.volume_id = volume['id']
    self.disk_type = volume['type']
    self.actual_disk_size = int(volume['size']) / (2 ** 30)  # In GB

  def _Delete(self):
    """Deletes the disk."""
    vol = self.cs.get_volume(self.name)
    if vol:
        self.cs.delete_volume(self.volume_id)

  def _Exists(self):
    """Returns true if the disk exists."""
    vol = self.cs.get_volume(self.name, self.project_id)
    if vol:
        return True
    return False

  @vm_util.Retry(max_retries=3)
  def Attach(self, vm):
    """Attaches the disk to a VM.

    Args:
      vm: The CloudStackVirtualMachine instance to which
      the disk will be attached.

    """

    res = self.cs.attach_volume(self.volume_id, vm.id)
    assert res, "Unable to attach volume"

    self.device_id = res['deviceid']

    self.device_path = "/dev/xvd" + \
        str(string.ascii_lowercase[self.device_id])

  def Detach(self):
    """Detaches the disk from a VM."""

    self.cs.detach_volume(self.volume_id)

  def _GetBestOfferingId(self, disk_size):
    """ Given a disk_size (in GB), try to find a disk
    offering that is atleast as big as the requested
    one.
    """

    disk_offerings = self.cs.list_disk_offerings()
    sorted_do = sorted(disk_offerings, key=lambda x: x['disksize'])

    for do in sorted_do:
        if int(do['disksize']) >= disk_size:
            return do['id']

    return None
<commit_msg>Pass project_id when deleting volume<commit_after># Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module containing classes related to CloudStack disks.

Disks can be created, deleted, attached to VMs, and detached from VMs.
"""

from perfkitbenchmarker import disk
from perfkitbenchmarker import vm_util
from perfkitbenchmarker import flags
from perfkitbenchmarker.cloudstack import util

import string
import logging

FLAGS = flags.FLAGS

class CloudStackDisk(disk.BaseDisk):
  """Object representing a Cloudstack Disk."""

  def __init__(self, disk_spec, name, zone_id, project_id=None):
    super(CloudStackDisk, self).__init__(disk_spec)

    self.cs = util.CsClient(
        FLAGS.CS_API_URL,
        FLAGS.CS_API_KEY,
        FLAGS.CS_API_SECRET
    )

    self.attached_vm_name = None
    self.attached_vm_id = None
    self.name = name

    self.zone_id = zone_id
    self.project_id = project_id

    self.disk_offering_id = self._GetBestOfferingId(self.disk_size)
    assert self.disk_offering_id, "Unable get disk offering of given size"

    if disk_spec.disk_type:
        logging.warn("Cloudstack does not support disk types")

  @vm_util.Retry(max_retries=3)
  def _Create(self):
    """Creates the disk."""

    volume = self.cs.create_volume(self.name,
                                   self.disk_offering_id,
                                   self.zone_id,
                                   self.project_id)

    assert volume, "Unable to create volume"

    self.volume_id = volume['id']
    self.disk_type = volume['type']
    self.actual_disk_size = int(volume['size']) / (2 ** 30)  # In GB

  def _Delete(self):
    """Deletes the disk."""
    vol = self.cs.get_volume(self.name, self.project_id)
    if vol:
        self.cs.delete_volume(self.volume_id)

  def _Exists(self):
    """Returns true if the disk exists."""
    vol = self.cs.get_volume(self.name, self.project_id)
    if vol:
        return True
    return False

  @vm_util.Retry(max_retries=3)
  def Attach(self, vm):
    """Attaches the disk to a VM.

    Args:
      vm: The CloudStackVirtualMachine instance to which
      the disk will be attached.

    """

    res = self.cs.attach_volume(self.volume_id, vm.id)
    assert res, "Unable to attach volume"

    self.device_id = res['deviceid']

    self.device_path = "/dev/xvd" + \
        str(string.ascii_lowercase[self.device_id])

  def Detach(self):
    """Detaches the disk from a VM."""

    self.cs.detach_volume(self.volume_id)

  def _GetBestOfferingId(self, disk_size):
    """ Given a disk_size (in GB), try to find a disk
    offering that is atleast as big as the requested
    one.
    """

    disk_offerings = self.cs.list_disk_offerings()
    sorted_do = sorted(disk_offerings, key=lambda x: x['disksize'])

    for do in sorted_do:
        if int(do['disksize']) >= disk_size:
            return do['id']

    return None
<|endoftext|>


<commit_before>"""
.. _tut-filter-resample:

Filtering and resampling data
=============================

Some artifacts are restricted to certain frequencies and can therefore
be fixed by filtering. An artifact that typically affects only some
frequencies is due to the power line.

Power-line noise is a noise created by the electrical network.
It is composed of sharp peaks at 50Hz (or 60Hz depending on your
geographical location). Some peaks may also be present at the harmonic
frequencies, i.e. the integer multiples of
the power-line frequency, e.g. 100Hz, 150Hz, ... (or 120Hz, 180Hz, ...).

This tutorial covers some basics of how to filter data in MNE-Python.
For more in-depth information about filter design in general and in
MNE-Python in particular, check out
:ref:`disc-filtering`.
"""

import numpy as np
import mne
from mne.datasets import sample

data_path = sample.data_path()
raw_fname = data_path + '/MEG/sample/sample_audvis_raw.fif'
proj_fname = data_path + '/MEG/sample/sample_audvis_eog_proj.fif'

tmin, tmax = 0, 20  # use the first 20s of data

# Setup for reading the raw data (save memory by cropping the raw data
# before loading it)
raw = mne.io.read_raw_fif(raw_fname)
raw.crop(tmin, tmax).load_data()
raw.info['bads'] = ['MEG 2443', 'EEG 053']  # bads + 2 more

fmin, fmax = 2, 300  # look at frequencies between 2 and 300Hz
n_fft = 2048  # the FFT size (n_fft). Ideally a power of 2

# Pick a subset of channels (here for speed reasons)
selection = mne.read_selection('Left-temporal')
picks = mne.pick_types(raw.info, meg='mag', eeg=False, eog=False,
                       stim=False, exclude='bads', selection=selection)
raw.plot_psd(area_mode='range', tmax=10.0, picks=picks, average=False)

###############################################################################
# Removing power-line noise with notch filtering
# ----------------------------------------------
#
# Removing power-line noise can be done with a Notch filter, directly on the
# Raw object, specifying an array of frequency to be cut off:

raw.notch_filter(np.arange(60, 241, 60), picks=picks, fir_design='firwin')
raw.plot_psd(area_mode='range', tmax=10.0, picks=picks, average=False)

###############################################################################
# Removing power-line noise with low-pass filtering
# -------------------------------------------------
#
# If you're only interested in low frequencies, below the peaks of power-line
# noise you can simply low pass filter the data.

# low pass filtering below 50 Hz
raw.filter(None, 50., fir_design='firwin')
raw.plot_psd(area_mode='range', tmax=10.0, picks=picks, average=False)

###############################################################################
# High-pass filtering to remove slow drifts
# -----------------------------------------
#
# To remove slow drifts, you can high pass.
#
# .. warning:: In several applications such as event-related potential (ERP)
#              and event-related field (ERF) analysis, high-pass filters with
#              cutoff frequencies greater than 0.1 Hz are usually considered
#              problematic since they significantly change the shape of the
#              resulting averaged waveform (see examples in
#              :ref:`tut_filtering_hp_problems`). In such applications, apply
#              high-pass filters with caution.

raw.filter(1., None, fir_design='firwin')
raw.plot_psd(area_mode='range', tmax=10.0, picks=picks, average=False)

###############################################################################
# To do the low-pass and high-pass filtering in one step you can do
# a so-called *band-pass* filter by running the following:

# band-pass filtering in the range 1 Hz - 50 Hz
raw.filter(1, 50., fir_design='firwin')

###############################################################################
# Downsampling and decimation
# ---------------------------
#
# When performing experiments where timing is critical, a signal with a high
# sampling rate is desired. However, having a signal with a much higher
# sampling rate than necessary needlessly consumes memory and slows down
# computations operating on the data. To avoid that, you can downsample
# your time series. Since downsampling raw data reduces the timing precision
# of events, it is recommended only for use in procedures that do not require
# optimal precision, e.g. computing EOG or ECG projectors on long recordings.
#
# .. note:: A *downsampling* operation performs a low-pass (to prevent
#           aliasing) followed by *decimation*, which selects every
#           :math:`N^{th}` sample from the signal. See
#           :func:`scipy.signal.resample` and
#           :func:`scipy.signal.resample_poly` for examples.
#
# Data resampling can be done with *resample* methods.

raw.resample(100, npad="auto")  # set sampling frequency to 100Hz
raw.plot_psd(area_mode='range', tmax=10.0, picks=picks, average=True)

###############################################################################
# To avoid this reduction in precision, the suggested pipeline for
# processing final data to be analyzed is:
#
#    1. low-pass the data with :meth:`mne.io.Raw.filter`.
#    2. Extract epochs with :class:`mne.Epochs`.
#    3. Decimate the Epochs object using :meth:`mne.Epochs.decimate` or the
#       ``decim`` argument to the :class:`mne.Epochs` object.
#
# We also provide the convenience methods :meth:`mne.Epochs.resample` and
# :meth:`mne.Evoked.resample` to downsample or upsample data, but these are
# less optimal because they will introduce edge artifacts into every epoch,
# whereas filtering the raw data will only introduce edge artifacts only at
# the start and end of the recording.
<commit_msg>DOC: update filtering & resampling tutorial (#6872)<commit_after># -*- coding: utf-8 -*-
"""
.. _tut-filter-resample:

Filtering and resampling data
=============================

This tutorial covers filtering and resampling, and gives examples of how
filtering can be used for artifact repair.

.. contents:: Page contents
   :local:
   :depth: 2

We begin as always by importing the necessary Python modules and loading some
:ref:`example data <sample-dataset>`. We'll also crop the data to 60 seconds
(to save memory on the documentation server):
"""

import os
import numpy as np
import matplotlib.pyplot as plt
import mne

sample_data_folder = mne.datasets.sample.data_path()
sample_data_raw_file = os.path.join(sample_data_folder, 'MEG', 'sample',
                                    'sample_audvis_raw.fif')
raw = mne.io.read_raw_fif(sample_data_raw_file)
raw.crop(0, 60).load_data()  # use just 60 seconds of data, to save memory

###############################################################################
# Background on filtering
# ^^^^^^^^^^^^^^^^^^^^^^^
#
# A filter removes or suppresses parts of a signal. Usually, filters act on
# specific *frequency ranges* of a signal — for example, suppressing all
# frequency components above or below a certain cutoff value. There are *many*
# ways of designing digital filters; see :ref:`disc-filtering` for a longer
# discussion of the various approaches to filtering physiological signals in
# MNE-Python.
#
#
# Repairing artifacts by filtering
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
#
# Artifacts that are restricted to a narrow range of frequencies can sometimes
# be repaired by filtering the data. Two examples of frequency-restricted
# artifacts are slow drifts and power line noise. Here we illustrate how each
# of these can be repaired by filtering.
#
#
# Slow drifts
# ~~~~~~~~~~~
#
# Low-frequency drifts in raw data can usually be spotted by plotting a fairly
# long span of data with the :meth:`~mne.io.Raw.plot` method, though it is
# helpful to disable channel-wise DC shift correction to make slow drifts
# more readily visible. Here we plot 60 seconds, showing all the magnetometer
# channels:

mag_channels = mne.pick_types(raw.info, meg='mag')
raw.plot(duration=60, order=mag_channels, proj=False,
         n_channels=len(mag_channels), remove_dc=False)

###############################################################################
# A half-period of this slow drift appears to last around 10 seconds, so a full
# period would be 20 seconds, i.e., :math:`\frac{1}{20} \mathrm{Hz}`. To be
# sure those components are excluded, we want our highpass to be *higher* than
# that, so let's try :math:`\frac{1}{10} \mathrm{Hz}` and :math:`\frac{1}{5}
# \mathrm{Hz}` filters to see which works best:

for cutoff in (0.1, 0.2):
    raw_highpass = raw.copy().filter(l_freq=cutoff, h_freq=None)
    fig = raw_highpass.plot(duration=60, order=mag_channels, proj=False,
                            n_channels=len(mag_channels), remove_dc=False)
    fig.subplots_adjust(top=0.95)
    fig.suptitle('High-pass filtered at {} Hz'.format(cutoff), size='xx-large',
                 weight='bold')

###############################################################################
# Looks like 0.1 Hz was not quite high enough to fully remove the slow drifts.
# Notice that the text output summarizes the relevant characteristics of the
# filter that was created. If you want to visualize the filter, you can pass
# the same arguments used in the call to :meth:`raw.filter()
# <mne.io.Raw.filter>` above to the function :func:`mne.filter.create_filter`
# to get the filter parameters, and then pass the filter parameters to
# :func:`mne.viz.plot_filter`. :func:`~mne.filter.create_filter` also requires
# parameters ``data`` (a :class:`NumPy array <numpy.ndarray>`) and ``sfreq``
# (the sampling frequency of the data), so we'll extract those from our
# :class:`~mne.io.Raw` object:

filter_params = mne.filter.create_filter(raw.get_data(), raw.info['sfreq'],
                                         l_freq=0.2, h_freq=None)

###############################################################################
# Notice that the output is the same as when we applied this filter to the data
# using :meth:`raw.filter() <mne.io.Raw.filter>`. You can now pass the filter
# parameters (and the sampling frequency) to :func:`~mne.viz.plot_filter` to
# plot the filter:

mne.viz.plot_filter(filter_params, raw.info['sfreq'], flim=(0.01, 5))

###############################################################################
# Power line noise
# ~~~~~~~~~~~~~~~~
#
# Power line noise is an environmental artifact that manifests as persistent
# oscillations centered around the `AC power line frequency`_. Power line
# artifacts are easiest to see on plots of the spectrum, so we'll use
# :meth:`~mne.io.Raw.plot_psd` to illustrate. We'll also write a little
# function that adds arrows to the spectrum plot to highlight the artifacts:

def add_arrows(axes):
    # add some arrows at 60 Hz and its harmonics
    for ax in axes:
        freqs = ax.lines[-1].get_xdata()
        psds = ax.lines[-1].get_ydata()
        for freq in (60, 120, 180, 240):
            idx = np.searchsorted(freqs, freq)
            # get ymax of a small region around the freq. of interest
            y = psds[(idx - 4):(idx + 5)].max()
            ax.arrow(x=freqs[idx], y=y + 18, dx=0, dy=-12, color='red',
                     width=0.1, head_width=3, length_includes_head=True)

fig = raw.plot_psd(fmax=250, average=True)
add_arrows(fig.axes[:2])

###############################################################################
# It should be evident that MEG channels are more susceptible to this kind of
# interference. Removing power-line noise can be done with a notch filter,
# applied directly to the :class:`~mne.io.Raw` object, specifying an array of
# frequencies to be attenuated. Since the EEG channels are relatively
# unaffected by the power line noise, we'll also specify a ``picks`` argument
# so that only the magnetometers and gradiometers get filtered:

meg_picks = mne.pick_types(raw.info)  # meg=True, eeg=False are the defaults
freqs = (60, 120, 180, 240)
raw_notch = raw.copy().notch_filter(freqs=freqs, picks=meg_picks)
for title, data in zip(['Un', 'Notch '], [raw, raw_notch]):
    fig = data.plot_psd(fmax=250, average=True)
    fig.subplots_adjust(top=0.85)
    fig.suptitle('{}filtered'.format(title), size='xx-large', weight='bold')
    add_arrows(fig.axes[:2])

###############################################################################
# :meth:`~mne.io.Raw.notch_filter` also has parameters to control the notch
# width, transition bandwidth and other aspects of the filter. See the
# docstring for details.
#
#
# Resampling
# ^^^^^^^^^^
#
# EEG and MEG recordings are notable for their high temporal precision, and are
# often recorded with sampling rates around 1000 Hz or higher. This is good
# when precise timing of events is important to the experimental design or
# analysis plan, but also consumes more memory and computational resources when
# processing the data. In cases where high-frequency components of the signal
# are not of interest and precise timing is not needed (e.g., computing EOG or
# ECG projectors on a long recording), downsampling the signal can be a useful
# time-saver.
#
# In MNE-Python, downsampling effectively applies a low-pass filter to the
# signal to avoid `aliasing`_, so you don't need to explicitly filter it
# yourself before using the :meth:`~mne.io.Raw.resample` method. The effect is
# most clearly seen in the PSD plot, where a dashed vertical line indicates the
# filter cutoff; the original data had an existing lowpass at around 172 Hz
# (see ``raw.info['lowpass']``), and the data resampled to 200 Hz gets
# automatically lowpass filtered at 100 Hz:

raw_downsampled = raw.copy().resample(sfreq=200)

for data, title in zip([raw, raw_downsampled], ['Original', 'Downsampled']):
    fig = data.plot_psd(average=True)
    fig.subplots_adjust(top=0.9)
    fig.suptitle(title)
    plt.setp(fig.axes, xlim=(0, 300))

###############################################################################
# In addition to :meth:`raw.resample() <mne.io.Raw.resample>` there are also
# methods :meth:`epochs.resample() <mne.Epochs.resample>` and
# :meth:`evoked.resample() <mne.Evoked.resample>`. However, because resampling
# involves filtering, there are some pitfalls to resampling at different points
# in the analysis stream:
#
# - Performing resampling on :class:`~mne.io.Raw` data (*before* epoching) will
#   negatively affect the temporal precision of Event arrays, by causing
#   `jitter`_ in the event timing. This reduced temporal precision will
#   propagate to subsequent epoching operations.
#
# - Performing resampling *after* epoching can introduce edge artifacts *on
#   every epoch*, whereas filtering the :class:`~mne.io.Raw` object will only
#   introduce artifacts at the start and end of the recording (which is often
#   far enough from the first and last epochs to have no affect on the
#   analysis).
#
# The following section suggests best practices to mitigate both of these
# issues.
#
#
# Best practices
# ~~~~~~~~~~~~~~
#
# To avoid the reduction in temporal precision of events that comes with
# resampling a :class:`~mne.io.Raw` object, and also avoid the edge artifacts
# that come with filtering an :class:`~mne.Epochs` or :class:`~mne.Evoked`
# object, the best practice is to:
#
# 1. low-pass filter the :class:`~mne.io.Raw` data at or below
#    :math:`\frac{1}{3}` of the desired sample rate, then
#
# 2. decimate the data after epoching, by either passing the ``decim``
#    parameter to the :class:`~mne.Epochs` constructor, or using the
#    :meth:`~mne.Epochs.decimate` method after the :class:`~mne.Epochs` have
#    been created.
#
# Note that this method is exact only when the original sampling frequency is
# an integer multiple of the desired new sampling frequency. Since the sampling
# frequency of our example data is 600.614990234375 Hz, ending up with a
# specific sampling frequency like (say) 90 Hz will not be possible:

current_sfreq = raw.info['sfreq']
desired_sfreq = 90  # Hz
decim = np.round(current_sfreq / desired_sfreq).astype(int)
obtained_sfreq = current_sfreq / decim
lowpass_freq = obtained_sfreq / 3.

raw_filtered = raw.copy().filter(l_freq=None, h_freq=lowpass_freq)
events = mne.find_events(raw_filtered)
epochs = mne.Epochs(raw_filtered, events, decim=decim)

print('desired sampling frequency was {} Hz; decim factor of {} yielded an '
      'actual sampling frequency of {} Hz.'
      .format(desired_sfreq, decim, epochs.info['sfreq']))

###############################################################################
# .. warning::
#    The relationship :math:`\mathrm{lowpass~cutoff} \leq \frac{1}{3}~
#    \mathrm{new~sampling~frequency}` is a fairly safe rule of thumb based on
#    MNE-Python's default lowpass filter settings. If you use a customized
#    lowpass filter (specifically, if your transition bandwidth is wider than
#    0.5× the lowpass cutoff), downsampling to 3× the lowpass cutoff may still
#    not be enough to avoid `aliasing`_, and MNE-Python will not warn you about
#    it (because the :class:`raw.info <mne.Info>` object only keeps track of
#    the lowpass cutoff, not the transition bandwidth).
#
#
# If for some reason you cannot follow the above-recommended best practices,
# you should at the very least either:
#
# 1. resample the data *after* epoching, and make your epochs long enough that
#    edge effects from the filtering do not affect the temporal span of the
#    epoch that you hope to analyze / interpret; or
#
# 2. perform resampling on the :class:`~mne.io.Raw` object and its
#    corresponding Events array *simultaneously* so that they stay more or less
#    in synch. This can be done by passing the Events array as the
#    ``events`` parameter to :meth:`raw.resample() <mne.io.Raw.resample>`.
#
#
# .. LINKS
#
# .. _`AC power line frequency`:
#    https://en.wikipedia.org/wiki/Mains_electricity
# .. _`aliasing`: https://en.wikipedia.org/wiki/Anti-aliasing_filter
# .. _`jitter`: https://en.wikipedia.org/wiki/Jitter
<|endoftext|>


<commit_before>"""
syslog2irc.processor
~~~~~~~~~~~~~~~~~~~~

:Copyright: 2007-2021 Jochen Kupperschmidt
:License: MIT, see LICENSE for details.
"""

import logging
from time import sleep
from typing import Any, Callable, Iterator, Optional, Set, Tuple

from syslogmp import Message as SyslogMessage

from .network import Port
from .router import Router
from .signals import (
    irc_channel_joined,
    message_approved,
    message_received,
    syslog_message_received,
)

MESSAGE_TEXT_ENCODING = 'utf-8'

logger = logging.getLogger(__name__)

class Processor:
    def __init__(
        self,
        router: Router,
        syslog_message_formatter: Optional[
            Callable[[SyslogMessage], str]
        ] = None,
    ) -> None:
        super(Processor, self).__init__()
        self.router = router

        if syslog_message_formatter is not None:
            self.syslog_message_formatter = syslog_message_formatter
        else:
            self.syslog_message_formatter = format_syslog_message

    def connect_to_signals(self) -> None:
        irc_channel_joined.connect(self.router.enable_channel)
        syslog_message_received.connect(self.handle_syslog_message)
        message_received.connect(self.handle_message)

    def handle_syslog_message(
        self,
        port: Port,
        source_address: Optional[Tuple[str, int]] = None,
        message: Optional[SyslogMessage] = None,
    ) -> None:
        """Process an incoming syslog message."""
        channel_names = self.router.get_channel_names_for_port(port)

        formatted_source = f'{source_address[0]}:{source_address[1]:d}'
        formatted_message = self.syslog_message_formatter(message)
        text = f'{formatted_source} {formatted_message}'

        message_received.send(
            channel_names=channel_names,
            text=text,
            source_address=source_address,
        )

    def handle_message(
        self,
        sender: Any,
        *,
        channel_names: Optional[Set[str]] = None,
        text: Optional[str] = None,
        source_address: Optional[Tuple[str, int]] = None,
    ) -> None:
        """Process an incoming message."""
        for channel_name in channel_names:
            if self.router.is_channel_enabled(channel_name):
                message_approved.send(channel_name=channel_name, text=text)

    def run(self, seconds_to_sleep: float = 0.5) -> None:
        """Run the main loop."""
        try:
            while True:
                sleep(seconds_to_sleep)
        except KeyboardInterrupt:
            pass

        logger.info('Shutting down ...')

def format_syslog_message(message: SyslogMessage) -> str:
    """Format a syslog message to be displayed on IRC."""

    def _generate() -> Iterator[str]:
        if message.timestamp is not None:
            timestamp_format = '%Y-%m-%d %H:%M:%S'
            formatted_timestamp = message.timestamp.strftime(timestamp_format)
            yield f'[{formatted_timestamp}] '

        if message.hostname is not None:
            yield f'({message.hostname}) '

        severity_name = message.severity.name

        # Important: The message text is a byte string.
        message_text = message.message.decode(MESSAGE_TEXT_ENCODING)

        # Remove leading and trailing newlines. Those would result in
        # additional lines on IRC with the usual metadata but with an
        # empty message text.
        message_text = message_text.strip('\n')

        yield f'[{severity_name}]: {message_text}'

    return ''.join(_generate())
<commit_msg>Do not send source address in `message_received` event<commit_after>"""
syslog2irc.processor
~~~~~~~~~~~~~~~~~~~~

:Copyright: 2007-2021 Jochen Kupperschmidt
:License: MIT, see LICENSE for details.
"""

import logging
from time import sleep
from typing import Any, Callable, Iterator, Optional, Set, Tuple

from syslogmp import Message as SyslogMessage

from .network import Port
from .router import Router
from .signals import (
    irc_channel_joined,
    message_approved,
    message_received,
    syslog_message_received,
)

MESSAGE_TEXT_ENCODING = 'utf-8'

logger = logging.getLogger(__name__)

class Processor:
    def __init__(
        self,
        router: Router,
        syslog_message_formatter: Optional[
            Callable[[SyslogMessage], str]
        ] = None,
    ) -> None:
        super(Processor, self).__init__()
        self.router = router

        if syslog_message_formatter is not None:
            self.syslog_message_formatter = syslog_message_formatter
        else:
            self.syslog_message_formatter = format_syslog_message

    def connect_to_signals(self) -> None:
        irc_channel_joined.connect(self.router.enable_channel)
        syslog_message_received.connect(self.handle_syslog_message)
        message_received.connect(self.handle_message)

    def handle_syslog_message(
        self,
        port: Port,
        source_address: Optional[Tuple[str, int]] = None,
        message: Optional[SyslogMessage] = None,
    ) -> None:
        """Process an incoming syslog message."""
        channel_names = self.router.get_channel_names_for_port(port)

        formatted_source = f'{source_address[0]}:{source_address[1]:d}'
        formatted_message = self.syslog_message_formatter(message)
        text = f'{formatted_source} {formatted_message}'

        message_received.send(channel_names=channel_names, text=text)

    def handle_message(
        self,
        sender: Any,
        *,
        channel_names: Optional[Set[str]] = None,
        text: Optional[str] = None,
    ) -> None:
        """Process an incoming message."""
        for channel_name in channel_names:
            if self.router.is_channel_enabled(channel_name):
                message_approved.send(channel_name=channel_name, text=text)

    def run(self, seconds_to_sleep: float = 0.5) -> None:
        """Run the main loop."""
        try:
            while True:
                sleep(seconds_to_sleep)
        except KeyboardInterrupt:
            pass

        logger.info('Shutting down ...')

def format_syslog_message(message: SyslogMessage) -> str:
    """Format a syslog message to be displayed on IRC."""

    def _generate() -> Iterator[str]:
        if message.timestamp is not None:
            timestamp_format = '%Y-%m-%d %H:%M:%S'
            formatted_timestamp = message.timestamp.strftime(timestamp_format)
            yield f'[{formatted_timestamp}] '

        if message.hostname is not None:
            yield f'({message.hostname}) '

        severity_name = message.severity.name

        # Important: The message text is a byte string.
        message_text = message.message.decode(MESSAGE_TEXT_ENCODING)

        # Remove leading and trailing newlines. Those would result in
        # additional lines on IRC with the usual metadata but with an
        # empty message text.
        message_text = message_text.strip('\n')

        yield f'[{severity_name}]: {message_text}'

    return ''.join(_generate())
<|endoftext|>


<commit_before>from __future__ import unicode_literals

"""
SQLite natively supports only the types TEXT, INTEGER, REAL, BLOB and NULL.
And RQLite always answers 'bytes' values.

Converters transforms RQLite answers to Python native types.
Adapters transforms Python native types to RQLite-aware values.
"""

import binascii
import datetime
import functools
import numbers
import re
import sqlite3

from .exceptions import InterfaceError

PARSE_DECLTYPES = 1
PARSE_COLNAMES = 2

def _decoder(conv_func):
    """ The Python sqlite3 interface returns always byte strings.
        This function converts the received value to a regular string before
        passing it to the receiver function.
    """
    return lambda s: conv_func(s.decode('utf-8'))

def _adapt_date(val):
    return val.isoformat()

def _adapt_datetime(val):
    return val.isoformat(b" ")

def _convert_date(val):
    return datetime.date(*map(int, val.split('T')[0].split("-")))

def _convert_timestamp(val):
    datepart, timepart = val.split("T")
    year, month, day = map(int, datepart.split("-"))
    timepart_full = timepart.strip('Z').split(".")
    hours, minutes, seconds = map(int, timepart_full[0].split(":"))
    if len(timepart_full) == 2:
        microseconds = int('{:0<6.6}'.format(timepart_full[1].decode()))
    else:
        microseconds = 0

    val = datetime.datetime(year, month, day, hours, minutes, seconds, microseconds)
    return val

adapters = {
    float: lambda x: x,
    int: lambda x: x,
    bool: int,
    unicode: lambda x: x.encode('utf-8'),
    type(None): lambda x: None,
    datetime.date: _adapt_date,
    datetime.datetime: _adapt_datetime,

}
adapters = {(type_, sqlite3.PrepareProtocol): val for type_, val in adapters.items()}

converters = {
    'UNICODE': lambda x: x.decode('utf-8'),
    'INTEGER': int,
    'BOOL': bool,
    'FLOAT': float,
    'REAL': float,
    'NULL': lambda x: None,
    'BLOB': lambda x: x,
    'DATE': _convert_date,
    'TIMESTAMP': _convert_timestamp,
    # For PRAGMA results, the go-sqlite3 SQLiteRows.DeclTypes method
    # returns empty strings for all of the types, but the rows can
    # contain int or None values that need to pass through here.
    '': lambda x: x.encode('utf-8') if hasattr(x, 'encode') else x,
}

# Non-native converters will be decoded from base64 before fed into converter
_native_converters = ('BOOL', 'FLOAT', 'INTEGER', 'REAL', 'NUMBER', 'NULL', 'DATE', 'TIMESTAMP', '')

# SQLite TEXT affinity: https://www.sqlite.org/datatype3.html
_text_affinity_re = re.compile(r'CHAR|CLOB|TEXT')

def register_converter(type_string, function):
    converters[type_string.upper()] = function

def register_adapter(type_, function):
    adapters[(type_, sqlite3.PrepareProtocol)] = function

def _convert_to_python(column_name, type_, parse_decltypes=False, parse_colnames=False):
    """
    Tries to mimic stock sqlite3 module behaviours.

    PARSE_COLNAMES have precedence over PARSE_DECLTYPES on _sqlite/cursor.c code
    """
    converter = None
    type_upper = None

    if type_ == '':     # q="select 3.0" -> type='' column_name='3.0' value=3
        if column_name.isdigit():
            type_ = 'int'
        elif all([slice.isdigit() for slice in column_name.partition('.')[::2]]):   # 3.14
            type_ = 'real'

    if '[' in column_name and ']' in column_name and parse_colnames:
        type_upper = column_name.upper().partition('[')[-1].partition(']')[0]
        if type_upper in converters:
            converter = converters[type_upper]

    if not converter:
        type_upper = type_.upper()
        if parse_decltypes:
            ## From: https://github.com/python/cpython/blob/c72b6008e0578e334f962ee298279a23ba298856/Modules/_sqlite/cursor.c#L167
            # /* Converter names are split at '(' and blanks.
            #  * This allows 'INTEGER NOT NULL' to be treated as 'INTEGER' and
            #  * 'NUMBER(10)' to be treated as 'NUMBER', for example.
            #  * In other words, it will work as people expect it to work.*/
            type_upper = type_upper.partition('(')[0].partition(' ')[0]
        if type_upper in converters:
            if type_upper in _native_converters or parse_decltypes:
                converter = converters[type_upper]

    if converter:
        if type_upper not in _native_converters:
            converter = functools.partial(_decode_base64_converter, converter)
    elif _text_affinity_re.search(type_upper):
        # Python's sqlite3 module has a text_factory attribute which
        # returns unicode by default.
        pass
    else:
        converter = _conditional_string_decode_base64

    return converter

def _adapt_from_python(value):
    if not isinstance(value, basestring):
        try:
            adapted = adapters[(type(value), sqlite3.PrepareProtocol)](value)
        except KeyError, e:
            # No adapter registered. Let the object adapt itself via PEP-246.
            # It has been rejected by the BDFL, but is still implemented
            # on stdlib sqlite3 module even on Python 3 !!
            if hasattr(value, '__adapt__'):
                adapted = value.__adapt__(sqlite3.PrepareProtocol)
            elif hasattr(value, '__conform__'):
                adapted = value.__conform__(sqlite3.PrepareProtocol)
            else:
                raise InterfaceError(e)
    else:
        adapted = value

    # The adapter could had returned a string
    if isinstance(adapted, basestring):
        adapted = _escape_string(adapted)
    elif adapted is None:
        adapted = 'NULL'
    else:
        adapted = str(adapted)

    return adapted

def _escape_string(value):
    return (type(value)(b"'%s'")) % (value.replace(b"'", b"''"))

def _column_stripper(column_name, parse_colnames=False):
    return column_name.partition(' ')[0] if parse_colnames else column_name

def _decode_base64_converter(converter, value):
    return converter(value.decode('base64'))

def _conditional_string_decode_base64(value):
    if isinstance(value, basestring):
        value = value.decode('base64')
    return value
<commit_msg>extensions: minimal python3 compat<commit_after>from __future__ import unicode_literals

"""
SQLite natively supports only the types TEXT, INTEGER, REAL, BLOB and NULL.
And RQLite always answers 'bytes' values.

Converters transforms RQLite answers to Python native types.
Adapters transforms Python native types to RQLite-aware values.
"""

import binascii
import datetime
import functools
import numbers
import re
import sqlite3
import sys

from .exceptions import InterfaceError

if sys.version_info[0] >= 3:
    basestring = str
    unicode = str

PARSE_DECLTYPES = 1
PARSE_COLNAMES = 2

def _decoder(conv_func):
    """ The Python sqlite3 interface returns always byte strings.
        This function converts the received value to a regular string before
        passing it to the receiver function.
    """
    return lambda s: conv_func(s.decode('utf-8'))

def _adapt_date(val):
    return val.isoformat()

def _adapt_datetime(val):
    return val.isoformat(b" ")

def _convert_date(val):
    return datetime.date(*map(int, val.split('T')[0].split("-")))

def _convert_timestamp(val):
    datepart, timepart = val.split("T")
    year, month, day = map(int, datepart.split("-"))
    timepart_full = timepart.strip('Z').split(".")
    hours, minutes, seconds = map(int, timepart_full[0].split(":"))
    if len(timepart_full) == 2:
        microseconds = int('{:0<6.6}'.format(timepart_full[1].decode()))
    else:
        microseconds = 0

    val = datetime.datetime(year, month, day, hours, minutes, seconds, microseconds)
    return val

adapters = {
    float: lambda x: x,
    int: lambda x: x,
    bool: int,
    unicode: lambda x: x.encode('utf-8'),
    type(None): lambda x: None,
    datetime.date: _adapt_date,
    datetime.datetime: _adapt_datetime,

}
adapters = {(type_, sqlite3.PrepareProtocol): val for type_, val in adapters.items()}

converters = {
    'UNICODE': lambda x: x.decode('utf-8'),
    'INTEGER': int,
    'BOOL': bool,
    'FLOAT': float,
    'REAL': float,
    'NULL': lambda x: None,
    'BLOB': lambda x: x,
    'DATE': _convert_date,
    'TIMESTAMP': _convert_timestamp,
    # For PRAGMA results, the go-sqlite3 SQLiteRows.DeclTypes method
    # returns empty strings for all of the types, but the rows can
    # contain int or None values that need to pass through here.
    '': lambda x: x.encode('utf-8') if hasattr(x, 'encode') else x,
}

# Non-native converters will be decoded from base64 before fed into converter
_native_converters = ('BOOL', 'FLOAT', 'INTEGER', 'REAL', 'NUMBER', 'NULL', 'DATE', 'TIMESTAMP', '')

# SQLite TEXT affinity: https://www.sqlite.org/datatype3.html
_text_affinity_re = re.compile(r'CHAR|CLOB|TEXT')

def register_converter(type_string, function):
    converters[type_string.upper()] = function

def register_adapter(type_, function):
    adapters[(type_, sqlite3.PrepareProtocol)] = function

def _convert_to_python(column_name, type_, parse_decltypes=False, parse_colnames=False):
    """
    Tries to mimic stock sqlite3 module behaviours.

    PARSE_COLNAMES have precedence over PARSE_DECLTYPES on _sqlite/cursor.c code
    """
    converter = None
    type_upper = None

    if type_ == '':     # q="select 3.0" -> type='' column_name='3.0' value=3
        if column_name.isdigit():
            type_ = 'int'
        elif all([slice.isdigit() for slice in column_name.partition('.')[::2]]):   # 3.14
            type_ = 'real'

    if '[' in column_name and ']' in column_name and parse_colnames:
        type_upper = column_name.upper().partition('[')[-1].partition(']')[0]
        if type_upper in converters:
            converter = converters[type_upper]

    if not converter:
        type_upper = type_.upper()
        if parse_decltypes:
            ## From: https://github.com/python/cpython/blob/c72b6008e0578e334f962ee298279a23ba298856/Modules/_sqlite/cursor.c#L167
            # /* Converter names are split at '(' and blanks.
            #  * This allows 'INTEGER NOT NULL' to be treated as 'INTEGER' and
            #  * 'NUMBER(10)' to be treated as 'NUMBER', for example.
            #  * In other words, it will work as people expect it to work.*/
            type_upper = type_upper.partition('(')[0].partition(' ')[0]
        if type_upper in converters:
            if type_upper in _native_converters or parse_decltypes:
                converter = converters[type_upper]

    if converter:
        if type_upper not in _native_converters:
            converter = functools.partial(_decode_base64_converter, converter)
    elif _text_affinity_re.search(type_upper):
        # Python's sqlite3 module has a text_factory attribute which
        # returns unicode by default.
        pass
    else:
        converter = _conditional_string_decode_base64

    return converter

def _adapt_from_python(value):
    if not isinstance(value, basestring):
        try:
            adapted = adapters[(type(value), sqlite3.PrepareProtocol)](value)
        except KeyError as e:
            # No adapter registered. Let the object adapt itself via PEP-246.
            # It has been rejected by the BDFL, but is still implemented
            # on stdlib sqlite3 module even on Python 3 !!
            if hasattr(value, '__adapt__'):
                adapted = value.__adapt__(sqlite3.PrepareProtocol)
            elif hasattr(value, '__conform__'):
                adapted = value.__conform__(sqlite3.PrepareProtocol)
            else:
                raise InterfaceError(e)
    else:
        adapted = value

    # The adapter could had returned a string
    if isinstance(adapted, basestring):
        adapted = _escape_string(adapted)
    elif adapted is None:
        adapted = 'NULL'
    else:
        adapted = str(adapted)

    return adapted

def _escape_string(value):
    return (type(value)(b"'%s'")) % (value.replace(b"'", b"''"))

def _column_stripper(column_name, parse_colnames=False):
    return column_name.partition(' ')[0] if parse_colnames else column_name

def _decode_base64_converter(converter, value):
    return converter(value.decode('base64'))

def _conditional_string_decode_base64(value):
    if isinstance(value, basestring):
        value = value.decode('base64')
    return value
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import weakref
import struct
import re
import sys

from cymysql.err import Warning, Error, InterfaceError, DataError, \
             DatabaseError, OperationalError, IntegrityError, InternalError, \
            NotSupportedError, ProgrammingError

PYTHON3 = sys.version_info[0] > 2

class Cursor(object):
    '''
    This is the object you use to interact with the database.
    '''
    def __init__(self, connection):
        '''
        Do not create an instance of a Cursor yourself. Call
        connections.Connection.cursor().
        '''
        self.connection = connection
        self.arraysize = 1
        self._executed = None
        self.messages = []
        self._result = None

    def __del__(self):
        '''
        When this gets GC'd close it.
        '''
        self.close()

    def errorhandler(self, errorclass, errorvalue):
        if self.connection:
            self.connection.errorhandler(self, errorclass, errorvalue)
        else:
            raise errorclass(errorvalue)

    @property
    def rowcount(self):
        if self._result and self._result.affected_rows is not None:
            return self._result.affected_rows
        return -1

    @property
    def description(self):
        return self._result.description if self._result else None

    @property
    def lastrowid(self):
        return self._result.insert_id if self._result else None

    def close(self):
        '''
        Closing a cursor just exhausts all remaining data.
        '''
        if not self.connection:
            return
        try:
            while self.nextset():
                pass
        except:
            pass

        self.connection = None

    def _get_db(self):
        if not self.connection:
            self.errorhandler(ProgrammingError, (-1, "cursor closed"))
        return self.connection

    def _check_executed(self):
        if not self._executed:
            self.errorhandler(ProgrammingError, (-1, "execute() first"))

    def _flush(self):
        if self._result:
            self._result.read_rest_rowdata_packet()

    def setinputsizes(self, *args):
        """Does nothing, required by DB API."""

    def setoutputsizes(self, *args):
        """Does nothing, required by DB API."""

    def nextset(self):
        ''' Get the next query set '''
        if self._executed:
            self.fetchall()
        del self.messages[:]

        if not self._result or not self._result.has_next:
            return None
        connection = self._get_db()
        connection.next_result()
        self._do_get_result()
        return True

    def execute(self, query, args=None):
        ''' Execute a query '''
        from sys import exc_info

        conn = self._get_db()
        if hasattr(conn, '_last_execute_cursor') and not conn._last_execute_cursor() is None:
            conn._last_execute_cursor()._flush()

        charset = conn.charset
        del self.messages[:]

        if PYTHON3 and (not isinstance(query, str)):
            query = query.decode(charset)
        if (not PYTHON3) and isinstance(query, unicode):
            query = query.encode(charset)

        if args is not None:
            if isinstance(args, tuple) or isinstance(args, list):
                escaped_args = tuple(conn.escape(arg) for arg in args)
            elif isinstance(args, dict):
                escaped_args = dict((key, conn.escape(val)) for (key, val) in args.items())
            else:
                #If it's not a dictionary let's try escaping it anyways.
                #Worst case it will throw a Value error
                escaped_args = conn.escape(args)

            query = query % escaped_args

        try:
            self._query(query)
        except:
            exc, value, tb = exc_info()
            del tb
            self.messages.append((exc,value))
            self.errorhandler(exc, value)

        self._executed = query
        conn._last_execute_cursor = weakref.ref(self)

    def executemany(self, query, args):
        ''' Run several data against one query '''
        del self.messages[:]

        rowcount = 0
        for params in args:
            self.execute(query, params)
            if self.rowcount != -1:
                rowcount += self.rowcount
        self._result = None
        return rowcount

    def callproc(self, procname, args=()):
        """Execute stored procedure procname with args

        procname -- string, name of procedure to execute on server

        args -- Sequence of parameters to use with procedure

        Returns the original args.

        Compatibility warning: PEP-249 specifies that any modified
        parameters must be returned. This is currently impossible
        as they are only available by storing them in a server
        variable and then retrieved by a query. Since stored
        procedures return zero or more result sets, there is no
        reliable way to get at OUT or INOUT parameters via callproc.
        The server variables are named @_procname_n, where procname
        is the parameter above and n is the position of the parameter
        (from zero). Once all result sets generated by the procedure
        have been fetched, you can issue a SELECT @_procname_0, ...
        query using .execute() to get any OUT or INOUT values.

        Compatibility warning: The act of calling a stored procedure
        itself creates an empty result set. This appears after any
        result sets generated by the procedure. This is non-standard
        behavior with respect to the DB-API. Be sure to use nextset()
        to advance through all result sets; otherwise you may get
        disconnected.
        """
        conn = self._get_db()
        for index, arg in enumerate(args):
            q = "SET @_%s_%d=%s" % (procname, index, conn.escape(arg))
            if isinstance(q, unicode):
                q = q.encode(conn.charset)
            self._query(q)
            self.nextset()

        q = "CALL %s(%s)" % (procname,
                             ','.join(['@_%s_%d' % (procname, i)
                                       for i in range(len(args))]))
        if isinstance(q, unicode):
            q = q.encode(conn.charset)
        self._query(q)
        self._executed = q

        return args

    def fetchone(self):
        ''' Fetch the next row '''
        self._check_executed()
        if self._result is None:
            return None
        return self._result.fetchone()

    def fetchmany(self, size=None):
        ''' Fetch several rows '''
        self._check_executed()
        size = size or self.arraysize
        if self._result is None:
            return None
        result = []
        for i in range(size):
            r = self._result.fetchone()
            if not r:
                break
            result.append(r)
        return result

    def fetchall(self):
        ''' Fetch all the rows '''
        self._check_executed()
        if self._result is None:
            return None
        result = []

        r = self._result.fetchone()
        while r:
            result.append(r)
            r = self._result.fetchone()

        return result

    def _query(self, q):
        conn = self._get_db()
        self._last_executed = q
        conn.query(q)
        self._do_get_result()

    def _do_get_result(self):
        conn = self._get_db()
        self._result = conn._result

    def __iter__(self):
        return iter(self.fetchone, None)

    Warning = Warning
    Error = Error
    InterfaceError = InterfaceError
    DatabaseError = DatabaseError
    DataError = DataError
    OperationalError = OperationalError
    IntegrityError = IntegrityError
    InternalError = InternalError
    ProgrammingError = ProgrammingError
    NotSupportedError = NotSupportedError

class DictCursor(Cursor):
    """A cursor which returns results as a dictionary"""

    def execute(self, query, args=None):
        result = super(DictCursor, self).execute(query, args)
        if self.description:
            self._fields = [ field[0] for field in self.description ]
        return result

    def fetchone(self):
        ''' Fetch the next row '''
        self._check_executed()
        if self._result is None:
            return None
        r = super(DictCursor, self).fetchone()
        if not r:
            return None
        return  dict(zip(self._fields, r))

    def fetchmany(self, size=None):
        ''' Fetch several rows '''
        self._check_executed()
        if self._result is None:
            return None
        result = [ dict(zip(self._fields, r)) for r in super(DictCursor, self).fetchmany(size)]
        return tuple(result)

    def fetchall(self):
        ''' Fetch all the rows '''
        self._check_executed()
        if self._result is None:
            return None
        result = [ dict(zip(self._fields, r)) for r in super(DictCursor, self).fetchall() ]
        return tuple(result)

<commit_msg>remove no use import<commit_after># -*- coding: utf-8 -*-
import weakref
import struct
import sys

from cymysql.err import Warning, Error, InterfaceError, DataError, \
             DatabaseError, OperationalError, IntegrityError, InternalError, \
            NotSupportedError, ProgrammingError

PYTHON3 = sys.version_info[0] > 2

class Cursor(object):
    '''
    This is the object you use to interact with the database.
    '''
    def __init__(self, connection):
        '''
        Do not create an instance of a Cursor yourself. Call
        connections.Connection.cursor().
        '''
        self.connection = connection
        self.arraysize = 1
        self._executed = None
        self.messages = []
        self._result = None

    def __del__(self):
        '''
        When this gets GC'd close it.
        '''
        self.close()

    def errorhandler(self, errorclass, errorvalue):
        if self.connection:
            self.connection.errorhandler(self, errorclass, errorvalue)
        else:
            raise errorclass(errorvalue)

    @property
    def rowcount(self):
        if self._result and self._result.affected_rows is not None:
            return self._result.affected_rows
        return -1

    @property
    def description(self):
        return self._result.description if self._result else None

    @property
    def lastrowid(self):
        return self._result.insert_id if self._result else None

    def close(self):
        '''
        Closing a cursor just exhausts all remaining data.
        '''
        if not self.connection:
            return
        try:
            while self.nextset():
                pass
        except:
            pass

        self.connection = None

    def _get_db(self):
        if not self.connection:
            self.errorhandler(ProgrammingError, (-1, "cursor closed"))
        return self.connection

    def _check_executed(self):
        if not self._executed:
            self.errorhandler(ProgrammingError, (-1, "execute() first"))

    def _flush(self):
        if self._result:
            self._result.read_rest_rowdata_packet()

    def setinputsizes(self, *args):
        """Does nothing, required by DB API."""

    def setoutputsizes(self, *args):
        """Does nothing, required by DB API."""

    def nextset(self):
        ''' Get the next query set '''
        if self._executed:
            self.fetchall()
        del self.messages[:]

        if not self._result or not self._result.has_next:
            return None
        connection = self._get_db()
        connection.next_result()
        self._do_get_result()
        return True

    def execute(self, query, args=None):
        ''' Execute a query '''
        from sys import exc_info

        conn = self._get_db()
        if hasattr(conn, '_last_execute_cursor') and not conn._last_execute_cursor() is None:
            conn._last_execute_cursor()._flush()

        charset = conn.charset
        del self.messages[:]

        if PYTHON3 and (not isinstance(query, str)):
            query = query.decode(charset)
        if (not PYTHON3) and isinstance(query, unicode):
            query = query.encode(charset)

        if args is not None:
            if isinstance(args, tuple) or isinstance(args, list):
                escaped_args = tuple(conn.escape(arg) for arg in args)
            elif isinstance(args, dict):
                escaped_args = dict((key, conn.escape(val)) for (key, val) in args.items())
            else:
                #If it's not a dictionary let's try escaping it anyways.
                #Worst case it will throw a Value error
                escaped_args = conn.escape(args)

            query = query % escaped_args

        try:
            self._query(query)
        except:
            exc, value, tb = exc_info()
            del tb
            self.messages.append((exc,value))
            self.errorhandler(exc, value)

        self._executed = query
        conn._last_execute_cursor = weakref.ref(self)

    def executemany(self, query, args):
        ''' Run several data against one query '''
        del self.messages[:]

        rowcount = 0
        for params in args:
            self.execute(query, params)
            if self.rowcount != -1:
                rowcount += self.rowcount
        self._result = None
        return rowcount

    def callproc(self, procname, args=()):
        """Execute stored procedure procname with args

        procname -- string, name of procedure to execute on server

        args -- Sequence of parameters to use with procedure

        Returns the original args.

        Compatibility warning: PEP-249 specifies that any modified
        parameters must be returned. This is currently impossible
        as they are only available by storing them in a server
        variable and then retrieved by a query. Since stored
        procedures return zero or more result sets, there is no
        reliable way to get at OUT or INOUT parameters via callproc.
        The server variables are named @_procname_n, where procname
        is the parameter above and n is the position of the parameter
        (from zero). Once all result sets generated by the procedure
        have been fetched, you can issue a SELECT @_procname_0, ...
        query using .execute() to get any OUT or INOUT values.

        Compatibility warning: The act of calling a stored procedure
        itself creates an empty result set. This appears after any
        result sets generated by the procedure. This is non-standard
        behavior with respect to the DB-API. Be sure to use nextset()
        to advance through all result sets; otherwise you may get
        disconnected.
        """
        conn = self._get_db()
        for index, arg in enumerate(args):
            q = "SET @_%s_%d=%s" % (procname, index, conn.escape(arg))
            if isinstance(q, unicode):
                q = q.encode(conn.charset)
            self._query(q)
            self.nextset()

        q = "CALL %s(%s)" % (procname,
                             ','.join(['@_%s_%d' % (procname, i)
                                       for i in range(len(args))]))
        if isinstance(q, unicode):
            q = q.encode(conn.charset)
        self._query(q)
        self._executed = q

        return args

    def fetchone(self):
        ''' Fetch the next row '''
        self._check_executed()
        if self._result is None:
            return None
        return self._result.fetchone()

    def fetchmany(self, size=None):
        ''' Fetch several rows '''
        self._check_executed()
        size = size or self.arraysize
        if self._result is None:
            return None
        result = []
        for i in range(size):
            r = self._result.fetchone()
            if not r:
                break
            result.append(r)
        return result

    def fetchall(self):
        ''' Fetch all the rows '''
        self._check_executed()
        if self._result is None:
            return None
        result = []

        r = self._result.fetchone()
        while r:
            result.append(r)
            r = self._result.fetchone()

        return result

    def _query(self, q):
        conn = self._get_db()
        self._last_executed = q
        conn.query(q)
        self._do_get_result()

    def _do_get_result(self):
        conn = self._get_db()
        self._result = conn._result

    def __iter__(self):
        return iter(self.fetchone, None)

    Warning = Warning
    Error = Error
    InterfaceError = InterfaceError
    DatabaseError = DatabaseError
    DataError = DataError
    OperationalError = OperationalError
    IntegrityError = IntegrityError
    InternalError = InternalError
    ProgrammingError = ProgrammingError
    NotSupportedError = NotSupportedError

class DictCursor(Cursor):
    """A cursor which returns results as a dictionary"""

    def execute(self, query, args=None):
        result = super(DictCursor, self).execute(query, args)
        if self.description:
            self._fields = [ field[0] for field in self.description ]
        return result

    def fetchone(self):
        ''' Fetch the next row '''
        self._check_executed()
        if self._result is None:
            return None
        r = super(DictCursor, self).fetchone()
        if not r:
            return None
        return  dict(zip(self._fields, r))

    def fetchmany(self, size=None):
        ''' Fetch several rows '''
        self._check_executed()
        if self._result is None:
            return None
        result = [ dict(zip(self._fields, r)) for r in super(DictCursor, self).fetchmany(size)]
        return tuple(result)

    def fetchall(self):
        ''' Fetch all the rows '''
        self._check_executed()
        if self._result is None:
            return None
        result = [ dict(zip(self._fields, r)) for r in super(DictCursor, self).fetchall() ]
        return tuple(result)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import datetime
import pytest

from bs4 import BeautifulSoup

from django.template import Template, Context

from components.correlations.models import Correlation
from components.people.factories import GroupFactory

<commit_msg>Flake dat.<commit_after># -*- coding: utf-8 -*-
<|endoftext|>


<commit_before># Copyright 2014 PerfKitBenchmarker Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module containing build tools installation and cleanup functions."""
import logging
from absl import flags
from perfkitbenchmarker import os_types

FLAGS = flags.FLAGS
flags.DEFINE_string('gcc_version', None, 'Version of gcc to use. Benchmarks '
                    'that utilize gcc compilation should ensure reinstallation '
                    'of GCC. Default is set by the OS package manager.')
flags.DEFINE_boolean('force_build_gcc_from_source', False, 'Whether to force '
                     'building GCC from source.')
flags.DEFINE_boolean('build_fortran', False, 'Whether to build fortran '
                     'alongside c and c++ when building GCC.')

def YumInstall(vm):
  """Installs build tools on the VM."""
  vm.InstallPackageGroup('Development Tools')
  if FLAGS.gcc_version:
    Reinstall(vm, version=FLAGS.gcc_version)

def AptInstall(vm):
  """Installs build tools on the VM."""
  vm.InstallPackages('build-essential git libtool autoconf automake')
  if FLAGS.gcc_version:
    Reinstall(vm, version=FLAGS.gcc_version)

def BuildGccFromSource(vm, gcc_version):
  """Install a specific version of gcc by compiling from source.

  Args:
    vm: VirtualMachine object.
    gcc_version: string. GCC version.

  Taken from: https://gist.github.com/nchaigne/ad06bc867f911a3c0d32939f1e930a11
  """
  if gcc_version == '9' or gcc_version == '9.2':
    gcc_version = '9.2.0'
  logging.info('Compiling GCC %s', gcc_version)

  # build GCC on scratch disks for speed if possible
  build_dir = vm.GetScratchDir() if vm.scratch_disks else 'build_tools'
  vm.RemoteCommand(f'cd {build_dir} && wget https://ftp.gnu.org/gnu/gcc/gcc-'
                   f'{gcc_version}/gcc-{gcc_version}.tar.gz')
  vm.RemoteCommand(f'cd {build_dir} && tar xzvf gcc-{gcc_version}.tar.gz')
  vm.RemoteCommand(f'cd {build_dir} && mkdir -p obj.gcc-{gcc_version}')
  vm.RemoteCommand(f'cd {build_dir}/gcc-{gcc_version} && '
                   './contrib/download_prerequisites')
  enable_languages = 'c,c++' + ',fortran' if FLAGS.build_fortran else ''
  vm.RemoteCommand(f'cd {build_dir}/obj.gcc-{gcc_version} && '
                   f'../gcc-{gcc_version}/configure '
                   f'--disable-multilib --enable-languages={enable_languages}')
  # TODO(user): Measure GCC compilation time as a benchmark.
  vm.RemoteCommand(f'cd {build_dir}/obj.gcc-{gcc_version} && '
                   f'time make -j {vm.NumCpusForBenchmark()}')
  vm.RemoteCommand(f'cd {build_dir}/obj.gcc-{gcc_version} && sudo make install')
  vm.RemoteCommand('sudo rm -rf /usr/bin/gcc && '
                   'sudo ln -s /usr/local/bin/gcc /usr/bin/gcc')
  vm.RemoteCommand('sudo rm -rf /usr/bin/g++ && '
                   'sudo ln -s /usr/local/bin/g++ /usr/bin/g++')
  if FLAGS.build_fortran:
    vm.RemoteCommand('sudo rm -rf /usr/bin/gfortran && '
                     'sudo ln -s /usr/local/bin/gfortran /usr/bin/gfortran')

  if '11' in gcc_version:
    # https://stackoverflow.com/a/65384705
    vm.RemoteCommand(
        f'sudo cp {build_dir}/obj.gcc-{gcc_version}/x86_64-pc-linux-gnu/'
        'libstdc++-v3/src/.libs/* /usr/lib/x86_64-linux-gnu/',
        ignore_failure=True)
    vm.RemoteCommand(
        f'sudo cp {build_dir}/obj.gcc-{gcc_version}/aarch64-unknown-linux-gnu/'
        'libstdc++-v3/src/.libs/* /usr/lib/aarch64-linux-gnu/',
        ignore_failure=True)

def GetVersion(vm, pkg):
  """Get version of package using -dumpversion."""
  out, _ = vm.RemoteCommand(
      '{pkg} -dumpversion'.format(pkg=pkg), ignore_failure=True)
  return out.rstrip()

def GetVersionInfo(vm, pkg):
  """Get compiler version info for package using --version."""
  out, _ = vm.RemoteCommand(
      '{pkg} --version'.format(pkg=pkg), ignore_failure=True)
  # return first line of pkg --version
  return out.splitlines()[0] if out else None

def Reinstall(vm, version='4.7'):
  """Install specific version of gcc.

  Args:
    vm: VirtualMachine object.
    version: string. GCC version.
  """
  if vm.BASE_OS_TYPE != os_types.DEBIAN or FLAGS.force_build_gcc_from_source:
    BuildGccFromSource(vm, version)
    logging.info('GCC info: %s', GetVersion(vm, 'gcc'))
    return
  vm.Install('ubuntu_toolchain')
  for pkg in ('gcc', 'gfortran', 'g++'):
    version_string = GetVersion(vm, pkg)
    if version in version_string:
      logging.info('Have expected version of %s: %s', pkg, version_string)
      continue
    else:
      new_pkg = pkg + '-' + version
      vm.InstallPackages(new_pkg)
      vm.RemoteCommand('sudo rm -f /usr/bin/{pkg}'.format(pkg=pkg))
      vm.RemoteCommand('sudo ln -s /usr/bin/{new_pkg} /usr/bin/{pkg}'.format(
          new_pkg=new_pkg, pkg=pkg))
      logging.info('Updated version of %s: Old: %s New: %s', pkg,
                   version_string, GetVersion(vm, pkg))
<commit_msg>Correctly set language support in SpecCPU based on fortran flag.<commit_after># Copyright 2014 PerfKitBenchmarker Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module containing build tools installation and cleanup functions."""
import logging
from absl import flags
from perfkitbenchmarker import os_types

FLAGS = flags.FLAGS
flags.DEFINE_string('gcc_version', None, 'Version of gcc to use. Benchmarks '
                    'that utilize gcc compilation should ensure reinstallation '
                    'of GCC. Default is set by the OS package manager.')
flags.DEFINE_boolean('force_build_gcc_from_source', False, 'Whether to force '
                     'building GCC from source.')
flags.DEFINE_boolean('build_fortran', False, 'Whether to build fortran '
                     'alongside c and c++ when building GCC.')

def YumInstall(vm):
  """Installs build tools on the VM."""
  vm.InstallPackageGroup('Development Tools')
  if FLAGS.gcc_version:
    Reinstall(vm, version=FLAGS.gcc_version)

def AptInstall(vm):
  """Installs build tools on the VM."""
  vm.InstallPackages('build-essential git libtool autoconf automake')
  if FLAGS.gcc_version:
    Reinstall(vm, version=FLAGS.gcc_version)

def BuildGccFromSource(vm, gcc_version):
  """Install a specific version of gcc by compiling from source.

  Args:
    vm: VirtualMachine object.
    gcc_version: string. GCC version.

  Taken from: https://gist.github.com/nchaigne/ad06bc867f911a3c0d32939f1e930a11
  """
  if gcc_version == '9' or gcc_version == '9.2':
    gcc_version = '9.2.0'
  logging.info('Compiling GCC %s', gcc_version)

  # build GCC on scratch disks for speed if possible
  build_dir = vm.GetScratchDir() if vm.scratch_disks else 'build_tools'
  vm.RemoteCommand(f'cd {build_dir} && wget https://ftp.gnu.org/gnu/gcc/gcc-'
                   f'{gcc_version}/gcc-{gcc_version}.tar.gz')
  vm.RemoteCommand(f'cd {build_dir} && tar xzvf gcc-{gcc_version}.tar.gz')
  vm.RemoteCommand(f'cd {build_dir} && mkdir -p obj.gcc-{gcc_version}')
  vm.RemoteCommand(f'cd {build_dir}/gcc-{gcc_version} && '
                   './contrib/download_prerequisites')
  enable_languages = 'c,c++' + (',fortran' if FLAGS.build_fortran else '')
  vm.RemoteCommand(f'cd {build_dir}/obj.gcc-{gcc_version} && '
                   f'../gcc-{gcc_version}/configure '
                   f'--disable-multilib --enable-languages={enable_languages}')
  # TODO(user): Measure GCC compilation time as a benchmark.
  vm.RemoteCommand(f'cd {build_dir}/obj.gcc-{gcc_version} && '
                   f'time make -j {vm.NumCpusForBenchmark()}')
  vm.RemoteCommand(f'cd {build_dir}/obj.gcc-{gcc_version} && sudo make install')
  vm.RemoteCommand('sudo rm -rf /usr/bin/gcc && '
                   'sudo ln -s /usr/local/bin/gcc /usr/bin/gcc')
  vm.RemoteCommand('sudo rm -rf /usr/bin/g++ && '
                   'sudo ln -s /usr/local/bin/g++ /usr/bin/g++')
  if FLAGS.build_fortran:
    vm.RemoteCommand('sudo rm -rf /usr/bin/gfortran && '
                     'sudo ln -s /usr/local/bin/gfortran /usr/bin/gfortran')

  if '11' in gcc_version:
    # https://stackoverflow.com/a/65384705
    vm.RemoteCommand(
        f'sudo cp {build_dir}/obj.gcc-{gcc_version}/x86_64-pc-linux-gnu/'
        'libstdc++-v3/src/.libs/* /usr/lib/x86_64-linux-gnu/',
        ignore_failure=True)
    vm.RemoteCommand(
        f'sudo cp {build_dir}/obj.gcc-{gcc_version}/aarch64-unknown-linux-gnu/'
        'libstdc++-v3/src/.libs/* /usr/lib/aarch64-linux-gnu/',
        ignore_failure=True)

def GetVersion(vm, pkg):
  """Get version of package using -dumpversion."""
  out, _ = vm.RemoteCommand(
      '{pkg} -dumpversion'.format(pkg=pkg), ignore_failure=True)
  return out.rstrip()

def GetVersionInfo(vm, pkg):
  """Get compiler version info for package using --version."""
  out, _ = vm.RemoteCommand(
      '{pkg} --version'.format(pkg=pkg), ignore_failure=True)
  # return first line of pkg --version
  return out.splitlines()[0] if out else None

def Reinstall(vm, version='4.7'):
  """Install specific version of gcc.

  Args:
    vm: VirtualMachine object.
    version: string. GCC version.
  """
  if vm.BASE_OS_TYPE != os_types.DEBIAN or FLAGS.force_build_gcc_from_source:
    BuildGccFromSource(vm, version)
    logging.info('GCC info: %s', GetVersion(vm, 'gcc'))
    return
  vm.Install('ubuntu_toolchain')
  for pkg in ('gcc', 'gfortran', 'g++'):
    version_string = GetVersion(vm, pkg)
    if version in version_string:
      logging.info('Have expected version of %s: %s', pkg, version_string)
      continue
    else:
      new_pkg = pkg + '-' + version
      vm.InstallPackages(new_pkg)
      vm.RemoteCommand('sudo rm -f /usr/bin/{pkg}'.format(pkg=pkg))
      vm.RemoteCommand('sudo ln -s /usr/bin/{new_pkg} /usr/bin/{pkg}'.format(
          new_pkg=new_pkg, pkg=pkg))
      logging.info('Updated version of %s: Old: %s New: %s', pkg,
                   version_string, GetVersion(vm, pkg))
<|endoftext|>


<commit_before>__author__ = 'sarath'
<commit_msg>twitter API handler<commit_after>import os

__author__ = 'sarath'

import ujson
import time
import sys
from __builtin__ import file

sys.path.append(os.path.join(os.path.dirname(__file__), '..'))

from domain.Listing import Listing

from TwitterAPI import TwitterAPI, TwitterRequestError

class TwitterSearch:
    def __init__(self):
        self.api = TwitterAPI('9bV6PsfoyLOr2EBL4JInYBVsz',
                              'gYBmmAdd1KWZuMgzI9i8U5wgeBvojvyZNXnXISqi3C3UFcqEan',
                              '3287471448-PgnQvj1GqbBJ8GaVR85XjBE6jQP0qvvo1MJOWKo',
                              'LKgpSjSZwptVoQ0QAB4rtl0DpUijLz8JJBhoN2EtqZl9C')

    def search_show(self, listing):
        results = self.api.request('users/search', {'q': listing.show_name, 'count': 3})
        count = 1
        for twitter_msg in results:
            if self.is_verified(twitter_msg) and count <= 3:
                return self.save_listing_data(twitter_msg)
        time.sleep(1)
        return {}

    def is_verified(self, twitter_msg):
        return 'verified' in twitter_msg and twitter_msg['verified']

    def save_listing_data(self, twitter_msg):
        fans = twitter_msg['followers_count']
        verified = twitter_msg['verified']
        id = twitter_msg['id']
        screen_name = twitter_msg['screen_name']
        return {'fans': fans, 'verified': verified, 'id': id, 'screen_name': screen_name}

if __name__ == '__main__':
    search = TwitterSearch()
    listing_file = file(sys.argv[1], 'r')
    scores = open(sys.argv[2], 'a')
    for line in listing_file:
        value = line.strip()
        print(value)
        listing = Listing(value)
        try:
            score = search.search_show(listing)
        except TwitterRequestError:
            print('wait for 15 min', listing)
            time.sleep(15 * 60)
            score = search.search_show(listing)
        scores.write(value + '\t' + ujson.dumps(score) + "\n")
    scores.close()<|endoftext|>