<commit_before>from django.utils import timezone

from django.conf import settings
from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _
from django.db import models
from django.db.models.signals import post_save, post_delete
from django.dispatch import receiver
from django.template.defaultfilters import slugify

from cms import utils, tasks
from cms.git.models import GitPage, GitCategory
from gitmodel import exceptions

ENG_UK = 'eng_UK'
SWH_TZ = 'swh_TZ'  # Swahili
SWH_KE = 'swh_KE'  # Swahili
THA_TH = 'tha_TH'  # Thai
IND_ID = 'ind_ID'  # Bahasa

LANGUAGE_CHOICES = (
    (ENG_UK, 'English (United Kingdom)'),
    (SWH_TZ, 'Swahili (Tanzania)'),
    (SWH_KE, 'Swahili (Kenya)'),
    (THA_TH, 'Thai (Thailand)'),
    (IND_ID, 'Bahasa (Indonesia)')
)

class Category(models.Model):
    """
    Category model to be used for categorization of content. Categories are
    high level constructs to be used for grouping and organizing content,
    thus creating a site's table of contents.
    """
    uuid = models.CharField(
        max_length=32,
        blank=True,
        null=True,
        unique=True,
        db_index=True,
        editable=False)
    title = models.CharField(
        max_length=200,
        help_text='Short descriptive name for this category.',
    )
    subtitle = models.CharField(
        max_length=200,
        blank=True,
        null=True,
        default='',
        help_text='Some titles may be the same and cause confusion in admin '
                  'UI. A subtitle makes a distinction.',
    )
    slug = models.SlugField(
        max_length=255,
        db_index=True,
        help_text='Short descriptive unique name for use in urls.',
    )
    last_author = models.ForeignKey(
        User,
        blank=True,
        null=True,
        related_name='category_last_author'
    )
    language = models.CharField(
        max_length=6,
        blank=True,
        null=True,
        choices=LANGUAGE_CHOICES)
    source = models.ForeignKey(
        'self',
        blank=True,
        null=True)
    featured_in_navbar = models.BooleanField(
        _('Featured in navigation bar'),
        default=False,
        help_text=_(
            'If checked this category will be displayed on the top'
            'navigation bar. It will always appear on the homepage.'))

    class Meta:
        ordering = ('title',)
        verbose_name = 'category'
        verbose_name_plural = 'categories'

    def __unicode__(self):
        return self.title

    def save(self, *args, **kwargs):
        # use django slugify filter to slugify
        if not self.slug:
            self.slug = slugify(self.title)

        super(Category, self).save(*args, **kwargs)

class Post(models.Model):

    class Meta:
        ordering = ('-created_at',)

    uuid = models.CharField(
        max_length=32,
        blank=True,
        null=True,
        unique=True,
        db_index=True,
        editable=False)
    title = models.CharField(
        _("Title"),
        max_length=200, help_text=_('A short descriptive title.'),
    )
    subtitle = models.CharField(
        max_length=200,
        blank=True,
        null=True,
        default='',
        help_text=_(
            'Some titles may be the same and cause confusion in admin UI. '
            'A subtitle makes a distinction.'),
    )
    slug = models.SlugField(
        max_length=255,
        db_index=True,
        help_text='Short descriptive unique name for use in urls.',
    )
    description = models.TextField(
        help_text=_(
            'A short description. More verbose than the title but'
            'limited to one or two sentences.'),
        blank=True,
        null=True,
    )
    content = models.TextField(blank=True, null=True)
    created_at = models.DateTimeField(
        _('Created Date & Time'),
        blank=True,
        db_index=True,
        help_text=_(
            'Date and time on which this item was created. This is'
            'automatically set on creation, but can be changed subsequently.')
    )
    modified_at = models.DateTimeField(
        _('Modified Date & Time'),
        db_index=True,
        editable=False,
        auto_now=True,
        help_text=_(
            'Date and time on which this item was last modified. This'
            'is automatically set each time the item is saved.')
    )
    owner = models.ForeignKey(
        User,
        blank=True,
        null=True,
    )
    last_author = models.ForeignKey(
        User,
        blank=True,
        null=True,
        related_name='post_last_author'
    )
    primary_category = models.ForeignKey(
        Category,
        blank=True,
        null=True,
        related_name="primary_modelbase_set",
    )

    featured = models.BooleanField(
        _('Feature on homepage'),
        default=False,
        help_text=_(
            'If checked this post will be displayed on the homepage.'))

    featured_in_category = models.BooleanField(
        _('Feature in category post listing'),
        default=False,
        help_text=_(
            'If checked this post will be displayed in the category\'s '
            'list of featured posts.'))

    language = models.CharField(
        max_length=6,
        blank=True,
        null=True,
        choices=LANGUAGE_CHOICES)
    source = models.ForeignKey(
        'self',
        blank=True,
        null=True)

    def save(self, *args, **kwargs):
        # use django slugify filter to slugify
        if not self.slug:
            self.slug = slugify(self.title)

        # set created time to now if not already set.
        if not self.created_at:
            self.created_at = timezone.now()

        super(Post, self).save(*args, **kwargs)

    def __unicode__(self):
        if self.subtitle:
            return '%s - %s' % (self.title, self.subtitle)
        else:
            return self.title

def celery_push_to_git():
    if hasattr(settings, 'SSH_PUBKEY_PATH') and hasattr(
            settings, 'SSH_PRIVKEY_PATH'):
        tasks.celery_push_to_git.delay(
            settings.GIT_REPO_PATH,
            settings.SSH_PUBKEY_PATH,
            settings.SSH_PRIVKEY_PATH,
            settings.SSH_PASSPHRASE)

@receiver(post_save, sender=Post)
def auto_save_post_to_git(sender, instance, created, **kwargs):
    def update_fields(page, post):
        page.title = instance.title
        page.subtitle = instance.subtitle
        page.slug = instance.slug
        page.description = instance.description
        page.content = instance.content
        page.created_at = instance.created_at
        page.modified_at = instance.modified_at
        page.language = instance.language
        page.featured_in_category = instance.featured_in_category
        page.featured = instance.featured

        if instance.primary_category:
            category = GitCategory.get(instance.primary_category.uuid)
            page.primary_category = category

        if instance.source:
            source = GitPage.get(instance.source.uuid)
            page.source = source

    author = utils.get_author_from_user(instance.last_author)

    if created:
        page = GitPage()
        update_fields(page, instance)
        page.save(
            True, message='Page created: %s' % instance.title,
            author=author)

        # store the page's uuid on the Post instance without triggering `save`
        Post.objects.filter(pk=instance.pk).update(uuid=page.uuid)
    else:
        try:
            page = GitPage.get(instance.uuid)
            update_fields(page, instance)
            page.save(
                True, message='Page updated: %s' % instance.title,
                author=author)
        except exceptions.DoesNotExist:
            page = GitPage()
            update_fields(page, instance)
            page.save(
                True, message='Page re-created: %s' % instance.title,
                author=author)
            Post.objects.filter(pk=instance.pk).update(uuid=page.uuid)

    utils.sync_repo()
    celery_push_to_git()

@receiver(post_delete, sender=Post)
def auto_delete_post_to_git(sender, instance, **kwargs):
    author = utils.get_author_from_user(instance.last_author)
    GitPage.delete(
        instance.uuid, True, message='Page deleted: %s' % instance.title,
        author=author)
    utils.sync_repo()
    celery_push_to_git()

@receiver(post_save, sender=Category)
def auto_save_category_to_git(sender, instance, created, **kwargs):
    def update_fields(category, post):
        category.title = instance.title
        category.subtitle = instance.subtitle
        category.slug = instance.slug
        category.language = instance.language
        category.featured_in_navbar = instance.featured_in_navbar

        if instance.source and instance.uuid:
            source = GitCategory.get(instance.source.uuid)
            category.source = source

    author = utils.get_author_from_user(instance.last_author)

    if created:
        category = GitCategory()
        update_fields(category, instance)
        category.save(
            True, message='Category created: %s' % instance.title,
            author=author)

        # store the page's uuid on the Post instance without triggering `save`
        Category.objects.filter(pk=instance.pk).update(uuid=category.uuid)
    else:
        try:
            category = GitCategory.get(instance.uuid)
            update_fields(category, instance)
            category.save(
                True, message='Category updated: %s' % instance.title,
                author=author)
        except exceptions.DoesNotExist:
            category = GitCategory()
            update_fields(category, instance)
            category.save(
                True, message='Category re-updated: %s' % instance.title,
                author=author)
            Category.objects.filter(pk=instance.pk).update(uuid=category.uuid)

    utils.sync_repo()
    celery_push_to_git()

@receiver(post_delete, sender=Category)
def auto_delete_category_to_git(sender, instance, **kwargs):
    author = utils.get_author_from_user(instance.last_author)
    GitCategory.delete(
        instance.uuid, True, message='Category deleted: %s' % instance.title,
        author=author)
    utils.sync_repo()
    celery_push_to_git()
<commit_msg>fix copy-replace error<commit_after>from django.utils import timezone

from django.conf import settings
from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _
from django.db import models
from django.db.models.signals import post_save, post_delete
from django.dispatch import receiver
from django.template.defaultfilters import slugify

from cms import utils, tasks
from cms.git.models import GitPage, GitCategory
from gitmodel import exceptions

ENG_UK = 'eng_UK'
SWH_TZ = 'swh_TZ'  # Swahili
SWH_KE = 'swh_KE'  # Swahili
THA_TH = 'tha_TH'  # Thai
IND_ID = 'ind_ID'  # Bahasa

LANGUAGE_CHOICES = (
    (ENG_UK, 'English (United Kingdom)'),
    (SWH_TZ, 'Swahili (Tanzania)'),
    (SWH_KE, 'Swahili (Kenya)'),
    (THA_TH, 'Thai (Thailand)'),
    (IND_ID, 'Bahasa (Indonesia)')
)

class Category(models.Model):
    """
    Category model to be used for categorization of content. Categories are
    high level constructs to be used for grouping and organizing content,
    thus creating a site's table of contents.
    """
    uuid = models.CharField(
        max_length=32,
        blank=True,
        null=True,
        unique=True,
        db_index=True,
        editable=False)
    title = models.CharField(
        max_length=200,
        help_text='Short descriptive name for this category.',
    )
    subtitle = models.CharField(
        max_length=200,
        blank=True,
        null=True,
        default='',
        help_text='Some titles may be the same and cause confusion in admin '
                  'UI. A subtitle makes a distinction.',
    )
    slug = models.SlugField(
        max_length=255,
        db_index=True,
        help_text='Short descriptive unique name for use in urls.',
    )
    last_author = models.ForeignKey(
        User,
        blank=True,
        null=True,
        related_name='category_last_author'
    )
    language = models.CharField(
        max_length=6,
        blank=True,
        null=True,
        choices=LANGUAGE_CHOICES)
    source = models.ForeignKey(
        'self',
        blank=True,
        null=True)
    featured_in_navbar = models.BooleanField(
        _('Featured in navigation bar'),
        default=False,
        help_text=_(
            'If checked this category will be displayed on the top'
            'navigation bar. It will always appear on the homepage.'))

    class Meta:
        ordering = ('title',)
        verbose_name = 'category'
        verbose_name_plural = 'categories'

    def __unicode__(self):
        return self.title

    def save(self, *args, **kwargs):
        # use django slugify filter to slugify
        if not self.slug:
            self.slug = slugify(self.title)

        super(Category, self).save(*args, **kwargs)

class Post(models.Model):

    class Meta:
        ordering = ('-created_at',)

    uuid = models.CharField(
        max_length=32,
        blank=True,
        null=True,
        unique=True,
        db_index=True,
        editable=False)
    title = models.CharField(
        _("Title"),
        max_length=200, help_text=_('A short descriptive title.'),
    )
    subtitle = models.CharField(
        max_length=200,
        blank=True,
        null=True,
        default='',
        help_text=_(
            'Some titles may be the same and cause confusion in admin UI. '
            'A subtitle makes a distinction.'),
    )
    slug = models.SlugField(
        max_length=255,
        db_index=True,
        help_text='Short descriptive unique name for use in urls.',
    )
    description = models.TextField(
        help_text=_(
            'A short description. More verbose than the title but'
            'limited to one or two sentences.'),
        blank=True,
        null=True,
    )
    content = models.TextField(blank=True, null=True)
    created_at = models.DateTimeField(
        _('Created Date & Time'),
        blank=True,
        db_index=True,
        help_text=_(
            'Date and time on which this item was created. This is'
            'automatically set on creation, but can be changed subsequently.')
    )
    modified_at = models.DateTimeField(
        _('Modified Date & Time'),
        db_index=True,
        editable=False,
        auto_now=True,
        help_text=_(
            'Date and time on which this item was last modified. This'
            'is automatically set each time the item is saved.')
    )
    owner = models.ForeignKey(
        User,
        blank=True,
        null=True,
    )
    last_author = models.ForeignKey(
        User,
        blank=True,
        null=True,
        related_name='post_last_author'
    )
    primary_category = models.ForeignKey(
        Category,
        blank=True,
        null=True,
        related_name="primary_modelbase_set",
    )

    featured = models.BooleanField(
        _('Feature on homepage'),
        default=False,
        help_text=_(
            'If checked this post will be displayed on the homepage.'))

    featured_in_category = models.BooleanField(
        _('Feature in category post listing'),
        default=False,
        help_text=_(
            'If checked this post will be displayed in the category\'s '
            'list of featured posts.'))

    language = models.CharField(
        max_length=6,
        blank=True,
        null=True,
        choices=LANGUAGE_CHOICES)
    source = models.ForeignKey(
        'self',
        blank=True,
        null=True)

    def save(self, *args, **kwargs):
        # use django slugify filter to slugify
        if not self.slug:
            self.slug = slugify(self.title)

        # set created time to now if not already set.
        if not self.created_at:
            self.created_at = timezone.now()

        super(Post, self).save(*args, **kwargs)

    def __unicode__(self):
        if self.subtitle:
            return '%s - %s' % (self.title, self.subtitle)
        else:
            return self.title

def celery_push_to_git():
    if hasattr(settings, 'SSH_PUBKEY_PATH') and hasattr(
            settings, 'SSH_PRIVKEY_PATH'):
        tasks.push_to_git.delay(
            settings.GIT_REPO_PATH,
            settings.SSH_PUBKEY_PATH,
            settings.SSH_PRIVKEY_PATH,
            settings.SSH_PASSPHRASE)

@receiver(post_save, sender=Post)
def auto_save_post_to_git(sender, instance, created, **kwargs):
    def update_fields(page, post):
        page.title = instance.title
        page.subtitle = instance.subtitle
        page.slug = instance.slug
        page.description = instance.description
        page.content = instance.content
        page.created_at = instance.created_at
        page.modified_at = instance.modified_at
        page.language = instance.language
        page.featured_in_category = instance.featured_in_category
        page.featured = instance.featured

        if instance.primary_category:
            category = GitCategory.get(instance.primary_category.uuid)
            page.primary_category = category

        if instance.source:
            source = GitPage.get(instance.source.uuid)
            page.source = source

    author = utils.get_author_from_user(instance.last_author)

    if created:
        page = GitPage()
        update_fields(page, instance)
        page.save(
            True, message='Page created: %s' % instance.title,
            author=author)

        # store the page's uuid on the Post instance without triggering `save`
        Post.objects.filter(pk=instance.pk).update(uuid=page.uuid)
    else:
        try:
            page = GitPage.get(instance.uuid)
            update_fields(page, instance)
            page.save(
                True, message='Page updated: %s' % instance.title,
                author=author)
        except exceptions.DoesNotExist:
            page = GitPage()
            update_fields(page, instance)
            page.save(
                True, message='Page re-created: %s' % instance.title,
                author=author)
            Post.objects.filter(pk=instance.pk).update(uuid=page.uuid)

    utils.sync_repo()
    celery_push_to_git()

@receiver(post_delete, sender=Post)
def auto_delete_post_to_git(sender, instance, **kwargs):
    author = utils.get_author_from_user(instance.last_author)
    GitPage.delete(
        instance.uuid, True, message='Page deleted: %s' % instance.title,
        author=author)
    utils.sync_repo()
    celery_push_to_git()

@receiver(post_save, sender=Category)
def auto_save_category_to_git(sender, instance, created, **kwargs):
    def update_fields(category, post):
        category.title = instance.title
        category.subtitle = instance.subtitle
        category.slug = instance.slug
        category.language = instance.language
        category.featured_in_navbar = instance.featured_in_navbar

        if instance.source and instance.uuid:
            source = GitCategory.get(instance.source.uuid)
            category.source = source

    author = utils.get_author_from_user(instance.last_author)

    if created:
        category = GitCategory()
        update_fields(category, instance)
        category.save(
            True, message='Category created: %s' % instance.title,
            author=author)

        # store the page's uuid on the Post instance without triggering `save`
        Category.objects.filter(pk=instance.pk).update(uuid=category.uuid)
    else:
        try:
            category = GitCategory.get(instance.uuid)
            update_fields(category, instance)
            category.save(
                True, message='Category updated: %s' % instance.title,
                author=author)
        except exceptions.DoesNotExist:
            category = GitCategory()
            update_fields(category, instance)
            category.save(
                True, message='Category re-updated: %s' % instance.title,
                author=author)
            Category.objects.filter(pk=instance.pk).update(uuid=category.uuid)

    utils.sync_repo()
    celery_push_to_git()

@receiver(post_delete, sender=Category)
def auto_delete_category_to_git(sender, instance, **kwargs):
    author = utils.get_author_from_user(instance.last_author)
    GitCategory.delete(
        instance.uuid, True, message='Category deleted: %s' % instance.title,
        author=author)
    utils.sync_repo()
    celery_push_to_git()
<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
#===============================================================================
# Filename      : tasks
# Author        : Vincent BESANCON <besancon.vincent@gmail.com>
# Description   : Nagios module to be used by fabric.
#                 Contains functions to execute tasks on Nagios satellites.
#-------------------------------------------------------------------------------
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#===============================================================================

from fabric.api import env, roles, parallel, settings, hide, get, task, sudo, cd, put, puts
from fabric.colors import blue, yellow, green, red
from monitoring.fabric.nagios.log import Log

#-------------------------------------------------------------------------------
# Task: nagios.deploy
#-------------------------------------------------------------------------------
@task
@roles('satellites')
@parallel
def deploy(no_service=False):
    """
    Deploy Nagios configuration on all satellites.

    USAGE
    -----

      $ fab nagios.deploy[:OPTIONS][,OPTIONS...]

    OPTIONS
    -------

      no_service=<True|False>       If set to True, show warning for hosts that
                                    have no service defined. (Default: False)
    """

    env.linewise = True

    with settings(hide('warnings', 'stdout', 'running'), warn_only=True):
        print blue('===> Starting deploying new Nagios configuration on %s...' % yellow(env.host, bold=True), bold=True)
        nagios_config_result = sudo('/usr/local/nagios/admin/update-nagios-config.sh', user='nagios')
        get('/usr/local/nagios/var/last_export.log', 'log/nagdeploy_%(host)s.log')
        log = Log(env.host)

        # Check results
        if nagios_config_result.failed:
            puts(red(\
'''
 _   _       _         _ _              _ _
| \ | | __ _| |__   __| (_)_ __   ___  | | |
|  \| |/ _` | '_ \ / _` | | '_ \ / _ \ | | |
| |\  | (_| | | | | (_| | | | | |  __/ |_|_|
|_| \_|\__,_|_| |_|\__,_|_|_| |_|\___| (_|_)

''', bold=True))
            puts(red("\nErrors must be corrected:\n=========================\n\n%s\n" % log.errors))
        else:
            if log.duplicates:
                puts(red("\nDuplicates exist:\n=================\n\n%s\n" % log.duplicates))
            if log.host_has_no_service and no_service:
                puts(yellow("\nHosts that have no service defined:\n===================================\n\n%s\n" % log.host_has_no_service))

#-------------------------------------------------------------------------------
# Task: nagios.setup
#-------------------------------------------------------------------------------
@task
@roles('satellites')
def setup():
    """Deploy Nagios system image changes on all satellites."""
    print blue('==== Deploying new Nagios system image on %s ====' % yellow(env.host, bold=True), bold=True)
    sudo('rm -rf /tmp/sat_update')
    sudo('svn export --depth files "http://monitoring-dc.app.corp/svn/nagios_dc_system_image/trunk" "/tmp/sat_update"')
    with cd('/tmp/sat_update'):
        sudo('make init')

#-------------------------------------------------------------------------------
# Task: nagios.clean_old_graphs
#-------------------------------------------------------------------------------
@task
@roles('satellites')
def clean_old_graphs():
    """Clean the old graphs of removed hosts from satellites."""
    sudo('find /usr/local/pnp4nagios/var/perfdata -mindepth 1 -type d -ctime +15 -exec rm -rfv {} +' ,user='nagios')

#-------------------------------------------------------------------------------
# Task: nagios.list_graphs
#-------------------------------------------------------------------------------
@task
@roles('satellites')
def list_graphs():
    """Show a list of all hosts that have graphs. (debug)"""
    sudo('find /usr/local/pnp4nagios/var/perfdata -mindepth 1 -type d -exec ls -1drt {} +' ,user='nagios')

#-------------------------------------------------------------------------------
# Task: nagios.update_cssh
#-------------------------------------------------------------------------------
@task
@roles('admin')
def update_cssh():
    """Update Cluster SSH list of satellites on the Administration Server."""
    header = """\
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# THIS FILE IS MANAGED BY MONITORING TOOLS
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
#
# If you edit this file manually, damn god ! you risk to loose what have been
# done here ! Advice is to use Monitoring Tools.
#
"""
    nagprod = 'nagprod\t\t%s\n' % ' '.join(env.roledefs['satellites'])
    nagdev = 'nagdev\t\t%s %s\n' % ( ' '.join(env.roledefs['satellite_dev']), ' '.join(env.roledefs['satellite_tpl']) )

    conf = [header, nagprod, nagdev]

    with open('tmp/clusters', 'w') as clusters_file:
        clusters_file.writelines(conf)

    put('tmp/clusters', '/etc/clusters', use_sudo=True, mode=0644)
<commit_msg>Remove nagios related tasks, not needed.<commit_after><|endoftext|>


<commit_before># THIS SCRIPT WAS PRODUCED VIA THE NCBI HACKATHON IN AUGUST 2015
# WRITTEN (INITIALLY) BY PAUL CANTALUPO, HIROKO OHMIYA, ALLISSA DILLMAN, AND RUSSELL DURRETT

# TO DO - UPDATE TO SUBREAD PACKAGE INSTEAD OF HTSEQ (TOO SLOW)

# FASTAREF='/home/ubuntu/russ/ncbi/GCF_000001405.30_GRCh38.ens77_genomic.fna'
HISATREF = "/resources/ensembl/hisat_indexes/Homo_sapiens.GRCh38.dna.toplevel"
#HISATREF = "/resources/ensembl/hisat_indexes/Homo_sapiens.GRCh38.dna.toplevel"
#GFFFILE = "/home/ubuntu/refs/GCF_000001405.30_GRCh38.ens77_genomic.gff"
GTFFILE = "/resources/ensembl/Ensembl.GRCh38.77.gtf"
#set the number of threads to use in alignments
THREADS=6

#set the filename of the file with the list of accessions
filename = "/home/ubuntu/accessions"

# EXECUTABLE LOCATIONS (some on path)
HISAT=" /home/ubuntu/install/hisat/hisat "
PICARD=" java -jar /home/ubuntu/install/picard-tools-1.138/picard.jar "
FEATURECOUNTS="/home/ubuntu/install/subread-1.4.6-p4-Linux-x86_64/bin/featureCounts"
#HTSEQ=" ~/HTSeq-0.6.1/build/scripts-2.7/htseq-count "

SAMTOOLS=" samtools "

SAMPLES = [line.rstrip('\n') for line in open(filename)]

rule all:
	input: expand("{sample}.GRCh38.ens77.featureCounts.counts", sample=SAMPLES)
#	input: expand("{sample}.GRCh38.ens77.HTSeq.counts", sample=SAMPLES)

rule clean:
	shell: "rm -fr SRR*"

# rule perform_counting:
# 	output: "{sample}.GRCh38.ens77.HTSeq.counts"
# 	input: "{sample}.GRCh38.ens77.hisat.sorted.bam.bai"
# 	log: "log/{wildcards.sample}.counting.log"
# 	message: "performing counting of reads on genes in {input}"
# 	shell: "time {HTSEQ} -m intersection-nonempty -i gene -s no -f bam {wildcards.sample}.GRCh38.ens77.hisat.sorted.bam {GFFFILE} > {wildcards.sample}.GRCh38.ens77.HTSeq.counts 2> {wildcards.sample}.HTseq-count.log"

rule perform_counting:
	output: "{sample}.GRCh38.ens77.featureCounts.counts"
	input: "{sample}.GRCh38.ens77.hisat.sorted.bam.bai"
	log: "log/{wildcards.sample}.featureCounts.log"
	threads: THREADS
	message: "performing counting of reads on genes in {input}"
	shell: "time {FEATURECOUNTS}  -T {threads} --primary --ignore-dup -F GTF -t exon -g gene_id -a {GTFFILE} -o {wildcards.sample}.GRCh38.ens77.featureCounts.counts {wildcards.sample}.GRCh38.ens77.hisat.sorted.bam  2> {wildcards.sample}.featureCounts.log"

#IF COUNTING THEN JUST REPORT ONE MAX HIT PER READ ?  --primary fixes that?
# PAIRED END?...  -p  and  -P

rule qc_check:
	output: touch("{sample}.qc_check.done")
	input: "{sample}.GRCh38.ens77.hisat.crsm"
	log: "log/{wildcards.sample}.perlqc.log"
	message: "checking quality stats of {input} with perl script"
	shell: "time perl qc.pl --maplogfile {wildcards.sample}.hisat.two.log --metricsfile {wildcards.sample}.GRCh38.ens77.hisat.crsm --sra {wildcards.sample} 2> {wildcards.sample}.perl_qc.log"

rule picard_rnaseq_qual:
	output: "{sample}.GRCh38.ens77.hisat.crsm"
	input: "{sample}.GRCh38.ens77.hisat.sorted.bam.bai"
	log: "log/{wildcards.sample}.picard_rnametrics.log"
	message: "running picard rna qc stats on {input}"
	shell: "time {PICARD} CollectRnaSeqMetrics REF_FLAT=~/refs/refflat/ncbirefflat.txt STRAND=NONE INPUT={wildcards.sample}.GRCh38.ens77.hisat.sorted.bam OUTPUT={output} 2> {wildcards.sample}.picard_qual.log"

rule index_bam:
	output: "{sample}.GRCh38.ens77.hisat.sorted.bam.bai"
	input: "{sample}.GRCh38.ens77.hisat.sorted.bam"
	message: "indexing bam alignment file {input}"
	shell: "time {SAMTOOLS} index {input} {output} 2> {wildcards.sample}.index_bam.log"

rule sort_bam:
	output: "{sample}.GRCh38.ens77.hisat.sorted.bam"
	input: "{sample}.GRCh38.ens77.hisat.bam"
	message: "sorting {input} to {output}"
	shell: "time {SAMTOOLS} sort {input} {wildcards.sample}.GRCh38.ens77.hisat.sorted 2> {wildcards.sample}.sort_bam.log"

rule sam_to_bam:
	output: temp("{sample}.GRCh38.ens77.hisat.bam")
	input: "{sample}.GRCh38.ens77.hisat.sam"
	message: "converting sam to bam: {input} to {output}"
	shell: "time {SAMTOOLS} view -bS {input} > {output} 2> {wildcards.sample}.sam_to_bam.log"

rule hisat_alignment_two:
	output: temp("{sample}.GRCh38.ens77.hisat.sam")
	input: "{sample}.hisat.novel.splicesites.txt"
	threads: THREADS
	log: "log/{sample}.hisat.two.log"
	message: "running second pass hisat alignment with {threads} threads"
	shell: "time {HISAT} -D 15 -R 2 -N 0 -L 22 -i S,1,1.15 -x {HISATREF} -p {threads} --sra-acc {wildcards.sample} -t -S {wildcards.sample}.GRCh38.ens77.hisat.sam  2>& {log}"

#--novel-splicesite-infile {wildcards.sample}.hisat.novel.splicesites.txt

rule hisat_alignment_one:
	output: "{sample}.hisat.novel.splicesites.txt"   #, temp("{sample}.GRCh38.ens77.hisat.one.sam")
	threads: THREADS
	log: "log/{sample}.hisat.one.log"
	message: "hisat aligning reads from {wildcards.sample} to GRCh38.ens77 with {threads} threads to produce splicesites"
	shell: "time {HISAT} -D 15 -R 2 -N 0 -L 22 -i S,1,1.15 -x {HISATREF} -p {threads} --sra-acc {wildcards.sample} -t --novel-splicesite-outfile {wildcards.sample}.hisat.novel.splicesites.txt -S /dev/null 2>&{log} "

<commit_msg>format<commit_after># THIS SCRIPT WAS PRODUCED VIA THE NCBI HACKATHON IN AUGUST 2015
# WRITTEN (INITIALLY) BY PAUL CANTALUPO, HIROKO OHMIYA, ALLISSA DILLMAN, AND RUSSELL DURRETT

# TO DO - UPDATE TO SUBREAD PACKAGE INSTEAD OF HTSEQ (TOO SLOW)

# FASTAREF='/home/ubuntu/russ/ncbi/GCF_000001405.30_GRCh38.ens77_genomic.fna'
HISATREF = "/resources/ensembl/hisat_indexes/Homo_sapiens.GRCh38.dna.toplevel"
#HISATREF = "/resources/ensembl/hisat_indexes/Homo_sapiens.GRCh38.dna.toplevel"
#GFFFILE = "/home/ubuntu/refs/GCF_000001405.30_GRCh38.ens77_genomic.gff"
GTFFILE = "/resources/ensembl/Ensembl.GRCh38.77.gtf"
#set the number of threads to use in alignments
THREADS=6

#set the filename of the file with the list of accessions
filename = "/home/ubuntu/accessions"

# EXECUTABLE LOCATIONS (some on path)
HISAT=" /home/ubuntu/install/hisat/hisat "
PICARD=" java -jar /home/ubuntu/install/picard-tools-1.138/picard.jar "
FEATURECOUNTS="/home/ubuntu/install/subread-1.4.6-p4-Linux-x86_64/bin/featureCounts"
#HTSEQ=" ~/HTSeq-0.6.1/build/scripts-2.7/htseq-count "

SAMTOOLS=" samtools "

SAMPLES = [line.rstrip('\n') for line in open(filename)]

rule all:
	input: expand("{sample}.GRCh38.ens77.featureCounts.counts", sample=SAMPLES)
#	input: expand("{sample}.GRCh38.ens77.HTSeq.counts", sample=SAMPLES)

rule clean:
	shell: "rm -fr SRR*"

# rule perform_counting:
# 	output: "{sample}.GRCh38.ens77.HTSeq.counts"
# 	input: "{sample}.GRCh38.ens77.hisat.sorted.bam.bai"
# 	log: "log/{wildcards.sample}.counting.log"
# 	message: "performing counting of reads on genes in {input}"
# 	shell: "time {HTSEQ} -m intersection-nonempty -i gene -s no -f bam {wildcards.sample}.GRCh38.ens77.hisat.sorted.bam {GFFFILE} > {wildcards.sample}.GRCh38.ens77.HTSeq.counts 2> {wildcards.sample}.HTseq-count.log"

rule perform_counting:
	output: "{sample}.GRCh38.ens77.featureCounts.counts"
	input: "{sample}.GRCh38.ens77.hisat.sorted.bam.bai"
	log: "log/{wildcards.sample}.featureCounts.log"
	threads: THREADS
	message: "performing counting of reads on genes in {input}"
	shell: "time {FEATURECOUNTS}  -T {threads} --primary -F GTF -t exon -g gene_id -a {GTFFILE} -o {wildcards.sample}.GRCh38.ens77.featureCounts.counts {wildcards.sample}.GRCh38.ens77.hisat.sorted.bam  2> {wildcards.sample}.featureCounts.log"

#IF COUNTING THEN JUST REPORT ONE MAX HIT PER READ ?  --primary fixes that?
# PAIRED END?...  -p  and  -P

rule qc_check:
	output: touch("{sample}.qc_check.done")
	input: "{sample}.GRCh38.ens77.hisat.crsm"
	log: "log/{wildcards.sample}.perlqc.log"
	message: "checking quality stats of {input} with perl script"
	shell: "time perl qc.pl --maplogfile {wildcards.sample}.hisat.two.log --metricsfile {wildcards.sample}.GRCh38.ens77.hisat.crsm --sra {wildcards.sample} 2> {wildcards.sample}.perl_qc.log"

rule picard_rnaseq_qual:
	output: "{sample}.GRCh38.ens77.hisat.crsm"
	input: "{sample}.GRCh38.ens77.hisat.sorted.bam.bai"
	log: "log/{wildcards.sample}.picard_rnametrics.log"
	message: "running picard rna qc stats on {input}"
	shell: "time {PICARD} CollectRnaSeqMetrics REF_FLAT=~/refs/refflat/ncbirefflat.txt STRAND=NONE INPUT={wildcards.sample}.GRCh38.ens77.hisat.sorted.bam OUTPUT={output} 2> {wildcards.sample}.picard_qual.log"

rule index_bam:
	output: "{sample}.GRCh38.ens77.hisat.sorted.bam.bai"
	input: "{sample}.GRCh38.ens77.hisat.sorted.bam"
	message: "indexing bam alignment file {input}"
	shell: "time {SAMTOOLS} index {input} {output} 2> {wildcards.sample}.index_bam.log"

rule sort_bam:
	output: "{sample}.GRCh38.ens77.hisat.sorted.bam"
	input: "{sample}.GRCh38.ens77.hisat.bam"
	message: "sorting {input} to {output}"
	shell: "time {SAMTOOLS} sort {input} {wildcards.sample}.GRCh38.ens77.hisat.sorted 2> {wildcards.sample}.sort_bam.log"

rule sam_to_bam:
	output: temp("{sample}.GRCh38.ens77.hisat.bam")
	input: "{sample}.GRCh38.ens77.hisat.sam"
	message: "converting sam to bam: {input} to {output}"
	shell: "time {SAMTOOLS} view -bS {input} > {output} 2> {wildcards.sample}.sam_to_bam.log"

rule hisat_alignment_two:
	output: temp("{sample}.GRCh38.ens77.hisat.sam")
	input: "{sample}.hisat.novel.splicesites.txt"
	threads: THREADS
	log: "log/{sample}.hisat.two.log"
	message: "running second pass hisat alignment with {threads} threads"
	shell: "time {HISAT} -D 15 -R 2 -N 0 -L 22 -i S,1,1.15 -x {HISATREF} -p {threads} --sra-acc {wildcards.sample} -t -S {wildcards.sample}.GRCh38.ens77.hisat.sam  2>& {log}"

#--novel-splicesite-infile {wildcards.sample}.hisat.novel.splicesites.txt

rule hisat_alignment_one:
	output: "{sample}.hisat.novel.splicesites.txt"   #, temp("{sample}.GRCh38.ens77.hisat.one.sam")
	threads: THREADS
	log: "log/{sample}.hisat.one.log"
	message: "hisat aligning reads from {wildcards.sample} to GRCh38.ens77 with {threads} threads to produce splicesites"
	shell: "time {HISAT} -D 15 -R 2 -N 0 -L 22 -i S,1,1.15 -x {HISATREF} -p {threads} --sra-acc {wildcards.sample} -t --novel-splicesite-outfile {wildcards.sample}.hisat.novel.splicesites.txt -S /dev/null 2>&{log} "

<|endoftext|>


<commit_before>from __future__ import unicode_literals

import logging

from mopidy import backend
from mopidy.models import Album, Track, SearchResult, Ref

from .parsing import *  # noqa
from .query import Query
from .uritools import urisplit, uriunsplit

logger = logging.getLogger(__name__)

class InternetArchiveLibraryProvider(backend.LibraryProvider):

    BROWSE_FIELDS = ('identifier', 'title', 'mediatype'),

    SEARCH_FIELDS = ('identifier', 'title', 'creator', 'date', 'publicdate')

    def __init__(self, backend):
        super(InternetArchiveLibraryProvider, self).__init__(backend)
        self.root_directory = Ref.directory(
            uri=uriunsplit([backend.SCHEME, None, '/', None, None]),
            name=self.config['browse_label']
        )
        self.browse_query = self.backend.client.query_string({
            'mediatype': self.config['mediatypes'],
            'format': self.config['formats']
        }, group='OR')
        self.search_query = self.backend.client.query_string({
            'collection': self.config['collections'],
            'mediatype':  self.config['mediatypes'],
            'format': self.config['formats']
        }, group='OR')
        self.bookmarks = self._load_bookmarks(self.config['bookmarks'])
        self.collections = self._load_collections(self.config['collections'])

    @property
    def config(self):
        return self.backend.config[self.backend.SCHEME]

    def browse(self, uri):
        logger.debug("internetarchive browse %r", uri)

        try:
            if not uri:
                return [self.root_directory]
            elif uri == self.root_directory.uri:
                return self._browse_root()
            elif uri in self.bookmarks:
                return self._browse_bookmarks(urisplit(uri).userinfo)
            elif uri in self.collections:
                return self._browse_collection(urisplit(uri).path)
            else:
                return self._browse_item(urisplit(uri).path)
        except Exception as error:
            logger.error('Error browsing %r: %s', uri, error)
            return []

    def lookup(self, uri):
        logger.debug("internetarchive lookup %r", uri)

        try:
            _, _, identifier, _, filename = urisplit(uri)
            item = self.backend.client.metadata(identifier)
            if filename:
                files = [f for f in item['files'] if f['name'] == filename]
            else:
                files = self._files_by_format(item['files'])
            logger.debug("internetarchive files: %r", files)
            return self._item_to_tracks(item, files)
        except Exception as error:
            logger.error('Lookup error for %r: %s', uri, error)
            return []

    def find_exact(self, query=None, uris=None):
        logger.debug("internetarchive find %r", query)

        if not query:
            return
        try:
            return SearchResult(albums=self._find_albums(Query(query, True)))
        except Exception as error:
            logger.error('internetarchive find %r: %s', query, error)
            return None

    def search(self, query=None, uris=None):
        logger.debug("internetarchive search %r", query)

        if not query:
            return
        try:
            return SearchResult(albums=self._find_albums(Query(query, False)))
        except Exception as error:
            logger.error('internetarchive search %r: %s', query, error)
            return None

    def get_stream_url(self, uri):
        _, _, identifier, _, filename = urisplit(uri)
        return self.backend.client.geturl(identifier.lstrip('/'), filename)

    def _load_bookmarks(self, usernames):
        refs = {}
        for username in usernames:
            try:
                # raise error if username does not exit, cache for browsing
                self.backend.client.bookmarks(username)
            except Exception as e:
                logger.error('Error loading bookmarks for %r: %s', username, e)
                continue
            uri = self._bookmarks_uri(username)
            name = self.config['bookmarks_label'].format(username)
            refs[uri] = Ref.directory(uri=uri, name=name)
        if refs:
            logger.info("Loaded %d Internet Archive bookmarks", len(refs))
        return refs

    def _load_collections(self, identifiers):
        refs = {}
        for identifier in identifiers:
            try:
                item = self.backend.client.metadata(identifier + '/metadata')
            except Exception as e:
                logger.error('Error loading collection %r: %s', identifier, e)
                continue
            if item['mediatype'] != 'collection':
                logger.error('Item %r is not a collection', identifier)
                continue
            uri = self._item_uri(identifier)
            name = parse_title(item.get('title'), identifier)
            refs[uri] = Ref.directory(uri=uri, name=name)
        if refs:
            logger.info("Loaded %d Internet Archive collections", len(refs))
        return refs

    def _browse_root(self):
        # fix temporary (e.g. network) errors at startup
        if len(self.bookmarks) != len(self.config['bookmarks']):
            missing = filter(
                lambda v: self._bookmarks_uri(v) not in self.bookmarks,
                self.config['bookmarks']
            )
            self.bookmarks.update(self._load_bookmarks(missing))
        if len(self.collections) != len(self.config['collections']):
            missing = filter(
                lambda v: self._item_uri(v) not in self.collections,
                self.config['collections']
            )
            self.collections.update(self._load_collections(missing))
        return self.bookmarks.values() + self.collections.values()

    def _browse_bookmarks(self, username):
        result = self.backend.client.bookmarks(username)
        return [self._doc_to_ref(doc) for doc in result]

    def _browse_collection(self, identifier):
        qs = self.backend.client.query_string(dict(collection=identifier))
        result = self.backend.client.search(
            self.browse_query + ' AND ' + qs,
            fields=self.BROWSE_FIELDS,
            sort=self.config['sort_order'],
            rows=self.config['browse_limit']
        )
        return [self._doc_to_ref(doc) for doc in result]

    def _browse_item(self, identifier):
        item = self.backend.client.metadata(identifier)
        if item['metadata']['mediatype'] not in self.config['mediatypes']:
            return []
        refs = []
        for f in self._files_by_format(item['files']):
            uri = self._file_uri(identifier, f['name'])
            name = parse_title(f.get('title'), f['name'])
            refs.append(Ref.track(uri=uri, name=name))
        return refs

    def _find_albums(self, query):
        terms = {}
        for (field, values) in query.iteritems():
            if field == "any":
                terms[None] = values
            elif field == "album":
                terms['title'] = values
            elif field == "artist":
                terms['creator'] = values
            elif field == 'date':
                terms['date'] = values
        qs = self.backend.client.query_string(terms, group='AND')
        logger.debug('internetarchive query string: %r', qs)
        result = self.backend.client.search(
            self.search_query + ' AND ' + qs,
            fields=self.SEARCH_FIELDS,
            sort=self.config['sort_order'],
            rows=self.config['search_limit']
        )
        albums = [self._doc_to_album(doc) for doc in result]
        logger.debug("internetarchive found albums: %r", albums)
        return query.filter_albums(albums)

    def _bookmarks_uri(self, username):
        authority = '%s@archive.org' % username
        return uriunsplit([self.backend.SCHEME, authority, '/', None, None])

    def _item_uri(self, identifier):
        return uriunsplit([self.backend.SCHEME, None, identifier, None, None])

    def _file_uri(self, itemid, filename):
        return uriunsplit([self.backend.SCHEME, None, itemid, None, filename])

    def _doc_to_album(self, doc):
        return Album(
            uri=self._item_uri(doc['identifier']),
            name=parse_title(doc.get('title'), doc['identifier']),
            artists=parse_creator(doc.get('creator')),
            date=parse_date(doc.get('date'))
        )

    def _doc_to_ref(self, doc):
        return Ref.directory(
            uri=self._item_uri(doc['identifier']),
            name=parse_title(doc.get('title'), doc['identifier'])
        )

    def _item_to_tracks(self, item, files):
        metadata = item['metadata']
        identifier = metadata['identifier']
        album = self._doc_to_album(metadata)
        byname = {f['name']: f for f in item['files']}

        tracks = []
        for f in files:
            if 'original' in f and f['original'] in byname:
                orig = byname[f['original']]
                for key in orig:
                    if not key in f or f[key] in ('', 'tmp'):
                        f[key] = orig[key]

            tracks.append(Track(
                uri=self._file_uri(identifier, f['name']),
                name=parse_title(f.get('title'), f['name']),
                artists=parse_creator(f.get('creator'), album.artists),
                album=album,
                track_no=parse_track(f.get('track')),
                date=parse_date(f.get('date'), album.date),
                length=parse_length(f.get('length')),
                bitrate=parse_bitrate(f.get('bitrate')),
                last_modified=parse_mtime(f.get('mtime'))
            ))
        return tracks

    def _files_by_format(self, files):
        from collections import defaultdict
        byformat = defaultdict(list)
        for f in files:
            byformat[f['format'].lower()].append(f)
        for fmt in [fmt.lower() for fmt in self.config['formats']]:
            if fmt in byformat:
                return byformat[fmt]
            for key in byformat.keys():
                if fmt in key:
                    return byformat[key]
        return []
<commit_msg>Refactor parsing/traslators<commit_after>from __future__ import unicode_literals

import logging

from mopidy import backend
from mopidy.models import Album, Track, SearchResult, Ref

from .parsing import *  # noqa
from .query import Query
from .uritools import urisplit, uriunsplit

logger = logging.getLogger(__name__)

QUERY_MAP = {
    'any': None,
    'album': 'title',
    'artist': 'creator',
    'date': 'date'
}

class InternetArchiveLibraryProvider(backend.LibraryProvider):

    BROWSE_FIELDS = ('identifier', 'title', 'mediatype'),

    SEARCH_FIELDS = ('identifier', 'title', 'creator', 'date', 'publicdate')

    def __init__(self, backend):
        super(InternetArchiveLibraryProvider, self).__init__(backend)
        self.root_directory = Ref.directory(
            uri=uriunsplit([backend.SCHEME, None, '/', None, None]),
            name=self.config['browse_label']
        )
        self.browse_query = self.backend.client.query_string({
            'mediatype': self.config['mediatypes'],
            'format': self.config['formats']
        }, group='OR')
        self.search_query = self.backend.client.query_string({
            'collection': self.config['collections'],
            'mediatype':  self.config['mediatypes'],
            'format': self.config['formats']
        }, group='OR')
        self.bookmarks = self._load_bookmarks(self.config['bookmarks'])
        self.collections = self._load_collections(self.config['collections'])

    @property
    def config(self):
        return self.backend.config[self.backend.SCHEME]

    def browse(self, uri):
        logger.debug("internetarchive browse %r", uri)

        try:
            if not uri:
                return [self.root_directory]
            elif uri == self.root_directory.uri:
                return self._browse_root()
            elif uri in self.bookmarks:
                return self._browse_bookmarks(urisplit(uri).userinfo)
            elif uri in self.collections:
                return self._browse_collection(urisplit(uri).path)
            else:
                return self._browse_item(urisplit(uri).path)
        except Exception as error:
            logger.error('Error browsing %r: %s', uri, error)
            return []

    def lookup(self, uri):
        logger.debug("internetarchive lookup %r", uri)

        try:
            _, _, identifier, _, filename = urisplit(uri)
            item = self.backend.client.metadata(identifier)
            if filename:
                files = [f for f in item['files'] if f['name'] == filename]
            else:
                files = self._files_by_format(item['files'])
            logger.debug("internetarchive files: %r", files)
            return self._item_to_tracks(item, files)
        except Exception as error:
            logger.error('Lookup error for %r: %s', uri, error)
            return []

    def find_exact(self, query=None, uris=None):
        logger.debug("internetarchive find %r", query)

        if not query:
            return
        try:
            return SearchResult(albums=self._find_albums(Query(query, True)))
        except Exception as error:
            logger.error('internetarchive find %r: %s', query, error)
            return None

    def search(self, query=None, uris=None):
        logger.debug("internetarchive search %r", query)

        if not query:
            return
        try:
            return SearchResult(albums=self._find_albums(Query(query, False)))
        except Exception as error:
            logger.error('internetarchive search %r: %s', query, error)
            return None

    def get_stream_url(self, uri):
        _, _, identifier, _, filename = urisplit(uri)
        return self.backend.client.geturl(identifier.lstrip('/'), filename)

    def _load_bookmarks(self, usernames):
        refs = {}
        for username in usernames:
            try:
                # raise error if username does not exit, cache for browsing
                self.backend.client.bookmarks(username)
            except Exception as e:
                logger.error('Error loading bookmarks for %r: %s', username, e)
                continue
            uri = self._bookmarks_uri(username)
            name = self.config['bookmarks_label'].format(username)
            refs[uri] = Ref.directory(uri=uri, name=name)
        if refs:
            logger.info("Loaded %d Internet Archive bookmarks", len(refs))
        return refs

    def _load_collections(self, identifiers):
        refs = {}
        for identifier in identifiers:
            try:
                item = self.backend.client.metadata(identifier + '/metadata')
            except Exception as e:
                logger.error('Error loading collection %r: %s', identifier, e)
                continue
            if item['mediatype'] != 'collection':
                logger.error('Item %r is not a collection', identifier)
                continue
            uri = self._item_uri(identifier)
            name = parse_title(item.get('title'), identifier)
            refs[uri] = Ref.directory(uri=uri, name=name)
        if refs:
            logger.info("Loaded %d Internet Archive collections", len(refs))
        return refs

    def _browse_root(self):
        # fix temporary (e.g. network) errors at startup
        if len(self.bookmarks) != len(self.config['bookmarks']):
            missing = filter(
                lambda v: self._bookmarks_uri(v) not in self.bookmarks,
                self.config['bookmarks']
            )
            self.bookmarks.update(self._load_bookmarks(missing))
        if len(self.collections) != len(self.config['collections']):
            missing = filter(
                lambda v: self._item_uri(v) not in self.collections,
                self.config['collections']
            )
            self.collections.update(self._load_collections(missing))
        return self.bookmarks.values() + self.collections.values()

    def _browse_bookmarks(self, username):
        result = self.backend.client.bookmarks(username)
        return [self._doc_to_ref(doc) for doc in result]

    def _browse_collection(self, identifier):
        qs = self.backend.client.query_string({'collection': identifier})
        logger.debug('internetarchive query string: %r', qs)
        result = self.backend.client.search(
            self.browse_query + ' AND ' + qs,
            fields=self.BROWSE_FIELDS,
            sort=self.config['sort_order'],
            rows=self.config['browse_limit']
        )
        return [self._doc_to_ref(doc) for doc in result]

    def _browse_item(self, identifier):
        item = self.backend.client.metadata(identifier)
        if item['metadata']['mediatype'] not in self.config['mediatypes']:
            return []
        refs = []
        for f in self._files_by_format(item['files']):
            uri = self._file_uri(identifier, f['name'])
            name = parse_title(f.get('title'), f['name'])
            refs.append(Ref.track(uri=uri, name=name))
        return refs

    def _find_albums(self, query):
        qs = self.backend.client.query_string({
            QUERY_MAP[k]: query[k] for k in query if k in QUERY_MAP
        }, group='AND')
        logger.debug('internetarchive query string: %r', qs)
        result = self.backend.client.search(
            self.search_query + ' AND ' + qs,
            fields=self.SEARCH_FIELDS,
            sort=self.config['sort_order'],
            rows=self.config['search_limit']
        )
        albums = [self._doc_to_album(doc) for doc in result]
        logger.debug("internetarchive found albums: %r", albums)
        return query.filter_albums(albums)

    def _bookmarks_uri(self, username):
        authority = '%s@archive.org' % username
        return uriunsplit([self.backend.SCHEME, authority, '/', None, None])

    def _item_uri(self, identifier):
        return uriunsplit([self.backend.SCHEME, None, identifier, None, None])

    def _file_uri(self, itemid, filename):
        return uriunsplit([self.backend.SCHEME, None, itemid, None, filename])

    def _doc_to_album(self, doc):
        return Album(
            uri=self._item_uri(doc['identifier']),
            name=parse_title(doc.get('title'), doc['identifier']),
            artists=parse_creator(doc.get('creator')),
            date=parse_date(doc.get('date'))
        )

    def _doc_to_ref(self, doc):
        return Ref.directory(
            uri=self._item_uri(doc['identifier']),
            name=parse_title(doc.get('title'), doc['identifier'])
        )

    def _item_to_tracks(self, item, files):
        metadata = item['metadata']
        identifier = metadata['identifier']
        album = self._doc_to_album(metadata)
        byname = {f['name']: f for f in item['files']}

        tracks = []
        for f in files:
            if 'original' in f and f['original'] in byname:
                orig = byname[f['original']]
                for key in orig:
                    if not key in f or f[key] in ('', 'tmp'):
                        f[key] = orig[key]

            tracks.append(Track(
                uri=self._file_uri(identifier, f['name']),
                name=parse_title(f.get('title'), f['name']),
                artists=parse_creator(f.get('creator'), album.artists),
                album=album,
                track_no=parse_track(f.get('track')),
                date=parse_date(f.get('date'), album.date),
                length=parse_length(f.get('length')),
                bitrate=parse_bitrate(f.get('bitrate')),
                last_modified=parse_mtime(f.get('mtime'))
            ))
        return tracks

    def _files_by_format(self, files):
        from collections import defaultdict
        byformat = defaultdict(list)
        for f in files:
            byformat[f['format'].lower()].append(f)
        for fmt in [fmt.lower() for fmt in self.config['formats']]:
            if fmt in byformat:
                return byformat[fmt]
            for key in byformat.keys():
                if fmt in key:
                    return byformat[key]
        return []
<|endoftext|>


<commit_before># ----------------------------------------------------------------------------
# cocos2d
# Copyright (c) 2008 Daniel Moisset, Ricardo Quesada, Rayentray Tappa, Lucio Torre
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#   * Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in
#     the documentation and/or other materials provided with the
#     distribution.
#   * Neither the name of cocos2d nor the names of its
#     contributors may be used to endorse or promote products
#     derived from this software without specific prior written
#     permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
# ----------------------------------------------------------------------------
'''Grid data structure'''

__docformat__ = 'restructuredtext'

import pyglet
from pyglet import image
from pyglet.gl import *
from euclid import Point2, Point3

from director import director
import framegrabber

__all__ = ['GridBase',
           'Grid3D',
           'TiledGrid3D',
            ]

class GridBase(object):
    """
    A Scene that takes two scenes and makes a transition between them
    """
    texture = None

    def __init__(self):
        super(GridBase, self).__init__()
        self._active = False
        self.reuse_grid = 0     #! Number of times that this grid will be reused

    def init( self, grid ):
        '''Initializes the grid creating both a vertex_list for an independent-tiled grid
        and creating also a vertex_list_indexed for a "united" (non independent tile) grid.

        :Parameters:
            `grid` : euclid.Point2
                size of a 2D grid
        '''

        #: size of the grid. (rows, columns)
        self.grid = grid

        width, height = director.get_window_size()

        if self.texture is None:
            self.texture = image.Texture.create_for_size(
                    GL_TEXTURE_2D, width,
                    height, GL_RGBA)

        self.grabber = framegrabber.TextureGrabber()
        self.grabber.grab(self.texture)

        #: x pixels between each vertex (float)
        self.x_step = width / self.grid.x
        #: y pixels between each vertex (float)
        self.y_step = height / self.grid.y

        self._init()

    def before_draw( self ):
        '''Binds the framebuffer to a texture
        and set a 2d projection before binding
        to prevent calculating a new texture
        '''

        self._set_2d_projection()

        # capture before drawing
        self.grabber.before_render(self.texture)

    def after_draw( self, camera ):
        '''Called by CocosNode when the texture is already grabbed.
        The FrameBuffer will be unbound and the texture will be drawn

        :Parameters:
            `camera` : `Camera`
                The target's camera object.
        '''

        # capture after drawing
        self.grabber.after_render(self.texture)

        # after unbinding
        # set a 3d projection
        self._set_3d_projection()

        # and center the camera
        camera.locate( force=True )

        # blit
        glEnable(self.texture.target)
        glBindTexture(self.texture.target, self.texture.id)

        glPushAttrib(GL_COLOR_BUFFER_BIT)

        self._blit()

        glPopAttrib()
        glDisable(self.texture.target)

    def _set_active(self, bool):
        if self._active == bool:
            return
        self._active = bool
        if self._active == True:
            pass
        elif self._active == False:
            self.vertex_list.delete()
            # to restore the camera to default position
            director.set_projection()
        else:
            raise Exception("Invalid value for GridBase.active")

    def _get_active(self):
        return self._active

    active = property(_get_active, _set_active,
                      doc='''Determines whether the grid is active or not
                     :type: bool
                     ''')
    def _init(self):
        raise NotImplementedError('abstract')

    def _blit(self):
        raise NotImplementedError('abstract')

    def _on_resize(self):
        raise NotImplementedError('abstract')

    @classmethod
    def _set_3d_projection(cls):
        width, height = director.window.width, director.window.height

        glViewport(0, 0, width, height)
        glMatrixMode(GL_PROJECTION)
        glLoadIdentity()
        gluPerspective(60, 1.0*width/height, 0.1, 3000.0)
        glMatrixMode(GL_MODELVIEW)

    @classmethod
    def _set_2d_projection(cls):

#        director.set_2d_projection()
        width, height = director.get_window_size()
        glLoadIdentity()
        glViewport(0, 0, width, height)
        glMatrixMode(GL_PROJECTION)
        glLoadIdentity()
        glOrtho(0, width, 0, height, -100, 100)
        glMatrixMode(GL_MODELVIEW)

class Grid3D(GridBase):
    '''`Grid3D` is a 3D grid implementation. Each vertex has 3 dimensions: x,y,z

    The vindexed ertex array will be built with::

        self.vertex_list.vertices: x,y,z (floats)
        self.vertex_list.tex_coords: x,y,z (floats)
        self.vertex_list.colors: RGBA, with values from 0 - 255
    '''

    def _init( self ):
        # calculate vertex, textures depending on screen size
        idx_pts, ver_pts_idx, tex_pts_idx = self._calculate_vertex_points()

        #: indexed vertex array that can be transformed.
        #: it has these attributes:
        #:
        #:    - vertices
        #:    - colors
        #:    - tex_coords
        #:
        #: for more information refer to pyglet's documentation: pyglet.graphics.vertex_list_indexed
        self.vertex_list = pyglet.graphics.vertex_list_indexed( (self.grid.x+1) * (self.grid.y+1),
                            idx_pts, "t2f", "v3f/stream","c4B")

        #: original vertex array of the grid. (read-only)
        self.vertex_points = ver_pts_idx[:]
        self.vertex_list.vertices = ver_pts_idx
        self.vertex_list.tex_coords = tex_pts_idx
        self.vertex_list.colors = (255,255,255,255) * (self.grid.x+1) * (self.grid.y+1)

    def _blit(self ):
        self.vertex_list.draw(pyglet.gl.GL_TRIANGLES)

    def _calculate_vertex_points(self):
        w = float(self.texture.width)
        h = float(self.texture.height)

        index_points = []
        vertex_points_idx = []
        texture_points_idx = []

        for x in xrange(0,self.grid.x+1):
            for y in xrange(0,self.grid.y+1):
                vertex_points_idx += [-1,-1,-1]
                texture_points_idx += [-1,-1]

        for x in xrange(0, self.grid.x):
            for y in xrange(0, self.grid.y):
                x1 = x * self.x_step
                x2 = x1 + self.x_step
                y1 = y * self.y_step
                y2 = y1 + self.y_step

                #  d <-- c
                #        ^
                #        |
                #  a --> b
                a = x * (self.grid.y+1) + y
                b = (x+1) * (self.grid.y+1) + y
                c = (x+1) * (self.grid.y+1) + (y+1)
                d = x * (self.grid.y+1) + (y+1)

                # 2 triangles: a-b-d, b-c-d
                index_points += [ a, b, d, b, c, d]    # triangles

                l1 = ( a*3, b*3, c*3, d*3 )
                l2 = ( Point3(x1,y1,0), Point3(x2,y1,0), Point3(x2,y2,0), Point3(x1,y2,0) )

                #  building the vertex
                for i in xrange( len(l1) ):
                    vertex_points_idx[ l1[i] ] = l2[i].x
                    vertex_points_idx[ l1[i] + 1 ] = l2[i].y
                    vertex_points_idx[ l1[i] + 2 ] = l2[i].z

                # building the texels
                tex1 = ( a*2, b*2, c*2, d*2 )
                tex2 = ( Point2(x1,y1), Point2(x2,y1), Point2(x2,y2), Point2(x1,y2) )

                for i in xrange( len(tex1)):
                    texture_points_idx[ tex1[i] ] = tex2[i].x / w
                    texture_points_idx[ tex1[i] + 1 ] = tex2[i].y / h

        return ( index_points, vertex_points_idx, texture_points_idx )

    def get_vertex( self, x, y):
        '''Get the current vertex coordinate

        :Parameters:
            `x` : int
               x-vertex
            `y` : int
               y-vertex

        :rtype: (float, float, float)
        '''
        idx = (x * (self.grid.y+1) + y) * 3
        x = self.vertex_list.vertices[idx]
        y = self.vertex_list.vertices[idx+1]
        z = self.vertex_list.vertices[idx+2]
        return (x,y,z)

    def get_original_vertex( self, x, y):
        '''Get the original vertex coordinate.
        The original vertices are the ones weren't modified by the current action.

        :Parameters:
            `x` : int
               x-vertex
            `y` : int
               y-vertex

        :rtype: (float, float, float)
        '''
        idx = (x * (self.grid.y+1) + y) * 3
        x = self.vertex_points[idx]
        y = self.vertex_points[idx+1]
        z = self.vertex_points[idx+2]

        return (x,y,z)

    def set_vertex( self, x, y, v):
        '''Set a vertex point is a certain value

        :Parameters:
            `x` : int
               x-vertex
            `y` : int
               y-vertex
            `v` : (float, float, float)
                tuple value for the vertex
        '''
        idx = (x * (self.grid.y+1) + y) * 3
        self.vertex_list.vertices[idx] = int(v[0])
        self.vertex_list.vertices[idx+1] = int(v[1])
        self.vertex_list.vertices[idx+2] = int(v[2])

class TiledGrid3D(GridBase):
    '''`TiledGrid3D` is a 3D grid implementation. It differs from `Grid3D` in that
    the tiles can be separated from the grid.

    The vertex array will be built with::

        self.vertex_list.vertices: x,y,z (floats)
        self.vertex_list.tex_coords: x,y (floats)
        self.vertex_list.colors: RGBA, with values from 0 - 255
    '''
    def _init( self ):
        # calculate vertex, textures depending on screen size
        ver_pts, tex_pts = self._calculate_vertex_points()

        #: vertex array that can be transformed.
        #: it has these attributes:
        #:
        #:    - vertices
        #:    - colors
        #:    - tex_coords
        #:
        #: for more information refer to pyglet's documentation: pyglet.graphics.vertex_list
        self.vertex_list = pyglet.graphics.vertex_list(self.grid.x * self.grid.y * 4,
                            "t2f", "v3f/stream","c4B")
        #: original vertex array of the grid. (read-only)
        self.vertex_points = ver_pts[:]
        self.vertex_list.vertices = ver_pts
        self.vertex_list.tex_coords = tex_pts
        self.vertex_list.colors = (255,255,255,255) * self.grid.x * self.grid.y * 4

    def _blit(self ):
        self.vertex_list.draw(pyglet.gl.GL_QUADS)

    def _calculate_vertex_points(self):
        w = float(self.texture.width)
        h = float(self.texture.height)

        vertex_points = []
        texture_points = []

        for x in xrange(0, self.grid.x):
            for y in xrange(0, self.grid.y):
                x1 = x * self.x_step
                x2 = x1 + self.x_step
                y1 = y * self.y_step
                y2 = y1 + self.y_step

                # Building the tiles' vertex and texture points
                vertex_points += [x1, y1, 0, x2, y1, 0, x2, y2, 0, x1, y2, 0 ]
                texture_points += [x1/w, y1/h, x2/w, y1/h, x2/w, y2/h, x1/w, y2/h]

        # Generates a quad for each tile, to perform tiles effect
        return (vertex_points, texture_points)

    def set_tile(self, x, y, coords):
        '''Set the 4 tile coordinates

        Coordinates positions::

            3 <-- 2
                  ^
                  |
            0 --> 1

        :Parameters:
            `x` : int
                x coodinate of the tile
            `y` : int
                y coordinate of the tile
            `coords` : [ float, float, float, float, float, float, float, float, float, float, float, float ]
                The 4 coordinates in the format (x0, y0, z0, x1, y1, z1,..., x3, y3, z3)
        '''
        idx = (self.grid.y * x + y) * 4 * 3
        self.vertex_list.vertices[idx:idx+12] = coords

    def get_original_tile(self, x, y):
        '''Get the 4-original tile coordinates.

        Coordinates positions::

            3 <-- 2
                  ^
                  |
            0 --> 1

        :Parameters:
            `x` : int
                x coordinate of the tile
            `y` : int
                y coordinate of the tile

        :rtype: [ float, float, float, float, float, float, float, float, float, float, float, float ]
        :returns: The 4 coordinates with the following order: x0, y0, z0, x1, y1, z1,...,x3, y3, z3
        '''
        idx = (self.grid.y * x + y) * 4 * 3
        return self.vertex_points[idx:idx+12]

    def get_tile(self, x, y):
        '''Get the current tile coordinates.

        Coordinates positions::

            3 <-- 2
                  ^
                  |
            0 --> 1

        :Parameters:
            `x` : int
                x coordinate of the tile
            `y` : int
                y coordinate of the tile

        :rtype: [ float, float, float, float, float, float, float, float, float, float, float, float ]
        :returns: The 4 coordinates with the following order: x0, y0, z0, x1, y1, z1,...,x3, y3, z3
        '''
        idx = (self.grid.y * x + y) * 4 * 3
        return self.vertex_list.vertices[idx:idx+12]
<commit_msg>fix issue 152: tilegid actions view wrong when resized<commit_after># ----------------------------------------------------------------------------
# cocos2d
# Copyright (c) 2008 Daniel Moisset, Ricardo Quesada, Rayentray Tappa, Lucio Torre
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#   * Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in
#     the documentation and/or other materials provided with the
#     distribution.
#   * Neither the name of cocos2d nor the names of its
#     contributors may be used to endorse or promote products
#     derived from this software without specific prior written
#     permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
# ----------------------------------------------------------------------------
'''Grid data structure'''

__docformat__ = 'restructuredtext'

import pyglet
from pyglet import image
from pyglet.gl import *
from euclid import Point2, Point3

from director import director
import framegrabber

__all__ = ['GridBase',
           'Grid3D',
           'TiledGrid3D',
            ]

class GridBase(object):
    """
    A Scene that takes two scenes and makes a transition between them
    """
    texture = None

    def __init__(self):
        super(GridBase, self).__init__()
        self._active = False
        self.reuse_grid = 0     #! Number of times that this grid will be reused

    def init( self, grid ):
        '''Initializes the grid creating both a vertex_list for an independent-tiled grid
        and creating also a vertex_list_indexed for a "united" (non independent tile) grid.

        :Parameters:
            `grid` : euclid.Point2
                size of a 2D grid
        '''

        #: size of the grid. (rows, columns)
        self.grid = grid

        width, height = director.get_window_size()

        if self.texture is None:
            self.texture = image.Texture.create_for_size(
                    GL_TEXTURE_2D, width,
                    height, GL_RGBA)

        self.grabber = framegrabber.TextureGrabber()
        self.grabber.grab(self.texture)

        #: x pixels between each vertex (float)
        self.x_step = width / self.grid.x
        #: y pixels between each vertex (float)
        self.y_step = height / self.grid.y

        self._init()

    def before_draw( self ):
        '''Binds the framebuffer to a texture
        and set a 2d projection before binding
        to prevent calculating a new texture
        '''

        self._set_2d_projection()

        # capture before drawing
        self.grabber.before_render(self.texture)

    def after_draw( self, camera ):
        '''Called by CocosNode when the texture is already grabbed.
        The FrameBuffer will be unbound and the texture will be drawn

        :Parameters:
            `camera` : `Camera`
                The target's camera object.
        '''

        # capture after drawing
        self.grabber.after_render(self.texture)

        # after unbinding
        # set a 3d projection
        self._set_3d_projection()

        # and center the camera
        camera.locate( force=True )

        # blit
        glEnable(self.texture.target)
        glBindTexture(self.texture.target, self.texture.id)

        glPushAttrib(GL_COLOR_BUFFER_BIT)

        self._blit()

        glPopAttrib()
        glDisable(self.texture.target)

    def _set_active(self, bool):
        if self._active == bool:
            return
        self._active = bool
        if self._active == True:
            pass
        elif self._active == False:
            self.vertex_list.delete()
            # to restore the camera to default position
            director.set_projection()
        else:
            raise Exception("Invalid value for GridBase.active")

    def _get_active(self):
        return self._active

    active = property(_get_active, _set_active,
                      doc='''Determines whether the grid is active or not
                     :type: bool
                     ''')
    def _init(self):
        raise NotImplementedError('abstract')

    def _blit(self):
        raise NotImplementedError('abstract')

    def _on_resize(self):
        raise NotImplementedError('abstract')

    @classmethod
    def _set_3d_projection(cls):
        glViewport(director._offset_x, director._offset_y, director._usable_width, director._usable_height)
        glMatrixMode(GL_PROJECTION)
        glLoadIdentity()
        gluPerspective(60, 1.0*director._usable_width/director._usable_height, 0.1, 3000.0)
        glMatrixMode(GL_MODELVIEW)

    @classmethod
    def _set_2d_projection(cls):

#        director.set_2d_projection()
        width, height = director.get_window_size()
        glLoadIdentity()
        glViewport(0, 0, width, height)
        glMatrixMode(GL_PROJECTION)
        glLoadIdentity()
        glOrtho(0, width, 0, height, -100, 100)
        glMatrixMode(GL_MODELVIEW)

class Grid3D(GridBase):
    '''`Grid3D` is a 3D grid implementation. Each vertex has 3 dimensions: x,y,z

    The vindexed ertex array will be built with::

        self.vertex_list.vertices: x,y,z (floats)
        self.vertex_list.tex_coords: x,y,z (floats)
        self.vertex_list.colors: RGBA, with values from 0 - 255
    '''

    def _init( self ):
        # calculate vertex, textures depending on screen size
        idx_pts, ver_pts_idx, tex_pts_idx = self._calculate_vertex_points()

        #: indexed vertex array that can be transformed.
        #: it has these attributes:
        #:
        #:    - vertices
        #:    - colors
        #:    - tex_coords
        #:
        #: for more information refer to pyglet's documentation: pyglet.graphics.vertex_list_indexed
        self.vertex_list = pyglet.graphics.vertex_list_indexed( (self.grid.x+1) * (self.grid.y+1),
                            idx_pts, "t2f", "v3f/stream","c4B")

        #: original vertex array of the grid. (read-only)
        self.vertex_points = ver_pts_idx[:]
        self.vertex_list.vertices = ver_pts_idx
        self.vertex_list.tex_coords = tex_pts_idx
        self.vertex_list.colors = (255,255,255,255) * (self.grid.x+1) * (self.grid.y+1)

    def _blit(self ):
        self.vertex_list.draw(pyglet.gl.GL_TRIANGLES)

    def _calculate_vertex_points(self):
        w = float(self.texture.width)
        h = float(self.texture.height)

        index_points = []
        vertex_points_idx = []
        texture_points_idx = []

        for x in xrange(0,self.grid.x+1):
            for y in xrange(0,self.grid.y+1):
                vertex_points_idx += [-1,-1,-1]
                texture_points_idx += [-1,-1]

        for x in xrange(0, self.grid.x):
            for y in xrange(0, self.grid.y):
                x1 = x * self.x_step
                x2 = x1 + self.x_step
                y1 = y * self.y_step
                y2 = y1 + self.y_step

                #  d <-- c
                #        ^
                #        |
                #  a --> b
                a = x * (self.grid.y+1) + y
                b = (x+1) * (self.grid.y+1) + y
                c = (x+1) * (self.grid.y+1) + (y+1)
                d = x * (self.grid.y+1) + (y+1)

                # 2 triangles: a-b-d, b-c-d
                index_points += [ a, b, d, b, c, d]    # triangles

                l1 = ( a*3, b*3, c*3, d*3 )
                l2 = ( Point3(x1,y1,0), Point3(x2,y1,0), Point3(x2,y2,0), Point3(x1,y2,0) )

                #  building the vertex
                for i in xrange( len(l1) ):
                    vertex_points_idx[ l1[i] ] = l2[i].x
                    vertex_points_idx[ l1[i] + 1 ] = l2[i].y
                    vertex_points_idx[ l1[i] + 2 ] = l2[i].z

                # building the texels
                tex1 = ( a*2, b*2, c*2, d*2 )
                tex2 = ( Point2(x1,y1), Point2(x2,y1), Point2(x2,y2), Point2(x1,y2) )

                for i in xrange( len(tex1)):
                    texture_points_idx[ tex1[i] ] = tex2[i].x / w
                    texture_points_idx[ tex1[i] + 1 ] = tex2[i].y / h

        return ( index_points, vertex_points_idx, texture_points_idx )

    def get_vertex( self, x, y):
        '''Get the current vertex coordinate

        :Parameters:
            `x` : int
               x-vertex
            `y` : int
               y-vertex

        :rtype: (float, float, float)
        '''
        idx = (x * (self.grid.y+1) + y) * 3
        x = self.vertex_list.vertices[idx]
        y = self.vertex_list.vertices[idx+1]
        z = self.vertex_list.vertices[idx+2]
        return (x,y,z)

    def get_original_vertex( self, x, y):
        '''Get the original vertex coordinate.
        The original vertices are the ones weren't modified by the current action.

        :Parameters:
            `x` : int
               x-vertex
            `y` : int
               y-vertex

        :rtype: (float, float, float)
        '''
        idx = (x * (self.grid.y+1) + y) * 3
        x = self.vertex_points[idx]
        y = self.vertex_points[idx+1]
        z = self.vertex_points[idx+2]

        return (x,y,z)

    def set_vertex( self, x, y, v):
        '''Set a vertex point is a certain value

        :Parameters:
            `x` : int
               x-vertex
            `y` : int
               y-vertex
            `v` : (float, float, float)
                tuple value for the vertex
        '''
        idx = (x * (self.grid.y+1) + y) * 3
        self.vertex_list.vertices[idx] = int(v[0])
        self.vertex_list.vertices[idx+1] = int(v[1])
        self.vertex_list.vertices[idx+2] = int(v[2])

class TiledGrid3D(GridBase):
    '''`TiledGrid3D` is a 3D grid implementation. It differs from `Grid3D` in that
    the tiles can be separated from the grid.

    The vertex array will be built with::

        self.vertex_list.vertices: x,y,z (floats)
        self.vertex_list.tex_coords: x,y (floats)
        self.vertex_list.colors: RGBA, with values from 0 - 255
    '''
    def _init( self ):
        # calculate vertex, textures depending on screen size
        ver_pts, tex_pts = self._calculate_vertex_points()

        #: vertex array that can be transformed.
        #: it has these attributes:
        #:
        #:    - vertices
        #:    - colors
        #:    - tex_coords
        #:
        #: for more information refer to pyglet's documentation: pyglet.graphics.vertex_list
        self.vertex_list = pyglet.graphics.vertex_list(self.grid.x * self.grid.y * 4,
                            "t2f", "v3f/stream","c4B")
        #: original vertex array of the grid. (read-only)
        self.vertex_points = ver_pts[:]
        self.vertex_list.vertices = ver_pts
        self.vertex_list.tex_coords = tex_pts
        self.vertex_list.colors = (255,255,255,255) * self.grid.x * self.grid.y * 4

    def _blit(self ):
        self.vertex_list.draw(pyglet.gl.GL_QUADS)

    def _calculate_vertex_points(self):
        w = float(self.texture.width)
        h = float(self.texture.height)

        vertex_points = []
        texture_points = []

        for x in xrange(0, self.grid.x):
            for y in xrange(0, self.grid.y):
                x1 = x * self.x_step
                x2 = x1 + self.x_step
                y1 = y * self.y_step
                y2 = y1 + self.y_step

                # Building the tiles' vertex and texture points
                vertex_points += [x1, y1, 0, x2, y1, 0, x2, y2, 0, x1, y2, 0 ]
                texture_points += [x1/w, y1/h, x2/w, y1/h, x2/w, y2/h, x1/w, y2/h]

        # Generates a quad for each tile, to perform tiles effect
        return (vertex_points, texture_points)

    def set_tile(self, x, y, coords):
        '''Set the 4 tile coordinates

        Coordinates positions::

            3 <-- 2
                  ^
                  |
            0 --> 1

        :Parameters:
            `x` : int
                x coodinate of the tile
            `y` : int
                y coordinate of the tile
            `coords` : [ float, float, float, float, float, float, float, float, float, float, float, float ]
                The 4 coordinates in the format (x0, y0, z0, x1, y1, z1,..., x3, y3, z3)
        '''
        idx = (self.grid.y * x + y) * 4 * 3
        self.vertex_list.vertices[idx:idx+12] = coords

    def get_original_tile(self, x, y):
        '''Get the 4-original tile coordinates.

        Coordinates positions::

            3 <-- 2
                  ^
                  |
            0 --> 1

        :Parameters:
            `x` : int
                x coordinate of the tile
            `y` : int
                y coordinate of the tile

        :rtype: [ float, float, float, float, float, float, float, float, float, float, float, float ]
        :returns: The 4 coordinates with the following order: x0, y0, z0, x1, y1, z1,...,x3, y3, z3
        '''
        idx = (self.grid.y * x + y) * 4 * 3
        return self.vertex_points[idx:idx+12]

    def get_tile(self, x, y):
        '''Get the current tile coordinates.

        Coordinates positions::

            3 <-- 2
                  ^
                  |
            0 --> 1

        :Parameters:
            `x` : int
                x coordinate of the tile
            `y` : int
                y coordinate of the tile

        :rtype: [ float, float, float, float, float, float, float, float, float, float, float, float ]
        :returns: The 4 coordinates with the following order: x0, y0, z0, x1, y1, z1,...,x3, y3, z3
        '''
        idx = (self.grid.y * x + y) * 4 * 3
        return self.vertex_list.vertices[idx:idx+12]
<|endoftext|>