<commit_before># -*- coding: utf-8 -*-

import os

try:
  # This part is surrounded in try/except because the config.py file is
  # also used in the run.py script which is used to compile/minify the client
  # side files (*.less, *.coffee, *.js) and is not aware of the GAE
  from datetime import datetime
  import model
  CONFIG_DB = model.Config.get_master_db()
  SECRET_KEY = CONFIG_DB.flask_secret_key.encode('ascii')
  CURRENT_VERSION_ID = os.environ.get('CURRENT_VERSION_ID')
  CURRENT_VERSION_NAME = CURRENT_VERSION_ID.split('.')[0]
  CURRENT_VERSION_TIMESTAMP = long(CURRENT_VERSION_ID.split('.')[1]) >> 28
  CURRENT_VERSION_DATE = datetime.fromtimestamp(CURRENT_VERSION_TIMESTAMP)
except:
  pass

PRODUCTION = os.environ.get('SERVER_SOFTWARE', '').startswith('Google App Eng')
DEVELOPMENT = not PRODUCTION
DEBUG = DEVELOPMENT

DEFAULT_DB_LIMIT = 64

###############################################################################
# Client modules, also used by the run.py script.
###############################################################################
STYLES = [
    'src/style/style.less',
  ]

SCRIPTS_MODULES = [
    'libs',
    'scripts',
  ]

SCRIPTS = {
    'libs': [
        'src/lib/jquery.js',
        'src/lib/moment.js',
        'src/lib/nprogress/nprogress.js',
        'src/lib/bootstrap/js/alert.js',
        'src/lib/bootstrap/js/button.js',
        'src/lib/bootstrap/js/transition.js',
        'src/lib/bootstrap/js/collapse.js',
        'src/lib/bootstrap/js/dropdown.js',
        'src/lib/bootstrap/js/tooltip.js',
      ],
    'scripts': [
        'src/script/common/service.coffee',
        'src/script/common/util.coffee',
        'src/script/site/app.coffee',
        'src/script/site/admin.coffee',
        'src/script/site/profile.coffee',
        'src/script/site/user.coffee',
      ],
  }
<commit_msg>Added CURRENT_APP_NAME to config.py<commit_after># -*- coding: utf-8 -*-

import os

try:
  # This part is surrounded in try/except because the config.py file is
  # also used in the run.py script which is used to compile/minify the client
  # side files (*.less, *.coffee, *.js) and is not aware of the GAE
  from datetime import datetime
  from google.appengine.api import app_identity
  import model

  CONFIG_DB = model.Config.get_master_db()
  SECRET_KEY = CONFIG_DB.flask_secret_key.encode('ascii')
  CURRENT_VERSION_ID = os.environ.get('CURRENT_VERSION_ID')
  CURRENT_VERSION_NAME = CURRENT_VERSION_ID.split('.')[0]
  CURRENT_VERSION_TIMESTAMP = long(CURRENT_VERSION_ID.split('.')[1]) >> 28
  CURRENT_VERSION_DATE = datetime.fromtimestamp(CURRENT_VERSION_TIMESTAMP)
  CURRENT_APP_NAME = app_identity.get_application_id()
except:
  pass

PRODUCTION = os.environ.get('SERVER_SOFTWARE', '').startswith('Google App Eng')
DEVELOPMENT = not PRODUCTION
DEBUG = DEVELOPMENT

DEFAULT_DB_LIMIT = 64

###############################################################################
# Client modules, also used by the run.py script.
###############################################################################
STYLES = [
    'src/style/style.less',
  ]

SCRIPTS_MODULES = [
    'libs',
    'scripts',
  ]

SCRIPTS = {
    'libs': [
        'src/lib/jquery.js',
        'src/lib/moment.js',
        'src/lib/nprogress/nprogress.js',
        'src/lib/bootstrap/js/alert.js',
        'src/lib/bootstrap/js/button.js',
        'src/lib/bootstrap/js/transition.js',
        'src/lib/bootstrap/js/collapse.js',
        'src/lib/bootstrap/js/dropdown.js',
        'src/lib/bootstrap/js/tooltip.js',
      ],
    'scripts': [
        'src/script/common/service.coffee',
        'src/script/common/util.coffee',
        'src/script/site/app.coffee',
        'src/script/site/admin.coffee',
        'src/script/site/profile.coffee',
        'src/script/site/user.coffee',
      ],
  }
<|endoftext|>


<commit_before>from django.db import models
from django.db.models import signals
# from validatedfile.fields import ValidatedFileField
from django.contrib.auth.models import User
from django.utils.text import slugify

# Receive the pre_delete signal and delete the file associated with the model instance.
from django.db.models.signals import pre_delete
from django.dispatch.dispatcher import receiver

import sys, os, time, yaml, errno
from uuid import uuid1, uuid4

from taggit.managers import TaggableManager
from host.settings import MEDIA_ROOT

from validatedfile.fields import ValidatedFileField
validatedfile_rules = [(
    [ValidatedFileField],
    [],
    {
        "content_types": ["content_types", {"default": []}],
        "max_upload_size": ["max_upload_size", {"default": 0}],
        "mime_lookup_length": ["mime_lookup_length", {"default": 4096}],
    }
)]

from south.modelsinspector import add_introspection_rules
add_introspection_rules(validatedfile_rules, ["^validatedfile\.fields\.ValidatedFileField"])

def mkdir_p(path, perms=None):
    try:
        if perms:
            os.makedirs(path, perms)
        else:
            os.makedirs(path)
    except OSError as exc: # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else: raise

def path_for_game_file():
    def path_generator(instance, filename):
        ext = filename.split('.')[-1]
        basename = filename.split('.')[0]
        # set path.
        if instance.game_group:
            library_dir = slugify(instance.game_group.name) if instance.game_group else "ungrouped"
            path = "uploads/{library_dir}/games/".format(library_dir=library_dir)
        else:
            path = "uploads/unclaimed/" # dump problematic files where we can see them.

        # set permissions on path.
        mkdir_p(MEDIA_ROOT + path, 0o775)

        # since this is going into the group directory, along with Library files, make sure the filename is unique.
        filename = "{0}_{1}.{2}".format(basename, int(time.time()), ext)

        # return the whole path to the file
        return os.path.join(path, filename)
    return path_generator

class Game(models.Model):
    name = models.CharField(max_length=255)
    description = models.TextField(blank=True, null=True)
    game_group = models.ForeignKey("GameGroup")
    game_spec = ValidatedFileField(
                        null = True,
                        blank = True,
                        upload_to = path_for_game_file(),
                        max_length = 250,
                        max_upload_size = 204800, # 200KB
                        content_types = ['application/x-yaml','text/yaml','text/plain','text/plain; charset=us-ascii', 'text/plain; charset=utf-8'])
    tags = TaggableManager(blank=True)

    def slug(self):
        return slugify(self.name)

    def __unicode__(self):
        return self.name

    def save(self, *args, **kwargs):
        # create Round objects as nec. set a 'parsed' flag to true when done.
        super(Game, self).save(*args, **kwargs)

        game_yaml_file_path = MEDIA_ROOT + str(self.game_spec)
        game_yaml_file = None
        try:
            game_yaml_file = open(game_yaml_file_path, 'r')
            game_yaml = yaml.load(game_yaml_file)

            # for a first stab at game YAML validation,
            # we try to create a Round object for each Round in the uploaded file.
            # if this fails, we pass back a validation error.
            rounds = game_yaml.get('rounds')
            for i, round in enumerate(rounds):
                rnd, created = Round.objects.get_or_create(game=self, nbr=(i+1), points=value)
                if created:
                    rnd.save()

            # if we've gotten this far, note it in the db.
            self.parsed = True
            super(Game, self).save(*args, **kwargs)

        except:
            e = sys.exc_info()[0]
            print( "Error: %s" % e )

        finally:
            if (game_yaml_file):
                game_yaml_file.close()

@receiver(pre_delete, sender=Game)
def game_delete(sender, instance, **kwargs):
    # Pass false so FileField doesn't save the model.
    instance.game_spec.delete(False)

class Round(models.Model):
    game = models.ForeignKey(Game)
    nbr = models.IntegerField()
    correct_answer = models.CharField(max_length=32)
    points = models.IntegerField()

class Struct:
    def __init__(self, **entries):
        self.__dict__.update(entries)

class GameReport(models.Model):
    payload = models.TextField(blank=True,null=True)
    student = models.ForeignKey(User, unique=False)
    game = models.ForeignKey(Game, unique=False)
    created_on = models.DateTimeField(auto_now_add=True)

    def _decode_payload(self):
        args = json.loads(self.payload)
        s = Struct(**args)
        return s
    decoded_payload = property(_decode_payload)

class GameGroup(models.Model):
    name = models.CharField(max_length=255)
    games = models.ManyToManyField(Game, blank=True, null=True)
    users = models.ManyToManyField(User, blank=True, null=True, through="Membership")
    library_files = models.ManyToManyField("LibraryFile", blank=True, null=True)

    def slug(self):
        return slugify(self.name)

    def __unicode__(self):
        return self.name

class Role(models.Model):
    name = models.CharField(max_length=64)

    def __unicode__(self):
        return self.name

class Membership(models.Model):
    user = models.ForeignKey(User)
    game_group = models.ForeignKey(GameGroup)
    role = models.ForeignKey(Role)

    def __unicode__(self):
        group_name = self.game_group.name
        if group_name==None:
            group_name = "Unknown Group"
        role_name = self.role.name
        if role_name==None:
            role_name = "Unknown Role"

        return "{}:{}".format(group_name, role_name)

    def games(self):
        return self.game_group.games

def path_for_media_type():
    def path_generator(instance, filename):
        ext = filename.split('.')[-1]
        # set path.
        if instance.game_group:
            library_dir = slugify(instance.game_group.name) if instance.game_group else "ungrouped"
            path = "uploads/{library_dir}/custom_{media_type}/".format(library_dir=library_dir, media_type=instance.media_type)
        else:
            path = "uploads/unclaimed/" # dump problematic files where we can see them.

        # return the whole path to the file
        return os.path.join(path, filename)
    return path_generator

class LibraryFile(models.Model):
    MEDIA_CHOICES = (
        ('js', 'js'),
    )
    media_type = models.CharField(max_length=64, choices=MEDIA_CHOICES)
    description = models.TextField(blank=True, null=True)
    game_group = models.ForeignKey(GameGroup)
    file = models.FileField(
                        null = True,
                        blank = True,
                        upload_to = path_for_media_type())

    def __unicode__(self):
        return self.file.name

@receiver(pre_delete, sender=LibraryFile)
def library_file_delete(sender, instance, **kwargs):
    # Pass false so FileField doesn't save the model.
    instance.file.delete(False)
<commit_msg>fix bug in Membership model __unicode__(), when group or role is undefined.<commit_after>from django.db import models
from django.db.models import signals
# from validatedfile.fields import ValidatedFileField
from django.contrib.auth.models import User
from django.utils.text import slugify

# Receive the pre_delete signal and delete the file associated with the model instance.
from django.db.models.signals import pre_delete
from django.dispatch.dispatcher import receiver

import sys, os, time, yaml, errno
from uuid import uuid1, uuid4

from taggit.managers import TaggableManager
from host.settings import MEDIA_ROOT

from validatedfile.fields import ValidatedFileField
validatedfile_rules = [(
    [ValidatedFileField],
    [],
    {
        "content_types": ["content_types", {"default": []}],
        "max_upload_size": ["max_upload_size", {"default": 0}],
        "mime_lookup_length": ["mime_lookup_length", {"default": 4096}],
    }
)]

from south.modelsinspector import add_introspection_rules
add_introspection_rules(validatedfile_rules, ["^validatedfile\.fields\.ValidatedFileField"])

def mkdir_p(path, perms=None):
    try:
        if perms:
            os.makedirs(path, perms)
        else:
            os.makedirs(path)
    except OSError as exc: # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else: raise

def path_for_game_file():
    def path_generator(instance, filename):
        ext = filename.split('.')[-1]
        basename = filename.split('.')[0]
        # set path.
        if instance.game_group:
            library_dir = slugify(instance.game_group.name) if instance.game_group else "ungrouped"
            path = "uploads/{library_dir}/games/".format(library_dir=library_dir)
        else:
            path = "uploads/unclaimed/" # dump problematic files where we can see them.

        # set permissions on path.
        mkdir_p(MEDIA_ROOT + path, 0o775)

        # since this is going into the group directory, along with Library files, make sure the filename is unique.
        filename = "{0}_{1}.{2}".format(basename, int(time.time()), ext)

        # return the whole path to the file
        return os.path.join(path, filename)
    return path_generator

class Game(models.Model):
    name = models.CharField(max_length=255)
    description = models.TextField(blank=True, null=True)
    game_group = models.ForeignKey("GameGroup")
    game_spec = ValidatedFileField(
                        null = True,
                        blank = True,
                        upload_to = path_for_game_file(),
                        max_length = 250,
                        max_upload_size = 204800, # 200KB
                        content_types = ['application/x-yaml','text/yaml','text/plain','text/plain; charset=us-ascii', 'text/plain; charset=utf-8'])
    tags = TaggableManager(blank=True)

    def slug(self):
        return slugify(self.name)

    def __unicode__(self):
        return self.name

    def save(self, *args, **kwargs):
        # create Round objects as nec. set a 'parsed' flag to true when done.
        super(Game, self).save(*args, **kwargs)

        game_yaml_file_path = MEDIA_ROOT + str(self.game_spec)
        game_yaml_file = None
        try:
            game_yaml_file = open(game_yaml_file_path, 'r')
            game_yaml = yaml.load(game_yaml_file)

            # for a first stab at game YAML validation,
            # we try to create a Round object for each Round in the uploaded file.
            # if this fails, we pass back a validation error.
            rounds = game_yaml.get('rounds')
            for i, round in enumerate(rounds):
                rnd, created = Round.objects.get_or_create(game=self, nbr=(i+1), points=value)
                if created:
                    rnd.save()

            # if we've gotten this far, note it in the db.
            self.parsed = True
            super(Game, self).save(*args, **kwargs)

        except:
            e = sys.exc_info()[0]
            print( "Error: %s" % e )

        finally:
            if (game_yaml_file):
                game_yaml_file.close()

@receiver(pre_delete, sender=Game)
def game_delete(sender, instance, **kwargs):
    # Pass false so FileField doesn't save the model.
    instance.game_spec.delete(False)

class Round(models.Model):
    game = models.ForeignKey(Game)
    nbr = models.IntegerField()
    correct_answer = models.CharField(max_length=32)
    points = models.IntegerField()

class Struct:
    def __init__(self, **entries):
        self.__dict__.update(entries)

class GameReport(models.Model):
    payload = models.TextField(blank=True,null=True)
    student = models.ForeignKey(User, unique=False)
    game = models.ForeignKey(Game, unique=False)
    created_on = models.DateTimeField(auto_now_add=True)

    def _decode_payload(self):
        args = json.loads(self.payload)
        s = Struct(**args)
        return s
    decoded_payload = property(_decode_payload)

class GameGroup(models.Model):
    name = models.CharField(max_length=255)
    games = models.ManyToManyField(Game, blank=True, null=True)
    users = models.ManyToManyField(User, blank=True, null=True, through="Membership")
    library_files = models.ManyToManyField("LibraryFile", blank=True, null=True)

    def slug(self):
        return slugify(self.name)

    def __unicode__(self):
        return self.name

class Role(models.Model):
    name = models.CharField(max_length=64)

    def __unicode__(self):
        return self.name

class Membership(models.Model):
    user = models.ForeignKey(User)
    game_group = models.ForeignKey(GameGroup)
    role = models.ForeignKey(Role)

    def __unicode__(self):
        group_name = self.game_group.name
        role_name = self.role.name
        try:
            return "{}:{}".format(group_name, role_name)
        except:
            return "unknown"

    def games(self):
        return self.game_group.games

def path_for_media_type():
    def path_generator(instance, filename):
        ext = filename.split('.')[-1]
        # set path.
        if instance.game_group:
            library_dir = slugify(instance.game_group.name) if instance.game_group else "ungrouped"
            path = "uploads/{library_dir}/custom_{media_type}/".format(library_dir=library_dir, media_type=instance.media_type)
        else:
            path = "uploads/unclaimed/" # dump problematic files where we can see them.

        # return the whole path to the file
        return os.path.join(path, filename)
    return path_generator

class LibraryFile(models.Model):
    MEDIA_CHOICES = (
        ('js', 'js'),
    )
    media_type = models.CharField(max_length=64, choices=MEDIA_CHOICES)
    description = models.TextField(blank=True, null=True)
    game_group = models.ForeignKey(GameGroup)
    file = models.FileField(
                        null = True,
                        blank = True,
                        upload_to = path_for_media_type())

    def __unicode__(self):
        return self.file.name

@receiver(pre_delete, sender=LibraryFile)
def library_file_delete(sender, instance, **kwargs):
    # Pass false so FileField doesn't save the model.
    instance.file.delete(False)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import hashlib

class BaseX(object):
  @classmethod
  def retrieve_one_by(cls, name, value):
    cls_db_list = cls.query(getattr(cls, name) == value).fetch(1)
    if cls_db_list:
      return cls_db_list[0]
    return None

class ConfigX(object):
  @classmethod
  def get_master_db(cls):
    return cls.get_or_insert('master')

class UserX(object):
  @property
  def avatar_url(self):
    return '//gravatar.com/avatar/%s?d=identicon&r=x' % (
        hashlib.md5((self.email or self.name).encode('utf-8')).hexdigest().lower()
      )
<commit_msg>Support for size of gravatar image<commit_after># -*- coding: utf-8 -*-

import hashlib

class BaseX(object):
  @classmethod
  def retrieve_one_by(cls, name, value):
    cls_db_list = cls.query(getattr(cls, name) == value).fetch(1)
    if cls_db_list:
      return cls_db_list[0]
    return None

class ConfigX(object):
  @classmethod
  def get_master_db(cls):
    return cls.get_or_insert('master')

class UserX(object):
  @property
  def avatar_url(self, size=80):
    return '//gravatar.com/avatar/%s?d=identicon&r=x&s=%d' % (
        hashlib.md5((self.email or self.name).encode('utf-8')).hexdigest().lower(),
        size
      )
<|endoftext|>


<commit_before>#!/usr/bin/env python

# # write-good-py
# ## Inspired by Brian Ford
#
# This is designed as a naive linter for the English Language, using specific modules.
# This program is also developed in literal form, where the program is it's own documentation.
# (Inspired by CoffeeScript Literal programs.)

# # Development Notes:
#
# * No Tabs used, 4 Spaces Instead
# * Dependencies should be as few as possible.
# * Unlike the original write-good, developing this project as a single script file, ala Bottle.py, is preferable.
# *It allows it to be a drop-in file, to get up and running inside any other project out there.*
# * When commenting code, the comment goes above the line that is being commented about.
# * Try and use a Markdown flavour of code, though beginning each line with a '#' symbol.
# * Flags should only be utilised for functions that are optional to the core function. (If in doubt, go to [Issues](https://github.com/shakna-israel/write-good-py/issues)

# # Versioning Notes:
#
# This project was developed under Python 2.7 originally, though support for Python 3.x is incoming.
# But as of this commit, it remains untested.

# # Dependencies
#
import sys
import os

# Debugging tools
global verbose
verbose = False

# Print a helpful statement about arguments.
def help():
    # Fetch the global verbose setting, to allow for complex debugging
    global verbose
    # Print a simple help statement of what to do.
    print "./write-good.py FILENAME"

# Grab a file from the commandline argument, and store it into memory:
def get_file():
    # Fetch the global verbose setting, to allow for complex debugging
    global verbose
    # Check if any commandline arguments are issued to the linter.
    try:
        # Expect the user to supply the filename as the first argument.
        # This lets the developer integrate write-good-py into all sorts of scripting environments, and doesn't put the weight of interfacing at this end.
        # Also means that we don't need a dozen flags. Flags are for optional things.
        if sys.argv[1]:
            FILENAME = sys.argv[1]
        else:
            if verbose:
                print sys.argv[1]
        # Check if the argument the user has given is actually a file.
        if os.path.isfile(FILENAME):
            # Attempt to open the file, if it fails, throw an exception.
            try:
                file = open(FILENAME,"r")
                # For each line in the file, let's run the annotate function.
                for line in file.read().split('\n'):
                    annotate(file)
                # Close the file to remove any locks.
                file.close()
            except:
                raise Exception, "File could not be opened. Do you have permissions?"
            except Exception as exception:
                print "Exception: %s" % exception
        else:
            # If the user has given an argument that is not a file, raise an exception for that.
            try:
                raise Exception, "File either does not exist, is not a file, or you do not have permissions to the file."
            except Exception as exception:
                print "Exception: %s" % exception
                help()
    # If no arguments are found, then run a function for that.
    except IndexError as exception:
        if verbose:
            print "Exception: %s" % exception
        help()
    # If any other exceptions come up, send them to the verbose system, rather than crashing on them.
    except:
        if verbose:
            print "Except: %s" % exception
            help()

# This is not the final function. This program will have a main function, and that will be the final function to handle everything and which way to take things. It's just for testing.
get_file()
<commit_msg>Enabled Verbose Mode from CLI<commit_after>#!/usr/bin/env python

# # write-good-py
# ## Inspired by Brian Ford
#
# This is designed as a naive linter for the English Language, using specific modules.
# This program is also developed in literal form, where the program is it's own documentation.
# (Inspired by CoffeeScript Literal programs.)

# # Development Notes:
#
# * No Tabs used, 4 Spaces Instead
# * Dependencies should be as few as possible.
# * Unlike the original write-good, developing this project as a single script file, ala Bottle.py, is preferable.
# *It allows it to be a drop-in file, to get up and running inside any other project out there.*
# * When commenting code, the comment goes above the line that is being commented about.
# * Try and use a Markdown flavour of code, though beginning each line with a '#' symbol.
# * Flags should only be utilised for functions that are optional to the core function. (If in doubt, go to [Issues](https://github.com/shakna-israel/write-good-py/issues)

# # Versioning Notes:
#
# This project was developed under Python 2.7 originally, though support for Python 3.x is incoming.
# But as of this commit, it remains untested.

# # Dependencies
#
import sys
import os

# Debugging tools
global verbose
verbose = False

# Check if verbose enabled
def check_verbose():
    global verbose
    # Try and see if a flag exists
    try:
        # If a flag exists, and is *-v*
        if sys.argv[2] == "-v":
            # Make Verbose Mode True.
            verbose = True
            # If we're in verbose mode, enable it.
            if verbose == True:
                print "Verbose mode enabled."
        else:
            # If the flag isn't *-v*, set Verbose Mode to disabled.
            verbose = False
    except:
        # If there is no flag, set Verbose Mode to disabled.
        verbose = False

# Print a helpful statement about arguments.
def help():
    # Fetch the global verbose setting, to allow for complex debugging
    global verbose
    # Print a simple help statement of what to do.
    print "./write-good.py FILENAME -FLAG"
    print "-v is Verbose Mode"
# Grab a file from the commandline argument, and store it into memory:
def get_file():
    # Fetch the global verbose setting, to allow for complex debugging
    global verbose
    # Check if any commandline arguments are issued to the linter.
    try:
        # Expect the user to supply the filename as the first argument.
        # This lets the developer integrate write-good-py into all sorts of scripting environments, and doesn't put the weight of interfacing at this end.
        # Also means that we don't need a dozen flags. Flags are for optional things.
        if sys.argv[1]:
            FILENAME = sys.argv[1]
        else:
            if verbose:
                print sys.argv[1]
        # Check if the argument the user has given is actually a file.
        if os.path.isfile(FILENAME):
            # Attempt to open the file, if it fails, throw an exception.
            try:
                file = open(FILENAME,"r")
                # For each line in the file, let's run the annotate function.
                for line in file.read().split('\n'):
                    annotate(file)
                # Close the file to remove any locks.
                file.close()
            except Exception as exception:
                raise Exception, "File could not be opened. Do you have permissions?"
                print "Exception: %s" % exception
        else:
            # If the user has given an argument that is not a file, raise an exception for that.
            try:
                raise Exception, "File either does not exist, is not a file, or you do not have permissions to the file."
            except Exception as exception:
                print "Exception: %s" % exception
                help()
    # If no arguments are found, then run a function for that.
    except IndexError as exception:
        if verbose:
            print "Exception: %s" % exception
        help()
    # If any other exceptions come up, send them to the verbose system, rather than crashing on them.
    except:
        if verbose:
            print "Except: %s" % exception
            help()

# This is not the final function. This program will have a main function, and that will be the final function to handle everything and which way to take things. It's just for testing.
check_verbose()
get_file()
<|endoftext|>


<commit_before>from flask import Flask
from werkzeug.wsgi import DispatcherMiddleware

application = DispatcherMiddleware(Flask(__name__), {
    '/one': Flask(__name__),
    '/two': Flask(__name__),
})

if __name__ == '__main__':
    from werkzeug.serving import run_simple
    run_simple('localhost', 5000, application, use_reloader=True)
<commit_msg>Add the idea.<commit_after>from flask import Flask, request
from werkzeug.wsgi import DispatcherMiddleware
from werkzeug.test import create_environ, run_wsgi_app

class FlaskDrunk(Flask):
    def __init__(self, import_name, *args, **kwargs):
        super(FlaskDrunk, self).__init__(import_name, *args, **kwargs)
        self.add_url_rule('/invite/', endpoint='party', view_func=self.party)
        self.dispatcher = None
        self.drunks = []

    def party(self):
        self.dispatcher = request.environ.get('mc_dispatcher')
        self.dispatcher.attendees.append(self)
        self.drunks = self.dispatcher.attendees
        return repr(self)

    @property
    def buddies(self):
        return [buddy for buddy in self.drunks if buddy is not self]

    def receive(self, sender, message):
        if message == ('ping', None):
            return ('pong', None)

class MC(DispatcherMiddleware):
    def __init__(self, app, mounts=None, base_url=None):
        super(MC, self).__init__(app, mounts=mounts)
        self.base_url = None
        self.attendees = []
        environ = create_environ(path='/invite/', base_url=self.base_url)
        environ['mc_dispatcher'] = self
        for application in [app] + mounts.values():
            run_wsgi_app(application, environ)

# Demonstrate.
root = FlaskDrunk(__name__)
one = FlaskDrunk(__name__)
two = FlaskDrunk(__name__)

root.debug = True
one.debug = True
two.debug = True

@root.route('/', endpoint='index')
def root_index():
    if not root.buddies:
        return 'I have no friends.'
    buddy = root.buddies[0]
    received = buddy.receive(root, ('ping', None))
    return 'received %r from %r' % (received, buddy)

application = MC(root, {
    '/one': one,
    '/two': two,
})

if __name__ == '__main__':
    from werkzeug.serving import run_simple
    run_simple('localhost', 5000, application, use_reloader=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# name:             website.py
# author:           Harold Bradley III
# email:            harold@bradleystudio.net
# date:             11/11/2015
#
# description:      A class to manage websites
#

try:
    from ext_pylib.files import Dir, File
    from ext_pylib.prompt import prompt, prompt_str, warn_prompt
except ImportError:
    raise ImportError('ext_pylib must be installed to run ww')

from htaccess import Htaccess
import os
import tarfile
from vhost import Vhost
from website_domain import WebsiteDomain
from ww import settings as s

# localhost(function)
#   a python decorator that creates a temporary host entry before
#   the function is called and removes it after it is completed.
def localhost(function):
    def function_wrapper(self, *args, **kwargs):
        print 'Adding temporary host entry.'
        os.system("echo '127.0.0.1 " + self.domain + "' | cat >> /etc/hosts")
        function(self, *args, **kwargs)
        print 'Removing temporary host entry.'
        os.system("sed -i '/^127\.0\.0\.1 " + self.domain + "$/d' /etc/hosts")
    return function_wrapper

# merge_atts(atts, new_atts)
#   Merges two dictionaries with the second overwriting the corresponding
#   values of the first and returns the result.
def merge_atts(atts, new_atts):
    for k, v in new_atts.iteritems():
        if (k in atts and isinstance(atts[k], dict)
                and isinstance(new_atts[k], dict)):
            atts[k] = merge_atts(atts[k], new_atts[k])
        else:
            atts[k] = new_atts[k]
    return atts

# Website(website)
#   a class that describes a generic website with the following properties:
#       domain      Website domain
#       htdocs      Website root directory
#       assets      Website assets directory
#       logs        Website log directory
#       access_log  Website access log file
#       error_log   Website error log file
#       vhost       Apache vhost config file
#       htaccess    Website root htaccess file
#   methods:
#       install()  - installs a new website
#       remove()  - removes an existing website
#       pack  - packs an existing website
#       unpack  - unpacks an existing packed website into current server
#       migrate  - migrates a website from existing settings to default
#                  settings
#       verify()  - verifies a website installation
#       repair()  - verifies website forcing repairs
class Website(object):

    def __init__(self, domain, atts = {}):
        """Initializes a new Website instance."""

        print '[*] SSL not yet implemented in Website class.'

        self.domain = WebsiteDomain(domain)
        self.dirs = {}
        self.files = {}

        default_atts = {
                'root' : {
                    'path'  : s.WWW_DIR + self.domain,
                    'perms' : 0775,
                    'owner' : s.WWW_USR,
                    'group' : s.WWW_USR,
                },
                'htdocs' : {
                    'path'  : s.WWW_DIR + self.domain + '/htdocs/',
                    'perms' : 0775,
                    'owner' : s.WWW_USR,
                    'group' : s.WWW_USR,
                },
                'assets' : {
                    'path'  : s.WWW_DIR + self.domain + '/assets/',
                    'perms' : 0775,
                    'owner' : 'root',
                    'group' : s.WWW_ADMIN,
                },
                'logs' : {
                    'path'  : s.WWW_DIR + self.domain + '/log/',
                    'perms' : 0770,
                    'owner' : 'root',
                    'group' : s.WWW_ADMIN,
                },
                'access_log' : {
                    'path'  : None,
                    'perms' : 0750,
                    'owner' : 'root',
                    'group' : s.WWW_ADMIN,
                },
                'error_log' : {
                    'path'  : None,
                    'perms' : 0750,
                    'owner' : 'root',
                    'group' : s.WWW_ADMIN,
                },
                'vhost' : {
                    'path'  : s.VHOST_PATH + self.domain + '.conf',
                    'perms' : 0644,
                    'owner' : 'root',
                    'group' : 'root',
                },
                'htaccess' : {
                    'path'  : None,
                    'perms' : 0664,
                    'owner' : s.WWW_USR,
                    'group' : s.WWW_USR,
                    'h5g'   : True,
                },
            }

        atts = merge_atts(default_atts, atts)

        self.vhost = Vhost(self.domain.name, atts['vhost'])

        # If an apache vhost config already exists, try to parse it
        if self.vhost.exists():
            print str(self.vhost) + ' already exists.'
            if prompt('Parse existing vhost configuration?'):
                atts = merge_atts(atts, self.vhost.parse())
        # TODO: Convert...or migrate?

        self.root   = Dir(atts['root'])
        self.htdocs = Dir(atts['htdocs'])
        self.assets = Dir(atts['assets'])
        self.logs   = Dir(atts['logs'])

        # Set path of htaccess, access_log, and error_log after htdocs is set.
        if not atts['htaccess']['path']:
            atts['htaccess']['path'] = self.htdocs.path + '.htaccess'
        if not atts['access_log']['path']:
            atts['access_log']['path'] = self.logs.path + s.SITE_ACCESS_LOG
        if not atts['error_log']['path']:
            atts['error_log']['path'] = self.logs.path + s.SITE_ERROR_LOG

        self.htaccess  = Htaccess(atts['htaccess'])
        self.access_log = File(atts['access_log'])
        self.error_log = File(atts['error_log'])

        # Setup vhost placeholders after attributes have been defined
        self.vhost.placeholders = {
                '#WEBSITE#'    : self.domain.name,
                '#HTDOCS#'     : self.htdocs.path,
                '#EMAIL#'      : s.SITE_ADMIN_EMAIL,
                '#ACCESS_LOG#' : self.access_log.path,
                '#ERROR_LOG#'  : self.error_log.path
            }

    def __str__(self):
        """Returns a string with relevant instance information."""
        string =  '\n\n----------------------------------------------------------'
        string += '\n                        - Website -'
        string += '\n----------------------------------------------------------'
        string += '\n  domain:           ' + str(self.domain)
        string += '\n  vhost config:     ' + str(self.vhost)
        string += '\n  htdocs directory: ' + str(self.htdocs)
        string += '\n  assets:           ' + str(self.assets)
        string += '\n  logs:             ' + str(self.logs)
        string += '\n----------------------------------------------------------\n'
        return string

    def __repr__(self):
        """Returns a python string that evaluates to the object instance."""
        return "%s('%s', {'htdocs' : %s, 'assets' : %s, 'logs' : %s, " + \
               "'access_log' : %s, 'error_log' : %s, 'htaccess' : %s, 'vhost' : %s})" % (
            self.__class__.__name__,
            self.domain,
            self.htdocs._atts_(),
            self.assets._atts_(),
            self.logs._atts_(),
            self.access_log._atts_(),
            self.error_log._atts_(),
            self.htaccess._atts_(),
            self.vhost._atts_()
            )

    ################
    # Public Methods

    def install(self):
        """Installs website to server"""
        # Check if domain is already installed
        if self.is_installed():
            print self.domain + ' is already installed.'

        self.root.create()
        self.htdocs.create()
        self.assets.create()
        self.logs.create()
        self.access_log.create()
        self.error_log.create()
        self.htaccess.create()

        self.vhost.create()

        self.domain.set_ip()

        print str(self)
        print 'Installation complete.'

    def remove(self, ask=True):
        """Removes website from server"""
        self.vhost.disable(ask)

        self.vhost.remove(ask)
        self.access_log.remove(ask)
        self.error_log.remove(ask)
        self.htaccess.remove(ask)
        self.logs.remove(ask)
        self.htdocs.remove(ask)
        self.assets.remove(ask)
        self.root.remove(ask)

        self.domain.set_ip()

    def pack(self):
        pass

    def unpack(self):
        pass

    def migrate(self, old_website = None):
        if not old_website:
            old_website = self.existing
        # make sure there's nothing conflicting
        # install() new
        # this overwrites vhostconf...
        # for dirs and files in old copy
        # this includes vhost...

    def verify(self, repair = False):
        """Verifies website installation"""
        result = all([self.vhost.verify(repair),
                      self.access_log.verify(repair),
                      self.error_log.verify(repair),
                      self.htaccess.verify(repair),
                      self.logs.verify(repair),
                      self.htdocs.verify(repair),
                      self.assets.verify(repair),
                      self.root.verify(repair),
                      self.domain.verify(repair)])
        print self
        return result

    def repair(self):
        """Repairs website installation"""
        print 'Repairing ' + self.domain + '...'
        if self.verify(True):
            print 'Repair completed successfully. [OK]'
        else:
            print 'Repair resulted in errors. [ERROR]'

    def is_installed(self):
        """Returns true if vhost is enabled"""
        return self.vhost.is_enabled()
<commit_msg>Updates ww in accordance with changes to ext_pylib.<commit_after>#!/usr/bin/env python
#
# name:             website.py
# author:           Harold Bradley III
# email:            harold@bradleystudio.net
# date:             11/11/2015
#
# description:      A class to manage websites
#

try:
    from ext_pylib.files import Dir, File
    from ext_pylib.prompt import prompt, prompt_str, warn_prompt
except ImportError:
    raise ImportError('ext_pylib must be installed to run ww')

from htaccess import Htaccess
import os
import tarfile
from vhost import Vhost
from website_domain import WebsiteDomain
from ww import settings as s

# localhost(function)
#   a python decorator that creates a temporary host entry before
#   the function is called and removes it after it is completed.
def localhost(function):
    def function_wrapper(self, *args, **kwargs):
        print 'Adding temporary host entry.'
        os.system("echo '127.0.0.1 " + self.domain + "' | cat >> /etc/hosts")
        function(self, *args, **kwargs)
        print 'Removing temporary host entry.'
        os.system("sed -i '/^127\.0\.0\.1 " + self.domain + "$/d' /etc/hosts")
    return function_wrapper

# merge_atts(atts, new_atts)
#   Merges two dictionaries with the second overwriting the corresponding
#   values of the first and returns the result.
def merge_atts(atts, new_atts):
    for k, v in new_atts.iteritems():
        if (k in atts and isinstance(atts[k], dict)
                and isinstance(new_atts[k], dict)):
            atts[k] = merge_atts(atts[k], new_atts[k])
        else:
            atts[k] = new_atts[k]
    return atts

# Website(website)
#   a class that describes a generic website with the following properties:
#       domain      Website domain
#       htdocs      Website root directory
#       assets      Website assets directory
#       logs        Website log directory
#       access_log  Website access log file
#       error_log   Website error log file
#       vhost       Apache vhost config file
#       htaccess    Website root htaccess file
#   methods:
#       install()  - installs a new website
#       remove()  - removes an existing website
#       pack  - packs an existing website
#       unpack  - unpacks an existing packed website into current server
#       migrate  - migrates a website from existing settings to default
#                  settings
#       verify()  - verifies a website installation
#       repair()  - verifies website forcing repairs
class Website(object):

    def __init__(self, domain, atts = {}):
        """Initializes a new Website instance."""

        print '[*] SSL not yet implemented in Website class.'

        self.domain = WebsiteDomain(domain)
        self.dirs = {}
        self.files = {}

        default_atts = {
                'root' : {
                    'path'  : s.WWW_DIR + self.domain,
                    'perms' : 0775,
                    'owner' : s.WWW_USR,
                    'group' : s.WWW_USR,
                },
                'htdocs' : {
                    'path'  : s.WWW_DIR + self.domain + '/htdocs/',
                    'perms' : 0775,
                    'owner' : s.WWW_USR,
                    'group' : s.WWW_USR,
                },
                'assets' : {
                    'path'  : s.WWW_DIR + self.domain + '/assets/',
                    'perms' : 0775,
                    'owner' : 'root',
                    'group' : s.WWW_ADMIN,
                },
                'logs' : {
                    'path'  : s.WWW_DIR + self.domain + '/log/',
                    'perms' : 0770,
                    'owner' : 'root',
                    'group' : s.WWW_ADMIN,
                },
                'access_log' : {
                    'path'  : None,
                    'perms' : 0750,
                    'owner' : 'root',
                    'group' : s.WWW_ADMIN,
                },
                'error_log' : {
                    'path'  : None,
                    'perms' : 0750,
                    'owner' : 'root',
                    'group' : s.WWW_ADMIN,
                },
                'vhost' : {
                    'path'  : s.VHOST_PATH + self.domain + '.conf',
                    'perms' : 0644,
                    'owner' : 'root',
                    'group' : 'root',
                },
                'htaccess' : {
                    'path'  : None,
                    'perms' : 0664,
                    'owner' : s.WWW_USR,
                    'group' : s.WWW_USR,
                    'h5g'   : True,
                },
            }

        atts = merge_atts(default_atts, atts)

        self.vhost = Vhost(self.domain.name, atts['vhost'])

        # If an apache vhost config already exists, try to parse it
        if self.vhost.exists():
            print str(self.vhost) + ' already exists.'
            if prompt('Parse existing vhost configuration?'):
                atts = merge_atts(atts, self.vhost.parse())
        # TODO: Convert...or migrate?

        self.root   = Dir(atts['root'])
        self.htdocs = Dir(atts['htdocs'])
        self.assets = Dir(atts['assets'])
        self.logs   = Dir(atts['logs'])

        # Set path of htaccess, access_log, and error_log after htdocs is set.
        if not atts['htaccess']['path']:
            atts['htaccess']['path'] = self.htdocs.path + '.htaccess'
        if not atts['access_log']['path']:
            atts['access_log']['path'] = self.logs.path + s.SITE_ACCESS_LOG
        if not atts['error_log']['path']:
            atts['error_log']['path'] = self.logs.path + s.SITE_ERROR_LOG

        self.htaccess  = Htaccess(atts['htaccess'])
        self.access_log = File(atts['access_log'])
        self.error_log = File(atts['error_log'])

        # Setup vhost placeholders after attributes have been defined
        self.vhost.placeholders = {
                '#WEBSITE#'    : self.domain.name,
                '#HTDOCS#'     : self.htdocs.path,
                '#EMAIL#'      : s.SITE_ADMIN_EMAIL,
                '#ACCESS_LOG#' : self.access_log.path,
                '#ERROR_LOG#'  : self.error_log.path
            }

    def __str__(self):
        """Returns a string with relevant instance information."""
        string =  '\n\n----------------------------------------------------------'
        string += '\n                        - Website -'
        string += '\n----------------------------------------------------------'
        string += '\n  domain:           ' + str(self.domain)
        string += '\n  vhost config:     ' + str(self.vhost)
        string += '\n  htdocs directory: ' + str(self.htdocs)
        string += '\n  assets:           ' + str(self.assets)
        string += '\n  logs:             ' + str(self.logs)
        string += '\n----------------------------------------------------------\n'
        return string

    def __repr__(self):
        """Returns a python string that evaluates to the object instance."""
        return "%s('%s', {'htdocs' : %s, 'assets' : %s, 'logs' : %s, " + \
               "'access_log' : %s, 'error_log' : %s, 'htaccess' : %s, 'vhost' : %s})" % (
            self.__class__.__name__,
            self.domain,
            self.htdocs.get_atts(),
            self.assets.get_atts(),
            self.logs.get_atts(),
            self.access_log.get_atts(),
            self.error_log.get_atts(),
            self.htaccess.get_atts(),
            self.vhost.get_atts()
            )

    ################
    # Public Methods

    def install(self):
        """Installs website to server"""
        # Check if domain is already installed
        if self.is_installed():
            print self.domain + ' is already installed.'

        self.root.create()
        self.htdocs.create()
        self.assets.create()
        self.logs.create()
        self.access_log.create()
        self.error_log.create()
        self.htaccess.create()

        self.vhost.create()

        self.domain.set_ip()

        print str(self)
        print 'Installation complete.'

    def remove(self, ask=True):
        """Removes website from server"""
        self.vhost.disable(ask)

        self.vhost.remove(ask)
        self.access_log.remove(ask)
        self.error_log.remove(ask)
        self.htaccess.remove(ask)
        self.logs.remove(ask)
        self.htdocs.remove(ask)
        self.assets.remove(ask)
        self.root.remove(ask)

        self.domain.set_ip()

    def pack(self):
        pass

    def unpack(self):
        pass

    def migrate(self, old_website = None):
        if not old_website:
            old_website = self.existing
        # make sure there's nothing conflicting
        # install() new
        # this overwrites vhostconf...
        # for dirs and files in old copy
        # this includes vhost...

    def verify(self, repair = False):
        """Verifies website installation"""
        result = all([self.vhost.verify(repair),
                      self.access_log.verify(repair),
                      self.error_log.verify(repair),
                      self.htaccess.verify(repair),
                      self.logs.verify(repair),
                      self.htdocs.verify(repair),
                      self.assets.verify(repair),
                      self.root.verify(repair),
                      self.domain.verify(repair)])
        print self
        return result

    def repair(self):
        """Repairs website installation"""
        print 'Repairing ' + self.domain + '...'
        if self.verify(True):
            print 'Repair completed successfully. [OK]'
        else:
            print 'Repair resulted in errors. [ERROR]'

    def is_installed(self):
        """Returns true if vhost is enabled"""
        return self.vhost.is_enabled()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from collections import defaultdict, OrderedDict
from typing import Iterable, Iterator

class DataToken:
    def __init__(self, offset, word, lowercase, pos_tag, mwe_tag, parent_offset, strength, supersense, sentence_id):
        self.offset = offset
        self.word = word
        self.lowercase_lemma = lowercase
        self.pos_tag = pos_tag
        self.mwe_tag = mwe_tag
        self.parent_offset = None if parent_offset == '0' else int(parent_offset)
        self.strength = strength
        self.supersense = supersense
        self.sentence_id = sentence_id

class DataSentence:
    def __init__(self):
        self._tokens: OrderedDict = OrderedDict()
        self.sentence_id = None

    def __iter__(self) -> Iterator[DataToken]:
        return iter(self._tokens)

    def __getitem__(self, item) -> DataToken:
        return self._tokens[item]

    def __bool__(self) -> bool:
        return bool(self._tokens)

    def append(self, token: DataToken):
        if not self.sentence_id:
            self.sentence_id = token.sentence_id
        else:
            if token.sentence_id != self.sentence_id:
                raise ValueError("sentence IDs do not match: {} (sentence) != {} (token)".format(self.sentence_id,
                                                                                token.sentence_id))
        self._tokens[int(token.offset)] = token

    @property
    def sentence(self) -> str:
        return ' '.join(map(lambda t: t.word, self._tokens.values()))

def read_data_file(datafile: str):
    data_sentences = defaultdict(DataSentence)
    with open(datafile) as df:
        for line in df:
            if not line.isspace():
                token = DataToken(*line.strip(' \n').split('\t'))
                data_sentences[token.sentence_id].append(token)
    return data_sentences

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('training-data', help='the datafile to train with')
    parser.add_argument('testing-data', help='the datafile to test with')
    args = parser.parse_args()

    # Read the data.
    training_sentences = read_data_file(args.training_data)
    testing_sentences = read_data_file(args.testing_data)
<commit_msg>Update data file reading<commit_after>#!/usr/bin/env python3

from collections import defaultdict, OrderedDict
from typing import Iterable, Iterator

class DataToken:
    def __init__(self, offset, word, lowercase, pos_tag, mwe_tag, parent_offset, strength, supersense, sentence_id):
        self.offset = offset
        self.word = word
        self.lowercase_lemma = lowercase
        self.pos_tag = pos_tag
        self.mwe_tag = mwe_tag
        self.parent_offset = None if parent_offset == '0' else int(parent_offset)
        self.strength = strength
        self.supersense = supersense
        self.sentence_id = sentence_id

class DataSentence:
    def __init__(self):
        self._tokens: OrderedDict = OrderedDict()
        self.sentence_id = None

    def __iter__(self) -> Iterator[DataToken]:
        return iter(self._tokens)

    def __getitem__(self, item) -> DataToken:
        return self._tokens[item]

    def __bool__(self) -> bool:
        return bool(self._tokens)

    def append(self, token: DataToken):
        if not self.sentence_id:
            self.sentence_id = token.sentence_id
        else:
            if token.sentence_id != self.sentence_id:
                raise ValueError("sentence IDs do not match: {} (sentence) != {} (token)".format(self.sentence_id,
                                                                                token.sentence_id))
        self._tokens[int(token.offset)] = token

    @property
    def sentence(self) -> str:
        return ' '.join(map(lambda t: t.word, self._tokens.values()))

def read_data_file(datafile: str) -> Iterable[DataSentence]:
    data_sentences = defaultdict(DataSentence)
    with open(datafile) as df:
        for line in df:
            if not line.isspace():
                # Strip extraneous whitespace from the line, then split it on the tabs. There should be exactly enough
                # fields to fill the DataToken.
                token = DataToken(*(line.strip(' \n').split('\t')))
                data_sentences[token.sentence_id].append(token)
    return data_sentences.values()

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('training-data', help='the datafile to train with')
    parser.add_argument('testing-data', help='the datafile to test with')
    args = parser.parse_args()

    # Read the data.
    training_sentences = read_data_file(args.training_data)
    testing_sentences = read_data_file(args.testing_data)
<|endoftext|>


<commit_before># script to measure nu_max and delta_nu for all chaplin stars

import numpy as np
import matplotlib.pyplot as plt
import glob
from isochrones.dartmouth import Dartmouth_Isochrone
from isochrones import StarModel
import kplr
from useful_things import color2teff
from scaling_relations import delta_nu, nu_max
import superpgram as spg
from header_time import real_footprint, real_footprint_sc
from fft import load_data
import h5py
import nufft
from find_delta_nu import find_delta_nu as deltan
from astero import astero
import os

# using the kic colours, use isochrones to produce posterior samples
# of the physical stellar parameters
def colour2physical(kid, plot=False):

    # fetch photometry
    client = kplr.API()
    star = client.star(kid)
    mags = {'H': (star.kic_hmag, 0.08),
            'J': (star.kic_jmag, 0.08),
            'K': (star.kic_kmag, 0.08),
            'g':(star.kic_gmag, 0.08),
            'r':(star.kic_rmag, 0.08)}

    # find the mass and radius using isochrones
    dar = Dartmouth_Isochrone()
    smod_phot = StarModel(dar, **mags)
    result = smod_phot.fit_mcmc()

    # save samples and triangle plot
    if plot:
        smod_phot.triangle_plots("test", format="pdf")
    smod_phot.save_hdf("samples_%s.h5" % str(int(kid)))
    max_like = smod_phot.maxlike()
    print max_like
    np.savetxt("maxlike.txt", max_like)

# load samples and find astero parameters
def samples2astero(fname, max_like=False):
    print fname

    if max_like:
        max_like = np.genfromtxt(fname).T
    else:
        with h5py.File(fname) as f:
            dataset = f["/samples"][...]

    # find the nu_max and delta_nu
#     nm = nu_max(mass, radius, teff)
#     dn = delta_nu(mass, radius)

# compute a periodogram around that area using short cadence
def make_pgram(x, y, yerr, ivar, nm, dn, fft=False):

    # compute spg or fft
    fs = np.arange(nm-(10*dn), nm+(10*dn), 1e-2) * 1e-6
    if fft:
        comp_pgram = nufft.nufft3(x, y, fs*2*np.pi)
        pgram = np.imag(comp_pgram)**2 + np.real(comp_pgram)**2
    else:
        starts, stops, centres = real_footprint_sc(x)
        pgram = spg.superpgram(starts, stops, y, ivar, fs*2*np.pi)

    # save spg
    np.savetxt("pgram_%s.txt" % str(int(kid)), np.transpose((fs, pgram)))
    plt.clf()
    plt.plot(fs, pgram)
    plt.savefig("pgram_%s" % str(int(kid)))
    return fs, pgram

# add up the signal for stars of the same type... how? try adding both the
# matrix and the collapsed
def add_up(kids):
    fname = "pow_%s.txt" % str(int(kids[0]))
    if os.path.isfile(fname):
        pos, power = np.genfromtxt("pow_%s.txt" % str(int(kids[0]))).T
        for i, kid in enumerate(kids[1:]):
            kid = str(int(kid))
            data = np.genfromtxt("pow_%s.txt" % kid).T
            power += data[-1]
    plt.clf()
    plt.plot(pos, power, "k")
    plt.savefig("test")

# calculates the pgram and dnu, etc
def calc_pgram_and_dnu(kid, dn, nm):
        D = "/Users/angusr/.kplr/data/lightcurves/%s" % kid.zfill(9)
        x, y, yerr, ivar = load_data(kid, D, sc=True)
        if len(x):  # only run if you find data
            print "data found for ", kid
            print "dnu = ", dn, "nm = ", nm
            fs, pgram = make_pgram(x, y, yerr, ivar, nm, dn, kid)

            # run delta nu
            width = .1
            fs, pgram = np.genfromtxt("pgram_%s.txt" % str(int(kid))).T
            mx, my, lags, pos, acor, power = deltan(fs, pgram, str(int(kid)),
                                                    width, sub=100,
                                                    truths=[dn, nm],
                                                    smooth=False)
            np.savetxt("acor_%s.txt" % kid, np.transpose((lags, acor)))
            np.savetxt("pow_%s.txt" % kid, np.transpose((pos, power)))

            plt.clf()
            plt.subplot(2, 1, 1)
            plt.plot(lags, acor, "k")
            plt.axvline(nm, color="r", linestyle="--")
            plt.subplot(2, 1, 2)
            plt.plot(pos, power, "k")
            plt.savefig("test_%s" % kid)

#     add_up(kids[l])

if __name__ == "__main__":

#     kid = "6442183"
#     dn = 65.
#     nm = 1160.
#     calc_pgram_and_dnu(kid, dn, nm)

    data = astero()
    kids = data.iKID
    dnu, nm = data.idnu, data.inu_max

    # select stars with similar nu_max
    l = (500 < nm) * (nm < 600)
    print len(kids[l])

#     fname = "samples_%s.h5" % kid
#     fname = "maxlike.txt"

#     colour2physical(kid)
#     samples2astero(fname)

    for i, kid in enumerate(kids[l][:3]):
        kid = str(int(kid))
        calc_pgram_and_dnu(kid, dnu[l][i], nm[l][i])
<commit_msg>initial<commit_after># script to measure nu_max and delta_nu for all chaplin stars

import numpy as np
import matplotlib.pyplot as plt
import glob
from isochrones.dartmouth import Dartmouth_Isochrone
from isochrones import StarModel
import kplr
from useful_things import color2teff
from scaling_relations import delta_nu, nu_max
import superpgram as spg
from header_time import real_footprint, real_footprint_sc
from fft import load_data
import h5py
import nufft
from find_delta_nu import find_delta_nu as deltan
from astero import astero
import os
import scipy.interpolate as spi

# using the kic colours, use isochrones to produce posterior samples
# of the physical stellar parameters
def colour2physical(kid, plot=False):

    # fetch photometry
    client = kplr.API()
    star = client.star(kid)
    mags = {'H': (star.kic_hmag, 0.08),
            'J': (star.kic_jmag, 0.08),
            'K': (star.kic_kmag, 0.08),
            'g':(star.kic_gmag, 0.08),
            'r':(star.kic_rmag, 0.08)}

    # find the mass and radius using isochrones
    dar = Dartmouth_Isochrone()
    smod_phot = StarModel(dar, **mags)
    result = smod_phot.fit_mcmc()

    # save samples and triangle plot
    if plot:
        smod_phot.triangle_plots("test", format="pdf")
    smod_phot.save_hdf("samples_%s.h5" % str(int(kid)))
    max_like = smod_phot.maxlike()
    print max_like
    np.savetxt("maxlike.txt", max_like)

# load samples and find astero parameters
def samples2astero(fname, max_like=False):
    print fname

    if max_like:
        max_like = np.genfromtxt(fname).T
    else:
        with h5py.File(fname) as f:
            dataset = f["/samples"][...]

    # find the nu_max and delta_nu
#     nm = nu_max(mass, radius, teff)
#     dn = delta_nu(mass, radius)

# compute a periodogram around that area using short cadence
def make_pgram(kid, x, y, yerr, ivar, nm, dn, fft=False):

    # compute spg or fft
    fs = np.arange(nm-(10*dn), nm+(10*dn), 1e-2) * 1e-6
    if fft:
        comp_pgram = nufft.nufft3(x, y, fs*2*np.pi)
        pgram = np.imag(comp_pgram)**2 + np.real(comp_pgram)**2
    else:
        starts, stops, centres = real_footprint_sc(x)
        pgram = spg.superpgram(starts, stops, y, ivar, fs*2*np.pi)

    # save spg
    np.savetxt("pgram_%s.txt" % str(int(kid)), np.transpose((fs, pgram)))
    plt.clf()
    plt.plot(fs, pgram)
    plt.savefig("pgram_%s" % str(int(kid)))
    return fs, pgram

def interp(grid, x, y):
    f = spi.interp1d(x, y)
    ynew = f(grid)
    return ynew

# add up the signal for stars of the same type... how? try adding both the
# matrix and the collapsed
def add_up(kids):
    fname = "pow_%s.txt" % str(int(kids[0]))
    if os.path.isfile(fname):
        pos, power = np.genfromtxt("pow_%s.txt" % str(int(kids[0]))).T
        for i, kid in enumerate(kids[1:]):
            kid = str(int(kid))
            data = np.genfromtxt("pow_%s.txt" % kid).T
            power += data[-1]
    plt.clf()
    plt.plot(pos, power, "k")
    plt.savefig("test")

# calculates the pgram and dnu, etc
def calc_pgram_and_dnu(kid, dn, nm):
    D = "/Users/angusr/.kplr/data/lightcurves/%s" % kid.zfill(9)
    x, y, yerr, ivar = load_data(kid, D, sc=True)
    if len(x):  # only run if you find data
        print "data found for ", kid
        print "dnu = ", dn, "nm = ", nm
        fs, pgram = make_pgram(kid, x, y, yerr, ivar, nm, dn, kid)

        # run delta nu
        width = .2  # fs range is 20*dn, so this is about 4*dn
        fs, pgram = np.genfromtxt("pgram_%s.txt" % str(int(kid))).T
        mx, my, lags, pos, acor, power = deltan(fs, pgram, str(int(kid)),
                                                width, sub=100,
                                                truths=[dn, nm],
                                                smooth=False)
        np.savetxt("acor_%s.txt" % kid, np.transpose((lags, acor)))
        np.savetxt("pow_%s.txt" % kid, np.transpose((pos, power)))

        plt.clf()
        plt.subplot(2, 1, 1)
        plt.plot(lags*1e6, acor, "k")
        plt.subplot(2, 1, 2)
        plt.plot(pos, power, "k")
        plt.savefig("test_%s" % kid)

    return lags, acor, pos, power

def add_up(kids, dns, nms, load=False):

    lag_list, pos_list, acor_list, power_list = [], [], [], []
    for i, kid in enumerate(kids[:3]):
        kid = str(int(kid))
        if load:
            lags, acor = np.genfromtxt("acor_%s.txt" % kid).T
            pos, power = np.genfromtxt("pow_%s.txt" % kid).T
        else:
            lags, acor, pos, power = calc_pgram_and_dnu(kid, dnu[i], nm[i])

        # save acfs and powers
        lag_list.append(lags)
        acor_list.append(acor)
        pos_list.append(pos)
        power_list.append(power)

    # find the shortest acf so that you interpolate the others onto it
#     lag_size = np.array([len(j) for j in lag_list])
#     pos_size = np.array([len(j) for j in pos_list])
#     l1 = np.where(lag_size==max(lag_size))[0]
#     l2 = np.where(pos_size==max(pos_size))[0]
#     lag_grid = lag_list[l1]
#     pos_grid = pos_list[l2]
    dl = lag_list[0][1] - lag_list[0][0]
    print dl
    max_start = max([j[0] for j in lag_list])
    min_end = min([j[0] for j in lag_list])
    lag_grid = np.arange(max_start, min_end, dl)
    print len(lag_grid), len(lag_list[0])
    assert 0

    # interpolate so that you can add up the signals
    lag_interps = np.zeros((len(lag_list), len(lag_grid)))
    pos_interps = np.zeros((len(pos_list), len(pos_grid)))
    for i, lags in enumerate(lag_list):
        l = lags > lag_grid[0]
        print min(lag_grid), min(lags[l])
        print max(lag_grid), max(lags[l])
        new_acor = interp(lag_grid, lags[l], acor_list[i][l])
        l = pos > pos_grid[0]
        print min(pos_grid), min(pos[l])
        new_power = interp(pos_grid, pos_list[i][l], power_list[i][l])
        lag_interps[i, :] = new_acor
        pos_interps[i, :] = new_power

    # make plots of summed acfs and power thingys
    plt.clf()
    plt.subplot(2, 1, 1)
    plt.plot(lag_grid, np.sum(lag_interps, axis=0))
    plt.subplot(2, 1, 2)
    plt.plot(pos_grid, np.sum(pos_interps, axis=0))
    plt.savefig("test")

if __name__ == "__main__":

#     kid = "6442183"
#     dn = 65.
#     nm = 1160.
#     calc_pgram_and_dnu(kid, dn, nm)

    data = astero()
    kids = data.iKID
    dnu, nm = data.idnu, data.inu_max

    # select stars with similar nu_max
    l = (500 < nm) * (nm < 600)
    print len(kids[l])

#     fname = "samples_%s.h5" % kid
#     fname = "maxlike.txt"

#     colour2physical(kid)
#     samples2astero(fname)

    add_up(kids[l], dnu[l], nm[l])
<|endoftext|>


<commit_before>"""
desitarget.cmx.cmx_cuts
========================

`Target Selection for DESI commissioning (cmx) derived from `the wiki`_.

A collection of helpful (static) methods to check whether an object's
flux passes a given selection criterion (*e.g.* STD_DITHER).

.. _`the Gaia data model`: https://gea.esac.esa.int/archive/documentation/GDR2/Gaia_archive/chap_datamodel/sec_dm_main_tables/ssec_dm_gaia_source.html
.. _`the wiki`: https://desi.lbl.gov/trac/wiki/TargetSelectionWG/CommissioningTargets
"""

from time import time
import numpy as np

import warnings

import astropy.units as u
from astropy.coordinates import SkyCoord
from astropy.table import Table, Row

from desitarget import io
from desitarget.cuts import _psflike
from desitarget.internal import sharedmem
from desitarget.cmx.cmx_targetmask import cmx_mask

#ADM set up the DESI default logger
from desiutil.log import get_logger
log = get_logger()

#ADM start the clock
start = time()

def passesSTD_logic(gfracflux=None, rfracflux=None, zfracflux=None,
                    objtype=None, isgaia=None, pmra=None, pmdec=None,
                    aen=None, dupsource=None, paramssolved=None,
                    primary=None):
    """The default logic/mask cuts for commissioning stars

    Parameters
    ----------
    gfracflux, rfracflux, zfracflux : :class:`array_like` or :class:`None`
        Profile-weighted fraction of the flux from other sources divided
        by the total flux in g, r and z bands.
    objtype : :class:`array_like` or :class:`None`
        The Legacy Surveys TYPE to restrict to point sources.
    isgaia : :class:`boolean array_like` or :class:`None`
       ``True`` if there is a match between this object in the Legacy
       Surveys and in Gaia.
    pmra, pmdec : :class:`array_like` or :class:`None`
        Gaia-based proper motion in RA and Dec and parallax
        (same units as the Gaia data model).
    aen : :class:`array_like` or :class:`None`
        Gaia Astrometric Excess Noise (as in the Gaia Data Model).
    dupsource : :class:`array_like` or :class:`None`
        Whether the source is a duplicate in Gaia (as in the Gaia Data model).
    paramssolved : :class:`array_like` or :class:`None`
        How many parameters were solved for in Gaia (as in the Gaia Data model).
    primary : :class:`array_like` or :class:`None`
        If given, the ``BRICK_PRIMARY`` column of the catalogue.

    Returns
    -------
    :class:`array_like`
        True if and only if the object passes the logic cuts for cmx stars.

    Notes
    -----
    - Current version (08/30/18) is version 4 on `the wiki`_.
    - See also `the Gaia data model`_.
    """
    if primary is None:
        primary = np.ones_like(isgaia, dtype='?')

    std = primary.copy()

    # ADM A point source with a Gaia match.
    std &= _psflike(objtype)
    std &= isgaia

    # ADM An Isolated source.
    fracflux = [gfracflux, rfracflux, zfracflux]
    with warnings.catch_warnings():
        warnings.simplefilter('ignore') # fracflux can be Inf/NaN.
        for bandint in (0, 1, 2):  # g, r, z.
            std &= fracflux[bandint] < 0.01

    # ADM No obvious issues with the astrometry.
    std &= (aen < 1) & (paramssolved == 31)

    # ADM Finite proper motions.
    std & = np.isfinite(pmra) & np.isfinite(pmdec)

    # ADM Unique source (not a duplicated source).
    std &= ~dupsource

    return std

def isSTD_dither(obs_gflux=None, obs_rflux=None, obs_zflux=None,
                 gfracflux=None, rfracflux=None, zfracflux=None,
                 objtype=None, gaia=None, aen=None, pmra=None, pmdec=None,
                 objtype=None, primary=None):
    """Gaia stars for dithering tests during commissioning

    Parameters
    ----------
    obs_gflux, obs_rflux, obs_zflux : :class:`array_like` or :class:`None`
        The flux in nano-maggies of g, r, z bands WITHOUT any
        Galactic extinction correction.
    gfracflux, rfracflux, zfracflux : :class:`array_like` or :class:`None`
        Profile-weighted fraction of the flux from other sources divided
        by the total flux in g, r and z bands.
    objtype : :class:`array_like` or :class:`None`
        The Legacy Surveys TYPE to restrict to point sources.
    gaia : :class:`boolean array_like` or :class:`None`
       ``True`` if there is a match between this object in the Legacy
       Surveys and in Gaia.
    pmra, pmdec : :class:`array_like` or :class:`None`
        Gaia-based proper motion in RA and Dec and parallax
        (same units as the Gaia data model).
    aen : :class:`array_like` or :class:`None`
        Gaia Astrometric Excess Noise (as in the Gaia Data Model).
    dupsource : :class:`array_like` or :class:`None`
        Whether the source is a duplicate in Gaia (as in the Gaia Data model).
    paramssolved : :class:`array_like` or :class:`None`
        How many parameters were solved for in Gaia (as in the Gaia Data model).
    primary : :class:`array_like` or :class:`None`
        If given, the ``BRICK_PRIMARY`` column of the catalogue.

    Returns
    -------
    :class:`array_like`
        True if and only if the object is Gaia dithering target.

    Notes
    -----
    - Current version (08/30/18) is version 4 on `the wiki`_.
    - See also `the Gaia data model`_.
    """
    if primary is None:
        primary = np.ones_like(rflux, dtype='?')

    std = primary.copy()
    std &= _psflike(objtype)
    std &= gaia

    bgs &= rflux > 10**((22.5-20.0)/2.5)
    bgs &= rflux <= 10**((22.5-19.5)/2.5)

    return bgs

    isdither &= (gflux > 0)

    return isdither

def apply_cuts(objects):
    """Perform commissioning (cmx) target selection on objects, return target mask arrays

    Parameters
    ----------
    objects: numpy structured array with UPPERCASE columns needed for
        target selection, OR a string tractor/sweep filename

    Returns
    -------
    :class:`~numpy.ndarray`
        commissioning target selection bitmask flags for each object

    See desitarget.cmx.cmx_targetmask.cmx_mask for the definition of each bit
    """
    #- Check if objects is a filename instead of the actual data
    if isinstance(objects, str):
        objects = io.read_tractor(objects)

    #- Ensure uppercase column names if astropy Table
    if isinstance(objects, (Table, Row)):
        for col in list(objects.columns.values()):
            if not col.name.isupper():
                col.name = col.name.upper()

    # ADM Currently only coded for objects with Gaia matches
    # ADM (e.g. DR7 or above). Fail for earlier Data Releases.
    release = objects['RELEASE']
    if release < 7000:
        log.critical('Commissioning cuts only coded for DR7 or above')
        raise ValueError

    # ADM The observed g/r/z fluxes.
    obs_gflux = objects['FLUX_G']
    obs_rflux = objects['FLUX_R']
    obs_zflux = objects['FLUX_Z']

    # ADM The de-extincted g/r/z/ fluxes.
    from desitarget.cuts import unextinct_fluxes
    flux = unextinct_fluxes(objects)
    gflux = flux['GFLUX']
    rflux = flux['RFLUX']
    zflux = flux['ZFLUX']

    # ADM The Legacy Surveys object type and fracflux flags.
    objtype = objects['TYPE']
    gfracflux = objects['FRACFLUX_G']
    rfracflux = objects['FRACFLUX_R']
    zfracflux = objects['FRACFLUX_Z']

    # ADM Add the Gaia columns...
    # ADM if we don't have REF_CAT in the sweeps use the
    # ADM minimum value of REF_ID to identify Gaia sources. This will
    # ADM introduce a small number (< 0.001%) of Tycho-only sources.
    isgaia = objects['REF_ID'] > 0
    if "REF_CAT" in objects.dtype.names:
        isgaia = (objects['REF_CAT'] == b'G2') | (objects['REF_CAT'] == 'G2')
    pmra = objects['PMRA']
    pmdec = objects['PMDEC']
    pmraivar = objects['PMRA_IVAR']
    gaiaaen = objects['GAIA_ASTROMETRIC_EXCESS_NOISE']
    gaiadupsource = objects['GAIA_DUPLICATED_SOURCE']

    # ADM If proper motion is not NaN, 31 parameters were solved for
    # ADM in Gaia astrometry. Or, gaiaparamssolved should be 3 for NaNs).
    # ADM In the sweeps, NaN has not been preserved...but PMRA_IVAR == 0
    # ADM in the sweeps is equivalent to PMRA of NaN in Gaia.
    if 'GAIA_ASTROMETRIC_PARAMS_SOLVED' in objects.dtype.names:
        gaiaparamssolved = objects['GAIA_ASTROMETRIC_PARAMS_SOLVED']
    else:
        gaiaparamssolved = np.zeros_like(gaia)+31
        w = np.where( np.isnan(pmra) | (pmraivar == 0) )[0]
        if len(w) > 0:
            gaiaparamssolved[w] = 3

    # ADM initially, every object passes the cuts (is True)
    primary = np.ones_like(objects, dtype=bool)

    #ADM determine if an object passes the default logic for cmx stars
    isgood = passesSTD_logic(
        gfracflux=gfracflux, rfracflux=rfracflux, zfracflux=zfracflux,
        objtype=objtype, isgaia=isgaia, pmra=pmra, pmdec=pmdec,
        aen=gaiaaen, dupsource=gaiadupsource, paramssolved=gaiaparamssolved,
        primary=primary
    )

    dither = isSTD_dither(gflux=gflux)

    # ADM Construct the targetflag bits
    cmx_target  = dither * cmx_mask.STD_GAIA

    return cmx_target

def select_targets(infiles, numproc=4, gaiamatch=False,
            gaiadir='/project/projectdirs/cosmo/work/gaia/chunks-gaia-dr2-astrom'):
    """Process input files in parallel to select commissioning (cmx) targets

    Parameters
    ----------
    infiles : :class:`list` or `str`
        A list of input filenames (tractor or sweep files) OR a single filename
    numproc : :class:`int`, optional, defaults to 4
        The number of parallel processes to use
    gaiamatch : :class:`boolean`, optional, defaults to ``False``
        If ``True``, match to Gaia DR2 chunks files and populate Gaia columns
    gaiadir : :class:`str`, optional, defaults to Gaia DR2 path at NERSC
        Root directory of a Gaia Data Release as used by the Legacy Surveys.

    Returns
    -------
    :class:`~numpy.ndarray`
        The subset of input targets which pass the cuts, including extra
        columns for `DESI_TARGET`

    Notes
    -----
        - if numproc==1, use serial code instead of parallel
    """
    from desiutil.log import get_logger
    log = get_logger()

    #- Convert single file to list of files
    if isinstance(infiles,str):
        infiles = [infiles,]

    #- Sanity check that files exist before going further
    for filename in infiles:
        if not os.path.exists(filename):
            raise ValueError("{} doesn't exist".format(filename))

    def _finalize_targets(objects, desi_target):
        #- desi_target includes BGS_ANY and MWS_ANY, so we can filter just
        #- on desi_target != 0
        keep = (desi_target != 0)
        objects = objects[keep]
        desi_target = desi_target[keep]

        #- Add *_target mask columns
        #ADM note that only desi_target is defined for commissioning
        #ADM so just pass that around
        targets = desitarget.targets.finalize(
            objects, desi_target, desi_target, desi_target)

        return io.fix_tractor_dr1_dtype(targets)

    #- functions to run on every brick/sweep file
    def _select_targets_file(filename):
        '''Returns targets in filename that pass the cuts'''
        objects = io.read_tractor(filename)
        desi_target, bgs_target, mws_target = apply_cuts(objects,gaiamatch,gaiadir)

        return _finalize_targets(objects, desi_target, desi_target, desi_target)

    # Counter for number of bricks processed;
    # a numpy scalar allows updating nbrick in python 2
    # c.f https://www.python.org/dev/peps/pep-3104/
    nbrick = np.zeros((), dtype='i8')

    t0 = time()
    def _update_status(result):
        ''' wrapper function for the critical reduction operation,
            that occurs on the main parallel process '''
        if nbrick%50 == 0 and nbrick>0:
            rate = nbrick / (time() - t0)
            log.info('{} files; {:.1f} files/sec'.format(nbrick, rate))

        nbrick[...] += 1    # this is an in-place modification
        return result

    #- Parallel process input files
    if numproc > 1:
        pool = sharedmem.MapReduce(np=numproc)
        with pool:
            targets = pool.map(_select_targets_file, infiles, reduce=_update_status)
    else:
        targets = list()
        for x in infiles:
            targets.append(_update_status(_select_targets_file(x)))

    targets = np.concatenate(targets)

    return targets
<commit_msg>add STD_GAIA commissioning class<commit_after>"""
desitarget.cmx.cmx_cuts
========================

`Target Selection for DESI commissioning (cmx) derived from `the wiki`_.

A collection of helpful (static) methods to check whether an object's
flux passes a given selection criterion (*e.g.* STD_DITHER).

.. _`the Gaia data model`: https://gea.esac.esa.int/archive/documentation/GDR2/Gaia_archive/chap_datamodel/sec_dm_main_tables/ssec_dm_gaia_source.html
.. _`the wiki`: https://desi.lbl.gov/trac/wiki/TargetSelectionWG/CommissioningTargets
"""

from time import time
import numpy as np

import warnings

import astropy.units as u
from astropy.coordinates import SkyCoord
from astropy.table import Table, Row

from desitarget import io
from desitarget.cuts import _psflike
from desitarget.internal import sharedmem
from desitarget.cmx.cmx_targetmask import cmx_mask

#ADM set up the DESI default logger
from desiutil.log import get_logger
log = get_logger()

#ADM start the clock
start = time()

def passesSTD_logic(gfracflux=None, rfracflux=None, zfracflux=None,
                    objtype=None, isgaia=None, pmra=None, pmdec=None,
                    aen=None, dupsource=None, paramssolved=None,
                    primary=None):
    """The default logic/mask cuts for commissioning stars.

    Parameters
    ----------
    gfracflux, rfracflux, zfracflux : :class:`array_like` or :class:`None`
        Profile-weighted fraction of the flux from other sources divided
        by the total flux in g, r and z bands.
    objtype : :class:`array_like` or :class:`None`
        The Legacy Surveys TYPE to restrict to point sources.
    isgaia : :class:`boolean array_like` or :class:`None`
       ``True`` if there is a match between this object in the Legacy
       Surveys and in Gaia.
    pmra, pmdec : :class:`array_like` or :class:`None`
        Gaia-based proper motion in RA and Dec and parallax
        (same units as the Gaia data model).
    aen : :class:`array_like` or :class:`None`
        Gaia Astrometric Excess Noise (as in the Gaia Data Model).
    dupsource : :class:`array_like` or :class:`None`
        Whether the source is a duplicate in Gaia (as in the Gaia Data model).
    paramssolved : :class:`array_like` or :class:`None`
        How many parameters were solved for in Gaia (as in the Gaia Data model).
    primary : :class:`array_like` or :class:`None`
        If given, the ``BRICK_PRIMARY`` column of the catalogue.

    Returns
    -------
    :class:`array_like`
        True if and only if the object passes the logic cuts for cmx stars.

    Notes
    -----
    - Current version (08/30/18) is version 4 on `the wiki`_.
    - See also `the Gaia data model`_.
    """
    if primary is None:
        primary = np.ones_like(isgaia, dtype='?')

    std = primary.copy()

    # ADM A point source with a Gaia match.
    std &= _psflike(objtype)
    std &= isgaia

    # ADM An Isolated source.
    fracflux = [gfracflux, rfracflux, zfracflux]
    with warnings.catch_warnings():
        warnings.simplefilter('ignore') # fracflux can be Inf/NaN.
        for bandint in (0, 1, 2):  # g, r, z.
            std &= fracflux[bandint] < 0.01

    # ADM No obvious issues with the astrometry.
    std &= (aen < 1) & (paramssolved == 31)

    # ADM Finite proper motions.
    std & = np.isfinite(pmra) & np.isfinite(pmdec)

    # ADM Unique source (not a duplicated source).
    std &= ~dupsource

    return std

def isSTD_dither(obs_gflux=None, obs_rflux=None, obs_zflux=None,
                 isgood=None, primary=None):
    """Gaia stars for dithering tests during commissioning.

    Parameters
    ----------
    obs_gflux, obs_rflux, obs_zflux : :class:`array_like` or :class:`None`
        The flux in nano-maggies of g, r, z bands WITHOUT any
        Galactic extinction correction.
    isgood : :class:`array_like` or :class:`None`
        ``True`` for objects that pass the logic cuts in
        :func:`~desitarget.cmx.cmx_cuts.passesSTD_logic`.
    primary : :class:`array_like` or :class:`None`
        If given, the ``BRICK_PRIMARY`` column of the catalogue.

    Returns
    -------
    :class:`array_like`
        True if and only if the object is Gaia dithering target.

    Notes
    -----
    - Current version (08/30/18) is version 4 on `the wiki`_.
    - See also `the Gaia data model`_.
    """
    if primary is None:
        primary = np.ones_like(rflux, dtype='?')

    isdither = primary.copy()
    # ADM passes all of the default logic cuts.
    isdither &= isgood

    # ADM not too bright in g, r, z (> 15 mags)
    isdither &= obs_gflux < 10**((22.5-15.0)/2.5)
    isdither &= obs_rflux < 10**((22.5-15.0)/2.5)
    isdither &= obs_zflux < 10**((22.5-15.0)/2.5)

    return isdither

def apply_cuts(objects):
    """Perform commissioning (cmx) target selection on objects, return target mask arrays

    Parameters
    ----------
    objects: numpy structured array with UPPERCASE columns needed for
        target selection, OR a string tractor/sweep filename

    Returns
    -------
    :class:`~numpy.ndarray`
        commissioning target selection bitmask flags for each object

    See desitarget.cmx.cmx_targetmask.cmx_mask for the definition of each bit
    """
    #- Check if objects is a filename instead of the actual data
    if isinstance(objects, str):
        objects = io.read_tractor(objects)

    #- Ensure uppercase column names if astropy Table
    if isinstance(objects, (Table, Row)):
        for col in list(objects.columns.values()):
            if not col.name.isupper():
                col.name = col.name.upper()

    # ADM Currently only coded for objects with Gaia matches
    # ADM (e.g. DR7 or above). Fail for earlier Data Releases.
    release = objects['RELEASE']
    if release < 7000:
        log.critical('Commissioning cuts only coded for DR7 or above')
        raise ValueError

    # ADM The observed g/r/z fluxes.
    obs_gflux = objects['FLUX_G']
    obs_rflux = objects['FLUX_R']
    obs_zflux = objects['FLUX_Z']

    # ADM The de-extincted g/r/z/ fluxes.
    from desitarget.cuts import unextinct_fluxes
    flux = unextinct_fluxes(objects)
    gflux = flux['GFLUX']
    rflux = flux['RFLUX']
    zflux = flux['ZFLUX']

    # ADM The Legacy Surveys object type and fracflux flags.
    objtype = objects['TYPE']
    gfracflux = objects['FRACFLUX_G']
    rfracflux = objects['FRACFLUX_R']
    zfracflux = objects['FRACFLUX_Z']

    # ADM Add the Gaia columns...
    # ADM if we don't have REF_CAT in the sweeps use the
    # ADM minimum value of REF_ID to identify Gaia sources. This will
    # ADM introduce a small number (< 0.001%) of Tycho-only sources.
    isgaia = objects['REF_ID'] > 0
    if "REF_CAT" in objects.dtype.names:
        isgaia = (objects['REF_CAT'] == b'G2') | (objects['REF_CAT'] == 'G2')
    pmra = objects['PMRA']
    pmdec = objects['PMDEC']
    pmraivar = objects['PMRA_IVAR']
    gaiaaen = objects['GAIA_ASTROMETRIC_EXCESS_NOISE']
    gaiadupsource = objects['GAIA_DUPLICATED_SOURCE']

    # ADM If proper motion is not NaN, 31 parameters were solved for
    # ADM in Gaia astrometry. Or, gaiaparamssolved should be 3 for NaNs).
    # ADM In the sweeps, NaN has not been preserved...but PMRA_IVAR == 0
    # ADM in the sweeps is equivalent to PMRA of NaN in Gaia.
    if 'GAIA_ASTROMETRIC_PARAMS_SOLVED' in objects.dtype.names:
        gaiaparamssolved = objects['GAIA_ASTROMETRIC_PARAMS_SOLVED']
    else:
        gaiaparamssolved = np.zeros_like(gaia)+31
        w = np.where( np.isnan(pmra) | (pmraivar == 0) )[0]
        if len(w) > 0:
            gaiaparamssolved[w] = 3

    # ADM initially, every object passes the cuts (is True)
    primary = np.ones_like(objects, dtype=bool)

    #ADM determine if an object passes the default logic for cmx stars
    isgood = passesSTD_logic(
        gfracflux=gfracflux, rfracflux=rfracflux, zfracflux=zfracflux,
        objtype=objtype, isgaia=isgaia, pmra=pmra, pmdec=pmdec,
        aen=gaiaaen, dupsource=gaiadupsource, paramssolved=gaiaparamssolved,
        primary=primary
    )

    dither = isSTD_dither(
        obs_gflux=obs_gflux, obs_rflux=obs_rflux, obs_zflux=obs_zflux,
        isgood=isgood, primary=primary):
    )

    # ADM Construct the targetflag bits
    cmx_target  = dither * cmx_mask.STD_GAIA

    return cmx_target

def select_targets(infiles, numproc=4, gaiamatch=False,
            gaiadir='/project/projectdirs/cosmo/work/gaia/chunks-gaia-dr2-astrom'):
    """Process input files in parallel to select commissioning (cmx) targets

    Parameters
    ----------
    infiles : :class:`list` or `str`
        A list of input filenames (tractor or sweep files) OR a single filename
    numproc : :class:`int`, optional, defaults to 4
        The number of parallel processes to use
    gaiamatch : :class:`boolean`, optional, defaults to ``False``
        If ``True``, match to Gaia DR2 chunks files and populate Gaia columns
    gaiadir : :class:`str`, optional, defaults to Gaia DR2 path at NERSC
        Root directory of a Gaia Data Release as used by the Legacy Surveys.

    Returns
    -------
    :class:`~numpy.ndarray`
        The subset of input targets which pass the cuts, including extra
        columns for `DESI_TARGET`

    Notes
    -----
        - if numproc==1, use serial code instead of parallel
    """
    from desiutil.log import get_logger
    log = get_logger()

    #- Convert single file to list of files
    if isinstance(infiles,str):
        infiles = [infiles,]

    #- Sanity check that files exist before going further
    for filename in infiles:
        if not os.path.exists(filename):
            raise ValueError("{} doesn't exist".format(filename))

    def _finalize_targets(objects, desi_target):
        #- desi_target includes BGS_ANY and MWS_ANY, so we can filter just
        #- on desi_target != 0
        keep = (desi_target != 0)
        objects = objects[keep]
        desi_target = desi_target[keep]

        #- Add *_target mask columns
        #ADM note that only desi_target is defined for commissioning
        #ADM so just pass that around
        targets = desitarget.targets.finalize(
            objects, desi_target, desi_target, desi_target)

        return io.fix_tractor_dr1_dtype(targets)

    #- functions to run on every brick/sweep file
    def _select_targets_file(filename):
        '''Returns targets in filename that pass the cuts'''
        objects = io.read_tractor(filename)
        desi_target, bgs_target, mws_target = apply_cuts(objects,gaiamatch,gaiadir)

        return _finalize_targets(objects, desi_target, desi_target, desi_target)

    # Counter for number of bricks processed;
    # a numpy scalar allows updating nbrick in python 2
    # c.f https://www.python.org/dev/peps/pep-3104/
    nbrick = np.zeros((), dtype='i8')

    t0 = time()
    def _update_status(result):
        ''' wrapper function for the critical reduction operation,
            that occurs on the main parallel process '''
        if nbrick%50 == 0 and nbrick>0:
            rate = nbrick / (time() - t0)
            log.info('{} files; {:.1f} files/sec'.format(nbrick, rate))

        nbrick[...] += 1    # this is an in-place modification
        return result

    #- Parallel process input files
    if numproc > 1:
        pool = sharedmem.MapReduce(np=numproc)
        with pool:
            targets = pool.map(_select_targets_file, infiles, reduce=_update_status)
    else:
        targets = list()
        for x in infiles:
            targets.append(_update_status(_select_targets_file(x)))

    targets = np.concatenate(targets)

    return targets
<|endoftext|>


<commit_before># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from warnings import warn

from datetime import datetime
from sqlalchemy.orm import validates
from pyfarm.warning import ColumnStateChangeWarning

class StateValidationMixin(object):
    """
    Mixin that adds a `state` column and uses a class
    level `STATE_ENUM` attribute to assist in validation.
    """
    @validates("state")
    def validate_state(self, key, value):
        if value not in self.STATE_ENUM.values():
            # string which represents what states are valid
            valid_states = ", ".join(
                "%s (%s)" % (value, self.STATE_ENUM.get(value))
                for value in sorted(self.STATE_ENUM.values()))

            msg = "%s is not a valid state, valid states " % value
            msg += "are %s" % valid_states
            raise ValueError(msg)

        return value

class StateChangedMixin(object):
    """
    Mixin which adds a static method to be used when the model
    state changes
    """
    @staticmethod
    def stateChangedEvent(target, new_value, old_value, initiator):
        """update the datetime objects depending on the new value"""
        if target.id is None:
            pass

        elif new_value == target.STATE_ENUM.RUNNING:
            target.time_started = datetime.now()

            if hasattr(target, "attempts"):
                target.attempts += 1

        elif new_value in (target.STATE_ENUM.DONE, target.STATE_ENUM.FAILED):
            if target.time_started is None:
                msg = "job %s has not been started yet, state is " % target.id
                msg += "being set to %s" % target.STATE_ENUM.get(new_value)
                warn(msg,  ColumnStateChangeWarning)

            target.time_finished = datetime.now()<commit_msg>docstring<commit_after># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Module containing mixins which can be used by multiple models.
"""

from warnings import warn
from datetime import datetime
from sqlalchemy.orm import validates
from pyfarm.warning import ColumnStateChangeWarning

class StateValidationMixin(object):
    """
    Mixin that adds a `state` column and uses a class
    level `STATE_ENUM` attribute to assist in validation.
    """
    @validates("state")
    def validate_state(self, key, value):
        if value not in self.STATE_ENUM.values():
            # string which represents what states are valid
            valid_states = ", ".join(
                "%s (%s)" % (value, self.STATE_ENUM.get(value))
                for value in sorted(self.STATE_ENUM.values()))

            msg = "%s is not a valid state, valid states " % value
            msg += "are %s" % valid_states
            raise ValueError(msg)

        return value

class StateChangedMixin(object):
    """
    Mixin which adds a static method to be used when the model
    state changes
    """
    @staticmethod
    def stateChangedEvent(target, new_value, old_value, initiator):
        """update the datetime objects depending on the new value"""
        if target.id is None:
            pass

        elif new_value == target.STATE_ENUM.RUNNING:
            target.time_started = datetime.now()

            if hasattr(target, "attempts"):
                target.attempts += 1

        elif new_value in (target.STATE_ENUM.DONE, target.STATE_ENUM.FAILED):
            if target.time_started is None:
                msg = "job %s has not been started yet, state is " % target.id
                msg += "being set to %s" % target.STATE_ENUM.get(new_value)
                warn(msg,  ColumnStateChangeWarning)

            target.time_finished = datetime.now()<|endoftext|>