<commit_before>#    Copyright 2014 Rustici Software
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.

import unittest

if __name__ == '__main__':
    from main import setup_tincan_path
    setup_tincan_path()
from tincan.activity import Activity
from tincan.activity_definition import ActivityDefinition
from tincan.interaction_component_list import InteractionComponentList
from tincan.language_map import LanguageMap

class ActivityTest(unittest.TestCase):

    def test_InitEmpty(self):
        activity = Activity()
        self.assertIsNone(activity.id)

    def test_InitExceptionEmptyId(self):
        with self.assertRaises(ValueError):
            activity = Activity(id='')

    def test_Init(self):
        activity = Activity(id='test', definition=ActivityDefinition(), object_type='Activity')
        self.activityVerificationHelper(activity)

    def test_InitExceptionObjectType(self):
        with self.assertRaises(ValueError):
            activity = Activity(object_type='NotActivity')

    def test_InitAnonDefinition(self):
        # these are arbitrary parameters - the ActivityDefinition is tested in ActivityDefinition_test
        activity = Activity(definition={'name': {'en-US': 'test'}, 'scale': []})
        self.assertIsInstance(activity.definition, ActivityDefinition)
        self.assertEqual(activity.definition.name, {'en-US': 'test'})
        self.assertIsInstance(activity.definition.name, LanguageMap)
        self.assertEqual(activity.definition.scale, [])
        self.assertIsInstance(activity.definition.scale, InteractionComponentList)

    def test_InitUnpack(self):
        obj = {'id': 'test', 'definition': ActivityDefinition(), 'object_type': 'Activity'}
        activity = Activity(**obj)
        self.activityVerificationHelper(activity)

    def test_InitUnpackExceptionEmptyId(self):
        obj = {'id': ''}
        with self.assertRaises(ValueError):
            activity = Activity(**obj)

    def test_InitUnpakeExceptionObjectType(self):
        obj = {'object_type': 'NotActivity'}
        with self.assertRaises(ValueError):
            activity = Activity(**obj)

    def test_FromJSON(self):
        activity = Activity.from_json('{"id": "test", "definition": {}, "object_type": "Activity"}')
        self.activityVerificationHelper(activity)

    def test_FromJSONExceptionObjectType(self):
        with self.assertRaises(ValueError):
            activity = Activity.from_json('{"object_type": "NotActivity"}')

    def test_FromJSONExcpetionEmptyId(self):
        with self.assertRaises(ValueError):
            activity = Activity.from_json('{"id": ""}')

    def test_FromJSONExceptionEmptyObject(self):
        activity = Activity.from_json('{}')
        self.assertIsInstance(activity, Activity)
        self.assertIsNone(activity.id, None)

    def test_AsVersionEmpty(self):
        activity = Activity()
        activity2 = activity.as_version()
        self.assertEqual(activity2, {})

    def test_asVersionNotEmpty(self):
        activity = Activity(id='test')
        activity2 = activity.as_version()
        self.assertEqual(activity2, {'id': 'test'})

    def test_asVersion(self):
        activity = Activity(id='test', definition=ActivityDefinition(), object_type='Activity')
        activity2 = activity.as_version()
        self.assertEqual(activity2, {'id': 'test', 'definition': {}, 'objectType': 'Activity'})

    def test_ToJSONFromJSON(self):
        json_str = '{"id": "test", "definition": {}, "object_type": "Activity"}'
        check_str = '{"definition": {}, "id": "test", "objectType": "Activity"}'
        activity = Activity.from_json(json_str)
        self.activityVerificationHelper(activity)
        self.assertEqual(activity.to_json(), check_str)

    def test_ToJSON(self):
        check_str = '{"definition": {}, "id": "test", "objectType": "Activity"}'
        activity = Activity(**{'id':'test', 'definition':{}, 'object_type':'Activity'})
        self.assertEqual(activity.to_json(), check_str)

    def test_setDefinitionException(self):
        activity = Activity()
        with self.assertRaises(AttributeError):
            activity.definition = {"invalid": "definition"}

    def test_setDefinition(self):
        activity = Activity()
        activity.definition = ActivityDefinition()
        self.assertIsInstance(activity.definition, ActivityDefinition)

    def test_setObjectTypeException(self):
        activity = Activity()
        with self.assertRaises(ValueError):
            activity.object_type = 'NotActivity'

    def test_setObjectType(self):
        activity = Activity()
        activity.object_type = 'Activity'
        self.assertEqual(activity.object_type, 'Activity')

    def test_setIdException(self):
        activity = Activity()
        with self.assertRaises(ValueError):
            activity.id = ''

    def test_setId(self):
        activity = Activity()
        activity.id = 'test'
        self.assertEqual(activity.id, 'test')

    def activityVerificationHelper(self, activity):
        self.assertEqual(activity.id, 'test')
        self.assertIsInstance(activity.definition, ActivityDefinition)
        self.assertEqual(activity.object_type, 'Activity')

if __name__ == '__main__':
    suite = unittest.TestLoader().loadTestsFromTestCase(ActivityTest)
    unittest.TextTestRunner(verbosity=2).run(suite)<commit_msg>Test name typo fix<commit_after>#    Copyright 2014 Rustici Software
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.

import unittest

if __name__ == '__main__':
    from main import setup_tincan_path
    setup_tincan_path()
from tincan.activity import Activity
from tincan.activity_definition import ActivityDefinition
from tincan.interaction_component_list import InteractionComponentList
from tincan.language_map import LanguageMap

class ActivityTest(unittest.TestCase):

    def test_InitEmpty(self):
        activity = Activity()
        self.assertIsNone(activity.id)

    def test_InitExceptionEmptyId(self):
        with self.assertRaises(ValueError):
            activity = Activity(id='')

    def test_Init(self):
        activity = Activity(id='test', definition=ActivityDefinition(), object_type='Activity')
        self.activityVerificationHelper(activity)

    def test_InitExceptionObjectType(self):
        with self.assertRaises(ValueError):
            activity = Activity(object_type='NotActivity')

    def test_InitAnonDefinition(self):
        # these are arbitrary parameters - the ActivityDefinition is tested in ActivityDefinition_test
        activity = Activity(definition={'name': {'en-US': 'test'}, 'scale': []})
        self.assertIsInstance(activity.definition, ActivityDefinition)
        self.assertEqual(activity.definition.name, {'en-US': 'test'})
        self.assertIsInstance(activity.definition.name, LanguageMap)
        self.assertEqual(activity.definition.scale, [])
        self.assertIsInstance(activity.definition.scale, InteractionComponentList)

    def test_InitUnpack(self):
        obj = {'id': 'test', 'definition': ActivityDefinition(), 'object_type': 'Activity'}
        activity = Activity(**obj)
        self.activityVerificationHelper(activity)

    def test_InitUnpackExceptionEmptyId(self):
        obj = {'id': ''}
        with self.assertRaises(ValueError):
            activity = Activity(**obj)

    def test_InitUnpackExceptionObjectType(self):
        obj = {'object_type': 'NotActivity'}
        with self.assertRaises(ValueError):
            activity = Activity(**obj)

    def test_FromJSON(self):
        activity = Activity.from_json('{"id": "test", "definition": {}, "object_type": "Activity"}')
        self.activityVerificationHelper(activity)

    def test_FromJSONExceptionObjectType(self):
        with self.assertRaises(ValueError):
            activity = Activity.from_json('{"object_type": "NotActivity"}')

    def test_FromJSONExcpetionEmptyId(self):
        with self.assertRaises(ValueError):
            activity = Activity.from_json('{"id": ""}')

    def test_FromJSONExceptionEmptyObject(self):
        activity = Activity.from_json('{}')
        self.assertIsInstance(activity, Activity)
        self.assertIsNone(activity.id, None)

    def test_AsVersionEmpty(self):
        activity = Activity()
        activity2 = activity.as_version()
        self.assertEqual(activity2, {})

    def test_asVersionNotEmpty(self):
        activity = Activity(id='test')
        activity2 = activity.as_version()
        self.assertEqual(activity2, {'id': 'test'})

    def test_asVersion(self):
        activity = Activity(id='test', definition=ActivityDefinition(), object_type='Activity')
        activity2 = activity.as_version()
        self.assertEqual(activity2, {'id': 'test', 'definition': {}, 'objectType': 'Activity'})

    def test_ToJSONFromJSON(self):
        json_str = '{"id": "test", "definition": {}, "object_type": "Activity"}'
        check_str = '{"definition": {}, "id": "test", "objectType": "Activity"}'
        activity = Activity.from_json(json_str)
        self.activityVerificationHelper(activity)
        self.assertEqual(activity.to_json(), check_str)

    def test_ToJSON(self):
        check_str = '{"definition": {}, "id": "test", "objectType": "Activity"}'
        activity = Activity(**{'id':'test', 'definition':{}, 'object_type':'Activity'})
        self.assertEqual(activity.to_json(), check_str)

    def test_setDefinitionException(self):
        activity = Activity()
        with self.assertRaises(AttributeError):
            activity.definition = {"invalid": "definition"}

    def test_setDefinition(self):
        activity = Activity()
        activity.definition = ActivityDefinition()
        self.assertIsInstance(activity.definition, ActivityDefinition)

    def test_setObjectTypeException(self):
        activity = Activity()
        with self.assertRaises(ValueError):
            activity.object_type = 'NotActivity'

    def test_setObjectType(self):
        activity = Activity()
        activity.object_type = 'Activity'
        self.assertEqual(activity.object_type, 'Activity')

    def test_setIdException(self):
        activity = Activity()
        with self.assertRaises(ValueError):
            activity.id = ''

    def test_setId(self):
        activity = Activity()
        activity.id = 'test'
        self.assertEqual(activity.id, 'test')

    def activityVerificationHelper(self, activity):
        self.assertEqual(activity.id, 'test')
        self.assertIsInstance(activity.definition, ActivityDefinition)
        self.assertEqual(activity.object_type, 'Activity')

if __name__ == '__main__':
    suite = unittest.TestLoader().loadTestsFromTestCase(ActivityTest)
    unittest.TextTestRunner(verbosity=2).run(suite)
<|endoftext|>


<commit_before>#!/usr/bin/python2.5
#
# Copyright 2009 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Student Proposal (Model) query functions.
"""

__authors__ = [
  '"Lennard de Rijk" <ljvderijk@gmail.com>',
  ]

from soc.logic.models import base
from soc.logic.models import student as student_logic
from soc.models import student_proposal

import soc.models.linkable
import soc.models.student_proposal

class Logic(base.Logic):
  """Logic methods for the Student Proposal model.
  """

  def __init__(self, model=soc.models.student_proposal.StudentProposal,
               base_model=soc.models.linkable.Linkable,
               scope_logic=student_logic):
    """Defines the name, key_name and model for this entity.
    """

    super(Logic, self).__init__(model=model, base_model=base_model,
                                scope_logic=scope_logic)

  def getRankerFor(self, entity):
    """Returns the ranker for the given Student Proposal.

    Args:
      entity: Student Proposal entity for which the ranker should be returned

    Returns:
      Ranker object which is used to rank the given entity
    """

    from soc.logic.models.ranker_root import logic as ranker_root_logic

    fields = {'link_id': student_proposal.DEF_RANKER_NAME,
        'scope': entity.org}

    ranker_root = ranker_root_logic.getForFields(fields, unique=True)
    ranker = ranker_root_logic.getRootFromEntity(ranker_root)

    return ranker

  def getProposalsToBeAcceptedForOrg(self, org_entity, stepsize=25):
    """Returns all StudentProposals which will be accepted into the program
    for the given organization.

    params:
      org_entity: the Organization for which the proposals should be checked
      stepsize: optional parameter to specify the ammount of Student Proposals
                that should be retrieved per roundtrip to the datastore

    returns:
      List with all StudentProposal which will be accepted into the program
    """

    # check if there are already slots taken by this org
    fields = {'org': org_entity,
              'status': 'accepted'}

    query = self.getQueryForFields(fields)

    slots_left_to_assign = max(0, org_entity.slots - query.count())

    if slots_left_to_assign == 0:
      # no slots left so return nothing
      return []

    fields = {'org': org_entity,
              'status': 'pending'}
    order = ['-score']

    # get the the number of proposals that would be assigned a slot
    query = self.getQueryForFields(
        fields, order=order)

    proposals = query.fetch(slots_left_to_assign)
    proposals = [i for i in proposals if i.mentor]

    offset = slots_left_to_assign

    # retrieve as many additional proposals as needed in case the top
    # N do not have a mentor assigned
    while len(proposals) < slots_left_to_assign:
      new_proposals = query.fetch(stepsize, offset=offset)

      if not new_proposals:
        # we ran out of proposals`
        break

      new_proposals = [i for i in new_proposals if i.mentor]
      proposals += new_proposals
      offset += stepsize

    # cut off any superfluous proposals
    return proposals[:slots_left_to_assign]

  def _onCreate(self, entity):
    """Adds this proposal to the organization ranker entity.
    """

    ranker = self.getRankerFor(entity)
    ranker.SetScore(entity.key().id_or_name(), [entity.score])

    super(Logic, self)._onCreate(entity)

  def _updateField(self, entity, entity_properties, name):
    """Called when the fields of the student_proposal are updated.

      - Update the ranker if the score changes and keep the score within bounds
      - Remove the entity from the ranker when the status changes to invalid or rejected
    """

    value = entity_properties[name]

    if name == 'score':
      # keep the score within bounds
      min_score, max_score = student_proposal.DEF_SCORE

      value = max(min_score, min(value, max_score-1))
      entity_properties[name] = value

      # update the ranker
      ranker = self.getRankerFor(entity)
      ranker.SetScore(entity.key().id_or_name(), [value])

    if name == 'status':

      if value in ['invalid', 'rejected'] and entity.status != value:
        # the proposal is going into invalid or rejected state
        # remove the score from the ranker
        ranker = self.getRankerFor(entity)

        # entries in the ranker can be removed by setting the score to None
        ranker.SetScore(entity.key().id_or_name(), None)

    return super(Logic, self)._updateField(entity, entity_properties, name)

  def delete(self, entity):
    """Removes Ranker entry and all ReviewFollowers before deleting the entity.

    Args:
      entity: an existing entity in datastore
    """

    from soc.logic.models.review_follower import logic as review_follower_logic

    # entries in the ranker can be removed by setting the score to None
    ranker = self.getRankerFor(entity)
    ranker.SetScore(entity.key().id_or_name(), None)

    # get all the ReviewFollwers that have this entity as it's scope
    fields = {'scope': entity}

    # TODO be sure that this captures all the followers
    followers = review_follower_logic.getForFields(fields)

    for follower in followers:
      review_follower_logic.delete(follower)

    # call to super to complete the deletion
    super(Logic, self).delete(entity)

logic = Logic()
<commit_msg>Renamed stepsize to step_size in Student Proposal Logic.<commit_after>#!/usr/bin/python2.5
#
# Copyright 2009 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Student Proposal (Model) query functions.
"""

__authors__ = [
  '"Lennard de Rijk" <ljvderijk@gmail.com>',
  ]

from soc.logic.models import base
from soc.logic.models import student as student_logic
from soc.models import student_proposal

import soc.models.linkable
import soc.models.student_proposal

class Logic(base.Logic):
  """Logic methods for the Student Proposal model.
  """

  def __init__(self, model=soc.models.student_proposal.StudentProposal,
               base_model=soc.models.linkable.Linkable,
               scope_logic=student_logic):
    """Defines the name, key_name and model for this entity.
    """

    super(Logic, self).__init__(model=model, base_model=base_model,
                                scope_logic=scope_logic)

  def getRankerFor(self, entity):
    """Returns the ranker for the given Student Proposal.

    Args:
      entity: Student Proposal entity for which the ranker should be returned

    Returns:
      Ranker object which is used to rank the given entity
    """

    from soc.logic.models.ranker_root import logic as ranker_root_logic

    fields = {'link_id': student_proposal.DEF_RANKER_NAME,
        'scope': entity.org}

    ranker_root = ranker_root_logic.getForFields(fields, unique=True)
    ranker = ranker_root_logic.getRootFromEntity(ranker_root)

    return ranker

  def getProposalsToBeAcceptedForOrg(self, org_entity, step_size=25):
    """Returns all StudentProposals which will be accepted into the program
    for the given organization.

    params:
      org_entity: the Organization for which the proposals should be checked
      step_size: optional parameter to specify the amount of Student Proposals
                that should be retrieved per roundtrip to the datastore

    returns:
      List with all StudentProposal which will be accepted into the program
    """

    # check if there are already slots taken by this org
    fields = {'org': org_entity,
              'status': 'accepted'}

    query = self.getQueryForFields(fields)

    slots_left_to_assign = max(0, org_entity.slots - query.count())

    if slots_left_to_assign == 0:
      # no slots left so return nothing
      return []

    fields = {'org': org_entity,
              'status': 'pending'}
    order = ['-score']

    # get the the number of proposals that would be assigned a slot
    query = self.getQueryForFields(
        fields, order=order)

    proposals = query.fetch(slots_left_to_assign)
    proposals = [i for i in proposals if i.mentor]

    offset = slots_left_to_assign

    # retrieve as many additional proposals as needed in case the top
    # N do not have a mentor assigned
    while len(proposals) < slots_left_to_assign:
      new_proposals = query.fetch(step_size, offset=offset)

      if not new_proposals:
        # we ran out of proposals`
        break

      new_proposals = [i for i in new_proposals if i.mentor]
      proposals += new_proposals
      offset += step_size

    # cut off any superfluous proposals
    return proposals[:slots_left_to_assign]

  def _onCreate(self, entity):
    """Adds this proposal to the organization ranker entity.
    """

    ranker = self.getRankerFor(entity)
    ranker.SetScore(entity.key().id_or_name(), [entity.score])

    super(Logic, self)._onCreate(entity)

  def _updateField(self, entity, entity_properties, name):
    """Called when the fields of the student_proposal are updated.

      - Update the ranker if the score changes and keep the score within bounds
      - Remove the entity from the ranker when the status changes to invalid or rejected
    """

    value = entity_properties[name]

    if name == 'score':
      # keep the score within bounds
      min_score, max_score = student_proposal.DEF_SCORE

      value = max(min_score, min(value, max_score-1))
      entity_properties[name] = value

      # update the ranker
      ranker = self.getRankerFor(entity)
      ranker.SetScore(entity.key().id_or_name(), [value])

    if name == 'status':

      if value in ['invalid', 'rejected'] and entity.status != value:
        # the proposal is going into invalid or rejected state
        # remove the score from the ranker
        ranker = self.getRankerFor(entity)

        # entries in the ranker can be removed by setting the score to None
        ranker.SetScore(entity.key().id_or_name(), None)

    return super(Logic, self)._updateField(entity, entity_properties, name)

  def delete(self, entity):
    """Removes Ranker entry and all ReviewFollowers before deleting the entity.

    Args:
      entity: an existing entity in datastore
    """

    from soc.logic.models.review_follower import logic as review_follower_logic

    # entries in the ranker can be removed by setting the score to None
    ranker = self.getRankerFor(entity)
    ranker.SetScore(entity.key().id_or_name(), None)

    # get all the ReviewFollwers that have this entity as it's scope
    fields = {'scope': entity}

    # TODO be sure that this captures all the followers
    followers = review_follower_logic.getForFields(fields)

    for follower in followers:
      review_follower_logic.delete(follower)

    # call to super to complete the deletion
    super(Logic, self).delete(entity)

logic = Logic()
<|endoftext|>


<commit_before>#!/usr/bin/env python
import re
import string
import sys; sys.path.append(".."); sys.path.append("../data/extras")
from data.extras import chooseone
from fireplace import cards
from fireplace.cards import debug, game, classic, naxxramas, gvg, blackrock
from fireplace.enums import CardType

import buffs

GREEN = "\033[92m"
YELLOW = "\033[33m"
RED = "\033[91m"
ENDC = "\033[0m"
PREFIXES = {
	GREEN: "Implemented: ",
	YELLOW: "Potentially implemented: ",
	RED: "Not implemented: ",
}

SOLVED_KEYWORDS = [
	"Windfury", "Charge", "Divine Shield", "Taunt", "Stealth",
	"Spell Damage \+[0-9]+",
	"Can't be targeted by spells or Hero Powers",
	"Overload: \([0-9]+\)",
	"50% chance to attack the wrong enemy",
	"Can't Attack",
]

def cleanup_description(description):
	ret = description
	ret = re.sub("(" + "|".join(SOLVED_KEYWORDS) + ")", "", ret)
	ret = re.sub("<[^>]*>", "", ret)
	exclude_chars = string.punctuation + string.whitespace
	ret = "".join([ch for ch in ret if ch not in exclude_chars])
	return ret

def potentially_implemented(card):
	return card.auras or "Enrage" in card.description or hasattr(chooseone, card.id)

def main():
	for id in sorted(cards.db):
		card = cards.db[id]
		if not cleanup_description(card.description):
			# Minions without card text or with basic abilities are implemented
			color = GREEN
		elif card.type == CardType.ENCHANTMENT:
			if id in buffs.__dict__:
				color = GREEN
		else:
			for set in (debug, game, classic, naxxramas, gvg, blackrock):
				if hasattr(set, id):
					color = GREEN
					break
			else:
				if potentially_implemented(card):
					color = YELLOW
				else:
					color = RED
		print(color + PREFIXES[color] + card.name + ENDC + " (%s)" % (id))

if __name__ == "__main__":
	main()
<commit_msg>implemented.py: Also strip <b> tags<commit_after>#!/usr/bin/env python
import re
import string
import sys; sys.path.append(".."); sys.path.append("../data/extras")
from data.extras import chooseone
from fireplace import cards
from fireplace.cards import debug, game, classic, naxxramas, gvg, blackrock
from fireplace.enums import CardType

import buffs

GREEN = "\033[92m"
YELLOW = "\033[33m"
RED = "\033[91m"
ENDC = "\033[0m"
PREFIXES = {
	GREEN: "Implemented: ",
	YELLOW: "Potentially implemented: ",
	RED: "Not implemented: ",
}

SOLVED_KEYWORDS = [
	"Windfury", "Charge", "Divine Shield", "Taunt", "Stealth",
	"Spell Damage \+[0-9]+",
	"Can't be targeted by spells or Hero Powers",
	"Overload: \([0-9]+\)",
	"50% chance to attack the wrong enemy",
	"Can't Attack",
]

def cleanup_description(description):
	ret = description
	ret = re.sub("(<b>|</b>)", "", ret)
	ret = re.sub("(" + "|".join(SOLVED_KEYWORDS) + ")", "", ret)
	ret = re.sub("<[^>]*>", "", ret)
	exclude_chars = string.punctuation + string.whitespace
	ret = "".join([ch for ch in ret if ch not in exclude_chars])
	return ret

def potentially_implemented(card):
	return card.auras or "Enrage" in card.description or hasattr(chooseone, card.id)

def main():
	for id in sorted(cards.db):
		card = cards.db[id]
		if not cleanup_description(card.description):
			# Minions without card text or with basic abilities are implemented
			color = GREEN
		elif card.type == CardType.ENCHANTMENT:
			if id in buffs.__dict__:
				color = GREEN
		else:
			for set in (debug, game, classic, naxxramas, gvg, blackrock):
				if hasattr(set, id):
					color = GREEN
					break
			else:
				if potentially_implemented(card):
					color = YELLOW
				else:
					color = RED
		print(color + PREFIXES[color] + card.name + ENDC + " (%s)" % (id))

if __name__ == "__main__":
	main()
<|endoftext|>


<commit_before>import os
from glob import glob
import re
import logging
import shutil
import time
import pyfits
import numpy as np
import numpy.ma as ma
import argparse
from configobj import ConfigObj

import Ska.DBI
import Ska.arc5gl
from Chandra.Time import DateTime
import Ska.File

logger = logging.getLogger('aca0 fetch')
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler())

# borrowed from eng_archive
archfiles_hdr_cols = ('tstart', 'tstop', 'startmjf', 'startmnf',
                      'stopmjf', 'stopmnf',
                      'tlmver', 'ascdsver', 'revision', 'date',
                      'imgsize')

dtype = [('level', '|S4'), ('instrum', '|S7'), ('content', '|S13'),
         ('arc5gl_query', '|S27'), ('fileglob', '|S9')]
filetype = np.rec.fromrecords([('L0', 'PCAD', 'ACADATA', 'ACA0', '*fits.gz')],
                               dtype=dtype)[0]

aca_dtype = [('TIME', '>f8'), ('QUALITY', '>i4'), ('MJF', '>i4'),
             ('MNF', '>i4'),
             ('END_INTEG_TIME', '>f8'), ('INTEG', '>f4'), ('GLBSTAT', '|u1'),
             ('COMMCNT', '|u1'), ('COMMPROG', '|u1'), ('IMGFID1', '|u1'),
             ('IMGNUM1', '|u1'), ('IMGFUNC1', '|u1'), ('IMGSTAT', '|u1'),
             ('IMGROW0', '>i2'), ('IMGCOL0', '>i2'), ('IMGSCALE', '>i2'),
             ('BGDAVG', '>i2'), ('IMGFID2', '|u1'), ('IMGNUM2', '|u1'),
             ('IMGFUNC2', '|u1'), ('BGDRMS', '>i2'), ('TEMPCCD', '>f4'),
             ('TEMPHOUS', '>f4'), ('TEMPPRIM', '>f4'), ('TEMPSEC', '>f4'),
             ('BGDSTAT', '|u1'), ('IMGFID3', '|u1'), ('IMGNUM3', '|u1'),
             ('IMGFUNC3', '|u1'), ('IMGFID4', '|u1'), ('IMGNUM4', '|u1'),
             ('IMGFUNC4', '|u1'), ('IMGRAW', '>f4', (64,)),
             ('HD3TLM62', '|u1'),
             ('HD3TLM63', '|u1'), ('HD3TLM64', '|u1'), ('HD3TLM65', '|u1'),
             ('HD3TLM66', '|u1'), ('HD3TLM67', '|u1'), ('HD3TLM72', '|u1'),
             ('HD3TLM73', '|u1'), ('HD3TLM74', '|u1'), ('HD3TLM75', '|u1'),
             ('HD3TLM76', '|u1'), ('HD3TLM77', '|u1'),
             ('IMGSIZE', '>i4'),
             ]

#config = ConfigObj("aca0.conf")
config = dict(data_root='/data/aca/archive/aca0',
              temp_root='/data/aca/archive/temp',
              days_at_once=30.0)

def get_options():
    parser = argparse.ArgumentParser(
        description="Fetch aca level 0 products and make a file archive")
    defaults = dict(config)
    parser.set_defaults(**defaults)
    parser.add_argument("--data-root",
                        help="parent directory for all data")
    parser.add_argument("--temp-root",
                        help="parent temp directory")
    parser.add_argument("--datestart",
                        help="start date for retrieve (defaults to max date of archived files)")
    parser.add_argument("--datestop",
                        help="stop date for retrieve (defaults to now)")
    parser.add_argument("--days-at-once",
                        type=float,
                        help="if over this number, bin to chunks of this number of days")
    opt = parser.parse_args()
    return opt

def get_slot_data(tstart, tstop, slot, imgsize=[4, 6, 8],
                  db=None, data_root=config['data_root']):
    if not db:
        dbfile = os.path.join(config['data_root'], 'archfiles.db3')
        db = Ska.DBI.DBI(dbi='sqlite', server=dbfile)
    data_files = get_interval_files(tstart, tstop, slots=[slot],
                                    imgsize=imgsize, db=db)
    if not len(data_files):
        # return an empty masked array
        return ma.zeros(0, dtype=aca_dtype)
    rows = np.sum(data_files['rows'])
    zero_row = ma.zeros(1, dtype=aca_dtype)
    zero_row.mask = True
    all_rows = zero_row.repeat(rows)
    rowcount = 0
    for f in data_files:
        fp = os.path.join(data_root,
                          str(f['year']), str(f['doy']), f['filename'])
        hdu = pyfits.open(fp)
        chunk = hdu[1].data
        for fname in all_rows.dtype.names:
            if fname == 'IMGRAW' or fname == 'IMGSIZE':
                continue
            if fname in chunk.dtype.names:
                all_rows[fname][rowcount:(rowcount + len(chunk))] \
                    = chunk.field(fname)
                all_rows[fname][rowcount:(rowcount + len(chunk))].mask \
                    = False
        imgsize = int(np.sqrt(chunk[0].field('IMGRAW').size))
        all_rows['IMGSIZE'][rowcount:(rowcount + len(chunk))] = imgsize
        all_rows['IMGRAW'].reshape(rows, 8, 8)[
            rowcount:(rowcount + len(chunk)), 0:imgsize, 0:imgsize] = (
            chunk.field('IMGRAW').reshape(len(chunk), imgsize, imgsize))
        rowcount += len(chunk)
    return all_rows

def get_interval_files(tstart, tstop, slot, imgsize=[4, 6, 8], db=None):
    if not db:
        dbfile = os.path.join(config['data_root'], 'archfiles.db3')
        db = Ska.DBI.DBI(dbi='sqlite', server=dbfile)
    tstart = DateTime(tstart).secs
    tstop = DateTime(tstop).secs
    imgsize_str = ','.join([str(x) for x in imgsize])
    db_query = ('SELECT * FROM archfiles '
                'WHERE tstop > %f '
                'AND tstart < %f '
                'AND slot == %d '
                'AND imgsize in (%s) '
                'order by filetime asc '
                % (tstart, tstop, slot, imgsize_str))
    return db.fetchall(db_query)

def get_archive_files(filetype, datestart, datestop):
    """Update FITS file archive with arc5gl and ingest files into msid (HDF5) archive"""

    # Retrieve CXC archive files in a temp directory with arc5gl
    arc5 = Ska.arc5gl.Arc5gl(echo=True)
    n_queries = 1
    times = np.linspace(datestart.secs, datestop.secs, n_queries + 1)

    logger.info('********** %s %s **********'
                % (filetype['content'], time.ctime()))

    for t0, t1 in zip(times[:-1], times[1:]):
        arc5.sendline('tstart=%s' % DateTime(t0).date)
        arc5.sendline('tstop=%s' % DateTime(t1).date)
        arc5.sendline('get %s' % filetype['arc5gl_query'].lower())

    return sorted(glob(filetype['fileglob']))

def read_archfile(i, f, archfiles, db):
    """Read filename ``f`` with index ``i`` (position within list of filenames).  The
    file has type ``filetype`` and will be added to MSID file at row index ``row``.
    ``colnames`` is the list of column names for the content type (not used here).
    """
    # Check if filename is already in archfiles.
    # If so then abort further processing.
    filename = os.path.basename(f)
    colnames = archfiles_hdr_cols
    if db.fetchall('SELECT filename FROM archfiles WHERE filename=?',
                   (filename,)):
        logger.info(
            'File %s already in archfiles - unlinking and skipping' % f)
        os.unlink(f)
        return None

    # Read FITS archive file and accumulate data into dats list and header into headers dict
    logger.info('Reading (%d / %d) %s' % (i, len(archfiles), filename))
    hdus = pyfits.open(f)
    hdu = hdus[1]
    #dat = converters.convert(hdu.data, filetype['content'])

    # Accumlate relevant info about archfile that will be ingested into
    # MSID h5 files.  Commit info before h5 ingest so if there is a failure
    # the needed info will be available to do the repair.
    archfiles_row = dict((x, hdu.header.get(x.upper()))
                         for x in archfiles_hdr_cols)
    archfiles_row['checksum'] = hdu._checksum
    #archfiles_row['rowstart'] = row
    #archfiles_row['rowstop'] = row + len(dat)
    imgsize_sq = hdu.data[0].field('IMGRAW').shape[0]
    archfiles_row['imgsize'] = int(np.sqrt(imgsize_sq))
    archfiles_row['slot'] = int(re.search(
            r'acaf\d+N\d{3}_(\d)_img0.fits(\.gz)?',
            filename).group(1))
    archfiles_row['filename'] = filename
    archfiles_row['filetime'] = int(
        re.search(r'(\d+)', archfiles_row['filename']).group(1))
    filedate = DateTime(archfiles_row['filetime']).date
    year, doy = (int(x)
                 for x in re.search(r'(\d\d\d\d):(\d\d\d)', filedate).groups())
    archfiles_row['year'] = year
    archfiles_row['doy'] = doy
    archfiles_row['rows'] = len(hdu.data)
    hdus.close()
    return  archfiles_row

def move_archive_files(filetype, archfiles, opt):

    if not os.path.exists(opt.data_root):
        os.makedirs(opt.data_root)

    for f in archfiles:
        if not os.path.exists(f):
            continue
        basename = os.path.basename(f)
        tstart = re.search(r'(\d+)', str(basename)).group(1)
        datestart = DateTime(tstart).date
        year, doy = re.search(r'(\d\d\d\d):(\d\d\d)', datestart).groups()

        archdir = os.path.abspath(os.path.join(opt.data_root,
                                               year,
                                               doy))
        archfile = os.path.abspath(os.path.join(archdir, basename))

        if not os.path.exists(archdir):
            os.makedirs(archdir)

        if not os.path.exists(archfile):
            logger.info('mv %s %s' % (os.path.abspath(f), archfile))
#            if not opt.dry_run:
#                if not opt.occ:
#            shutil.copy2(f, stagedir)
            shutil.move(f, archfile)

        if os.path.exists(f):
            logger.info('Unlinking %s' % os.path.abspath(f))
            os.unlink(f)

#def get_arch_info(i, f, archfiles, db):
#    """Read filename ``f`` with index ``i`` (position within list of filenames).  The
#    file has type ``filetype`` and will be added to MSID file at row index ``row``.
#    ``colnames`` is the list of column names for the content type (not used here).
#    """
#
#    filename = os.path.basename(f)
#    # Read FITS archive file and accumulate data into dats list and header into headers dict
#    logger.debug('Reading (%d / %d) %s' % (i, len(archfiles), filename))
#    hdus = pyfits.open(f)
#    hdu = hdus[1]
#
#    # Accumlate relevant info about archfile that will be ingested into
#    # MSID h5 files.  Commit info before h5 ingest so if there is a failure
#    # the needed info will be available to do the repair.
#    archfiles_row = dict((x, hdu.header.get(x.upper()))
#                         for x in archfiles_hdr_cols)
#    archfiles_row['checksum'] = hdu._checksum
#    archfiles_row['filename'] = filename
#    archfiles_row['filetime'] = int(
#        re.search(r'(\d+)', archfiles_row['filename']).group(1))
#    filedate = DateTime(archfiles_row['filetime']).date
#    year, doy = (int(x)
#                 for x in re.search(r'(\d\d\d\d):(\d\d\d)', filedate).groups())
#    archfiles_row['year'] = year
#    archfiles_row['doy'] = doy
#
#    hdus.close()
#    return archfiles_row

def update_archive(opt):
    contentdir = os.path.join(opt.data_root)
    if not os.path.exists(contentdir):
        os.makedirs(contentdir)
    archdb = os.path.join(contentdir, 'archfiles.db3')

    db = Ska.DBI.DBI(dbi='sqlite', server=archdb, autocommit=False)
    if opt.datestart:
        datestart = DateTime(opt.datestart)
    else:
        # Get datestart as the most-recent file time from archfiles table
        # will need min-of-max-slot-datestart
        last_time = min([db.fetchone(
                    "select max(filetime) from archfiles where slot = %d"
                    % s)['max(filetime)'] for s in range(0, 8)])
        datestart = DateTime(last_time)
    datestop = DateTime(opt.datestop)
    padding_seconds = 10000
    for tstart in np.arange(datestart.day_start().secs,
                            datestop.day_end().secs,
                            opt.days_at_once * 86400):

        # set times for a chunk
        range_tstart = tstart - padding_seconds
        range_tstop = tstart + opt.days_at_once * 86400
        if range_tstop > datestop.day_end().secs:
            range_tstop = datestop.day_end().secs
        range_tstop += padding_seconds
        print DateTime(range_tstart).date + " " + DateTime(range_tstop).date
        # make a temporary directory
        tmpdir = Ska.File.TempDir(dir=opt.temp_root)
        dirname = tmpdir.name
        print dirname
        # get the files
        with Ska.File.chdir(dirname):
            archfiles = get_archive_files(filetype,
                                          DateTime(range_tstart),
                                          DateTime(range_tstop))
            #archfiles = glob(os.path.join(dirname, '*'))
            for i, f in enumerate(archfiles):
                arch_info = read_archfile(i, f, archfiles, db)
                if arch_info:
                    move_archive_files(filetype, [f], opt)
                    db.insert(arch_info, 'archfiles')
        db.commit()

def main():
    opt = get_options()
    update_archive(opt)

if __name__ == '__main__':
    main()
<commit_msg>Use more idiomatic python in masked array handling<commit_after>import os
from glob import glob
import re
import logging
import shutil
import time
import pyfits
import numpy as np
import numpy.ma as ma
import argparse
from configobj import ConfigObj

import Ska.DBI
import Ska.arc5gl
from Chandra.Time import DateTime
import Ska.File

logger = logging.getLogger('aca0 fetch')
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler())

# borrowed from eng_archive
archfiles_hdr_cols = ('tstart', 'tstop', 'startmjf', 'startmnf',
                      'stopmjf', 'stopmnf',
                      'tlmver', 'ascdsver', 'revision', 'date',
                      'imgsize')

dtype = [('level', '|S4'), ('instrum', '|S7'), ('content', '|S13'),
         ('arc5gl_query', '|S27'), ('fileglob', '|S9')]
filetype = np.rec.fromrecords([('L0', 'PCAD', 'ACADATA', 'ACA0', '*fits.gz')],
                               dtype=dtype)[0]

aca_dtype = [('TIME', '>f8'), ('QUALITY', '>i4'), ('MJF', '>i4'),
             ('MNF', '>i4'),
             ('END_INTEG_TIME', '>f8'), ('INTEG', '>f4'), ('GLBSTAT', '|u1'),
             ('COMMCNT', '|u1'), ('COMMPROG', '|u1'), ('IMGFID1', '|u1'),
             ('IMGNUM1', '|u1'), ('IMGFUNC1', '|u1'), ('IMGSTAT', '|u1'),
             ('IMGROW0', '>i2'), ('IMGCOL0', '>i2'), ('IMGSCALE', '>i2'),
             ('BGDAVG', '>i2'), ('IMGFID2', '|u1'), ('IMGNUM2', '|u1'),
             ('IMGFUNC2', '|u1'), ('BGDRMS', '>i2'), ('TEMPCCD', '>f4'),
             ('TEMPHOUS', '>f4'), ('TEMPPRIM', '>f4'), ('TEMPSEC', '>f4'),
             ('BGDSTAT', '|u1'), ('IMGFID3', '|u1'), ('IMGNUM3', '|u1'),
             ('IMGFUNC3', '|u1'), ('IMGFID4', '|u1'), ('IMGNUM4', '|u1'),
             ('IMGFUNC4', '|u1'), ('IMGRAW', '>f4', (64,)),
             ('HD3TLM62', '|u1'),
             ('HD3TLM63', '|u1'), ('HD3TLM64', '|u1'), ('HD3TLM65', '|u1'),
             ('HD3TLM66', '|u1'), ('HD3TLM67', '|u1'), ('HD3TLM72', '|u1'),
             ('HD3TLM73', '|u1'), ('HD3TLM74', '|u1'), ('HD3TLM75', '|u1'),
             ('HD3TLM76', '|u1'), ('HD3TLM77', '|u1'),
             ('IMGSIZE', '>i4'),
             ]

#config = ConfigObj("aca0.conf")
config = dict(data_root='/data/aca/archive/aca0',
              temp_root='/data/aca/archive/temp',
              days_at_once=30.0)

def get_options():
    parser = argparse.ArgumentParser(
        description="Fetch aca level 0 products and make a file archive")
    defaults = dict(config)
    parser.set_defaults(**defaults)
    parser.add_argument("--data-root",
                        help="parent directory for all data")
    parser.add_argument("--temp-root",
                        help="parent temp directory")
    parser.add_argument("--datestart",
                        help="start date for retrieve (defaults to max date of archived files)")
    parser.add_argument("--datestop",
                        help="stop date for retrieve (defaults to now)")
    parser.add_argument("--days-at-once",
                        type=float,
                        help="if over this number, bin to chunks of this number of days")
    opt = parser.parse_args()
    return opt

def get_slot_data(tstart, tstop, slot, imgsize=[4, 6, 8],
                  db=None, data_root=config['data_root']):
    if not db:
        dbfile = os.path.join(config['data_root'], 'archfiles.db3')
        db = Ska.DBI.DBI(dbi='sqlite', server=dbfile)
    data_files = get_interval_files(tstart, tstop, slots=[slot],
                                    imgsize=imgsize, db=db)
    if not len(data_files):
        # return an empty masked array
        return ma.zeros(0, dtype=aca_dtype)
    rows = np.sum(data_files['rows'])
    zero_row = ma.zeros(1, dtype=aca_dtype)
    zero_row.mask = ma.masked
    all_rows = zero_row.repeat(rows)
    rowcount = 0
    for f in data_files:
        fp = os.path.join(data_root,
                          str(f['year']), str(f['doy']), f['filename'])
        hdu = pyfits.open(fp)
        chunk = hdu[1].data
        for fname in all_rows.dtype.names:
            if fname == 'IMGRAW' or fname == 'IMGSIZE':
                continue
            if fname in chunk.dtype.names:
                all_rows[fname][rowcount:(rowcount + len(chunk))] \
                    = chunk.field(fname)
        imgsize = int(np.sqrt(chunk[0].field('IMGRAW').size))
        all_rows['IMGSIZE'][rowcount:(rowcount + len(chunk))] = imgsize
        all_rows['IMGRAW'].reshape(rows, 8, 8)[
            rowcount:(rowcount + len(chunk)), 0:imgsize, 0:imgsize] = (
            chunk.field('IMGRAW').reshape(len(chunk), imgsize, imgsize))
        rowcount += len(chunk)
    return all_rows

def get_interval_files(tstart, tstop, slot, imgsize=[4, 6, 8], db=None):
    if not db:
        dbfile = os.path.join(config['data_root'], 'archfiles.db3')
        db = Ska.DBI.DBI(dbi='sqlite', server=dbfile)
    tstart = DateTime(tstart).secs
    tstop = DateTime(tstop).secs
    imgsize_str = ','.join([str(x) for x in imgsize])
    db_query = ('SELECT * FROM archfiles '
                'WHERE tstop > %f '
                'AND tstart < %f '
                'AND slot == %d '
                'AND imgsize in (%s) '
                'order by filetime asc '
                % (tstart, tstop, slot, imgsize_str))
    return db.fetchall(db_query)

def get_archive_files(filetype, datestart, datestop):
    """Update FITS file archive with arc5gl and ingest files into msid (HDF5) archive"""

    # Retrieve CXC archive files in a temp directory with arc5gl
    arc5 = Ska.arc5gl.Arc5gl(echo=True)
    n_queries = 1
    times = np.linspace(datestart.secs, datestop.secs, n_queries + 1)

    logger.info('********** %s %s **********'
                % (filetype['content'], time.ctime()))

    for t0, t1 in zip(times[:-1], times[1:]):
        arc5.sendline('tstart=%s' % DateTime(t0).date)
        arc5.sendline('tstop=%s' % DateTime(t1).date)
        arc5.sendline('get %s' % filetype['arc5gl_query'].lower())

    return sorted(glob(filetype['fileglob']))

def read_archfile(i, f, archfiles, db):
    """Read filename ``f`` with index ``i`` (position within list of filenames).  The
    file has type ``filetype`` and will be added to MSID file at row index ``row``.
    ``colnames`` is the list of column names for the content type (not used here).
    """
    # Check if filename is already in archfiles.
    # If so then abort further processing.
    filename = os.path.basename(f)
    colnames = archfiles_hdr_cols
    if db.fetchall('SELECT filename FROM archfiles WHERE filename=?',
                   (filename,)):
        logger.info(
            'File %s already in archfiles - unlinking and skipping' % f)
        os.unlink(f)
        return None

    # Read FITS archive file and accumulate data into dats list and header into headers dict
    logger.info('Reading (%d / %d) %s' % (i, len(archfiles), filename))
    hdus = pyfits.open(f)
    hdu = hdus[1]
    #dat = converters.convert(hdu.data, filetype['content'])

    # Accumlate relevant info about archfile that will be ingested into
    # MSID h5 files.  Commit info before h5 ingest so if there is a failure
    # the needed info will be available to do the repair.
    archfiles_row = dict((x, hdu.header.get(x.upper()))
                         for x in archfiles_hdr_cols)
    archfiles_row['checksum'] = hdu._checksum
    #archfiles_row['rowstart'] = row
    #archfiles_row['rowstop'] = row + len(dat)
    imgsize_sq = hdu.data[0].field('IMGRAW').shape[0]
    archfiles_row['imgsize'] = int(np.sqrt(imgsize_sq))
    archfiles_row['slot'] = int(re.search(
            r'acaf\d+N\d{3}_(\d)_img0.fits(\.gz)?',
            filename).group(1))
    archfiles_row['filename'] = filename
    archfiles_row['filetime'] = int(
        re.search(r'(\d+)', archfiles_row['filename']).group(1))
    filedate = DateTime(archfiles_row['filetime']).date
    year, doy = (int(x)
                 for x in re.search(r'(\d\d\d\d):(\d\d\d)', filedate).groups())
    archfiles_row['year'] = year
    archfiles_row['doy'] = doy
    archfiles_row['rows'] = len(hdu.data)
    hdus.close()
    return  archfiles_row

def move_archive_files(filetype, archfiles, opt):

    if not os.path.exists(opt.data_root):
        os.makedirs(opt.data_root)

    for f in archfiles:
        if not os.path.exists(f):
            continue
        basename = os.path.basename(f)
        tstart = re.search(r'(\d+)', str(basename)).group(1)
        datestart = DateTime(tstart).date
        year, doy = re.search(r'(\d\d\d\d):(\d\d\d)', datestart).groups()

        archdir = os.path.abspath(os.path.join(opt.data_root,
                                               year,
                                               doy))
        archfile = os.path.abspath(os.path.join(archdir, basename))

        if not os.path.exists(archdir):
            os.makedirs(archdir)

        if not os.path.exists(archfile):
            logger.info('mv %s %s' % (os.path.abspath(f), archfile))
#            if not opt.dry_run:
#                if not opt.occ:
#            shutil.copy2(f, stagedir)
            shutil.move(f, archfile)

        if os.path.exists(f):
            logger.info('Unlinking %s' % os.path.abspath(f))
            os.unlink(f)

#def get_arch_info(i, f, archfiles, db):
#    """Read filename ``f`` with index ``i`` (position within list of filenames).  The
#    file has type ``filetype`` and will be added to MSID file at row index ``row``.
#    ``colnames`` is the list of column names for the content type (not used here).
#    """
#
#    filename = os.path.basename(f)
#    # Read FITS archive file and accumulate data into dats list and header into headers dict
#    logger.debug('Reading (%d / %d) %s' % (i, len(archfiles), filename))
#    hdus = pyfits.open(f)
#    hdu = hdus[1]
#
#    # Accumlate relevant info about archfile that will be ingested into
#    # MSID h5 files.  Commit info before h5 ingest so if there is a failure
#    # the needed info will be available to do the repair.
#    archfiles_row = dict((x, hdu.header.get(x.upper()))
#                         for x in archfiles_hdr_cols)
#    archfiles_row['checksum'] = hdu._checksum
#    archfiles_row['filename'] = filename
#    archfiles_row['filetime'] = int(
#        re.search(r'(\d+)', archfiles_row['filename']).group(1))
#    filedate = DateTime(archfiles_row['filetime']).date
#    year, doy = (int(x)
#                 for x in re.search(r'(\d\d\d\d):(\d\d\d)', filedate).groups())
#    archfiles_row['year'] = year
#    archfiles_row['doy'] = doy
#
#    hdus.close()
#    return archfiles_row

def update_archive(opt):
    contentdir = os.path.join(opt.data_root)
    if not os.path.exists(contentdir):
        os.makedirs(contentdir)
    archdb = os.path.join(contentdir, 'archfiles.db3')

    db = Ska.DBI.DBI(dbi='sqlite', server=archdb, autocommit=False)
    if opt.datestart:
        datestart = DateTime(opt.datestart)
    else:
        # Get datestart as the most-recent file time from archfiles table
        # will need min-of-max-slot-datestart
        last_time = min([db.fetchone(
                    "select max(filetime) from archfiles where slot = %d"
                    % s)['max(filetime)'] for s in range(0, 8)])
        datestart = DateTime(last_time)
    datestop = DateTime(opt.datestop)
    padding_seconds = 10000
    for tstart in np.arange(datestart.day_start().secs,
                            datestop.day_end().secs,
                            opt.days_at_once * 86400):

        # set times for a chunk
        range_tstart = tstart - padding_seconds
        range_tstop = tstart + opt.days_at_once * 86400
        if range_tstop > datestop.day_end().secs:
            range_tstop = datestop.day_end().secs
        range_tstop += padding_seconds
        print DateTime(range_tstart).date + " " + DateTime(range_tstop).date
        # make a temporary directory
        tmpdir = Ska.File.TempDir(dir=opt.temp_root)
        dirname = tmpdir.name
        print dirname
        # get the files
        with Ska.File.chdir(dirname):
            archfiles = get_archive_files(filetype,
                                          DateTime(range_tstart),
                                          DateTime(range_tstop))
            #archfiles = glob(os.path.join(dirname, '*'))
            for i, f in enumerate(archfiles):
                arch_info = read_archfile(i, f, archfiles, db)
                if arch_info:
                    move_archive_files(filetype, [f], opt)
                    db.insert(arch_info, 'archfiles')
        db.commit()

def main():
    opt = get_options()
    update_archive(opt)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import os
from datetime import datetime
from stat import ST_CTIME
import re
import logging
import csv

from dipper.sources.PostgreSQLSource import PostgreSQLSource
from dipper.models.Model import Model
from dipper import config
from dipper.models.Reference import Reference
from dipper.models.BiolinkVocabulary import BioLinkVocabulary as blv

LOG = logging.getLogger(__name__)

class EOM(PostgreSQLSource):
    """
    Elements of Morphology is a resource from NHGRI that has definitions of
    morphological abnormalities, together with image depictions.
    We pull those relationships, as well as our local mapping of equivalences
    between EOM and HP terminologies.

    The website is crawled monthly by NIF's DISCO crawler system,
    which we utilize here.
    Be sure to have pg user/password connection details in your conf.yaml file,
    like:
    dbauth : {'disco' : {'user' : '<username>', 'password' : '<password>'}}

    Monarch-curated data for the HP to EOM mapping is stored at
    https://raw.githubusercontent.com/obophenotype/human-phenotype-ontology/master/src/mappings/hp-to-eom-mapping.tsv

    Since this resource is so small, the entirety of it is the "test" set.

    """

    # we are using the production view here; should we be using services?
    tables = [
        'dvp.pr_nlx_157874_1'
    ]

    GHRAW = 'https://raw.githubusercontent.com/obophenotype/human-phenotype-ontology'
    files = {
        'map': {  # in 2019 this repo has beem untouched in four years
            'file': 'hp-to-eom-mapping.tsv',
            'url': GHRAW + '/master/src/mappings/hp-to-eom-mapping.tsv',
            'columns': [
                'morphology_term_id',
                'morphology_term_label',
                'HP ID',
                'HP Label',
                'Notes'
            ]
        }
    }
    resources = {
        'tables': {
            'url': 'nif-db.crbs.ucsd.edu:5432',
            'file': 'dvp.pr_nlx_157874_1',
            'columns': [
                # head -1 dvp.pr_nlx_157874_1 | tr '\t' '\n' |sed "s|\(.*\)|'\1',|g"
                'morphology_term_id',
                'morphology_term_num',
                'morphology_term_label',
                'morphology_term_url',
                'terminology_category_label',
                'terminology_category_url',
                'subcategory',
                'objective_definition',
                'subjective_definition',
                'comments',
                'synonyms',
                'replaces',
                'small_figure_url',
                'large_figure_url',
                'e_uid',
                'v_uid',
                'v_uuid',
                'v_lastmodified',
                'v_status',
                'v_lastmodified_epoch'
            ],
        },
    }

    def __init__(self, graph_type, are_bnodes_skolemized):
        super().__init__(
            graph_type,
            are_bnodes_skolemized,
            'eom',
            ingest_title='Elements of Morphology',
            ingest_url='http://elementsofmorphology.nih.gov',
            data_rights='http://www.genome.gov/copyright.cfm',
            license_url='https://creativecommons.org/publicdomain/mark/1.0/'
            # file_handle=None
        )

    def fetch(self, is_dl_forced=False):
        '''connection details for DISCO'''
        cxn = {}
        cxn['host'] = 'nif-db.crbs.ucsd.edu'
        cxn['database'] = 'disco_crawler'
        cxn['port'] = '5432'
        cxn['user'] = config.get_config()['user']['disco']
        cxn['password'] = config.get_config()['keys'][cxn['user']]

        self.dataset.setFileAccessUrl(
            'jdbc:postgresql://'+cxn['host']+':'+cxn['port']+'/'+cxn['database'],
            is_object_literal=True)

        # process the tables
        # self.fetch_from_pgdb(self.tables,cxn,100)  #for testing
        self.fetch_from_pgdb(self.tables, cxn)

        self.get_files(is_dl_forced)

        # FIXME: Everything needed for data provenance?
        fstat = os.stat('/'.join((self.rawdir, 'dvp.pr_nlx_157874_1')))
        filedate = datetime.utcfromtimestamp(fstat[ST_CTIME]).strftime("%Y-%m-%d")
        self.dataset.setVersion(filedate)

    def parse(self, limit=None):
        '''
            Over ride Source.parse inherited via PostgreSQLSource
        '''

        if limit is not None:
            LOG.info("Only parsing first %s rows of each file", limit)

        if self.test_only:
            self.test_mode = True

        LOG.info("Parsing files...")

        self._process_nlx_157874_1_view(
            '/'.join((self.rawdir, 'dvp.pr_nlx_157874_1')), limit)
        self._map_eom_terms(
            '/'.join((self.rawdir, self.files['map']['file'])), limit)

        LOG.info("Finished parsing.")

        # since it's so small,
        # we default to copying the entire graph to the test set
        self.testgraph = self.graph

    def _process_nlx_157874_1_view(self, raw, limit=None):
        """
        This table contains the Elements of Morphology data .
        Note that foaf:depiction is inverse of foaf:depicts relationship.

        Since it is bad form to have two definitions,
        we concatenate the two into one string.

        Turtle:
            <eom id> a owl:Class
                rdf:label Literal(eom label)
                oboInOwl:has_related_synonym Literal(synonym list)
                IAO:definition Literal(objective_def. subjective def)
                foaf:depiction Literal(small_image_url),
                               Literal(large_image_url)
                foaf:page Literal(page_url)
                rdfs:comment Literal(long commented text)

        TEC_note: URL are not literals.

        :param raw:
        :param limit:
        :return:
        """

        src_key = 'tables'
        model = Model(self.graph)
        col = self.resources[src_key]['columns']
        with open(raw, 'r') as rawread:
            reader = csv.reader(rawread, delimiter='\t', quotechar='\"')
            row = next(reader)
            if not self.check_fileheader(col, row):
                pass

            for row in reader:
                # head -1 dvp.pr_nlx_157874_1|tr '\t' '\n'|
                # sed "s|\(.*\)|# \1 = row[col.index('\1')]|g"

                morphology_term_id = row[col.index('morphology_term_id')].strip()
                # morphology_term_num = row[col.index('morphology_term_num')]
                morphology_term_label = row[col.index('morphology_term_label')].strip()
                morphology_term_url = row[col.index('morphology_term_url')].strip()
                # terminology_category_label = row[
                #   col.index('terminology_category_label')]
                # terminology_category_url = row[col.index('terminology_category_url')]
                # subcategory = row[col.index('subcategory')]
                objective_definition = row[col.index('objective_definition')].strip()
                subjective_definition = row[col.index('subjective_definition')].strip()
                comments = row[col.index('comments')].strip()
                synonyms = row[col.index('synonyms')].strip()
                replaces = row[col.index('replaces')].strip()
                small_figure_url = row[col.index('small_figure_url')].strip()
                large_figure_url = row[col.index('large_figure_url')].strip()
                # e_uid = row[col.index('e_uid')]
                # v_uid = row[col.index('v_uid')]
                # v_uuid = row[col.index('v_uuid')]
                # v_lastmodified = row[col.index('v_lastmodified')]
                # v_status = row[col.index('v_status')]
                # v_lastmodified_epoch = row[col.index('v_lastmodified_epoch')]

                # Add morphology term to graph as a class
                # with label, type, and description.
                model.addClassToGraph(morphology_term_id, morphology_term_label,
                                      blv.terms.PhenotypicFeature.value)

                # Assemble the description text

                if subjective_definition != '' and not (
                        re.match(r'.+\.$', subjective_definition)):
                    # add a trailing period.
                    subjective_definition = subjective_definition + '.'
                if objective_definition != '' and not (
                        re.match(r'.+\.$', objective_definition)):
                    # add a trailing period.
                    objective_definition = objective_definition + '.'

                definition = '  '.join(
                    (objective_definition, subjective_definition))

                model.addDefinition(morphology_term_id, definition,
                                    class_category=blv.terms.PhenotypicFeature.value)

                # <term id> FOAF:depicted_by literal url
                # <url> type foaf:depiction

                # do we want both images?
                # morphology_term_id has depiction small_figure_url
                if small_figure_url != '':
                    model.addDepiction(morphology_term_id, small_figure_url,
                                       subject_category=blv.terms.PhenotypicFeature.value,
                                       object_category=
                                       blv.terms.InformationContentEntity.value)

                # morphology_term_id has depiction large_figure_url
                if large_figure_url != '':
                    model.addDepiction(morphology_term_id, large_figure_url,
                                       subject_category=blv.terms.PhenotypicFeature.value,
                                       object_category=blv.terms.PhenotypicFeature.value)

                # morphology_term_id has comment comments
                if comments != '':
                    model.addComment(morphology_term_id, comments,
                                     subject_category=blv.terms.PhenotypicFeature.value)

                for syn in synonyms.split(';'):
                    model.addSynonym(
                        morphology_term_id, syn.strip(),
                        self.globaltt['has_exact_synonym'],
                        class_category=blv.terms.PhenotypicFeature.value)

                # morphology_term_id has_related_synonym replaces (; delimited)
                if replaces not in ['', synonyms]:
                    for syn in replaces.split(';'):
                        model.addSynonym(
                            morphology_term_id, syn.strip(),
                            self.globaltt['has_related_synonym'],
                            class_category=blv.terms.PhenotypicFeature.value)

                # <morphology_term_id> <foaf:page> morphology_term_url
                if morphology_term_id is not None:
                    reference = Reference(
                        self.graph, morphology_term_id, self.globaltt['web page'])

                    # TEC 201905:
                    # Not so sure we need explicit   <eom_uri> <webpage> <eom_url>.
                    # since <eom_uri> IS the <eom_url>.

                    reference.addPage(morphology_term_id, morphology_term_url,
                                      subject_category=blv.terms.PhenotypicFeature.value)

                if limit is not None and reader.line_num > limit:
                    break

    def _map_eom_terms(self, raw, limit=None):
        """
        This table contains the HP ID mappings from the local tsv file.
        Triples:
            <eom id> owl:equivalentClass <hp id>
        :param raw:
        :param limit:
        :return:
        """

        model = Model(self.graph)
        line_counter = 0
        col = self.files['map']['columns']
        with open(raw, 'r') as reader:
            line = reader.readline().strip()
            line = line.strip('/n')
            if self.check_fileheader(col, line.split('\t')):
                pass
            for line in reader:
                line_counter += 1
                row = line.strip('\n').split('\t')

                morphology_term_id = row[col.index('morphology_term_id')].strip()
                # morphology_term_label = row[col.index('morphology_term_label')]
                hp_id = row[col.index('HP ID')].strip()
                # hp_label = row[col.index('HP_Label')]
                # notes = row[col.index('Notes')]

                # Sub out the underscores for colons.
                hp_id = re.sub('_', ':', hp_id)
                if re.match(".*HP:.*", hp_id):
                    # add the HP term as a class
                    model.addClassToGraph(hp_id, None,
                                          class_category=blv.terms.PhenotypicFeature.value)  # TEC subclass of phenotype??
                    # Add the HP ID as an equivalent class
                    model.addEquivalentClass(morphology_term_id, hp_id,
                                             subject_category=
                                             blv.terms.PhenotypicFeature.value,
                                             object_category=
                                             blv.terms.PhenotypicFeature.value)
                else:
                    LOG.warning('No matching HP term for %s', morphology_term_id)

                if limit is not None and line_counter > limit:
                    break

    def getTestSuite(self):
        import unittest
        from tests.test_eom import EOMTestCase

        test_suite = unittest.TestLoader().loadTestsFromTestCase(EOMTestCase)

        return test_suite
<commit_msg>Lint (long line)<commit_after>import os
from datetime import datetime
from stat import ST_CTIME
import re
import logging
import csv

from dipper.sources.PostgreSQLSource import PostgreSQLSource
from dipper.models.Model import Model
from dipper import config
from dipper.models.Reference import Reference
from dipper.models.BiolinkVocabulary import BioLinkVocabulary as blv

LOG = logging.getLogger(__name__)

class EOM(PostgreSQLSource):
    """
    Elements of Morphology is a resource from NHGRI that has definitions of
    morphological abnormalities, together with image depictions.
    We pull those relationships, as well as our local mapping of equivalences
    between EOM and HP terminologies.

    The website is crawled monthly by NIF's DISCO crawler system,
    which we utilize here.
    Be sure to have pg user/password connection details in your conf.yaml file,
    like:
    dbauth : {'disco' : {'user' : '<username>', 'password' : '<password>'}}

    Monarch-curated data for the HP to EOM mapping is stored at
    https://raw.githubusercontent.com/obophenotype/human-phenotype-ontology/master/src/mappings/hp-to-eom-mapping.tsv

    Since this resource is so small, the entirety of it is the "test" set.

    """

    # we are using the production view here; should we be using services?
    tables = [
        'dvp.pr_nlx_157874_1'
    ]

    GHRAW = 'https://raw.githubusercontent.com/obophenotype/human-phenotype-ontology'
    files = {
        'map': {  # in 2019 this repo has beem untouched in four years
            'file': 'hp-to-eom-mapping.tsv',
            'url': GHRAW + '/master/src/mappings/hp-to-eom-mapping.tsv',
            'columns': [
                'morphology_term_id',
                'morphology_term_label',
                'HP ID',
                'HP Label',
                'Notes'
            ]
        }
    }
    resources = {
        'tables': {
            'url': 'nif-db.crbs.ucsd.edu:5432',
            'file': 'dvp.pr_nlx_157874_1',
            'columns': [
                # head -1 dvp.pr_nlx_157874_1 | tr '\t' '\n' |sed "s|\(.*\)|'\1',|g"
                'morphology_term_id',
                'morphology_term_num',
                'morphology_term_label',
                'morphology_term_url',
                'terminology_category_label',
                'terminology_category_url',
                'subcategory',
                'objective_definition',
                'subjective_definition',
                'comments',
                'synonyms',
                'replaces',
                'small_figure_url',
                'large_figure_url',
                'e_uid',
                'v_uid',
                'v_uuid',
                'v_lastmodified',
                'v_status',
                'v_lastmodified_epoch'
            ],
        },
    }

    def __init__(self, graph_type, are_bnodes_skolemized):
        super().__init__(
            graph_type,
            are_bnodes_skolemized,
            'eom',
            ingest_title='Elements of Morphology',
            ingest_url='http://elementsofmorphology.nih.gov',
            data_rights='http://www.genome.gov/copyright.cfm',
            license_url='https://creativecommons.org/publicdomain/mark/1.0/'
            # file_handle=None
        )

    def fetch(self, is_dl_forced=False):
        '''connection details for DISCO'''
        cxn = {}
        cxn['host'] = 'nif-db.crbs.ucsd.edu'
        cxn['database'] = 'disco_crawler'
        cxn['port'] = '5432'
        cxn['user'] = config.get_config()['user']['disco']
        cxn['password'] = config.get_config()['keys'][cxn['user']]

        self.dataset.setFileAccessUrl(
            'jdbc:postgresql://'+cxn['host']+':'+cxn['port']+'/'+cxn['database'],
            is_object_literal=True)

        # process the tables
        # self.fetch_from_pgdb(self.tables,cxn,100)  #for testing
        self.fetch_from_pgdb(self.tables, cxn)

        self.get_files(is_dl_forced)

        # FIXME: Everything needed for data provenance?
        fstat = os.stat('/'.join((self.rawdir, 'dvp.pr_nlx_157874_1')))
        filedate = datetime.utcfromtimestamp(fstat[ST_CTIME]).strftime("%Y-%m-%d")
        self.dataset.setVersion(filedate)

    def parse(self, limit=None):
        '''
            Over ride Source.parse inherited via PostgreSQLSource
        '''

        if limit is not None:
            LOG.info("Only parsing first %s rows of each file", limit)

        if self.test_only:
            self.test_mode = True

        LOG.info("Parsing files...")

        self._process_nlx_157874_1_view(
            '/'.join((self.rawdir, 'dvp.pr_nlx_157874_1')), limit)
        self._map_eom_terms(
            '/'.join((self.rawdir, self.files['map']['file'])), limit)

        LOG.info("Finished parsing.")

        # since it's so small,
        # we default to copying the entire graph to the test set
        self.testgraph = self.graph

    def _process_nlx_157874_1_view(self, raw, limit=None):
        """
        This table contains the Elements of Morphology data .
        Note that foaf:depiction is inverse of foaf:depicts relationship.

        Since it is bad form to have two definitions,
        we concatenate the two into one string.

        Turtle:
            <eom id> a owl:Class
                rdf:label Literal(eom label)
                oboInOwl:has_related_synonym Literal(synonym list)
                IAO:definition Literal(objective_def. subjective def)
                foaf:depiction Literal(small_image_url),
                               Literal(large_image_url)
                foaf:page Literal(page_url)
                rdfs:comment Literal(long commented text)

        TEC_note: URL are not literals.

        :param raw:
        :param limit:
        :return:
        """

        src_key = 'tables'
        model = Model(self.graph)
        col = self.resources[src_key]['columns']
        with open(raw, 'r') as rawread:
            reader = csv.reader(rawread, delimiter='\t', quotechar='\"')
            row = next(reader)
            if not self.check_fileheader(col, row):
                pass

            for row in reader:
                # head -1 dvp.pr_nlx_157874_1|tr '\t' '\n'|
                # sed "s|\(.*\)|# \1 = row[col.index('\1')]|g"

                morphology_term_id = row[col.index('morphology_term_id')].strip()
                # morphology_term_num = row[col.index('morphology_term_num')]
                morphology_term_label = row[col.index('morphology_term_label')].strip()
                morphology_term_url = row[col.index('morphology_term_url')].strip()
                # terminology_category_label = row[
                #   col.index('terminology_category_label')]
                # terminology_category_url = row[col.index('terminology_category_url')]
                # subcategory = row[col.index('subcategory')]
                objective_definition = row[col.index('objective_definition')].strip()
                subjective_definition = row[col.index('subjective_definition')].strip()
                comments = row[col.index('comments')].strip()
                synonyms = row[col.index('synonyms')].strip()
                replaces = row[col.index('replaces')].strip()
                small_figure_url = row[col.index('small_figure_url')].strip()
                large_figure_url = row[col.index('large_figure_url')].strip()
                # e_uid = row[col.index('e_uid')]
                # v_uid = row[col.index('v_uid')]
                # v_uuid = row[col.index('v_uuid')]
                # v_lastmodified = row[col.index('v_lastmodified')]
                # v_status = row[col.index('v_status')]
                # v_lastmodified_epoch = row[col.index('v_lastmodified_epoch')]

                # Add morphology term to graph as a class
                # with label, type, and description.
                model.addClassToGraph(morphology_term_id, morphology_term_label,
                                      blv.terms.PhenotypicFeature.value)

                # Assemble the description text

                if subjective_definition != '' and not (
                        re.match(r'.+\.$', subjective_definition)):
                    # add a trailing period.
                    subjective_definition = subjective_definition + '.'
                if objective_definition != '' and not (
                        re.match(r'.+\.$', objective_definition)):
                    # add a trailing period.
                    objective_definition = objective_definition + '.'

                definition = '  '.join(
                    (objective_definition, subjective_definition))

                model.addDefinition(morphology_term_id, definition,
                                    class_category=blv.terms.PhenotypicFeature.value)

                # <term id> FOAF:depicted_by literal url
                # <url> type foaf:depiction

                # do we want both images?
                # morphology_term_id has depiction small_figure_url
                if small_figure_url != '':
                    model.addDepiction(morphology_term_id, small_figure_url,
                                       subject_category=blv.terms.PhenotypicFeature.value,
                                       object_category=
                                       blv.terms.InformationContentEntity.value)

                # morphology_term_id has depiction large_figure_url
                if large_figure_url != '':
                    model.addDepiction(morphology_term_id, large_figure_url,
                                       subject_category=blv.terms.PhenotypicFeature.value,
                                       object_category=blv.terms.PhenotypicFeature.value)

                # morphology_term_id has comment comments
                if comments != '':
                    model.addComment(morphology_term_id, comments,
                                     subject_category=blv.terms.PhenotypicFeature.value)

                for syn in synonyms.split(';'):
                    model.addSynonym(
                        morphology_term_id, syn.strip(),
                        self.globaltt['has_exact_synonym'],
                        class_category=blv.terms.PhenotypicFeature.value)

                # morphology_term_id has_related_synonym replaces (; delimited)
                if replaces not in ['', synonyms]:
                    for syn in replaces.split(';'):
                        model.addSynonym(
                            morphology_term_id, syn.strip(),
                            self.globaltt['has_related_synonym'],
                            class_category=blv.terms.PhenotypicFeature.value)

                # <morphology_term_id> <foaf:page> morphology_term_url
                if morphology_term_id is not None:
                    reference = Reference(
                        self.graph, morphology_term_id, self.globaltt['web page'])

                    # TEC 201905:
                    # Not so sure we need explicit   <eom_uri> <webpage> <eom_url>.
                    # since <eom_uri> IS the <eom_url>.

                    reference.addPage(morphology_term_id, morphology_term_url,
                                      subject_category=blv.terms.PhenotypicFeature.value)

                if limit is not None and reader.line_num > limit:
                    break

    def _map_eom_terms(self, raw, limit=None):
        """
        This table contains the HP ID mappings from the local tsv file.
        Triples:
            <eom id> owl:equivalentClass <hp id>
        :param raw:
        :param limit:
        :return:
        """

        model = Model(self.graph)
        line_counter = 0
        col = self.files['map']['columns']
        with open(raw, 'r') as reader:
            line = reader.readline().strip()
            line = line.strip('/n')
            if self.check_fileheader(col, line.split('\t')):
                pass
            for line in reader:
                line_counter += 1
                row = line.strip('\n').split('\t')

                morphology_term_id = row[col.index('morphology_term_id')].strip()
                # morphology_term_label = row[col.index('morphology_term_label')]
                hp_id = row[col.index('HP ID')].strip()
                # hp_label = row[col.index('HP_Label')]
                # notes = row[col.index('Notes')]

                # Sub out the underscores for colons.
                hp_id = re.sub('_', ':', hp_id)
                if re.match(".*HP:.*", hp_id):
                    # add the HP term as a class
                    model.addClassToGraph(hp_id, None,
                                          class_category=blv.terms.PhenotypicFeature.
                                          value)  # TEC subclass of phenotype??
                    # Add the HP ID as an equivalent class
                    model.addEquivalentClass(morphology_term_id, hp_id,
                                             subject_category=
                                             blv.terms.PhenotypicFeature.value,
                                             object_category=
                                             blv.terms.PhenotypicFeature.value)
                else:
                    LOG.warning('No matching HP term for %s', morphology_term_id)

                if limit is not None and line_counter > limit:
                    break

    def getTestSuite(self):
        import unittest
        from tests.test_eom import EOMTestCase

        test_suite = unittest.TestLoader().loadTestsFromTestCase(EOMTestCase)

        return test_suite
<|endoftext|>


<commit_before><commit_msg>fix ApplicationsUnsolicitedFilter<commit_after><|endoftext|>


<commit_before>#!/usr/bin/python2.5
#
# Copyright 2009 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Student Proposal (Model) query functions.
"""

__authors__ = [
  '"Lennard de Rijk" <ljvderijk@gmail.com>',
  ]

from soc.logic.models import base
from soc.logic.models import student as student_logic
from soc.models import student_proposal

import soc.models.linkable
import soc.models.student_proposal

class Logic(base.Logic):
  """Logic methods for the Student Proposal model.
  """

  def __init__(self, model=soc.models.student_proposal.StudentProposal,
               base_model=soc.models.linkable.Linkable,
               scope_logic=student_logic):
    """Defines the name, key_name and model for this entity.
    """

    super(Logic, self).__init__(model=model, base_model=base_model,
                                scope_logic=scope_logic)

  def getRankerFor(self, entity):
    """Returns the ranker for the given Student Proposal.

    Args:
      entity: Student Proposal entity for which the ranker should be returned

    Returns:
      Ranker object which is used to rank the given entity
    """

    from soc.logic.models.ranker_root import logic as ranker_root_logic

    fields = {'link_id': student_proposal.DEF_RANKER_NAME,
        'scope': entity.org}

    ranker_root = ranker_root_logic.getForFields(fields, unique=True)
    ranker = ranker_root_logic.getRootFromEntity(ranker_root)

    return ranker

  def _onCreate(self, entity):
    """Adds this proposal to the organization ranker entity.
    """

    ranker = self.getRankerFor(entity)
    ranker.SetScore(entity.key().name(), [entity.score])

    super(Logic, self)._onCreate(entity)

  def _updateField(self, entity, entity_properties, name):
    """Called when the fields of the student_proposal are updated.

      - Update the ranker if the score changes and keep the score within bounds
      - Remove the entity from the ranker when the status changes to invalid or rejected
    """

    value = entity_properties[name]

    if name == 'score':
      # keep the score within bounds
      min_score, max_score = student_proposal.DEF_SCORE

      value = max(min_score, min(value, max_score-1))
      entity_properties[name] = value

      # update the ranker
      ranker = self.getRankerFor(entity)
      ranker.SetScore(entity.key().name(), [value])

    if name == 'status':

      if value in ['invalid', 'rejected'] and entity.status != value:
        # the proposal is going into invalid or rejected state
        # remove the score from the ranker
        ranker = self.getRankerFor(entity)

        # entries in the ranker can be removed by setting the score to None
        ranker.SetScore(entity.key().name(), None)

    return super(Logic, self)._updateField(entity, entity_properties, name)

  def delete(self, entity):
    """Removes Ranker entry and all ReviewFollowers before deleting the entity.

    Args:
      entity: an existing entity in datastore
    """

    from soc.models.logic.review_follower import logic as review_follower_logic

    # entries in the ranker can be removed by setting the score to None
    ranker = self.getRankerFor(entity)
    ranker.SetScore(entity.key().name(), None)

    # get all the ReviewFollwers that have this entity as it's scope
    fields = {'scope': entity}

    # TODO be sure that this captures all the followers
    followers = review_follower_logic.getForFields(fields)

    for follower in followers:
      review_follower_logic.delete(follower)

    # call to super to complete the deletion
    super(Logic, self).delete(entity)

logic = Logic()
<commit_msg>Fixed an Epic Fail in StudentProposal logic ^_^.<commit_after>#!/usr/bin/python2.5
#
# Copyright 2009 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Student Proposal (Model) query functions.
"""

__authors__ = [
  '"Lennard de Rijk" <ljvderijk@gmail.com>',
  ]

from soc.logic.models import base
from soc.logic.models import student as student_logic
from soc.models import student_proposal

import soc.models.linkable
import soc.models.student_proposal

class Logic(base.Logic):
  """Logic methods for the Student Proposal model.
  """

  def __init__(self, model=soc.models.student_proposal.StudentProposal,
               base_model=soc.models.linkable.Linkable,
               scope_logic=student_logic):
    """Defines the name, key_name and model for this entity.
    """

    super(Logic, self).__init__(model=model, base_model=base_model,
                                scope_logic=scope_logic)

  def getRankerFor(self, entity):
    """Returns the ranker for the given Student Proposal.

    Args:
      entity: Student Proposal entity for which the ranker should be returned

    Returns:
      Ranker object which is used to rank the given entity
    """

    from soc.logic.models.ranker_root import logic as ranker_root_logic

    fields = {'link_id': student_proposal.DEF_RANKER_NAME,
        'scope': entity.org}

    ranker_root = ranker_root_logic.getForFields(fields, unique=True)
    ranker = ranker_root_logic.getRootFromEntity(ranker_root)

    return ranker

  def _onCreate(self, entity):
    """Adds this proposal to the organization ranker entity.
    """

    ranker = self.getRankerFor(entity)
    ranker.SetScore(entity.key().name(), [entity.score])

    super(Logic, self)._onCreate(entity)

  def _updateField(self, entity, entity_properties, name):
    """Called when the fields of the student_proposal are updated.

      - Update the ranker if the score changes and keep the score within bounds
      - Remove the entity from the ranker when the status changes to invalid or rejected
    """

    value = entity_properties[name]

    if name == 'score':
      # keep the score within bounds
      min_score, max_score = student_proposal.DEF_SCORE

      value = max(min_score, min(value, max_score-1))
      entity_properties[name] = value

      # update the ranker
      ranker = self.getRankerFor(entity)
      ranker.SetScore(entity.key().name(), [value])

    if name == 'status':

      if value in ['invalid', 'rejected'] and entity.status != value:
        # the proposal is going into invalid or rejected state
        # remove the score from the ranker
        ranker = self.getRankerFor(entity)

        # entries in the ranker can be removed by setting the score to None
        ranker.SetScore(entity.key().name(), None)

    return super(Logic, self)._updateField(entity, entity_properties, name)

  def delete(self, entity):
    """Removes Ranker entry and all ReviewFollowers before deleting the entity.

    Args:
      entity: an existing entity in datastore
    """

    from soc.logic.models.review_follower import logic as review_follower_logic

    # entries in the ranker can be removed by setting the score to None
    ranker = self.getRankerFor(entity)
    ranker.SetScore(entity.key().name(), None)

    # get all the ReviewFollwers that have this entity as it's scope
    fields = {'scope': entity}

    # TODO be sure that this captures all the followers
    followers = review_follower_logic.getForFields(fields)

    for follower in followers:
      review_follower_logic.delete(follower)

    # call to super to complete the deletion
    super(Logic, self).delete(entity)

logic = Logic()
<|endoftext|>