<commit_before>
from Song import Song
from State import state
from player import PlayerEventCallbacks
from utils import *
import math, random
import appinfo

class RandomFileQueueGen:
	randomQuality = 0.5

	def __init__(self, dir):
		import os
		from RandomFileQueue import RandomFileQueue
		self.fileQueue = RandomFileQueue(
			rootdir = os.path.expanduser(dir),
			fileexts = appinfo.formats)

	def next(self):
		return self.fileQueue.getNextFile()

	def __iter__(self):
		while True:
			yield self.next()

class RandomFromSongDatabaseGen:
	randomQuality = 0.0

	def __init__(self):
		from SongStore import SongStore
		self.database = SongStore(appinfo.userdir)

		def loadDatabase():
			import utils

			self.database.open()

			for dir in appinfo.musicdirs:
				self.database.addMany(utils.getSongsFromDirectory(dir))

			self.database.close()

			self.randomQuality = 0.5
			print "Done loading songs"

		from threading import Thread
		loadDatabaseThread = Thread(target=loadDatabase, name="loadDatabase")
		loadDatabaseThread.start()

	def next(self):
		try:
			oldSong = state.recentlyPlayedList.getLastN(1)[0]
		except:
			oldSong = None

		self.database.open()
		songs = self.database.getRandomSongs(oldSong=oldSong, limit=1)
		self.database.close()

		return next(iter(songs))

	def __iter__(self):
		while True:
			yield self.next()

class RandomSongs:
	randomQuality = 0.5
	def __init__(self, generators):
		self.generators = [gen() for gen in generators]
	def next(self):
		while True:
			generators = list(self.generators)
			if not generators:
				raise StopIteration
			qualitySum = sum([gen.randomQuality for gen in generators])
			self.randomQuality = qualitySum / len(generators)
			r = random.random() * qualitySum
			i = 0
			gen = generators[i]
			while i < len(generators)-1 and r > gen.randomQuality:
				r -= gen.randomQuality
				i += 1
				gen = generators[i]
			try:
				return next(gen)
			except StopIteration:
				#print "warning: generator", gen, "raised StopIteration"
				#sys.excepthook(*sys.exc_info())
				pass
			generators.pop(i)
	def __iter__(self):
		while True:
			yield self.next()

from collections import deque
from threading import Lock

class InfQueue:
	def __init__(self):
		self.generator = RandomSongs([
			RandomFromSongDatabaseGen,
			lambda: RandomSongs([
				(lambda: RandomFileQueueGen(dir)) for dir in appinfo.musicdirs])
		])
		self.checkNextNForBest = 10
		self.checkLastNForContext = 10

	def calcContextMatchScore(self, song):
		count = 0
		lastSongs = state.recentlyPlayedList.getLastN(self.checkLastNForContext)
		lastSongs = filter(lambda s: not s.skipped, lastSongs)
		if not lastSongs: return 0.0
		for lastSong in lastSongs:
			count += max(intersectFuzzySets(song.tags, lastSong.tags).values() + [0])
		s = float(count) / self.checkLastNForContext
		# We likely get small values here. Boost a bit but keep in [0,1] range. sqrt is a good fit.
		return math.sqrt(s)

	def calcRating(self, song):
		import rating
		song.rating = rating.getRating(song.url, default=0.0)
		return song.rating

	def calcScore(self, song):
		scores = []
		scores += [self.calcRating(song) * random.gauss(1, 0.5)]
		scores += [self.calcContextMatchScore(song) * random.gauss(1, 0.5)]
		return sum(scores) + random.gauss(1, 0.5)

	def getNextSong(self):
		filenames = takeN(self.generator, self.checkNextNForBest)
		songs = map(Song, filenames)
		scores = map(lambda song: (self.calcScore(song), song), songs)
		best = max(scores)
		song = best[1]
		return song

class MainQueue:
	def __init__(self):
		self.lock = Lock()
		self.manualQueue = deque()
		self.infiniteQueue = InfQueue()

	def getNextSong(self):
		with self.lock:
			if self.manualQueue:
				return self.manualQueue.popleft()
			return self.infiniteQueue.getNextSong()

	def fillUpTo(self, n=10):
		with self.lock:
			while len(self.manualQueue) < n:
				self.manualQueue.append(self.infiniteQueue.getNextSong())

queue = MainQueue()

def getNextSong():
	return queue.getNextSong()

def queueMain():
	for ev,args,kwargs in state.updates.read():
		if ev is PlayerEventCallbacks.onSongChange:
			queue.fillUpTo()
<commit_msg>move SongStore loading. if it fails, it wont break the whole queue<commit_after>
from Song import Song
from State import state
from player import PlayerEventCallbacks
from utils import *
import math, random
import appinfo

class RandomFileQueueGen:
	randomQuality = 0.5

	def __init__(self, dir):
		import os
		from RandomFileQueue import RandomFileQueue
		self.fileQueue = RandomFileQueue(
			rootdir = os.path.expanduser(dir),
			fileexts = appinfo.formats)

	def next(self):
		return self.fileQueue.getNextFile()

	def __iter__(self):
		while True:
			yield self.next()

class RandomFromSongDatabaseGen:
	randomQuality = 0.0

	def __init__(self):
		def loadDatabase():
			from SongStore import SongStore
			self.database = SongStore(appinfo.userdir)

			import utils

			self.database.open()

			for dir in appinfo.musicdirs:
				self.database.addMany(utils.getSongsFromDirectory(dir))

			self.database.close()

			self.randomQuality = 0.5
			print "Done loading songs"

		from threading import Thread
		loadDatabaseThread = Thread(target=loadDatabase, name="loadDatabase")
		loadDatabaseThread.start()

	def next(self):
		try:
			oldSong = state.recentlyPlayedList.getLastN(1)[0]
		except:
			oldSong = None

		self.database.open()
		songs = self.database.getRandomSongs(oldSong=oldSong, limit=1)
		self.database.close()

		return next(iter(songs))

	def __iter__(self):
		while True:
			yield self.next()

class RandomSongs:
	randomQuality = 0.5
	def __init__(self, generators):
		self.generators = [gen() for gen in generators]
	def next(self):
		while True:
			generators = list(self.generators)
			if not generators:
				raise StopIteration
			qualitySum = sum([gen.randomQuality for gen in generators])
			self.randomQuality = qualitySum / len(generators)
			r = random.random() * qualitySum
			i = 0
			gen = generators[i]
			while i < len(generators)-1 and r > gen.randomQuality:
				r -= gen.randomQuality
				i += 1
				gen = generators[i]
			try:
				return next(gen)
			except StopIteration:
				#print "warning: generator", gen, "raised StopIteration"
				#sys.excepthook(*sys.exc_info())
				pass
			generators.pop(i)
	def __iter__(self):
		while True:
			yield self.next()

from collections import deque
from threading import Lock

class InfQueue:
	def __init__(self):
		self.generator = RandomSongs([
			RandomFromSongDatabaseGen,
			lambda: RandomSongs([
				(lambda: RandomFileQueueGen(dir)) for dir in appinfo.musicdirs])
		])
		self.checkNextNForBest = 10
		self.checkLastNForContext = 10

	def calcContextMatchScore(self, song):
		count = 0
		lastSongs = state.recentlyPlayedList.getLastN(self.checkLastNForContext)
		lastSongs = filter(lambda s: not s.skipped, lastSongs)
		if not lastSongs: return 0.0
		for lastSong in lastSongs:
			count += max(intersectFuzzySets(song.tags, lastSong.tags).values() + [0])
		s = float(count) / self.checkLastNForContext
		# We likely get small values here. Boost a bit but keep in [0,1] range. sqrt is a good fit.
		return math.sqrt(s)

	def calcRating(self, song):
		import rating
		song.rating = rating.getRating(song.url, default=0.0)
		return song.rating

	def calcScore(self, song):
		scores = []
		scores += [self.calcRating(song) * random.gauss(1, 0.5)]
		scores += [self.calcContextMatchScore(song) * random.gauss(1, 0.5)]
		return sum(scores) + random.gauss(1, 0.5)

	def getNextSong(self):
		filenames = takeN(self.generator, self.checkNextNForBest)
		songs = map(Song, filenames)
		scores = map(lambda song: (self.calcScore(song), song), songs)
		best = max(scores)
		song = best[1]
		return song

class MainQueue:
	def __init__(self):
		self.lock = Lock()
		self.manualQueue = deque()
		self.infiniteQueue = InfQueue()

	def getNextSong(self):
		with self.lock:
			if self.manualQueue:
				return self.manualQueue.popleft()
			return self.infiniteQueue.getNextSong()

	def fillUpTo(self, n=10):
		with self.lock:
			while len(self.manualQueue) < n:
				self.manualQueue.append(self.infiniteQueue.getNextSong())

queue = MainQueue()

def getNextSong():
	return queue.getNextSong()

def queueMain():
	for ev,args,kwargs in state.updates.read():
		if ev is PlayerEventCallbacks.onSongChange:
			queue.fillUpTo()
<|endoftext|>


<commit_before>import urllib2
import urlparse
from xml.etree import ElementTree
from django.http import HttpResponse
from tastypie.authentication import Authentication
from tastypie.authorization import Authorization
from tastypie.resources import Resource
from tastypie.serializers import Serializer
from tastypie.throttle import CacheThrottle
from corehq.apps.grapevine.forms import GrapevineBackendForm
from corehq.apps.sms.util import clean_phone_number
from corehq.apps.sms.mixin import SMSBackend
from couchdbkit.ext.django.schema import *
from xml.sax.saxutils import escape, unescape
from django.conf import settings
from corehq.apps.sms.api import incoming as incoming_sms
import logging

logger = logging.getLogger(__name__)

TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
    <gviSmsMessage>
        <affiliateCode>{affiliate_code}</affiliateCode>
        <authenticationCode>{auth_code}</authenticationCode>
        <messageType>text</messageType>
        <recipientList>
            <message>{message}</message>
            <recipient>
                <msisdn>{msisdn}</msisdn>
            </recipient>
        </recipientList>
    </gviSmsMessage>"""

class GrapevineBackend(SMSBackend):
    affiliate_code = StringProperty()
    authentication_code = StringProperty()

    @classmethod
    def get_opt_in_keywords(cls):
        return ["START"]

    @classmethod
    def get_opt_out_keywords(cls):
        return ["STOP", "END", "CANCEL", "UNSUBSCRIBE", "QUIT"]

    @classmethod
    def get_api_id(cls):
        return "GVI"

    @classmethod
    def get_generic_name(cls):
        return "Grapevine"

    @classmethod
    def get_template(cls):
        return "grapevine/backend.html"

    @classmethod
    def get_form_class(cls):
        return GrapevineBackendForm

    def send(self, msg, delay=True, *args, **kwargs):
        phone_number = clean_phone_number(msg.phone_number)
        text = msg.text.encode("utf-8")

        data = TEMPLATE.format(
            affiliate_code=escape(self.affiliate_code),
            auth_code=escape(self.authentication_code),
            message=escape(text),
            msisdn=escape(phone_number)
        )

        url = "http://www.gvi.bms9.vine.co.za/httpInputhandler/ApplinkUpload"
        req = urllib2.Request(url, data)
        response = urllib2.urlopen(req, timeout=settings.SMS_GATEWAY_TIMEOUT)
        resp = response.read()

class SmsMessage(object):
    phonenumber = ''
    text = ''

    def __init__(self, phonenumber=None, text=None):
        self.phonenumber = phonenumber
        self.text = unescape(text) if text else ''

    @property
    def is_complete(self):
        return bool(self.phonenumber)

class UrlencodedDeserializer(Serializer):
    formats = ['json', 'jsonp', 'xml', 'yaml', 'html', 'plist', 'urlencode']
    content_types = {
        'json': 'application/json',
        'jsonp': 'text/javascript',
        'xml': 'application/xml',
        'yaml': 'text/yaml',
        'html': 'text/html',
        'plist': 'application/x-plist',
        'urlencode': 'application/x-www-form-urlencoded',
    }

    def from_urlencode(self, data,options=None):
        """ handles basic form encoded url posts """
        qs = dict((k, v if len(v)>1 else v[0] )
            for k, v in urlparse.parse_qs(data).iteritems())

        return qs

    def to_urlencode(self,content):
        pass

class SimpleApiAuthentication(Authentication):
    def is_authenticated(self, request, **kwargs):
        user = self.get_identifier(request)
        key = request.GET.get('apikey')

        expected_key = getattr(settings, 'SIMPLE_API_KEYS', {}).get(user)
        if not expected_key:
            logger.warning("No apikey defined for user '%s'" % user)
        return expected_key and key == expected_key

    def get_identifier(self, request):
        return request.GET.get('apiuser', 'nouser')

class GrapevineResource(Resource):
    """
    Handles grapevine messaging requests

    * incoming SMS
        <gviSms>
            <smsDateTime>2013-10-29T12:55:58</smsDateTime>
            <gatewayIdentifier>vodacomPremMGA2Rx1</gatewayIdentifier>
            <cellNumber>27827891099</cellNumber>
            <smsLocation>30665</smsLocation>
            <content>Another test</content>
        </gviSms>

    * Replies to SMS
        <gviSmsResponse xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
            <responseDateTime>2013-10-29T13:19:07</responseDateTime>
            <recipient>
                <msisdn>27827891099</msisdn>
            </recipient>
            <responseType>reply</responseType>
            <response>Test reply</response>
        </gviSmsResponse>

    * SMS Status reports (not currently implemented)
        <gviSmsResponse xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
            <responseDateTime>2013-10-29T13:15:52</responseDateTime>
            <submitDateTime>2013-10-29T13:15:49</submitDateTime>
            <recipient>
                <msisdn>27827891099</msisdn>
            </recipient>
            <responseType>receipt</responseType>
            <status>
                0
                <reason>Message is delivered to destination. stat:DELIVRD</reason>
            </status>
        </gviSmsResponse>

    smsDateTime: The date and time when the original SMS arrived at GVI's SMS gateway.
    gatewayIdentifier: Identifies the network and the rate of the SMS.
    cellNumber: The number (in international MSISDN format) of the mobile phone that sent the SMS.
    smsLocation: The short code to which the SMS was sent.
    content: The message text of the SMS message.
    """
    class Meta:
        resource_name = 'sms'
        object_class = SmsMessage
        authorization = Authorization()
        allowed_methods = ['post']
        serializer = UrlencodedDeserializer()
        throttle = CacheThrottle(throttle_at=600, timeframe=60, expiration=86400)
        authentication = SimpleApiAuthentication()

    def detail_uri_kwargs(self, bundle_or_obj):
        return {}

    def full_hydrate(self, bundle):
        if not bundle.data or not bundle.data.get('XML'):
            return bundle

        root = ElementTree.fromstring(bundle.data['XML'])
        if root.tag == 'gviSms':
            date_string = root.find('smsDateTime').text
            phone_number = root.find('cellNumber').text
            content_text = root.find('content').text
            bundle.obj = SmsMessage(phone_number, content_text)

        elif root.tag == 'gviSmsResponse':
            date_string = root.find('responseDateTime').text
            phone_number = root.find('recipient/msisdn').text
            resp_type = root.find('responseType').text  # receipt, reply or error

            if resp_type == 'reply':
                response_text = root.find('response').text
                bundle.obj = SmsMessage(phone_number, response_text)

        return bundle

    def obj_create(self, bundle, request=None, **kwargs):
        bundle = self.full_hydrate(bundle)

        if bundle.obj.is_complete:
            incoming_sms(bundle.obj.phonenumber, bundle.obj.text, GrapevineBackend.get_api_id())

        return bundle

    def post_list(self, request, **kwargs):
        super(GrapevineResource, self).post_list(request, **kwargs)
        # respond with 200 OK instead of 201 CREATED
        return HttpResponse()

<commit_msg>Fix inbound grapevine issue<commit_after>import urllib2
import urlparse
from xml.etree import ElementTree
from django.http import HttpResponse
from tastypie.authentication import Authentication
from tastypie.authorization import Authorization
from tastypie.resources import Resource
from tastypie.serializers import Serializer
from tastypie.throttle import CacheThrottle
from corehq.apps.grapevine.forms import GrapevineBackendForm
from corehq.apps.sms.util import clean_phone_number
from corehq.apps.sms.mixin import SMSBackend
from couchdbkit.ext.django.schema import *
from xml.sax.saxutils import escape, unescape
from django.conf import settings
from corehq.apps.sms.api import incoming as incoming_sms
import logging

logger = logging.getLogger(__name__)

TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
    <gviSmsMessage>
        <affiliateCode>{affiliate_code}</affiliateCode>
        <authenticationCode>{auth_code}</authenticationCode>
        <messageType>text</messageType>
        <recipientList>
            <message>{message}</message>
            <recipient>
                <msisdn>{msisdn}</msisdn>
            </recipient>
        </recipientList>
    </gviSmsMessage>"""

class GrapevineBackend(SMSBackend):
    affiliate_code = StringProperty()
    authentication_code = StringProperty()

    @classmethod
    def get_opt_in_keywords(cls):
        return ["START"]

    @classmethod
    def get_opt_out_keywords(cls):
        return ["STOP", "END", "CANCEL", "UNSUBSCRIBE", "QUIT"]

    @classmethod
    def get_api_id(cls):
        return "GVI"

    @classmethod
    def get_generic_name(cls):
        return "Grapevine"

    @classmethod
    def get_template(cls):
        return "grapevine/backend.html"

    @classmethod
    def get_form_class(cls):
        return GrapevineBackendForm

    def send(self, msg, delay=True, *args, **kwargs):
        phone_number = clean_phone_number(msg.phone_number)
        text = msg.text.encode("utf-8")

        data = TEMPLATE.format(
            affiliate_code=escape(self.affiliate_code),
            auth_code=escape(self.authentication_code),
            message=escape(text),
            msisdn=escape(phone_number)
        )

        url = "http://www.gvi.bms9.vine.co.za/httpInputhandler/ApplinkUpload"
        req = urllib2.Request(url, data)
        response = urllib2.urlopen(req, timeout=settings.SMS_GATEWAY_TIMEOUT)
        resp = response.read()

class SmsMessage(object):
    phonenumber = ''
    text = ''

    def __init__(self, phonenumber=None, text=None):
        self.phonenumber = phonenumber
        self.text = unescape(text) if text else ''

    @property
    def is_complete(self):
        return bool(self.phonenumber)

class UrlencodedDeserializer(Serializer):
    formats = ['json', 'jsonp', 'xml', 'yaml', 'html', 'plist', 'urlencode']
    content_types = {
        'json': 'application/json',
        'jsonp': 'text/javascript',
        'xml': 'application/xml',
        'yaml': 'text/yaml',
        'html': 'text/html',
        'plist': 'application/x-plist',
        'urlencode': 'application/x-www-form-urlencoded',
    }

    def from_urlencode(self, data,options=None):
        """ handles basic form encoded url posts """
        qs = dict((k, v if len(v)>1 else v[0] )
            for k, v in urlparse.parse_qs(data).iteritems())

        return qs

    def to_urlencode(self,content):
        pass

class SimpleApiAuthentication(Authentication):
    def is_authenticated(self, request, **kwargs):
        user = self.get_identifier(request)
        key = request.GET.get('apikey')

        expected_key = getattr(settings, 'SIMPLE_API_KEYS', {}).get(user)
        if not expected_key:
            logger.warning("No apikey defined for user '%s'" % user)
        return expected_key and key == expected_key

    def get_identifier(self, request):
        return request.GET.get('apiuser', 'nouser')

class GrapevineResource(Resource):
    """
    Handles grapevine messaging requests

    * incoming SMS
        <gviSms>
            <smsDateTime>2013-10-29T12:55:58</smsDateTime>
            <gatewayIdentifier>vodacomPremMGA2Rx1</gatewayIdentifier>
            <cellNumber>27827891099</cellNumber>
            <smsLocation>30665</smsLocation>
            <content>Another test</content>
        </gviSms>

    * Replies to SMS
        <gviSmsResponse xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
            <responseDateTime>2013-10-29T13:19:07</responseDateTime>
            <recipient>
                <msisdn>27827891099</msisdn>
            </recipient>
            <responseType>reply</responseType>
            <response>Test reply</response>
        </gviSmsResponse>

    * SMS Status reports (not currently implemented)
        <gviSmsResponse xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
            <responseDateTime>2013-10-29T13:15:52</responseDateTime>
            <submitDateTime>2013-10-29T13:15:49</submitDateTime>
            <recipient>
                <msisdn>27827891099</msisdn>
            </recipient>
            <responseType>receipt</responseType>
            <status>
                0
                <reason>Message is delivered to destination. stat:DELIVRD</reason>
            </status>
        </gviSmsResponse>

    smsDateTime: The date and time when the original SMS arrived at GVI's SMS gateway.
    gatewayIdentifier: Identifies the network and the rate of the SMS.
    cellNumber: The number (in international MSISDN format) of the mobile phone that sent the SMS.
    smsLocation: The short code to which the SMS was sent.
    content: The message text of the SMS message.
    """
    class Meta:
        resource_name = 'sms'
        object_class = SmsMessage
        authorization = Authorization()
        allowed_methods = ['post']
        serializer = UrlencodedDeserializer()
        throttle = CacheThrottle(throttle_at=600, timeframe=60, expiration=86400)
        authentication = SimpleApiAuthentication()

    def detail_uri_kwargs(self, bundle_or_obj):
        return {}

    def full_hydrate(self, bundle):
        if not bundle.data or not bundle.data.get('XML'):
            return bundle

        # http://bugs.python.org/issue11033
        xml = bundle.data['XML'].encode('utf-8')

        root = ElementTree.fromstring(xml)
        if root.tag == 'gviSms':
            date_string = root.find('smsDateTime').text
            phone_number = root.find('cellNumber').text
            content_text = root.find('content').text
            bundle.obj = SmsMessage(phone_number, content_text)

        elif root.tag == 'gviSmsResponse':
            date_string = root.find('responseDateTime').text
            phone_number = root.find('recipient/msisdn').text
            resp_type = root.find('responseType').text  # receipt, reply or error

            if resp_type == 'reply':
                response_text = root.find('response').text
                bundle.obj = SmsMessage(phone_number, response_text)

        return bundle

    def obj_create(self, bundle, request=None, **kwargs):
        bundle = self.full_hydrate(bundle)

        if bundle.obj.is_complete:
            incoming_sms(bundle.obj.phonenumber, bundle.obj.text, GrapevineBackend.get_api_id())

        return bundle

    def post_list(self, request, **kwargs):
        super(GrapevineResource, self).post_list(request, **kwargs)
        # respond with 200 OK instead of 201 CREATED
        return HttpResponse()

<|endoftext|>


<commit_before>"""Tests API Serializers."""
import mock
from django.core import mail
from django.test import RequestFactory, TestCase
from opaque_keys.edx.keys import CourseKey
from rest_framework.exceptions import ValidationError

from course_discovery.apps.core.tests.factories import UserFactory
from course_discovery.apps.core.tests.helpers import make_image_file
from course_discovery.apps.course_metadata.tests import toggle_switch
from course_discovery.apps.course_metadata.tests.factories import OrganizationFactory, PersonFactory
from course_discovery.apps.ietf_language_tags.models import LanguageTag
from course_discovery.apps.publisher.api.serializers import (CourseRevisionSerializer, CourseRunSerializer,
                                                             CourseRunStateSerializer, CourseStateSerializer,
                                                             CourseUserRoleSerializer, GroupUserSerializer)
from course_discovery.apps.publisher.choices import CourseRunStateChoices, CourseStateChoices, PublisherUserRole
from course_discovery.apps.publisher.models import CourseRun, CourseState, Seat
from course_discovery.apps.publisher.tests.factories import (CourseFactory, CourseRunFactory, CourseRunStateFactory,
                                                             CourseStateFactory, CourseUserRoleFactory,
                                                             OrganizationExtensionFactory, SeatFactory)

class CourseUserRoleSerializerTests(TestCase):
    serializer_class = CourseUserRoleSerializer

    def setUp(self):
        super(CourseUserRoleSerializerTests, self).setUp()
        self.request = RequestFactory()
        self.course_user_role = CourseUserRoleFactory(role=PublisherUserRole.MarketingReviewer)
        self.request.user = self.course_user_role.user

    def get_expected_data(self):
        return {
            'course': self.course_user_role.course.id,
            'user': self.course_user_role.user.id,
            'role': self.course_user_role.role,
            'changed_by': self.course_user_role.user
        }

    def test_validation(self):
        serializer = self.serializer_class(self.course_user_role, context={'request': self.request})
        validated_data = serializer.validate(serializer.data)
        self.assertEqual(validated_data, self.get_expected_data())

    def test_update(self):
        """
        Test that user role assignment changed on update.
        """
        new_user = UserFactory()
        serializer = self.serializer_class(self.course_user_role, context={'request': self.request})
        course_role = serializer.update(self.course_user_role, {'user': new_user})
        self.assertEqual(course_role.user, new_user)
        self.assertEqual(len(mail.outbox), 1)

    def test_update_with_error(self):
        """
        Test that whole transaction roll backed if error in email sending.
        """
        new_user = UserFactory()
        serializer = self.serializer_class(self.course_user_role, context={'request': self.request})
        with mock.patch('django.core.mail.message.EmailMessage.send', side_effect=TypeError):
            with self.assertRaises(Exception):
                course_role = serializer.update(self.course_user_role, {'user': new_user})
                self.assertNotEqual(course_role.user, new_user)

class GroupUserSerializerTests(TestCase):

    def test_data_with_full_name(self):
        """ Verify that UserSerializer serialize the user object. """

        user = UserFactory(full_name='Test User')
        serializer = GroupUserSerializer(user)

        expected = {'id': user.id, 'full_name': user.full_name}
        self.assertDictEqual(serializer.data, expected)

    def test_data_without_full_name(self):
        """ Verify that UserSerializer serialize the user object. """

        user = UserFactory(full_name='', first_name='', last_name='')
        serializer = GroupUserSerializer(user)

        expected = {'id': user.id, 'full_name': user.username}
        self.assertDictEqual(serializer.data, expected)

class CourseRunSerializerTests(TestCase):
    serializer_class = CourseRunSerializer

    def setUp(self):
        super(CourseRunSerializerTests, self).setUp()
        self.course_run = CourseRunFactory()
        self.course_run.lms_course_id = 'course-v1:edX+DemoX+Demo_Course'
        self.request = RequestFactory()
        self.user = UserFactory()
        self.request.user = self.user
        self.course_state = CourseRunStateFactory(course_run=self.course_run, owner_role=PublisherUserRole.Publisher)

    def get_expected_data(self):
        return {
            'lms_course_id': self.course_run.lms_course_id,
            'changed_by': self.user,
            'preview_url': self.course_run.preview_url
        }

    def test_validate_lms_course_id(self):
        """ Verify that serializer raises error if 'lms_course_id' has invalid format. """
        self.course_run.lms_course_id = 'invalid-course-id'
        self.course_run.save()  # pylint: disable=no-member
        serializer = self.serializer_class(self.course_run)
        with self.assertRaises(ValidationError):
            serializer.validate_lms_course_id(self.course_run.lms_course_id)

    def test_validate_preview_url(self):
        """ Verify that serializer raises error if 'preview_url' has invalid format. """
        self.course_run.preview_url = 'invalid-preview-url'
        self.course_run.save()  # pylint: disable=no-member
        serializer = self.serializer_class(self.course_run)
        with self.assertRaises(ValidationError):
            serializer.validate_preview_url(self.course_run.preview_url)

    def test_serializer_with_valid_data(self):
        """ Verify that serializer validate course_run object. """
        serializer = self.serializer_class(self.course_run, context={'request': self.request})
        self.assertEqual(self.get_expected_data(), serializer.validate(serializer.data))

    def test_update_preview_url(self):
        """ Verify that course 'owner_role' will be changed to course_team after updating
        course run with preview url.
        """
        self.course_run.preview_url = ''
        self.course_run.save()
        serializer = self.serializer_class(self.course_run)
        serializer.update(self.course_run, {'preview_url': 'https://example.com/abc/course'})

        self.assertEqual(self.course_state.owner_role, PublisherUserRole.CourseTeam)
        self.assertEqual(self.course_run.preview_url, serializer.data['preview_url'])

    def test_update_lms_course_id(self):
        """ Verify that 'changed_by' also updated after updating course_run's lms_course_id."""
        self.course_run.preview_url = None
        self.course_run.save()

        serializer = self.serializer_class(self.course_run, context={'request': self.request})
        serializer.update(self.course_run, serializer.validate(serializer.data))

        self.assertEqual(self.course_run.lms_course_id, serializer.data['lms_course_id'])
        self.assertEqual(self.course_run.changed_by, self.user)

    def test_update_with_transaction_rollback(self):
        """
        Verify that transaction roll backed if an error occurred.
        """
        self.course_run.preview_url = ''
        self.course_run.save()
        serializer = self.serializer_class(self.course_run)

        with self.assertRaises(Exception):
            serializer.update(self.course_run, {'preview_url': 'invalid_url'})
            self.assertFalse(self.course_run.preview_url)

    def test_transaction_roll_back_with_error_on_email(self):
        """
        Verify that transaction is roll backed if error occurred during email sending.
        """
        toggle_switch('enable_publisher_email_notifications', True)
        self.course_run.preview_url = ''
        self.course_run.save()
        serializer = self.serializer_class(self.course_run)
        self.assertEqual(self.course_run.course_run_state.owner_role, PublisherUserRole.Publisher)

        with self.assertRaises(Exception):
            serializer.update(self.course_run, {'preview_url': 'https://example.com/abc/course'})

        self.course_run = CourseRun.objects.get(id=self.course_run.id)
        self.assertFalse(self.course_run.preview_url)
        # Verify that owner role not changed.
        self.assertEqual(self.course_run.course_run_state.owner_role, PublisherUserRole.Publisher)

class CourseRevisionSerializerTests(TestCase):
    def test_course_revision_serializer(self):
        """ Verify that CourseRevisionSerializer serialize the course revision object. """

        course = CourseFactory()
        course.title = 'updated title'
        course.save()
        revision = course.history.first()
        serializer = CourseRevisionSerializer(revision)

        expected = {
            'history_id': revision.history_id,
            'title': revision.title,
            'number': revision.number,
            'short_description': revision.short_description,
            'full_description': revision.full_description,
            'expected_learnings': revision.expected_learnings,
            'prerequisites': revision.prerequisites,
            'primary_subject': revision.primary_subject.name,
            'secondary_subject': revision.secondary_subject.name,
            'tertiary_subject': revision.tertiary_subject.name,
            'level_type': revision.level_type.name
        }

        self.assertDictEqual(serializer.data, expected)

    def test_course_revision_serializer_without_subjects(self):
        """ Verify that CourseRevisionSerializer serialize the course revision object
        even if subject fields are not available.
        """

        course = CourseFactory(primary_subject=None, secondary_subject=None, tertiary_subject=None, level_type=None)
        course.title = 'updated title'
        course.save()
        revision = course.history.first()
        serializer = CourseRevisionSerializer(revision)

        expected = {
            'history_id': revision.history_id,
            'title': revision.title,
            'number': revision.number,
            'short_description': revision.short_description,
            'full_description': revision.full_description,
            'expected_learnings': revision.expected_learnings,
            'prerequisites': revision.prerequisites,
            'primary_subject': None,
            'secondary_subject': None,
            'tertiary_subject': None,
            'level_type': None
        }

        self.assertDictEqual(serializer.data, expected)

class CourseStateSerializerTests(TestCase):
    serializer_class = CourseStateSerializer

    def setUp(self):
        super(CourseStateSerializerTests, self).setUp()
        self.course_state = CourseStateFactory(name=CourseStateChoices.Draft)
        self.request = RequestFactory()
        self.user = UserFactory()
        self.request.user = self.user

        CourseUserRoleFactory(
            course=self.course_state.course, role=PublisherUserRole.CourseTeam, user=self.user
        )

    def test_update(self):
        """
        Verify that we can update course workflow state with serializer.
        """
        course = self.course_state.course
        course.image = make_image_file('test_banner.jpg')
        course.save()
        course.organizations.add(OrganizationExtensionFactory().organization)

        self.assertNotEqual(self.course_state, CourseStateChoices.Review)
        serializer = self.serializer_class(self.course_state, context={'request': self.request})
        data = {'name': CourseStateChoices.Review}
        serializer.update(self.course_state, data)

        self.course_state = CourseState.objects.get(course=self.course_state.course)
        self.assertEqual(self.course_state.name, CourseStateChoices.Review)
        self.assertEqual(self.course_state.owner_role, PublisherUserRole.MarketingReviewer)

    def test_update_with_error(self):
        """
        Verify that serializer raises `ValidationError` with wrong transition.
        """
        serializer = self.serializer_class(self.course_state, context={'request': self.request})
        data = {'name': CourseStateChoices.Approved}

        with self.assertRaises(ValidationError):
            serializer.update(self.course_state, data)

class CourseRunStateSerializerTests(TestCase):
    serializer_class = CourseRunStateSerializer

    def setUp(self):
        super(CourseRunStateSerializerTests, self).setUp()
        self.run_state = CourseRunStateFactory(name=CourseRunStateChoices.Draft)
        self.course_run = self.run_state.course_run
        self.request = RequestFactory()
        self.user = UserFactory()
        self.request.user = self.user
        CourseStateFactory(name=CourseStateChoices.Approved, course=self.course_run.course)

        SeatFactory(course_run=self.course_run, type=Seat.AUDIT)
        language_tag = LanguageTag(code='te-st', name='Test Language')
        language_tag.save()
        self.course_run.transcript_languages.add(language_tag)
        self.course_run.language = language_tag
        organization = OrganizationFactory()
        self.course_run.course.organizations.add(organization)
        self.course_run.save()
        self.course_run.staff.add(PersonFactory())

        toggle_switch('enable_publisher_email_notifications', True)
        CourseUserRoleFactory(
            course=self.course_run.course, role=PublisherUserRole.CourseTeam, user=self.user
        )
        CourseUserRoleFactory(
            course=self.course_run.course, role=PublisherUserRole.ProjectCoordinator, user=UserFactory()
        )

    def test_update(self):
        """
        Verify that we can update course-run workflow state name and preview_accepted with serializer.
        """
        CourseUserRoleFactory(
            course=self.course_run.course, role=PublisherUserRole.Publisher, user=UserFactory()
        )

        self.assertNotEqual(self.run_state, CourseRunStateChoices.Review)

        self.course_run.lms_course_id = 'course-v1:edX+DemoX+Demo_Course'
        serializer = self.serializer_class(self.run_state, context={'request': self.request})
        data = {'name': CourseRunStateChoices.Review}
        serializer.update(self.run_state, data)

        self.assertEqual(self.run_state.name, CourseRunStateChoices.Review)

        self.assertFalse(self.run_state.preview_accepted)
        serializer.update(self.run_state, {'preview_accepted': True})

        self.assertTrue(CourseRun.objects.get(id=self.course_run.id).preview_url)
        self.assertEqual(len(mail.outbox), 1)
        course_key = CourseKey.from_string(self.course_run.lms_course_id)
        subject = 'Publication requested: {course_name} {run_name}'.format(
            course_name=self.course_run.course.title,
            run_name=course_key.run
        )
        self.assertIn(subject, str(mail.outbox[0].subject))

    def test_update_with_error(self):
        """
        Verify that serializer raises `ValidationError` with wrong transition.
        """
        serializer = self.serializer_class(self.run_state, context={'request': self.request})
        data = {'name': CourseRunStateChoices.Published}

        with self.assertRaises(ValidationError):
            serializer.update(self.run_state, data)
            self.assertEqual(len(mail.outbox), 1)

    def test_update_with_transaction_roll_back(self):
        """
        Verify that transaction roll back all db changes.
        """
        self.assertNotEqual(self.run_state, CourseRunStateChoices.Review)
        serializer = self.serializer_class(self.run_state, context={'request': self.request})
        data = {'name': CourseRunStateChoices.Review}
        with self.assertRaises(Exception):
            serializer.update(self.run_state, data)
            self.assertEqual(self.run_state.name, CourseRunStateChoices.Review)

            self.assertFalse(self.run_state.preview_accepted)
            serializer.update(self.run_state, {'preview_accepted': True})

            self.assertFalse(CourseRun.objects.get(id=self.course_run.id).preview_url)
            self.assertEqual(len(mail.outbox), 0)
<commit_msg>docstrings/comments added in /course_discovery/apps/publisher/api/tests/test_serializers.py<commit_after>"""Tests API Serializers."""
import mock
from django.core import mail
from django.test import RequestFactory, TestCase
from opaque_keys.edx.keys import CourseKey
from rest_framework.exceptions import ValidationError

from course_discovery.apps.core.tests.factories import UserFactory
from course_discovery.apps.core.tests.helpers import make_image_file
from course_discovery.apps.course_metadata.tests import toggle_switch
from course_discovery.apps.course_metadata.tests.factories import OrganizationFactory, PersonFactory
from course_discovery.apps.ietf_language_tags.models import LanguageTag
from course_discovery.apps.publisher.api.serializers import (CourseRevisionSerializer, CourseRunSerializer,
                                                             CourseRunStateSerializer, CourseStateSerializer,
                                                             CourseUserRoleSerializer, GroupUserSerializer)
from course_discovery.apps.publisher.choices import CourseRunStateChoices, CourseStateChoices, PublisherUserRole
from course_discovery.apps.publisher.models import CourseRun, CourseState, Seat
from course_discovery.apps.publisher.tests.factories import (CourseFactory, CourseRunFactory, CourseRunStateFactory,
                                                             CourseStateFactory, CourseUserRoleFactory,
                                                             OrganizationExtensionFactory, SeatFactory)

class CourseUserRoleSerializerTests(TestCase):
    serializer_class = CourseUserRoleSerializer

    def setUp(self):
        super(CourseUserRoleSerializerTests, self).setUp()
        self.request = RequestFactory()
        self.course_user_role = CourseUserRoleFactory(role=PublisherUserRole.MarketingReviewer)
        self.request.user = self.course_user_role.user

    def get_expected_data(self):
        """ Helper method which will return expected serialize data. """
        return {
            'course': self.course_user_role.course.id,
            'user': self.course_user_role.user.id,
            'role': self.course_user_role.role,
            'changed_by': self.course_user_role.user
        }

    def test_validation(self):
        """ Verify that serializer validate data. """

        # we are passing request to context because we need 'changed_by' user in validated values.
        serializer = self.serializer_class(self.course_user_role, context={'request': self.request})
        validated_data = serializer.validate(serializer.data)
        self.assertEqual(validated_data, self.get_expected_data())

    def test_update(self):
        """
        Test that user role assignment changed on update.
        """
        new_user = UserFactory()
        serializer = self.serializer_class(self.course_user_role, context={'request': self.request})
        course_role = serializer.update(self.course_user_role, {'user': new_user})
        self.assertEqual(course_role.user, new_user)
        self.assertEqual(len(mail.outbox), 1)

    def test_update_with_error(self):
        """
        Test that whole transaction roll backed if error in email sending.
        """
        new_user = UserFactory()
        serializer = self.serializer_class(self.course_user_role, context={'request': self.request})
        with mock.patch('django.core.mail.message.EmailMessage.send', side_effect=TypeError):
            with self.assertRaises(Exception):
                course_role = serializer.update(self.course_user_role, {'user': new_user})
                self.assertNotEqual(course_role.user, new_user)

class GroupUserSerializerTests(TestCase):

    def test_data_with_full_name(self):
        """ Verify that UserSerializer serialize the user object. """

        user = UserFactory(full_name='Test User')
        serializer = GroupUserSerializer(user)

        expected = {'id': user.id, 'full_name': user.full_name}
        self.assertDictEqual(serializer.data, expected)

    def test_data_without_full_name(self):
        """ Verify that UserSerializer serialize the user object using username
        if full_name is not available.
        """

        user = UserFactory(full_name='', first_name='', last_name='')
        serializer = GroupUserSerializer(user)

        expected = {'id': user.id, 'full_name': user.username}
        self.assertDictEqual(serializer.data, expected)

class CourseRunSerializerTests(TestCase):
    serializer_class = CourseRunSerializer

    def setUp(self):
        super(CourseRunSerializerTests, self).setUp()
        self.course_run = CourseRunFactory()
        self.course_run.lms_course_id = 'course-v1:edX+DemoX+Demo_Course'
        self.request = RequestFactory()
        self.user = UserFactory()
        self.request.user = self.user
        self.course_state = CourseRunStateFactory(course_run=self.course_run, owner_role=PublisherUserRole.Publisher)

    def get_expected_data(self):
        """ Helper method which will return expected serialize data. """
        return {
            'lms_course_id': self.course_run.lms_course_id,
            'changed_by': self.user,
            'preview_url': self.course_run.preview_url
        }

    def test_validate_lms_course_id(self):
        """ Verify that serializer raises error if 'lms_course_id' has invalid format. """
        self.course_run.lms_course_id = 'invalid-course-id'
        self.course_run.save()  # pylint: disable=no-member
        serializer = self.serializer_class(self.course_run)
        with self.assertRaises(ValidationError):
            serializer.validate_lms_course_id(self.course_run.lms_course_id)

    def test_validate_preview_url(self):
        """ Verify that serializer raises error if 'preview_url' has invalid format. """
        self.course_run.preview_url = 'invalid-preview-url'
        self.course_run.save()  # pylint: disable=no-member
        serializer = self.serializer_class(self.course_run)
        with self.assertRaises(ValidationError):
            serializer.validate_preview_url(self.course_run.preview_url)

    def test_serializer_with_valid_data(self):
        """ Verify that serializer validate course_run object. """
        serializer = self.serializer_class(self.course_run, context={'request': self.request})
        self.assertEqual(self.get_expected_data(), serializer.validate(serializer.data))

    def test_update_preview_url(self):
        """ Verify that course 'owner_role' will be changed to course_team after updating
        course run with preview url.
        """
        self.course_run.preview_url = ''
        self.course_run.save()
        serializer = self.serializer_class(self.course_run)
        serializer.update(self.course_run, {'preview_url': 'https://example.com/abc/course'})

        self.assertEqual(self.course_state.owner_role, PublisherUserRole.CourseTeam)
        self.assertEqual(self.course_run.preview_url, serializer.data['preview_url'])

    def test_update_lms_course_id(self):
        """ Verify that 'changed_by' also updated after updating course_run's lms_course_id."""
        self.course_run.preview_url = None
        self.course_run.save()

        serializer = self.serializer_class(self.course_run, context={'request': self.request})
        serializer.update(self.course_run, serializer.validate(serializer.data))

        self.assertEqual(self.course_run.lms_course_id, serializer.data['lms_course_id'])
        self.assertEqual(self.course_run.changed_by, self.user)

    def test_update_with_transaction_rollback(self):
        """
        Verify that transaction roll backed if an error occurred.
        """
        self.course_run.preview_url = ''
        self.course_run.save()
        serializer = self.serializer_class(self.course_run)

        with self.assertRaises(Exception):
            serializer.update(self.course_run, {'preview_url': 'invalid_url'})
            self.assertFalse(self.course_run.preview_url)

    def test_transaction_roll_back_with_error_on_email(self):
        """
        Verify that transaction is roll backed if error occurred during email sending.
        """
        toggle_switch('enable_publisher_email_notifications', True)
        self.course_run.preview_url = ''
        self.course_run.save()
        serializer = self.serializer_class(self.course_run)
        self.assertEqual(self.course_run.course_run_state.owner_role, PublisherUserRole.Publisher)

        with self.assertRaises(Exception):
            serializer.update(self.course_run, {'preview_url': 'https://example.com/abc/course'})

        self.course_run = CourseRun.objects.get(id=self.course_run.id)
        self.assertFalse(self.course_run.preview_url)
        # Verify that owner role not changed.
        self.assertEqual(self.course_run.course_run_state.owner_role, PublisherUserRole.Publisher)

class CourseRevisionSerializerTests(TestCase):
    def test_course_revision_serializer(self):
        """ Verify that CourseRevisionSerializer serialize the course revision object. """

        course = CourseFactory()
        course.title = 'updated title'
        course.save()
        revision = course.history.first()
        serializer = CourseRevisionSerializer(revision)

        expected = {
            'history_id': revision.history_id,
            'title': revision.title,
            'number': revision.number,
            'short_description': revision.short_description,
            'full_description': revision.full_description,
            'expected_learnings': revision.expected_learnings,
            'prerequisites': revision.prerequisites,
            'primary_subject': revision.primary_subject.name,
            'secondary_subject': revision.secondary_subject.name,
            'tertiary_subject': revision.tertiary_subject.name,
            'level_type': revision.level_type.name
        }

        self.assertDictEqual(serializer.data, expected)

    def test_course_revision_serializer_without_subjects(self):
        """ Verify that CourseRevisionSerializer serialize the course revision object
        even if subject fields are not available.
        """

        course = CourseFactory(primary_subject=None, secondary_subject=None, tertiary_subject=None, level_type=None)
        course.title = 'updated title'
        course.save()
        revision = course.history.first()
        serializer = CourseRevisionSerializer(revision)

        expected = {
            'history_id': revision.history_id,
            'title': revision.title,
            'number': revision.number,
            'short_description': revision.short_description,
            'full_description': revision.full_description,
            'expected_learnings': revision.expected_learnings,
            'prerequisites': revision.prerequisites,
            'primary_subject': None,
            'secondary_subject': None,
            'tertiary_subject': None,
            'level_type': None
        }

        self.assertDictEqual(serializer.data, expected)

class CourseStateSerializerTests(TestCase):
    serializer_class = CourseStateSerializer

    def setUp(self):
        super(CourseStateSerializerTests, self).setUp()
        self.course_state = CourseStateFactory(name=CourseStateChoices.Draft)
        self.request = RequestFactory()
        self.user = UserFactory()
        self.request.user = self.user

        CourseUserRoleFactory(
            course=self.course_state.course, role=PublisherUserRole.CourseTeam, user=self.user
        )

    def test_update(self):
        """
        Verify that we can update course workflow state with serializer.
        """
        course = self.course_state.course
        course.image = make_image_file('test_banner.jpg')
        course.save()
        course.organizations.add(OrganizationExtensionFactory().organization)

        self.assertNotEqual(self.course_state, CourseStateChoices.Review)
        serializer = self.serializer_class(self.course_state, context={'request': self.request})
        data = {'name': CourseStateChoices.Review}
        serializer.update(self.course_state, data)

        self.course_state = CourseState.objects.get(course=self.course_state.course)
        self.assertEqual(self.course_state.name, CourseStateChoices.Review)
        self.assertEqual(self.course_state.owner_role, PublisherUserRole.MarketingReviewer)

    def test_update_with_error(self):
        """
        Verify that serializer raises `ValidationError` with wrong transition.
        """
        serializer = self.serializer_class(self.course_state, context={'request': self.request})
        data = {'name': CourseStateChoices.Approved}

        with self.assertRaises(ValidationError):
            serializer.update(self.course_state, data)

class CourseRunStateSerializerTests(TestCase):
    serializer_class = CourseRunStateSerializer

    def setUp(self):
        super(CourseRunStateSerializerTests, self).setUp()
        self.run_state = CourseRunStateFactory(name=CourseRunStateChoices.Draft)
        self.course_run = self.run_state.course_run
        self.request = RequestFactory()
        self.user = UserFactory()
        self.request.user = self.user
        CourseStateFactory(name=CourseStateChoices.Approved, course=self.course_run.course)

        SeatFactory(course_run=self.course_run, type=Seat.AUDIT)
        language_tag = LanguageTag(code='te-st', name='Test Language')
        language_tag.save()
        self.course_run.transcript_languages.add(language_tag)
        self.course_run.language = language_tag
        organization = OrganizationFactory()
        self.course_run.course.organizations.add(organization)
        self.course_run.save()
        self.course_run.staff.add(PersonFactory())

        toggle_switch('enable_publisher_email_notifications', True)
        CourseUserRoleFactory(
            course=self.course_run.course, role=PublisherUserRole.CourseTeam, user=self.user
        )
        CourseUserRoleFactory(
            course=self.course_run.course, role=PublisherUserRole.ProjectCoordinator, user=UserFactory()
        )

    def test_update(self):
        """
        Verify that we can update course-run workflow state name and preview_accepted with serializer.
        """
        CourseUserRoleFactory(
            course=self.course_run.course, role=PublisherUserRole.Publisher, user=UserFactory()
        )

        self.assertNotEqual(self.run_state, CourseRunStateChoices.Review)

        self.course_run.lms_course_id = 'course-v1:edX+DemoX+Demo_Course'
        serializer = self.serializer_class(self.run_state, context={'request': self.request})
        data = {'name': CourseRunStateChoices.Review}
        serializer.update(self.run_state, data)

        self.assertEqual(self.run_state.name, CourseRunStateChoices.Review)

        self.assertFalse(self.run_state.preview_accepted)
        serializer.update(self.run_state, {'preview_accepted': True})

        self.assertTrue(CourseRun.objects.get(id=self.course_run.id).preview_url)
        self.assertEqual(len(mail.outbox), 1)
        course_key = CourseKey.from_string(self.course_run.lms_course_id)
        subject = 'Publication requested: {course_name} {run_name}'.format(
            course_name=self.course_run.course.title,
            run_name=course_key.run
        )
        self.assertIn(subject, str(mail.outbox[0].subject))

    def test_update_with_error(self):
        """
        Verify that serializer raises `ValidationError` with wrong transition.
        """
        serializer = self.serializer_class(self.run_state, context={'request': self.request})
        data = {'name': CourseRunStateChoices.Published}

        with self.assertRaises(ValidationError):
            serializer.update(self.run_state, data)
            self.assertEqual(len(mail.outbox), 1)

    def test_update_with_transaction_roll_back(self):
        """
        Verify that transaction roll back all db changes.
        """
        self.assertNotEqual(self.run_state, CourseRunStateChoices.Review)
        serializer = self.serializer_class(self.run_state, context={'request': self.request})
        data = {'name': CourseRunStateChoices.Review}
        with self.assertRaises(Exception):
            serializer.update(self.run_state, data)
            self.assertEqual(self.run_state.name, CourseRunStateChoices.Review)

            self.assertFalse(self.run_state.preview_accepted)
            serializer.update(self.run_state, {'preview_accepted': True})

            self.assertFalse(CourseRun.objects.get(id=self.course_run.id).preview_url)
            self.assertEqual(len(mail.outbox), 0)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

'This script process bands landsat, calculate radiance and reflectance and copy values of reflectace in db'

import os, sys, math, time, csv, io
import gdal
import numpy
import Pdbc

bandList = ['1','2','3','4','5','61','7']

class Landsat(object):
    def __init__(self, nameFolder):
        self.idLandsat = nameFolder
        self.metadata = self.readMetadata(self.idLandsat)
        self.getMetadata()

    def readMetadata(self, idLandsat):
        fileMTL = os.popen('ls '+idLandsat+'/*_MTL.txt').read().split('\n')[0]
        metadataFile = open(fileMTL,'r')
        metadata = {}
        for line in metadataFile:
            if not line.strip() == "END":
                val = line.strip().split('=')
                metadata [val[0].strip()] = val[1].strip().strip('"')
            else:
                break
        metadataFile.close()
        return metadata

    def acquireMetadata(self, band):

        metadatalist = []

        if ("RADIANCE_MAXIMUM_BAND_" + band) in self.metadata.keys():
            BANDFILE = "FILE_NAME_BAND_" + band
            LMAX = "RADIANCE_MAXIMUM_BAND_" + band
            LMIN = "RADIANCE_MINIMUM_BAND_" + band
            QCALMAX = "QUANTIZE_CAL_MAX_BAND_" + band
            QCALMIN = "QUANTIZE_CAL_MIN_BAND_" + band
            DATE = "DATE_ACQUIRED"
            metadatalist = [BANDFILE, LMAX, LMIN, QCALMAX, QCALMIN, DATE]

        elif ("LMAX_BAND" + band) in self.metadata.keys():
            BANDFILE = "BAND" + band + "_FILE_NAME"
            LMAX = "LMAX_BAND" + band
            LMIN = "LMIN_BAND" + band
            QCALMAX = "QCALMAX_BAND" + band
            QCALMIN = "QCALMIN_BAND" + band
            DATE ="ACQUISITION_DATE"
            metadatalist = [BANDFILE, LMAX, LMIN, QCALMAX, QCALMIN, DATE]

        else:
            print('There was a problem reading the metadata for this file.')

        return metadatalist

    def getMetadata(self):
        self.band = {}
        for b in bandList:
            metlist = self.acquireMetadata(b)
            BANDFILE = self.metadata[metlist[0]]
            LMAX = self.metadata[metlist[1]]
            LMIN = self.metadata[metlist[2]]
            QCALMAX = self.metadata[metlist[3]]
            QCALMIN = self.metadata[metlist[4]]
            self.DATE = self.metadata[metlist[5]]
            self.band[b] = Band(self.idLandsat, BANDFILE, LMAX, LMIN, QCALMAX, QCALMIN, self.DATE, b)

class Band(object):
    def __init__(self, idLandsat, BANDFILE, LMAX, LMIN, QCALMAX, QCALMIN, DATE, band):
        self.BANDFILE = BANDFILE
        self.LMAX = LMAX
        self.LMIN = LMIN
        self.QCALMAX = QCALMAX
        self.QCALMIN = QCALMIN
        self.DATE = DATE
        self.band = band
        self.dataset = gdal.Open(idLandsat+'/'+BANDFILE, gdal.GA_ReadOnly)
        self.geoTransform = self.dataset.GetGeoTransform()
        self.cols = self.dataset.RasterXSize
        self.rows = self.dataset.RasterYSize
        self.pixelWidth = self.geoTransform[1]
        self.pixelHeight = self.geoTransform[5]
        self.originX = self.geoTransform[0]
        self.originY = self.geoTransform[3]
        self.array = self.dataset.ReadAsArray()

class Point(object):
    def __init__(self, landsat, x , y, SIType):
        self.landsat = landsat
        self.x = x
        self.y = y
        self.SIType = SIType
        self.radiance = {}
        self.reflectance = {}

        for b in bandList:
            self.radiance[b] = self.calcRadiance(landsat.band[b].LMAX,
                                                         landsat.band[b].LMIN,
                                                         landsat.band[b].QCALMAX,
                                                         landsat.band[b].QCALMIN, landsat.band[b].array[x][y])

            if(b!='61'):
                self.reflectance[b] = self.calcReflectance(calcSolarDist(calcJDay(landsat.band[b].DATE)),
                                                                                getESUN(landsat.band[b].band, SIType),
                                                                                landsat.metadata['SUN_ELEVATION'],
                                                                                self.radiance[b])

            elif(self.radiance['61']>0):
                self.band6 = 1282.71 / numpy.log(((666.09 * 0.95) / self.radiance['61'] ) +1 )

        self.NDVI =  (self.reflectance['4'] - self.reflectance['3']) / (self.reflectance['4'] + self.reflectance['3'])

        self.idLandsat = landsat.idLandsat
        self.pixelWidth = landsat.band[str(1)].pixelWidth
        self.pixelHeight = landsat.band[str(1)].pixelHeight
        self.originX = landsat.band[str(1)].originX
        self.originY = landsat.band[str(1)].originY
        latitude = self.originX + (y * self.pixelWidth)
        longitude = self.originY + (x * self.pixelHeight)

        self.latitude = latitude - (latitude%250)
        self.longitude = longitude - (longitude%250)

    def cloud(self):
        if(self.reflectance['3']<0.08):
            return 'non-cloud' # filter one less 0.08 is not cloud
        elif((self.reflectance['2'] - self.reflectance['5']) / (self.reflectance['2'] + self.reflectance['5'])) > 0.7:
            return 'non-cloud' # filter two NDSI less 0.7 is not cloud
        elif(self.band6 > 300):
            return 'non-cloud' # filter three temp more 300 is not cloud
        elif((self.reflectance['5']/self.band6) > 225):
            return 'ambiguous' # filter four
        elif((self.reflectance['4']/self.reflectance['3']) > 2):
            return 'ambiguous' # filter five
        elif((self.reflectance['4']/self.reflectance['2']) > 2):
            return 'ambiguous' # filter six
        elif((self.reflectance['4']/self.reflectance['5']) < 1):
            return 'ambiguous' # filter seven
        elif((self.reflectance['5']/self.band6) > 210):
            return 'warn-cloud' # filter eight
        else:
            return 'cold-cloud' #filter eight

    def calcRadiance (self, LMAX, LMIN, QCALMAX, QCALMIN, QCAL):
        'This function calculate Spectral Radiance Scaling Method'
        LMAX = float(LMAX)
        LMIN = float(LMIN)
        QCALMAX = float(QCALMAX)
        QCALMIN = float(QCALMIN)
        offset = (LMAX - LMIN)/(QCALMAX-QCALMIN)
        radiance = offset * (QCAL - QCALMIN) + LMIN

        return radiance

    def calcReflectance(self, solarDist, ESUN, solarElevation, radianceRaster):

        #Value for solar zenith is 90 degrees minus solar elevation (angle from horizon to the center of the sun)
        #http://landsathandbook.gsfc.nasa.gov/data_properties/prog_sect6_3.html

        solarZenith = ((90.0 - (float(solarElevation)))*math.pi)/180 #Converted from degrees to radians
        solarDist = float(solarDist)
        ESUN = float(ESUN)
        radiance = radianceRaster

        reflectance = (math.pi * radiance * math.pow(solarDist, 2)) / (ESUN * math.cos(solarZenith))

        return reflectance

    def __str__(self):
        return "%s %s" % (self.x, self.y)

def calcSolarDist (jday):

    #Values taken from d.csv file which is a formatted version of the d.xls file
    #associated with the Landsat7 handbook, representing the distance of the sun
    #for each julian day (1-366).
    #landsathandbook.gsfc.nasa.gov/excel_docs/d.xls (included in the parent folder)
    #this line keeps the relative path were this script is executing

    jday = int(jday)
    dist = distances[jday - 1]

    return dist

def calcJDay (date):

    #Seperate date aspects into list (check for consistnecy in formatting of all
    #Landsat7 metatdata) YYYY-MM-DD
    dt = date.rsplit("-")

    #Cast each part of the date as a in integer in the 9 int tuple mktime
    t = time.mktime((int(dt[0]), int(dt[1]), int(dt[2]), 0, 0, 0, 0, 0, 0))

    #As part of the time package the 7th int in mktime is calulated as Julian Day
    #from the completion of other essential parts of the tuple
    jday = time.gmtime(t)[7]

    return jday

def getESUN(bandNum, SIType):
    SIType = SIType
    ESUN = {}
    #from NASA's Landsat7 User Handbook Table 11.3 http://landsathandbook.gsfc.nasa.gov/pdfs/Landsat7_Handbook.pdf
    #ETM+ Solar Spectral Irradiances(generated using the Thuillier solar spectrum)
    if SIType == 'ETM+ Thuillier':
        ESUN = {'b1':1997,'b2':1812,'b3':1533,'b4':1039,'b5':230.8,'b7':84.90,'b8':1362}

    #from NASA's Landsat7 User Handbook Table 11.3 http://landsathandbook.gsfc.nasa.gov/data_prod/prog_sect11_3.html
    #ETM+ Solar Spectral Irradiances (generated using the combined Chance-Kurucz Solar Spectrum within MODTRAN 5)
    if SIType == 'ETM+ ChKur':
        ESUN = {'b1':1970,'b2':1842,'b3':1547,'b4':1044,'b5':225.7,'b7':82.06,'b8':1369}

    #from NASA's Landsat7 User Handbook Table 9.1 http://landsathandbook.gsfc.nasa.gov/pdfs/Landsat7_Handbook.pdf
    #from the LPS ACCA algorith to correct for cloud cover
    if SIType == 'LPS ACAA Algorithm':
        ESUN = {'b1':1969,'b2':1840,'b3':1551,'b4':1044,'b5':225.7,'b7':82.06,'b8':1368}

    #from Revised Landsat-5 TM Radiometric Calibration Procedures and Postcalibration
    #Dynamic Ranges Gyanesh Chander and Brian Markham. Nov 2003. Table II. http://landsathandbook.gsfc.nasa.gov/pdfs/L5TMLUTIEEE2003.pdf
    #Landsat 4 ChKur
    if SIType == 'Landsat 5 ChKur':
        ESUN = {'b1':1957,'b2':1825,'b3':1557,'b4':1033,'b5':214.9,'b7':80.72}

    #from Revised Landsat-5 TM Radiometric Calibration Procedures and Postcalibration
    #Dynamic Ranges Gyanesh Chander and Brian Markham. Nov 2003. Table II. http://landsathandbook.gsfc.nasa.gov/pdfs/L5TMLUTIEEE2003.pdf
    #Landsat 4 ChKur
    if SIType == 'Landsat 4 ChKur':
        ESUN = {'b1':1957,'b2':1826,'b3':1554,'b4':1036,'b5':215,'b7':80.67}

    bandNum = 'b'+str(bandNum)

    return ESUN[bandNum]

def rasterToDB(nameFolders):
    for i in range(len(nameFolders)-1):
        if os.path.isfile(nameFolders[i]):
            os.system('rm -R '+nameFolders[i].split('.')[0])
        os.system('mkdir '+nameFolders[i].split('.')[0])
        os.system('tar -xvf '+nameFolders[i]+' -C '+nameFolders[i].split('.')[0])
        landsat = Landsat(nameFolders[i].split('.')[0])
        #date_landsat = '{0}\t{1}'.format(landsat.idLandsat, landsat.DATE)
        #db.copyToTable('date_landsat',io.StringIO(date_landsat))
        reflectance = []
        discarded = []
        for x in range(0,landsat.band['1'].rows,15):
            for y in range(0,landsat.band['1'].cols,15):
                SIType = 'ETM+ Thuillier'
                point = Point(landsat, x, y, SIType)
                if(landsat.band['1'].array[x][y]==0 or landsat.band['2'].array[x][y]==0 or
                 landsat.band['3'].array[x][y]==0 or landsat.band['4'].array[x][y]==0 or
                 landsat.band['5'].array[x][y]==0 or landsat.band['61'].array[x][y]==0):
                     discarded.append('{0}\t{1}\t{2}\t{3}'.format(point.idLandsat, point.latitude, point.longitude, 'not rated'))
                     continue
                if(point.NDVI>0.6):
                        cloud = point.cloud()
                        if(cloud == 'non-cloud'):
                            reflectance.append('{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}'.format(point.idLandsat,
                                                                                point.latitude,
                                                                                point.longitude,
                                                                                point.reflectance['1'],
                                                                                point.reflectance['2'],
                                                                                point.reflectance['3'],
                                                                                point.reflectance['4'],
                                                                                point.reflectance['5'],
                                                                                point.band6,
                                                                                point.reflectance['7']))

                        else:
                            discarded.append('{0}\t{1}\t{2}\t{3}'.format(point.idLandsat, point.latitude, point.longitude, cloud))
                else:
                    discarded.append('{0}\t{1}\t{2}\t{3}'.format(point.idLandsat, point.latitude, point.longitude, 'no vegetation'))

        reflectance = '\n'.join(reflectance)
        discarded = '\n'.join(discarded)
        db.copyToTable('reflectance',io.StringIO(reflectance))
        db.copyToTable('discarded',io.StringIO(discarded))

path = os.getcwd()
path += '/L7'
os.chdir(path)

f = open('../d.csv', "r")
lines = f.readlines()[2:]

distances = []
for x in range(len(lines)):
    distances.append(float(lines[x].strip().split(',')[1]))
f.close()

tiempo1 = time.time()

db = Pdbc.DBConnector('landsat', 'omar', '1234', 'localhost', '5432')

filterImg = 'LE7*2001*'
nameFolders = os.popen('ls '+filterImg+'bz').read().split("\n")

rasterToDB(nameFolders)

tiempo2 = time.time()

print(tiempo2-tiempo1)
<commit_msg>rasterToDB.py<commit_after>#!/usr/bin/env python3

'This script process bands landsat, calculate radiance and reflectance and copy values of reflectace in db'

import os, sys, math, time, csv, io
import gdal
import numpy
import Pdbc

bandList = ['1','2','3','4','5','61','7']

class Landsat(object):
    def __init__(self, nameFolder):
        self.idLandsat = nameFolder
        self.metadata = self.readMetadata(self.idLandsat)
        self.getMetadata()

    def readMetadata(self, idLandsat):
        fileMTL = os.popen('ls '+idLandsat+'/*_MTL.txt').read().split('\n')[0]
        metadataFile = open(fileMTL,'r')
        metadata = {}
        for line in metadataFile:
            if not line.strip() == "END":
                val = line.strip().split('=')
                metadata [val[0].strip()] = val[1].strip().strip('"')
            else:
                break
        metadataFile.close()
        return metadata

    def acquireMetadata(self, band):

        metadatalist = []

        if ("RADIANCE_MAXIMUM_BAND_" + band) in self.metadata.keys():
            BANDFILE = "FILE_NAME_BAND_" + band
            LMAX = "RADIANCE_MAXIMUM_BAND_" + band
            LMIN = "RADIANCE_MINIMUM_BAND_" + band
            QCALMAX = "QUANTIZE_CAL_MAX_BAND_" + band
            QCALMIN = "QUANTIZE_CAL_MIN_BAND_" + band
            DATE = "DATE_ACQUIRED"
            metadatalist = [BANDFILE, LMAX, LMIN, QCALMAX, QCALMIN, DATE]
            return metadatalist

        elif ("LMAX_BAND" + band) in self.metadata.keys():
            BANDFILE = "BAND" + band + "_FILE_NAME"
            LMAX = "LMAX_BAND" + band
            LMIN = "LMIN_BAND" + band
            QCALMAX = "QCALMAX_BAND" + band
            QCALMIN = "QCALMIN_BAND" + band
            DATE ="ACQUISITION_DATE"
            metadatalist = [BANDFILE, LMAX, LMIN, QCALMAX, QCALMIN, DATE]
            return metadatalist

        elif(band == '61'):
            if ("RADIANCE_MAXIMUM_BAND_6_VCID_1") in self.metadata.keys():
                BANDFILE = "FILE_NAME_BAND_6_VCID_1"
                LMAX = "RADIANCE_MAXIMUM_BAND_6_VCID_1"
                LMIN = "RADIANCE_MINIMUM_BAND_6_VCID_1"
                QCALMAX = "QUANTIZE_CAL_MAX_BAND_6_VCID_1"
                QCALMIN = "QUANTIZE_CAL_MIN_BAND_6_VCID_1"
                DATE = "DATE_ACQUIRED"
                metadatalist = [BANDFILE, LMAX, LMIN, QCALMAX, QCALMIN, DATE]
                return metadatalist

        else:
            print('There was a problem reading the metadata for this file.')

        return metadatalist

    def getMetadata(self):
        self.band = {}
        for b in bandList:
            metlist = self.acquireMetadata(b)
            BANDFILE = self.metadata[metlist[0]]
            LMAX = self.metadata[metlist[1]]
            LMIN = self.metadata[metlist[2]]
            QCALMAX = self.metadata[metlist[3]]
            QCALMIN = self.metadata[metlist[4]]
            self.DATE = self.metadata[metlist[5]]
            self.band[b] = Band(self.idLandsat, BANDFILE, LMAX, LMIN, QCALMAX, QCALMIN, self.DATE, b)

class Band(object):
    def __init__(self, idLandsat, BANDFILE, LMAX, LMIN, QCALMAX, QCALMIN, DATE, band):
        self.BANDFILE = BANDFILE
        self.LMAX = LMAX
        self.LMIN = LMIN
        self.QCALMAX = QCALMAX
        self.QCALMIN = QCALMIN
        self.DATE = DATE
        self.band = band
        self.dataset = gdal.Open(idLandsat+'/'+BANDFILE, gdal.GA_ReadOnly)
        self.geoTransform = self.dataset.GetGeoTransform()
        self.cols = self.dataset.RasterXSize
        self.rows = self.dataset.RasterYSize
        self.pixelWidth = self.geoTransform[1]
        self.pixelHeight = self.geoTransform[5]
        self.originX = self.geoTransform[0]
        self.originY = self.geoTransform[3]
        self.array = self.dataset.ReadAsArray()

class Point(object):
    def __init__(self, landsat, x , y, SIType):
        self.landsat = landsat
        self.x = x
        self.y = y
        self.SIType = SIType
        self.radiance = {}
        self.reflectance = {}

        for b in bandList:
            self.radiance[b] = self.calcRadiance(landsat.band[b].LMAX,
                                                         landsat.band[b].LMIN,
                                                         landsat.band[b].QCALMAX,
                                                         landsat.band[b].QCALMIN, landsat.band[b].array[x][y])

            if(b!='61'):
                self.reflectance[b] = self.calcReflectance(calcSolarDist(calcJDay(landsat.band[b].DATE)),
                                                                                getESUN(landsat.band[b].band, SIType),
                                                                                landsat.metadata['SUN_ELEVATION'],
                                                                                self.radiance[b])

            elif(self.radiance['61']>0):
                self.band6 = 1282.71 / numpy.log(((666.09 * 0.95) / self.radiance['61'] ) +1 )

        self.NDVI =  (self.reflectance['4'] - self.reflectance['3']) / (self.reflectance['4'] + self.reflectance['3'])

        self.idLandsat = landsat.idLandsat
        self.pixelWidth = landsat.band[str(1)].pixelWidth
        self.pixelHeight = landsat.band[str(1)].pixelHeight
        self.originX = landsat.band[str(1)].originX
        self.originY = landsat.band[str(1)].originY
        latitude = self.originX + (y * self.pixelWidth)
        longitude = self.originY + (x * self.pixelHeight)

        self.latitude = latitude - (latitude%450)
        self.longitude = longitude - (longitude%450)

    def cloud(self):
        if(self.reflectance['3']<0.08):
            return 'non-cloud' # filter one less 0.08 is not cloud
        elif((self.reflectance['2'] - self.reflectance['5']) / (self.reflectance['2'] + self.reflectance['5'])) > 0.7:
            return 'non-cloud' # filter two NDSI less 0.7 is not cloud
        elif(self.band6 > 300):
            return 'non-cloud' # filter three temp more 300 is not cloud
        elif((self.reflectance['5']/self.band6) > 225):
            return 'ambiguous' # filter four
        elif((self.reflectance['4']/self.reflectance['3']) > 2):
            return 'ambiguous' # filter five
        elif((self.reflectance['4']/self.reflectance['2']) > 2):
            return 'ambiguous' # filter six
        elif((self.reflectance['4']/self.reflectance['5']) < 1):
            return 'ambiguous' # filter seven
        elif((self.reflectance['5']/self.band6) > 210):
            return 'warn-cloud' # filter eight
        else:
            return 'cold-cloud' #filter eight

    def calcRadiance (self, LMAX, LMIN, QCALMAX, QCALMIN, QCAL):
        'This function calculate Spectral Radiance Scaling Method'
        LMAX = float(LMAX)
        LMIN = float(LMIN)
        QCALMAX = float(QCALMAX)
        QCALMIN = float(QCALMIN)
        offset = (LMAX - LMIN)/(QCALMAX-QCALMIN)
        radiance = offset * (QCAL - QCALMIN) + LMIN

        return radiance

    def calcReflectance(self, solarDist, ESUN, solarElevation, radianceRaster):

        #Value for solar zenith is 90 degrees minus solar elevation (angle from horizon to the center of the sun)
        #http://landsathandbook.gsfc.nasa.gov/data_properties/prog_sect6_3.html

        solarZenith = ((90.0 - (float(solarElevation)))*math.pi)/180 #Converted from degrees to radians
        solarDist = float(solarDist)
        ESUN = float(ESUN)
        radiance = radianceRaster

        reflectance = (math.pi * radiance * math.pow(solarDist, 2)) / (ESUN * math.cos(solarZenith))

        return reflectance

    def __str__(self):
        return "%s %s" % (self.x, self.y)

def calcSolarDist (jday):

    #Values taken from d.csv file which is a formatted version of the d.xls file
    #associated with the Landsat7 handbook, representing the distance of the sun
    #for each julian day (1-366).
    #landsathandbook.gsfc.nasa.gov/excel_docs/d.xls (included in the parent folder)
    #this line keeps the relative path were this script is executing

    jday = int(jday)
    dist = distances[jday - 1]

    return dist

def calcJDay (date):

    #Seperate date aspects into list (check for consistnecy in formatting of all
    #Landsat7 metatdata) YYYY-MM-DD
    dt = date.rsplit("-")

    #Cast each part of the date as a in integer in the 9 int tuple mktime
    t = time.mktime((int(dt[0]), int(dt[1]), int(dt[2]), 0, 0, 0, 0, 0, 0))

    #As part of the time package the 7th int in mktime is calulated as Julian Day
    #from the completion of other essential parts of the tuple
    jday = time.gmtime(t)[7]

    return jday

def getESUN(bandNum, SIType):
    SIType = SIType
    ESUN = {}
    #from NASA's Landsat7 User Handbook Table 11.3 http://landsathandbook.gsfc.nasa.gov/pdfs/Landsat7_Handbook.pdf
    #ETM+ Solar Spectral Irradiances(generated using the Thuillier solar spectrum)
    if SIType == 'ETM+ Thuillier':
        ESUN = {'b1':1997,'b2':1812,'b3':1533,'b4':1039,'b5':230.8,'b7':84.90,'b8':1362}

    #from NASA's Landsat7 User Handbook Table 11.3 http://landsathandbook.gsfc.nasa.gov/data_prod/prog_sect11_3.html
    #ETM+ Solar Spectral Irradiances (generated using the combined Chance-Kurucz Solar Spectrum within MODTRAN 5)
    if SIType == 'ETM+ ChKur':
        ESUN = {'b1':1970,'b2':1842,'b3':1547,'b4':1044,'b5':225.7,'b7':82.06,'b8':1369}

    #from NASA's Landsat7 User Handbook Table 9.1 http://landsathandbook.gsfc.nasa.gov/pdfs/Landsat7_Handbook.pdf
    #from the LPS ACCA algorith to correct for cloud cover
    if SIType == 'LPS ACAA Algorithm':
        ESUN = {'b1':1969,'b2':1840,'b3':1551,'b4':1044,'b5':225.7,'b7':82.06,'b8':1368}

    #from Revised Landsat-5 TM Radiometric Calibration Procedures and Postcalibration
    #Dynamic Ranges Gyanesh Chander and Brian Markham. Nov 2003. Table II. http://landsathandbook.gsfc.nasa.gov/pdfs/L5TMLUTIEEE2003.pdf
    #Landsat 4 ChKur
    if SIType == 'Landsat 5 ChKur':
        ESUN = {'b1':1957,'b2':1825,'b3':1557,'b4':1033,'b5':214.9,'b7':80.72}

    #from Revised Landsat-5 TM Radiometric Calibration Procedures and Postcalibration
    #Dynamic Ranges Gyanesh Chander and Brian Markham. Nov 2003. Table II. http://landsathandbook.gsfc.nasa.gov/pdfs/L5TMLUTIEEE2003.pdf
    #Landsat 4 ChKur
    if SIType == 'Landsat 4 ChKur':
        ESUN = {'b1':1957,'b2':1826,'b3':1554,'b4':1036,'b5':215,'b7':80.67}

    bandNum = 'b'+str(bandNum)

    return ESUN[bandNum]

def rasterToDB(nameFolders):
    for i in range(len(nameFolders)-1):
        if os.path.isfile(nameFolders[i]):
            os.system('rm -R '+nameFolders[i].split('.')[0])
        os.system('mkdir '+nameFolders[i].split('.')[0])
        os.system('tar -xvf '+nameFolders[i]+' -C '+nameFolders[i].split('.')[0])
        landsat = Landsat(nameFolders[i].split('.')[0])
        #date_landsat = '{0}\t{1}'.format(landsat.idLandsat, landsat.DATE)
        #db.copyToTable('date_landsat',io.StringIO(date_landsat))
        reflectance = []
        discarded = []
        for x in range(0,landsat.band['1'].rows,15):
            for y in range(0,landsat.band['1'].cols,15):
                SIType = 'ETM+ Thuillier'
                point = Point(landsat, x, y, SIType)
                if(landsat.band['1'].array[x][y]==0 or landsat.band['2'].array[x][y]==0 or
                 landsat.band['3'].array[x][y]==0 or landsat.band['4'].array[x][y]==0 or
                 landsat.band['5'].array[x][y]==0 or landsat.band['61'].array[x][y]==0):
                     discarded.append('{0}\t{1}\t{2}\t{3}'.format(point.idLandsat, point.latitude, point.longitude, 'not rated'))
                     continue
                if(point.NDVI>0.6):
                        cloud = point.cloud()
                        if(cloud == 'non-cloud'):
                            reflectance.append('{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}'.format(point.idLandsat,
                                                                                point.latitude,
                                                                                point.longitude,
                                                                                point.reflectance['1'],
                                                                                point.reflectance['2'],
                                                                                point.reflectance['3'],
                                                                                point.reflectance['4'],
                                                                                point.reflectance['5'],
                                                                                point.band6,
                                                                                point.reflectance['7']))

                        else:
                            discarded.append('{0}\t{1}\t{2}\t{3}'.format(point.idLandsat, point.latitude, point.longitude, cloud))
                else:
                    discarded.append('{0}\t{1}\t{2}\t{3}'.format(point.idLandsat, point.latitude, point.longitude, 'no vegetation'))

        reflectance = '\n'.join(reflectance)
        discarded = '\n'.join(discarded)
        db.copyToTable('reflectance',io.StringIO(reflectance))
        db.copyToTable('discarded',io.StringIO(discarded))
        os.system('rm -R '+nameFolders[i].split('.')[0])

path = os.getcwd()
path += '/L7'
os.chdir(path)

f = open('../d.csv', "r")
lines = f.readlines()[2:]

distances = []
for x in range(len(lines)):
    distances.append(float(lines[x].strip().split(',')[1]))
f.close()

tiempo1 = time.time()

db = Pdbc.DBConnector('landsat', 'omar', '1234', 'localhost', '5432')

filterImg = 'LE7*2001*'
nameFolders = os.popen('ls '+filterImg+'bz').read().split("\n")

rasterToDB(nameFolders)

tiempo2 = time.time()

print(tiempo2-tiempo1)
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import unicode_literals
from datetime import date, timedelta

import csv342 as csv
import io
import attr
import requests
import json

from celery.schedules import crontab
from celery.task import task
from celery.task.base import periodic_task
from django.conf import settings
from django.contrib.auth import get_user_model
from django.db.models import Q
from django.template import Context, Template
from django.template.loader import render_to_string

from corehq.apps.es.users import UserES
from corehq.apps.hqadmin.models import HistoricalPillowCheckpoint
from corehq.apps.hqwebapp.tasks import send_html_email_async
from corehq.util.soft_assert import soft_assert
from dimagi.utils.logging import notify_error
from dimagi.utils.django.email import send_HTML_email
from dimagi.utils.web import get_site_domain
from pillowtop.utils import get_couch_pillow_instances
from .utils import check_for_rewind

_soft_assert_superusers = soft_assert(notify_admins=True)

@periodic_task(run_every=crontab(hour=0, minute=0), queue='background_queue')
def check_pillows_for_rewind():
    for pillow in get_couch_pillow_instances():
        checkpoint = pillow.checkpoint
        has_rewound, historical_seq = check_for_rewind(checkpoint)
        if has_rewound:
            notify_error(
                message='Found seq number lower than previous for {}. '
                        'This could mean we are in a rewind state'.format(checkpoint.checkpoint_id),
                details={
                    'pillow checkpoint seq': checkpoint.get_current_sequence_id(),
                    'stored seq': historical_seq
                }
            )

@periodic_task(run_every=crontab(hour=0, minute=0), queue='background_queue')
def create_historical_checkpoints():
    today = date.today()
    thirty_days_ago = today - timedelta(days=30)
    HistoricalPillowCheckpoint.create_pillow_checkpoint_snapshots()
    HistoricalPillowCheckpoint.objects.filter(date_updated__lt=thirty_days_ago).delete()

@periodic_task(run_every=crontab(minute=0), queue='background_queue')
def check_non_dimagi_superusers():
    non_dimagis_superuser = ', '.join((get_user_model().objects.filter(
        (Q(is_staff=True) | Q(is_superuser=True)) & ~Q(username__endswith='@dimagi.com')
    ).values_list('username', flat=True)))
    if non_dimagis_superuser:
        message = "{non_dimagis} have superuser privileges".format(non_dimagis=non_dimagis_superuser)
        _soft_assert_superusers(False, message)
        notify_error(message=message)

@task(serializer='pickle', queue="email_queue")
def send_mass_emails(username, real_email, subject, html, text):
    if real_email:
        recipients = [{
            'username': h['username'],
            'first_name': h['first_name'] or 'CommCare User',
        } for h in UserES().web_users().run().hits]
    else:
        recipients = [{
            'username': username,
            'first_name': 'CommCare User',
        }]

    successes = []
    failures = []
    for recipient in recipients:
        context = recipient
        context.update({
            'url_prefix': '' if settings.STATIC_CDN else 'http://' + get_site_domain(),
        })

        html_template = Template(html)
        text_template = Template(text)
        text_content = render_to_string("hqadmin/email/mass_email_base.txt", {
            'email_body': text_template.render(Context(context)),
        })
        html_content = render_to_string("hqadmin/email/mass_email_base.html", {
            'email_body': html_template.render(Context(context)),
        })

        try:
            send_HTML_email(subject, recipient['username'], html_content, text_content=text_content)
            successes.append((recipient['username'], None))
        except Exception as e:
            failures.append((recipient['username'], e))

    message = (
        "Subject: {subject},\n"
        "Total successes: {success_count} \n Total errors: {failure_count} \n"
        "".format(
            subject=subject,
            success_count=len(successes),
            failure_count=len(failures))
    )

    send_html_email_async(
        "Mass email summary", username, message,
        text_content=message, file_attachments=[
            _mass_email_attachment('successes', successes),
            _mass_email_attachment('failures', failures)]
    )

@attr.s
class AbnormalUsageAlert(object):
    source = attr.ib()
    domain = attr.ib()
    message = attr.ib()

support_email = "support@dimagi.com"

@task(serializer='pickle', queue="email_queue")
def send_abnormal_usage_alert(alert):
    """ Sends an alert to #support and email to let support know when a domain is doing something weird

    :param alert: AbnormalUsageAlert object
    """

    subject = "{domain} is doing something interesting with the {source} in the {environment} env".format(
        domain=alert.domain,
        source=alert.source,
        environment=settings.SERVER_ENVIRONMENT
    )
    send_html_email_async(
        subject,
        support_email,
        alert.message
    )

def _mass_email_attachment(name, rows):
    csv_file = io.StringIO()
    writer = csv.writer(csv_file)
    writer.writerow(['Email', 'Error'])
    writer.writerows(rows)
    attachment = {
        'title': "mass_email_{}.csv".format(name),
        'mimetype': 'text/csv',
        'file_obj': csv_file,
    }
    return attachment
<commit_msg>Removed hard-coded support email<commit_after>from __future__ import absolute_import
from __future__ import unicode_literals
from datetime import date, timedelta

import csv342 as csv
import io
import attr
import requests
import json

from celery.schedules import crontab
from celery.task import task
from celery.task.base import periodic_task
from django.conf import settings
from django.contrib.auth import get_user_model
from django.db.models import Q
from django.template import Context, Template
from django.template.loader import render_to_string

from corehq.apps.es.users import UserES
from corehq.apps.hqadmin.models import HistoricalPillowCheckpoint
from corehq.apps.hqwebapp.tasks import send_html_email_async
from corehq.util.soft_assert import soft_assert
from dimagi.utils.logging import notify_error
from dimagi.utils.django.email import send_HTML_email
from dimagi.utils.web import get_site_domain
from pillowtop.utils import get_couch_pillow_instances
from .utils import check_for_rewind

_soft_assert_superusers = soft_assert(notify_admins=True)

@periodic_task(run_every=crontab(hour=0, minute=0), queue='background_queue')
def check_pillows_for_rewind():
    for pillow in get_couch_pillow_instances():
        checkpoint = pillow.checkpoint
        has_rewound, historical_seq = check_for_rewind(checkpoint)
        if has_rewound:
            notify_error(
                message='Found seq number lower than previous for {}. '
                        'This could mean we are in a rewind state'.format(checkpoint.checkpoint_id),
                details={
                    'pillow checkpoint seq': checkpoint.get_current_sequence_id(),
                    'stored seq': historical_seq
                }
            )

@periodic_task(run_every=crontab(hour=0, minute=0), queue='background_queue')
def create_historical_checkpoints():
    today = date.today()
    thirty_days_ago = today - timedelta(days=30)
    HistoricalPillowCheckpoint.create_pillow_checkpoint_snapshots()
    HistoricalPillowCheckpoint.objects.filter(date_updated__lt=thirty_days_ago).delete()

@periodic_task(run_every=crontab(minute=0), queue='background_queue')
def check_non_dimagi_superusers():
    non_dimagis_superuser = ', '.join((get_user_model().objects.filter(
        (Q(is_staff=True) | Q(is_superuser=True)) & ~Q(username__endswith='@dimagi.com')
    ).values_list('username', flat=True)))
    if non_dimagis_superuser:
        message = "{non_dimagis} have superuser privileges".format(non_dimagis=non_dimagis_superuser)
        _soft_assert_superusers(False, message)
        notify_error(message=message)

@task(serializer='pickle', queue="email_queue")
def send_mass_emails(username, real_email, subject, html, text):
    if real_email:
        recipients = [{
            'username': h['username'],
            'first_name': h['first_name'] or 'CommCare User',
        } for h in UserES().web_users().run().hits]
    else:
        recipients = [{
            'username': username,
            'first_name': 'CommCare User',
        }]

    successes = []
    failures = []
    for recipient in recipients:
        context = recipient
        context.update({
            'url_prefix': '' if settings.STATIC_CDN else 'http://' + get_site_domain(),
        })

        html_template = Template(html)
        text_template = Template(text)
        text_content = render_to_string("hqadmin/email/mass_email_base.txt", {
            'email_body': text_template.render(Context(context)),
        })
        html_content = render_to_string("hqadmin/email/mass_email_base.html", {
            'email_body': html_template.render(Context(context)),
        })

        try:
            send_HTML_email(subject, recipient['username'], html_content, text_content=text_content)
            successes.append((recipient['username'], None))
        except Exception as e:
            failures.append((recipient['username'], e))

    message = (
        "Subject: {subject},\n"
        "Total successes: {success_count} \n Total errors: {failure_count} \n"
        "".format(
            subject=subject,
            success_count=len(successes),
            failure_count=len(failures))
    )

    send_html_email_async(
        "Mass email summary", username, message,
        text_content=message, file_attachments=[
            _mass_email_attachment('successes', successes),
            _mass_email_attachment('failures', failures)]
    )

@attr.s
class AbnormalUsageAlert(object):
    source = attr.ib()
    domain = attr.ib()
    message = attr.ib()

@task(serializer='pickle', queue="email_queue")
def send_abnormal_usage_alert(alert):
    """ Sends an alert to #support and email to let support know when a domain is doing something weird

    :param alert: AbnormalUsageAlert object
    """

    subject = "{domain} is doing something interesting with the {source} in the {environment} env".format(
        domain=alert.domain,
        source=alert.source,
        environment=settings.SERVER_ENVIRONMENT
    )
    send_html_email_async(
        subject,
        settings.SUPPORT_EMAIL,
        alert.message
    )

def _mass_email_attachment(name, rows):
    csv_file = io.StringIO()
    writer = csv.writer(csv_file)
    writer.writerow(['Email', 'Error'])
    writer.writerows(rows)
    attachment = {
        'title': "mass_email_{}.csv".format(name),
        'mimetype': 'text/csv',
        'file_obj': csv_file,
    }
    return attachment
<|endoftext|>


<commit_before>import Polygon, cv2, numpy as np, time
from collections import namedtuple

w, h, z = 8, 8, 30
poly = ((1,2), (w-1,1), (w-1,h-1))

Point = namedtuple('Point', 'x y')
lattice = [Point(x,y) for x in xrange(w) for y in xrange(h)]

def normalize_point(p):
	return (p[0]/float(w), p[1]/float(h))
normed_poly = Polygon.Polygon([normalize_point(p) for p in poly])
normed_box = [Point(*normalize_point(p)) for p in [(0,0), (1,0), (1,1), (0,1)]]
normed_lattice = [Point(*normalize_point(p)) for p in lattice]

# ------------------------------------------------------------------------------

def psi_2d(p, e, j, k):
	def psi_1d(p, e):
		if e == 0:	return 1 if 0 <= p < 1 else 0
		else:		return (1 if 0 <= p < 0.5 else -1) if 0 <= p < 1 else 0
	return 2**j * psi_1d(2**j*p.x-k.x, e.x) * psi_1d(2**j*p.y-k.y, e.y)

def c(e, j, k):
	s = 0
	for p in normed_lattice:
		psi_p = psi_2d(p, e, j, k)
		if psi_p != 0:
			box = Polygon.Polygon([(b.x+p.x, b.y+p.y) for b in normed_box])
			px_poly = box & normed_poly
			s += px_poly.area() * psi_p
	return s

def g(p, d):
	s = 0
	# --------------------------------------------------------------------------
	j = 0
	for kx in xrange(2**j):
		for ky in xrange(2**j):
			k = Point(kx, ky)
			s += c(Point(0,0), j, k) * psi_2d(p, Point(0,0), j, k)
	# --------------------------------------------------------------------------
	for j in xrange(d):
		for kx in xrange(2**j):
			for ky in xrange(2**j):
				k = Point(kx, ky)
				for e in [Point(0,1), Point(1,0), Point(1,1)]:
					s += c(e, j, k) * psi_2d(p, e, j, k)
	return s

# ------------------------------------------------------------------------------
ts = time.time()
raster = [g(p, 4) for p in normed_lattice]
print 'time: %2.1fs' % (time.time()-ts)

raster = np.array(np.asarray(raster)*30, np.uint8)
raster = np.reshape(raster, (w,h))
raster = cv2.cvtColor(raster, cv2.COLOR_GRAY2BGR)
raster = cv2.resize(raster, (w*z,h*z), interpolation = cv2.INTER_NEAREST)

poly_vis = np.array([(p[1]*z, p[0]*z) for p in poly])
cv2.polylines(raster, [poly_vis],True,(255,255,0),lineType=cv2.CV_AA)
cv2.namedWindow('raster')
cv2.imshow('raster', raster)

cv2.waitKey(0)<commit_msg>clean-ups<commit_after>import Polygon, cv2, numpy as np, time, math
from collections import namedtuple

# ------------------------------------------------------------------------------
Point = namedtuple('Point', 'x y')

class Rasterizer:

    def __init__(self, poly, w, h):
        def normalize(p):
            return (p[0]/float(w), p[1]/float(h))
        box = [(0,0), (1,0), (1,1), (0,1)]
        lattice = [Point(x,y) for x in xrange(w) for y in xrange(h)]
        self._poly = Polygon.Polygon([normalize(p) for p in poly])
        self._box = [Point(*normalize(p)) for p in box]
        self._lattice = [Point(*normalize(p)) for p in lattice]
        self._w = w
        self._h = h

    def _psi(self, p, e, j, k):
        def psi_1d(p, e):
            if e == 0:   return 1 if 0 <= p < 1 else 0
            else:        return (1 if 0 <= p < 0.5 else -1) if 0 <= p < 1 else 0
        return 2**j * psi_1d(2**j*p.x-k.x, e.x) * psi_1d(2**j*p.y-k.y, e.y)

    def _c(self, e, j, k):
        s = 0
        for p in self._lattice:
            psi_p = self._psi(p, e, j, k)
            if psi_p != 0:
                box = Polygon.Polygon([(b.x+p.x, b.y+p.y) for b in self._box])
                px_poly = box & self._poly
                s += px_poly.area() * psi_p
        return s

    def _g(self, p):
        s = 0
        # ----------------------------------------------------------------------
        j = 0
        for kx in xrange(2**j):
            for ky in xrange(2**j):
                k = Point(kx, ky)
                s += self._c(Point(0,0), j,k) * self._psi(p, Point(0,0), j,k)
        # ----------------------------------------------------------------------
        max_j = int(math.ceil(math.log(max(w,h), 2)))
        for j in xrange(max_j+1):
            for kx in xrange(2**j):
                for ky in xrange(2**j):
                    k = Point(kx, ky)
                    for e in [Point(0,1), Point(1,0), Point(1,1)]:
                        s += self._c(e, j, k) * self._psi(p, e, j, k)
        return s

    def get(self):
        return np.reshape([self._g(p) for p in self._lattice],(self._w,self._h))

# ------------------------------------------------------------------------------

if __name__ == '__main__':
    w, h, z = 8, 8, 30
    poly = ((1,2), (w-1,1), (w-1,h-1))

    ts = time.time()
    raster = Rasterizer(poly, w, h).get()
    print 'time: %2.1fs' % (time.time()-ts)

    raster = np.array(np.asarray(raster)*30, np.uint8)
    raster = cv2.cvtColor(raster, cv2.COLOR_GRAY2BGR)
    raster = cv2.resize(raster, (w*z,h*z), interpolation = cv2.INTER_NEAREST)

    poly_vis = np.array([(p[1]*z, p[0]*z) for p in poly])
    cv2.polylines(raster, [poly_vis],True,(255,255,0),lineType=cv2.CV_AA)
    cv2.namedWindow('raster')
    cv2.imshow('raster', raster)

    cv2.waitKey(0)<|endoftext|>