<commit_before>from __future__ import print_function
from __future__ import absolute_import
from __future__ import division

# from itertools import zip_longest

import compas_rhino
# from compas.utilities import like_list
from compas_rhino.artists import _PrimitiveArtist

__all__ = ['LineArtist']

def list_like(target, value, fillvalue=None):
    p = len(target)

    if isinstance(value, list):
        matched_list = value
    else:
        matched_list = [value] * p

    d = len(matched_list)
    if d < p:
        matched_list.extend([fillvalue] * (p - d))

    return matched_list

class LineArtist(_PrimitiveArtist):
    """Artist for drawing ``Line`` objects.

    Parameters
    ----------
    line : :class:`compas.geometry.Line`
        A COMPAS line.
    layer : str (optional)
        The name of the layer that will contain the line.
        Default value is ``None``, in which case the current layer will be used.

    Examples
    --------
    >>>

    """

    __module__ = "compas_rhino.artists"

    def __init__(self, line, layer=None):
        super(LineArtist, self).__init__(line, layer=layer)
        self.settings.update({
            'color.line': (0, 0, 0)})

    def draw(self):
        """Draw the line.

        Returns
        -------
        guid: str
            The GUID of the created Rhino object.

        """
        start = list(self.primitive.start)
        end = list(self.primitive.end)
        lines = [{'start': start, 'end': end, 'color': self.settings['color.Line']}]
        guids = compas_rhino.draw_lines(lines, layer=self.settings['layer'], clear=False)
        if guids:
            return guids[0]

    @staticmethod
    def draw_collection(collection, color=None, layer=None, clear=False, group_collection=False, group_name=None):
        """Draw a collection of lines.

        Parameters
        ----------
        collection: list of compas.geometry.Line
            A collection of ``Line`` objects.
        color: tuple or list of tuple (optional)
            Color specification of the lines.
            If one RGB color is provided, it will be applied to all lines.
            If a list of RGB colors is provided, these colors are applied to the corresponding lines.
            A list of colors should have the same length as the collection, with one color per item.
            Default value is ``None`` in which case the default line color of the artist is used.
        layer: str (optional)
            The layer in which the objects of the collection should be created.
            Default is ``None``, in which case the default layer setting of the artist is used.
        clear: bool (optional)
            Clear the layer before drawing.
            Default is ``False``.
        group_collection: bool (optional)
            Flag for grouping the objects of the collection.
            Default is ``False``.
        group_name: str (optional).
            The name of the group.
            Default is ``None``.

        Returns
        -------
        guids: list
            A list of GUIDs if the collection is not grouped.
        groupname: str
            The name of the group if the collection objects are grouped.

        """
        colors = list_like(collection, color)
        lines = []
        for line, rgb in zip(collection, colors):
            lines.append({
                'start': list(line.start),
                'end': list(line.end),
                'color': rgb})
        guids = compas_rhino.draw_lines(lines, layer=layer, clear=clear)
        if not group_collection:
            return guids
        group = compas_rhino.rs.AddGroup(group_name)
        if group:
            compas_rhino.rs.AddObjectsToGroup(guids, group)
        return group

# ==============================================================================
# Main
# ==============================================================================

if __name__ == "__main__":

    pass
<commit_msg>fix typo<commit_after>from __future__ import print_function
from __future__ import absolute_import
from __future__ import division

# from itertools import zip_longest

import compas_rhino
# from compas.utilities import like_list
from compas_rhino.artists import _PrimitiveArtist

__all__ = ['LineArtist']

def list_like(target, value, fillvalue=None):
    p = len(target)

    if isinstance(value, list):
        matched_list = value
    else:
        matched_list = [value] * p

    d = len(matched_list)
    if d < p:
        matched_list.extend([fillvalue] * (p - d))

    return matched_list

class LineArtist(_PrimitiveArtist):
    """Artist for drawing ``Line`` objects.

    Parameters
    ----------
    line : :class:`compas.geometry.Line`
        A COMPAS line.
    layer : str (optional)
        The name of the layer that will contain the line.
        Default value is ``None``, in which case the current layer will be used.

    Examples
    --------
    >>>

    """

    __module__ = "compas_rhino.artists"

    def __init__(self, line, layer=None):
        super(LineArtist, self).__init__(line, layer=layer)
        self.settings.update({
            'color.line': (0, 0, 0)})

    def draw(self):
        """Draw the line.

        Returns
        -------
        guid: str
            The GUID of the created Rhino object.

        """
        start = list(self.primitive.start)
        end = list(self.primitive.end)
        lines = [{'start': start, 'end': end, 'color': self.settings['color.line']}]
        guids = compas_rhino.draw_lines(lines, layer=self.settings['layer'], clear=False)
        if guids:
            return guids[0]

    @staticmethod
    def draw_collection(collection, color=None, layer=None, clear=False, group_collection=False, group_name=None):
        """Draw a collection of lines.

        Parameters
        ----------
        collection: list of compas.geometry.Line
            A collection of ``Line`` objects.
        color: tuple or list of tuple (optional)
            Color specification of the lines.
            If one RGB color is provided, it will be applied to all lines.
            If a list of RGB colors is provided, these colors are applied to the corresponding lines.
            A list of colors should have the same length as the collection, with one color per item.
            Default value is ``None`` in which case the default line color of the artist is used.
        layer: str (optional)
            The layer in which the objects of the collection should be created.
            Default is ``None``, in which case the default layer setting of the artist is used.
        clear: bool (optional)
            Clear the layer before drawing.
            Default is ``False``.
        group_collection: bool (optional)
            Flag for grouping the objects of the collection.
            Default is ``False``.
        group_name: str (optional).
            The name of the group.
            Default is ``None``.

        Returns
        -------
        guids: list
            A list of GUIDs if the collection is not grouped.
        groupname: str
            The name of the group if the collection objects are grouped.

        """
        colors = list_like(collection, color)
        lines = []
        for line, rgb in zip(collection, colors):
            lines.append({
                'start': list(line.start),
                'end': list(line.end),
                'color': rgb})
        guids = compas_rhino.draw_lines(lines, layer=layer, clear=clear)
        if not group_collection:
            return guids
        group = compas_rhino.rs.AddGroup(group_name)
        if group:
            compas_rhino.rs.AddObjectsToGroup(guids, group)
        return group

# ==============================================================================
# Main
# ==============================================================================

if __name__ == "__main__":

    pass
<|endoftext|>


<commit_before>from zorg.adaptor import Adaptor
import urlparse
import os

# Check urllib for Python 2 or 3 compatability
try:
    from urllib import request as urllib_request
except ImportError:
    import urllib2 as urllib_request

class Camera(Adaptor):

    def __init__(self, options):
        super(Camera, self).__init__(options)

        self.url = options.get("url", "")
        self.cache_directory = options.get("cache_directory", ".cache")
        self.image_last_modified = None

    def download_image(self):
        """
        Download the image and return the
        local path to the image file.
        """
        split = urlparse.urlsplit(self.url)
        filename = split.path.split("/")[-1]

        # Ensure the directory to store the image cache exists
        if not os.path.exists(self.cache_directory):
            os.makedirs(self.cache_directory)

        filepath = os.path.join(self.cache_directory, filename)

        data = urllib_request.urlopen(self.url)
        with open(filepath, "wb") as image:
            image.write(data.read())

        return filepath

    def has_changed(self):
        """
        Method to check if an image has changed
        since it was last downloaded. By making
        a head request, this check can be done
        quicker that downloading and processing
        the whole file.
        """
        request = urllib_request.Request(self.url)
        request.get_method = lambda: 'HEAD'

        response = urllib_request.urlopen(request)
        information = response.info()

        if 'Last-Modified' in information:
            last_modified = information['Last-Modified']

            # Return False if the image has not been modified
            if last_modified == self.image_last_modified:
                return False

        self.image_last_modified = last_modified

        # Return True if the image has been modified
        # or if the image has no last-modified header
        return True
<commit_msg>Added urlparse to python 2 and 3 import check.<commit_after>from zorg.adaptor import Adaptor
import os

# Check urllib for Python 2 or 3 compatability
try:
    from urllib.parse import urlsplit
    from urllib import request as urllib_request
except ImportError:
    from urlparse import urlsplit
    import urllib2 as urllib_request

class Camera(Adaptor):

    def __init__(self, options):
        super(Camera, self).__init__(options)

        self.url = options.get("url", "")
        self.cache_directory = options.get("cache_directory", ".cache")
        self.image_last_modified = None

    def download_image(self):
        """
        Download the image and return the
        local path to the image file.
        """
        split = urlsplit(self.url)
        filename = split.path.split("/")[-1]

        # Ensure the directory to store the image cache exists
        if not os.path.exists(self.cache_directory):
            os.makedirs(self.cache_directory)

        filepath = os.path.join(self.cache_directory, filename)

        data = urllib_request.urlopen(self.url)
        with open(filepath, "wb") as image:
            image.write(data.read())

        return filepath

    def has_changed(self):
        """
        Method to check if an image has changed
        since it was last downloaded. By making
        a head request, this check can be done
        quicker that downloading and processing
        the whole file.
        """
        request = urllib_request.Request(self.url)
        request.get_method = lambda: 'HEAD'

        response = urllib_request.urlopen(request)
        information = response.info()

        if 'Last-Modified' in information:
            last_modified = information['Last-Modified']

            # Return False if the image has not been modified
            if last_modified == self.image_last_modified:
                return False

        self.image_last_modified = last_modified

        # Return True if the image has been modified
        # or if the image has no last-modified header
        return True
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from operator import itemgetter

character_frequency = {
    'e': 27,
    't': 26,
    'a': 25,
    'o': 24,
    'i': 23,
    'n': 22,
    's': 21,
    'r': 20,
    'h': 19,
    'l': 18,
    'd': 17,
    'c': 16,
    'u': 15,
    'm': 14,
    'f': 13,
    'p': 12,
    'g': 11,
    'w': 10,
    'y': 9,
    'b': 8,
    'v': 7,
    'k': 6,
    'x': 5,
    ' ': 4,
    'j': 3,
    'q': 2,
    'z': 1
}

def xor_singlechar(input_bytes, key_value):
    output = b''

    for char in input_bytes:
        output += bytes([char ^ key_value])

    return output

def xor_find_singlechar_key(ciphertext):
    candidates = list()

    for key_candidate in range(256):
        total_score = 0
        plaintext_candidate = xor_singlechar(ciphertext, key_candidate)

        for byte in plaintext_candidate:
            char_score = character_frequency.get(chr(byte), 0)
            total_score += char_score

        # We might as well include the plaintext in the output since
        # it’s used in some of the challenges.
        result = {
            'key': key_candidate,
            'score': total_score,
            'plaintext': plaintext_candidate
        }

        candidates.append(result)

    winner = max(candidates, key=itemgetter('score'))

    return winner

def popcount(x):
    """Return the number of set bits in an integer (a.k.a the Hamming weight).
    """

    return bin(x).count('1')

def hamming_distance(a, b):
    """Take two bytes objects and return the number of differing bits
    (a.k.a. the Hamming distance).
    """

    total_score = 0

    for (byte_a, byte_b) in zip(a, b):
        current_score = popcount(byte_a ^ byte_b)
        total_score += current_score

    return total_score
<commit_msg>Add the extend_buffer() function<commit_after>#!/usr/bin/env python3

from operator import itemgetter

character_frequency = {
    'e': 27,
    't': 26,
    'a': 25,
    'o': 24,
    'i': 23,
    'n': 22,
    's': 21,
    'r': 20,
    'h': 19,
    'l': 18,
    'd': 17,
    'c': 16,
    'u': 15,
    'm': 14,
    'f': 13,
    'p': 12,
    'g': 11,
    'w': 10,
    'y': 9,
    'b': 8,
    'v': 7,
    'k': 6,
    'x': 5,
    ' ': 4,
    'j': 3,
    'q': 2,
    'z': 1
}

def xor_singlechar(input_bytes, key_value):
    output = b''

    for char in input_bytes:
        output += bytes([char ^ key_value])

    return output

def xor_find_singlechar_key(ciphertext):
    candidates = list()

    for key_candidate in range(256):
        total_score = 0
        plaintext_candidate = xor_singlechar(ciphertext, key_candidate)

        for byte in plaintext_candidate:
            char_score = character_frequency.get(chr(byte), 0)
            total_score += char_score

        # We might as well include the plaintext in the output since
        # it’s used in some of the challenges.
        result = {
            'key': key_candidate,
            'score': total_score,
            'plaintext': plaintext_candidate
        }

        candidates.append(result)

    winner = max(candidates, key=itemgetter('score'))

    return winner

def popcount(x):
    """Return the number of set bits in an integer (a.k.a the Hamming weight).
    """

    return bin(x).count('1')

def hamming_distance(a, b):
    """Take two bytes objects and return the number of differing bits
    (a.k.a. the Hamming distance).
    """

    total_score = 0

    for (byte_a, byte_b) in zip(a, b):
        current_score = popcount(byte_a ^ byte_b)
        total_score += current_score

    return total_score

def extend_buffer(buffer, length):
    """Take a string or bytes object and repeat for "length" characters.
    """
    return (buffer*length)[:length]
<|endoftext|>


<commit_before>from datetime import date, datetime, time, timedelta
from zlib import crc32
from django.core.cache import cache
from django.db.models import Sum
from accounts.models import UserProfile
from race.models import Map, MapType, Run, BestRun, Server
from lib.chunks import chunks
from celery.task import task
from tml.tml import Teemap

@task(rate_limit='60/m', ignore_result=True)
def redo_ranks(run_id):
	logger = redo_ranks.get_logger()
	try:
		user_run = Run.objects.get(pk=run_id)
	except Run.DoesNotExist:
		logger.error("How is that possible? (Run doesn't exist)")
		return False
	map_obj = user_run.map
	user_best = BestRun.objects.get(map=map_obj, user=user_run.user)
	if not user_best.run_id == user_run.id:
		return
	runs = BestRun.objects.filter(map=map_obj)
	# ranked = player that receives points for his place
	ranked_count = len(BestRun.SCORING)
	# exclude anonymous and banned users from scoring
	ranked = runs.exclude(user__is_active=False)
	ranked = ranked.order_by('run__time')[:ranked_count]
	try:
		if user_run.time >= ranked[ranked_count-1].run.time:
			return
	except IndexError:
		pass
	i = 0
	for run in ranked:
		run.points = BestRun.SCORING[i]
		run.save()
		# FIXME it's 3 AM, sorry for that
		run.user.profile.points = BestRun.objects.filter(user=run.user).aggregate(
			Sum('points')
		)['points__sum']
		run.user.profile.save()
		i += 1
	runs.exclude(id__in=ranked.values_list('id', flat=True)).update(
		points=0
	)
	logger.info("Processed rank for \"{0}\" map.".format(map_obj))

@task(rate_limit='10/m', ignore_result=True)
def rebuild_map_rank(map_id):
	map_obj = Map.objects.get(pk=map_id)
	runs = BestRun.objects.filter(map=map_obj)
	# ranked = player that receives points for his place
	ranked_count = len(BestRun.SCORING)
	# exclude anonymous and banned users from scoring
	ranked = runs.exclude(user__is_active=False) \
		.order_by('run__time')[:ranked_count]
	i = 0
	for run in ranked:
		run.points = BestRun.SCORING[i]
		run.save()
		i += 1
	runs.exclude(id__in=ranked.values_list('id', flat=True)).update(
		points=0
	)
	logger = rebuild_map_rank.get_logger()
	logger.info("Rebuilt rank for map \"{0}\".".format(map_obj.name))

@task(rate_limit='1/m', ignore_result=True)
def rebuild_global_rank():
	runners = User.objects.annotate(Sum('bestrun__points'))
	for runner in runners:
		runner.points = runner.bestrun__points__sum
		runner.save()
	logger = rebuild_global_rank.get_logger()
	logger.info("Rebuilt global rank.")

class index_factory(object):
	INDEXES = dict(
		DEATHTILE = 2,
		UNHOOKABLE = 3,
		RED_FLAG = 195,
		BLUE_FLAG = 196,
		SHIELD = 197,
		HEART = 198,
		GRENADE = 200,
	)

	def __getattr__(self, attr):
		try:
			return self.INDEXES.get(attr)
		except TypeError:
			raise AttributeError(attr)
indexes = index_factory()

@task(ignore_result=True)
def retrieve_map_details(map_id):
	"""
	WARNING!
	This task is CPU/(and highly) RAM expensive.

	Checks for presence of specified tiles, counts some of them
	and at the end takes a beautiful photo of the map.

	Thanks for your help, erdbeere!
	"""
	logger = retrieve_map_details.get_logger()
	try:
		map_obj = Map.objects.get(pk=map_id)
	except Map.DoesNotExist:
		logger.error("Are you kidding me? (map doesn't exist)")
		return False

	# I actually can use that! Thanks, erd!
	teemap = Teemap().load(map_obj.map_file.path)
	logger.info("Loaded \"{0}\" map.".format(map_obj.name))

	logger.info("Counting map items...")
	has_unhookables = has_deathtiles = is_fastcap = False
	shield_count = heart_count = grenade_count = 0
	for tile in teemap.gamelayer.tiles:
		if tile.index == indexes.DEATHTILE:
			has_deathtiles = True
		elif tile.index == indexes.UNHOOKABLE:
			has_unhookables = True
		elif tile.index in (indexes.RED_FLAG, indexes.BLUE_FLAG):
			is_fastcap = True
		elif tile.index == indexes.SHIELD:
			shield_count += 1
		elif tile.index == indexes.HEART:
			heart_count += 1
		elif tile.index == indexes.GRENADE:
			grenade_count += 1
	map_obj.has_unhookables = has_unhookables
	map_obj.has_deathtiles = has_deathtiles
	map_obj.shield_count = shield_count
	map_obj.heart_count = heart_count
	map_obj.grenade_count = grenade_count
	logger.info("Finished counting map items.")

	# DISABLED due to huge (counted in GiBs) memory usage
	# logger.info("Rendering map screenshot.")
	# map_image = teemap.render(gamelayer_on_top=True)
	# map_image.save('{0}images/maps/full/{1}.png'.format(settings.MEDIA_ROOT,
	# 	map_obj.name))
	# logger.info("Finished rendering map screenshot.")
	# map_obj.has_image = True

	logger.info("Generating map CRC...")
	map_obj.crc = '{0:x}'.format(crc32(map_obj.map_file.read()) & 0xffffffff)
	map_obj.map_file.close()

	if is_fastcap:
		logger.info("Turning map into fastcap type...")
		map_obj.map_type = MapType.objects.get(slug='fastcap')
		logger.info("Creating a non-weapon twin...")
		new_map = Map(
			name="{0}-no-weapons".format(map_obj.name),
			author=map_obj.author,
			added_by=map_obj.added_by,
			map_file=map_obj.map_file,
			crc=map_obj.crc,
			map_type=MapType.objects.get(slug='fastcap-no-weapons'),
			has_unhookables=has_unhookables,
			has_deathtiles=has_deathtiles,
		)
		new_map.save()
	map_obj.save()
	logger.info("Finished processing \"{0}\" map.".format(map_obj.name))

@task(ignore_result=True)
def set_server_map(server_id, map_id):
	try:
		map_obj = Map.objects.get(pk=map_id)
		server = Server.objects.get(pk=server_id)
	except (Map.DoesNotExist, Server.DoesNotExist):
		return False
	server.played_map = map_obj
	server.save()

@task(ignore_result=True)
def update_user_points_history():
	# everyday, around 4:30 AM
	users = UserProfile.objects.values_list('id', flat=True)
	for chunk in list(chunks(users, 200)):
		update_user_points_history_chunked.delay(chunk)

@task(ignore_result=True)
def update_user_points_history_chunked(chunk):
	today = date.today()
	for user_id in chunk:
		user = UserProfile.objects.get(pk=user_id)
		user.yesterday_points = user.points
		if not user.points_history:
			# let's make a list
			user.points_history = []
		user.points_history.append((today, user.points))
		user.save()

@task(ignore_result=True)
def update_yesterday_runs():
	# everyday, around 0:00 AM
	yesterday = datetime.today() - timedelta(days=1)
	runs_yesterday = Run.objects.filter(created_at__range=
		(datetime.combine(yesterday, time.min),
		datetime.combine(yesterday, time.max)))
	cache.set('runs_yesterday', runs_yesterday, 0)

@task(ignore_result=True)
def update_totals():
	# every 15 minutes
	total_runs = Run.objects.count()
	total_playtime = Run.objects.aggregate(Sum('time'))['time__sum']
	cache.set('total_runs', total_runs, 0)
	cache.set('total_playtime', total_playtime, 0)
<commit_msg>changed, so points history task will write to database only once a day<commit_after>from datetime import date, datetime, time, timedelta
from zlib import crc32
from django.core.cache import cache
from django.db.models import Sum
from accounts.models import UserProfile
from race.models import Map, MapType, Run, BestRun, Server
from lib.chunks import chunks
from celery.task import task
from tml.tml import Teemap

@task(rate_limit='60/m', ignore_result=True)
def redo_ranks(run_id):
	logger = redo_ranks.get_logger()
	try:
		user_run = Run.objects.get(pk=run_id)
	except Run.DoesNotExist:
		logger.error("How is that possible? (Run doesn't exist)")
		return False
	map_obj = user_run.map
	user_best = BestRun.objects.get(map=map_obj, user=user_run.user)
	if not user_best.run_id == user_run.id:
		return
	runs = BestRun.objects.filter(map=map_obj)
	# ranked = player that receives points for his place
	ranked_count = len(BestRun.SCORING)
	# exclude anonymous and banned users from scoring
	ranked = runs.exclude(user__is_active=False)
	ranked = ranked.order_by('run__time')[:ranked_count]
	try:
		if user_run.time >= ranked[ranked_count-1].run.time:
			return
	except IndexError:
		pass
	i = 0
	for run in ranked:
		run.points = BestRun.SCORING[i]
		run.save()
		# FIXME it's 3 AM, sorry for that
		run.user.profile.points = BestRun.objects.filter(user=run.user).aggregate(
			Sum('points')
		)['points__sum']
		run.user.profile.save()
		i += 1
	runs.exclude(id__in=ranked.values_list('id', flat=True)).update(
		points=0
	)
	logger.info("Processed rank for \"{0}\" map.".format(map_obj))

@task(rate_limit='10/m', ignore_result=True)
def rebuild_map_rank(map_id):
	map_obj = Map.objects.get(pk=map_id)
	runs = BestRun.objects.filter(map=map_obj)
	# ranked = player that receives points for his place
	ranked_count = len(BestRun.SCORING)
	# exclude anonymous and banned users from scoring
	ranked = runs.exclude(user__is_active=False) \
		.order_by('run__time')[:ranked_count]
	i = 0
	for run in ranked:
		run.points = BestRun.SCORING[i]
		run.save()
		i += 1
	runs.exclude(id__in=ranked.values_list('id', flat=True)).update(
		points=0
	)
	logger = rebuild_map_rank.get_logger()
	logger.info("Rebuilt rank for map \"{0}\".".format(map_obj.name))

@task(rate_limit='1/m', ignore_result=True)
def rebuild_global_rank():
	runners = User.objects.annotate(Sum('bestrun__points'))
	for runner in runners:
		runner.points = runner.bestrun__points__sum
		runner.save()
	logger = rebuild_global_rank.get_logger()
	logger.info("Rebuilt global rank.")

class index_factory(object):
	INDEXES = dict(
		DEATHTILE = 2,
		UNHOOKABLE = 3,
		RED_FLAG = 195,
		BLUE_FLAG = 196,
		SHIELD = 197,
		HEART = 198,
		GRENADE = 200,
	)

	def __getattr__(self, attr):
		try:
			return self.INDEXES.get(attr)
		except TypeError:
			raise AttributeError(attr)
indexes = index_factory()

@task(ignore_result=True)
def retrieve_map_details(map_id):
	"""
	WARNING!
	This task is CPU/(and highly) RAM expensive.

	Checks for presence of specified tiles, counts some of them
	and at the end takes a beautiful photo of the map.

	Thanks for your help, erdbeere!
	"""
	logger = retrieve_map_details.get_logger()
	try:
		map_obj = Map.objects.get(pk=map_id)
	except Map.DoesNotExist:
		logger.error("Are you kidding me? (map doesn't exist)")
		return False

	# I actually can use that! Thanks, erd!
	teemap = Teemap().load(map_obj.map_file.path)
	logger.info("Loaded \"{0}\" map.".format(map_obj.name))

	logger.info("Counting map items...")
	has_unhookables = has_deathtiles = is_fastcap = False
	shield_count = heart_count = grenade_count = 0
	for tile in teemap.gamelayer.tiles:
		if tile.index == indexes.DEATHTILE:
			has_deathtiles = True
		elif tile.index == indexes.UNHOOKABLE:
			has_unhookables = True
		elif tile.index in (indexes.RED_FLAG, indexes.BLUE_FLAG):
			is_fastcap = True
		elif tile.index == indexes.SHIELD:
			shield_count += 1
		elif tile.index == indexes.HEART:
			heart_count += 1
		elif tile.index == indexes.GRENADE:
			grenade_count += 1
	map_obj.has_unhookables = has_unhookables
	map_obj.has_deathtiles = has_deathtiles
	map_obj.shield_count = shield_count
	map_obj.heart_count = heart_count
	map_obj.grenade_count = grenade_count
	logger.info("Finished counting map items.")

	# DISABLED due to huge (counted in GiBs) memory usage
	# logger.info("Rendering map screenshot.")
	# map_image = teemap.render(gamelayer_on_top=True)
	# map_image.save('{0}images/maps/full/{1}.png'.format(settings.MEDIA_ROOT,
	# 	map_obj.name))
	# logger.info("Finished rendering map screenshot.")
	# map_obj.has_image = True

	logger.info("Generating map CRC...")
	map_obj.crc = '{0:x}'.format(crc32(map_obj.map_file.read()) & 0xffffffff)
	map_obj.map_file.close()

	if is_fastcap:
		logger.info("Turning map into fastcap type...")
		map_obj.map_type = MapType.objects.get(slug='fastcap')
		logger.info("Creating a non-weapon twin...")
		new_map = Map(
			name="{0}-no-weapons".format(map_obj.name),
			author=map_obj.author,
			added_by=map_obj.added_by,
			map_file=map_obj.map_file,
			crc=map_obj.crc,
			map_type=MapType.objects.get(slug='fastcap-no-weapons'),
			has_unhookables=has_unhookables,
			has_deathtiles=has_deathtiles,
		)
		new_map.save()
	map_obj.save()
	logger.info("Finished processing \"{0}\" map.".format(map_obj.name))

@task(ignore_result=True)
def set_server_map(server_id, map_id):
	try:
		map_obj = Map.objects.get(pk=map_id)
		server = Server.objects.get(pk=server_id)
	except (Map.DoesNotExist, Server.DoesNotExist):
		return False
	server.played_map = map_obj
	server.save()

@task(ignore_result=True)
def update_user_points_history():
	# everyday, around 4:30 AM
	users = UserProfile.objects.values_list('id', flat=True)
	for chunk in list(chunks(users, 200)):
		update_user_points_history_chunked.delay(chunk)

@task(ignore_result=True)
def update_user_points_history_chunked(chunk):
	today = date.today()
	for user_id in chunk:
		user = UserProfile.objects.get(pk=user_id)
		if not user.points_history:
			# let's make a list
			user.points_history = []
		elif user.points_history[-1][0] == today:
			# making sure we save only one time per day
			continue
		user.yesterday_points = user.points
		user.points_history.append((today, user.points))
		user.save()

@task(ignore_result=True)
def update_yesterday_runs():
	# everyday, around 0:00 AM
	yesterday = datetime.today() - timedelta(days=1)
	runs_yesterday = Run.objects.filter(created_at__range=
		(datetime.combine(yesterday, time.min),
		datetime.combine(yesterday, time.max)))
	cache.set('runs_yesterday', runs_yesterday, 0)

@task(ignore_result=True)
def update_totals():
	# every 15 minutes
	total_runs = Run.objects.count()
	total_playtime = Run.objects.aggregate(Sum('time'))['time__sum']
	cache.set('total_runs', total_runs, 0)
	cache.set('total_playtime', total_playtime, 0)
<|endoftext|>


<commit_before>import numpy as np
import matplotlib.pyplot as plt
import triangle

"""Classes and methods for a Dataset of stars.

Provides the ability to initialize the dataset, modify it by adding or
removing spectra, changing label names, adding or removing labels.

Methods
-------
remove_stars

"""

class Dataset(object):
    """A class to represent a Dataset of stellar spectra and labels.

    Parameters
    ----------
    IDs: numpy ndarray, list
        Specify the names (or IDs, in whatever format) of the stars.
    spectra: numpy ndarray
        spectra[:,:,0] = wavelengths, pixel values
        spectra[:,:,1] = flux (spectrum)
        spectra[:,:,2] = flux error array
    labels: numpy ndarray, list, optional
        Training labels for training set, but None for test set

    Methods
    -------
    set_IDs
    set_spectra
    set_label_names
    set_label_values

    """

    def __init__(self, IDs, SNRs, spectra, label_names, label_values=None):
        self.IDs = IDs
        self.SNRs = SNRs
        self.spectra = spectra
        self.label_names = label_names
        self.label_values = label_values

    def set_IDs(self, IDs):
        self.IDs = IDs

    def set_spectra(self, spectra):
        self.spectra = spectra

    def set_label_names(self, label_names):
        self.label_names = label_names

    def set_label_values(self, label_values):
        self.label_values = label_values

def training_set_diagnostics(dataset):
    # Plot SNR distribution
    plt.hist(dataset.SNRs)
    plt.title("Distribution of SNR in the Training Set")
    figname = "trainingset_SNRdist.png"
    plt.savefig(figname)
    plt.close()
    print "Diagnostic for SNR of training set"
    print "Saved fig %s" %figname
    # Plot training label distributions
    for i in range(0, len(dataset.label_names)):
        name = dataset.label_names[i]
        vals = dataset.label_values[:,i]
        plt.hist(vals)
        # Note: label names cannot have slashes
        plt.title("Training Set Distribution of Label: %s" %name)
        figname = "trainingset_labeldist_%s.png" %name
        plt.savefig(figname)
        print "Diagnostic for coverage of training label space"
        print "Saved fig %s" %figname
        plt.close()
    # Plot all training labels against each other
    fig = triangle.corner(dataset.label_values, labels=dataset.label_names,
            show_titles=True, title_args = {"fontsize": 12})
    figname = "trainingset_labels_triangle.png"
    fig.savefig(figname)
    print "Diagnostic for plotting every training label against every other"
    print "Saved fig %s" %figname

def test_set_diagnostics(training_set, test_set):
    label_names = training_set.label_names
    nlabels = len(label_names)
    training_labels = training_set.label_values
    test_IDs = test_set.IDs
    mean = np.mean(training_labels, 0)
    stdev = np.std(training_labels, 0)
    lower = mean - 2 * stdev
    upper = mean + 2 * stdev
    for i in range(nlabels):
        label_name = label_names[i]
        test_vals = test_set.label_values[:,i]
        warning = np.logical_or(test_vals < lower[i], test_vals > upper[i])
        flagged_stars = test_IDs[warning]
        filename = "flagged_stars_%s.txt" %label_name
        output = open(filename, 'w')
        for star in test_IDs[warning]:
            output.write(star + '\n')
        output.close()
        print "Training label %s" %label_name
        print "flagged %s stars beyond 2-sig of training labels" %sum(warning)
        print "Saved list %s" %filename

def remove_stars(dataset, mask):
    """A method to remove a subset of stars from the Dataset.

    Parameters
    ---------
    mask: numpy ndarray of booleans. True means "remove"
    """
    ntokeep = 1-np.sum(mask)
    return Dataset(dataset.IDs[mask], dataset.spectra[mask],
            dataset.label_names, dataset.label_values[mask])
    print "New dataset constructed with %s stars included" %ntokeep
<commit_msg>triangle plot for Cannon output labels<commit_after>import numpy as np
import matplotlib.pyplot as plt
import triangle

"""Classes and methods for a Dataset of stars.

Provides the ability to initialize the dataset, modify it by adding or
removing spectra, changing label names, adding or removing labels.

Methods
-------
remove_stars

"""

class Dataset(object):
    """A class to represent a Dataset of stellar spectra and labels.

    Parameters
    ----------
    IDs: numpy ndarray, list
        Specify the names (or IDs, in whatever format) of the stars.
    spectra: numpy ndarray
        spectra[:,:,0] = wavelengths, pixel values
        spectra[:,:,1] = flux (spectrum)
        spectra[:,:,2] = flux error array
    labels: numpy ndarray, list, optional
        Training labels for training set, but None for test set

    Methods
    -------
    set_IDs
    set_spectra
    set_label_names
    set_label_values

    """

    def __init__(self, IDs, SNRs, spectra, label_names, label_values=None):
        self.IDs = IDs
        self.SNRs = SNRs
        self.spectra = spectra
        self.label_names = label_names
        self.label_values = label_values

    def set_IDs(self, IDs):
        self.IDs = IDs

    def set_spectra(self, spectra):
        self.spectra = spectra

    def set_label_names(self, label_names):
        self.label_names = label_names

    def set_label_values(self, label_values):
        self.label_values = label_values

def training_set_diagnostics(dataset):
    # Plot SNR distribution
    plt.hist(dataset.SNRs)
    plt.title("Distribution of SNR in the Training Set")
    figname = "trainingset_SNRdist.png"
    plt.savefig(figname)
    plt.close()
    print "Diagnostic for SNR of training set"
    print "Saved fig %s" %figname
    # Plot training label distributions
    for i in range(0, len(dataset.label_names)):
        name = dataset.label_names[i]
        vals = dataset.label_values[:,i]
        plt.hist(vals)
        # Note: label names cannot have slashes
        plt.title("Training Set Distribution of Label: %s" %name)
        figname = "trainingset_labeldist_%s.png" %name
        plt.savefig(figname)
        print "Diagnostic for coverage of training label space"
        print "Saved fig %s" %figname
        plt.close()
    # Plot all training labels against each other
    fig = triangle.corner(dataset.label_values, labels=dataset.label_names,
            show_titles=True, title_args = {"fontsize": 12})
    figname = "trainingset_labels_triangle.png"
    fig.savefig(figname)
    print "Diagnostic for plotting every training label against every other"
    print "Saved fig %s" %figname

def test_set_diagnostics(training_set, test_set):
    # 2-sigma check from training labels
    label_names = training_set.label_names
    nlabels = len(label_names)
    training_labels = training_set.label_values
    test_IDs = test_set.IDs
    mean = np.mean(training_labels, 0)
    stdev = np.std(training_labels, 0)
    lower = mean - 2 * stdev
    upper = mean + 2 * stdev
    for i in range(nlabels):
        label_name = label_names[i]
        test_vals = test_set.label_values[:,i]
        warning = np.logical_or(test_vals < lower[i], test_vals > upper[i])
        flagged_stars = test_IDs[warning]
        filename = "flagged_stars_%s.txt" %label_name
        output = open(filename, 'w')
        for star in test_IDs[warning]:
            output.write(star + '\n')
        output.close()
        print "Training label %s" %label_name
        print "flagged %s stars beyond 2-sig of training labels" %sum(warning)
        print "Saved list %s" %filename
    # Plot all output labels against each other
    fig = triangle.corner(test_set.label_values, labels=test_set.label_names,
            show_titles=True, title_args = {"fontsize": 12})
    figname = "testset_labels_triangle.png"
    fig.savefig(figname)
    print "Diagnostic for plotting every Cannon label against every other"
    print "Saved fig %s" %figname

def remove_stars(dataset, mask):
    """A method to remove a subset of stars from the Dataset.

    Parameters
    ---------
    mask: numpy ndarray of booleans. True means "remove"
    """
    ntokeep = 1-np.sum(mask)
    return Dataset(dataset.IDs[mask], dataset.spectra[mask],
            dataset.label_names, dataset.label_values[mask])
    print "New dataset constructed with %s stars included" %ntokeep
<|endoftext|>


<commit_before># coding: utf-8

import logging
import urllib

import config
from config import WHITE_LIST, IDENTIFIER, TRANSPORT_ID, LOGO_URL
from friends import get_friend_jid
from library.xmpp.protocol import (NodeProcessed, NS_REGISTER, NS_CAPTCHA, NS_GATEWAY,
                                   NS_DISCO_ITEMS, NS_DISCO_INFO, NS_VCARD, NS_PING, ERR_FEATURE_NOT_IMPLEMENTED,
                                   Error, NS_DATA, ERR_BAD_REQUEST, Node, ERR_REGISTRATION_REQUIRED,)

import library.xmpp.simplexml

from sender import stanza_send
from handler import Handler
from messaging import send_watcher_message
from captcha import captcha_accept
import user as user_api
from errors import AuthenticationException
import database
import forms
from singletone import Gateway

logger = logging.getLogger("vk4xmpp")

class IQ(object):
    def __init__(self, iq):
        self.jid_from = iq.getFrom()
        self.jid_to = iq.getTo()
        self.jid_from_str = self.jid_from.getStripped()
        self.jid_to_str = self.jid_to.getStripped()
        self.i_type = iq.getType()
        self.ns = iq.getQueryNS()
        self.node = iq.getTagAttr("query", "node")
        self.result = iq.buildReply("result")
        # self.result_str = self.result.getStripped()

    def __repr__(self):
        return '<IQ %s->%s>' % (self.jid_from_str, self.jid_to_str)

def generate_error(stanza, error=None, text=None):
    if not error:
        error = ERR_FEATURE_NOT_IMPLEMENTED
    error = Error(stanza, error, True)
    if text:
        etag = error.getTag("error")
        etag.setTagData("text", text)
    return error

def _send_form(_, iq, jid):
    logger.debug("sending register form to %s" % jid)
    result = iq.buildReply("result")
    result.setQueryPayload((forms.get_form(),))
    return result

def _process_form(gateway, iq, jid):
    logger.debug('received register form from %s' % jid)

    assert isinstance(gateway, Gateway)
    assert isinstance(jid, unicode)

    result = iq.buildReply("result")

    query = iq.getTag("query")

    token = query.getTag("x").getTag("field", {"var": "password"}).getTagData("value")

    try:
        token = token.split("#access_token=")[1].split("&")[0].strip()
    except (IndexError, AttributeError) as e:
        logger.debug('access token in raw format')

    logger.debug('form processed')

    user_attributes = database.get_description(jid)

    if not user_attributes:
        logger.debug('user %s is not in database' % jid)
        database.insert_user(jid, None, token, None, False)
        # user = TUser(self.gateway, token, jid)
        # user.token = token
    # if not database.is_client(jid):
    #     logger.debug('user %s is not in client list' % jid)
    #
    #     user.token = token
    else:
        raise NotImplementedError('already in database')

    try:
        user_api.connect(gateway, jid, token)
        user_api.initialize(gateway, jid)
    except AuthenticationException:
        logger.error("user %s connection failed (from iq)" % jid)
        result = generate_error(iq, ERR_BAD_REQUEST, "Incorrect password or access token")

    database.set_last_activity_now(jid)
    gateway.add_user(jid)
    send_watcher_message(gateway.component, "new user registered: %s" % jid)
    logger.debug('registration for %s completed' % jid)

    return result

class IQHandler(Handler):
    # sDict = {
    #     "users/total": "users",
    #     "users/online": "users",
    #     "memory/virtual": "KB",
    #     "memory/real": "KB",
    #     "cpu/percent": "percent",
    #     "cpu/time": "seconds"
    # }

    def __init__(self, gateway):
        super(IQHandler, self).__init__(gateway)

    def handle(self, transport, stanza):
        jid_from = stanza.getFrom()
        jid_from_str = jid_from.getStripped()
        jid_to = stanza.getTo()

        if WHITE_LIST and jid_from and jid_from.getDomain() not in WHITE_LIST:
            stanza_send(transport, generate_error(stanza, ERR_BAD_REQUEST, "You are not in white list"))
            raise NodeProcessed()

        from_is_client = database.is_client(jid_from_str)
        destination_is_transport = jid_to == config.TRANSPORT_ID
        ns_is_captcha = stanza.getTagAttr("captcha", "xmlns") == NS_CAPTCHA

        if from_is_client and destination_is_transport and ns_is_captcha:
            c_tag = stanza.getTag("captcha")
            cx_tag = c_tag.getTag("x", {}, NS_DATA)
            fcx_tag = cx_tag.getTag("field", {"var": "ocr"})
            c_value = fcx_tag.getTagData("value")
            captcha_accept(self.gateway, transport, c_value, jid_to, jid_from_str)

        ns = stanza.getQueryNS()

        mapping = {
            NS_REGISTER: self.iq_register_handler,
            NS_GATEWAY: self.iq_gateway_handler,
            NS_DISCO_INFO: self.iq_disco_handler,
            NS_DISCO_ITEMS: self.iq_disco_handler
        }

        try:
            mapping[ns](transport, stanza)
        except KeyError:
            tag = stanza.getTag("vCard") or stanza.getTag("ping")
            if tag and tag.getNamespace() == NS_VCARD:
                self.iq_vcard_handler(transport, stanza)
            elif tag and tag.getNamespace() == NS_PING:
                if jid_to == config.TRANSPORT_ID:
                    stanza_send(transport, stanza.buildReply("result"))

        raise NodeProcessed()

    def iq_register_handler(self, transport, stanza):
        jid = stanza.getFrom().getStripped()
        logger.debug('register handler for %s' % jid)

        destination_jid = stanza.getTo().getStripped()

        logger.debug('destination %s' % destination_jid)
        # iq_type = iq.getType()
        # iq_children = iq.getQueryChildren()
        # result = iq.buildReply("result")
        # if config.USER_LIMIT:
        #     count = self.calc_stats()[0]
        #     if count >= config.USER_LIMIT and not jid_from_str in self.clients:
        #         cl.send(self.iq_build_error(iq, xmpp.ERR_NOT_ALLOWED,
        #                                     _("Transport's admins limited registrations, sorry.")))
        #         raise xmpp.NodeProcessed

        if destination_jid != config.TRANSPORT_ID:
            logger.debug('register not to transport')
            return

        gateway = self.gateway

        try:
            handler = {'get': _send_form, 'set': _process_form}[stanza.getType()]
            stanza_send(transport, handler(gateway, stanza, jid))
        except (NotImplementedError, KeyError) as e:
            stanza_send(transport, generate_error(stanza, 0, "Requested feature not implemented: %s" % e))

    # def calc_stats(self):
    #     count_total = 0
    #     count_online = 0
    #     # TODO: Semaphore
    #     with Database(config.DB_FILE) as db:
    #         db("select count(*) from users")
    #         count_total = db.fetchone()[0]
    #     for key in self.clients:
    #         if hasattr(key, "vk") and key.vk.Online:
    #             count_online += 1
    #     return [count_total, count_online]

    # def iq_uptime_handler(self, cl, iq):
    #     jidFrom = iq.getFrom()
    #     jidTo = iq.getTo()
    #     iType = iq.getType()
    #     if iType == "get" and jidTo == config.TRANSPORT_ID:
    #         uptime = int(time.time() - self.gateway.start_time)
    #         result = xmpp.Iq("result", to=jidFrom)
    #         result.setID(iq.getID())
    #         result.setTag("query", {"seconds": str(uptime)}, xmpp.NS_LAST)
    #         result.setTagData("query", config.IDENTIFIER["name"])
    #         stanza_send(cl, result)
    #     raise xmpp.NodeProcessed()

    #
    # def iqVersionHandler(self, cl, iq):
    #     iType = iq.getType()
    #     jidTo = iq.getTo()
    #     if iType == "get" and jidTo == config.TRANSPORT_ID:
    #         result = iq.buildReply("result")
    #         Query = result.getTag("query")
    #         Query.setTagData("name", config.IDENTIFIER["name"])
    #         os_name = "{0} {2:.16} [{4}]".format(*os.uname())
    #         python_version = "{0} {1}.{2}.{3}".format(sys.subversion[0], *sys.version_info)
    #         # TODO: WTF
    #         Query.setTagData("version", 666)
    #         Query.setTagData("os", "%s / %s" % (os_name, python_version))
    #         stanza_send(cl, result)
    #     raise xmpp.NodeProcessed()

    # def iqStatsHandler(self, cl, iq):
    #     jid_to_str = iq.getTo()
    #     i_type = iq.getType()
    #     iq_children = iq.getQueryChildren()
    #     result = iq.buildReply("result")
    #     if i_type == "get" and jid_to_str == config.TRANSPORT_ID:
    #         querypayload = list()
    #         if not iq_children:
    #             keys = sorted(self.sDict.keys(), reverse=True)
    #             for key in keys:
    #                 Node = xmpp.Node("stat", {"name": key})
    #                 querypayload.append(Node)
    #         else:
    #             users = self.calc_stats()
    #             shell = os.popen("ps -o vsz,rss,%%cpu,time -p %s" % os.getpid()).readlines()
    #             memVirt, memReal, cpuPercent, cpuTime = shell[1].split()
    #             stats = {"users": users, "KB": [memVirt, memReal],
    #                      "percent": [cpuPercent], "seconds": [cpuTime]}
    #             for Child in iq_children:
    #                 if Child.getName() != "stat":
    #                     continue
    #                 name = Child.getAttr("name")
    #                 if name in self.sDict:
    #                     attr = self.sDict[name]
    #                     value = stats[attr].pop(0)
    #                     Node = xmpp.Node("stat", {"units": attr})
    #                     Node.setAttr("name", name)
    #                     Node.setAttr("value", value)
    #                     querypayload.append(Node)
    #         if querypayload:
    #             result.setQueryPayload(querypayload)
    #             stanza_send(cl, result)

    def iq_disco_handler(self, transport, iq_raw):
        logger.debug('handling disco')

        iq = IQ(iq_raw)

        if iq.i_type == "get" and not iq.node and iq.jid_to_str == TRANSPORT_ID:
            query = [Node("identity", IDENTIFIER)]
            result = iq_raw.buildReply("result")
            if iq.ns == NS_DISCO_INFO:
                for key in self.gateway.features:
                    node = Node("feature", {"var": key})
                    query.append(node)
                result.setQueryPayload(query)
            elif iq.ns == NS_DISCO_ITEMS:
                result.setQueryPayload(query)
            stanza_send(transport, str(result))
        raise NodeProcessed()

    @staticmethod
    def iq_gateway_handler(cl, iq):

        jid_to = iq.getTo()
        i_type = iq.getType()
        jid_to_str = jid_to.getStripped()
        iq_children = iq.getQueryChildren()

        if jid_to_str == config.TRANSPORT_ID:
            result = iq.buildReply("result")
            if i_type == "get" and not iq_children:
                query = Node("query", {"xmlns": NS_GATEWAY})
                query.setTagData("desc", "Enter phone number")
                query.setTag("prompt")
                result.setPayload([query])

            elif iq_children and i_type == "set":
                phone = ""
                for node in iq_children:
                    if node.getName() == "prompt":
                        phone = node.getData()
                        break
                if phone:
                    x_node = library.xmpp.simplexml.Node("prompt")
                    x_node.setData(phone[0])
                    result.setQueryPayload([x_node])
            else:
                raise NodeProcessed()
            stanza_send(cl, result)

    @staticmethod
    def vcard_get_photo(url, encode=True):
        logger.debug('vcard_get_photo')
        try:
            opener = urllib.urlopen(url)
            data = opener.read()
            if data and encode:
                data = data.encode("base64")
            return data
        except IOError:
            logger.debug('IO error while getting photo')
            pass

    def iq_vcard_build(self, tags):
        logger.debug('iq_vcard_build')
        vcard = Node("vCard", {"xmlns": NS_VCARD})
        for key in tags.keys():
            if key == "PHOTO":
                bval = vcard.setTag("PHOTO")
                bval.setTagData("BINVAL", self.vcard_get_photo(tags[key]))
            else:
                vcard.setTagData(key, tags[key])
        return vcard

    def iq_vcard_handler(self, cl, iq):
        logger.debug('iq_vcard_handler')
        jid_from = iq.getFrom()
        jid_to = iq.getTo()
        jid = jid_from.getStripped()
        jid_to_str = jid_to.getStripped()
        i_type = iq.getType()
        result = iq.buildReply("result")
        if i_type == "get":
            _DESC = '\n'.join(
                (config.DESC, "_" * 16, config.ADDITIONAL_ABOUT)) if config.ADDITIONAL_ABOUT else config.DESC
            if jid_to_str == config.TRANSPORT_ID:
                vcard = self.iq_vcard_build({"NICKNAME": "VK4XMPP Transport",
                                             "DESC": _DESC,
                                             "PHOTO": LOGO_URL,
                                             "URL": "http://simpleapps.ru"})
                result.setPayload([vcard])
            elif database.is_client(jid):
                friends = database.get_friends(jid)
                if friends:
                    friend_jid = get_friend_jid(jid_to_str, jid)
                    json = user_api.get_user_data(jid, friend_jid, ["screen_name", config.PHOTO_SIZE])
                    values = {"NICKNAME": json.get("name", str(json)),
                              "URL": "http://vk.com/id%s" % friend_jid,
                              "DESC": "Contact uses VK4XMPP Transport\n%s" % _DESC}
                    if friend_jid in friends:
                        values["PHOTO"] = json.get(config.PHOTO_SIZE) or config.URL_VCARD_NO_IMAGE
                    vcard = self.iq_vcard_build(values)
                    result.setPayload([vcard])
                else:
                    result = self.iq_build_error(iq, ERR_BAD_REQUEST, "Your friend-list is null.")
            else:
                result = self.iq_build_error(iq, ERR_REGISTRATION_REQUIRED,
                                             "You're not registered for this action.")
        else:
            raise NodeProcessed()
        stanza_send(cl, result)

def get_handler(gateway):
    return IQHandler(gateway).handle<commit_msg>refactor<commit_after># coding: utf-8

import logging
import urllib

import config
from config import WHITE_LIST, IDENTIFIER, TRANSPORT_ID, LOGO_URL
from friends import get_friend_jid
from library.xmpp.protocol import (NodeProcessed, NS_REGISTER, NS_CAPTCHA, NS_GATEWAY,
                                   NS_DISCO_ITEMS, NS_DISCO_INFO, NS_VCARD, NS_PING, ERR_FEATURE_NOT_IMPLEMENTED,
                                   Error, NS_DATA, ERR_BAD_REQUEST, Node, ERR_REGISTRATION_REQUIRED,)

import library.xmpp.simplexml

from sender import stanza_send
from handler import Handler
from messaging import send_watcher_message
from captcha import captcha_accept
import user as user_api
from errors import AuthenticationException
import database
import forms
from singletone import Gateway

logger = logging.getLogger("vk4xmpp")

class IQ(object):
    def __init__(self, iq):
        self.jid_from = iq.getFrom()
        self.jid_to = iq.getTo()
        self.jid_from_str = self.jid_from.getStripped()
        self.jid_to_str = self.jid_to.getStripped()
        self.i_type = iq.getType()
        self.ns = iq.getQueryNS()
        self.node = iq.getTagAttr("query", "node")
        self.result = iq.buildReply("result")
        # self.result_str = self.result.getStripped()

    def __repr__(self):
        return '<IQ %s->%s>' % (self.jid_from_str, self.jid_to_str)

def generate_error(stanza, error=None, text=None):
    if not error:
        error = ERR_FEATURE_NOT_IMPLEMENTED
    error = Error(stanza, error, True)
    if text:
        etag = error.getTag("error")
        etag.setTagData("text", text)
    return error

def _send_form(_, iq, jid):
    logger.debug("sending register form to %s" % jid)
    result = iq.buildReply("result")
    result.setQueryPayload((forms.get_form(),))
    return result

def _process_form(gateway, iq, jid):
    logger.debug('received register form from %s' % jid)

    assert isinstance(gateway, Gateway)
    assert isinstance(jid, unicode)

    result = iq.buildReply("result")

    query = iq.getTag("query")

    token = query.getTag("x").getTag("field", {"var": "password"}).getTagData("value")

    try:
        token = token.split("#access_token=")[1].split("&")[0].strip()
    except (IndexError, AttributeError) as e:
        logger.debug('access token in raw format')

    logger.debug('form processed')

    user_attributes = database.get_description(jid)

    if not user_attributes:
        logger.debug('user %s is not in database' % jid)
        database.insert_user(jid, None, token, None, False)
        # user = TUser(self.gateway, token, jid)
        # user.token = token
    # if not database.is_client(jid):
    #     logger.debug('user %s is not in client list' % jid)
    #
    #     user.token = token
    else:
        raise NotImplementedError('already in database')

    try:
        user_api.connect(gateway, jid, token)
        user_api.initialize(gateway, jid)
    except AuthenticationException:
        logger.error("user %s connection failed (from iq)" % jid)
        result = generate_error(iq, ERR_BAD_REQUEST, "Incorrect password or access token")

    database.set_last_activity_now(jid)
    gateway.add_user(jid)
    send_watcher_message(gateway.component, "new user registered: %s" % jid)
    logger.debug('registration for %s completed' % jid)

    return result

class IQHandler(Handler):
    # sDict = {
    #     "users/total": "users",
    #     "users/online": "users",
    #     "memory/virtual": "KB",
    #     "memory/real": "KB",
    #     "cpu/percent": "percent",
    #     "cpu/time": "seconds"
    # }

    def __init__(self, gateway):
        super(IQHandler, self).__init__(gateway)

    def handle(self, transport, stanza):
        jid_from = stanza.getFrom()
        jid_from_str = jid_from.getStripped()
        jid_to = stanza.getTo()

        if WHITE_LIST and jid_from and jid_from.getDomain() not in WHITE_LIST:
            stanza_send(transport, generate_error(stanza, ERR_BAD_REQUEST, "You are not in white list"))
            raise NodeProcessed()

        from_is_client = database.is_client(jid_from_str)
        destination_is_transport = jid_to == config.TRANSPORT_ID
        ns_is_captcha = stanza.getTagAttr("captcha", "xmlns") == NS_CAPTCHA

        if from_is_client and destination_is_transport and ns_is_captcha:
            c_tag = stanza.getTag("captcha")
            cx_tag = c_tag.getTag("x", {}, NS_DATA)
            fcx_tag = cx_tag.getTag("field", {"var": "ocr"})
            c_value = fcx_tag.getTagData("value")
            captcha_accept(self.gateway, transport, c_value, jid_to, jid_from_str)

        ns = stanza.getQueryNS()

        mapping = {
            NS_REGISTER: self.iq_register_handler,
            NS_GATEWAY: self.iq_gateway_handler,
            NS_DISCO_INFO: self.iq_disco_handler,
            NS_DISCO_ITEMS: self.iq_disco_handler
        }

        try:
            mapping[ns](transport, stanza)
        except KeyError:
            tag = stanza.getTag("vCard") or stanza.getTag("ping")
            if tag and tag.getNamespace() == NS_VCARD:
                self.iq_vcard_handler(transport, stanza)
            elif tag and tag.getNamespace() == NS_PING:
                if jid_to == config.TRANSPORT_ID:
                    stanza_send(transport, stanza.buildReply("result"))

        raise NodeProcessed()

    def iq_register_handler(self, transport, stanza):
        jid = stanza.getFrom().getStripped()
        logger.debug('register handler for %s' % jid)

        destination_jid = stanza.getTo().getStripped()

        logger.debug('destination %s' % destination_jid)
        # iq_type = iq.getType()
        # iq_children = iq.getQueryChildren()
        # result = iq.buildReply("result")
        # if config.USER_LIMIT:
        #     count = self.calc_stats()[0]
        #     if count >= config.USER_LIMIT and not jid_from_str in self.clients:
        #         cl.send(self.iq_build_error(iq, xmpp.ERR_NOT_ALLOWED,
        #                                     _("Transport's admins limited registrations, sorry.")))
        #         raise xmpp.NodeProcessed

        if destination_jid != config.TRANSPORT_ID:
            logger.debug('register not to transport')
            return

        gateway = self.gateway

        try:
            handler = {'get': _send_form, 'set': _process_form}[stanza.getType()]
            stanza_send(transport, handler(gateway, stanza, jid))
        except (NotImplementedError, KeyError) as e:
            stanza_send(transport, generate_error(stanza, 0, "Requested feature not implemented: %s" % e))

    # def calc_stats(self):
    #     count_total = 0
    #     count_online = 0
    #     # TODO: Semaphore
    #     with Database(config.DB_FILE) as db:
    #         db("select count(*) from users")
    #         count_total = db.fetchone()[0]
    #     for key in self.clients:
    #         if hasattr(key, "vk") and key.vk.Online:
    #             count_online += 1
    #     return [count_total, count_online]

    # def iq_uptime_handler(self, cl, iq):
    #     jidFrom = iq.getFrom()
    #     jidTo = iq.getTo()
    #     iType = iq.getType()
    #     if iType == "get" and jidTo == config.TRANSPORT_ID:
    #         uptime = int(time.time() - self.gateway.start_time)
    #         result = xmpp.Iq("result", to=jidFrom)
    #         result.setID(iq.getID())
    #         result.setTag("query", {"seconds": str(uptime)}, xmpp.NS_LAST)
    #         result.setTagData("query", config.IDENTIFIER["name"])
    #         stanza_send(cl, result)
    #     raise xmpp.NodeProcessed()

    #
    # def iqVersionHandler(self, cl, iq):
    #     iType = iq.getType()
    #     jidTo = iq.getTo()
    #     if iType == "get" and jidTo == config.TRANSPORT_ID:
    #         result = iq.buildReply("result")
    #         Query = result.getTag("query")
    #         Query.setTagData("name", config.IDENTIFIER["name"])
    #         os_name = "{0} {2:.16} [{4}]".format(*os.uname())
    #         python_version = "{0} {1}.{2}.{3}".format(sys.subversion[0], *sys.version_info)
    #         # TODO: WTF
    #         Query.setTagData("version", 666)
    #         Query.setTagData("os", "%s / %s" % (os_name, python_version))
    #         stanza_send(cl, result)
    #     raise xmpp.NodeProcessed()

    # def iqStatsHandler(self, cl, iq):
    #     jid_to_str = iq.getTo()
    #     i_type = iq.getType()
    #     iq_children = iq.getQueryChildren()
    #     result = iq.buildReply("result")
    #     if i_type == "get" and jid_to_str == config.TRANSPORT_ID:
    #         querypayload = list()
    #         if not iq_children:
    #             keys = sorted(self.sDict.keys(), reverse=True)
    #             for key in keys:
    #                 Node = xmpp.Node("stat", {"name": key})
    #                 querypayload.append(Node)
    #         else:
    #             users = self.calc_stats()
    #             shell = os.popen("ps -o vsz,rss,%%cpu,time -p %s" % os.getpid()).readlines()
    #             memVirt, memReal, cpuPercent, cpuTime = shell[1].split()
    #             stats = {"users": users, "KB": [memVirt, memReal],
    #                      "percent": [cpuPercent], "seconds": [cpuTime]}
    #             for Child in iq_children:
    #                 if Child.getName() != "stat":
    #                     continue
    #                 name = Child.getAttr("name")
    #                 if name in self.sDict:
    #                     attr = self.sDict[name]
    #                     value = stats[attr].pop(0)
    #                     Node = xmpp.Node("stat", {"units": attr})
    #                     Node.setAttr("name", name)
    #                     Node.setAttr("value", value)
    #                     querypayload.append(Node)
    #         if querypayload:
    #             result.setQueryPayload(querypayload)
    #             stanza_send(cl, result)

    def iq_disco_handler(self, transport, iq_raw):
        logger.debug('handling disco')

        iq = IQ(iq_raw)

        if iq.i_type == "get" and not iq.node and iq.jid_to_str == TRANSPORT_ID:
            query = [Node("identity", IDENTIFIER)]
            result = iq_raw.buildReply("result")
            if iq.ns == NS_DISCO_INFO:
                for key in self.gateway.features:
                    node = Node("feature", {"var": key})
                    query.append(node)
                result.setQueryPayload(query)
            elif iq.ns == NS_DISCO_ITEMS:
                result.setQueryPayload(query)
            stanza_send(transport, str(result))
        raise NodeProcessed()

    @staticmethod
    def iq_gateway_handler(cl, iq):

        jid_to = iq.getTo()
        i_type = iq.getType()
        jid_to_str = jid_to.getStripped()
        iq_children = iq.getQueryChildren()

        if jid_to_str == config.TRANSPORT_ID:
            result = iq.buildReply("result")
            if i_type == "get" and not iq_children:
                query = Node("query", {"xmlns": NS_GATEWAY})
                query.setTagData("desc", "Enter phone number")
                query.setTag("prompt")
                result.setPayload([query])

            elif iq_children and i_type == "set":
                phone = ""
                for node in iq_children:
                    if node.getName() == "prompt":
                        phone = node.getData()
                        break
                if phone:
                    x_node = library.xmpp.simplexml.Node("prompt")
                    x_node.setData(phone[0])
                    result.setQueryPayload([x_node])
            else:
                raise NodeProcessed()
            stanza_send(cl, result)

    @staticmethod
    def vcard_get_photo(url, encode=True):
        logger.debug('vcard_get_photo')
        try:
            opener = urllib.urlopen(url)
            data = opener.read()
            if data and encode:
                data = data.encode("base64")
            return data
        except IOError:
            logger.debug('IO error while getting photo')
            pass

    def iq_vcard_build(self, tags):
        logger.debug('iq_vcard_build')
        vcard = Node("vCard", {"xmlns": NS_VCARD})
        for key in tags.keys():
            if key == "PHOTO":
                bval = vcard.setTag("PHOTO")
                bval.setTagData("BINVAL", self.vcard_get_photo(tags[key]))
            else:
                vcard.setTagData(key, tags[key])
        return vcard

    def iq_vcard_handler(self, transport, stanza):
        logger.debug('iq_vcard_handler')
        jid_from = stanza.getFrom()
        jid_to = stanza.getTo()
        jid = jid_from.getStripped()
        jid_to_str = jid_to.getStripped()
        i_type = stanza.getType()
        result = stanza.buildReply("result")
        if i_type == "get":
            _DESC = '\n'.join(
                (config.DESC, "_" * 16, config.ADDITIONAL_ABOUT)) if config.ADDITIONAL_ABOUT else config.DESC
            if jid_to_str == config.TRANSPORT_ID:
                vcard = self.iq_vcard_build({"NICKNAME": "VK4XMPP Transport",
                                             "DESC": _DESC,
                                             "PHOTO": LOGO_URL,
                                             "URL": "http://simpleapps.ru"})
                result.setPayload([vcard])
            elif database.is_client(jid):
                friends = database.get_friends(jid)
                if friends:
                    friend_jid = get_friend_jid(jid_to_str, jid)
                    json = user_api.get_user_data(jid, friend_jid, ["screen_name", config.PHOTO_SIZE])
                    values = {"NICKNAME": json.get("name", str(json)),
                              "URL": "http://vk.com/id%s" % friend_jid,
                              "DESC": "Contact uses VK4XMPP Transport\n%s" % _DESC}
                    if friend_jid in friends:
                        values["PHOTO"] = json.get(config.PHOTO_SIZE) or config.URL_VCARD_NO_IMAGE
                    vcard = self.iq_vcard_build(values)
                    result.setPayload([vcard])
                else:
                    result = generate_error(stanza, ERR_BAD_REQUEST, 'Your friend list is empty')
            else:
                result = generate_error(stanza, ERR_REGISTRATION_REQUIRED, 'You are not registered for this action')
        else:
            raise NodeProcessed()
        stanza_send(transport, result)

def get_handler(gateway):
    return IQHandler(gateway).handle<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""grumprun compiles and runs a snippet of Python using Grumpy.

Usage: $ grumprun -m <module>             # Run the named module.
       $ echo 'print "hola!"' | grumprun  # Execute Python code from stdin.
"""

import argparse
import os
import random
import shutil
import string
import subprocess
import sys
import tempfile

from .compiler import imputil
from . import grumpc

module_tmpl = string.Template("""\
package main
import (
\t"os"
\t"grumpy"
\tmod "$package"
$imports
)
func main() {
\tgrumpy.ImportModule(grumpy.NewRootFrame(), "traceback")
\tos.Exit(grumpy.RunMain(mod.Code))
}
""")

def main(stream=None, modname=None, pep3147=False):
  gopath = os.getenv('GOPATH', None)
  if not gopath:
    print >> sys.stderr, 'GOPATH not set'
    return 1

  workdir = tempfile.mkdtemp()
  try:
    if modname:
      # Find the script associated with the given module.
      for d in gopath.split(os.pathsep):
        script = imputil.find_script(
            os.path.join(d, 'src', '__python__'), modname)
        if script:
          break
      else:
        print >> sys.stderr, "can't find module", modname
        return 1
    else:
      # Generate a dummy python script on the GOPATH.
      modname = ''.join(random.choice(string.ascii_letters) for _ in range(16))
      py_dir = os.path.join(workdir, 'src', '__python__')
      mod_dir = os.path.join(py_dir, modname)
      os.makedirs(mod_dir)
      script = os.path.join(py_dir, 'module.py')
      with open(script, 'w') as f:
        f.write(stream.read())
      gopath = gopath + os.pathsep + workdir
      os.putenv('GOPATH', gopath)
      # Compile the dummy script to Go using grumpc.
      with open(os.path.join(mod_dir, 'module.go'), 'w+') as dummy_file:
        transpiled = grumpc.main(script, modname=modname, pep3147=True)
        dummy_file.write(transpiled)

    names = imputil.calculate_transitive_deps(modname, script, gopath)
    # Make sure traceback is available in all Python binaries.
    names.add('traceback')
    go_main = os.path.join(workdir, 'main.go')
    package = _package_name(modname)
    imports = ''.join('\t_ "' + _package_name(name) + '"\n' for name in names)
    with open(go_main, 'w') as f:
      f.write(module_tmpl.substitute(package=package, imports=imports))
    return subprocess.Popen('go run ' + go_main, shell=True).wait()
  finally:
    shutil.rmtree(workdir)

def _package_name(modname):
  if modname.startswith('__go__/'):
    return '__python__/' + modname
  return '__python__/' + modname.replace('.', '/')
<commit_msg>Let Click handle exitcodes.<commit_after>#!/usr/bin/env python

# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""grumprun compiles and runs a snippet of Python using Grumpy.

Usage: $ grumprun -m <module>             # Run the named module.
       $ echo 'print "hola!"' | grumprun  # Execute Python code from stdin.
"""

import argparse
import os
import random
import shutil
import string
import subprocess
import sys
import tempfile

from .compiler import imputil
from . import grumpc

module_tmpl = string.Template("""\
package main
import (
\t"os"
\t"grumpy"
\tmod "$package"
$imports
)
func main() {
\tgrumpy.ImportModule(grumpy.NewRootFrame(), "traceback")
\tos.Exit(grumpy.RunMain(mod.Code))
}
""")

def main(stream=None, modname=None, pep3147=False):
  gopath = os.getenv('GOPATH', None)
  if not gopath:
    print >> sys.stderr, 'GOPATH not set'
    return 1

  workdir = tempfile.mkdtemp()
  try:
    if modname:
      # Find the script associated with the given module.
      for d in gopath.split(os.pathsep):
        script = imputil.find_script(
            os.path.join(d, 'src', '__python__'), modname)
        if script:
          break
      else:
        raise RuntimeError("can't find module '%s'", modname)
    else:
      # Generate a dummy python script on the GOPATH.
      modname = ''.join(random.choice(string.ascii_letters) for _ in range(16))
      py_dir = os.path.join(workdir, 'src', '__python__')
      mod_dir = os.path.join(py_dir, modname)
      os.makedirs(mod_dir)
      script = os.path.join(py_dir, 'module.py')
      with open(script, 'w') as f:
        f.write(stream.read())
      gopath = gopath + os.pathsep + workdir
      os.putenv('GOPATH', gopath)
      # Compile the dummy script to Go using grumpc.
      with open(os.path.join(mod_dir, 'module.go'), 'w+') as dummy_file:
        transpiled = grumpc.main(script, modname=modname, pep3147=True)
        dummy_file.write(transpiled)

    names = imputil.calculate_transitive_deps(modname, script, gopath)
    # Make sure traceback is available in all Python binaries.
    names.add('traceback')
    go_main = os.path.join(workdir, 'main.go')
    package = _package_name(modname)
    imports = ''.join('\t_ "' + _package_name(name) + '"\n' for name in names)
    with open(go_main, 'w') as f:
      f.write(module_tmpl.substitute(package=package, imports=imports))
    return subprocess.Popen('go run ' + go_main, shell=True).wait()
  finally:
    shutil.rmtree(workdir)

def _package_name(modname):
  if modname.startswith('__go__/'):
    return '__python__/' + modname
  return '__python__/' + modname.replace('.', '/')
<|endoftext|>