<commit_before>import json
import subprocess

from pathlib import Path

import os
from django.core.management import BaseCommand

class Command(BaseCommand):
    help = "Create Split TopoJSON files for districts and blocks"

    def handle(self, *args, **kwargs):
        input_dir = os.path.join(Path(__file__).parent.parent.parent, 'static', 'js', 'topojsons')
        output_dir = os.path.join(Path(__file__).parent.parent.parent, 'utils', 'topojson_util', 'static')
        # loading block topojson object
        block_topojson_filename = os.path.join(input_dir, 'blocks_v3.topojson.js')
        block_topojson_file = open(block_topojson_filename)
        block_topojson_file_content = block_topojson_file.read()
        # strip off 'var BLOCK_TOPOJSON = ' from the front of the file and '\n;' from the end
        block_topojson = json.loads(block_topojson_file_content[21:][:-2])

        # loading district topojson object
        district_topojson_filename = os.path.join(input_dir, 'districts_v2.topojson.js')
        district_topojson_file = open(district_topojson_filename)
        district_topojson_file_content = district_topojson_file.read()
        # strip off 'var DISTRICT_TOPOJSON = ' from the front of the file
        district_topojson = json.loads(district_topojson_file_content[24:][:-2])

        # create state district map. end result will look like this:
        # {
        #   "state name": {
        #     "districts": [
        #       "district name",
        #       "district name 2",
        #     ]
        #   }
        # }
        state_district_map = {}
        for district in block_topojson['objects'].keys():
            for state, data in district_topojson['objects'].items():
                for district2 in data['geometries']:
                    if district == district2['id']:
                        if state in state_district_map:
                            state_district_map[state]['districts'].append(district)
                        else:
                            state_district_map[state] = {}
                            state_district_map[state]['districts'] = [district]

        # create actual shape data files for each of the states
        for state, data in state_district_map.items():
            district_list = data['districts']
            districts = ','.join(district_list)
            file_name_part = state.replace(' &', '').replace('&', '').replace(' ', '_').lower()
            output_filename = '{}_blocks_v3.topojson'.format(file_name_part)
            output_file_path = os.path.join(output_dir, 'blocks', output_filename)

            # breaking block topojson for each state using mapshaper : https://www.npmjs.com/package/mapshaper
            mapshaper_command = "mapshaper {} -o target='{}' {}".format(
                block_topojson_filename, districts, output_file_path
            )
            subprocess.call(mapshaper_command, shell=True)

            # create new topojson file using the topojson created above
            with open(output_file_path) as state_topojson:
                state_topojson_file_content = state_topojson.read()

            # copy center, height and scale data into new topojson from orginal block topojson
            state_topojson_js_json = json.loads(state_topojson_file_content)
            for district in state_topojson_js_json['objects'].keys():
                district_data = state_topojson_js_json['objects'][district]
                district_data['center'] = block_topojson['objects'][district]['center']
                district_data['height'] = block_topojson['objects'][district]['height']
                district_data['scale'] = block_topojson['objects'][district]['scale']

            state_topojson_js = json.dumps(state_topojson_js_json)

            with open(output_file_path, 'w+') as state_topojson_js_file:
                state_topojson_js_file.write(state_topojson_js)

            data['file_name'] = output_filename

        # saving the state district data with file name of topojson for each state
        state_district_map_file = open(os.path.join(output_dir, 'district_topojson_data.json'), 'w+')
        state_district_map_file.write(json.dumps(state_district_map, indent=2))
        state_district_map_file.close()

        block_topojson_file.close()
        district_topojson_file.close()
        print('done')
<commit_msg>fix data generation script<commit_after>import json
import subprocess

from pathlib import Path

import os
from django.core.management import BaseCommand

class Command(BaseCommand):
    help = "Create Split TopoJSON files for districts and blocks"

    def handle(self, *args, **kwargs):
        input_dir = os.path.join(Path(__file__).parent.parent.parent, 'static', 'js', 'topojsons')
        output_dir = os.path.join(Path(__file__).parent.parent.parent, 'utils', 'topojson_util', 'static')
        # loading block topojson object
        block_topojson_filename = os.path.join(input_dir, 'blocks_v3.topojson.js')
        block_topojson_file = open(block_topojson_filename)
        block_topojson_file_content = block_topojson_file.read()
        # strip off 'var BLOCK_TOPOJSON = ' from the front of the file and '\n;' from the end
        block_topojson_text = block_topojson_file_content[21:][:-2]
        block_topojson = json.loads(block_topojson_file_content[21:][:-2])

        # the mapshaper command needs the variable assignment removed, so save it to a temporary file
        tmp_block_filename = os.path.join(input_dir, 'blocks_tmp.topojson')
        with open(tmp_block_filename, 'w+') as f:
            f.write(block_topojson_text)

        # loading district topojson object
        district_topojson_filename = os.path.join(input_dir, 'districts_v2.topojson.js')
        district_topojson_file = open(district_topojson_filename)
        district_topojson_file_content = district_topojson_file.read()
        # strip off 'var DISTRICT_TOPOJSON = ' from the front of the file
        district_topojson = json.loads(district_topojson_file_content[24:][:-2])

        # create state district map. end result will look like this:
        # {
        #   "state name": {
        #     "districts": [
        #       "district name",
        #       "district name 2",
        #     ]
        #   }
        # }
        state_district_map = {}
        for district in block_topojson['objects'].keys():
            for state, data in district_topojson['objects'].items():
                for district2 in data['geometries']:
                    if district == district2['id']:
                        if state in state_district_map:
                            state_district_map[state]['districts'].append(district)
                        else:
                            state_district_map[state] = {}
                            state_district_map[state]['districts'] = [district]

        # create actual shape data files for each of the states
        for state, data in state_district_map.items():
            district_list = data['districts']
            districts = ','.join(district_list)
            file_name_part = state.replace(' &', '').replace('&', '').replace(' ', '_').lower()
            output_filename = '{}_blocks_v3.topojson'.format(file_name_part)
            output_file_path = os.path.join(output_dir, 'blocks', output_filename)

            # breaking block topojson for each state using mapshaper : https://www.npmjs.com/package/mapshaper
            mapshaper_command = "mapshaper {} -o target='{}' {}".format(
                tmp_block_filename, districts, output_file_path
            )
            subprocess.call(mapshaper_command, shell=True)

            # create new topojson file using the topojson created above
            with open(output_file_path) as state_topojson:
                state_topojson_file_content = state_topojson.read()

            # copy center, height and scale data into new topojson from orginal block topojson
            state_topojson_js_json = json.loads(state_topojson_file_content)
            for district in state_topojson_js_json['objects'].keys():
                district_data = state_topojson_js_json['objects'][district]
                district_data['center'] = block_topojson['objects'][district]['center']
                district_data['height'] = block_topojson['objects'][district]['height']
                district_data['scale'] = block_topojson['objects'][district]['scale']

            state_topojson_js = json.dumps(state_topojson_js_json)

            with open(output_file_path, 'w+') as state_topojson_js_file:
                state_topojson_js_file.write(state_topojson_js)

            data['file_name'] = output_filename

        # saving the state district data with file name of topojson for each state
        state_district_map_file = open(os.path.join(output_dir, 'district_topojson_data.json'), 'w+')
        state_district_map_file.write(json.dumps(state_district_map, indent=2))
        state_district_map_file.close()

        block_topojson_file.close()
        district_topojson_file.close()
        os.remove(tmp_block_filename)
        print('done')
<|endoftext|>


<commit_before>#!/usr/bin/python

# This script does package renamings on source code (and .xml files and build files, etc), as part of migrating code to Jetpack.
# For example, this script may replace the text
#   import android.support.annotation.RequiresApi;
# with
#   import androidx.annotation.RequiresApi;

# See also b/74074903
import json
import os.path
import subprocess

class StringBuilder(object):
  def __init__(self, item=None):
    self.items = []
    if item is not None:
      self.add(item)

  def add(self, item):
    self.items.append(str(item))
    return self

  def __str__(self):
    return "".join(self.items)

class ExecutionConfig(object):
  """Stores configuration about this execution of the package renaming.
     For example, file paths of the source code.
     This config could potentially be affected by command-line arguments.
  """
  def __init__(self, jetifierConfig, sourceRoots, excludeDirs):
    self.jetifierConfig = jetifierConfig
    self.sourceRoots = sourceRoots
    self.excludeDirs = excludeDirs

class SourceRewriteRule(object):
  def __init__(self, fromName, toName):
    self.fromName = fromName
    self.toName = toName

  def serialize(self):
    return self.fromName + ":" + self.toName

class JetifierConfig(object):
  """Stores configuration about the renaming itself, such as package rename rules.
     This config isn't supposed to be affected by command-line arguments.
  """
  @staticmethod
  def parse(filePath):
    with open(filePath) as f:
      lines = f.readlines()
      nonCommentLines = [line for line in lines if not line.strip().startswith("#")]
      parsed = json.loads("".join(nonCommentLines))
      return JetifierConfig(parsed)

  def __init__(self, parsedJson):
    self.json = parsedJson

  def getTypesMap(self):
    rules = []
    for rule in self.json["rules"]:
      fromName = rule["from"].replace("/", ".").replace("(.*)", "")
      toName = rule["to"].replace("/", ".").replace("{0}", "")
      if not toName.startswith("ignore"):
        rules.append(SourceRewriteRule(fromName, toName))

    return rules

def createRewriteCommand(executionConfig):
  # create command to find source files
  finderTextBuilder = StringBuilder("find")
  for sourceRoot in executionConfig.sourceRoots:
    finderTextBuilder.add(" ").add(sourceRoot)
  for exclusion in executionConfig.excludeDirs:
    finderTextBuilder.add(" -name ").add(exclusion).add(" -prune -o")
  finderTextBuilder.add(" -iregex '.*\.java\|.*\.xml' -print")

  # create command to rewrite one source
  print("Building sed instructions")
  rewriterTextBuilder = StringBuilder()
  rewriteRules = executionConfig.jetifierConfig.getTypesMap()
  for rule in rewriteRules:
    rewriterTextBuilder.add("s|").add(rule.fromName).add("|").add(rule.toName).add("|g\n")
  scriptPath = "/tmp/jetifier-sed-script.txt"
  print("Writing " + scriptPath)
  with open(scriptPath, 'w') as scriptFile:
    scriptFile.write(str(rewriterTextBuilder))

  # create the command to do the rewrites
  fullCommand = "time " + str(finderTextBuilder) + " | xargs -n 1 --no-run-if-empty -P 64 sed -i -f " + scriptPath

  return fullCommand

def processConfig(executionConfig):
  print("Building rewrite command")
  rewriteCommand = createRewriteCommand(executionConfig)
  commandLength = len(rewriteCommand)
  print("""
Will run command:

""" + rewriteCommand + """

""")
  response = raw_input("Ok? [y/n]")
  if response == "y":
    subprocess.check_output(rewriteCommand, shell=True)

def main():
  pathOfThisFile = os.path.realpath(__file__)
  jetifierPath = os.path.abspath(os.path.join(pathOfThisFile, "..", ".."))

  jetifierConfigPath = os.path.join(jetifierPath, "core", "src", "main", "resources", "default.config")
  print("Parsing " + jetifierConfigPath)
  jetifierConfig = JetifierConfig.parse(jetifierConfigPath)

  sourceRoot = os.getcwd()
  excludeDirs = ["out", ".git", ".repo"]

  executionConfig = ExecutionConfig(jetifierConfig, [sourceRoot], excludeDirs)

  processConfig(executionConfig)

main()

<commit_msg>Update package rewriter script. am: afa1f99d63 am: 57f394ea68<commit_after>#!/usr/bin/python

# This script does package renamings on source code (and .xml files and build files, etc), as part of migrating code to Jetpack.
# For example, this script may replace the text
#   import android.support.annotation.RequiresApi;
# with
#   import androidx.annotation.RequiresApi;

# See also b/74074903
import json
import os.path
import subprocess

HARDCODED_RULES_REVERSE = [
  "s|androidx.core.media.MediaBrowserCompat|android.support.v4.media.MediaBrowserCompat|g\n",
  "s|androidx.core.media.MediaDescriptionCompat|android.support.v4.media.MediaDescriptionCompat|g\n",
  "s|androidx.core.media.MediaMetadataCompat|android.support.v4.media.MediaMetadataCompat|g\n",
  "s|androidx.core.media.RatingCompat|android.support.v4.media.RatingCompat|g\n",
  "s|androidx.core.media.session.MediaControllerCompat|android.support.v4.media.session.MediaControllerCompat|g\n",
  "s|androidx.core.media.session.MediaSessionCompat|android.support.v4.media.session.MediaSessionCompat|g\n",
  "s|androidx.core.media.session.ParcelableVolumeInfo|android.support.v4.media.session.ParcelableVolumeInfo|g\n",
  "s|androidx.core.media.session.PlaybackStateCompat|android.support.v4.media.session.PlaybackStateCompat|g\n",
]

class StringBuilder(object):
  def __init__(self, item=None):
    self.items = []
    if item is not None:
      self.add(item)

  def add(self, item):
    self.items.append(str(item))
    return self

  def __str__(self):
    return "".join(self.items)

class ExecutionConfig(object):
  """Stores configuration about this execution of the package renaming.
     For example, file paths of the source code.
     This config could potentially be affected by command-line arguments.
  """
  def __init__(self, jetifierConfig, sourceRoots, excludeDirs):
    self.jetifierConfig = jetifierConfig
    self.sourceRoots = sourceRoots
    self.excludeDirs = excludeDirs

class SourceRewriteRule(object):
  def __init__(self, fromName, toName):
    self.fromName = fromName
    self.toName = toName

  def serialize(self):
    return self.fromName + ":" + self.toName

class JetifierConfig(object):
  """Stores configuration about the renaming itself, such as package rename rules.
     This config isn't supposed to be affected by command-line arguments.
  """
  @staticmethod
  def parse(filePath):
    with open(filePath) as f:
      lines = f.readlines()
      nonCommentLines = [line for line in lines if not line.strip().startswith("#")]
      parsed = json.loads("".join(nonCommentLines))
      return JetifierConfig(parsed)

  def __init__(self, parsedJson):
    self.json = parsedJson

  def getTypesMap(self):
    rules = []
    for rule in self.json["rules"]:
      fromName = rule["from"].replace("/", ".").replace("(.*)", "")
      toName = rule["to"].replace("/", ".").replace("{0}", "")
      if not toName.startswith("ignore"):
        rules.append(SourceRewriteRule(fromName, toName))

    return rules

def createRewriteCommand(executionConfig):
  # create command to find source files
  finderTextBuilder = StringBuilder("find")
  for sourceRoot in executionConfig.sourceRoots:
    finderTextBuilder.add(" ").add(sourceRoot)
  for exclusion in executionConfig.excludeDirs:
    finderTextBuilder.add(" -name ").add(exclusion).add(" -prune -o")
  finderTextBuilder.add(" -iregex '.*\.java\|.*\.xml\|.*\.cfg\|.*\.flags' -print")

  # create command to rewrite one source
  print("Building sed instructions")
  rewriterTextBuilder = StringBuilder()
  rewriteRules = executionConfig.jetifierConfig.getTypesMap()
  for rule in rewriteRules:
    rewriterTextBuilder.add("s|").add(rule.fromName).add("|").add(rule.toName).add("|g\n")
  for rule in HARDCODED_RULES_REVERSE:
    rewriterTextBuilder.add(rule)
  scriptPath = "/tmp/jetifier-sed-script.txt"
  print("Writing " + scriptPath)
  with open(scriptPath, 'w') as scriptFile:
    scriptFile.write(str(rewriterTextBuilder))

  # create the command to do the rewrites
  fullCommand = "time " + str(finderTextBuilder) + " | xargs -n 1 --no-run-if-empty -P 64 sed -i -f " + scriptPath

  return fullCommand

def processConfig(executionConfig):
  print("Building rewrite command")
  rewriteCommand = createRewriteCommand(executionConfig)
  commandLength = len(rewriteCommand)
  print("""
Will run command:

""" + rewriteCommand + """

""")
  response = raw_input("Ok? [y/n]")
  if response == "y":
    subprocess.check_output(rewriteCommand, shell=True)

def main():
  pathOfThisFile = os.path.realpath(__file__)
  jetifierPath = os.path.abspath(os.path.join(pathOfThisFile, "..", ".."))

  jetifierConfigPath = os.path.join(jetifierPath, "core", "src", "main", "resources", "default.config")
  print("Parsing " + jetifierConfigPath)
  jetifierConfig = JetifierConfig.parse(jetifierConfigPath)

  sourceRoot = os.getcwd()
  excludeDirs = ["out", ".git", ".repo"]

  executionConfig = ExecutionConfig(jetifierConfig, [sourceRoot], excludeDirs)

  processConfig(executionConfig)

main()

<|endoftext|>


<commit_before># BSD 3-Clause License
#
# Copyright (c) 2016-17, University of Liverpool
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""A module containing generic Figure related objects"""

__author__ = "Felix Simkovic"
__date__ = "08 Jan 2018"
__version__ = "0.2"

import os
import matplotlib.collections as mcoll
import matplotlib.pyplot as plt

class Figure(object):
    """A Figure class to store common features"""

    def __init__(self, ax=None, legend=True):
        """Initialise a new :obj:`conkit.plot.Figure` object

        Parameters
        ----------
        ax : :obj:`Axes <matplotlib.pyplot.Axes>`
           A pre-defined :obj:`Axes <matplotlib.pyplot.Axes>`

           If ``None`` is provided, a new plot will be created

        legend : bool, optional
           Draw a legend on the plot [default: True]

        """
        if ax is None:
            import matplotlib.pyplot as plt
            self._fig, self._ax = plt.subplots()
        else:
            self._fig = ax.get_figure()
            self._ax = ax
        self.legend = legend

    def __repr__(self):
        return self.__class__.__name__

    def _patch_scatter(self, x, y, facecolor="#ffffff", edgecolor="#000000", radius=0.5, linewidth=1.0):
        """Draw scatter points as :obj:`Circles <matplotlib.pyplot.Circle>` to control width for discrete data"""
        if len(x) != len(y):
            raise ValueError("Unequal x and y data provided")

        if isinstance(facecolor, str):
            fc = [facecolor] * len(x)
        else:
            if len(facecolor) != len(x):
                raise ValueError("Unequal x/y data and facecolors provided")
            fc = facecolor

        if isinstance(edgecolor, str):
            ec = [edgecolor] * len(x)
        else:
            if len(edgecolor) != len(x):
                raise ValueError("Unequal x/y data and edgecolors provided")
            ec = edgecolor

        if isinstance(linewidth, float) or isinstance(linewidth, int):
            lw = [linewidth] * len(x)
        else:
            if len(linewidth) != len(x):
                raise ValueError("Unequal x/y data and linewidths provided")
            lw = linewidth

        if isinstance(radius, float) or isinstance(radius, int):
            r = [radius] * len(x)
        else:
            if len(radius) != len(x):
                raise ValueError("Unequal x/y data and radii provided")
            r = radius

        # Credits to https://stackoverflow.com/a/48174228/3046533
        circles = [
            plt.Circle((xi, yi), facecolor=fci, edgecolor=eci, radius=ri, linewidth=lwi)
            for xi, yi, fci, eci, ri, lwi in zip(x, y, fc, ec, r, lw)
        ]
        patch_collection = mcoll.PatchCollection(circles, match_original=True)
        self.ax.add_collection(patch_collection)

    def savefig(self, filename, dpi=300, overwrite=False):
        if os.path.isfile(filename) and not overwrite:
            raise RuntimeError("File exists: %s! Please rename or remove." % filename)
        else:
            self._fig.savefig(filename, dpi=dpi, bbox_inches="tight")

    @property
    def fig(self):
        return self._fig

    @property
    def ax(self):
        return self._ax
<commit_msg>bug fix for Python2.7 compatibility<commit_after># BSD 3-Clause License
#
# Copyright (c) 2016-17, University of Liverpool
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""A module containing generic Figure related objects"""

__author__ = "Felix Simkovic"
__date__ = "08 Jan 2018"
__version__ = "0.2"

import os
import matplotlib.collections as mcoll
import matplotlib.pyplot as plt

class Figure(object):
    """A Figure class to store common features"""

    def __init__(self, ax=None, legend=True):
        """Initialise a new :obj:`conkit.plot.Figure` object

        Parameters
        ----------
        ax : :obj:`Axes <matplotlib.pyplot.Axes>`
           A pre-defined :obj:`Axes <matplotlib.pyplot.Axes>`

           If ``None`` is provided, a new plot will be created

        legend : bool, optional
           Draw a legend on the plot [default: True]

        """
        if ax is None:
            import matplotlib.pyplot as plt
            self._fig, self._ax = plt.subplots()
        else:
            self._fig = ax.get_figure()
            self._ax = ax
        self.legend = legend

    def __repr__(self):
        return self.__class__.__name__

    def _patch_scatter(self, x, y, facecolor="#ffffff", edgecolor="#000000", radius=0.5, linewidth=1.0):
        """Draw scatter points as :obj:`Circles <matplotlib.pyplot.Circle>` to control width for discrete data"""
        if len(x) != len(y):
            raise ValueError("Unequal x and y data provided")

        if isinstance(facecolor, str):
            fc = [facecolor] * len(x)
        else:
            if len(facecolor) != len(x):
                raise ValueError("Unequal x/y data and facecolors provided")
            fc = facecolor

        if isinstance(edgecolor, str):
            ec = [edgecolor] * len(x)
        else:
            if len(edgecolor) != len(x):
                raise ValueError("Unequal x/y data and edgecolors provided")
            ec = edgecolor

        if isinstance(linewidth, float) or isinstance(linewidth, int):
            lw = [linewidth] * len(x)
        else:
            if len(linewidth) != len(x):
                raise ValueError("Unequal x/y data and linewidths provided")
            lw = linewidth

        if isinstance(radius, float) or isinstance(radius, int):
            r = [radius] * len(x)
        else:
            if len(radius) != len(x):
                raise ValueError("Unequal x/y data and radii provided")
            r = radius

        # Credits to https://stackoverflow.com/a/48174228/3046533
        circles = [
            plt.Circle((xi, yi), facecolor=fci, edgecolor=eci, radius=ri, linewidth=lwi)
            for xi, yi, fci, eci, ri, lwi in zip(x, y, fc, ec, r, lw)
        ]
        if len(circles) > 0:
            patch_collection = mcoll.PatchCollection(circles, match_original=True)
            self.ax.add_collection(patch_collection)

    def savefig(self, filename, dpi=300, overwrite=False):
        if os.path.isfile(filename) and not overwrite:
            raise RuntimeError("File exists: %s! Please rename or remove." % filename)
        else:
            self._fig.savefig(filename, dpi=dpi, bbox_inches="tight")

    @property
    def fig(self):
        return self._fig

    @property
    def ax(self):
        return self._ax
<|endoftext|>


<commit_before>"""
Raster file input which can be read by rasterio.

Currently limited by extensions .tif, .vrt., .png and .jp2 but could be
extended easily.
"""

import fiona
from shapely.geometry import box
from rasterio.crs import CRS

from mapchete.formats import base
from mapchete.io.vector import reproject_geometry, read_vector_window

class InputData(base.InputData):
    """Main input class."""

    METADATA = {
        "driver_name": "vector_file",
        "data_type": "vector",
        "mode": "r",
        "file_extensions": ["shp", "geojson"]
    }

    def __init__(self, input_params):
        """Initialize."""
        super(InputData, self).__init__(input_params)
        self.path = input_params["path"]

    def open(self, tile, **kwargs):
        """Return InputTile."""
        return InputTile(tile, self, **kwargs)

    def bbox(self, out_crs=None):
        """Return data bounding box."""
        assert self.path
        assert self.pyramid
        if out_crs is None:
            out_crs = self.pyramid.crs
        with fiona.open(self.path) as inp:
            inp_crs = CRS(inp.crs)
            try:
                assert inp_crs.is_valid
            except AssertionError:
                print inp.crs
                raise IOError("CRS could not be read from %s" % self.path)
            bbox = box(*inp.bounds)
        # If soucre and target CRSes differ, segmentize and reproject
        if inp_crs != out_crs:
            return reproject_geometry(bbox, src_crs=inp_crs, dst_crs=out_crs)
        else:
            return bbox

class InputTile(base.InputTile):
    """Target Tile representation of input data."""

    def __init__(self, tile, vector_file):
        """Initialize."""
        self.tile = tile
        self.vector_file = vector_file
        self._cache = {}

    def read(self, validity_check=True):
        """Read reprojected and resampled numpy array for current Tile."""
        return self._read_from_cache(validity_check)

    def is_empty(self):
        """Check if there is data within this tile."""
        if not self.tile.bbox.intersects(self.vector_file.bbox()):
            return True
        if self.read():
            return False
        else:
            return True

    def _read_from_cache(self, validity_check):
        checked = "checked" if validity_check else "not_checked"
        if checked not in self._cache:
            self._cache[checked] = list(read_vector_window(
                self.vector_file.path, self.tile,
                validity_check=validity_check))
        return self._cache[checked]
<commit_msg>return empty Polygon for bounding box if vector file has no features<commit_after>"""
Raster file input which can be read by rasterio.

Currently limited by extensions .tif, .vrt., .png and .jp2 but could be
extended easily.
"""

import fiona
from shapely.geometry import box, Polygon
from rasterio.crs import CRS

from mapchete.formats import base
from mapchete.io.vector import reproject_geometry, read_vector_window

class InputData(base.InputData):
    """Main input class."""

    METADATA = {
        "driver_name": "vector_file",
        "data_type": "vector",
        "mode": "r",
        "file_extensions": ["shp", "geojson"]
    }

    def __init__(self, input_params):
        """Initialize."""
        super(InputData, self).__init__(input_params)
        self.path = input_params["path"]

    def open(self, tile, **kwargs):
        """Return InputTile."""
        return InputTile(tile, self, **kwargs)

    def bbox(self, out_crs=None):
        """Return data bounding box."""
        assert self.path
        assert self.pyramid
        if out_crs is None:
            out_crs = self.pyramid.crs
        with fiona.open(self.path) as inp:
            inp_crs = CRS(inp.crs)
            try:
                assert inp_crs.is_valid
            except AssertionError:
                print inp.crs
                raise IOError("CRS could not be read from %s" % self.path)
            bbox = box(*inp.bounds)
        # Check if file bounding box is empty.
        if len(set(inp.bounds)) == 1:
            return Polygon()
        # If soucre and target CRSes differ, segmentize and reproject
        if inp_crs != out_crs:
            return reproject_geometry(bbox, src_crs=inp_crs, dst_crs=out_crs)
        else:
            return bbox

class InputTile(base.InputTile):
    """Target Tile representation of input data."""

    def __init__(self, tile, vector_file):
        """Initialize."""
        self.tile = tile
        self.vector_file = vector_file
        self._cache = {}

    def read(self, validity_check=True):
        """Read reprojected and resampled numpy array for current Tile."""
        return self._read_from_cache(validity_check)

    def is_empty(self):
        """Check if there is data within this tile."""
        if not self.tile.bbox.intersects(self.vector_file.bbox()):
            return True
        if self.read():
            return False
        else:
            return True

    def _read_from_cache(self, validity_check):
        checked = "checked" if validity_check else "not_checked"
        if checked not in self._cache:
            self._cache[checked] = list(read_vector_window(
                self.vector_file.path, self.tile,
                validity_check=validity_check))
        return self._cache[checked]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.utils.functional import cached_property

from rest_framework import serializers

from formidable.models import Fieldidable
from formidable.serializers.items import ItemSerializer
from formidable.serializers.access import AccessSerializer
from formidable.serializers.validation import ValidationSerializer
from formidable.serializers.child_proxy import LazyChildProxy
from formidable.register import FieldSerializerRegister, load_serializer
from formidable.serializers.list import NestedListSerializer

BASE_FIELDS = (
    'slug', 'label', 'type_id', 'placeholder', 'helptext', 'default',
    'accesses', 'validations',
)

field_register = FieldSerializerRegister.get_instance()

class FieldListSerializer(NestedListSerializer):

    field_id = 'slug'
    parent_name = 'form_id'

    def __init__(self, *args, **kwargs):
        kwargs['child'] = LazyChildProxy(field_register)
        return super(FieldListSerializer, self).__init__(*args, **kwargs)

class FieldidableSerializer(serializers.ModelSerializer):

    type_id = None

    items = ItemSerializer(many=True)
    accesses = AccessSerializer(many=True)
    validations = ValidationSerializer(many=True, required=False)

    nested_objects = ['accesses', 'validations']

    class Meta:
        model = Fieldidable
        list_serializer_class = FieldListSerializer
        fields = '__all__'

    @cached_property
    def access_serializer(self):
        return self.fields['accesses']

    @cached_property
    def validations_serializer(self):
        return self.fields['validations']

    def create(self, validated_data):
        nested_data = self.extract_nested_data(validated_data)
        field = super(FieldidableSerializer, self).create(validated_data)
        self.create_nested_objects(field, nested_data)
        return field

    def create_nested_objects(self, field, nested_data):
        for name, data in nested_data.iteritems():
            self.fields[name].create(field, nested_data[name])

    def update_nested_objects(self, field, nested_data):
        for name, data in nested_data.iteritems():
            self.fields[name].update(
                getattr(field, name), field, nested_data[name]
            )

    def extract_nested_data(self, data):
        """
        Extract the data for nested object. By default DRF raise an execption
        when data for nested objet are found.
        Data are validated before, if nested_object is required we are sure
        to have the data.
        """
        res = {}
        for nested_object_name in self.nested_objects:
            if nested_object_name in data:
                res[nested_object_name] = data.pop(nested_object_name)
        return res

    def update(self, instance, validated_data):
        nested_data = self.extract_nested_data(validated_data)
        field = super(FieldidableSerializer, self).update(
            instance, validated_data
        )
        self.update_nested_objects(field, nested_data)
        return field

class FieldItemMixin(object):

    @cached_property
    def item_serializer(self):
        return self.fields['items']

    def create(self, validated_data):
        items_data = validated_data.pop('items')
        field = super(FieldItemMixin, self).create(validated_data)
        self.item_serializer.create(field, items_data)
        return field

    def update(self, instance, validated_data):
        items_kwargs = validated_data.pop('items')
        field = super(FieldItemMixin, self).update(instance, validated_data)
        self.item_serializer.update(field.items, field, items_kwargs)
        return field

@load_serializer(field_register)
class TextFieldSerializer(FieldidableSerializer):

    type_id = 'text'

    class Meta(FieldidableSerializer.Meta):
        fields = BASE_FIELDS

@load_serializer(field_register)
class ParagraphFieldSerializer(TextFieldSerializer):

    type_id = 'paragraph'

@load_serializer(field_register)
class DropdownFieldSerializer(FieldItemMixin, FieldidableSerializer):

    type_id = 'dropdown'

    class Meta(FieldidableSerializer.Meta):
        fields = BASE_FIELDS + ('items', 'multiple')

@load_serializer(field_register)
class CheckboxFieldSerializer(FieldItemMixin, FieldidableSerializer):

    type_id = 'checkbox'

    class Meta(FieldidableSerializer.Meta):
        fields = BASE_FIELDS + ('items',)

@load_serializer(field_register)
class CheckboxesFieldSerializer(FieldItemMixin, FieldidableSerializer):

    type_id = 'checkboxes'

    class Meta(FieldidableSerializer):
        fields = BASE_FIELDS + ('items', 'multiple')

@load_serializer(field_register)
class RadiosFieldSerializer(FieldItemMixin, FieldidableSerializer):

    type_id = 'radios'

    class Meta(FieldidableSerializer.Meta):
        fields = BASE_FIELDS + ('items', 'multiple')

@load_serializer(field_register)
class RadiosButtonsFieldSerializer(RadiosFieldSerializer):

    type_id = 'radiosButtons'

@load_serializer(field_register)
class FileFieldSerializer(FieldidableSerializer):

    type_id = 'file'

    class Meta(FieldidableSerializer.Meta):
        fields = BASE_FIELDS

@load_serializer(field_register)
class DateFieldSerializer(FieldidableSerializer):

    type_id = 'date'

    class Meta(FieldidableSerializer.Meta):
        fields = BASE_FIELDS

@load_serializer(field_register)
class EmailFieldSerializer(FieldidableSerializer):

    type_id = 'email'

    class Meta(FieldidableSerializer.Meta):
        fields = BASE_FIELDS

@load_serializer(field_register)
class NumberFieldSerializer(FieldidableSerializer):

    type_id = 'number'

    class Meta(FieldidableSerializer.Meta):
        fields = BASE_FIELDS
<commit_msg>refs #29, checkbox items has no items<commit_after># -*- coding: utf-8 -*-
from django.utils.functional import cached_property

from rest_framework import serializers

from formidable.models import Fieldidable
from formidable.serializers.items import ItemSerializer
from formidable.serializers.access import AccessSerializer
from formidable.serializers.validation import ValidationSerializer
from formidable.serializers.child_proxy import LazyChildProxy
from formidable.register import FieldSerializerRegister, load_serializer
from formidable.serializers.list import NestedListSerializer

BASE_FIELDS = (
    'slug', 'label', 'type_id', 'placeholder', 'helptext', 'default',
    'accesses', 'validations',
)

field_register = FieldSerializerRegister.get_instance()

class FieldListSerializer(NestedListSerializer):

    field_id = 'slug'
    parent_name = 'form_id'

    def __init__(self, *args, **kwargs):
        kwargs['child'] = LazyChildProxy(field_register)
        return super(FieldListSerializer, self).__init__(*args, **kwargs)

class FieldidableSerializer(serializers.ModelSerializer):

    type_id = None

    items = ItemSerializer(many=True)
    accesses = AccessSerializer(many=True)
    validations = ValidationSerializer(many=True, required=False)

    nested_objects = ['accesses', 'validations']

    class Meta:
        model = Fieldidable
        list_serializer_class = FieldListSerializer
        fields = '__all__'

    @cached_property
    def access_serializer(self):
        return self.fields['accesses']

    @cached_property
    def validations_serializer(self):
        return self.fields['validations']

    def create(self, validated_data):
        nested_data = self.extract_nested_data(validated_data)
        field = super(FieldidableSerializer, self).create(validated_data)
        self.create_nested_objects(field, nested_data)
        return field

    def create_nested_objects(self, field, nested_data):
        for name, data in nested_data.iteritems():
            self.fields[name].create(field, nested_data[name])

    def update_nested_objects(self, field, nested_data):
        for name, data in nested_data.iteritems():
            self.fields[name].update(
                getattr(field, name), field, nested_data[name]
            )

    def extract_nested_data(self, data):
        """
        Extract the data for nested object. By default DRF raise an execption
        when data for nested objet are found.
        Data are validated before, if nested_object is required we are sure
        to have the data.
        """
        res = {}
        for nested_object_name in self.nested_objects:
            if nested_object_name in data:
                res[nested_object_name] = data.pop(nested_object_name)
        return res

    def update(self, instance, validated_data):
        nested_data = self.extract_nested_data(validated_data)
        field = super(FieldidableSerializer, self).update(
            instance, validated_data
        )
        self.update_nested_objects(field, nested_data)
        return field

class FieldItemMixin(object):

    @cached_property
    def item_serializer(self):
        return self.fields['items']

    def create(self, validated_data):
        items_data = validated_data.pop('items')
        field = super(FieldItemMixin, self).create(validated_data)
        self.item_serializer.create(field, items_data)
        return field

    def update(self, instance, validated_data):
        items_kwargs = validated_data.pop('items')
        field = super(FieldItemMixin, self).update(instance, validated_data)
        self.item_serializer.update(field.items, field, items_kwargs)
        return field

@load_serializer(field_register)
class TextFieldSerializer(FieldidableSerializer):

    type_id = 'text'

    class Meta(FieldidableSerializer.Meta):
        fields = BASE_FIELDS

@load_serializer(field_register)
class ParagraphFieldSerializer(TextFieldSerializer):

    type_id = 'paragraph'

@load_serializer(field_register)
class DropdownFieldSerializer(FieldItemMixin, FieldidableSerializer):

    type_id = 'dropdown'

    class Meta(FieldidableSerializer.Meta):
        fields = BASE_FIELDS + ('items', 'multiple')

@load_serializer(field_register)
class CheckboxFieldSerializer(FieldidableSerializer):

    type_id = 'checkbox'

    class Meta(FieldidableSerializer.Meta):
        fields = BASE_FIELDS

@load_serializer(field_register)
class CheckboxesFieldSerializer(FieldItemMixin, FieldidableSerializer):

    type_id = 'checkboxes'

    class Meta(FieldidableSerializer):
        fields = BASE_FIELDS + ('items', 'multiple')

@load_serializer(field_register)
class RadiosFieldSerializer(FieldItemMixin, FieldidableSerializer):

    type_id = 'radios'

    class Meta(FieldidableSerializer.Meta):
        fields = BASE_FIELDS + ('items', 'multiple')

@load_serializer(field_register)
class RadiosButtonsFieldSerializer(RadiosFieldSerializer):

    type_id = 'radiosButtons'

@load_serializer(field_register)
class FileFieldSerializer(FieldidableSerializer):

    type_id = 'file'

    class Meta(FieldidableSerializer.Meta):
        fields = BASE_FIELDS

@load_serializer(field_register)
class DateFieldSerializer(FieldidableSerializer):

    type_id = 'date'

    class Meta(FieldidableSerializer.Meta):
        fields = BASE_FIELDS

@load_serializer(field_register)
class EmailFieldSerializer(FieldidableSerializer):

    type_id = 'email'

    class Meta(FieldidableSerializer.Meta):
        fields = BASE_FIELDS

@load_serializer(field_register)
class NumberFieldSerializer(FieldidableSerializer):

    type_id = 'number'

    class Meta(FieldidableSerializer.Meta):
        fields = BASE_FIELDS
<|endoftext|>


<commit_before>from .base import Config
from django.utils.functional import SimpleLazyObject

__version__ = '1.0a1'

try:
    from django.apps import AppConfig  # noqa
except ImportError:
    config = SimpleLazyObject(Config)
else:
    default_app_config = 'constance.apps.ConstanceConfig'
<commit_msg>1.0 \o/<commit_after>from .base import Config
from django.utils.functional import SimpleLazyObject

__version__ = '1.0'

try:
    from django.apps import AppConfig  # noqa
except ImportError:
    config = SimpleLazyObject(Config)
else:
    default_app_config = 'constance.apps.ConstanceConfig'
<|endoftext|>


<commit_before>from django.core.management.base import BaseCommand
from django.conf import settings
from django.utils import timezone

#from http://jamesmckay.net/2009/03/django-custom-managepy-commands-not-committing-transactions/
#Fix issue where db data in manage.py commands is not refreshed at all once they start running
from django.db import transaction
transaction.commit_unless_managed()

import requests
import urlparse
import time
import json
import logging
import sys
from ConfigParser import SafeConfigParser
from datetime import datetime

from controller.models import Submission, Grader
from controller.models import GraderStatus, SubmissionState

import controller.rubric_functions
import random
from controller import grader_util

log = logging.getLogger(__name__)

class Command(BaseCommand):
    args = "<filename>"
    help = "Poll grading controller and send items to be graded to ml"

    def handle(self, *args, **options):
        """
        Read from file
        """

        parser = SafeConfigParser()
        parser.read(args[0])

        print("Starting import...")
        print("Reading config from file {0}".format(args[0]))

        header_name = "importdata"
        location = parser.get(header_name, 'location')
        course_id = parser.get(header_name, 'course_id')
        problem_id = parser.get(header_name, 'problem_id')
        prompt_file = parser.get(header_name, 'prompt_file')
        essay_file = parser.get(header_name, 'essay_file')
        essay_limit = int(parser.get(header_name, 'essay_limit'))
        state = parser.get(header_name, "state")
        next_grader_type = parser.get(header_name, "next_grader")
        add_grader = parser.get(header_name, "add_grader_object") == "True"
        set_as_calibration = parser.get(header_name, "set_as_calibration") == "True"
        max_score= parser.get(header_name,"max_score")
        student_id = parser.get(header_name,'student_id')
        increment_ids = parser.get(header_name,'increment_ids')
        rubric_file = parser.get(header_name, 'rubric_file')
        import_rubric_scores = parser.get(header_name, 'import_rubric_scores') == "True"
        rubric_scores_file = parser.get(header_name, 'rubric_scores_file')

        rubric=open(settings.REPO_PATH / rubric_file).read()
        prompt=open(settings.REPO_PATH / prompt_file).read()

        score, text = [], []
        combined_raw = open(settings.REPO_PATH / essay_file).read()
        raw_lines = combined_raw.splitlines()
        for row in xrange(1, len(raw_lines)):
            score1, text1 = raw_lines[row].strip().split("\t")
            text.append(text1)
            score.append(int(score1))

        if increment_ids:
            student_id = int(student_id)

        if import_rubric_scores:
            rubric_scores=[]
            combined_raw = open(settings.REPO_PATH / rubric_scores_file).read()
            raw_lines = combined_raw.splitlines()
            for row in xrange(1, len(raw_lines)):
                rubric_score_row=[]
                for score_item in raw_lines[row].strip().split("\t"):
                    rubric_score_row.append(int(score_item))
                rubric_scores.append(rubric_score_row)

        for i in range(0, min(essay_limit, len(text))):
            sub = Submission(
                prompt=prompt,
                student_id=student_id,
                problem_id=problem_id,
                state=state,
                student_response=text[i],
                student_submission_time=timezone.now(),
                xqueue_submission_id="",
                xqueue_submission_key="",
                xqueue_queue_name="",
                location=location,
                course_id=course_id,
                next_grader_type=next_grader_type,
                posted_results_back_to_queue=True,
                previous_grader_type="BC",
                max_score=max_score,
                rubric=rubric,
                preferred_grader_type = next_grader_type,
            )

            sub.save()
            if add_grader:
                sub.previous_grader_type="IN"
                sub.save()
                grade = Grader(
                    score=score[i],
                    feedback="",
                    status_code=GraderStatus.success,
                    grader_id="",
                    grader_type="IN",
                    confidence=1,
                    is_calibration=set_as_calibration,
                )

                grade.submission = sub
                grade.save()

                success, rubric_targets=controller.rubric_functions.generate_targets_from_rubric(sub.rubric)
                scores=[]
                for z in xrange(0,len(rubric_targets)):
                    scores.append(random.randint(0,rubric_targets[z]))
                if import_rubric_scores:
                    score_item = rubric_scores[i]
                    if len(score_item) == len(scores):
                        scores = score_item
                        log.debug("Score: {0} Rubric Score: {1}".format(score[i], scores))

                controller.rubric_functions.generate_rubric_object(grade, scores, sub.rubric)

            if increment_ids:
                student_id+=1

        print ("Successfully imported {0} essays using configuration in file {1}.".format(
            min(essay_limit, len(text)),
            args[0],
        ))
<commit_msg>Make unique xqueue submission ids for added essays<commit_after>from django.core.management.base import BaseCommand
from django.conf import settings
from django.utils import timezone

#from http://jamesmckay.net/2009/03/django-custom-managepy-commands-not-committing-transactions/
#Fix issue where db data in manage.py commands is not refreshed at all once they start running
from django.db import transaction
transaction.commit_unless_managed()

import requests
import urlparse
import time
import json
import logging
import sys
from ConfigParser import SafeConfigParser
from datetime import datetime
import test_util

from controller.models import Submission, Grader
from controller.models import GraderStatus, SubmissionState

import controller.rubric_functions
import random
from controller import grader_util

log = logging.getLogger(__name__)

class Command(BaseCommand):
    args = "<filename>"
    help = "Poll grading controller and send items to be graded to ml"

    def handle(self, *args, **options):
        """
        Read from file
        """

        parser = SafeConfigParser()
        parser.read(args[0])

        print("Starting import...")
        print("Reading config from file {0}".format(args[0]))

        header_name = "importdata"
        location = parser.get(header_name, 'location')
        course_id = parser.get(header_name, 'course_id')
        problem_id = parser.get(header_name, 'problem_id')
        prompt_file = parser.get(header_name, 'prompt_file')
        essay_file = parser.get(header_name, 'essay_file')
        essay_limit = int(parser.get(header_name, 'essay_limit'))
        state = parser.get(header_name, "state")
        next_grader_type = parser.get(header_name, "next_grader")
        add_grader = parser.get(header_name, "add_grader_object") == "True"
        set_as_calibration = parser.get(header_name, "set_as_calibration") == "True"
        max_score= parser.get(header_name,"max_score")
        student_id = parser.get(header_name,'student_id')
        increment_ids = parser.get(header_name,'increment_ids')
        rubric_file = parser.get(header_name, 'rubric_file')
        import_rubric_scores = parser.get(header_name, 'import_rubric_scores') == "True"
        rubric_scores_file = parser.get(header_name, 'rubric_scores_file')

        rubric=open(settings.REPO_PATH / rubric_file).read()
        prompt=open(settings.REPO_PATH / prompt_file).read()

        score, text = [], []
        combined_raw = open(settings.REPO_PATH / essay_file).read()
        raw_lines = combined_raw.splitlines()
        for row in xrange(1, len(raw_lines)):
            score1, text1 = raw_lines[row].strip().split("\t")
            text.append(text1)
            score.append(int(score1))

        if increment_ids:
            student_id = int(student_id)

        if import_rubric_scores:
            rubric_scores=[]
            combined_raw = open(settings.REPO_PATH / rubric_scores_file).read()
            raw_lines = combined_raw.splitlines()
            for row in xrange(1, len(raw_lines)):
                rubric_score_row=[]
                for score_item in raw_lines[row].strip().split("\t"):
                    rubric_score_row.append(int(score_item))
                rubric_scores.append(rubric_score_row)

        for i in range(0, min(essay_limit, len(text))):
            sub = Submission(
                prompt=prompt,
                student_id=student_id,
                problem_id=problem_id,
                state=state,
                student_response=text[i],
                student_submission_time=timezone.now(),
                xqueue_submission_id=test_util.generate_new_xqueue_id(),
                xqueue_submission_key="",
                xqueue_queue_name="",
                location=location,
                course_id=course_id,
                next_grader_type=next_grader_type,
                posted_results_back_to_queue=True,
                previous_grader_type="BC",
                max_score=max_score,
                rubric=rubric,
                preferred_grader_type = next_grader_type,
            )

            sub.save()
            if add_grader:
                sub.previous_grader_type="IN"
                sub.save()
                grade = Grader(
                    score=score[i],
                    feedback="",
                    status_code=GraderStatus.success,
                    grader_id="",
                    grader_type="IN",
                    confidence=1,
                    is_calibration=set_as_calibration,
                )

                grade.submission = sub
                grade.save()

                success, rubric_targets=controller.rubric_functions.generate_targets_from_rubric(sub.rubric)
                scores=[]
                for z in xrange(0,len(rubric_targets)):
                    scores.append(random.randint(0,rubric_targets[z]))
                if import_rubric_scores:
                    score_item = rubric_scores[i]
                    if len(score_item) == len(scores):
                        scores = score_item
                        log.debug("Score: {0} Rubric Score: {1}".format(score[i], scores))

                controller.rubric_functions.generate_rubric_object(grade, scores, sub.rubric)

            if increment_ids:
                student_id+=1

        print ("Successfully imported {0} essays using configuration in file {1}.".format(
            min(essay_limit, len(text)),
            args[0],
        ))
<|endoftext|>


<commit_before>from __future__ import print_function, division, absolute_import, unicode_literals
from django.template import Library
from decimal import Decimal

register = Library()

def valid_numeric(arg):
    if isinstance(arg, (int, float, Decimal)):
        return arg
    try:
        return int(arg)
    except ValueError:
        return float(arg)

@register.filter
def sub(value, arg):
    """Subtracts the arg from the value."""
    try:
        return valid_numeric(value) - valid_numeric(arg)
    except (ValueError, TypeError):
        try:
            return value - arg
        except Exception:
            return ''
sub.is_safe = False

@register.filter
def mul(value, arg):
    """Multiplies the arg with the value."""
    try:
        return valid_numeric(value) * valid_numeric(arg)
    except (ValueError, TypeError):
        try:
            return value * arg
        except Exception:
            return ''
mul.is_safe = False

@register.filter()
def div(value, arg):
    """Divides the arg by the value."""
    try:
        return valid_numeric(value) / valid_numeric(arg)
    except (ValueError, TypeError):
        try:
            return value / arg
        except Exception:
            return ''
div.is_safe = False

@register.filter(name='abs')
def absolute(value):
    """Returns the absolute value."""
    try:
        return abs(valid_numeric(value))
    except (ValueError, TypeError):
        try:
            return abs(value)
        except Exception:
            return ''
absolute.is_safe = False

@register.filter(name='mod')
def mod(value, arg):
    """Returns the modulo value."""
    try:
        return valid_numeric(value) % valid_numeric(arg)
    except (ValueError, TypeError):
        try:
            return value % arg
        except Exception:
            return ''
mod.is_safe = False
<commit_msg>Code review fixes for previous 'Add Modulo' push<commit_after>from __future__ import print_function, division, absolute_import, unicode_literals
from django.template import Library
from decimal import Decimal

register = Library()

def valid_numeric(arg):
    if isinstance(arg, (int, float, Decimal)):
        return arg
    try:
        return int(arg)
    except ValueError:
        return float(arg)

@register.filter
def sub(value, arg):
    """Subtracts the arg from the value."""
    try:
        return valid_numeric(value) - valid_numeric(arg)
    except (ValueError, TypeError):
        try:
            return value - arg
        except Exception:
            return ''
sub.is_safe = False

@register.filter
def mul(value, arg):
    """Multiplies the arg with the value."""
    try:
        return valid_numeric(value) * valid_numeric(arg)
    except (ValueError, TypeError):
        try:
            return value * arg
        except Exception:
            return ''
mul.is_safe = False

@register.filter()
def div(value, arg):
    """Divides the arg by the value."""
    try:
        return valid_numeric(value) / valid_numeric(arg)
    except (ValueError, TypeError):
        try:
            return value / arg
        except Exception:
            return ''
div.is_safe = False

@register.filter(name='abs')
def absolute(value):
    """Returns the absolute value."""
    try:
        return abs(valid_numeric(value))
    except (ValueError, TypeError):
        try:
            return abs(value)
        except Exception:
            return ''
absolute.is_safe = False

@register.filter()
def mod(value, arg):
    """Returns the modulo value."""
    try:
        return valid_numeric(value) % valid_numeric(arg)
    except (ValueError, TypeError):
        try:
            return value % arg
        except Exception:
            return ''
mod.is_safe = False
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
#################################################################################
#                                                                               #
#    framework_helpers for OpenERP                                              #
#    Copyright (C) 2012 Akretion Benoît Guillot <benoit.guillot@akretion.com>   #
#                                                                               #
#    This program is free software: you can redistribute it and/or modify       #
#    it under the terms of the GNU Affero General Public License as             #
#    published by the Free Software Foundation, either version 3 of the         #
#    License, or (at your option) any later version.                            #
#                                                                               #
#    This program is distributed in the hope that it will be useful,            #
#    but WITHOUT ANY WARRANTY; without even the implied warranty of             #
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the              #
#    GNU Affero General Public License for more details.                        #
#                                                                               #
#    You should have received a copy of the GNU Affero General Public License   #
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.      #
#                                                                               #
#################################################################################

{
    'name': 'framework_helpers',
    'version': '0.1',
    'category': 'Generic Modules/Others',
    'license': 'AGPL-3',
    'description': """This module provides methods for the framework.""",
    'author': 'Akretion',
    'website': 'http://www.akretion.com/',
    'depends': ['base'],
    'init_xml': [],
    'update_xml': [],
    'demo_xml': [],
    'installable': True,
    'active': False,
}

<commit_msg>[FIX] framework_helpers: set the module version<commit_after># -*- encoding: utf-8 -*-
#################################################################################
#                                                                               #
#    framework_helpers for OpenERP                                              #
#    Copyright (C) 2012 Akretion Benoît Guillot <benoit.guillot@akretion.com>   #
#                                                                               #
#    This program is free software: you can redistribute it and/or modify       #
#    it under the terms of the GNU Affero General Public License as             #
#    published by the Free Software Foundation, either version 3 of the         #
#    License, or (at your option) any later version.                            #
#                                                                               #
#    This program is distributed in the hope that it will be useful,            #
#    but WITHOUT ANY WARRANTY; without even the implied warranty of             #
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the              #
#    GNU Affero General Public License for more details.                        #
#                                                                               #
#    You should have received a copy of the GNU Affero General Public License   #
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.      #
#                                                                               #
#################################################################################

{
    'name': 'framework_helpers',
    'version': '6.1.0',
    'category': 'Generic Modules/Others',
    'license': 'AGPL-3',
    'description': """This module provides methods for the framework.""",
    'author': 'Akretion',
    'website': 'http://www.akretion.com/',
    'depends': ['base'],
    'init_xml': [],
    'update_xml': [],
    'demo_xml': [],
    'installable': True,
    'active': False,
}

<|endoftext|>


<commit_before>import fileinput
import urllib.request
import re
import urllib.parse
import queue
import threading

class CompareMTGOBotPrices:
	def __init__(self):
		self.priceLists = []

	def start(self):
		self.loadSets()
		self.loadPriceLists()
		self.populatePriceLists()
		self.comparePriceLists()

	def loadSets(self):
		self.sets = []

		for code in fileinput.input('sets.txt'):
			self.sets.append(code.strip())

	def loadPriceLists(self):
		for url in fileinput.input('pricelists.txt'):
			url = url.strip()

			print('Loading file: ' + url)

			self.priceLists.append(PriceList(url))

	def populatePriceLists(self):
		q = queue.Queue()

		def populatePriceList(priceList):
			print('Requesting URL: ' + priceList.url)

			response = urllib.request.urlopen(priceList.url)

			response = response.read().decode('utf-8', 'ignore')

			priceList.load(response, self.sets)

		def processPriceListQueue (q):
			while True:
				populatePriceList(q.get())
				q.task_done()

		for i in range(8):
			worker = threading.Thread(target=processPriceListQueue, args=(q,))
			worker.setDaemon(True)
			worker.start()

		for priceList in self.priceLists:
			q.put(priceList)

		q.join()

	def comparePriceLists(self):
		for i in range(0, len(self.priceLists)):
			a = self.priceLists[i]

			aURL = urllib.parse.urlparse(a.url)

			for j in range(i + 1, len(self.priceLists)):
				b = self.priceLists[j]

				bURL = urllib.parse.urlparse(b.url)

				if aURL.netloc == bURL.netloc:
					continue

				print('Comparing ' + a.url + ' (A) and ' + b.url + ' (B)')

				comparison = a.compareTo(b)

				for cardComparison in comparison.aSellingLessThanB:
					sell = cardComparison.sellCard.sellPrice
					buy = cardComparison.buyCard.buyPrice

					if (cardComparison.profit < .25 or cardComparison.profit > 2):
						continue

					msg = cardComparison.buyCard.name + ':\nSELL ' + str(sell) + ' (A)'
					msg += ' BUY ' + str(buy) + ' (B) = ' + str(cardComparison.profit) + '\n'
					msg += cardComparison.sellCard.line + '\n'
					msg += cardComparison.buyCard.line + '\n\n'

					print(msg)

				for cardComparison in comparison.bSellingLessThanA:
					sell = cardComparison.sellCard.sellPrice
					buy = cardComparison.buyCard.buyPrice

					if (cardComparison.profit < .25 or cardComparison.profit > 2):
						continue

					msg = cardComparison.buyCard.name + ':\nSELL ' + str(sell) + ' (B)'
					msg += ' BUY ' + str(buy) + ' (A) = ' + str(cardComparison.profit) + '\n'
					msg += cardComparison.sellCard.line + '\n'
					msg += cardComparison.buyCard.line + '\n\n'

					print(msg)

class PriceList:
	def __init__(self, url):
		self.cards = []
		self.url = url

	def load (self, response, sets):
		#singlePricePattern = re.compile(r"([A-Z][a-z][a-zA-Z-',\/]+( [a-zA-Z-',\/0-9]+)*\*?).+?\b([0-9]+[0-9\.]*)")
		doublePricePattern = re.compile(r"([A-Z][a-z][a-zA-Z-',\/]+( [a-zA-Z-',\/0-9]+)*\*?).+?\b([0-9]+[0-9\.]*) +([0-9]+[0-9\.]*)")
		setsPattern = re.compile('\\b' + '|'.join(sets) + '\\b')

		for line in response.splitlines():
			#print('Parsing: ' + line)

			match = doublePricePattern.search(line)

			if match is None:
				continue # skip cards with only buy or sell price. can't tell if the price is buy or sell
				match = singlePricePattern.search(line)

				if match is None:
					continue

			setMatch = setsPattern.search(line)

			if setMatch is None:
				continue

			groups = match.groups()

			if (len(groups) >= 4):
				buy = float(groups[2])
				sell = float(groups[3])
			else:
				buy = None
				sell = float(groups[2])

			name = groups[0].strip()

			setCode = setMatch.group(0)

			if name.endswith('*'):
				foil = True

				name = name[0:len(name) - 1]
			else:
				if name.startswith('Foil '):
					foil = True

					name = name[len('Foil '):len(name)]
				else:
					foil = False

			card = Card(name, setCode, foil, buy, sell, line)

			if sell is None:
				sell = ''
			else:
				sell = str(sell)

			#print('Loaded: ' + str(card))

			self.cards.append(card)

	def compareTo (self, priceList):
		result = PriceListComparison(self, priceList)

		aSellingLessThanB = []
		bSellingLessThanA = []

		for a in self.cards:
			b = priceList.get(a)

			if b is None:
				continue

			if a.buyPrice is not None and b.sellPrice is not None:
				if a.buyPrice > b.sellPrice:
					bSellingLessThanA.append(CardComparison(a, b))

			if b.buyPrice is not None and a.sellPrice is not None:
				if b.buyPrice > a.sellPrice:
					aSellingLessThanB.append(CardComparison(b, a))

		result.aSellingLessThanB = sorted(aSellingLessThanB, key=lambda x: x.profit, reverse=True)
		result.bSellingLessThanA = sorted(bSellingLessThanA, key=lambda x: x.profit, reverse=True)

		return result

	def get (self, needle):
		for card in self.cards:
			if card.name == needle.name and card.foil == needle.foil and card.setCode == needle.setCode:
				return card

		return None

class Card:
	def __init__(self, name, setCode, foil, buyPrice, sellPrice, line):
		self.name = name
		self.buyPrice = buyPrice
		self.sellPrice = sellPrice
		self.line = line
		self.setCode = setCode
		self.foil = foil

	def __str__(self):
		return str({ 'name': self.name, 'setCode': self.setCode, 'foil': self.foil, 'buy': str(self.buyPrice), 'sell': str(self.sellPrice)})

class PriceListComparison:
	def __init__(self, a, b):
		self.priceListA = a
		self.priceListB = b
		self.aSellingLessThanB = []
		self.bSellingLessThanA = []

class CardComparison:
	def __init__(self, buy, sell):
		self.buyCard = buy
		self.sellCard = sell
		self.profit = buy.buyPrice - sell.sellPrice

compare = CompareMTGOBotPrices()
compare.start()<commit_msg>catch HTTP errors<commit_after>import fileinput
import urllib.request
import re
import urllib.parse
import queue
import threading
import urllib.error

class CompareMTGOBotPrices:
	def __init__(self):
		self.priceLists = []

	def start(self):
		self.loadSets()
		self.loadPriceLists()
		self.populatePriceLists()
		self.comparePriceLists()

	def loadSets(self):
		self.sets = []

		for code in fileinput.input('sets.txt'):
			self.sets.append(code.strip())

	def loadPriceLists(self):
		for url in fileinput.input('pricelists.txt'):
			url = url.strip()

			print('Loading file: ' + url)

			self.priceLists.append(PriceList(url))

	def populatePriceLists(self):
		q = queue.Queue()

		def populatePriceList(priceList):
			print('Requesting URL: ' + priceList.url)

			try:
				response = urllib.request.urlopen(priceList.url)

				response = response.read().decode('utf-8', 'ignore')

				priceList.load(response, self.sets)

				return True
			except urllib.error.HTTPError:
				return False

		def processPriceListQueue (q):
			while True:
				populatePriceList(q.get())
				q.task_done()

		for i in range(8):
			worker = threading.Thread(target=processPriceListQueue, args=(q,))
			worker.setDaemon(True)
			worker.start()

		for priceList in self.priceLists:
			q.put(priceList)

		q.join()

	def comparePriceLists(self):
		for i in range(0, len(self.priceLists)):
			a = self.priceLists[i]

			aURL = urllib.parse.urlparse(a.url)

			for j in range(i + 1, len(self.priceLists)):
				b = self.priceLists[j]

				bURL = urllib.parse.urlparse(b.url)

				if aURL.netloc == bURL.netloc:
					continue

				print('Comparing ' + a.url + ' (A) and ' + b.url + ' (B)')

				comparison = a.compareTo(b)

				for cardComparison in comparison.aSellingLessThanB:
					sell = cardComparison.sellCard.sellPrice
					buy = cardComparison.buyCard.buyPrice

					if (cardComparison.profit < .25 or cardComparison.profit > 2):
						continue

					msg = cardComparison.buyCard.name + ':\nSELL ' + str(sell) + ' (A)'
					msg += ' BUY ' + str(buy) + ' (B) = ' + str(cardComparison.profit) + '\n'
					msg += cardComparison.sellCard.line + '\n'
					msg += cardComparison.buyCard.line + '\n\n'

					print(msg)

				for cardComparison in comparison.bSellingLessThanA:
					sell = cardComparison.sellCard.sellPrice
					buy = cardComparison.buyCard.buyPrice

					if (cardComparison.profit < .25 or cardComparison.profit > 2):
						continue

					msg = cardComparison.buyCard.name + ':\nSELL ' + str(sell) + ' (B)'
					msg += ' BUY ' + str(buy) + ' (A) = ' + str(cardComparison.profit) + '\n'
					msg += cardComparison.sellCard.line + '\n'
					msg += cardComparison.buyCard.line + '\n\n'

					print(msg)

class PriceList:
	def __init__(self, url):
		self.cards = []
		self.url = url

	def load (self, response, sets):
		#singlePricePattern = re.compile(r"([A-Z][a-z][a-zA-Z-',\/]+( [a-zA-Z-',\/0-9]+)*\*?).+?\b([0-9]+[0-9\.]*)")
		doublePricePattern = re.compile(r"([A-Z][a-z][a-zA-Z-',\/]+( [a-zA-Z-',\/0-9]+)*\*?).+?\b([0-9]+[0-9\.]*) +([0-9]+[0-9\.]*)")
		setsPattern = re.compile('\\b' + '|'.join(sets) + '\\b')

		for line in response.splitlines():
			#print('Parsing: ' + line)

			match = doublePricePattern.search(line)

			if match is None:
				continue # skip cards with only buy or sell price. can't tell if the price is buy or sell
				match = singlePricePattern.search(line)

				if match is None:
					continue

			setMatch = setsPattern.search(line)

			if setMatch is None:
				continue

			groups = match.groups()

			if (len(groups) >= 4):
				buy = float(groups[2])
				sell = float(groups[3])
			else:
				buy = None
				sell = float(groups[2])

			name = groups[0].strip()

			setCode = setMatch.group(0)

			if name.endswith('*'):
				foil = True

				name = name[0:len(name) - 1]
			else:
				if name.startswith('Foil '):
					foil = True

					name = name[len('Foil '):len(name)]
				else:
					foil = False

			card = Card(name, setCode, foil, buy, sell, line)

			if sell is None:
				sell = ''
			else:
				sell = str(sell)

			#print('Loaded: ' + str(card))

			self.cards.append(card)

	def compareTo (self, priceList):
		result = PriceListComparison(self, priceList)

		aSellingLessThanB = []
		bSellingLessThanA = []

		for a in self.cards:
			b = priceList.get(a)

			if b is None:
				continue

			if a.buyPrice is not None and b.sellPrice is not None:
				if a.buyPrice > b.sellPrice:
					bSellingLessThanA.append(CardComparison(a, b))

			if b.buyPrice is not None and a.sellPrice is not None:
				if b.buyPrice > a.sellPrice:
					aSellingLessThanB.append(CardComparison(b, a))

		result.aSellingLessThanB = sorted(aSellingLessThanB, key=lambda x: x.profit, reverse=True)
		result.bSellingLessThanA = sorted(bSellingLessThanA, key=lambda x: x.profit, reverse=True)

		return result

	def get (self, needle):
		for card in self.cards:
			if card.name == needle.name and card.foil == needle.foil and card.setCode == needle.setCode:
				return card

		return None

class Card:
	def __init__(self, name, setCode, foil, buyPrice, sellPrice, line):
		self.name = name
		self.buyPrice = buyPrice
		self.sellPrice = sellPrice
		self.line = line
		self.setCode = setCode
		self.foil = foil

	def __str__(self):
		return str({ 'name': self.name, 'setCode': self.setCode, 'foil': self.foil, 'buy': str(self.buyPrice), 'sell': str(self.sellPrice)})

class PriceListComparison:
	def __init__(self, a, b):
		self.priceListA = a
		self.priceListB = b
		self.aSellingLessThanB = []
		self.bSellingLessThanA = []

class CardComparison:
	def __init__(self, buy, sell):
		self.buyCard = buy
		self.sellCard = sell
		self.profit = buy.buyPrice - sell.sellPrice

compare = CompareMTGOBotPrices()
compare.start()<|endoftext|>


<commit_before>import datetime
import logging
from email.utils import parseaddr
import os
import requests
import vobject

from django.contrib.auth.models import User
from django.contrib.sites.models import Site
from django.conf import settings
from django.core.cache import cache
from django.core.mail import EmailMultiAlternatives
from django.template.loader import get_template
from django.utils import timezone

import contacts as contact_settings
from contacts.models import (
    Book,
    Contact,
    ContactField,
    LogEntry,
    Tag,
)

sentry = logging.getLogger("raven")
logger = logging.getLogger("loggly_logs")

def process_incoming_email(message):
    _, sender = parseaddr(message.get('sender', ''))
    _, recipient = parseaddr(message.get('recipient', ''))

    try:
        user = User.objects.get(email=sender)
    except User.DoesNotExist:
        pass

    try:
        book_id = recipient.split('@')[0].split('-')[-1]
        book = Book.objects.filter_for_user(user, id=book_id)[0]
    except (Book.DoesNotExist, IndexError):
        pass

    if not (user and book):
        sentry.error("Somehow we got an email without user or book", extra=message)
        return

    contacts_updated = []
    tag = None
    to = message.get('To', '').split(',')
    to += (message.get('Cc', '').split(','))
    to += (message.get('Bcc', '').split(','))
    for item in set(to):
        name, email = parseaddr(item)
        if email and email != recipient:
            email_parts = email.split('@')
            if len(email_parts) > 1 and email_parts[1] == 'googlegroups.com':
                tag_name = name if name else email_parts[0]
                tag = Tag.objects.create(
                    book=book,
                    tag=tag_name,
                )
                continue
            try:
                contact = ContactField.objects.filter(
                    kind=contact_settings.FIELD_TYPE_EMAIL,
                    value=email,
                    contact__book=book,
                )[0].contact
            except (ContactField.DoesNotExist, IndexError):
                if not name:
                    name = email
                contact = Contact.objects.create(
                    name=name,
                    book=book,
                )
                ContactField.objects.create(
                    contact=contact,
                    kind=contact_settings.FIELD_TYPE_EMAIL,
                    value=email,
                    label="Email",
                    preferred=True,
                )
            log=LogEntry.objects.create(
                contact=contact,
                kind='email',
                logged_by=user,
                time=timezone.now(),
                notes='Subject: {}'.format(message.get('subject')),
            )
            contact.update_last_contact_from_log(log)
            contacts_updated.append(contact)
    if tag and contacts_updated:
        for contact in contacts_updated:
            contact.tags.add(tag)
            contact.save()

def process_vcard_upload(message):
    logger.info("Starting vcard processing")
    try:
        user = User.objects.get(id=message.get('user_id'))
        book = Book.objects.get(bookowner__user=user, id=message.get('book_id'))
    except Book.DoesNotExist:
        sentry.error("Bad book passed to vCard import job", exc_info=True)
        return
    except User.DoesNotExist:
        sentry.error("Bad user passed to vCard import job", exc_info=True)
        return
    try:
        url = message.get('url')
        local_filename = url.split('/')[-1]
        r = requests.get(url, stream=True)
        logger.info("vcard: Trying to write file")
        with open(local_filename, 'wb') as f:
            for chunk in r.iter_content(chunk_size=1024):
                if chunk:
                    f.write(chunk)
        logger.info("vcard: file written")
        with open(local_filename, 'r') as f:
            logger.info("vcard: file opened")
            count = 0
            cards = []
            current_lines = []
            possible_keys = set()
            for line in f:
                if line.strip() == 'END:VCARD':
                    current_lines.append(line)
                    card = ''.join(current_lines)
                    cards.append(card)
                    count += 1
                    current_lines = []
                else:
                    current_lines.append(line)
            for card in cards:
                v = vobject.readOne(card)
                logger.info(v)
                try:
                    name = str(v.n.value).strip()
                    if name:
                        contact = Contact.objects.create(book=book, name=name)
                        log = LogEntry.objects.create(contact=contact, kind='edit', logged_by=user)
                        contact.update_last_contact_from_log(log)
                    else:
                        continue
                except:
                    continue
                try:
                    for adr in v.adr_list:
                        ContactField.objects.create(
                            contact=contact,
                            label='Address',
                            kind='address',
                            value=str(adr.value),
                        )
                except:
                    pass
                try:
                    for email in v.email_list:
                        ContactField.objects.create(
                            contact=contact,
                            label='Email',
                            kind='email',
                            value=str(email.value),
                        )
                except:
                    pass
                try:
                    for url in v.url_list:
                        ContactField.objects.create(
                            contact=contact,
                            label='URL',
                            kind='url',
                            value=str(url.value),
                        )
                except:
                    pass
                try:
                    for bday in v.bday_list:
                        ContactField.objects.create(
                            contact=contact,
                            label='Birthday',
                            kind='date',
                            value=str(bday.value),
                        )
                except:
                    pass
                try:
                    for org in v.org_list:
                        if isinstance(org.value, list):
                            org = str(org.value[0])
                        else:
                            org = str(org.value)
                        ContactField.objects.create(
                            contact=contact,
                            label='Organization',
                            kind='text',
                            value=str(org),
                        )
                except:
                    pass
                try:
                    for title in v.title_list:
                        ContactField.objects.create(
                            contact=contact,
                            label='Title',
                            kind='text',
                            value=str(title.value),
                        )
                except:
                    pass
        os.remove(local_filename)
    except Exception as e:
        sentry.error("Error processing vCard upload", exc_info=True)
<commit_msg>Try not streaming vCard<commit_after>import datetime
import logging
from email.utils import parseaddr
import os
import requests
import vobject

from django.contrib.auth.models import User
from django.contrib.sites.models import Site
from django.conf import settings
from django.core.cache import cache
from django.core.mail import EmailMultiAlternatives
from django.template.loader import get_template
from django.utils import timezone

import contacts as contact_settings
from contacts.models import (
    Book,
    Contact,
    ContactField,
    LogEntry,
    Tag,
)

sentry = logging.getLogger("raven")
logger = logging.getLogger("loggly_logs")

def process_incoming_email(message):
    _, sender = parseaddr(message.get('sender', ''))
    _, recipient = parseaddr(message.get('recipient', ''))

    try:
        user = User.objects.get(email=sender)
    except User.DoesNotExist:
        pass

    try:
        book_id = recipient.split('@')[0].split('-')[-1]
        book = Book.objects.filter_for_user(user, id=book_id)[0]
    except (Book.DoesNotExist, IndexError):
        pass

    if not (user and book):
        sentry.error("Somehow we got an email without user or book", extra=message)
        return

    contacts_updated = []
    tag = None
    to = message.get('To', '').split(',')
    to += (message.get('Cc', '').split(','))
    to += (message.get('Bcc', '').split(','))
    for item in set(to):
        name, email = parseaddr(item)
        if email and email != recipient:
            email_parts = email.split('@')
            if len(email_parts) > 1 and email_parts[1] == 'googlegroups.com':
                tag_name = name if name else email_parts[0]
                tag = Tag.objects.create(
                    book=book,
                    tag=tag_name,
                )
                continue
            try:
                contact = ContactField.objects.filter(
                    kind=contact_settings.FIELD_TYPE_EMAIL,
                    value=email,
                    contact__book=book,
                )[0].contact
            except (ContactField.DoesNotExist, IndexError):
                if not name:
                    name = email
                contact = Contact.objects.create(
                    name=name,
                    book=book,
                )
                ContactField.objects.create(
                    contact=contact,
                    kind=contact_settings.FIELD_TYPE_EMAIL,
                    value=email,
                    label="Email",
                    preferred=True,
                )
            log=LogEntry.objects.create(
                contact=contact,
                kind='email',
                logged_by=user,
                time=timezone.now(),
                notes='Subject: {}'.format(message.get('subject')),
            )
            contact.update_last_contact_from_log(log)
            contacts_updated.append(contact)
    if tag and contacts_updated:
        for contact in contacts_updated:
            contact.tags.add(tag)
            contact.save()

def process_vcard_upload(message):
    logger.info("Starting vcard processing")
    try:
        user = User.objects.get(id=message.get('user_id'))
        book = Book.objects.get(bookowner__user=user, id=message.get('book_id'))
    except Book.DoesNotExist:
        sentry.error("Bad book passed to vCard import job", exc_info=True)
        return
    except User.DoesNotExist:
        sentry.error("Bad user passed to vCard import job", exc_info=True)
        return
    try:
        url = message.get('url')
        local_filename = url.split('/')[-1]
        r = requests.get(url)
        logger.info("vcard: Trying to write file")
        with open(local_filename, 'wb') as f:
            f.write(r.content)
        logger.info("vcard: file written")
        with open(local_filename, 'r') as f:
            logger.info("vcard: file opened")
            count = 0
            cards = []
            current_lines = []
            possible_keys = set()
            for line in f:
                if line.strip() == 'END:VCARD':
                    current_lines.append(line)
                    card = ''.join(current_lines)
                    cards.append(card)
                    count += 1
                    current_lines = []
                else:
                    current_lines.append(line)
            for card in cards:
                v = vobject.readOne(card)
                logger.info(v)
                try:
                    name = str(v.n.value).strip()
                    if name:
                        contact = Contact.objects.create(book=book, name=name)
                        log = LogEntry.objects.create(contact=contact, kind='edit', logged_by=user)
                        contact.update_last_contact_from_log(log)
                    else:
                        continue
                except:
                    continue
                try:
                    for adr in v.adr_list:
                        ContactField.objects.create(
                            contact=contact,
                            label='Address',
                            kind='address',
                            value=str(adr.value),
                        )
                except:
                    pass
                try:
                    for email in v.email_list:
                        ContactField.objects.create(
                            contact=contact,
                            label='Email',
                            kind='email',
                            value=str(email.value),
                        )
                except:
                    pass
                try:
                    for url in v.url_list:
                        ContactField.objects.create(
                            contact=contact,
                            label='URL',
                            kind='url',
                            value=str(url.value),
                        )
                except:
                    pass
                try:
                    for bday in v.bday_list:
                        ContactField.objects.create(
                            contact=contact,
                            label='Birthday',
                            kind='date',
                            value=str(bday.value),
                        )
                except:
                    pass
                try:
                    for org in v.org_list:
                        if isinstance(org.value, list):
                            org = str(org.value[0])
                        else:
                            org = str(org.value)
                        ContactField.objects.create(
                            contact=contact,
                            label='Organization',
                            kind='text',
                            value=str(org),
                        )
                except:
                    pass
                try:
                    for title in v.title_list:
                        ContactField.objects.create(
                            contact=contact,
                            label='Title',
                            kind='text',
                            value=str(title.value),
                        )
                except:
                    pass
        os.remove(local_filename)
    except Exception as e:
        sentry.error("Error processing vCard upload", exc_info=True)
<|endoftext|>


<commit_before>#! /usr/bin/env python
"""Fetch remote BMI models."""
from __future__ import print_function

import os
import sys
import re
import warnings

import yaml

from .utils import cd
from .git import git_repo_name, git_clone_or_update, git_repo_sha
from .project import empty_bmi_project, add_bmi_component
from . import api
from .errors import MissingFileError, ParseError

def _cache_dir_from_repo(repo, branch='master'):
    """Name of cache directory for a git repository.

    Parameters
    ----------
    repo : str
        URL of git repository.
    branch : str, optional
        Branch of the git repository.

    Returns
    -------
    str
        Name of the directory to use for the cache.
    """
    cache_dir = '${repo}-${sha}'.format(repo=git_repo_name(repo),
                                        sha=git_repo_sha(repo, branch=branch))
    return os.path.join('cache', cache_dir)

def _component_name_from_repo(repo, name=None):
    """Get fully-qualified component name from repository name.

    Parameters
    ----------
    repo : str
        Name of a repository.
    name : str, optional
        Name of component.

    Returns
    -------
    str
        Fully-qualified component name.
    """
    name = name or re.sub('-', '', os.path.basename(repo).title())

    return '.'.join(['model', name])

def _parse_repo_line(line):
    """Parse a line containing the location of a BMI repository.

    Parameters
    ----------
    line : str
        Location of repository.

    Returns
    -------
    (repo, branch)
        Tuple of repository url, and a branch name.
    """
    if ',' in line:
        repo, branch = line.split(',')
    else:
        repo, branch = line, 'master'

    return repo.strip(), branch.strip()

def _get_bmi_from_repo(repo, prefix='/usr/local'):
    """Get BMI description from a git repository.

    Parameters
    ----------
    repo : str
        url or path to repository.
    prefix : str
        Path to installation prefix for BMI implementation

    Returns
    -------
    dict
        BMI description.
    """
    if repo.startswith('http'):
        repo, branch = _parse_repo_line(repo)

        cache_dir = _cache_dir_from_repo(repo, branch=branch)

        git_clone_or_update(repo, dir=cache_dir, branch=branch)
    else:
        cache_dir = repo

    with cd(cache_dir) as _:
        api.execute_api_build(prefix=prefix)
        bmi = api.load_all()

    #bmi['name'] = _component_name_from_repo(repo, bmi.get('name', None))

    return bmi

def fetch_bmi_components(repos):
    proj = empty_bmi_project()
    for repo in repos:
        try:
            add_bmi_component(proj, _get_bmi_from_repo(repo,
                                                       prefix=args.prefix))
        except MissingFileError as err:
            warnings.warn('Skipping %s: missing file (%s)' % (repo, err))
        except ParseError as err:
            warnings.warn('Skipping %s: parse error (%s)' % (repo, err))

    return yaml.dump(proj, default_flow_style=False)
<commit_msg>Removed dollar signs from cache dir string.<commit_after>#! /usr/bin/env python
"""Fetch remote BMI models."""
from __future__ import print_function

import os
import sys
import re
import warnings

import yaml

from .utils import cd
from .git import git_repo_name, git_clone_or_update, git_repo_sha
from .project import empty_bmi_project, add_bmi_component
from . import api
from .errors import MissingFileError, ParseError

def _cache_dir_from_repo(repo, branch='master'):
    """Name of cache directory for a git repository.

    Parameters
    ----------
    repo : str
        URL of git repository.
    branch : str, optional
        Branch of the git repository.

    Returns
    -------
    str
        Name of the directory to use for the cache.
    """
    cache_dir = '{repo}-{sha}'.format(repo=git_repo_name(repo),
                                      sha=git_repo_sha(repo, branch=branch))
    return os.path.join('cache', cache_dir)

def _component_name_from_repo(repo, name=None):
    """Get fully-qualified component name from repository name.

    Parameters
    ----------
    repo : str
        Name of a repository.
    name : str, optional
        Name of component.

    Returns
    -------
    str
        Fully-qualified component name.
    """
    name = name or re.sub('-', '', os.path.basename(repo).title())

    return '.'.join(['model', name])

def _parse_repo_line(line):
    """Parse a line containing the location of a BMI repository.

    Parameters
    ----------
    line : str
        Location of repository.

    Returns
    -------
    (repo, branch)
        Tuple of repository url, and a branch name.
    """
    if ',' in line:
        repo, branch = line.split(',')
    else:
        repo, branch = line, 'master'

    return repo.strip(), branch.strip()

def _get_bmi_from_repo(repo, prefix='/usr/local'):
    """Get BMI description from a git repository.

    Parameters
    ----------
    repo : str
        url or path to repository.
    prefix : str
        Path to installation prefix for BMI implementation

    Returns
    -------
    dict
        BMI description.
    """
    if repo.startswith('http'):
        repo, branch = _parse_repo_line(repo)

        cache_dir = _cache_dir_from_repo(repo, branch=branch)

        git_clone_or_update(repo, dir=cache_dir, branch=branch)
    else:
        cache_dir = repo

    with cd(cache_dir) as _:
        api.execute_api_build(prefix=prefix)
        bmi = api.load_all()

    #bmi['name'] = _component_name_from_repo(repo, bmi.get('name', None))

    return bmi

def fetch_bmi_components(repos):
    proj = empty_bmi_project()
    for repo in repos:
        try:
            add_bmi_component(proj, _get_bmi_from_repo(repo,
                                                       prefix=args.prefix))
        except MissingFileError as err:
            warnings.warn('Skipping %s: missing file (%s)' % (repo, err))
        except ParseError as err:
            warnings.warn('Skipping %s: parse error (%s)' % (repo, err))

    return yaml.dump(proj, default_flow_style=False)
<|endoftext|>