<commit_before>import asyncio
import datetime
import traceback
import random
import os

import discord
import plotly.plotly as py
import plotly.graph_objs as go

import utils
import errors
from enums import PrivilegeLevel
from servermodule import ServerModule
import cmd

class ServerActivityStatistics(ServerModule):

   _SECRET_TOKEN = utils.SecretToken()

   RECOMMENDED_CMD_NAMES = ["stats", "serverstats"]

   MODULE_NAME = "Server Activity Statistics"
   MODULE_SHORT_DESCRIPTION = "Bot debugging tools."

   _HELP_SUMMARY_LINES = """
>>> PRIVILEGE LEVEL 8000 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
`{pf}stats [eval] [bin] [graph]` - Generate server statistics graph. (`{pf}stats` for details.)
   """.strip().splitlines()

   _HELP_DETAIL_LINES = """
>>> PRIVILEGE LEVEL 8000 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
`{pf}stats [eval] [bin] [graph]` - Generate server statistics graph. (`{pf}stats` for details.)
>>> PRIVILEGE LEVEL 9001 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
`{pf}stats login` - Re-read settings file to log into plotly.
   """.strip().splitlines()

   DEFAULT_SHARED_SETTINGS = {
      "login username": "PLACEHOLDER",
      "api key": "PLACEHOLDER",
   }

   _cmd_dict = {} # Command Dictionary

   async def _initialize(self, resources):
      self._res = resources
      self._client = self._res.client

      self._log_in_from_file()
      return

   async def msg_preprocessor(self, content, msg, default_cmd_prefix):
      return content

   async def process_cmd(self, substr, msg, privilege_level):
      (left, right) = utils.separate_left_word(substr)
      execute_regular_cmd = False
      try:
         cmd_to_execute = cmd.get(self._cmd_dict, left, privilege_level)
         execute_regular_cmd = True
      except (errors.InvalidCommandArgumentsError, errors.CommandPrivilegeError):
         pass
      if execute_regular_cmd:
         await cmd_to_execute(self, right, msg, privilege_level)
      else: # Now, let's process

         if privilege_level < PrivilegeLevel.ADMIN:
            raise errors.CommandPrivilegeError

         if len(substr) == 0:
            buf = "**How to use this command:**"
            buf += "\n\n" + self._get_usage_info()
            await self._client.send_msg(msg, buf)
            raise errors.OperationAborted

         eval_obj = None
         try:
            eval_obj = self._sg_argument1[left]
         except KeyError:
            buf = "Error: Unknown evaluation function."
            buf += "\n\n" + self._get_usage_info()
            await self._client.send_msg(msg, buf)
            raise errors.OperationAborted
         eval_obj = eval_obj(self) # Get the eval function

         bin_obj = None
         (left2, right2) = utils.separate_left_word(right)
         try:
            bin_obj = self._sg_argument2[left2]
         except KeyError:
            buf = "Error: Unknown binning function."
            buf += "\n\n" + self._get_usage_info()
            await self._client.send_msg(msg, buf)
            raise errors.OperationAborted
         bin_obj = bin_obj(self) # Get the bin function

         graph_fn = None
         (left3, right3) = utils.separate_left_word(right2)
         try:
            graph_fn = self._sg_argument3[left3]
         except KeyError:
            buf = "Error: Unknown graphing function."
            buf += "\n\n" + self._get_usage_info()
            await self._client.send_msg(msg, buf)
            raise errors.OperationAborted
         graph_fn = graph_fn(self) # Get the graphing function

         # right3 can be used for other things if need be...

         await self._client.send_msg(msg, "Generating graph. Please wait...")
         (data, x_vals) = await self._sg3_generate_graph_data(msg.channel, eval_obj["fn"], bin_obj["fn"])

         # Compile graph function kwargs
         graph_kwargs = {
            "title": eval_obj["title"] + " " + bin_obj["title"],

            "x_vals": x_vals,
            "y_vals": data,

            "x_axis_title": bin_obj["axis"],

            "y_axis_title": eval_obj["axis"],
         }

         await graph_fn(msg.channel, **graph_kwargs)

      return

   @cmd.add(_cmd_dict, "login")
   @cmd.minimum_privilege(PrivilegeLevel.BOT_OWNER)
   async def _cmdf_login(self, substr, msg, privilege_level):
      self._log_in_from_file()
      await self._client.send_msg(msg, "Login details have been loaded.")
      return

   ###############################################
   ### SIMPLE GRAPHING - STEP 1 (GRAPH VALUES) ###
   ###############################################

   # These return value evaluation functions.
   # Sees every message once with parameter "d" to evaluate.
   # Returned value accumulates, and this cumulative value is passed in with parameter "p".

   # You can think of this as the function that generates the y-value.

   # This factory method must return a dict where:
   #     fn    -> The value evaluation function.
   #     axis  -> Axis title
   #     title -> Customized text to use in the graph title.

   _sg_argument1 = {} # Command Dictionary
   _sg_arghelp1 = []

   _sg_arghelp1.append("`chars` - Total characters in messages.")
   @cmd.add(_sg_argument1, "chars")
   def _sg1_chars(self):
      ret = {
         "fn": lambda p, d: p + len(d["c"]),
         "axis": "total characters typed into messages",
         "title": "Characters Entered",
      }
      return ret

   _sg_arghelp1.append("`msgs` - Total messages sent.")
   @cmd.add(_sg_argument1, "msgs")
   def _sg1_msgs(self):
      ret = {
         "fn": lambda p, d: p + 1,
         "axis": "total messages sent",
         "title": "Messages Sent",
      }
      return ret

   _sg_arghelp1.append("`uuser` - Number of unique users that sent a message.")
   @cmd.add(_sg_argument1, "uuser")
   def _sg1_uuser(self):
      user_ids_seen = {}
      def uuser(p, d):
         user_id = d["a"]
         try:
            temp = user_ids_seen[user_id]
            return p
         except KeyError:
            user_ids_seen[user_id] = None
            return p + 1
      ret = {
         "fn": uuser,
         "axis": "unique users that sent a message",
         "title": "Unique Users",
      }
      return ret

   #############################################
   ### SIMPLE GRAPHING - STEP 2 (GRAPH BINS) ###
   #############################################

   # These return binning functions, which determines which x-value a message will be binned in.

   # This factory method must return a dict where:
   #     fn    -> The bin evaluation function.
   #     axis  -> Axis title
   #     title -> Customized text to use in the graph title.

   _sg_argument2 = {} # Command Dictionary
   _sg_arghelp2 = []

   _sg_arghelp2.append("`eachday` - Each day of the calendar.")
   @cmd.add(_sg_argument2, "eachday")
   def _sg2_eachday(self):
      now = utils.datetime_rounddown_to_day(datetime.datetime.utcnow())
      now += datetime.timedelta(days=1)
      ret = {
         "fn": lambda d: (now - d["t"]).days,
         "axis": "Each day of the server's life (left = earliest)",
         "title": "Each Day",
      }
      return ret

   ########################################################
   ### SIMPLE GRAPHING - STEP 3 (GENERATING GRAPH DATA) ###
   ########################################################

   # This function generates graph data based on the value evaluation
   # and binning functions.

   async def _sg3_generate_graph_data(self, channel, measured, bins):
      data_temp = {} # Maps day delta -> chars sent
      for ch in self._res.server.channels:
         for msg_dict in self._res.message_cache_read(self._res.server.id, ch.id):
            bin_value = bins(msg_dict)
            prev = 0
            try:
               prev = data_temp[bin_value]
            except KeyError: # May also catch msg_dict keyerror...
               pass
            data_temp[bin_value] = measured(prev, msg_dict)
      days_ago = 0
      data = []
      while bool(data_temp): # While dictionary still has data
         content_len = 0
         try:
            content_len = data_temp[days_ago]
            del data_temp[days_ago]
         except KeyError:
            pass
         data.insert(0, content_len)
         days_ago += 1

      x_vals = []
      i = 1
      for point in data:
         x_vals.append(i)
         i += 1

      return (data, x_vals)

   ##############################################
   ### SIMPLE GRAPHING - STEP 4 (DATA OUTPUT) ###
   ##############################################

   # This determines what form the output takes.

   _sg_argument3 = {} # Command Dictionary
   _sg_arghelp3 = []

   _sg_arghelp3.append("`vbar` - Vertical bar graph.")
   @cmd.add(_sg_argument3, "vbar")
   def _sg4_vbar(self):
      async def function(channel, **kwargs):
         plotly_data = [
            go.Bar(
               x = kwargs["x_vals"],
               y = kwargs["y_vals"]
            )
         ]
         plotly_layout = go.Layout(
            title = kwargs["title"],
            xaxis = dict(
               title = kwargs["x_axis_title"],
            ),
            yaxis = dict(
               title = kwargs["y_axis_title"],
            ),
         )
         await self._send_plotly_graph_object(channel, plotly_data, plotly_layout)
         return
      return function

   ######################
   ### OTHER SERVICES ###
   ######################

   def _log_in_from_file(self):
      shared_settings = self._res.get_shared_settings()

      if shared_settings is None:
         shared_settings = self.DEFAULT_SHARED_SETTINGS
         self._res.save_shared_settings(shared_settings)

      try:
         username = shared_settings["login username"]
      except KeyError:
         username = self.DEFAULT_SHARED_SETTINGS["login username"]
         shared_settings["login username"] = username

      try:
         api_key = shared_settings["api key"]
      except KeyError:
         api_key = self.DEFAULT_SHARED_SETTINGS["api key"]
         shared_settings["api key"] = api_key

      py.sign_in(username, api_key)

      self._res.save_shared_settings(shared_settings)
      return

   # This is a utility function used by graph generating functions.
   async def _send_plotly_graph_object(self, channel, data, layout):
      temp_filename = "temp" + str(random.getrandbits(128))
      temp_file_ext = ".png"
      try:
         py.image.save_as({'data':data, 'layout':layout}, temp_filename, format='png')
      except:
         print(traceback.format_exc())
         buf = "Unknown error occurred. Maybe you forgot to sign in..."
         buf += "\n(Bot owner must manually enter plotly login details in settings files,"
         buf += " and use `/stats login` for it take effect.)"
         await self._client.send_msg(msg, buf)
         raise errors.OperationAborted
      await self._client.perm_send_file(channel, temp_filename + temp_file_ext)
      os.remove(temp_filename + temp_file_ext)
      print("GRAPH SENT!")
      return

   def _get_usage_info(self):
      buf = "**Argument 1 (Message Evaluation) is one of:**"
      for line in self._sg_arghelp1:
         buf += "\n" + line
      buf += "\n\n**Argument 2 (Binning) is one of:**"
      for line in self._sg_arghelp2:
         buf += "\n" + line
      buf += "\n\n**Argument 3 (Graph Type) is one of:**"
      for line in self._sg_arghelp3:
         buf += "\n" + line
      buf += "\n\n**Example:** `" + self._res.cmd_prefix + self._cmd_names[0]
      buf += " chars eachday vbar` - Bar graph of all characters"
      buf += " received by the server each day."
      return buf

<commit_msg>Fixed uuser argument Server Activity Statistics.<commit_after>import asyncio
import datetime
import traceback
import random
import os

import discord
import plotly.plotly as py
import plotly.graph_objs as go

import utils
import errors
from enums import PrivilegeLevel
from servermodule import ServerModule
import cmd

class ServerActivityStatistics(ServerModule):

   _SECRET_TOKEN = utils.SecretToken()

   RECOMMENDED_CMD_NAMES = ["stats", "serverstats"]

   MODULE_NAME = "Server Activity Statistics"
   MODULE_SHORT_DESCRIPTION = "Bot debugging tools."

   _HELP_SUMMARY_LINES = """
>>> PRIVILEGE LEVEL 8000 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
`{pf}stats [eval] [bin] [graph]` - Generate server statistics graph. (`{pf}stats` for details.)
   """.strip().splitlines()

   _HELP_DETAIL_LINES = """
>>> PRIVILEGE LEVEL 8000 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
`{pf}stats [eval] [bin] [graph]` - Generate server statistics graph. (`{pf}stats` for details.)
>>> PRIVILEGE LEVEL 9001 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
`{pf}stats login` - Re-read settings file to log into plotly.
   """.strip().splitlines()

   DEFAULT_SHARED_SETTINGS = {
      "login username": "PLACEHOLDER",
      "api key": "PLACEHOLDER",
   }

   _cmd_dict = {} # Command Dictionary

   async def _initialize(self, resources):
      self._res = resources
      self._client = self._res.client

      self._log_in_from_file()
      return

   async def msg_preprocessor(self, content, msg, default_cmd_prefix):
      return content

   async def process_cmd(self, substr, msg, privilege_level):
      (left, right) = utils.separate_left_word(substr)
      execute_regular_cmd = False
      try:
         cmd_to_execute = cmd.get(self._cmd_dict, left, privilege_level)
         execute_regular_cmd = True
      except (errors.InvalidCommandArgumentsError, errors.CommandPrivilegeError):
         pass
      if execute_regular_cmd:
         await cmd_to_execute(self, right, msg, privilege_level)
      else: # Now, let's process

         if privilege_level < PrivilegeLevel.ADMIN:
            raise errors.CommandPrivilegeError

         if len(substr) == 0:
            buf = "**How to use this command:**"
            buf += "\n\n" + self._get_usage_info()
            await self._client.send_msg(msg, buf)
            raise errors.OperationAborted

         eval_obj = None
         try:
            eval_obj = self._sg_argument1[left]
         except KeyError:
            buf = "Error: Unknown evaluation function."
            buf += "\n\n" + self._get_usage_info()
            await self._client.send_msg(msg, buf)
            raise errors.OperationAborted
         eval_obj = eval_obj(self) # Get the eval function

         bin_obj = None
         (left2, right2) = utils.separate_left_word(right)
         try:
            bin_obj = self._sg_argument2[left2]
         except KeyError:
            buf = "Error: Unknown binning function."
            buf += "\n\n" + self._get_usage_info()
            await self._client.send_msg(msg, buf)
            raise errors.OperationAborted
         bin_obj = bin_obj(self) # Get the bin function

         graph_fn = None
         (left3, right3) = utils.separate_left_word(right2)
         try:
            graph_fn = self._sg_argument3[left3]
         except KeyError:
            buf = "Error: Unknown graphing function."
            buf += "\n\n" + self._get_usage_info()
            await self._client.send_msg(msg, buf)
            raise errors.OperationAborted
         graph_fn = graph_fn(self) # Get the graphing function

         # right3 can be used for other things if need be...

         await self._client.send_msg(msg, "Generating graph. Please wait...")
         (data, x_vals) = await self._sg3_generate_graph_data(msg.channel, eval_obj["fn"], bin_obj["fn"])

         # Compile graph function kwargs
         graph_kwargs = {
            "title": eval_obj["title"] + " " + bin_obj["title"],

            "x_vals": x_vals,
            "y_vals": data,

            "x_axis_title": bin_obj["axis"],

            "y_axis_title": eval_obj["axis"],
         }

         await graph_fn(msg.channel, **graph_kwargs)

      return

   @cmd.add(_cmd_dict, "login")
   @cmd.minimum_privilege(PrivilegeLevel.BOT_OWNER)
   async def _cmdf_login(self, substr, msg, privilege_level):
      self._log_in_from_file()
      await self._client.send_msg(msg, "Login details have been loaded.")
      return

   ###############################################
   ### SIMPLE GRAPHING - STEP 1 (GRAPH VALUES) ###
   ###############################################

   # These return value evaluation functions.
   # Sees every message once with parameter "d" to evaluate.
   # Returned value accumulates, and this cumulative value is passed in with parameter "p".
   # Associated bin value is parameter "b".

   # You can think of this as the function that generates the y-value.

   # This factory method must return a dict where:
   #     fn    -> The value evaluation function.
   #     axis  -> Axis title
   #     title -> Customized text to use in the graph title.

   _sg_argument1 = {} # Command Dictionary
   _sg_arghelp1 = []

   _sg_arghelp1.append("`chars` - Total characters in messages.")
   @cmd.add(_sg_argument1, "chars")
   def _sg1_chars(self):
      ret = {
         "fn": lambda p, d, b: p + len(d["c"]),
         "axis": "total characters typed into messages",
         "title": "Characters Entered",
      }
      return ret

   _sg_arghelp1.append("`msgs` - Total messages sent.")
   @cmd.add(_sg_argument1, "msgs")
   def _sg1_msgs(self):
      ret = {
         "fn": lambda p, d, b: p + 1,
         "axis": "total messages sent",
         "title": "Messages Sent",
      }
      return ret

   _sg_arghelp1.append("`uuser` - Number of unique users that sent a message.")
   @cmd.add(_sg_argument1, "uuser")
   def _sg1_uuser(self):
      bins_dict = {} # Maps bin value -> user id -> literally any possible value
      def uuser(p, d, b):
         user_id = d["a"]
         try:
            users_dict = bins_dict[b]
            try:
               temp = users_dict[user_id]
               return p
            except KeyError:
               users_dict[user_id] = None
               return p + 1
         except KeyError:
            bins_dict[b] = {user_id:None}
            return p + 1
      ret = {
         "fn": uuser,
         "axis": "unique users that sent a message",
         "title": "Unique Users",
      }
      return ret

   #############################################
   ### SIMPLE GRAPHING - STEP 2 (GRAPH BINS) ###
   #############################################

   # These return binning functions, which determines which x-value a message will be binned in.

   # This factory method must return a dict where:
   #     fn    -> The bin evaluation function.
   #     axis  -> Axis title
   #     title -> Customized text to use in the graph title.

   _sg_argument2 = {} # Command Dictionary
   _sg_arghelp2 = []

   _sg_arghelp2.append("`eachday` - Each day of the calendar.")
   @cmd.add(_sg_argument2, "eachday")
   def _sg2_eachday(self):
      now = utils.datetime_rounddown_to_day(datetime.datetime.utcnow())
      now += datetime.timedelta(days=1)
      ret = {
         "fn": lambda d: (now - d["t"]).days,
         "axis": "Each day of the server's life (left = earliest)",
         "title": "Each Day",
      }
      return ret

   ########################################################
   ### SIMPLE GRAPHING - STEP 3 (GENERATING GRAPH DATA) ###
   ########################################################

   # This function generates graph data based on the value evaluation
   # and binning functions.

   async def _sg3_generate_graph_data(self, channel, measured, bins):
      data_temp = {} # Maps day delta -> chars sent
      for ch in self._res.server.channels:
         for msg_dict in self._res.message_cache_read(self._res.server.id, ch.id):
            bin_value = bins(msg_dict)
            prev = 0
            try:
               prev = data_temp[bin_value]
            except KeyError: # May also catch msg_dict keyerror...
               pass
            data_temp[bin_value] = measured(prev, msg_dict, bin_value)
      days_ago = 0
      data = []
      while bool(data_temp): # While dictionary still has data
         content_len = 0
         try:
            content_len = data_temp[days_ago]
            del data_temp[days_ago]
         except KeyError:
            pass
         data.insert(0, content_len)
         days_ago += 1

      x_vals = []
      i = 1
      for point in data:
         x_vals.append(i)
         i += 1

      return (data, x_vals)

   ##############################################
   ### SIMPLE GRAPHING - STEP 4 (DATA OUTPUT) ###
   ##############################################

   # This determines what form the output takes.

   _sg_argument3 = {} # Command Dictionary
   _sg_arghelp3 = []

   _sg_arghelp3.append("`vbar` - Vertical bar graph.")
   @cmd.add(_sg_argument3, "vbar")
   def _sg4_vbar(self):
      async def function(channel, **kwargs):
         plotly_data = [
            go.Bar(
               x = kwargs["x_vals"],
               y = kwargs["y_vals"]
            )
         ]
         plotly_layout = go.Layout(
            title = kwargs["title"],
            xaxis = dict(
               title = kwargs["x_axis_title"],
            ),
            yaxis = dict(
               title = kwargs["y_axis_title"],
            ),
         )
         await self._send_plotly_graph_object(channel, plotly_data, plotly_layout)
         return
      return function

   ######################
   ### OTHER SERVICES ###
   ######################

   def _log_in_from_file(self):
      shared_settings = self._res.get_shared_settings()

      if shared_settings is None:
         shared_settings = self.DEFAULT_SHARED_SETTINGS
         self._res.save_shared_settings(shared_settings)

      try:
         username = shared_settings["login username"]
      except KeyError:
         username = self.DEFAULT_SHARED_SETTINGS["login username"]
         shared_settings["login username"] = username

      try:
         api_key = shared_settings["api key"]
      except KeyError:
         api_key = self.DEFAULT_SHARED_SETTINGS["api key"]
         shared_settings["api key"] = api_key

      py.sign_in(username, api_key)

      self._res.save_shared_settings(shared_settings)
      return

   # This is a utility function used by graph generating functions.
   async def _send_plotly_graph_object(self, channel, data, layout):
      temp_filename = "temp" + str(random.getrandbits(128))
      temp_file_ext = ".png"
      try:
         py.image.save_as({'data':data, 'layout':layout}, temp_filename, format='png')
      except:
         print(traceback.format_exc())
         buf = "Unknown error occurred. Maybe you forgot to sign in..."
         buf += "\n(Bot owner must manually enter plotly login details in settings files,"
         buf += " and use `/stats login` for it take effect.)"
         await self._client.send_msg(msg, buf)
         raise errors.OperationAborted
      await self._client.perm_send_file(channel, temp_filename + temp_file_ext)
      os.remove(temp_filename + temp_file_ext)
      print("GRAPH SENT!")
      return

   def _get_usage_info(self):
      buf = "**Argument 1 (Message Evaluation) is one of:**"
      for line in self._sg_arghelp1:
         buf += "\n" + line
      buf += "\n\n**Argument 2 (Binning) is one of:**"
      for line in self._sg_arghelp2:
         buf += "\n" + line
      buf += "\n\n**Argument 3 (Graph Type) is one of:**"
      for line in self._sg_arghelp3:
         buf += "\n" + line
      buf += "\n\n**Example:** `" + self._res.cmd_prefix + self._cmd_names[0]
      buf += " chars eachday vbar` - Bar graph of all characters"
      buf += " received by the server each day."
      return buf

<|endoftext|>


<commit_before>from django.http import HttpResponse, HttpResponseNotFound

from mint.django_rest.rbuilder import service
from mint.django_rest.deco import requires, return_xml, access, ACCESS

from mint.django_rest.rbuilder.packageworkspaces import models

class PackageWorkspaceService(service.BaseService):
    """docstring for PackageWorkspaceService"""

    @return_xml
    def rest_GET(self, request, workspace_id=None):
        """docstring for rest_GET"""
        if workspace_id:
            return self.mgr.getPackageWorkspace(workspace_id)
        else:
            return self.mgr.getPackageWorkspaces()

    # access.admin
    @requires('package_workspace')
    @return_xml
    def rest_POST(self, request, package_workspace):
        """docstring for rest_POST"""
        return self.mgr.addPackageWorkspace(package_workspace)

    # @access.admin
    @requires('package_workspace')
    @return_xml
    def rest_PUT(self, request, workspace_id, package_workspace):
        """docstring for rest_PUT"""
        return self.mgr.updatePackageWorkspace(workspace_id, package_workspace)

    # @access.admin
    def rest_DELETE(self, request, workspace_id):
        """docstring for rest_DELETE"""
        self.mgr.deletePackageWorkspace(workspace_id)
        response = HttpResponse(status=204)
        return response<commit_msg>trying to fix pepflakes errors<commit_after>from django.http import HttpResponse

from mint.django_rest.rbuilder import service
from mint.django_rest.deco import requires, return_xml #, access, ACCESS

# from mint.django_rest.rbuilder.packageworkspaces import models

class PackageWorkspaceService(service.BaseService):
    """docstring for PackageWorkspaceService"""

    @return_xml
    def rest_GET(self, request, workspace_id=None):
        """docstring for rest_GET"""
        if workspace_id:
            return self.mgr.getPackageWorkspace(workspace_id)
        else:
            return self.mgr.getPackageWorkspaces()

    # access.admin
    @requires('package_workspace')
    @return_xml
    def rest_POST(self, request, package_workspace):
        """docstring for rest_POST"""
        return self.mgr.addPackageWorkspace(package_workspace)

    # @access.admin
    @requires('package_workspace')
    @return_xml
    def rest_PUT(self, request, workspace_id, package_workspace):
        """docstring for rest_PUT"""
        return self.mgr.updatePackageWorkspace(workspace_id, package_workspace)

    # @access.admin
    def rest_DELETE(self, request, workspace_id):
        """docstring for rest_DELETE"""
        self.mgr.deletePackageWorkspace(workspace_id)
        response = HttpResponse(status=204)
        return response<|endoftext|>


<commit_before>import re
import os
import flask_mongorest
from flask import Blueprint
from flask_mongorest.resources import Resource
from flask_mongorest import operators as ops
from flask_mongorest.methods import List, Fetch, Create, Delete, Update, BulkUpdate, Download
from flask_mongorest.exceptions import UnknownFieldError
from mpcontribs.api.core import SwaggerView
from mpcontribs.api.contributions.document import Contributions
from mpcontribs.api.structures.document import Structures
from mpcontribs.api.tables.document import Tables

templates = os.path.join(
    os.path.dirname(flask_mongorest.__file__), 'templates'
)
contributions = Blueprint("contributions", __name__, template_folder=templates)
exclude = r'[^$.\s_~`^&(){}[\]\\;\'"/]'

# TODO data__ regex doesn't work through bravado/swagger client
class ContributionsResource(Resource):
    document = Contributions
    filters = {
        'id': [ops.In, ops.Exact],
        'project': [ops.In, ops.Exact],
        'identifier': [ops.In, ops.Contains, ops.Exact],
        'formula': [ops.In, ops.Contains, ops.Exact],
        'is_public': [ops.Boolean],
        re.compile(r'^data__((?!__).)*$'): [ops.Contains, ops.Gte, ops.Lte]
    }
    fields = ['id', 'project', 'identifier', 'formula', 'is_public']
    allowed_ordering = [
        'id', 'project', 'identifier', 'formula', 'is_public',
        re.compile(r'^data(__(' + exclude + ')+){1,3}$')
    ]
    paginate = True
    default_limit = 20
    max_limit = 200
    bulk_update_limit = 100
    download_formats = ['json', 'csv']

    @staticmethod
    def get_optional_fields():
        return ['data', 'structures', 'tables']

    def value_for_field(self, obj, field):
        # add structures and tables info to response if requested
        if field.startswith('structures'):
            field_split = field.split('.')
            field_len = len(field_split)
            if field_len > 2:
                raise UnknownFieldError
            elif field_len == 2:
                if field_split[1] in Structures._fields:
                    # TODO return structure subfields if nested fields requested
                    # TODO find a way to not re-query structures for every subfield?
                    return f'TODO requested {field_split[1]}'
                else:
                    # requested latest structure for label
                    objects = Structures.objects.only('id')
                    objects = objects.filter(contribution=obj.id, label=field_split[1])
                    objects = objects.order_by('-id').limit(1)
                    if not objects:
                        raise UnknownFieldError
                    return objects[0].id
            elif field_len == 1:
                # return full structures only if download requested
                full = bool(self.view_method == Download and
                            self.params.get('format') == 'json' and
                            self.params.get('_fields') == '_all')
                mask = ['id', 'lattice', 'sites', 'charge', 'klass', 'module'] if full else ['id', 'label']
                objects = Structures.objects.only(*mask).filter(contribution=obj.id).order_by('-id')
                if full:
                    from mpcontribs.api.structures.views import StructuresResource
                    sr = StructuresResource(view_method=Download)
                value = [] if full else {}
                for s in objects:
                    if full:
                        value.append(sr.serialize(s, fields=mask))
                    else:  # only return newest structure for each label
                        if not s.label in value:
                            value[s.label] = s.id
                return value

        elif field == 'tables':
            tables = Tables.objects.only('id', 'name').filter(contribution=obj.id)
            return [{'id': t.id, 'name': t.name} for t in tables]
        else:
            raise UnknownFieldError

class ContributionsView(SwaggerView):
    resource = ContributionsResource
    methods = [List, Fetch, Download, Create, Delete, Update, BulkUpdate]
<commit_msg>api: return multiple structures<commit_after>import re
import os
import flask_mongorest
from flask import Blueprint
from flask_mongorest.resources import Resource
from flask_mongorest import operators as ops
from flask_mongorest.methods import List, Fetch, Create, Delete, Update, BulkUpdate, Download
from flask_mongorest.exceptions import UnknownFieldError
from mpcontribs.api.core import SwaggerView
from mpcontribs.api.contributions.document import Contributions
from mpcontribs.api.structures.document import Structures
from mpcontribs.api.tables.document import Tables

templates = os.path.join(
    os.path.dirname(flask_mongorest.__file__), 'templates'
)
contributions = Blueprint("contributions", __name__, template_folder=templates)
exclude = r'[^$.\s_~`^&(){}[\]\\;\'"/]'

# TODO data__ regex doesn't work through bravado/swagger client
class ContributionsResource(Resource):
    document = Contributions
    filters = {
        'id': [ops.In, ops.Exact],
        'project': [ops.In, ops.Exact],
        'identifier': [ops.In, ops.Contains, ops.Exact],
        'formula': [ops.In, ops.Contains, ops.Exact],
        'is_public': [ops.Boolean],
        re.compile(r'^data__((?!__).)*$'): [ops.Contains, ops.Gte, ops.Lte]
    }
    fields = ['id', 'project', 'identifier', 'formula', 'is_public']
    allowed_ordering = [
        'id', 'project', 'identifier', 'formula', 'is_public',
        re.compile(r'^data(__(' + exclude + ')+){1,3}$')
    ]
    paginate = True
    default_limit = 20
    max_limit = 200
    bulk_update_limit = 100
    download_formats = ['json', 'csv']

    @staticmethod
    def get_optional_fields():
        return ['data', 'structures', 'tables']

    def value_for_field(self, obj, field):
        # add structures and tables info to response if requested
        if field.startswith('structures'):
            from mpcontribs.api.structures.views import StructuresResource
            field_split = field.split('.')
            field_len = len(field_split)
            if field_len > 2:
                raise UnknownFieldError
            elif field_len == 2:
                if field_split[1] in Structures._fields:
                    # TODO return structure subfields if nested fields requested
                    # TODO find a way to not re-query structures for every subfield?
                    return f'TODO requested {field_split[1]}'
                else:
                    # requested structure(s) for label
                    mask = ['id', 'name']
                    objects = Structures.objects.only(*mask)
                    objects = objects.filter(contribution=obj.id, label=field_split[1]).order_by('-id')
                    if not objects:
                        raise UnknownFieldError
                    sr = StructuresResource(view_method=List)
                    return [sr.serialize(o, fields=mask) for o in objects]
            elif field_len == 1:
                # return full structures only if download requested
                full = bool(self.view_method == Download and
                            self.params.get('format') == 'json' and
                            self.params.get('_fields') == '_all')
                mask = ['id', 'lattice', 'sites', 'charge', 'klass', 'module'] if full else ['id', 'label']
                objects = Structures.objects.only(*mask).filter(contribution=obj.id).order_by('-id')
                if full:
                    sr = StructuresResource(view_method=Download)
                value = [] if full else {}
                for s in objects:
                    if full:
                        value.append(sr.serialize(s, fields=mask))
                    else:  # only return newest structure for each label
                        if not s.label in value:
                            value[s.label] = s.id
                return value

        elif field == 'tables':
            tables = Tables.objects.only('id', 'name').filter(contribution=obj.id)
            return [{'id': t.id, 'name': t.name} for t in tables]
        else:
            raise UnknownFieldError

class ContributionsView(SwaggerView):
    resource = ContributionsResource
    methods = [List, Fetch, Download, Create, Delete, Update, BulkUpdate]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import json
from hashlib import md5
from flask_mongoengine import Document
from mongoengine import signals
from mongoengine.fields import StringField, FloatField, ListField, DictField
from pymatgen import Structure
from pymatgen.io.cif import CifWriter

class Structures(Document):
    name = StringField(required=True, help_text="name")
    lattice = DictField(required=True, help_text="lattice")
    sites = ListField(DictField(), required=True, help_text="sites")
    charge = FloatField(null=True, help_text="charge")
    md5 = StringField(regex=r"^[a-z0-9]{32}$", unique=True, help_text="md5 sum")
    cif = StringField(help_text="CIF string")
    meta = {"collection": "structures", "indexes": ["name", "md5"]}

    @classmethod
    def pre_save_post_validation(cls, sender, document, **kwargs):
        from mpcontribs.api.structures.views import StructuresResource

        resource = StructuresResource()
        d = resource.serialize(document, fields=["lattice", "sites", "charge"])
        s = json.dumps(d, sort_keys=True).encode("utf-8")
        document.md5 = md5(s).hexdigest()
        structure = Structure.from_dict(d)
        document.cif = CifWriter(structure, symprec=1e-10).__str__()

signals.pre_save_post_validation.connect(
    Structures.pre_save_post_validation, sender=Structures
)
<commit_msg>api: catch CIFWriter exception<commit_after># -*- coding: utf-8 -*-
import json
from hashlib import md5
from flask_mongoengine import Document
from mongoengine import signals
from mongoengine.fields import StringField, FloatField, ListField, DictField
from pymatgen import Structure
from pymatgen.io.cif import CifWriter

class Structures(Document):
    name = StringField(required=True, help_text="name")
    lattice = DictField(required=True, help_text="lattice")
    sites = ListField(DictField(), required=True, help_text="sites")
    charge = FloatField(null=True, help_text="charge")
    md5 = StringField(regex=r"^[a-z0-9]{32}$", unique=True, help_text="md5 sum")
    cif = StringField(help_text="CIF string")
    meta = {"collection": "structures", "indexes": ["name", "md5"]}

    @classmethod
    def pre_save_post_validation(cls, sender, document, **kwargs):
        from mpcontribs.api.structures.views import StructuresResource

        resource = StructuresResource()
        d = resource.serialize(document, fields=["lattice", "sites", "charge"])
        s = json.dumps(d, sort_keys=True).encode("utf-8")
        document.md5 = md5(s).hexdigest()
        structure = Structure.from_dict(d)

        try:
            writer = CifWriter(structure, symprec=1e-10)
        except TypeError:
            # save CIF string without symmetry information
            writer = CifWriter(structure)

        document.cif = writer.__str__()

signals.pre_save_post_validation.connect(
    Structures.pre_save_post_validation, sender=Structures
)
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2021 The Google Research Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3
"""Models for distillation."""

import tensorflow as tf
import tensorflow_model_optimization as tfmot
from non_semantic_speech_benchmark.distillation.layers import CompressedDense
from non_semantic_speech_benchmark.export_model import tf_frontend
# pylint: disable=g-direct-tensorflow-import
from tensorflow.python.keras.applications import mobilenet_v3 as v3_util
# pylint: enable=g-direct-tensorflow-import

@tf.function
def _sample_to_features(x):
  return tf_frontend.compute_frontend_features(x, 16000, overlap_seconds=79)

def get_keras_model(bottleneck_dimension,
                    output_dimension,
                    alpha=1.0,
                    mobilenet_size='small',
                    frontend=True,
                    avg_pool=False,
                    compressor=None,
                    qat=False,
                    tflite=False):
  """Make a Keras student model."""

  def _map_fn_lambda(x):
    return tf.map_fn(_sample_to_features, x, dtype=tf.float64)

  def _map_mobilenet_func(mnet_size):
    mnet_size_map = {
        'tiny': mobilenetv3_tiny,
        'small': tf.keras.applications.MobileNetV3Small,
        'large': tf.keras.applications.MobileNetV3Large,
    }
    if mnet_size.lower() not in mnet_size_map:
      raise ValueError('Unknown MobileNet size %s.' % mnet_size)
    return mnet_size_map[mnet_size.lower()]

  # TFLite use-cases usually use non-batched inference, and this also enables
  # hardware acceleration.
  num_batches = 1 if tflite else None
  if frontend:
    model_in = tf.keras.Input((None,), name='audio_samples')
    feats = tf.keras.layers.Lambda(_map_fn_lambda)(model_in)
    feats.shape.assert_is_compatible_with([None, None, 96, 64])
    feats = tf.transpose(feats, [0, 2, 1, 3])
    feats = tf.reshape(feats, [-1, 96, 64, 1])
  else:
    model_in = tf.keras.Input((96, 64, 1),
                              name='log_mel_spectrogram',
                              batch_size=num_batches)
    feats = model_in
  model = _map_mobilenet_func(mobilenet_size)(
      input_shape=[96, 64, 1],
      alpha=alpha,
      minimalistic=False,
      include_top=False,
      weights=None,
      pooling='avg' if avg_pool else None,
      dropout_rate=0.0)
  model_out = model(feats)
  if avg_pool:
    model_out.shape.assert_is_compatible_with([None, None])
  else:
    model_out.shape.assert_is_compatible_with([None, 3, 2, None])
  if bottleneck_dimension:
    if compressor is not None:
      bottleneck = CompressedDense(
          bottleneck_dimension,
          compression_obj=compressor,
          name='distilled_output')
    else:
      bottleneck = tf.keras.layers.Dense(
          bottleneck_dimension, name='distilled_output')
      if qat:
        bottleneck = tfmot.quantization.keras.quantize_annotate_layer(
            bottleneck)
    embeddings = tf.keras.layers.Flatten()(model_out)
    embeddings = bottleneck(embeddings)

    if tflite:
      # We generate TFLite models just for the embeddings.
      output_model = tf.keras.Model(inputs=model_in, outputs=embeddings)
      if compressor is not None:
        # If model employs compression, this ensures that the TFLite model
        # just uses the smaller matrices for inference.
        output_model.get_layer('distilled_output').kernel = None
        output_model.get_layer(
            'distilled_output').compression_op.a_matrix_tfvar = None
      return output_model
  else:
    embeddings = tf.keras.layers.Flatten(name='distilled_output')(model_out)
  if output_dimension:
    output = tf.keras.layers.Dense(
        output_dimension, name='embedding_to_target')(embeddings)
  else:
    output = embeddings
  output_model = tf.keras.Model(inputs=model_in, outputs=output)
  return output_model

def mobilenetv3_tiny(input_shape=None,
                     alpha=1.0,
                     minimalistic=False,
                     include_top=True,
                     weights='imagenet',
                     input_tensor=None,
                     classes=1000,
                     pooling=None,
                     dropout_rate=0.2,
                     classifier_activation='softmax'):
  """Makes MobileNetV3 model."""

  def stack_fn(x, kernel, activation, se_ratio):

    # Using blocks from MobileNetV3 saves a lot of code duplication.
    # pylint: disable=protected-access
    def depth(d):
      return v3_util._depth(d * alpha)

    x = v3_util._inverted_res_block(x, 1, depth(16), 3, 2, se_ratio,
                                    v3_util.relu, 0)
    x = v3_util._inverted_res_block(x, 72. / 16, depth(24), 3, 2, None,
                                    v3_util.relu, 1)
    x = v3_util._inverted_res_block(x, 88. / 24, depth(24), 3, 1, None,
                                    v3_util.relu, 2)
    x = v3_util._inverted_res_block(x, 4, depth(40), kernel, 2, se_ratio,
                                    activation, 3)
    x = v3_util._inverted_res_block(x, 6, depth(40), kernel, 1, se_ratio,
                                    activation, 4)
    x = v3_util._inverted_res_block(x, 6, depth(40), kernel, 1, se_ratio,
                                    activation, 5)
    x = v3_util._inverted_res_block(x, 3, depth(48), kernel, 1, se_ratio,
                                    activation, 6)
    x = v3_util._inverted_res_block(x, 6, depth(96), kernel, 2, se_ratio,
                                    activation, 8)
    x = v3_util._inverted_res_block(x, 6, depth(96), kernel, 1, se_ratio,
                                    activation, 9)
    # pylint: enable=protected-access
    return x

  return v3_util.MobileNetV3(stack_fn, 512, input_shape, alpha, 'tiny',
                             minimalistic, include_top, weights, input_tensor,
                             classes, pooling, dropout_rate,
                             classifier_activation)
<commit_msg>Fix the test breakage caused by recent change in keras.application Mobilenetv3.<commit_after># coding=utf-8
# Copyright 2021 The Google Research Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3
"""Models for distillation."""

import tensorflow as tf
import tensorflow_model_optimization as tfmot
from non_semantic_speech_benchmark.distillation.layers import CompressedDense
from non_semantic_speech_benchmark.export_model import tf_frontend
# pylint: disable=g-direct-tensorflow-import
from tensorflow.python.keras.applications import mobilenet_v3 as v3_util
# pylint: enable=g-direct-tensorflow-import

@tf.function
def _sample_to_features(x):
  return tf_frontend.compute_frontend_features(x, 16000, overlap_seconds=79)

def get_keras_model(bottleneck_dimension,
                    output_dimension,
                    alpha=1.0,
                    mobilenet_size='small',
                    frontend=True,
                    avg_pool=False,
                    compressor=None,
                    qat=False,
                    tflite=False):
  """Make a Keras student model."""

  def _map_fn_lambda(x):
    return tf.map_fn(_sample_to_features, x, dtype=tf.float64)

  def _map_mobilenet_func(mnet_size):
    mnet_size_map = {
        'tiny': mobilenetv3_tiny,
        'small': tf.keras.applications.MobileNetV3Small,
        'large': tf.keras.applications.MobileNetV3Large,
    }
    if mnet_size.lower() not in mnet_size_map:
      raise ValueError('Unknown MobileNet size %s.' % mnet_size)
    return mnet_size_map[mnet_size.lower()]

  # TFLite use-cases usually use non-batched inference, and this also enables
  # hardware acceleration.
  num_batches = 1 if tflite else None
  if frontend:
    model_in = tf.keras.Input((None,), name='audio_samples')
    feats = tf.keras.layers.Lambda(_map_fn_lambda)(model_in)
    feats.shape.assert_is_compatible_with([None, None, 96, 64])
    feats = tf.transpose(feats, [0, 2, 1, 3])
    feats = tf.reshape(feats, [-1, 96, 64, 1])
  else:
    model_in = tf.keras.Input((96, 64, 1),
                              name='log_mel_spectrogram',
                              batch_size=num_batches)
    feats = model_in
  model = _map_mobilenet_func(mobilenet_size)(
      input_shape=[96, 64, 1],
      alpha=alpha,
      minimalistic=False,
      include_top=False,
      weights=None,
      pooling='avg' if avg_pool else None,
      dropout_rate=0.0)
  model_out = model(feats)
  if avg_pool:
    model_out.shape.assert_is_compatible_with([None, None])
  else:
    model_out.shape.assert_is_compatible_with([None, 1, 1, None])
  if bottleneck_dimension:
    if compressor is not None:
      bottleneck = CompressedDense(
          bottleneck_dimension,
          compression_obj=compressor,
          name='distilled_output')
    else:
      bottleneck = tf.keras.layers.Dense(
          bottleneck_dimension, name='distilled_output')
      if qat:
        bottleneck = tfmot.quantization.keras.quantize_annotate_layer(
            bottleneck)
    embeddings = tf.keras.layers.Flatten()(model_out)
    embeddings = bottleneck(embeddings)

    if tflite:
      # We generate TFLite models just for the embeddings.
      output_model = tf.keras.Model(inputs=model_in, outputs=embeddings)
      if compressor is not None:
        # If model employs compression, this ensures that the TFLite model
        # just uses the smaller matrices for inference.
        output_model.get_layer('distilled_output').kernel = None
        output_model.get_layer(
            'distilled_output').compression_op.a_matrix_tfvar = None
      return output_model
  else:
    embeddings = tf.keras.layers.Flatten(name='distilled_output')(model_out)
  if output_dimension:
    output = tf.keras.layers.Dense(
        output_dimension, name='embedding_to_target')(embeddings)
  else:
    output = embeddings
  output_model = tf.keras.Model(inputs=model_in, outputs=output)
  return output_model

def mobilenetv3_tiny(input_shape=None,
                     alpha=1.0,
                     minimalistic=False,
                     include_top=True,
                     weights='imagenet',
                     input_tensor=None,
                     classes=1000,
                     pooling=None,
                     dropout_rate=0.2,
                     classifier_activation='softmax'):
  """Makes MobileNetV3 model."""

  def stack_fn(x, kernel, activation, se_ratio):

    # Using blocks from MobileNetV3 saves a lot of code duplication.
    # pylint: disable=protected-access
    def depth(d):
      return v3_util._depth(d * alpha)

    x = v3_util._inverted_res_block(x, 1, depth(16), 3, 2, se_ratio,
                                    v3_util.relu, 0)
    x = v3_util._inverted_res_block(x, 72. / 16, depth(24), 3, 2, None,
                                    v3_util.relu, 1)
    x = v3_util._inverted_res_block(x, 88. / 24, depth(24), 3, 1, None,
                                    v3_util.relu, 2)
    x = v3_util._inverted_res_block(x, 4, depth(40), kernel, 2, se_ratio,
                                    activation, 3)
    x = v3_util._inverted_res_block(x, 6, depth(40), kernel, 1, se_ratio,
                                    activation, 4)
    x = v3_util._inverted_res_block(x, 6, depth(40), kernel, 1, se_ratio,
                                    activation, 5)
    x = v3_util._inverted_res_block(x, 3, depth(48), kernel, 1, se_ratio,
                                    activation, 6)
    x = v3_util._inverted_res_block(x, 6, depth(96), kernel, 2, se_ratio,
                                    activation, 8)
    x = v3_util._inverted_res_block(x, 6, depth(96), kernel, 1, se_ratio,
                                    activation, 9)
    # pylint: enable=protected-access
    return x

  return v3_util.MobileNetV3(stack_fn, 512, input_shape, alpha, 'tiny',
                             minimalistic, include_top, weights, input_tensor,
                             classes, pooling, dropout_rate,
                             classifier_activation)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#     This file is part of mozaik_communication, an Odoo module.
#
#     Copyright (c) 2015 ACSONE SA/NV (<http://acsone.eu>)
#
#     mozaik_communication is free software:
#     you can redistribute it and/or
#     modify it under the terms of the GNU Affero General Public License
#     as published by the Free Software Foundation, either version 3 of
#     the License, or (at your option) any later version.
#
#     mozaik_communication is distributed in the hope that it will
#     be useful but WITHOUT ANY WARRANTY; without even the implied warranty of
#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#     GNU Affero General Public License for more details.
#
#     You should have received a copy of the
#     GNU Affero General Public License
#     along with mozaik_communication.
#     If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from openerp.osv import orm

class email_coordinate(orm.Model):

    def _get_linked_mailing(self, cr, uid, ids, context=None):
        '''
        :rtype: [integer]
        :rparam: list of `mail.mass_mailing.contact` associated with a partner
            id into `ids`
        '''
        mailing_ids = []
        partner_ids = self.get_linked_partners(cr, uid, ids, context=context)
        if partner_ids:
            mailing_ids = self.pool['mail.mass_mailing.contact'].search(
                cr, uid, [('partner_id', 'in', partner_ids)], context=context)
        return mailing_ids

    _inherit = 'email.coordinate'
<commit_msg>Migrate mozaik_communication/email_coordinate.py to new API<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#     This file is part of mozaik_communication, an Odoo module.
#
#     Copyright (c) 2015 ACSONE SA/NV (<http://acsone.eu>)
#
#     mozaik_communication is free software:
#     you can redistribute it and/or
#     modify it under the terms of the GNU Affero General Public License
#     as published by the Free Software Foundation, either version 3 of
#     the License, or (at your option) any later version.
#
#     mozaik_communication is distributed in the hope that it will
#     be useful but WITHOUT ANY WARRANTY; without even the implied warranty of
#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#     GNU Affero General Public License for more details.
#
#     You should have received a copy of the
#     GNU Affero General Public License
#     along with mozaik_communication.
#     If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from openerp import api, models

class EmailCoordinate(models.Model):

    _inherit = 'email.coordinate'

    @api.multi
    @api.returns('mail.mass_mailing.contact')
    def _get_linked_mailing(self):
        '''
        :rtype: [integer]
        :rparam: list of `mail.mass_mailing.contact` associated with a partner
            id into `ids`
        '''
        mailing_contacts = []
        partner_ids = self.get_linked_partners()
        if partner_ids:
            mailing_contacts = self.env['mail.mass_mailing.contact'].search(
                [('partner_id', 'in', partner_ids)])
        return mailing_contacts

<|endoftext|>


<commit_before>import base64
import functools
import logging
from collections import namedtuple

from django.conf import settings
from django.contrib.auth import login
from django.core.urlresolvers import reverse
from django.db.models import Q
from django.http import HttpResponseRedirect
from django.utils.http import is_safe_url

from rest_framework import generics
from rest_framework.response import Response
from rest_framework.views import APIView
from tower import ugettext_lazy as _
from waffle.decorators import waffle_switch

from olympia.amo import messages
from olympia.api.jwt_auth.views import JWTProtectedView
from olympia.users.models import UserProfile
from olympia.accounts.serializers import (
    UserProfileSerializer, AccountSourceSerializer)

from . import verify

log = logging.getLogger('accounts')

STUB_FXA_USER = namedtuple('FxAUser', ['source'])('fxa')

def safe_redirect(url, action):
    if not is_safe_url(url):
        url = reverse('home')
    log.info('Redirecting after {} to: {}'.format(action, url))
    return HttpResponseRedirect(url)

def find_user(identity):
    """Try to find a user for a Firefox Accounts profile. If the account
    hasn't been migrated we'll need to do the lookup by email but we should
    use the ID after that so check both. If we get multiple users we're in
    some weird state where the accounts need to be merged but that behaviour
    hasn't been defined so let it raise.
    """
    try:
        return UserProfile.objects.get(
            Q(fxa_id=identity['uid']) | Q(email=identity['email']))
    except UserProfile.DoesNotExist:
        return None
    except UserProfile.MultipleObjectsReturned:
        # This shouldn't happen, so let it raise.
        log.error(
            'Found multiple users for {email} and {uid}'.format(**identity))
        raise

def register_user(request, identity):
    user = UserProfile.objects.create_user(
        email=identity['email'], username=None, fxa_id=identity['uid'])
    log.info('Created user {} from FxA'.format(user))
    login(request, user)
    return user

def login_user(request, user, identity):
    if (user.fxa_id != identity['uid'] or
            user.email != identity['email']):
        log.info(
            'Updating user info from FxA for {pk}. Old {old_email} {old_uid} '
            'New {new_email} {new_uid}'.format(
                pk=user.pk, old_email=user.email, old_uid=user.fxa_id,
                new_email=identity['email'], new_uid=identity['uid']))
        if not user.fxa_migrated():
            messages.success(
                request,
                _('Great job! You can now log in to Add-ons with your '
                  'Firefox Account.'),
                extra_tags='fxa')
        user.update(fxa_id=identity['uid'], email=identity['email'])
    log.info('Logging in user {} from FxA'.format(user))
    login(request, user)

def with_user(fn):
    @functools.wraps(fn)
    def inner(self, request):
        data = request.GET if request.method == 'GET' else request.DATA
        state_parts = data.get('state', '').split(':', 1)
        state = state_parts[0]
        if 'code' not in data:
            log.info('No code provided.')
            return Response({'error': 'No code provided.'}, status=422)
        elif (not request.session.get('fxa_state') or
                request.session['fxa_state'] != state):
            log.info('State mismatch. URL: {url} Session: {session}'.format(
                url=data.get('state'),
                session=request.session.get('fxa_state'),
            ))
            return Response({'error': 'State mismatch.'}, status=400)

        try:
            identity = verify.fxa_identify(data['code'],
                                           config=settings.FXA_CONFIG)
        except verify.IdentificationError:
            log.info('Profile not found. Code: {}'.format(data['code']))
            return Response({'error': 'Profile not found.'}, status=401)
        else:
            identity_user = find_user(identity)
            if request.user.is_authenticated():
                if identity_user is not None and identity_user != request.user:
                    log.info('Conflict finding user during FxA login. '
                             'request.user: {}, identity_user: {}'.format(
                                 request.user.pk, identity_user.pk))
                    return Response({'error': 'User mismatch.'}, status=422)
                elif request.user.fxa_migrated():
                    log.info('User already migrated. '
                             'request.user: {}, identity_user: {}'.format(
                                 request.user, identity_user))
                    return Response(
                        {'error': 'User already migrated.'}, status=422)
                else:
                    user = request.user
            else:
                user = identity_user
            next_path = None
            if len(state_parts) == 2:
                # The = signs will be stripped off so we need to add them back
                # but it only cares if there are too few so add 4 of them.
                encoded_path = state_parts[1] + '===='
                try:
                    next_path = base64.urlsafe_b64decode(str(encoded_path))
                except TypeError:
                    log.info('Error decoding next_path {}'.format(
                        encoded_path))
                    pass
            if not is_safe_url(next_path):
                next_path = None
            return fn(self, request, user=user, identity=identity,
                      next_path=next_path)
    return inner

class LoginView(APIView):

    @waffle_switch('fxa-auth')
    @with_user
    def post(self, request, user, identity, next_path):
        if user is None:
            return Response({'error': 'User does not exist.'}, status=422)
        else:
            login_user(request, user, identity)
            return Response({'email': identity['email']})

class RegisterView(APIView):

    @waffle_switch('fxa-auth')
    @with_user
    def post(self, request, user, identity, next_path):
        if user is not None:
            return Response({'error': 'That account already exists.'},
                            status=422)
        else:
            user = register_user(request, identity)
            return Response({'email': user.email})

class AuthenticateView(APIView):

    @waffle_switch('fxa-auth')
    @with_user
    def get(self, request, user, identity, next_path):
        if user is None:
            register_user(request, identity)
            return safe_redirect(reverse('users.edit'), 'register')
        else:
            login_user(request, user, identity)
            return safe_redirect(next_path, 'login')

class ProfileView(JWTProtectedView, generics.RetrieveAPIView):
    serializer_class = UserProfileSerializer

    def retrieve(self, request, *args, **kw):
        return Response(self.get_serializer(request.user).data)

class AccountSourceView(generics.RetrieveAPIView):
    serializer_class = AccountSourceSerializer

    @waffle_switch('fxa-auth')
    def retrieve(self, request, *args, **kwargs):
        email = request.GET.get('email')
        try:
            user = UserProfile.objects.get(email=email)
        except UserProfile.DoesNotExist:
            # Use the stub FxA user with source='fxa' when the account doesn't
            # exist. This will make it more difficult to discover if an email
            # address has an account associated with it.
            user = STUB_FXA_USER
        return Response(self.get_serializer(user).data)
<commit_msg>Use unicode when adding a message<commit_after>import base64
import functools
import logging
from collections import namedtuple

from django.conf import settings
from django.contrib.auth import login
from django.core.urlresolvers import reverse
from django.db.models import Q
from django.http import HttpResponseRedirect
from django.utils.http import is_safe_url

from rest_framework import generics
from rest_framework.response import Response
from rest_framework.views import APIView
from tower import ugettext_lazy as _
from waffle.decorators import waffle_switch

from olympia.amo import messages
from olympia.api.jwt_auth.views import JWTProtectedView
from olympia.users.models import UserProfile
from olympia.accounts.serializers import (
    UserProfileSerializer, AccountSourceSerializer)

from . import verify

log = logging.getLogger('accounts')

STUB_FXA_USER = namedtuple('FxAUser', ['source'])('fxa')

def safe_redirect(url, action):
    if not is_safe_url(url):
        url = reverse('home')
    log.info('Redirecting after {} to: {}'.format(action, url))
    return HttpResponseRedirect(url)

def find_user(identity):
    """Try to find a user for a Firefox Accounts profile. If the account
    hasn't been migrated we'll need to do the lookup by email but we should
    use the ID after that so check both. If we get multiple users we're in
    some weird state where the accounts need to be merged but that behaviour
    hasn't been defined so let it raise.
    """
    try:
        return UserProfile.objects.get(
            Q(fxa_id=identity['uid']) | Q(email=identity['email']))
    except UserProfile.DoesNotExist:
        return None
    except UserProfile.MultipleObjectsReturned:
        # This shouldn't happen, so let it raise.
        log.error(
            'Found multiple users for {email} and {uid}'.format(**identity))
        raise

def register_user(request, identity):
    user = UserProfile.objects.create_user(
        email=identity['email'], username=None, fxa_id=identity['uid'])
    log.info('Created user {} from FxA'.format(user))
    login(request, user)
    return user

def login_user(request, user, identity):
    if (user.fxa_id != identity['uid'] or
            user.email != identity['email']):
        log.info(
            'Updating user info from FxA for {pk}. Old {old_email} {old_uid} '
            'New {new_email} {new_uid}'.format(
                pk=user.pk, old_email=user.email, old_uid=user.fxa_id,
                new_email=identity['email'], new_uid=identity['uid']))
        if not user.fxa_migrated():
            messages.success(
                request,
                _(u'Great job! You can now log in to Add-ons with your '
                  'Firefox Account.'),
                extra_tags='fxa')
        user.update(fxa_id=identity['uid'], email=identity['email'])
    log.info('Logging in user {} from FxA'.format(user))
    login(request, user)

def with_user(fn):
    @functools.wraps(fn)
    def inner(self, request):
        data = request.GET if request.method == 'GET' else request.DATA
        state_parts = data.get('state', '').split(':', 1)
        state = state_parts[0]
        if 'code' not in data:
            log.info('No code provided.')
            return Response({'error': 'No code provided.'}, status=422)
        elif (not request.session.get('fxa_state') or
                request.session['fxa_state'] != state):
            log.info('State mismatch. URL: {url} Session: {session}'.format(
                url=data.get('state'),
                session=request.session.get('fxa_state'),
            ))
            return Response({'error': 'State mismatch.'}, status=400)

        try:
            identity = verify.fxa_identify(data['code'],
                                           config=settings.FXA_CONFIG)
        except verify.IdentificationError:
            log.info('Profile not found. Code: {}'.format(data['code']))
            return Response({'error': 'Profile not found.'}, status=401)
        else:
            identity_user = find_user(identity)
            if request.user.is_authenticated():
                if identity_user is not None and identity_user != request.user:
                    log.info('Conflict finding user during FxA login. '
                             'request.user: {}, identity_user: {}'.format(
                                 request.user.pk, identity_user.pk))
                    return Response({'error': 'User mismatch.'}, status=422)
                elif request.user.fxa_migrated():
                    log.info('User already migrated. '
                             'request.user: {}, identity_user: {}'.format(
                                 request.user, identity_user))
                    return Response(
                        {'error': 'User already migrated.'}, status=422)
                else:
                    user = request.user
            else:
                user = identity_user
            next_path = None
            if len(state_parts) == 2:
                # The = signs will be stripped off so we need to add them back
                # but it only cares if there are too few so add 4 of them.
                encoded_path = state_parts[1] + '===='
                try:
                    next_path = base64.urlsafe_b64decode(str(encoded_path))
                except TypeError:
                    log.info('Error decoding next_path {}'.format(
                        encoded_path))
                    pass
            if not is_safe_url(next_path):
                next_path = None
            return fn(self, request, user=user, identity=identity,
                      next_path=next_path)
    return inner

class LoginView(APIView):

    @waffle_switch('fxa-auth')
    @with_user
    def post(self, request, user, identity, next_path):
        if user is None:
            return Response({'error': 'User does not exist.'}, status=422)
        else:
            login_user(request, user, identity)
            return Response({'email': identity['email']})

class RegisterView(APIView):

    @waffle_switch('fxa-auth')
    @with_user
    def post(self, request, user, identity, next_path):
        if user is not None:
            return Response({'error': 'That account already exists.'},
                            status=422)
        else:
            user = register_user(request, identity)
            return Response({'email': user.email})

class AuthenticateView(APIView):

    @waffle_switch('fxa-auth')
    @with_user
    def get(self, request, user, identity, next_path):
        if user is None:
            register_user(request, identity)
            return safe_redirect(reverse('users.edit'), 'register')
        else:
            login_user(request, user, identity)
            return safe_redirect(next_path, 'login')

class ProfileView(JWTProtectedView, generics.RetrieveAPIView):
    serializer_class = UserProfileSerializer

    def retrieve(self, request, *args, **kw):
        return Response(self.get_serializer(request.user).data)

class AccountSourceView(generics.RetrieveAPIView):
    serializer_class = AccountSourceSerializer

    @waffle_switch('fxa-auth')
    def retrieve(self, request, *args, **kwargs):
        email = request.GET.get('email')
        try:
            user = UserProfile.objects.get(email=email)
        except UserProfile.DoesNotExist:
            # Use the stub FxA user with source='fxa' when the account doesn't
            # exist. This will make it more difficult to discover if an email
            # address has an account associated with it.
            user = STUB_FXA_USER
        return Response(self.get_serializer(user).data)
<|endoftext|>


<commit_before>from __future__ import unicode_literals

try:
    import configparser
except ImportError:
    import ConfigParser as configparser

import logging
import os.path
from datetime import datetime

import io
from dateutil import parser as date_parser

class ConfigurationError(Exception):
    pass

class Configuration(object):
    """ Object holding the configuration """
    properties = {'otx': True, 'misp': True, 'server': True, 'timestamp': False, 'author': False,
                  'distribution': False, 'threat_level': False, 'analysis': False, 'update_timestamp': False,
                  'publish': False, 'tlp': False, 'discover_tags': False, 'to_ids': False, 'author_tag': False, 'bulk_tag': False}
    simulation_properties = {'otx': True, 'misp': True, 'server': True, 'timestamp': False, 'author': False,
                             'distribution': False, 'threat_level': False, 'analysis': False, 'update_timestamp': False,
                             'publish': False, 'tlp': False, 'discover_tags': False, 'to_ids': False, 'author_tag': False, 'bulk_tag': False}
    defaults = {'distribution': 0, 'threat_level': 4, 'analysis': 2, 'timestamp': datetime.utcfromtimestamp(0),
                'update_timestamp': False}
    config_section = 'otx_misp'

    def __init__(self, arguments):
        self.config = configparser.SafeConfigParser(allow_no_value=True)
        if arguments.config and os.path.isfile(arguments.config):
            self.config.read(arguments.config)
        if not self.config.has_section(self.config_section):
            self.config.add_section(self.config_section)
        self.arguments = arguments
        self.original_config = self._clone_config(self.config)
        self._populate_config()

    @staticmethod
    def _clone_config(config):
        clone = configparser.SafeConfigParser(allow_no_value=True)
        for section in config.sections():
            clone.add_section(section)
            for option in config.options(section):
                clone.set(section, option, config.get(section, option))
        return clone

    def _populate_config(self):
        if self.arguments.simulate:
            self.properties = self.simulation_properties
        for key, required in list(self.simulation_properties.items()):
            value = getattr(self.arguments, key, None)
            if isinstance(value, bool):
                if value:
                    value = 'yes'
                else:
                    value = 'no'
            elif isinstance(value, (list, tuple)):
                value = ','.join(value)
            elif isinstance(value, datetime):
                value = value.isoformat()
            elif isinstance(value, int):
                value = '{}'.format(value)
            if value is not None:
                self.config.set(self.config_section, key, value)
            if required:
                try:
                    value = self.config.get(self.config_section, key)
                except configparser.NoOptionError:
                    raise ConfigurationError("Missing required parameter: '--{}'".format(key))
                if value is None:
                    raise ConfigurationError("Missing required parameter: '--{}'".format(key))

    def __getattr__(self, item):
        if item not in self.arguments:
            raise AttributeError
        value = getattr(self.arguments, item, None)
        if value is None and item in self.defaults:
            value = self.defaults[item]
        if item not in self.properties or not self.config.has_option(self.config_section, item):
            return value
        if isinstance(value, bool):
            return self.config.getboolean(self.config_section, item)
        elif isinstance(value, (list, tuple)):
            parameter = self.config.get(self.config_section, item)
            return parameter.split(',')
        elif isinstance(value, datetime):
            parameter = self.config.get(self.config_section, item)
            if parameter is not None:
                try:
                    return date_parser.parse(parameter)
                except:
                    pass
        elif isinstance(value, int):
            return self.config.getint(self.config_section, item)
        return self.config.get(self.config_section, item)

    def write(self, fp):
        if self.write_config:
            config = self.config
        else:
            config = self.original_config
        if self.update_timestamp:
            config.set(self.config_section, 'timestamp', datetime.utcnow().isoformat())
        return config.write(fp)
<commit_msg>Add title dedup option<commit_after>from __future__ import unicode_literals

try:
    import configparser
except ImportError:
    import ConfigParser as configparser

import logging
import os.path
from datetime import datetime

import io
from dateutil import parser as date_parser

class ConfigurationError(Exception):
    pass

class Configuration(object):
    """ Object holding the configuration """
    properties = {'otx': True, 'misp': True, 'server': True, 'timestamp': False, 'author': False,
                  'distribution': False, 'threat_level': False, 'analysis': False, 'update_timestamp': False,
                  'publish': False, 'tlp': False, 'discover_tags': False, 'to_ids': False, 'author_tag': False, 'bulk_tag': False,
                  'dedup_titles': False}
    simulation_properties = {'otx': True, 'misp': True, 'server': True, 'timestamp': False, 'author': False,
                             'distribution': False, 'threat_level': False, 'analysis': False, 'update_timestamp': False,
                             'publish': False, 'tlp': False, 'discover_tags': False, 'to_ids': False, 'author_tag': False, 'bulk_tag': False,
                             'dedup_titles': False}
    defaults = {'distribution': 0, 'threat_level': 4, 'analysis': 2, 'timestamp': datetime.utcfromtimestamp(0),
                'update_timestamp': False}
    config_section = 'otx_misp'

    def __init__(self, arguments):
        self.config = configparser.SafeConfigParser(allow_no_value=True)
        if arguments.config and os.path.isfile(arguments.config):
            self.config.read(arguments.config)
        if not self.config.has_section(self.config_section):
            self.config.add_section(self.config_section)
        self.arguments = arguments
        self.original_config = self._clone_config(self.config)
        self._populate_config()

    @staticmethod
    def _clone_config(config):
        clone = configparser.SafeConfigParser(allow_no_value=True)
        for section in config.sections():
            clone.add_section(section)
            for option in config.options(section):
                clone.set(section, option, config.get(section, option))
        return clone

    def _populate_config(self):
        if self.arguments.simulate:
            self.properties = self.simulation_properties
        for key, required in list(self.simulation_properties.items()):
            value = getattr(self.arguments, key, None)
            if isinstance(value, bool):
                if value:
                    value = 'yes'
                else:
                    value = 'no'
            elif isinstance(value, (list, tuple)):
                value = ','.join(value)
            elif isinstance(value, datetime):
                value = value.isoformat()
            elif isinstance(value, int):
                value = '{}'.format(value)
            if value is not None:
                self.config.set(self.config_section, key, value)
            if required:
                try:
                    value = self.config.get(self.config_section, key)
                except configparser.NoOptionError:
                    raise ConfigurationError("Missing required parameter: '--{}'".format(key))
                if value is None:
                    raise ConfigurationError("Missing required parameter: '--{}'".format(key))

    def __getattr__(self, item):
        if item not in self.arguments:
            raise AttributeError
        value = getattr(self.arguments, item, None)
        if value is None and item in self.defaults:
            value = self.defaults[item]
        if item not in self.properties or not self.config.has_option(self.config_section, item):
            return value
        if isinstance(value, bool):
            return self.config.getboolean(self.config_section, item)
        elif isinstance(value, (list, tuple)):
            parameter = self.config.get(self.config_section, item)
            return parameter.split(',')
        elif isinstance(value, datetime):
            parameter = self.config.get(self.config_section, item)
            if parameter is not None:
                try:
                    return date_parser.parse(parameter)
                except:
                    pass
        elif isinstance(value, int):
            return self.config.getint(self.config_section, item)
        return self.config.get(self.config_section, item)

    def write(self, fp):
        if self.write_config:
            config = self.config
        else:
            config = self.original_config
        if self.update_timestamp:
            config.set(self.config_section, 'timestamp', datetime.utcnow().isoformat())
        return config.write(fp)
<|endoftext|>