<commit_before>import enum
import dis
import math
import opcode as _opcode
import sys
import types

import bytecode as _bytecode

@enum.unique
class Compare(enum.IntEnum):
    LT = 0
    LE = 1
    EQ = 2
    NE = 3
    GT = 4
    GE = 5
    IN = 6
    NOT_IN = 7
    IS = 8
    IS_NOT = 9
    EXC_MATCH = 10

UNSET = object()

def const_key(obj):
    # Python implmentation of the C function _PyCode_ConstantKey()
    # of Python 3.6

    obj_type = type(obj)
    # Note: check obj_type == test_type rather than isinstance(obj, test_type)
    # to not merge instance of subtypes

    if (obj is None
        or obj is Ellipsis
            or obj_type in {int, bool, bytes, str, types.CodeType}):
        return (obj_type, obj)

    if obj_type == float:
        # all we need is to make the tuple different in either the 0.0
        # or -0.0 case from all others, just to avoid the "coercion".
        if obj == 0.0 and math.copysign(1.0, obj) < 0:
            return (obj_type, obj, None)
        else:
            return (obj_type, obj)

    if obj_type == complex:
        # For the complex case we must make complex(x, 0.)
        # different from complex(x, -0.) and complex(0., y)
        # different from complex(-0., y), for any x and y.
        # All four complex zeros must be distinguished.
        real_negzero = (obj.real == 0.0 and math.copysign(1.0, obj.real) < 0.0)
        imag_negzero = (obj.imag == 0.0 and math.copysign(1.0, obj.imag) < 0.0)

        # use True, False and None singleton as tags for the real and imag
        # sign, to make tuples different
        if real_negzero and imag_negzero:
            return (obj_type, obj, True)
        elif imag_negzero:
            return (obj_type, obj, False)
        elif real_negzero:
            return (obj_type, obj, None)
        else:
            return (obj_type, obj)

    if type(obj) == tuple:
        key = tuple(const_key(item) for item in obj)
        return (obj_type, obj, key)

    if type(obj) == frozenset:
        key = frozenset(const_key(item) for item in obj)
        return (obj_type, obj, key)

    # for other types, use the object identifier as an unique identifier
    # to ensure that they are seen as unequal.
    return (obj_type, obj, id(obj))

def _check_lineno(lineno):
    if not isinstance(lineno, int):
        raise TypeError("lineno must be an int")
    if lineno < 1:
        raise ValueError("invalid lineno")

class SetLineno:
    __slots__ = ('_lineno',)

    def __init__(self, lineno):
        _check_lineno(lineno)
        self._lineno = lineno

    @property
    def lineno(self):
        return self._lineno

    def __eq__(self, other):
        if not isinstance(other, SetLineno):
            return False
        return (self._lineno == other._lineno)

class Label:
    __slots__ = ()

class _Variable:
    __slots__ = ('name',)

    def __init__(self, name):
        self.name = name

    def __eq__(self, other):
        if type(self) != type(other):
            return False
        return (self.name == other.name)

    def __str__(self):
        return self.name

    def __repr__(self):
        return '<%s %r>' % (self.__class__.__name__, self.name)

class CellVar(_Variable):
    __slots__ = ()

class FreeVar(_Variable):
    __slots__ = ()

def _check_arg_int(name, arg):
    if not isinstance(arg, int):
        raise TypeError("operation %s argument must be an int, "
                        "got %s"
                        % (name, type(arg).__name__))

    if not(0 <= arg <= 2147483647):
        raise ValueError("operation %s argument must be in "
                         "the range 0..2,147,483,647"
                         % name)

_stack_effects = {
    # NOTE: the entries are all 3-tuples.  Entry[0/False] is non-taken jumps.
    # Entry[1/True] is for taken jumps.  Entry[2] is the max of the others.
    # The final entry as the max is maintained manually here but verified in a
    # unittest.

    # opcodes not in dis.stack_effect
    _opcode.opmap['EXTENDED_ARG']: (0, 0, 0),
    _opcode.opmap['NOP']: (0, 0, 0),

    # Jump taken/not-taken are different:
    _opcode.opmap['JUMP_IF_TRUE_OR_POP']: (-1, 0, 0),
    _opcode.opmap['JUMP_IF_FALSE_OR_POP']: (-1, 0, 0),
    _opcode.opmap['FOR_ITER']: (1, -1, 1),
    _opcode.opmap['SETUP_WITH']: (1, 6, 6),
    _opcode.opmap['SETUP_ASYNC_WITH']: (0, 5, 5),
    _opcode.opmap['SETUP_EXCEPT']: (0, 6, 6),   # as of 3.7, below for <=3.6
    _opcode.opmap['SETUP_FINALLY']: (0, 6, 6),  # as of 3.7, below for <=3.6
}

# More stack effect values that are unique to the version of Python.
if not (sys.version_info >= (3, 7)):
    _stack_effects.update({
        _opcode.opmap['SETUP_WITH']: (7, 7, 7),
        _opcode.opmap['SETUP_EXCEPT']: (6, 9, 9),
        _opcode.opmap['SETUP_FINALLY']: (6, 9, 9),
    })

class Instr:
    """Abstract instruction."""

    __slots__ = ('_name', '_opcode', '_arg', '_lineno')

    def __init__(self, name, arg=UNSET, *, lineno=None):
        self._set(name, arg, lineno)

    def _check_arg(self, name, opcode, arg):
        if opcode >= _opcode.HAVE_ARGUMENT:
            if arg is UNSET:
                raise ValueError("operation %s requires an argument" % name)
        else:
            if arg is not UNSET:
                raise ValueError("operation %s has no argument" % name)

        if self._has_jump(opcode):
            if not isinstance(arg, (Label, _bytecode.BasicBlock)):
                raise TypeError("operation %s argument type must be "
                                "Label or BasicBlock, got %s"
                                % (name, type(arg).__name__))

        elif opcode in _opcode.hasfree:
            if not isinstance(arg, (CellVar, FreeVar)):
                raise TypeError("operation %s argument must be CellVar "
                                "or FreeVar, got %s"
                                % (name, type(arg).__name__))

        elif (opcode in _opcode.haslocal
              or opcode in _opcode.hasname):
            if not isinstance(arg, str):
                raise TypeError("operation %s argument must be a str, "
                                "got %s"
                                % (name, type(arg).__name__))

        elif opcode in _opcode.hasconst:
            if isinstance(arg, Label):
                raise ValueError("label argument cannot be used "
                                 "in %s operation" % name)
            if isinstance(arg, _bytecode.BasicBlock):
                raise ValueError("block argument cannot be used "
                                 "in %s operation" % name)

        elif opcode in _opcode.hascompare:
            if not isinstance(arg, Compare):
                raise TypeError("operation %s argument type must be "
                                "Compare, got %s"
                                % (name, type(arg).__name__))

        elif opcode >= _opcode.HAVE_ARGUMENT:
            _check_arg_int(name, arg)

    def _set(self, name, arg, lineno):
        if not isinstance(name, str):
            raise TypeError("operation name must be a str")
        try:
            opcode = _opcode.opmap[name]
        except KeyError:
            raise ValueError("invalid operation name")

        # check lineno
        if lineno is not None:
            _check_lineno(lineno)

        self._check_arg(name, opcode, arg)

        opcode = _opcode.opmap[name]
        self._name = name
        self._opcode = opcode
        self._arg = arg
        self._lineno = lineno

    def set(self, name, arg=UNSET):
        """Modify the instruction in-place.

        Replace name and arg attributes. Don't modify lineno.
        """
        self._set(name, arg, self._lineno)

    def require_arg(self):
        """Does the instruction require an argument?"""
        return (self._opcode >= _opcode.HAVE_ARGUMENT)

    @property
    def name(self):
        return self._name

    @name.setter
    def name(self, name):
        self._set(name, self._arg, self._lineno)

    @property
    def opcode(self):
        return self._opcode

    @opcode.setter
    def opcode(self, op):
        if not isinstance(op, int):
            raise TypeError("operator code must be an int")
        if 0 <= op <= 255:
            name = _opcode.opname[op]
            valid = (name != '<%r>' % op)
        else:
            valid = False
        if not valid:
            raise ValueError("invalid operator code")

        self._set(name, self._arg, self._lineno)

    @property
    def arg(self):
        return self._arg

    @arg.setter
    def arg(self, arg):
        self._set(self._name, arg, self._lineno)

    @property
    def lineno(self):
        return self._lineno

    @lineno.setter
    def lineno(self, lineno):
        self._set(self._name, self._arg, lineno)

    def stack_effect(self, jump=None):
        """Return stack effect of the instruction.

        Some opcodes have different stack effect when jump to the target and
        when not jump. The jump parameter indicates whether to return the jump
        taken or not-taken scenario.  Specifying None means to return the
        maximum value.
        """

        effect = _stack_effects.get(self._opcode, None)
        if effect is not None:
            return effect[-1] if jump is None else effect[jump]

        # TODO: if dis.stack_effect ever expands to take the 'jump' parameter
        # then we should pass that through, and perhaps remove some of the
        # overrides that are set up in _init_stack_effects()

        # All opcodes whose arguments are not represented by integers have
        # a stack_effect indepent of their argument.
        arg = (self._arg if isinstance(self._arg, int) else
               0 if self._opcode >= _opcode.HAVE_ARGUMENT else None)
        return dis.stack_effect(self._opcode, arg)

    def copy(self):
        return self.__class__(self._name, self._arg, lineno=self._lineno)

    def __repr__(self):
        if self._arg is not UNSET:
            return ('<%s arg=%r lineno=%s>'
                    % (self._name, self._arg, self._lineno))
        else:
            return ('<%s lineno=%s>'
                    % (self._name, self._lineno))

    def _cmp_key(self, labels=None):
        arg = self._arg
        if self._opcode in _opcode.hasconst:
            arg = const_key(arg)
        elif isinstance(arg, Label) and labels is not None:
            arg = labels[arg]
        return (self._lineno, self._name, arg)

    def __eq__(self, other):
        if type(self) != type(other):
            return False
        return self._cmp_key() == other._cmp_key()

    @staticmethod
    def _has_jump(opcode):
        return (opcode in _opcode.hasjrel
                or opcode in _opcode.hasjabs)

    def has_jump(self):
        return self._has_jump(self._opcode)

    def is_cond_jump(self):
        """Is a conditional jump?"""
        # Ex: POP_JUMP_IF_TRUE, JUMP_IF_FALSE_OR_POP
        return ('JUMP_IF_' in self._name)

    def is_uncond_jump(self):
        """Is an unconditional jump?"""
        return self.name in {'JUMP_FORWARD', 'JUMP_ABSOLUTE'}

    def is_final(self):
        if self._name in {'RETURN_VALUE', 'RAISE_VARARGS',
                          'BREAK_LOOP', 'CONTINUE_LOOP'}:
            return True
        if self.is_uncond_jump():
            return True
        return False
<commit_msg>fix comparison to sys.version_info<commit_after>import enum
import dis
import math
import opcode as _opcode
import sys
import types

import bytecode as _bytecode

@enum.unique
class Compare(enum.IntEnum):
    LT = 0
    LE = 1
    EQ = 2
    NE = 3
    GT = 4
    GE = 5
    IN = 6
    NOT_IN = 7
    IS = 8
    IS_NOT = 9
    EXC_MATCH = 10

UNSET = object()

def const_key(obj):
    # Python implmentation of the C function _PyCode_ConstantKey()
    # of Python 3.6

    obj_type = type(obj)
    # Note: check obj_type == test_type rather than isinstance(obj, test_type)
    # to not merge instance of subtypes

    if (obj is None
        or obj is Ellipsis
            or obj_type in {int, bool, bytes, str, types.CodeType}):
        return (obj_type, obj)

    if obj_type == float:
        # all we need is to make the tuple different in either the 0.0
        # or -0.0 case from all others, just to avoid the "coercion".
        if obj == 0.0 and math.copysign(1.0, obj) < 0:
            return (obj_type, obj, None)
        else:
            return (obj_type, obj)

    if obj_type == complex:
        # For the complex case we must make complex(x, 0.)
        # different from complex(x, -0.) and complex(0., y)
        # different from complex(-0., y), for any x and y.
        # All four complex zeros must be distinguished.
        real_negzero = (obj.real == 0.0 and math.copysign(1.0, obj.real) < 0.0)
        imag_negzero = (obj.imag == 0.0 and math.copysign(1.0, obj.imag) < 0.0)

        # use True, False and None singleton as tags for the real and imag
        # sign, to make tuples different
        if real_negzero and imag_negzero:
            return (obj_type, obj, True)
        elif imag_negzero:
            return (obj_type, obj, False)
        elif real_negzero:
            return (obj_type, obj, None)
        else:
            return (obj_type, obj)

    if type(obj) == tuple:
        key = tuple(const_key(item) for item in obj)
        return (obj_type, obj, key)

    if type(obj) == frozenset:
        key = frozenset(const_key(item) for item in obj)
        return (obj_type, obj, key)

    # for other types, use the object identifier as an unique identifier
    # to ensure that they are seen as unequal.
    return (obj_type, obj, id(obj))

def _check_lineno(lineno):
    if not isinstance(lineno, int):
        raise TypeError("lineno must be an int")
    if lineno < 1:
        raise ValueError("invalid lineno")

class SetLineno:
    __slots__ = ('_lineno',)

    def __init__(self, lineno):
        _check_lineno(lineno)
        self._lineno = lineno

    @property
    def lineno(self):
        return self._lineno

    def __eq__(self, other):
        if not isinstance(other, SetLineno):
            return False
        return (self._lineno == other._lineno)

class Label:
    __slots__ = ()

class _Variable:
    __slots__ = ('name',)

    def __init__(self, name):
        self.name = name

    def __eq__(self, other):
        if type(self) != type(other):
            return False
        return (self.name == other.name)

    def __str__(self):
        return self.name

    def __repr__(self):
        return '<%s %r>' % (self.__class__.__name__, self.name)

class CellVar(_Variable):
    __slots__ = ()

class FreeVar(_Variable):
    __slots__ = ()

def _check_arg_int(name, arg):
    if not isinstance(arg, int):
        raise TypeError("operation %s argument must be an int, "
                        "got %s"
                        % (name, type(arg).__name__))

    if not(0 <= arg <= 2147483647):
        raise ValueError("operation %s argument must be in "
                         "the range 0..2,147,483,647"
                         % name)

_stack_effects = {
    # NOTE: the entries are all 3-tuples.  Entry[0/False] is non-taken jumps.
    # Entry[1/True] is for taken jumps.  Entry[2] is the max of the others.
    # The final entry as the max is maintained manually here but verified in a
    # unittest.

    # opcodes not in dis.stack_effect
    _opcode.opmap['EXTENDED_ARG']: (0, 0, 0),
    _opcode.opmap['NOP']: (0, 0, 0),

    # Jump taken/not-taken are different:
    _opcode.opmap['JUMP_IF_TRUE_OR_POP']: (-1, 0, 0),
    _opcode.opmap['JUMP_IF_FALSE_OR_POP']: (-1, 0, 0),
    _opcode.opmap['FOR_ITER']: (1, -1, 1),
    _opcode.opmap['SETUP_WITH']: (1, 6, 6),
    _opcode.opmap['SETUP_ASYNC_WITH']: (0, 5, 5),
    _opcode.opmap['SETUP_EXCEPT']: (0, 6, 6),   # as of 3.7, below for <=3.6
    _opcode.opmap['SETUP_FINALLY']: (0, 6, 6),  # as of 3.7, below for <=3.6
}

# More stack effect values that are unique to the version of Python.
if sys.version_info < (3, 7):
    _stack_effects.update({
        _opcode.opmap['SETUP_WITH']: (7, 7, 7),
        _opcode.opmap['SETUP_EXCEPT']: (6, 9, 9),
        _opcode.opmap['SETUP_FINALLY']: (6, 9, 9),
    })

class Instr:
    """Abstract instruction."""

    __slots__ = ('_name', '_opcode', '_arg', '_lineno')

    def __init__(self, name, arg=UNSET, *, lineno=None):
        self._set(name, arg, lineno)

    def _check_arg(self, name, opcode, arg):
        if opcode >= _opcode.HAVE_ARGUMENT:
            if arg is UNSET:
                raise ValueError("operation %s requires an argument" % name)
        else:
            if arg is not UNSET:
                raise ValueError("operation %s has no argument" % name)

        if self._has_jump(opcode):
            if not isinstance(arg, (Label, _bytecode.BasicBlock)):
                raise TypeError("operation %s argument type must be "
                                "Label or BasicBlock, got %s"
                                % (name, type(arg).__name__))

        elif opcode in _opcode.hasfree:
            if not isinstance(arg, (CellVar, FreeVar)):
                raise TypeError("operation %s argument must be CellVar "
                                "or FreeVar, got %s"
                                % (name, type(arg).__name__))

        elif (opcode in _opcode.haslocal
              or opcode in _opcode.hasname):
            if not isinstance(arg, str):
                raise TypeError("operation %s argument must be a str, "
                                "got %s"
                                % (name, type(arg).__name__))

        elif opcode in _opcode.hasconst:
            if isinstance(arg, Label):
                raise ValueError("label argument cannot be used "
                                 "in %s operation" % name)
            if isinstance(arg, _bytecode.BasicBlock):
                raise ValueError("block argument cannot be used "
                                 "in %s operation" % name)

        elif opcode in _opcode.hascompare:
            if not isinstance(arg, Compare):
                raise TypeError("operation %s argument type must be "
                                "Compare, got %s"
                                % (name, type(arg).__name__))

        elif opcode >= _opcode.HAVE_ARGUMENT:
            _check_arg_int(name, arg)

    def _set(self, name, arg, lineno):
        if not isinstance(name, str):
            raise TypeError("operation name must be a str")
        try:
            opcode = _opcode.opmap[name]
        except KeyError:
            raise ValueError("invalid operation name")

        # check lineno
        if lineno is not None:
            _check_lineno(lineno)

        self._check_arg(name, opcode, arg)

        opcode = _opcode.opmap[name]
        self._name = name
        self._opcode = opcode
        self._arg = arg
        self._lineno = lineno

    def set(self, name, arg=UNSET):
        """Modify the instruction in-place.

        Replace name and arg attributes. Don't modify lineno.
        """
        self._set(name, arg, self._lineno)

    def require_arg(self):
        """Does the instruction require an argument?"""
        return (self._opcode >= _opcode.HAVE_ARGUMENT)

    @property
    def name(self):
        return self._name

    @name.setter
    def name(self, name):
        self._set(name, self._arg, self._lineno)

    @property
    def opcode(self):
        return self._opcode

    @opcode.setter
    def opcode(self, op):
        if not isinstance(op, int):
            raise TypeError("operator code must be an int")
        if 0 <= op <= 255:
            name = _opcode.opname[op]
            valid = (name != '<%r>' % op)
        else:
            valid = False
        if not valid:
            raise ValueError("invalid operator code")

        self._set(name, self._arg, self._lineno)

    @property
    def arg(self):
        return self._arg

    @arg.setter
    def arg(self, arg):
        self._set(self._name, arg, self._lineno)

    @property
    def lineno(self):
        return self._lineno

    @lineno.setter
    def lineno(self, lineno):
        self._set(self._name, self._arg, lineno)

    def stack_effect(self, jump=None):
        """Return stack effect of the instruction.

        Some opcodes have different stack effect when jump to the target and
        when not jump. The jump parameter indicates whether to return the jump
        taken or not-taken scenario.  Specifying None means to return the
        maximum value.
        """

        effect = _stack_effects.get(self._opcode, None)
        if effect is not None:
            return effect[-1] if jump is None else effect[jump]

        # TODO: if dis.stack_effect ever expands to take the 'jump' parameter
        # then we should pass that through, and perhaps remove some of the
        # overrides that are set up in _init_stack_effects()

        # All opcodes whose arguments are not represented by integers have
        # a stack_effect indepent of their argument.
        arg = (self._arg if isinstance(self._arg, int) else
               0 if self._opcode >= _opcode.HAVE_ARGUMENT else None)
        return dis.stack_effect(self._opcode, arg)

    def copy(self):
        return self.__class__(self._name, self._arg, lineno=self._lineno)

    def __repr__(self):
        if self._arg is not UNSET:
            return ('<%s arg=%r lineno=%s>'
                    % (self._name, self._arg, self._lineno))
        else:
            return ('<%s lineno=%s>'
                    % (self._name, self._lineno))

    def _cmp_key(self, labels=None):
        arg = self._arg
        if self._opcode in _opcode.hasconst:
            arg = const_key(arg)
        elif isinstance(arg, Label) and labels is not None:
            arg = labels[arg]
        return (self._lineno, self._name, arg)

    def __eq__(self, other):
        if type(self) != type(other):
            return False
        return self._cmp_key() == other._cmp_key()

    @staticmethod
    def _has_jump(opcode):
        return (opcode in _opcode.hasjrel
                or opcode in _opcode.hasjabs)

    def has_jump(self):
        return self._has_jump(self._opcode)

    def is_cond_jump(self):
        """Is a conditional jump?"""
        # Ex: POP_JUMP_IF_TRUE, JUMP_IF_FALSE_OR_POP
        return ('JUMP_IF_' in self._name)

    def is_uncond_jump(self):
        """Is an unconditional jump?"""
        return self.name in {'JUMP_FORWARD', 'JUMP_ABSOLUTE'}

    def is_final(self):
        if self._name in {'RETURN_VALUE', 'RAISE_VARARGS',
                          'BREAK_LOOP', 'CONTINUE_LOOP'}:
            return True
        if self.is_uncond_jump():
            return True
        return False
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from datetime import datetime

import scrapy
from bs4 import BeautifulSoup
from scrapy.conf import settings
from scrapy.crawler import CrawlerProcess
from scrapy.selector import Selector
from scrapy.utils.log import configure_logging
from scrapy.utils.project import get_project_settings
from twisted.internet import reactor, defer

from cookpad.items import RecipeItem, RecipeURLItem
from cookpad.mongodal import MongoDAL
from cookpad.mssqlodal import MsSQLDAL
import sys

class CookpadrSpider(scrapy.Spider):
    name = "cookpadr"

    def start_requests(self):
        urls = []

        if settings['IS_MSSQLDB']:
            self.msSQLDAL = MsSQLDAL()
            results = self.msSQLDAL.read(query="USP_RecipesSpider_readall",app_name=self.name)
        else:
            mongodal = MongoDAL()
            results = mongodal.read_mongo(collection="recipes_spider")

        for result in results:
            urls.append(result['url'])

        for i, url in enumerate(urls):
            yield scrapy.Request(url='https://cookpad.com/eg/%D9%88%D8%B5%D9%81%D8%A7%D8%AA/' + url,
                                 meta={'dont_merge_cookies': False} \
                                 , callback=self.parse, dont_filter=True, encoding='utf-8', errback=self.errback)

    def parse(self, response):
        page = response.body.decode("utf-8")
        soup = BeautifulSoup(page, 'html.parser')

        recipe_likes = soup.find('span', attrs={'class': 'field-group__hide subtle'}).text.strip()

        likes = 0
        try:
            #likes = (0, int(recipe_likes.strip()))[len(recipe_likes.strip()) > 0]
            likes = int(recipe_likes)
        except:
            likes = 0
        self.logger.debug("likes: "+str(likes)+ " recipe_likes: "+recipe_likes)
        if likes != 0:

            recipe_name = soup.find('h1', {'class': "recipe-show__title recipe-title strong field-group--no-container-xs"}).text.strip()
            self.logger.debug(recipe_name)

            author_name = soup.find('span', attrs={'itemprop': "author"}).text.strip().replace("'","-")
            self.logger.debug(author_name)

            author_url = soup.find('span', attrs={'itemprop': "author"}).parent['href']
            self.logger.debug(author_url)

            recipe_id = soup.find('div', attrs={'class': 'bookmark-button '})['id'].replace('bookmark_recipe_', '')
            self.logger.debug(recipe_id)

            try:
                recipe_image = [x['src'] for x in soup.findAll('img', {'alt': recipe_name})][0]
            except:
                recipe_image = ''
            desc = soup.find(attrs={'name': 'description'})['content']

            # recipe_image = recipe_image_div.find('a',{'data-target':'#modal'})["href"]
            recipe_ingredients = []
            index = 1
            for i in soup.find_all('li', {'class': 'ingredient'}):
                if i.text.strip() != '':
                    recipe_ingredients.append({'in': index, 'n': i.text.strip().replace("'","-")})
                    index = index + 1

            index = 1
            recipe_instructions = []
            for i in soup.find_all('li', {'class': 'step numbered-list__item card-sm'}):
                step = i.find('p').text.strip()
                try:
                    imageUrl = [x['src'] for x in i.findAll('img')][0]\
                        .replace('//global.cpcdn.com/en/assets/blank_step-17c926f7cd09f48ae848b5dfe68bcf26cf84cf2129001eee9513dc6c062d83bc.jpg','')
                except:
                    imageUrl = ''
                recipe_instructions.append({'in': index, 'txt': step.replace("'","-"), 'img': imageUrl})
                del step, imageUrl

                index = index + 1

            recipe_tags = []
            for i in soup.find_all('ul', {"class": 'list-inline'}):
                for x in i.find_all('a'):
                    if('/eg/search/' in x['href']):
                        recipe_tags.append(x.text.strip().replace("'","-"))

            recipe = RecipeItem()
            recipe["n"] = recipe_name.replace("'","-")
            recipe["src"] = response.url.replace('https://cookpad.com/eg/%D9%88%D8%B5%D9%81%D8%A7%D8%AA/', '')
            recipe["rcpe_id"] = (recipe_id, int(recipe_id.strip()))[len(recipe_id.strip()) > 0]
            recipe["ingrd"] = recipe_ingredients
            recipe["instrct"] = recipe_instructions
            recipe["img"] = recipe_image
            recipe["auth"] = {'n': author_name, 'src': author_url}
            recipe["tags"] = recipe_tags
            recipe["likes"] = likes
            recipe["pub"] = datetime.utcnow().isoformat()
            # recipe["etag"] = response.headers.get(b'Etag').decode("utf-8")
            # recipe["desc"] = desc
            del page, soup, recipe_name, author_name, author_url, recipe_id, likes
            del recipe_image, recipe_likes, recipe_ingredients, index
            del recipe_tags, recipe_instructions

            self.logger.debug(recipe)

            return recipe
        else:
            del page, soup, recipe_likes, likes

            if(self.logger.isEnabledFor(10)):
                self.logger.debug("0 Likes")
            pass

    def errback(self, response):
        if settings['IS_MSSQLDB']:
            src = response.request.url.replace('https://cookpad.com/eg/%D9%88%D8%B5%D9%81%D8%A7%D8%AA/', '')
            self.msSQLDAL.execute_none_query(query="exec USP_Recipes_NotFound_insert @src='%s'", sp_params=(src.replace('\r\n', '')), \
                                        app_name='MsSQLDBPipeline-' + self.name)
        pass

class ExtractlinksSpider(scrapy.Spider):
    name = "extractLinks"
    allowed_domains = ["cookpad.com"]
    base_url = 'https://cookpad.com/eg/وصفات?page=%s'
    start_urls = [base_url % 1]
    pageid = 1
    max_page_Id = 1020

    def parse(self, response):
        recipes = Selector(response).xpath('//div[@class="masonry__item"]/div/a[@class="link-unstyled"]/@href').extract()

        for recipe in recipes:
            item = RecipeURLItem()
            item['url'] = recipe.replace('/eg/%D9%88%D8%B5%D9%81%D8%A7%D8%AA/', '')
            yield item

        #if len(recipes) > 0 :
        if len(recipes) > 0 and self.pageid < self.max_page_Id:
            self.pageid = self.pageid + 1
            next_page = self.base_url % self.pageid
            yield scrapy.Request(url=next_page, callback=self.parse, meta={'dont_merge_cookies': False}, \
            dont_filter=True \
            , encoding='utf-8', errback=self.errback)

    def errback(self, response):
        pass

def notThreadSafe(x):
    """do something that isn't thread-safe"""
    pass

configure_logging()
project_settings = get_project_settings()

runner = CrawlerProcess(settings=project_settings)

@defer.inlineCallbacks
def crawl():
    #yield runner.crawl(ExtractlinksSpider)
    yield runner.crawl(CookpadrSpider)
    reactor.stop()

try:
    crawl()
    reactor.run()  # the script will block here until the last crawl call is finished
except:
    pass
<commit_msg>2018-05-15<commit_after># -*- coding: utf-8 -*-
from datetime import datetime

import scrapy
from bs4 import BeautifulSoup
from scrapy.conf import settings
from scrapy.crawler import CrawlerProcess
from scrapy.selector import Selector
from scrapy.utils.log import configure_logging
from scrapy.utils.project import get_project_settings
from twisted.internet import reactor, defer

from cookpad.items import RecipeItem, RecipeURLItem
from cookpad.mongodal import MongoDAL
from cookpad.mssqlodal import MsSQLDAL
import sys
import re

class CookpadrSpider(scrapy.Spider):
    name = "cookpadr"

    def start_requests(self):
        urls = []

        if settings['IS_MSSQLDB']:
            self.msSQLDAL = MsSQLDAL()
            results = self.msSQLDAL.read(query="USP_RecipesSpider_readall",app_name=self.name)
        else:
            mongodal = MongoDAL()
            results = mongodal.read_mongo(collection="recipes_spider")

        for result in results:
            urls.append(result['url'])

        for i, url in enumerate(urls):
            yield scrapy.Request(url='https://cookpad.com/eg/%D9%88%D8%B5%D9%81%D8%A7%D8%AA/' + url,
                                 meta={'dont_merge_cookies': False} \
                                 , callback=self.parse, dont_filter=True, encoding='utf-8', errback=self.errback)

    def parse(self, response):
        page = response.body.decode("utf-8")
        soup = BeautifulSoup(page, 'html.parser')

        recipe_likes = soup.find('span', attrs={'class': 'field-group__hide subtle'}).text.strip()

        likes = 0
        try:
            #likes = (0, int(recipe_likes.strip()))[len(recipe_likes.strip()) > 0]
            likes = int(re.sub("[^0-9]", "",recipe_likes))
        except:
            likes = 0
        self.logger.debug("likes: "+str(likes)+ " recipe_likes: "+recipe_likes)
        if likes != 0:

            recipe_name = soup.find('h1', {'class': "recipe-show__title recipe-title strong field-group--no-container-xs"}).text.strip()
            self.logger.debug(recipe_name)

            author_name = soup.find('span', attrs={'itemprop': "author"}).text.strip().replace("'","-")
            self.logger.debug(author_name)

            author_url = soup.find('span', attrs={'itemprop': "author"}).parent['href']
            self.logger.debug(author_url)

            recipe_id = soup.find('div', attrs={'class': 'bookmark-button '})['id'].replace('bookmark_recipe_', '')
            self.logger.debug(recipe_id)

            try:
                recipe_image = [x['src'] for x in soup.findAll('img', {'alt': recipe_name})][0]
            except:
                recipe_image = ''
            desc = soup.find(attrs={'name': 'description'})['content']

            # recipe_image = recipe_image_div.find('a',{'data-target':'#modal'})["href"]
            recipe_ingredients = []
            index = 1
            for i in soup.find_all('li', {'class': 'ingredient'}):
                if i.text.strip() != '':
                    recipe_ingredients.append({'in': index, 'n': i.text.strip().replace("'","-")})
                    index = index + 1

            index = 1
            recipe_instructions = []
            for i in soup.find_all('li', {'class': 'step numbered-list__item card-sm'}):
                step = i.find('p').text.strip()
                try:
                    imageUrl = [x['src'] for x in i.findAll('img')][0]\
                        .replace('//global.cpcdn.com/en/assets/blank_step-17c926f7cd09f48ae848b5dfe68bcf26cf84cf2129001eee9513dc6c062d83bc.jpg','')
                except:
                    imageUrl = ''
                recipe_instructions.append({'in': index, 'txt': step.replace("'","-"), 'img': imageUrl})
                del step, imageUrl

                index = index + 1

            recipe_tags = []
            for i in soup.find_all('ul', {"class": 'list-inline'}):
                for x in i.find_all('a'):
                    if('/eg/search/' in x['href']):
                        recipe_tags.append(x.text.strip().replace("'","-"))

            recipe = RecipeItem()
            recipe["n"] = recipe_name.replace("'","-")
            recipe["src"] = response.url.replace('https://cookpad.com/eg/%D9%88%D8%B5%D9%81%D8%A7%D8%AA/', '')
            recipe["rcpe_id"] = (recipe_id, int(recipe_id.strip()))[len(recipe_id.strip()) > 0]
            recipe["ingrd"] = recipe_ingredients
            recipe["instrct"] = recipe_instructions
            recipe["img"] = recipe_image
            recipe["auth"] = {'n': author_name, 'src': author_url}
            recipe["tags"] = recipe_tags
            recipe["likes"] = likes
            recipe["pub"] = datetime.utcnow().isoformat()
            # recipe["etag"] = response.headers.get(b'Etag').decode("utf-8")
            # recipe["desc"] = desc
            del page, soup, recipe_name, author_name, author_url, recipe_id, likes
            del recipe_image, recipe_likes, recipe_ingredients, index
            del recipe_tags, recipe_instructions

            self.logger.debug(recipe)

            return recipe
        else:
            del page, soup, recipe_likes, likes

            if(self.logger.isEnabledFor(10)):
                self.logger.debug("0 Likes")
            pass

    def errback(self, response):
        if settings['IS_MSSQLDB']:
            src = response.request.url.replace('https://cookpad.com/eg/%D9%88%D8%B5%D9%81%D8%A7%D8%AA/', '')
            self.msSQLDAL.execute_none_query(query="exec USP_Recipes_NotFound_insert @src='%s'", sp_params=(src.replace('\r\n', '')), \
                                        app_name='MsSQLDBPipeline-' + self.name)
        pass

class ExtractlinksSpider(scrapy.Spider):
    name = "extractLinks"
    allowed_domains = ["cookpad.com"]
    base_url = 'https://cookpad.com/eg/وصفات?page=%s'
    start_urls = [base_url % 1]
    pageid = 1
    max_page_Id = 1020

    def parse(self, response):
        recipes = Selector(response).xpath('//div[@class="masonry__item"]/div/a[@class="link-unstyled"]/@href').extract()

        for recipe in recipes:
            item = RecipeURLItem()
            item['url'] = recipe.replace('/eg/%D9%88%D8%B5%D9%81%D8%A7%D8%AA/', '')
            yield item

        #if len(recipes) > 0 :
        if len(recipes) > 0 and self.pageid < self.max_page_Id:
            self.pageid = self.pageid + 1
            next_page = self.base_url % self.pageid
            yield scrapy.Request(url=next_page, callback=self.parse, meta={'dont_merge_cookies': False}, \
            dont_filter=True \
            , encoding='utf-8', errback=self.errback)

    def errback(self, response):
        pass

def notThreadSafe(x):
    """do something that isn't thread-safe"""
    pass

configure_logging()
project_settings = get_project_settings()

runner = CrawlerProcess(settings=project_settings)

@defer.inlineCallbacks
def crawl():
    #yield runner.crawl(ExtractlinksSpider)
    yield runner.crawl(CookpadrSpider)
    reactor.stop()

try:
    crawl()
    reactor.run()  # the script will block here until the last crawl call is finished
except:
    pass
<|endoftext|>


<commit_before>import datetime
import re
from collections import defaultdict

import dateutil
from django.contrib.postgres.fields.jsonb import KeyTextTransform
from django.db.models import Count

from corehq.apps.es import UserES
from custom.icds_reports.models import AwcLocation, ICDSAuditEntryRecord, AggAwc
from custom.icds_reports.utils import india_now

class DashBoardUsage:

    title = 'Dashboard usage'
    required_fields = ['state_id', 'state_name', 'district_id', 'district_name', 'block_id', 'block_name']
    location_types = ['state_id', 'district_id', 'block_id']
    agg_required_fields = ['state_id', 'district_id', 'block_id', 'is_launched']
    location_test_fields = ['state_is_test', 'district_is_test', 'block_is_test', 'supervisor_is_test',
                            'awc_is_test']

    roles = {
        '.nod': 'Nodal Officer',
        '.ncd': 'Consultant(Nutrition & Child Development)',
        '.bcc': 'Consultant (Behaviour Change Communication & Capacity Building)',
        '.sdc': 'Consultant (Social Development & Community Mobilization)',
        '.mne': 'Consultant (Monitoring & Evaluation and Decentralized Planning)',
        '.fm': 'Consultant (Financial Management)',
        '.shd': 'Consultant (Procurement) (State Helpdesk)',
        '.pa': 'Project Associate',
        '.acc': 'Accountant',
        '.cta': 'Central Training Agent',
        '.dhd': 'District Coordinator (District Helpdesk)',
        '.dpa': 'District Project Assistant',
        '.dc': 'District Collector',
        '.bhd': 'Block Coordinator (Block Helpdesk)',
        '.bpa': 'Block Project Assistant',
        '.dpo': 'District Programme Officer (DPO)',
        '.cdpo': 'Child Development Project Officer (CDPO)',

    }

    def __init__(self, couch_user, domain):
        self.user = couch_user
        self.domain = domain
        self.agg_list = None
        self.sql_locations = None
        self.national_user = False

    def get_role_from_username(self, username):
        for key, value in self.roles.items():
            if key.lower() in username:
                return value
        return 'N/A'

    def convert_boolean_to_string(self, value):
        return 'yes' if value else 'no'

    def get_users_by_location(self, user_supported_locations):
        if not self.national_user:
            user_query = UserES().mobile_users().domain(self.domain).location(
                user_supported_locations
            ).fields(['username', 'assigned_location_ids', 'last_login'])
        else:
            user_query = UserES().mobile_users().domain(self.domain
                                                        ).fields(['username',
                                                                  'assigned_location_ids',
                                                                  'last_login'])
        return [u for u in user_query.run().hits]

    def get_names_from_ids(self, location_id):
        return location_id.split('_')[0] + '_name'

    def convert_rs_to_matrix(self, results_queryset, main_location_type=None):
        """

        :param results_queryset: AwcLocation values queryset
        :param main_location_type: parent location type to fetch only locations below that level
        :return: location matrix similar to a dataframe and locations ids of all the locations below the given
         location
        """
        location_matrix = []
        location_ids = []
        if main_location_type is not None:
            sub_location_types = self.location_types[self.location_types.index(main_location_type) + 1:]
        else:
            sub_location_types = self.location_types

        for result in results_queryset:
            row_data = [result['state_id'], result['district_id'], result['block_id']]
            location_matrix.append(row_data)
            # adding only descendants to the main location
            for sub_location in sub_location_types:
                if result[sub_location] not in location_ids:
                    location_ids.append(result[sub_location])
        return location_matrix, location_ids

    def get_location_id_string_from_location_type(self, location_type):
        return location_type + '_id'

    def get_location_type_string_from_location_id(self, location_id):
        return location_id.replace('_id', '')

    def check_if_date_in_last_week(self, date):
        if date is None:
            return False
        d = dateutil.parser.parse(date).strftime("%Y-%m-%d")
        d = datetime.datetime.strptime(d, "%Y-%m-%d")
        now = datetime.datetime.now()
        return (now - d).days < 7

    def prepare_is_launched_agg_list(self, location_type=None, location_id=None):
        """
        populates the location_id vs is_launched status and location_id vs location_type mappings for the
         descendants of given location
        :param location_type:
        :param location_id:
        :return: None
        """
        filter_dict = {'aggregation_level': 5,
                       'month': datetime.date.today().replace(day=1)}
        if location_type is not None:
            location_type = self.get_location_id_string_from_location_type(location_type)
            sub_location_types = self.agg_required_fields[self.location_types.index(location_type) + 1:]
            filter_dict[location_type] = location_id
        else:
            sub_location_types = self.agg_required_fields

        aggregate_records = AggAwc.objects.filter(**filter_dict).values(*sub_location_types)

        aggregate_records_dict = defaultdict(str)
        location_type_id_mapping = defaultdict(str)

        for aggregate_record in aggregate_records:
            for sub_location_type in sub_location_types:
                if sub_location_type != 'is_launched':
                    if aggregate_record[sub_location_type] not in aggregate_records_dict or not\
                            aggregate_records_dict[aggregate_record[sub_location_type]]:
                        aggregate_records_dict[aggregate_record[sub_location_type]] =\
                            aggregate_record['is_launched']
                    if aggregate_record[sub_location_type] not in location_type_id_mapping:
                        location_type_id_mapping[aggregate_record[sub_location_type]] = sub_location_type

        self.agg_list = aggregate_records_dict
        self.sql_locations = location_type_id_mapping

    def get_excel_data(self):
        excel_rows = []
        filters = [['Generated at', india_now()]]
        headers = ['Sr.No', 'State/UT Name', 'District Name', 'Block Name', 'Username', 'Level', 'Role',
                   'Launched?', 'Last Login', 'Logged in the last week?', 'Total', 'Child', 'Pregnant Women',
                   'Demographics', 'System Usage', 'AWC infrastructure', 'Child Growth Monitoring List',
                   'ICDS - CAS Monthly Register', 'AWW Performance Report', 'LS Performance Report',
                   'Take Home Ration']
        excel_rows.append(headers)
        serial_count = 0
        logged_in_user_locations = list(self.user.get_sql_locations(self.domain))
        if not logged_in_user_locations:
            self.national_user = True

        loop_counter = 0
        # Need to fetch all users for a national user
        while (self.national_user and loop_counter < 1) or len(logged_in_user_locations) > loop_counter:
            # getting the location types to retrieve for this user location

            location_type_filter = {
                'aggregation_level': 5
            }
            if not self.national_user:
                user_location = logged_in_user_locations[loop_counter]
                user_location_type_name = \
                    self.get_location_id_string_from_location_type(user_location.location_type_name)
                location_type_filter[user_location_type_name]: user_location.get_id
            else:
                user_location_type_name = None
            for test_location in self.location_test_fields:
                location_type_filter[test_location] = 0

            all_awc_locations = AwcLocation.objects.filter(**location_type_filter).values(*self.required_fields)
            # converting the result set to matrix to fetch ancestors for a given location
            location_matrix, location_ids =\
                self.convert_rs_to_matrix(all_awc_locations, user_location_type_name)

            users = self.get_users_by_location(location_ids)

            dashboard_uname_rx = re.compile(r'^\d*\.[a-zA-Z]*@.*')

            usernames = [user['username'] for user in users if dashboard_uname_rx.match(user['username'])]

            records = list(ICDSAuditEntryRecord.objects.filter(url='/a/icds-dashboard-qa/cas_export',
                                                               username__in=usernames)
                           .annotate(indicator=KeyTextTransform('indicator', 'post_data')).values('indicator',
                                                                                'username')
                           .annotate(count=Count('indicator')).order_by('username', 'indicator'))

            if self.national_user:
                self.prepare_is_launched_agg_list()
            else:
                self.prepare_is_launched_agg_list(user_location.location_type_name, user_location.get_id)

            user_counts = defaultdict(int)
            user_indicators = defaultdict(list)
            for record in records:
                user_counts[record['username']] += record['count']
                user_indicators[record['username']].append(record['count'])
            # accumulating the indicator counts
            for user in users:
                indicator_count = user_indicators[user['username']]
                user_sql_location_ids = user['assigned_location_ids']
                if isinstance(user_sql_location_ids, str):
                    user_sql_location_ids = [user_sql_location_ids]

                for user_sql_location in user_sql_location_ids:
                    # getting the location type to look up in matrix
                    if user_sql_location not in self.sql_locations:
                        continue
                    location_type_id = self.sql_locations[user_sql_location]
                    column_index = self.location_types.index(location_type_id)
                    user_location_row = None
                    # iterating and getting the db row from matrix
                    for row in location_matrix:
                        if row[column_index] == user_sql_location:
                            user_location_row = row
                            break
                    user_location_type = self.get_location_type_string_from_location_id(location_type_id)

                    if user_location_row is not None:
                        if user_location_type == 'state':
                            user_location_row[1] = 'All'
                            user_location_row[2] = 'All'
                        if user_location_type == 'district':
                            user_location_row[2] == 'All'
                        serial_count += 1

                        excel = [serial_count, user_location_row[0], user_location_row[1],
                                 user_location_row[2], user['username'], user_location_type,
                                 self.get_role_from_username(user['username']),
                                 self.convert_boolean_to_string(self.agg_list[user_sql_location]),
                                 user['last_login'],
                                 self.convert_boolean_to_string(self.check_if_date_in_last_week(
                                     user['last_login'])), user_counts[user['username']]]
                        excel.extend(indicator_count)
                        excel_rows.append(excel)
            loop_counter += 1

        return [
            [
                self.title,
                excel_rows
            ],
            [
                'Export Info',
                filters
            ]
        ]
<commit_msg>regex issue fix<commit_after>import datetime
import re
from collections import defaultdict

import dateutil
from django.contrib.postgres.fields.jsonb import KeyTextTransform
from django.db.models import Count

from corehq.apps.es import UserES
from custom.icds_reports.models import AwcLocation, ICDSAuditEntryRecord, AggAwc
from custom.icds_reports.utils import india_now

class DashBoardUsage:

    title = 'Dashboard usage'
    required_fields = ['state_id', 'state_name', 'district_id', 'district_name', 'block_id', 'block_name']
    location_types = ['state_id', 'district_id', 'block_id']
    agg_required_fields = ['state_id', 'district_id', 'block_id', 'is_launched']
    location_test_fields = ['state_is_test', 'district_is_test', 'block_is_test', 'supervisor_is_test',
                            'awc_is_test']

    roles = {
        '.nod': 'Nodal Officer',
        '.ncd': 'Consultant(Nutrition & Child Development)',
        '.bcc': 'Consultant (Behaviour Change Communication & Capacity Building)',
        '.sdc': 'Consultant (Social Development & Community Mobilization)',
        '.mne': 'Consultant (Monitoring & Evaluation and Decentralized Planning)',
        '.fm': 'Consultant (Financial Management)',
        '.shd': 'Consultant (Procurement) (State Helpdesk)',
        '.pa': 'Project Associate',
        '.acc': 'Accountant',
        '.cta': 'Central Training Agent',
        '.dhd': 'District Coordinator (District Helpdesk)',
        '.dpa': 'District Project Assistant',
        '.dc': 'District Collector',
        '.bhd': 'Block Coordinator (Block Helpdesk)',
        '.bpa': 'Block Project Assistant',
        '.dpo': 'District Programme Officer (DPO)',
        '.cdpo': 'Child Development Project Officer (CDPO)',

    }

    def __init__(self, couch_user, domain):
        self.user = couch_user
        self.domain = domain
        self.agg_list = None
        self.sql_locations = None
        self.national_user = False

    def get_role_from_username(self, username):
        for key, value in self.roles.items():
            if key.lower() in username:
                return value
        return 'N/A'

    def convert_boolean_to_string(self, value):
        return 'yes' if value else 'no'

    def get_users_by_location(self, user_supported_locations):
        if not self.national_user:
            user_query = UserES().mobile_users().domain(self.domain).location(
                user_supported_locations
            ).fields(['username', 'assigned_location_ids', 'last_login'])
        else:
            user_query = UserES().mobile_users().domain(self.domain
                                                        ).fields(['username',
                                                                  'assigned_location_ids',
                                                                  'last_login'])
        return [u for u in user_query.run().hits]

    def get_names_from_ids(self, location_id):
        return location_id.split('_')[0] + '_name'

    def convert_rs_to_matrix(self, results_queryset, main_location_type=None):
        """

        :param results_queryset: AwcLocation values queryset
        :param main_location_type: parent location type to fetch only locations below that level
        :return: location matrix similar to a dataframe and locations ids of all the locations below the given
         location
        """
        location_matrix = []
        location_ids = []
        if main_location_type is not None:
            sub_location_types = self.location_types[self.location_types.index(main_location_type) + 1:]
        else:
            sub_location_types = self.location_types

        for result in results_queryset:
            row_data = [result['state_id'], result['district_id'], result['block_id']]
            location_matrix.append(row_data)
            # adding only descendants to the main location
            for sub_location in sub_location_types:
                if result[sub_location] not in location_ids:
                    location_ids.append(result[sub_location])
        return location_matrix, location_ids

    def get_location_id_string_from_location_type(self, location_type):
        return location_type + '_id'

    def get_location_type_string_from_location_id(self, location_id):
        return location_id.replace('_id', '')

    def check_if_date_in_last_week(self, date):
        if date is None:
            return False
        d = dateutil.parser.parse(date).strftime("%Y-%m-%d")
        d = datetime.datetime.strptime(d, "%Y-%m-%d")
        now = datetime.datetime.now()
        return (now - d).days < 7

    def prepare_is_launched_agg_list(self, location_type=None, location_id=None):
        """
        populates the location_id vs is_launched status and location_id vs location_type mappings for the
         descendants of given location
        :param location_type:
        :param location_id:
        :return: None
        """
        filter_dict = {'aggregation_level': 5,
                       'month': datetime.date.today().replace(day=1)}
        if location_type is not None:
            location_type = self.get_location_id_string_from_location_type(location_type)
            sub_location_types = self.agg_required_fields[self.location_types.index(location_type) + 1:]
            filter_dict[location_type] = location_id
        else:
            sub_location_types = self.agg_required_fields

        aggregate_records = AggAwc.objects.filter(**filter_dict).values(*sub_location_types)

        aggregate_records_dict = defaultdict(str)
        location_type_id_mapping = defaultdict(str)

        for aggregate_record in aggregate_records:
            for sub_location_type in sub_location_types:
                if sub_location_type != 'is_launched':
                    if aggregate_record[sub_location_type] not in aggregate_records_dict or not\
                            aggregate_records_dict[aggregate_record[sub_location_type]]:
                        aggregate_records_dict[aggregate_record[sub_location_type]] =\
                            aggregate_record['is_launched']
                    if aggregate_record[sub_location_type] not in location_type_id_mapping:
                        location_type_id_mapping[aggregate_record[sub_location_type]] = sub_location_type

        self.agg_list = aggregate_records_dict
        self.sql_locations = location_type_id_mapping

    def get_excel_data(self):
        excel_rows = []
        filters = [['Generated at', india_now()]]
        headers = ['Sr.No', 'State/UT Name', 'District Name', 'Block Name', 'Username', 'Level', 'Role',
                   'Launched?', 'Last Login', 'Logged in the last week?', 'Total', 'Child', 'Pregnant Women',
                   'Demographics', 'System Usage', 'AWC infrastructure', 'Child Growth Monitoring List',
                   'ICDS - CAS Monthly Register', 'AWW Performance Report', 'LS Performance Report',
                   'Take Home Ration']
        excel_rows.append(headers)
        serial_count = 0
        logged_in_user_locations = list(self.user.get_sql_locations(self.domain))
        if not logged_in_user_locations:
            self.national_user = True

        loop_counter = 0
        # Need to fetch all users for a national user
        while (self.national_user and loop_counter < 1) or len(logged_in_user_locations) > loop_counter:
            # getting the location types to retrieve for this user location

            location_type_filter = {
                'aggregation_level': 5
            }
            if not self.national_user:
                user_location = logged_in_user_locations[loop_counter]
                user_location_type_name = \
                    self.get_location_id_string_from_location_type(user_location.location_type_name)
                location_type_filter[user_location_type_name]: user_location.get_id
            else:
                user_location_type_name = None
            for test_location in self.location_test_fields:
                location_type_filter[test_location] = 0

            all_awc_locations = AwcLocation.objects.filter(**location_type_filter).values(*self.required_fields)
            # converting the result set to matrix to fetch ancestors for a given location
            location_matrix, location_ids =\
                self.convert_rs_to_matrix(all_awc_locations, user_location_type_name)

            users = self.get_users_by_location(location_ids)

            dashboard_uname_rx = re.compile(r'^\d*\.[a-zA-Z]*@.*')

            usernames = []
            for user in users:
                if dashboard_uname_rx.match(user['username']):
                    usernames.append(user['username'])

            records = list(ICDSAuditEntryRecord.objects.filter(url='/a/icds-dashboard-qa/cas_export',
                                                               username__in=usernames)
                           .annotate(indicator=KeyTextTransform('indicator', 'post_data')).values('indicator',
                                                                                'username')
                           .annotate(count=Count('indicator')).order_by('username', 'indicator'))

            if self.national_user:
                self.prepare_is_launched_agg_list()
            else:
                self.prepare_is_launched_agg_list(user_location.location_type_name, user_location.get_id)

            user_counts = defaultdict(int)
            user_indicators = defaultdict(list)
            for record in records:
                user_counts[record['username']] += record['count']
                user_indicators[record['username']].append(record['count'])
            # accumulating the indicator counts
            for user in users:
                indicator_count = user_indicators[user['username']]
                user_sql_location_ids = user['assigned_location_ids']
                if isinstance(user_sql_location_ids, str):
                    user_sql_location_ids = [user_sql_location_ids]

                for user_sql_location in user_sql_location_ids:
                    # getting the location type to look up in matrix
                    if user_sql_location not in self.sql_locations:
                        continue
                    location_type_id = self.sql_locations[user_sql_location]
                    column_index = self.location_types.index(location_type_id)
                    user_location_row = None
                    # iterating and getting the db row from matrix
                    for row in location_matrix:
                        if row[column_index] == user_sql_location:
                            user_location_row = row
                            break
                    user_location_type = self.get_location_type_string_from_location_id(location_type_id)

                    if user_location_row is not None:
                        if user_location_type == 'state':
                            user_location_row[1] = 'All'
                            user_location_row[2] = 'All'
                        if user_location_type == 'district':
                            user_location_row[2] == 'All'
                        serial_count += 1

                        excel = [serial_count, user_location_row[0], user_location_row[1],
                                 user_location_row[2], user['username'], user_location_type,
                                 self.get_role_from_username(user['username']),
                                 self.convert_boolean_to_string(self.agg_list[user_sql_location]),
                                 user['last_login'],
                                 self.convert_boolean_to_string(self.check_if_date_in_last_week(
                                     user['last_login'])), user_counts[user['username']]]
                        excel.extend(indicator_count)
                        excel_rows.append(excel)
            loop_counter += 1

        return [
            [
                self.title,
                excel_rows
            ],
            [
                'Export Info',
                filters
            ]
        ]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Fri Oct 13 07:55:15 2017

@author: Rachael Mansbach

Script to do a full (MPI) data analysis run using the cluster module

Description of necessary events:

1) Find all cluster IDs
2) Write out cluster sizes and cluster IDs at each time step
3) Plot Mass-averaged cluster size of contact, aligned, and optical clusters
both separately and in the same plot, including standard deviation over runs
and save raw mu2 data
4) Compute linear and nonlinear Smoluchowski fits & plot for contact, optical,
and aligned clusters
5) Compute, plot, and save data for the correlation integral of the final
snapshot
"""
from __future__ import absolute_import, division, print_function
from time import time
import clustering as cl
import gsd.hoomd
import os.path as op
import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import pdb
save_path = SSS
data_path=save_path
#Matlab setup

plt.ioff()
font = {'weight' : 'bold',
        'size'   : 22}

matplotlib.rc('font', **font)

runs = 1

ttotal = 799
tstart = 0
ats = {'contact':17,'optical':12,'aligned':6}
#molno = 4
molno = 10648
c1=float(BBB)
c2=0.35
c3=0.35
c1 = max(0.5,(c1/100.)*1.1225)
cs={'contact':c1,'optical':c2,'aligned':c3}
cutoff = {'contact':c1*c1,'optical':c2*c2,'aligned':c3*c3}
compairs = np.array([[0,6],[1,7],[2,8],[3,9],[4,10],[5,11]])
molnolabel = 10000
AAdlabel = AAA
SCdlabel = SCSCSC
BBdlabel = BBB
dt = 1.0
rank=0
emax = 73.5
estep = 0.147
atype = u'LS'
combeadtype = 'E'
colors = {'contact':'red','optical':'blue','aligned':'olive'}

fbase = 'mols'+str(molnolabel)+'_' + str(AAdlabel)+'-'\
        +str(SCdlabel)+'-'+str(BBdlabel)+'_short_run'

fnames = []
ldfnames = []
for i in range(runs):
    fname = op.join(data_path,fbase + str(i+1) + '.gsd')
    fnames.append(fname)
    ldfname = op.join(data_path,fbase+'ldistrib'+str(i+1))
    ldfnames.append(ldfname)
start = time()
traj = gsd.hoomd.open(fname)
box = traj[0].configuration.box[0:3]
Systs = [cl.SnapSystem(traj,ats,molno,cutoff,
                       compairs=compairs,ttotal=ttotal,tstart=tstart,
                       atype=atype) for fname in fnames]
end = time()
if rank == 0:
    print("Time to setup clusters: ",end-start)

#Find all cluster IDs
start = time()
lind = 0
for Syst in Systs:
    ldfnameC = ldfnames[lind]+'_C.dat'
    ldfnameO = ldfnames[lind]+'_O.dat'
    ldfnameA = ldfnames[lind]+'_A.dat'
    Syst.get_clusters_serial('contact',box,lcompute=ldfnameC)
    Syst.get_clusters_serial('optical',box,lcompute=ldfnameO)
    Syst.get_clusters_serial('aligned',box,lcompute=ldfnameA)
    lind += 1
end = time()
if rank == 0:
    print("Time to get clusters: ",end-start)

if rank == 0:
    run = 0
    mu2s = {'contact':np.zeros([ttotal,runs]),
            'optical':np.zeros([ttotal,runs]),
            'aligned':np.zeros([ttotal,runs])}
    start = time()
    for Syst in Systs:
        for ctype in ['contact','optical','aligned']:
            #write out cluster sizes and cluster IDs at each time step
            cidName = fbase + 'cut'+str(cs[ctype]) + str(run+1) + ctype + '-CIDs.dat'
            cszName = fbase + 'cut' + str(cs[ctype]) + str(run+1) + ctype + '-sizes.dat'
            Syst.writeCIDs(ctype,op.join(save_path,cidName))
            Syst.writeSizes(ctype,op.join(save_path,cszName))
            #compute mass-averaged cluster size versus time
            #pdb.set_trace()

        run += 1
    end = time()
    print("Time to write out clusters: ",end-start)
<commit_msg>minor script modification for cutoffs<commit_after># -*- coding: utf-8 -*-
"""
Created on Fri Oct 13 07:55:15 2017

@author: Rachael Mansbach

Script to do a full (MPI) data analysis run using the cluster module

Description of necessary events:

1) Find all cluster IDs
2) Write out cluster sizes and cluster IDs at each time step
3) Plot Mass-averaged cluster size of contact, aligned, and optical clusters
both separately and in the same plot, including standard deviation over runs
and save raw mu2 data
4) Compute linear and nonlinear Smoluchowski fits & plot for contact, optical,
and aligned clusters
5) Compute, plot, and save data for the correlation integral of the final
snapshot
"""
from __future__ import absolute_import, division, print_function
from time import time
import clustering as cl
import gsd.hoomd
import os.path as op
import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import pdb
save_path = SSS
data_path=save_path
#Matlab setup

plt.ioff()
font = {'weight' : 'bold',
        'size'   : 22}

matplotlib.rc('font', **font)

runs = 1

ttotal = 799
tstart = 0
ats = {'contact':17,'optical':12,'aligned':6}
#molno = 4
molno = 10648
c1=float(BBB)
c2=0.35
c3=0.35
c1 = max(1.1,(c1/100.)*1.1225+0.1)
cs={'contact':c1,'optical':c2,'aligned':c3}
cutoff = {'contact':c1*c1,'optical':c2*c2,'aligned':c3*c3}
compairs = np.array([[0,6],[1,7],[2,8],[3,9],[4,10],[5,11]])
molnolabel = 10000
AAdlabel = AAA
SCdlabel = SCSCSC
BBdlabel = BBB
dt = 1.0
rank=0
emax = 73.5
estep = 0.147
atype = u'LS'
combeadtype = 'E'
colors = {'contact':'red','optical':'blue','aligned':'olive'}

fbase = 'mols'+str(molnolabel)+'_' + str(AAdlabel)+'-'\
        +str(SCdlabel)+'-'+str(BBdlabel)+'_short_run'

fnames = []
ldfnames = []
for i in range(runs):
    fname = op.join(data_path,fbase + str(i+1) + '.gsd')
    fnames.append(fname)
    ldfname = op.join(data_path,fbase+'ldistrib'+str(i+1))
    ldfnames.append(ldfname)
start = time()
traj = gsd.hoomd.open(fname)
box = traj[0].configuration.box[0:3]
Systs = [cl.SnapSystem(traj,ats,molno,cutoff,
                       compairs=compairs,ttotal=ttotal,tstart=tstart,
                       atype=atype) for fname in fnames]
end = time()
if rank == 0:
    print("Time to setup clusters: ",end-start)

#Find all cluster IDs
start = time()
lind = 0
for Syst in Systs:
    ldfnameC = ldfnames[lind]+'_C.dat'
    ldfnameO = ldfnames[lind]+'_O.dat'
    ldfnameA = ldfnames[lind]+'_A.dat'
    Syst.get_clusters_serial('contact',box,lcompute=ldfnameC)
    Syst.get_clusters_serial('optical',box,lcompute=ldfnameO)
    Syst.get_clusters_serial('aligned',box,lcompute=ldfnameA)
    lind += 1
end = time()
if rank == 0:
    print("Time to get clusters: ",end-start)

if rank == 0:
    run = 0
    mu2s = {'contact':np.zeros([ttotal,runs]),
            'optical':np.zeros([ttotal,runs]),
            'aligned':np.zeros([ttotal,runs])}
    start = time()
    for Syst in Systs:
        for ctype in ['contact','optical','aligned']:
            #write out cluster sizes and cluster IDs at each time step
            cidName = fbase + 'cut'+str(cs[ctype]) + str(run+1) + ctype + '-CIDs.dat'
            cszName = fbase + 'cut' + str(cs[ctype]) + str(run+1) + ctype + '-sizes.dat'
            Syst.writeCIDs(ctype,op.join(save_path,cidName))
            Syst.writeSizes(ctype,op.join(save_path,cszName))
            #compute mass-averaged cluster size versus time
            #pdb.set_trace()

        run += 1
    end = time()
    print("Time to write out clusters: ",end-start)
<|endoftext|>


<commit_before>"""
Copyright 2013 Steven Diamond, 2017 Robin Verschueren

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import numpy as np

import cvxpy.settings as s
from cvxpy.constraints import SOC
from cvxpy.reductions.solvers.conic_solvers.scs_conif import (SCS,
                                                              dims_to_solver_dict)
from cvxpy.reductions.solvers.conic_solvers.conic_solver import ConicSolver
from cvxpy.reductions.solution import Solution, failure_solution
from cvxpy.reductions.solvers import utilities
from scipy.sparse import dok_matrix

class GUROBI(SCS):
    """
    An interface for the Gurobi solver.

    * WARNING * This implementation takes an inadvisable approach by directly
    inheriting from a concrete solver (SCS). This implementation should not be
    used as a reference when writing other solver interfaces.

    TODO: simplify this file so it doesn't inherit from SCS; see gurobi_qpif.py
        for a very clean implementation of a quadratic programming interface.
    """

    # Solver capabilities.
    MIP_CAPABLE = True
    SUPPORTED_CONSTRAINTS = ConicSolver.SUPPORTED_CONSTRAINTS + [SOC]

    # Map of Gurobi status to CVXPY status.
    STATUS_MAP = {2: s.OPTIMAL,
                  3: s.INFEASIBLE,
                  4: s.SOLVER_ERROR,  # Triggers reoptimize.
                  5: s.UNBOUNDED,
                  6: s.SOLVER_ERROR,
                  7: s.SOLVER_ERROR,
                  8: s.SOLVER_ERROR,
                  # TODO could be anything.
                  # means time expired.
                  9: s.OPTIMAL_INACCURATE,
                  10: s.SOLVER_ERROR,
                  11: s.SOLVER_ERROR,
                  12: s.SOLVER_ERROR,
                  13: s.SOLVER_ERROR}

    def name(self):
        """The name of the solver.
        """
        return s.GUROBI

    def import_solver(self):
        """Imports the solver.
        """
        import gurobipy
        gurobipy  # For flake8

    def accepts(self, problem):
        """Can Gurobi solve the problem?
        """
        # TODO check if is matrix stuffed.
        if not problem.objective.args[0].is_affine():
            return False
        for constr in problem.constraints:
            if type(constr) not in self.SUPPORTED_CONSTRAINTS:
                return False
            for arg in constr.args:
                if not arg.is_affine():
                    return False
        return True

    def apply(self, problem):
        """Returns a new problem and data for inverting the new solution.

        Returns
        -------
        tuple
            (dict of arguments needed for the solver, inverse data)
        """
        data, inv_data = super(GUROBI, self).apply(problem)
        variables = problem.x
        data[s.BOOL_IDX] = [int(t[0]) for t in variables.boolean_idx]
        data[s.INT_IDX] = [int(t[0]) for t in variables.integer_idx]
        inv_data['is_mip'] = data[s.BOOL_IDX] or data[s.INT_IDX]

        return data, inv_data

    def invert(self, solution, inverse_data):
        """Returns the solution to the original problem given the inverse_data.
        """
        status = solution['status']
        attr = {s.EXTRA_STATS: solution['model']}

        primal_vars = None
        dual_vars = None
        if status in s.SOLUTION_PRESENT:
            opt_val = solution['value'] + inverse_data[s.OFFSET]
            primal_vars = {inverse_data[GUROBI.VAR_ID]: solution['primal']}
            if "eq_dual" in solution and not inverse_data['is_mip']:
                eq_dual = utilities.get_dual_values(
                    solution['eq_dual'],
                    utilities.extract_dual_value,
                    inverse_data[GUROBI.EQ_CONSTR])
                leq_dual = utilities.get_dual_values(
                    solution['ineq_dual'],
                    utilities.extract_dual_value,
                    inverse_data[GUROBI.NEQ_CONSTR])
                eq_dual.update(leq_dual)
                dual_vars = eq_dual
            return Solution(status, opt_val, primal_vars, dual_vars, attr)
        else:
            return failure_solution(status)

        return Solution(status, opt_val, primal_vars, dual_vars, attr)

    def solve_via_data(self, data, warm_start, verbose, solver_opts, solver_cache=None):
        """Returns the result of the call to the solver.

        Parameters
        ----------
        data : dict
            Data used by the solver.
        warm_start : bool
            Not used.
        verbose : bool
            Should the solver print output?
        solver_opts : dict
            Additional arguments for the solver.

        Returns
        -------
        tuple
            (status, optimal value, primal, equality dual, inequality dual)
        """
        import gurobipy

        c = data[s.C]
        b = data[s.B]
        A = dok_matrix(data[s.A])
        # Save the dok_matrix.
        data[s.A] = A
        dims = dims_to_solver_dict(data[s.DIMS])

        n = c.shape[0]

        model = gurobipy.Model()
        # Pass through verbosity
        model.setParam("OutputFlag", verbose)

        variables = []
        for i in range(n):
            # Set variable type.
            if i in data[s.BOOL_IDX]:
                vtype = gurobipy.GRB.BINARY
            elif i in data[s.INT_IDX]:
                vtype = gurobipy.GRB.INTEGER
            else:
                vtype = gurobipy.GRB.CONTINUOUS
            variables.append(
                model.addVar(
                    obj=c[i],
                    name="x_%d" % i,
                    vtype=vtype,
                    # Gurobi's default LB is 0 (WHY???)
                    lb=-gurobipy.GRB.INFINITY,
                    ub=gurobipy.GRB.INFINITY)
            )
        model.update()

        eq_constrs = self.add_model_lin_constr(model, variables,
                                               range(dims[s.EQ_DIM]),
                                               gurobipy.GRB.EQUAL,
                                               A, b)
        leq_start = dims[s.EQ_DIM]
        leq_end = dims[s.EQ_DIM] + dims[s.LEQ_DIM]
        ineq_constrs = self.add_model_lin_constr(model, variables,
                                                 range(leq_start, leq_end),
                                                 gurobipy.GRB.LESS_EQUAL,
                                                 A, b)
        soc_start = leq_end
        soc_constrs = []
        new_leq_constrs = []
        for constr_len in dims[s.SOC_DIM]:
            soc_end = soc_start + constr_len
            soc_constr, new_leq, new_vars = self.add_model_soc_constr(
                model, variables, range(soc_start, soc_end),
                A, b
            )
            soc_constrs.append(soc_constr)
            new_leq_constrs += new_leq
            variables += new_vars
            soc_start += constr_len

        gur_constrs = eq_constrs + ineq_constrs + new_leq_constrs + soc_constrs
        model.update()

        # Set parameters
        # TODO user option to not compute duals.
        model.setParam("QCPDual", True)
        for key, value in solver_opts.items():
            model.setParam(key, value)

        solution = {}
        try:
            model.optimize()
            # Reoptimize if INF_OR_UNBD, to get definitive answer.
            if model.Status == 4:
                model.setParam("DualReductions", 0)
                model.optimize()
            solution["value"] = model.ObjVal
            solution["primal"] = np.array([v.X for v in variables])

            # Only add duals if not a MIP.
            # Not sure why we need to negate the following,
            # but need to in order to be consistent with other solvers.
            if not (data[s.BOOL_IDX] or data[s.INT_IDX]):
                vals = []
                for lc in gur_constrs:
                    if lc is not None:
                        if isinstance(lc, gurobipy.QConstr):
                            vals.append(lc.QCPi)
                        else:
                            vals.append(lc.Pi)
                    else:
                        vals.append(0)
                solution["y"] = -np.array(vals)
                solution[s.EQ_DUAL] = solution["y"][0:dims[s.EQ_DIM]]
                solution[s.INEQ_DUAL] = solution["y"][dims[s.EQ_DIM]:]
        except Exception:
            pass
        solution[s.SOLVE_TIME] = model.Runtime
        solution["status"] = self.STATUS_MAP.get(model.Status,
                                                 s.SOLVER_ERROR)
        if solution["status"] == s.SOLVER_ERROR and model.SolCount:
            solution["status"] = s.OPTIMAL_INACCURATE
        solution["model"] = model

        return solution

    def add_model_lin_constr(self, model, variables,
                             rows, ctype,
                             mat, vec):
        """Adds EQ/LEQ constraints to the model using the data from mat and vec.

        Parameters
        ----------
        model : GUROBI model
            The problem model.
        variables : list
            The problem variables.
        rows : range
            The rows to be constrained.
        ctype : GUROBI constraint type
            The type of constraint.
        mat : SciPy COO matrix
            The matrix representing the constraints.
        vec : NDArray
            The constant part of the constraints.

        Returns
        -------
        list
            A list of constraints.
        """
        import gurobipy
        constr = []
        expr_list = {i: [] for i in rows}
        for (i, j), c in mat.items():
            v = variables[j]
            try:
                expr_list[i].append((c, v))
            except Exception:
                pass
        for i in rows:
            # Ignore empty constraints.
            if expr_list[i]:
                expr = gurobipy.LinExpr(expr_list[i])
                constr.append(
                    model.addConstr(expr, ctype, vec[i])
                )
            else:
                constr.append(None)
        return constr

    def add_model_soc_constr(self, model, variables,
                             rows, mat, vec):
        """Adds SOC constraint to the model using the data from mat and vec.

        Parameters
        ----------
        model : GUROBI model
            The problem model.
        variables : list
            The problem variables.
        rows : range
            The rows to be constrained.
        mat : SciPy COO matrix
            The matrix representing the constraints.
        vec : NDArray
            The constant part of the constraints.

        Returns
        -------
        tuple
            A tuple of (QConstr, list of Constr, and list of variables).
        """
        import gurobipy
        # Assume first expression (i.e. t) is nonzero.
        expr_list = {i: [] for i in rows}
        for (i, j), c in mat.items():
            v = variables[j]
            try:
                expr_list[i].append((c, v))
            except Exception:
                pass
        lin_expr_list = [vec[i] - gurobipy.LinExpr(expr_list[i]) for i in rows]

        # Make a variable and equality constraint for each term.
        soc_vars = [
            model.addVar(
                obj=0,
                name="soc_t_%d" % rows[0],
                vtype=gurobipy.GRB.CONTINUOUS,
                lb=0,
                ub=gurobipy.GRB.INFINITY)
        ]
        for i in rows[1:]:
            soc_vars += [
                model.addVar(
                    obj=0,
                    name="soc_x_%d" % i,
                    vtype=gurobipy.GRB.CONTINUOUS,
                    lb=-gurobipy.GRB.INFINITY,
                    ub=gurobipy.GRB.INFINITY)
            ]
        model.update()

        new_lin_constrs = []
        for i, _ in enumerate(lin_expr_list):
            new_lin_constrs += [
                model.addConstr(soc_vars[i] == lin_expr_list[i])
            ]

        t_term = soc_vars[0]*soc_vars[0]
        x_term = gurobipy.quicksum([var*var for var in soc_vars[1:]])
        return (model.addQConstr(x_term <= t_term),
                new_lin_constrs,
                soc_vars)
<commit_msg>fixed #954<commit_after>"""
Copyright 2013 Steven Diamond, 2017 Robin Verschueren

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import numpy as np

import cvxpy.settings as s
from cvxpy.constraints import SOC
from cvxpy.reductions.solvers.conic_solvers.scs_conif import (SCS,
                                                              dims_to_solver_dict)
from cvxpy.reductions.solvers.conic_solvers.conic_solver import ConicSolver
from cvxpy.reductions.solution import Solution, failure_solution
from cvxpy.reductions.solvers import utilities
from scipy.sparse import dok_matrix

class GUROBI(SCS):
    """
    An interface for the Gurobi solver.

    * WARNING * This implementation takes an inadvisable approach by directly
    inheriting from a concrete solver (SCS). This implementation should not be
    used as a reference when writing other solver interfaces.

    TODO: simplify this file so it doesn't inherit from SCS; see gurobi_qpif.py
        for a very clean implementation of a quadratic programming interface.
    """

    # Solver capabilities.
    MIP_CAPABLE = True
    SUPPORTED_CONSTRAINTS = ConicSolver.SUPPORTED_CONSTRAINTS + [SOC]

    # Map of Gurobi status to CVXPY status.
    STATUS_MAP = {2: s.OPTIMAL,
                  3: s.INFEASIBLE,
                  4: s.SOLVER_ERROR,  # Triggers reoptimize.
                  5: s.UNBOUNDED,
                  6: s.SOLVER_ERROR,
                  7: s.SOLVER_ERROR,
                  8: s.SOLVER_ERROR,
                  # TODO could be anything.
                  # means time expired.
                  9: s.OPTIMAL_INACCURATE,
                  10: s.SOLVER_ERROR,
                  11: s.SOLVER_ERROR,
                  12: s.SOLVER_ERROR,
                  13: s.SOLVER_ERROR}

    def name(self):
        """The name of the solver.
        """
        return s.GUROBI

    def import_solver(self):
        """Imports the solver.
        """
        import gurobipy
        gurobipy  # For flake8

    def accepts(self, problem):
        """Can Gurobi solve the problem?
        """
        # TODO check if is matrix stuffed.
        if not problem.objective.args[0].is_affine():
            return False
        for constr in problem.constraints:
            if type(constr) not in self.SUPPORTED_CONSTRAINTS:
                return False
            for arg in constr.args:
                if not arg.is_affine():
                    return False
        return True

    def apply(self, problem):
        """Returns a new problem and data for inverting the new solution.

        Returns
        -------
        tuple
            (dict of arguments needed for the solver, inverse data)
        """
        data, inv_data = super(GUROBI, self).apply(problem)
        variables = problem.x
        data[s.BOOL_IDX] = [int(t[0]) for t in variables.boolean_idx]
        data[s.INT_IDX] = [int(t[0]) for t in variables.integer_idx]
        inv_data['is_mip'] = data[s.BOOL_IDX] or data[s.INT_IDX]

        return data, inv_data

    def invert(self, solution, inverse_data):
        """Returns the solution to the original problem given the inverse_data.
        """
        status = solution['status']
        attr = {s.EXTRA_STATS: solution['model'],
                s.SOLVE_TIME: solution[s.SOLVE_TIME]}

        primal_vars = None
        dual_vars = None
        if status in s.SOLUTION_PRESENT:
            opt_val = solution['value'] + inverse_data[s.OFFSET]
            primal_vars = {inverse_data[GUROBI.VAR_ID]: solution['primal']}
            if "eq_dual" in solution and not inverse_data['is_mip']:
                eq_dual = utilities.get_dual_values(
                    solution['eq_dual'],
                    utilities.extract_dual_value,
                    inverse_data[GUROBI.EQ_CONSTR])
                leq_dual = utilities.get_dual_values(
                    solution['ineq_dual'],
                    utilities.extract_dual_value,
                    inverse_data[GUROBI.NEQ_CONSTR])
                eq_dual.update(leq_dual)
                dual_vars = eq_dual
            return Solution(status, opt_val, primal_vars, dual_vars, attr)
        else:
            return failure_solution(status)

        return Solution(status, opt_val, primal_vars, dual_vars, attr)

    def solve_via_data(self, data, warm_start, verbose, solver_opts, solver_cache=None):
        """Returns the result of the call to the solver.

        Parameters
        ----------
        data : dict
            Data used by the solver.
        warm_start : bool
            Not used.
        verbose : bool
            Should the solver print output?
        solver_opts : dict
            Additional arguments for the solver.

        Returns
        -------
        tuple
            (status, optimal value, primal, equality dual, inequality dual)
        """
        import gurobipy

        c = data[s.C]
        b = data[s.B]
        A = dok_matrix(data[s.A])
        # Save the dok_matrix.
        data[s.A] = A
        dims = dims_to_solver_dict(data[s.DIMS])

        n = c.shape[0]

        model = gurobipy.Model()
        # Pass through verbosity
        model.setParam("OutputFlag", verbose)

        variables = []
        for i in range(n):
            # Set variable type.
            if i in data[s.BOOL_IDX]:
                vtype = gurobipy.GRB.BINARY
            elif i in data[s.INT_IDX]:
                vtype = gurobipy.GRB.INTEGER
            else:
                vtype = gurobipy.GRB.CONTINUOUS
            variables.append(
                model.addVar(
                    obj=c[i],
                    name="x_%d" % i,
                    vtype=vtype,
                    # Gurobi's default LB is 0 (WHY???)
                    lb=-gurobipy.GRB.INFINITY,
                    ub=gurobipy.GRB.INFINITY)
            )
        model.update()

        eq_constrs = self.add_model_lin_constr(model, variables,
                                               range(dims[s.EQ_DIM]),
                                               gurobipy.GRB.EQUAL,
                                               A, b)
        leq_start = dims[s.EQ_DIM]
        leq_end = dims[s.EQ_DIM] + dims[s.LEQ_DIM]
        ineq_constrs = self.add_model_lin_constr(model, variables,
                                                 range(leq_start, leq_end),
                                                 gurobipy.GRB.LESS_EQUAL,
                                                 A, b)
        soc_start = leq_end
        soc_constrs = []
        new_leq_constrs = []
        for constr_len in dims[s.SOC_DIM]:
            soc_end = soc_start + constr_len
            soc_constr, new_leq, new_vars = self.add_model_soc_constr(
                model, variables, range(soc_start, soc_end),
                A, b
            )
            soc_constrs.append(soc_constr)
            new_leq_constrs += new_leq
            variables += new_vars
            soc_start += constr_len

        gur_constrs = eq_constrs + ineq_constrs + new_leq_constrs + soc_constrs
        model.update()

        # Set parameters
        # TODO user option to not compute duals.
        model.setParam("QCPDual", True)
        for key, value in solver_opts.items():
            model.setParam(key, value)

        solution = {}
        try:
            model.optimize()
            # Reoptimize if INF_OR_UNBD, to get definitive answer.
            if model.Status == 4:
                model.setParam("DualReductions", 0)
                model.optimize()
            solution["value"] = model.ObjVal
            solution["primal"] = np.array([v.X for v in variables])

            # Only add duals if not a MIP.
            # Not sure why we need to negate the following,
            # but need to in order to be consistent with other solvers.
            if not (data[s.BOOL_IDX] or data[s.INT_IDX]):
                vals = []
                for lc in gur_constrs:
                    if lc is not None:
                        if isinstance(lc, gurobipy.QConstr):
                            vals.append(lc.QCPi)
                        else:
                            vals.append(lc.Pi)
                    else:
                        vals.append(0)
                solution["y"] = -np.array(vals)
                solution[s.EQ_DUAL] = solution["y"][0:dims[s.EQ_DIM]]
                solution[s.INEQ_DUAL] = solution["y"][dims[s.EQ_DIM]:]
        except Exception:
            pass
        solution[s.SOLVE_TIME] = model.Runtime
        solution["status"] = self.STATUS_MAP.get(model.Status,
                                                 s.SOLVER_ERROR)
        if solution["status"] == s.SOLVER_ERROR and model.SolCount:
            solution["status"] = s.OPTIMAL_INACCURATE
        solution["model"] = model

        return solution

    def add_model_lin_constr(self, model, variables,
                             rows, ctype,
                             mat, vec):
        """Adds EQ/LEQ constraints to the model using the data from mat and vec.

        Parameters
        ----------
        model : GUROBI model
            The problem model.
        variables : list
            The problem variables.
        rows : range
            The rows to be constrained.
        ctype : GUROBI constraint type
            The type of constraint.
        mat : SciPy COO matrix
            The matrix representing the constraints.
        vec : NDArray
            The constant part of the constraints.

        Returns
        -------
        list
            A list of constraints.
        """
        import gurobipy
        constr = []
        expr_list = {i: [] for i in rows}
        for (i, j), c in mat.items():
            v = variables[j]
            try:
                expr_list[i].append((c, v))
            except Exception:
                pass
        for i in rows:
            # Ignore empty constraints.
            if expr_list[i]:
                expr = gurobipy.LinExpr(expr_list[i])
                constr.append(
                    model.addConstr(expr, ctype, vec[i])
                )
            else:
                constr.append(None)
        return constr

    def add_model_soc_constr(self, model, variables,
                             rows, mat, vec):
        """Adds SOC constraint to the model using the data from mat and vec.

        Parameters
        ----------
        model : GUROBI model
            The problem model.
        variables : list
            The problem variables.
        rows : range
            The rows to be constrained.
        mat : SciPy COO matrix
            The matrix representing the constraints.
        vec : NDArray
            The constant part of the constraints.

        Returns
        -------
        tuple
            A tuple of (QConstr, list of Constr, and list of variables).
        """
        import gurobipy
        # Assume first expression (i.e. t) is nonzero.
        expr_list = {i: [] for i in rows}
        for (i, j), c in mat.items():
            v = variables[j]
            try:
                expr_list[i].append((c, v))
            except Exception:
                pass
        lin_expr_list = [vec[i] - gurobipy.LinExpr(expr_list[i]) for i in rows]

        # Make a variable and equality constraint for each term.
        soc_vars = [
            model.addVar(
                obj=0,
                name="soc_t_%d" % rows[0],
                vtype=gurobipy.GRB.CONTINUOUS,
                lb=0,
                ub=gurobipy.GRB.INFINITY)
        ]
        for i in rows[1:]:
            soc_vars += [
                model.addVar(
                    obj=0,
                    name="soc_x_%d" % i,
                    vtype=gurobipy.GRB.CONTINUOUS,
                    lb=-gurobipy.GRB.INFINITY,
                    ub=gurobipy.GRB.INFINITY)
            ]
        model.update()

        new_lin_constrs = []
        for i, _ in enumerate(lin_expr_list):
            new_lin_constrs += [
                model.addConstr(soc_vars[i] == lin_expr_list[i])
            ]

        t_term = soc_vars[0]*soc_vars[0]
        x_term = gurobipy.quicksum([var*var for var in soc_vars[1:]])
        return (model.addQConstr(x_term <= t_term),
                new_lin_constrs,
                soc_vars)
<|endoftext|>