<commit_before>################################################################################
#Formula de Calculo -> AC = min( int( ECTS + 24)/60+1; numero de anos do curso)#
################################################################################

import unittest

### Functions ###

def calculaCreditosNecessariosParaProximoAno(creditosFeitos, nroAnosCurso):
	creditosExtra = 0.5
	concluido = False
	proximoAnoCurricular = calculaAnoCurricularCorrente(creditosFeitos, nroAnosCurso) + 1
	if proximoAnoCurricular > nroAnosCurso:
		print "Frequentas o ultimo ano curricular do curso."
		return 0
	while concluido == False:
		tentativa = calculaAnoCurricularCorrente(creditosFeitos + creditosExtra, nroAnosCurso)
		if tentativa >= proximoAnoCurricular:
			concluido = True
		else:
			creditosExtra = creditosExtra + 0.5
	return creditosExtra

def calculaAnoCurricularCorrente(creditosFeitos, nroAnosCurso):
	anoCurricular = min((creditosFeitos + 24) // 60 + 1, nroAnosCurso)
	return anoCurricular

### Interface ###

anosCurso = input('Quantos anos tem o curso? ')
if anosCurso > 10:
	raise ValueError("O curso nao pode ter mais de 10 anos.")
creditos = input('Quantos creditos tens aprovados? ')
print ('O teu ano curricular -> ' + str(calculaAnoCurricularCorrente(creditos, anosCurso)))
print ('Faltam-te ' + str(calculaCreditosNecessariosParaProximoAno(creditos, anosCurso)) + ' creditos para passar de ano.')

### Tests ###

class TesteCalculoAnoCurricular(unittest.TestCase):
	def test1Ano(self):
		self.assertEqual(calculaAnoCurricularCorrente(10,3), 1)
	def test2Ano(self):
		self.assertEqual(calculaAnoCurricularCorrente(36,3), 2)
	def test3Ano(self):
		self.assertEqual(calculaAnoCurricularCorrente(96,3), 3)

if __name__ == '__main__':
    unittest.main()

<commit_msg>added some extra tests<commit_after>################################################################################
#Formula de Calculo -> AC = min( int( ECTS + 24)/60+1; numero de anos do curso)#
################################################################################

import unittest

### Functions ###

def calculaCreditosNecessariosParaProximoAno(creditosFeitos, nroAnosCurso):
	creditosExtra = 0.5
	concluido = False
	proximoAnoCurricular = calculaAnoCurricularCorrente(creditosFeitos, nroAnosCurso) + 1
	if proximoAnoCurricular > nroAnosCurso:
		print "Frequentas o ultimo ano curricular do curso."
		return 0
	while concluido == False:
		tentativa = calculaAnoCurricularCorrente(creditosFeitos + creditosExtra, nroAnosCurso)
		if tentativa >= proximoAnoCurricular:
			concluido = True
		else:
			creditosExtra = creditosExtra + 0.5
	return creditosExtra

def calculaAnoCurricularCorrente(creditosFeitos, nroAnosCurso):
	anoCurricular = min((creditosFeitos + 24) // 60 + 1, nroAnosCurso)
	return anoCurricular

### Interface ###

anosCurso = input('Quantos anos tem o curso? ')
if anosCurso > 10:
	raise ValueError("O curso nao pode ter mais de 10 anos.")
if anosCurso < 1:
	raise ValueError("O curso nao pode ter menos de 1 ano.")
creditos = input('Quantos creditos tens aprovados? ')
print ('O teu ano curricular -> ' + str(calculaAnoCurricularCorrente(creditos, anosCurso)))
print ('Faltam-te ' + str(calculaCreditosNecessariosParaProximoAno(creditos, anosCurso)) + ' creditos para passar de ano.')

### Tests ###

class TesteCalculoAnoCurricular(unittest.TestCase):
	def test1Ano(self):
		self.assertEqual(calculaAnoCurricularCorrente(10,3), 1)
	def test2Ano(self):
		self.assertEqual(calculaAnoCurricularCorrente(36,3), 2)
	def test3Ano(self):
		self.assertEqual(calculaAnoCurricularCorrente(96,3), 3)
	def testCreditosPara2Ano(self):
		self.assertEqual(calculaCreditosNecessariosParaProximoAno(30,3), 6.0)
	def testCreditosPara3Ano(self):
		self.assertEqual(calculaCreditosNecessariosParaProximoAno(64,3), 32.0)
	def testFinalista(self):
		self.assertEqual(calculaCreditosNecessariosParaProximoAno(96,3), 0.0)

if __name__ == '__main__':
    unittest.main()

<|endoftext|>


<commit_before>import colorsys
from ..random import random

# RGB
def rgb(c1, c2, c3):
    return ((c1 * 127.5) + 127.5, (c2 * 127.5) + 127.5, (c3 * 127.5) + 127.5)

# HSL
def hsl(c1, c2, c3):
    (r, g, b) = colorsys.hls_to_rgb((c1 * 0.5) + 0.5, (c3 * 0.5) + 0.5, (c2 * 0.5) + 0.5)
    return (r * 255.0, g * 255.0, b * 255.0)

# Generate
def generateColors():
    color1 = float(random()) / float(0xFFFFFFFFFFFFFFFF)
    return color1, (((3.0 * color1) / (70.0 * float(0xFFFFFFFFFFFFFFFF))) + (1.0 / 10.0)) % 1.0

colors = [rgb, hsl]
colorsLength = len(colors)
<commit_msg>generate constant colors<commit_after>import colorsys
from ..random import random

# RGB
def rgb(c1, c2, c3):
    return ((c1 * 127.5) + 127.5, (c2 * 127.5) + 127.5, (c3 * 127.5) + 127.5)

# HSL
def hsl(c1, c2, c3):
    (r, g, b) = colorsys.hls_to_rgb((c1 * 0.5) + 0.5, (c3 * 0.5) + 0.5, (c2 * 0.5) + 0.5)
    return (r * 255.0, g * 255.0, b * 255.0)

# Generate
def generateColors():
    color1 = float(random()) / float(0xFFFFFFFFFFFFFFFF)
    color2 = color1 + (1.0 / 7.0)

    if color2 > 1.0:
        return generateColors()

    return color1, color2

colors = [rgb, hsl]
colorsLength = len(colors)
<|endoftext|>


<commit_before># -*- coding: utf-8; -*-

import inspect
import os.path
import re
import shlex
import subprocess

from ano.commands.base import Command
from ano.environment import BoardModels
from ano.exc import Abort
import ano.filters
from ano.utils import SpaceList, list_subdirs
import jinja2
from jinja2.runtime import StrictUndefined

class Build(Command):
    """
    Build a project in the current directory and produce a ready-to-upload
    firmware file.

    The project is expected to have a `src' subdirectory where all its sources
    are located. This directory is scanned recursively to find
    *.[c|cpp|pde|ino] files. They are compiled and linked into resulting
    firmware hex-file.

    Also any external library dependencies are tracked automatically. If a
    source file includes any library found among standard Arduino libraries or
    a library placed in `lib' subdirectory of the project, the library gets
    built too.

    Build artifacts are placed in `.build' subdirectory of the project.
    """

    name = 'build'
    help_line = "Build firmware from the current directory project"

    default_make = 'make'
    default_cc = 'avr-gcc'
    default_cxx = 'avr-g++'
    default_ar = 'avr-ar'
    default_objcopy = 'avr-objcopy'
    default_memsize= 'avr-size'

    default_cppflags = '-ffunction-sections -fdata-sections -g -Os -w'
    default_cflags = ''
    default_cxxflags = '-fno-exceptions'
    default_ldflags = '-Os --gc-sections'
    default_arch = 'AVR'

    def setup_arg_parser(self, parser):
        super(Build, self).setup_arg_parser(parser)
        self.e.add_board_model_arg(parser)
        self.e.add_arduino_dist_arg(parser)

        parser.add_argument('--make', metavar='MAKE',
                            default=self.default_make,
                            help='Specifies the make tool to use. If '
                            'a full path is not given, searches in Arduino '
                            'directories before PATH. Default: "%(default)s".')

        parser.add_argument('--cc', metavar='COMPILER',
                            default=self.default_cc,
                            help='Specifies the compiler used for C files. If '
                            'a full path is not given, searches in Arduino '
                            'directories before PATH. Default: "%(default)s".')

        parser.add_argument('--cxx', metavar='COMPILER',
                            default=self.default_cxx,
                            help='Specifies the compiler used for C++ files. '
                            'If a full path is not given, searches in Arduino '
                            'directories before PATH. Default: "%(default)s".')

        parser.add_argument('--memsize', metavar='MEMSIZE',
                            default=self.default_memsize,
                            help='Specifies the tool used to determine memory '
                            ' size. Default: "%(default)s".')

        parser.add_argument('--ar', metavar='AR',
                            default=self.default_ar,
                            help='Specifies the AR tool to use. If a full path '
                            'is not given, searches in Arduino directories '
                            'before PATH. Default: "%(default)s".')

        parser.add_argument('--objcopy', metavar='OBJCOPY',
                            default=self.default_objcopy,
                            help='Specifies the OBJCOPY to use. If a full path '
                            'is not given, searches in Arduino directories '
                            'before PATH. Default: "%(default)s".')

        parser.add_argument('-f', '--cppflags', metavar='FLAGS',
                            default=self.default_cppflags,
                            help='Flags that will be passed to the compiler. '
                            'Note that multiple (space-separated) flags must '
                            'be surrounded by quotes, e.g. '
                            '`--cppflags="-DC1 -DC2"\' specifies flags to define '
                            'the constants C1 and C2. Default: "%(default)s".')

        parser.add_argument('--cflags', metavar='FLAGS',
                            default=self.default_cflags,
                            help='Like --cppflags, but the flags specified are '
                            'only passed to compilations of C source files. '
                            'Default: "%(default)s".')

        parser.add_argument('--cxxflags', metavar='FLAGS',
                            default=self.default_cxxflags,
                            help='Like --cppflags, but the flags specified '
                            'are only passed to compilations of C++ source '
                            'files. Default: "%(default)s".')

        parser.add_argument('--ldflags', metavar='FLAGS',
                            default=self.default_ldflags,
                            help='Like --cppflags, but the flags specified '
                            'are only passed during the linking stage. Note '
                            'these flags should be specified as if `ld\' were '
                            'being invoked directly (i.e. the `-Wl,\' prefix '
                            'should be omitted). Default: "%(default)s".')

        parser.add_argument('--arch', metavar='ARCH',
                            default=self.default_arch,
                            help='Specifies the architecture to be passed '
                            'during compilation as  -DARDUINO_ARCH_<ARCH>. '
                            'Default: "%(default)s".')

        parser.add_argument('-v', '--verbose', default=False, action='store_true',
                            help='Verbose make output')

    def discover(self, args):
        board = self.e.board_model(args.board_model)

        core_place = os.path.join(board['_coredir'], 'cores', board['build']['core'])
        core_header = 'Arduino.h' if self.e.arduino_lib_version.major else 'WProgram.h'
        self.e.find_dir('arduino_core_dir', [core_header], [core_place],
                        human_name='Arduino core library')

        if self.e.arduino_lib_version.major:
            variants_place = os.path.join(board['_coredir'], 'variants')
            self.e.find_dir('arduino_variants_dir', ['.'], [variants_place],
                            human_name='Arduino variants directory')

        self.e.find_arduino_dir('arduino_core_libraries_dir', [os.path.join(board['_coredir'], 'libraries')],
                                human_name='Arduino core libraries', optional=True)

        self.e.find_arduino_dir('arduino_libraries_dir', ['libraries'],
                                human_name='Arduino standard libraries', optional=True)

        self.e.find_arduino_user_dir('arduino_user_libraries_dir', ['libraries'],
                                human_name='Arduino user libraries', optional=True)

        toolset = [
            ('make', args.make),
            ('cc', args.cc),
            ('cxx', args.cxx),
            ('ar', args.ar),
            ('objcopy', args.objcopy),
            ('memsize', args.memsize )
        ]

        for tool_key, tool_binary in toolset:
            self.e.find_arduino_tool(
                tool_key, ['hardware', 'tools', 'avr', 'bin'],
                items=[tool_binary], human_name=tool_binary)

    def setup_flags(self, args):
        board = self.e.board_model(args.board_model)
        boardVariant = args.cpu if ('cpu' in args) else None;

        mcu = '-mmcu=' + BoardModels.getValueForVariant(board, boardVariant, 'build', 'mcu')
        # Hard-code the flags that are essential to building the sketch
        self.e['cppflags'] = SpaceList([
            mcu,
            '-DF_CPU=' + BoardModels.getValueForVariant(board, boardVariant, 'build', 'f_cpu'),
            '-DARDUINO=' + str(self.e.arduino_lib_version.as_int()),
            '-DARDUINO_ARCH_' + args.arch.upper(),
            '-I' + self.e['arduino_core_dir'],
        ])
        # Add additional flags as specified
        self.e['cppflags'] += SpaceList(shlex.split(args.cppflags))

        try:
            self.e['cppflags'].append('-DUSB_VID=%s' % BoardModels.getValueForVariant(board, boardVariant, 'build', 'vid'))
        except KeyError:
            None
        try:
            self.e['cppflags'].append('-DUSB_PID=%s' % BoardModels.getValueForVariant(board, boardVariant, 'build', 'pid'))
        except KeyError:
            None

        if self.e.arduino_lib_version.major:
            variant_dir = os.path.join(self.e.arduino_variants_dir,
                                       board['build']['variant'])
            self.e.cppflags.append('-I' + variant_dir)

        self.e['cflags'] = SpaceList(shlex.split(args.cflags))
        self.e['cxxflags'] = SpaceList(shlex.split(args.cxxflags))

        # Again, hard-code the flags that are essential to building the sketch
        self.e['ldflags'] = SpaceList([mcu])
        self.e['ldflags'] += SpaceList([
            '-Wl,' + flag for flag in shlex.split(args.ldflags)
        ])

        self.e['names'] = {
            'obj': '%s.o',
            'lib': 'lib%s.a',
            'cpp': '%s.cpp',
            'deps': '%s.d',
        }

    def create_jinja(self, verbose):
        templates_dir = os.path.join(os.path.dirname(__file__), '..', 'make')
        self.jenv = jinja2.Environment(
            loader=jinja2.FileSystemLoader(templates_dir),
            undefined=StrictUndefined, # bark on Undefined render
            extensions=['jinja2.ext.do'])

        # inject @filters from ano.filters
        for name, f in inspect.getmembers(ano.filters, lambda x: getattr(x, 'filter', False)):
            self.jenv.filters[name] = f

        # inject globals
        self.jenv.globals['e'] = self.e
        self.jenv.globals['v'] = '' if verbose else '@'
        self.jenv.globals['slash'] = os.path.sep
        self.jenv.globals['SpaceList'] = SpaceList

    def render_template(self, source, target, **ctx):
        template = self.jenv.get_template(source)
        contents = template.render(**ctx)
        out_path = os.path.join(self.e.build_dir, target)
        with open(out_path, 'wt') as f:
            f.write(contents)

        return out_path

    def make(self, makefile, **kwargs):
        makefile = self.render_template(makefile + '.jinja', makefile, **kwargs)
        ret = subprocess.call([self.e.make, '-f', makefile, 'all'])
        if ret != 0:
            raise Abort("Make failed with code %s" % ret)

    def recursive_inc_lib_flags(self, libdirs):
        flags = SpaceList()
        for d in libdirs:
            flags.append('-I' + d)
            flags.extend('-I' + subd for subd in list_subdirs(d, recursive=True, exclude=['examples', 'extras']))
        return flags

    def _scan_dependencies(self, dirName, lib_dirs, inc_flags):
        output_filepath = os.path.join(self.e.build_dir, os.path.basename(dirName), 'dependencies.d')
        self.make('Makefile.deps', inc_flags=inc_flags, src_dir=dirName, output_filepath=output_filepath)
        self.e['deps'].append(output_filepath)

        # search for dependencies on libraries
        # for this scan dependency file generated by make
        # with regexes to find entries that start with
        # libraries dirname
        regexes = dict((lib, re.compile(r'\s' + lib + re.escape(os.path.sep))) for lib in lib_dirs)
        used_libs = set()
        with open(output_filepath) as f:
            for line in f:
                for lib, regex in regexes.iteritems():
                    if regex.search(line) and lib != dirName:
                        used_libs.add(lib)

        return used_libs

    def check_memory(self, args):
        board = self.e.board_model(args.board_model)
        boardVariant = args.cpu if ('cpu' in args) else None;
        flash_max = sram_max = 0

        try:
            flash_max = int(BoardModels.getValueForVariant(board, boardVariant,
                'upload', 'maximum_size'))
        except KeyError:
            pass

        try:
            sram_max = int(BoardModels.getValueForVariant(board, boardVariant,
                'upload', 'maximum_data_size'))
        except KeyError:
            pass

        firmware = os.path.join(self.e.build_dir, "firmware.elf")
        output = subprocess.Popen( [self.e.memsize, "--format=sysv", firmware],
            stdout=subprocess.PIPE).communicate()[0]
        text_size = int(re.search('\.text\s+(\d+)', output).group(1))
        data_size = int(re.search('\.data\s+(\d+)', output).group(1))
        bss_size = int(re.search('\.bss\s+(\d+)', output).group(1))

        flash_size = text_size + data_size
        sram_size = data_size + bss_size
        flash_pct = flash_size * 100 / flash_max if flash_max > 0 else 0
        sram_pct = sram_size * 100 / sram_max if sram_max > 0 else 0

        if flash_max > 0:
            print "Sketch uses {:,d} bytes ({:d}%) of " \
                    "program storage space.\nMaxiumum is {:,d} bytes.".format(
                            flash_size, flash_pct, flash_max )
        else:
            print "Sketch uses {:,d} bytes of program storage space.\n" \
                    "Maxiumum is unknown.".format( flash_size )

        if sram_max > 0:
            print "Global variables use {:,d} bytes ({:d}%) of dynamic " \
                    "memory,\nleaving {:,d} bytes for local varialbes. " \
                    "Maxiumum is {:,d} bytes.".format( sram_size, sram_pct,
                            sram_max - sram_size, sram_max )
        else:
            print "Global variables use {:,d} bytes of dynamic memory.\n" \
                    "Maxiumum is unknown.".format( sram_size )

        if flash_pct > 99:
            print "\033[91mSketch too big; see " \
            "http://www.arduino.cc/en/Guide/Troubleshooting#size\nfor tips " \
            "on reducing it.\033[0m"
        if sram_pct >= 75:
            print "\033[91mLow memory available, stability problems may " \
                "occur.\033[0m"

    def scan_dependencies(self):
        self.e['deps'] = SpaceList()

        lib_dirs = [self.e.arduino_core_dir] + \
            list_subdirs(self.e.lib_dir) + \
            list_subdirs(self.e.arduino_libraries_dir) + \
            list_subdirs(self.e.arduino_core_libraries_dir) + \
            list_subdirs(self.e.arduino_user_libraries_dir)
        inc_flags = self.recursive_inc_lib_flags(lib_dirs)

        # If lib A depends on lib B it have to appear before B in final
        # list so that linker could link all together correctly
        # but order of `_scan_dependencies` is not defined, so...

        # 1. Get dependencies of sources in arbitrary order
        used_libs = list(self._scan_dependencies(self.e.src_dir, lib_dirs, inc_flags))

        # 2. Get dependencies of dependency libs themselves: existing dependencies
        # are moved to the end of list maintaining order, new dependencies are appended
        scanned_libs = set()
        while scanned_libs != set(used_libs):
            for lib in set(used_libs) - scanned_libs:
                dep_libs = self._scan_dependencies(lib, lib_dirs, inc_flags)

                i = 0
                for ulib in used_libs[:]:
                    if ulib in dep_libs:
                        # dependency lib used already, move it to the tail
                        used_libs.append(used_libs.pop(i))
                        dep_libs.remove(ulib)
                    else:
                        i += 1

                # append new dependencies to the tail
                used_libs.extend(dep_libs)
                scanned_libs.add(lib)

        self.e['used_libs'] = used_libs
        self.e['cppflags'].extend(self.recursive_inc_lib_flags(used_libs))

    def run(self, args):
        self.discover(args)
        self.setup_flags(args)
        self.create_jinja(verbose=args.verbose)
        self.make('Makefile.sketch')
        self.scan_dependencies()
        self.make('Makefile')
        self.check_memory(args)
<commit_msg>Fixed typo in build output<commit_after># -*- coding: utf-8; -*-

import inspect
import os.path
import re
import shlex
import subprocess

from ano.commands.base import Command
from ano.environment import BoardModels
from ano.exc import Abort
import ano.filters
from ano.utils import SpaceList, list_subdirs
import jinja2
from jinja2.runtime import StrictUndefined

class Build(Command):
    """
    Build a project in the current directory and produce a ready-to-upload
    firmware file.

    The project is expected to have a `src' subdirectory where all its sources
    are located. This directory is scanned recursively to find
    *.[c|cpp|pde|ino] files. They are compiled and linked into resulting
    firmware hex-file.

    Also any external library dependencies are tracked automatically. If a
    source file includes any library found among standard Arduino libraries or
    a library placed in `lib' subdirectory of the project, the library gets
    built too.

    Build artifacts are placed in `.build' subdirectory of the project.
    """

    name = 'build'
    help_line = "Build firmware from the current directory project"

    default_make = 'make'
    default_cc = 'avr-gcc'
    default_cxx = 'avr-g++'
    default_ar = 'avr-ar'
    default_objcopy = 'avr-objcopy'
    default_memsize= 'avr-size'

    default_cppflags = '-ffunction-sections -fdata-sections -g -Os -w'
    default_cflags = ''
    default_cxxflags = '-fno-exceptions'
    default_ldflags = '-Os --gc-sections'
    default_arch = 'AVR'

    def setup_arg_parser(self, parser):
        super(Build, self).setup_arg_parser(parser)
        self.e.add_board_model_arg(parser)
        self.e.add_arduino_dist_arg(parser)

        parser.add_argument('--make', metavar='MAKE',
                            default=self.default_make,
                            help='Specifies the make tool to use. If '
                            'a full path is not given, searches in Arduino '
                            'directories before PATH. Default: "%(default)s".')

        parser.add_argument('--cc', metavar='COMPILER',
                            default=self.default_cc,
                            help='Specifies the compiler used for C files. If '
                            'a full path is not given, searches in Arduino '
                            'directories before PATH. Default: "%(default)s".')

        parser.add_argument('--cxx', metavar='COMPILER',
                            default=self.default_cxx,
                            help='Specifies the compiler used for C++ files. '
                            'If a full path is not given, searches in Arduino '
                            'directories before PATH. Default: "%(default)s".')

        parser.add_argument('--memsize', metavar='MEMSIZE',
                            default=self.default_memsize,
                            help='Specifies the tool used to determine memory '
                            ' size. Default: "%(default)s".')

        parser.add_argument('--ar', metavar='AR',
                            default=self.default_ar,
                            help='Specifies the AR tool to use. If a full path '
                            'is not given, searches in Arduino directories '
                            'before PATH. Default: "%(default)s".')

        parser.add_argument('--objcopy', metavar='OBJCOPY',
                            default=self.default_objcopy,
                            help='Specifies the OBJCOPY to use. If a full path '
                            'is not given, searches in Arduino directories '
                            'before PATH. Default: "%(default)s".')

        parser.add_argument('-f', '--cppflags', metavar='FLAGS',
                            default=self.default_cppflags,
                            help='Flags that will be passed to the compiler. '
                            'Note that multiple (space-separated) flags must '
                            'be surrounded by quotes, e.g. '
                            '`--cppflags="-DC1 -DC2"\' specifies flags to define '
                            'the constants C1 and C2. Default: "%(default)s".')

        parser.add_argument('--cflags', metavar='FLAGS',
                            default=self.default_cflags,
                            help='Like --cppflags, but the flags specified are '
                            'only passed to compilations of C source files. '
                            'Default: "%(default)s".')

        parser.add_argument('--cxxflags', metavar='FLAGS',
                            default=self.default_cxxflags,
                            help='Like --cppflags, but the flags specified '
                            'are only passed to compilations of C++ source '
                            'files. Default: "%(default)s".')

        parser.add_argument('--ldflags', metavar='FLAGS',
                            default=self.default_ldflags,
                            help='Like --cppflags, but the flags specified '
                            'are only passed during the linking stage. Note '
                            'these flags should be specified as if `ld\' were '
                            'being invoked directly (i.e. the `-Wl,\' prefix '
                            'should be omitted). Default: "%(default)s".')

        parser.add_argument('--arch', metavar='ARCH',
                            default=self.default_arch,
                            help='Specifies the architecture to be passed '
                            'during compilation as  -DARDUINO_ARCH_<ARCH>. '
                            'Default: "%(default)s".')

        parser.add_argument('-v', '--verbose', default=False, action='store_true',
                            help='Verbose make output')

    def discover(self, args):
        board = self.e.board_model(args.board_model)

        core_place = os.path.join(board['_coredir'], 'cores', board['build']['core'])
        core_header = 'Arduino.h' if self.e.arduino_lib_version.major else 'WProgram.h'
        self.e.find_dir('arduino_core_dir', [core_header], [core_place],
                        human_name='Arduino core library')

        if self.e.arduino_lib_version.major:
            variants_place = os.path.join(board['_coredir'], 'variants')
            self.e.find_dir('arduino_variants_dir', ['.'], [variants_place],
                            human_name='Arduino variants directory')

        self.e.find_arduino_dir('arduino_core_libraries_dir', [os.path.join(board['_coredir'], 'libraries')],
                                human_name='Arduino core libraries', optional=True)

        self.e.find_arduino_dir('arduino_libraries_dir', ['libraries'],
                                human_name='Arduino standard libraries', optional=True)

        self.e.find_arduino_user_dir('arduino_user_libraries_dir', ['libraries'],
                                human_name='Arduino user libraries', optional=True)

        toolset = [
            ('make', args.make),
            ('cc', args.cc),
            ('cxx', args.cxx),
            ('ar', args.ar),
            ('objcopy', args.objcopy),
            ('memsize', args.memsize )
        ]

        for tool_key, tool_binary in toolset:
            self.e.find_arduino_tool(
                tool_key, ['hardware', 'tools', 'avr', 'bin'],
                items=[tool_binary], human_name=tool_binary)

    def setup_flags(self, args):
        board = self.e.board_model(args.board_model)
        boardVariant = args.cpu if ('cpu' in args) else None;

        mcu = '-mmcu=' + BoardModels.getValueForVariant(board, boardVariant, 'build', 'mcu')
        # Hard-code the flags that are essential to building the sketch
        self.e['cppflags'] = SpaceList([
            mcu,
            '-DF_CPU=' + BoardModels.getValueForVariant(board, boardVariant, 'build', 'f_cpu'),
            '-DARDUINO=' + str(self.e.arduino_lib_version.as_int()),
            '-DARDUINO_ARCH_' + args.arch.upper(),
            '-I' + self.e['arduino_core_dir'],
        ])
        # Add additional flags as specified
        self.e['cppflags'] += SpaceList(shlex.split(args.cppflags))

        try:
            self.e['cppflags'].append('-DUSB_VID=%s' % BoardModels.getValueForVariant(board, boardVariant, 'build', 'vid'))
        except KeyError:
            None
        try:
            self.e['cppflags'].append('-DUSB_PID=%s' % BoardModels.getValueForVariant(board, boardVariant, 'build', 'pid'))
        except KeyError:
            None

        if self.e.arduino_lib_version.major:
            variant_dir = os.path.join(self.e.arduino_variants_dir,
                                       board['build']['variant'])
            self.e.cppflags.append('-I' + variant_dir)

        self.e['cflags'] = SpaceList(shlex.split(args.cflags))
        self.e['cxxflags'] = SpaceList(shlex.split(args.cxxflags))

        # Again, hard-code the flags that are essential to building the sketch
        self.e['ldflags'] = SpaceList([mcu])
        self.e['ldflags'] += SpaceList([
            '-Wl,' + flag for flag in shlex.split(args.ldflags)
        ])

        self.e['names'] = {
            'obj': '%s.o',
            'lib': 'lib%s.a',
            'cpp': '%s.cpp',
            'deps': '%s.d',
        }

    def create_jinja(self, verbose):
        templates_dir = os.path.join(os.path.dirname(__file__), '..', 'make')
        self.jenv = jinja2.Environment(
            loader=jinja2.FileSystemLoader(templates_dir),
            undefined=StrictUndefined, # bark on Undefined render
            extensions=['jinja2.ext.do'])

        # inject @filters from ano.filters
        for name, f in inspect.getmembers(ano.filters, lambda x: getattr(x, 'filter', False)):
            self.jenv.filters[name] = f

        # inject globals
        self.jenv.globals['e'] = self.e
        self.jenv.globals['v'] = '' if verbose else '@'
        self.jenv.globals['slash'] = os.path.sep
        self.jenv.globals['SpaceList'] = SpaceList

    def render_template(self, source, target, **ctx):
        template = self.jenv.get_template(source)
        contents = template.render(**ctx)
        out_path = os.path.join(self.e.build_dir, target)
        with open(out_path, 'wt') as f:
            f.write(contents)

        return out_path

    def make(self, makefile, **kwargs):
        makefile = self.render_template(makefile + '.jinja', makefile, **kwargs)
        ret = subprocess.call([self.e.make, '-f', makefile, 'all'])
        if ret != 0:
            raise Abort("Make failed with code %s" % ret)

    def recursive_inc_lib_flags(self, libdirs):
        flags = SpaceList()
        for d in libdirs:
            flags.append('-I' + d)
            flags.extend('-I' + subd for subd in list_subdirs(d, recursive=True, exclude=['examples', 'extras']))
        return flags

    def _scan_dependencies(self, dirName, lib_dirs, inc_flags):
        output_filepath = os.path.join(self.e.build_dir, os.path.basename(dirName), 'dependencies.d')
        self.make('Makefile.deps', inc_flags=inc_flags, src_dir=dirName, output_filepath=output_filepath)
        self.e['deps'].append(output_filepath)

        # search for dependencies on libraries
        # for this scan dependency file generated by make
        # with regexes to find entries that start with
        # libraries dirname
        regexes = dict((lib, re.compile(r'\s' + lib + re.escape(os.path.sep))) for lib in lib_dirs)
        used_libs = set()
        with open(output_filepath) as f:
            for line in f:
                for lib, regex in regexes.iteritems():
                    if regex.search(line) and lib != dirName:
                        used_libs.add(lib)

        return used_libs

    def check_memory(self, args):
        board = self.e.board_model(args.board_model)
        boardVariant = args.cpu if ('cpu' in args) else None;
        flash_max = sram_max = 0

        try:
            flash_max = int(BoardModels.getValueForVariant(board, boardVariant,
                'upload', 'maximum_size'))
        except KeyError:
            pass

        try:
            sram_max = int(BoardModels.getValueForVariant(board, boardVariant,
                'upload', 'maximum_data_size'))
        except KeyError:
            pass

        firmware = os.path.join(self.e.build_dir, "firmware.elf")
        output = subprocess.Popen( [self.e.memsize, "--format=sysv", firmware],
            stdout=subprocess.PIPE).communicate()[0]
        text_size = int(re.search('\.text\s+(\d+)', output).group(1))
        data_size = int(re.search('\.data\s+(\d+)', output).group(1))
        bss_size = int(re.search('\.bss\s+(\d+)', output).group(1))

        flash_size = text_size + data_size
        sram_size = data_size + bss_size
        flash_pct = flash_size * 100 / flash_max if flash_max > 0 else 0
        sram_pct = sram_size * 100 / sram_max if sram_max > 0 else 0

        if flash_max > 0:
            print "Sketch uses {:,d} bytes ({:d}%) of " \
                    "program storage space.\nMaxiumum is {:,d} bytes.".format(
                            flash_size, flash_pct, flash_max )
        else:
            print "Sketch uses {:,d} bytes of program storage space.\n" \
                    "Maxiumum is unknown.".format( flash_size )

        if sram_max > 0:
            print "Global variables use {:,d} bytes ({:d}%) of dynamic " \
                    "memory,\nleaving {:,d} bytes for local variables. " \
                    "Maxiumum is {:,d} bytes.".format( sram_size, sram_pct,
                            sram_max - sram_size, sram_max )
        else:
            print "Global variables use {:,d} bytes of dynamic memory.\n" \
                    "Maxiumum is unknown.".format( sram_size )

        if flash_pct > 99:
            print "\033[91mSketch too big; see " \
            "http://www.arduino.cc/en/Guide/Troubleshooting#size\nfor tips " \
            "on reducing it.\033[0m"
        if sram_pct >= 75:
            print "\033[91mLow memory available, stability problems may " \
                "occur.\033[0m"

    def scan_dependencies(self):
        self.e['deps'] = SpaceList()

        lib_dirs = [self.e.arduino_core_dir] + \
            list_subdirs(self.e.lib_dir) + \
            list_subdirs(self.e.arduino_libraries_dir) + \
            list_subdirs(self.e.arduino_core_libraries_dir) + \
            list_subdirs(self.e.arduino_user_libraries_dir)
        inc_flags = self.recursive_inc_lib_flags(lib_dirs)

        # If lib A depends on lib B it have to appear before B in final
        # list so that linker could link all together correctly
        # but order of `_scan_dependencies` is not defined, so...

        # 1. Get dependencies of sources in arbitrary order
        used_libs = list(self._scan_dependencies(self.e.src_dir, lib_dirs, inc_flags))

        # 2. Get dependencies of dependency libs themselves: existing dependencies
        # are moved to the end of list maintaining order, new dependencies are appended
        scanned_libs = set()
        while scanned_libs != set(used_libs):
            for lib in set(used_libs) - scanned_libs:
                dep_libs = self._scan_dependencies(lib, lib_dirs, inc_flags)

                i = 0
                for ulib in used_libs[:]:
                    if ulib in dep_libs:
                        # dependency lib used already, move it to the tail
                        used_libs.append(used_libs.pop(i))
                        dep_libs.remove(ulib)
                    else:
                        i += 1

                # append new dependencies to the tail
                used_libs.extend(dep_libs)
                scanned_libs.add(lib)

        self.e['used_libs'] = used_libs
        self.e['cppflags'].extend(self.recursive_inc_lib_flags(used_libs))

    def run(self, args):
        self.discover(args)
        self.setup_flags(args)
        self.create_jinja(verbose=args.verbose)
        self.make('Makefile.sketch')
        self.scan_dependencies()
        self.make('Makefile')
        self.check_memory(args)
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License..

import abc
import collections
import glob

from anvil import colorizer
from anvil import date
from anvil import exceptions as excp
from anvil import log as logging
from anvil import packager
from anvil import phase
from anvil import pip
from anvil import shell as sh
from anvil import utils

LOG = logging.getLogger(__name__)

PhaseFunctors = collections.namedtuple('PhaseFunctors', ['start', 'run', 'end'])

class Action(object):
    __meta__ = abc.ABCMeta
    NAME = None

    def __init__(self, distro, cfg, root_dir, **kargs):
        self.distro = distro
        self.cfg = cfg
        self.keep_old = kargs.get('keep_old', False)
        self.force = kargs.get('force', False)
        self.root_dir = root_dir

    @abc.abstractmethod
    def _run(self, persona, component_order, instances):
        """Run the phases of processing for this action.

        Subclasses are expected to override this method to
        do something useful.
        """

    def _order_components(self, components):
        """Returns the components in the order they should be processed.
        """
        # Duplicate the list to avoid problems if it is updated later.
        return list(components)

    def get_component_dirs(self, component):
        component_dir = sh.joinpths(self.root_dir, component)
        trace_dir = sh.joinpths(component_dir, 'traces')
        app_dir = sh.joinpths(component_dir, 'app')
        cfg_dir = sh.joinpths(component_dir, 'config')
        return {
            'app_dir': app_dir,
            'cfg_dir': cfg_dir,
            'component_dir': component_dir,
            'root_dir': self.root_dir,
            'trace_dir': trace_dir,
        }

    def _construct_instances(self, persona):
        """
        Create component objects for each component in the persona.
        """
        components = persona.wanted_components
        desired_subsystems = persona.wanted_subsystems or {}
        component_opts = persona.component_options or {}
        instances = {}
        pip_factory = packager.PackagerFactory(self.distro, pip.Packager)
        pkg_factory = packager.PackagerFactory(self.distro, self.distro.get_default_package_manager_cls())
        for c in components:
            (cls, my_info) = self.distro.extract_component(c, self.NAME)
            LOG.debug("Constructing class %s" % (cls))
            cls_kvs = {}
            cls_kvs['runner'] = self
            cls_kvs.update(self.get_component_dirs(c))
            cls_kvs['subsystem_info'] = my_info.get('subsystems') or {}
            cls_kvs['all_instances'] = instances
            cls_kvs['name'] = c
            cls_kvs['keep_old'] = self.keep_old
            cls_kvs['desired_subsystems'] = desired_subsystems.get(c) or set()
            cls_kvs['options'] = component_opts.get(c) or {}
            cls_kvs['pip_factory'] = pip_factory
            cls_kvs['packager_factory'] = pkg_factory
            # The above is not overrideable...
            for (k, v) in my_info.items():
                if k not in cls_kvs:
                    cls_kvs[k] = v
                else:
                    LOG.warn("You can not override component constructor variable named %s.", colorizer.quote(k))
            instances[c] = cls(**cls_kvs)
        return instances

    def _verify_components(self, component_order, instances):
        LOG.info("Verifying that the components are ready to rock-n-roll.")
        for c in component_order:
            instances[c].verify()

    def _warm_components(self, component_order, instances):
        LOG.info("Warming up component configurations.")
        for c in component_order:
            instances[c].warm_configs()

    def _run_phase(self, functors, component_order, instances, phase_name):
        """
        Run a given 'functor' across all of the components, in order.
        """
        component_results = dict()
        phase_recorder = phase.PhaseRecorder(sh.joinpths(self.root_dir, "%s.%s.phases" % (self.NAME, phase_name.lower())))
        for c in component_order:
            instance = instances[c]
            if phase_recorder.has_ran(instance.component_name):
                LOG.debug("Skipping phase named %r for component %r since it already happened.", phase_name, c)
            else:
                try:
                    with phase_recorder.mark(instance.component_name):
                        if functors.start:
                            functors.start(instance)
                        result = None
                        if functors.run:
                            result = functors.run(instance)
                        if functors.end:
                            functors.end(instance, result)
                        component_results[instance] = result
                except (excp.NoTraceException) as e:
                    if self.force:
                        LOG.debug("Skipping exception: %s" % (e))
                    else:
                        raise
        return component_results

    def _delete_phase_files(self, names):
        phase_dir = self.root_dir
        for n in names:
            for fn in glob.glob("%s/%s.*.phases" % (phase_dir, n)):
                if sh.isfile(fn):
                    sh.unlink(fn)

    def run(self, persona):
        instances = self._construct_instances(persona)
        component_order = self._order_components(persona.wanted_components)
        LOG.info("Processing components for action %s.", colorizer.quote(self.NAME or "???"))
        utils.log_iterable(component_order,
                        header="Activating in the following order",
                        logger=LOG)
        self._verify_components(component_order, instances)
        self._warm_components(component_order, instances)
        self._run(persona, component_order, instances)
        return component_order
<commit_msg>Update how phases are tracked<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License..

import abc
import collections
import glob

from anvil import colorizer
from anvil import date
from anvil import exceptions as excp
from anvil import log as logging
from anvil import packager
from anvil import phase
from anvil import pip
from anvil import shell as sh
from anvil import utils

LOG = logging.getLogger(__name__)

PhaseFunctors = collections.namedtuple('PhaseFunctors', ['start', 'run', 'end'])

class Action(object):
    __meta__ = abc.ABCMeta
    NAME = None

    def __init__(self, distro, cfg, root_dir, **kargs):
        self.distro = distro
        self.cfg = cfg
        self.keep_old = kargs.get('keep_old', False)
        self.force = kargs.get('force', False)
        self.root_dir = root_dir

    @abc.abstractmethod
    def _run(self, persona, component_order, instances):
        """Run the phases of processing for this action.

        Subclasses are expected to override this method to
        do something useful.
        """

    def _order_components(self, components):
        """Returns the components in the order they should be processed.
        """
        # Duplicate the list to avoid problems if it is updated later.
        return list(components)

    def get_component_dirs(self, component):
        component_dir = sh.joinpths(self.root_dir, component)
        trace_dir = sh.joinpths(component_dir, 'traces')
        app_dir = sh.joinpths(component_dir, 'app')
        cfg_dir = sh.joinpths(component_dir, 'config')
        return {
            'app_dir': app_dir,
            'cfg_dir': cfg_dir,
            'component_dir': component_dir,
            'root_dir': self.root_dir,
            'trace_dir': trace_dir,
        }

    def _construct_instances(self, persona):
        """
        Create component objects for each component in the persona.
        """
        components = persona.wanted_components
        desired_subsystems = persona.wanted_subsystems or {}
        component_opts = persona.component_options or {}
        instances = {}
        pip_factory = packager.PackagerFactory(self.distro, pip.Packager)
        pkg_factory = packager.PackagerFactory(self.distro, self.distro.get_default_package_manager_cls())
        for c in components:
            (cls, my_info) = self.distro.extract_component(c, self.NAME)
            LOG.debug("Constructing class %s" % (cls))
            cls_kvs = {}
            cls_kvs['runner'] = self
            cls_kvs.update(self.get_component_dirs(c))
            cls_kvs['subsystem_info'] = my_info.get('subsystems') or {}
            cls_kvs['all_instances'] = instances
            cls_kvs['name'] = c
            cls_kvs['keep_old'] = self.keep_old
            cls_kvs['desired_subsystems'] = desired_subsystems.get(c) or set()
            cls_kvs['options'] = component_opts.get(c) or {}
            cls_kvs['pip_factory'] = pip_factory
            cls_kvs['packager_factory'] = pkg_factory
            # The above is not overrideable...
            for (k, v) in my_info.items():
                if k not in cls_kvs:
                    cls_kvs[k] = v
                else:
                    LOG.warn("You can not override component constructor variable named %s.", colorizer.quote(k))
            instances[c] = cls(**cls_kvs)
        return instances

    def _verify_components(self, component_order, instances):
        LOG.info("Verifying that the components are ready to rock-n-roll.")
        for c in component_order:
            instances[c].verify()

    def _warm_components(self, component_order, instances):
        LOG.info("Warming up component configurations.")
        for c in component_order:
            instances[c].warm_configs()

    def _run_phase(self, functors, component_order, instances, phase_name):
        """
        Run a given 'functor' across all of the components, in order.
        """
        component_results = dict()
        phase_recorder = phase.PhaseRecorder(sh.joinpths(self.root_dir, "%s.%s.phases" % (self.NAME, phase_name.lower())))
        for c in component_order:
            instance = instances[c]
            if phase_recorder.has_ran(instance.component_name):
                LOG.debug("Skipping phase named %r for component %r since it already happened.", phase_name, c)
            else:
                try:
                    with phase_recorder.mark(instance.component_name):
                        if functors.start:
                            functors.start(instance)
                        result = None
                        if functors.run:
                            result = functors.run(instance)
                        if functors.end:
                            functors.end(instance, result)
                        component_results[instance] = result
                except (excp.NoTraceException) as e:
                    if self.force:
                        LOG.debug("Skipping exception: %s" % (e))
                    else:
                        raise
        return component_results

    def _delete_phase_files(self, names):
        for n in names:
            phases_path = sh.joinpths(self.root_dir, '%s.*.phases' % (n))
            for fn in glob.glob(phases_path):
                if sh.isfile(fn):
                    sh.unlink(fn)

    def run(self, persona):
        instances = self._construct_instances(persona)
        component_order = self._order_components(persona.wanted_components)
        LOG.info("Processing components for action %s.", colorizer.quote(self.NAME or "???"))
        utils.log_iterable(component_order,
                        header="Activating in the following order",
                        logger=LOG)
        self._verify_components(component_order, instances)
        self._warm_components(component_order, instances)
        self._run(persona, component_order, instances)
        return component_order
<|endoftext|>


<commit_before>from lxml import html as lhtml
from pyquery import PyQuery as pq
from lxml import etree
import requests
from models import AmazonMongoTradeIn, AmazonMongoTradeIn_NJ, Amazon_Textbook_Section_NR, Amazon_NR, Price_NR, Book_NR, ProfitableBooks_NR, MetaTable_NR
import re
import tasks

def getROI(theBuy, theSell):
    theBuy = float(theBuy)
    theSell = float(theSell)
    actb = (theBuy - (theSell + 3.99)) / (theSell + 3.99)
    actb = round(actb * 100, 2)
    return actb

def isGoodProfit(obj):
    theBuy = float(obj.latest_price.buy)
    theSell = float(obj.latest_price.sell)
    actb = (theBuy - (theSell + 3.99)) / (theSell + 3.99)
    actb = round(actb * 100, 2)
    if actb >= 50.0:
        return True
    return False

def retrievePage(url, proxy=None):

    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11", }

    '''if (proxy):
        theProxy = Proxy.objects.order_by('?')[0]
        proxy = {theProxy.proxy_type:theProxy.ip_and_port}
        r = requests.get(url,proxies=proxy,headers=headers)
    else:'''
    r = requests.get(url, headers=headers)
    return r.content

def toAscii(content):
    return unicode(content).encode('ascii', 'ignore')

def createOrUpdateMetaField(keyvalue, value1):
    try:
        obj = MetaTable_NR.objects.get(metakey=keyvalue)
    except MetaTable_NR.DoesNotExist:
        obj = MetaTable_NR(metakey=keyvalue, metatype="INTEGER")
    obj.int_field = value1
    obj.save()

def updateBookCounts():
    createOrUpdateMetaField("totalIndexed", AmazonMongoTradeIn.objects.count())
    createOrUpdateMetaField("totalBooks", AmazonMongoTradeIn.objects.count())
    createOrUpdateMetaField("totalProfitable", ProfitableBooks_NR.objects.all().count())

'''def addProxy(type, proxy):
    p = Proxy(proxy_type=type, ip_and_port=proxy)
    p.save()'''

# thread = ta.models.AmazonMongo.objects.get().fields(slice__prices=-1)

def checkProfitable(a):
    # print a
    price = a.prices[-1]
    if price.buy:
        if price.buy > price.sell:
            b = ProfitableBooks_NR()
            b.amazon = a.amazon
            b.price = price
            b.timestamp = price.timestamp
            b.save()

def getProfitableBooks():
    AmazonMongoTradeIn.objects.all().delete()
    objs = AmazonMongoTradeIn.objects.values_list('id', flat=True)
    tasks.process_lots_of_items_profitable.delay(objs)

def detailAllBooks():
    objs = AmazonMongoTradeIn.objects.values_list('id', flat=True)
    # print 'Objs len is %d' % (len(objs),)
    # print 'ok done with that'
    tasks.process_lots_of_items(objs)

'''
   New Detail book:

   if div#more-buying-choice-content-div
   if children divs == 3
     parse first-child with span.price
     parse third-child with div 2 span
'''
#//div[contains(concat(' ',normalize-space(@class),' '),' result')]

def countBooksInCategory(url):
    '''Counts how may books are in each category, to intelligently scrape the correct number of pages.'''
    print 'counting book in %s' % (url,)
    content = retrievePage("http://amazon.com" + url)
    d = pq(content)

    s = d("#resultCount")
    resultsNoComma = 0

    if len(s):
        txt = s[0].text_content().strip()
        #print txt
        matches = re.search(r'of\s+([,\d]+) Results', txt)

        if matches != None:
            resultsNoComma = re.sub(",", "", matches.group(1))

    return resultsNoComma

# Edited on Apr27
def getAmazonBooksOnTradeinPage(url, page):
    '''Gets all the books on the tradein page'''
    print ('Getting books for ' + url + '&page=' + str(page))
    content = retrievePage("http://amazon.com" + url + "&page=" + str(page))
    d = pq(content)

    s = d('.list.results > div')
    for result in s:
        aa = d('.image a',result)
        title = d('span.lrg.bold',result).text()
        re_product_code = re.compile(r'/dp/(.*?)/')
        pc = re.findall(re_product_code, aa[0].get('href'))

        # print aa
        # print title
        # print pc

        am = AmazonMongoTradeIn_NJ()

        am.pckey = pc[0]
        am.title = title
        am.productcode = pc[0]
        am.profitable = 0

        am.save()

# not needed with latest
def getDepartmentContainer(containers):
    for container in containers:
# Selects [New Releases, Departments, ...]
        h2s = container.cssselect("h2")
        for s in h2s:
        #s = container.cssselect("h2")
            print s.text_content()
        #s = container.cssselect("div.narrowItemHeading")
            matches = re.search(r'Department', s.text_content())
            if matches:
                return container
    return None

# not needed with latest
def getFormatContainer(containers):
    for container in containers:
        s = container.cssselect("div.narrowItemHeading")
        matches = re.search(r'Format', s[0].text_content())
        if matches:
            return container
    return None

# Works on Apr 27
def addFacetToScan(url):
    content = retrievePage(url)

    d = pq(content)

    thecontainer = d('.expand')
    if thecontainer is None:
        return

    currentHeading = d('li strong')
    currentHeading = currentHeading.parent()
    lst = currentHeading.parent()
    children = lst.children('li')
    index = children.index(currentHeading[0])
    s = children[index:]
    s = s[1:]

    #Get all navigable directories
    # s = thecontainer[0].xpath("./li/a/span[@class='refinementLink']")
    # No more to get -- this is a leaf node so add it to scan
    if not len(s):
        # print 'Adding finally!'
        addCategoryToScan(url)

    for cat in s:
    # Get to the anchor tag
        p1 = pq(cat)
        el = p1('a')
        if el:
            tasks.task_addFacetToScan.delay("http://www.amazon.com" + el[0].get('href'))
            # print 'Adding facet %s' % ("http://www.amazon.com" + el[0].get('href'),)
            # addFacetToScan("http://www.amazon.com" + el[0].get('href'))
    # print 'Made it thru!'

# Works on Apr 27
def addCategoryToScan(url):
    content = retrievePage(url)
    html = lhtml.fromstring(content)
    d = pq(content)
    s = d('h1#breadCrumb')
    # s = html.xpath("//h1[@id='breadCrumb']")
    breadcrumb = toAscii(s[0].text_content())

    # print breadcrumb
    # containers = html.xpath(".//*[@id='bestRefinement']")
    containers = d('#bestRefinement > a')
    # thecontainer = getFormatContainer(containers)

    #currently always going here - Apr27
    if containers is None:
        ats = Amazon_Textbook_Section_NR(title=breadcrumb, url=url)
        try:
            ats.save()
        except:
            pass
        return

    # s = thecontainer.xpath(".//div[@class='refinement']")
    for cat in containers:
        el = cat
        if len(el):
            ats = Amazon_Textbook_Section_NR(title=breadcrumb + " " + el.text_content(), url=el.get('href'))
            ats.save()

def scanCategories():
    objs = Amazon_Textbook_Section_NR.objects.all()
    for obj in objs:
        # scanCategoryAndAddBooks(obj)
        tasks.task_scanCategoryAndAddBooks.delay(obj)

def scanCategoryAndAddBooks(cat):
    books = countBooksInCategory(cat.url)
    print "Counted " + str(books)
    pages = int(books) / 12 + 1
    for i in range(1, pages + 1):
        tasks.scanTradeInPage.delay(cat.url, i)
        # tasks.scanTradeInPage(cat.url,i)

def parseUsedPage(amnj):
    # if not am.latest_price:
    #     am.price = Price_NR()
    url = 'http://www.amazon.com/gp/offer-listing/%s/ref=dp_olp_used?ie=UTF8&condition=used' % (amnj.amazon.productcode)
    try:
        content = retrievePage(url)
    except:
        return
    # html = lhtml.fromstring(content)
    d = pq(content)
    matches = re.search(r'a \$?(\d*\.\d{2}) Amazon.com Gift Card', d[0].text_content())
    buyprice = None
    if matches != None:
        buyprice = matches.group(1)

    # results = html.xpath("//tbody[@class='result']")
    results = d('.olpOffer')

    for result in results:
        if re.search('Acceptable', d('.olpCondition',result)[0].text_content()):
            continue
        if re.search('nternational', d('.comments',result)[0].text_content()):
            continue

        sellprice = re.match('\$?(\d*\.\d{2})', d('.olpOfferPrice',result)[0].text_content())
        if sellprice != None and buyprice != None:
            sellprice = sellprice.group(1)
            amnj.buy = buyprice
            amnj.sell = sellprice
            # price = Price_NR(buy=buyprice, sell=sellprice)
        else:
            # price = Price_NR()
            amnj.buy = buyprice
            amnj.sell = sellprice

        if amnj.buy and amnj.sell:
            roi = getROI(amnj.buy, amnj.sell)
            if roi:
                amnj.profitable = roi
            else:
                amnj.profitable = 0
        amnj.save()
        #print result.cssselect('.condition')[0].text_content()
        break

def fetchPage(url, add):
    content = retrievePage(url)
    # f = open('/home/seocortex/dropbox/web/static/testing-' + add + '.html', 'w')
    # f.write(content)
    # f.close()
<commit_msg>Thanks, Duke.<commit_after>from lxml import html as lhtml
from pyquery import PyQuery as pq
from lxml import etree
import requests
from models import AmazonMongoTradeIn, AmazonMongoTradeIn_NJ, Amazon_Textbook_Section_NR, Amazon_NR, Price_NR, Book_NR, ProfitableBooks_NR, MetaTable_NR
import re
import tasks

def getROI(theBuy, theSell):
    theBuy = float(theBuy)
    theSell = float(theSell)
    actb = (theBuy - (theSell + 3.99)) / (theSell + 3.99)
    actb = round(actb * 100, 2)
    return actb

def isGoodProfit(obj):
    theBuy = float(obj.latest_price.buy)
    theSell = float(obj.latest_price.sell)
    actb = (theBuy - (theSell + 3.99)) / (theSell + 3.99)
    actb = round(actb * 100, 2)
    if actb >= 50.0:
        return True
    return False

def retrievePage(url, proxy=None):

    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11", }

    '''if (proxy):
        theProxy = Proxy.objects.order_by('?')[0]
        proxy = {theProxy.proxy_type:theProxy.ip_and_port}
        r = requests.get(url,proxies=proxy,headers=headers)
    else:'''
    r = requests.get(url, headers=headers)
    return r.content

def toAscii(content):
    return unicode(content).encode('ascii', 'ignore')

def createOrUpdateMetaField(keyvalue, value1):
    try:
        obj = MetaTable_NR.objects.get(metakey=keyvalue)
    except MetaTable_NR.DoesNotExist:
        obj = MetaTable_NR(metakey=keyvalue, metatype="INTEGER")
    obj.int_field = value1
    obj.save()

def updateBookCounts():
    createOrUpdateMetaField("totalIndexed", AmazonMongoTradeIn.objects.count())
    createOrUpdateMetaField("totalBooks", AmazonMongoTradeIn.objects.count())
    createOrUpdateMetaField("totalProfitable", ProfitableBooks_NR.objects.all().count())

'''def addProxy(type, proxy):
    p = Proxy(proxy_type=type, ip_and_port=proxy)
    p.save()'''

# thread = ta.models.AmazonMongo.objects.get().fields(slice__prices=-1)

def checkProfitable(a):
    # print a
    price = a.prices[-1]
    if price.buy:
        if price.buy > price.sell:
            b = ProfitableBooks_NR()
            b.amazon = a.amazon
            b.price = price
            b.timestamp = price.timestamp
            b.save()

def getProfitableBooks():
    AmazonMongoTradeIn.objects.all().delete()
    objs = AmazonMongoTradeIn.objects.values_list('id', flat=True)
    tasks.process_lots_of_items_profitable.delay(objs)

def detailAllBooks():
    objs = AmazonMongoTradeIn.objects.values_list('id', flat=True)
    # print 'Objs len is %d' % (len(objs),)
    # print 'ok done with that'
    tasks.process_lots_of_items(objs)

'''
   New Detail book:

   if div#more-buying-choice-content-div
   if children divs == 3
     parse first-child with span.price
     parse third-child with div 2 span
'''
#//div[contains(concat(' ',normalize-space(@class),' '),' result')]

def countBooksInCategory(url):
    '''Counts how may books are in each category, to intelligently scrape the correct number of pages.'''
    print 'counting book in %s' % (url,)
    content = retrievePage("http://amazon.com" + url)
    d = pq(content)

    s = d("#resultCount")
    resultsNoComma = 0

    if len(s):
        txt = s[0].text_content().strip()
        #print txt
        matches = re.search(r'of\s+([,\d]+) Results', txt)

        if matches != None:
            resultsNoComma = re.sub(",", "", matches.group(1))

    return resultsNoComma

# Edited on Apr27
def getAmazonBooksOnTradeinPage(url, page):
    '''Gets all the books on the tradein page'''
    print ('Getting books for ' + url + '&page=' + str(page))
    content = retrievePage("http://amazon.com" + url + "&page=" + str(page))
    d = pq(content)

    s = d('.list.results > div')
    for result in s:
        aa = d('.image a',result)
        title = d('span.lrg.bold',result).text()
        re_product_code = re.compile(r'/dp/(.*?)/')
        pc = re.findall(re_product_code, aa[0].get('href'))

        # print aa
        # print title
        # print pc

        am = AmazonMongoTradeIn_NJ()

        am.pckey = pc[0]
        am.title = title
        am.productcode = pc[0]
        am.profitable = 0

        am.save()

# not needed with latest
def getDepartmentContainer(containers):
    for container in containers:
# Selects [New Releases, Departments, ...]
        h2s = container.cssselect("h2")
        for s in h2s:
        #s = container.cssselect("h2")
            print s.text_content()
        #s = container.cssselect("div.narrowItemHeading")
            matches = re.search(r'Department', s.text_content())
            if matches:
                return container
    return None

# not needed with latest
def getFormatContainer(containers):
    for container in containers:
        s = container.cssselect("div.narrowItemHeading")
        matches = re.search(r'Format', s[0].text_content())
        if matches:
            return container
    return None

# Works on Apr 27
def addFacetToScan(url):
    content = retrievePage(url)

    d = pq(content)

    thecontainer = d('.expand')
    if thecontainer is None:
        return

    currentHeading = d('li strong')
    currentHeading = currentHeading.parent()
    lst = currentHeading.parent()
    children = lst.children('li')
    index = children.index(currentHeading[0])
    s = children[index:]
    s = s[1:]

    #Get all navigable directories
    # s = thecontainer[0].xpath("./li/a/span[@class='refinementLink']")
    # No more to get -- this is a leaf node so add it to scan
    if not len(s):
        # print 'Adding finally!'
        addCategoryToScan(url)

    for cat in s:
    # Get to the anchor tag
        p1 = pq(cat)
        el = p1('a')
        if el:
            tasks.task_addFacetToScan.delay("http://www.amazon.com" + el[0].get('href'))
            # print 'Adding facet %s' % ("http://www.amazon.com" + el[0].get('href'),)
            # addFacetToScan("http://www.amazon.com" + el[0].get('href'))
    # print 'Made it thru!'

# Works on Apr 27
def addCategoryToScan(url):
    content = retrievePage(url)
    html = lhtml.fromstring(content)
    d = pq(content)
    s = d('h1#breadCrumb')
    # s = html.xpath("//h1[@id='breadCrumb']")
    breadcrumb = toAscii(s[0].text_content())

    # print breadcrumb
    # containers = html.xpath(".//*[@id='bestRefinement']")
    containers = d('#bestRefinement > a')
    # thecontainer = getFormatContainer(containers)

    #currently always going here - Apr27
    if containers is None:
        ats = Amazon_Textbook_Section_NR(title=breadcrumb, url=url)
        try:
            ats.save()
        except:
            pass
        return

    # s = thecontainer.xpath(".//div[@class='refinement']")
    for cat in containers:
        el = cat
        if len(el):
            ats = Amazon_Textbook_Section_NR(title=breadcrumb + " " + el.text_content(), url=el.get('href'))
            ats.save()

def scanCategories():
    objs = Amazon_Textbook_Section_NR.objects.all()
    for obj in objs:
        # scanCategoryAndAddBooks(obj)
        tasks.task_scanCategoryAndAddBooks.delay(obj)

def scanCategoryAndAddBooks(cat):
    books = countBooksInCategory(cat.url)
    print "Counted " + str(books)
    pages = int(books) / 12 + 1
    for i in range(1, pages + 1):
        tasks.scanTradeInPage.delay(cat.url, i)
        # tasks.scanTradeInPage(cat.url,i)

def parseUsedPage(amnj):
    # if not am.latest_price:
    #     am.price = Price_NR()
    url = 'http://www.amazon.com/gp/offer-listing/%s/ref=dp_olp_used?ie=UTF8&condition=used' % (amnj.productcode)
    try:
        content = retrievePage(url)
    except:
        return
    # html = lhtml.fromstring(content)
    d = pq(content)
    matches = re.search(r'a \$?(\d*\.\d{2}) Amazon.com Gift Card', d[0].text_content())
    buyprice = None
    if matches != None:
        buyprice = matches.group(1)

    # results = html.xpath("//tbody[@class='result']")
    results = d('.olpOffer')

    for result in results:
        if re.search('Acceptable', d('.olpCondition',result)[0].text_content()):
            continue
        if re.search('nternational', d('.comments',result)[0].text_content()):
            continue

        sellprice = re.match('\$?(\d*\.\d{2})', d('.olpOfferPrice',result)[0].text_content())
        if sellprice != None and buyprice != None:
            sellprice = sellprice.group(1)
            amnj.buy = buyprice
            amnj.sell = sellprice
            # price = Price_NR(buy=buyprice, sell=sellprice)
        else:
            # price = Price_NR()
            amnj.buy = buyprice
            amnj.sell = sellprice

        if amnj.buy and amnj.sell:
            roi = getROI(amnj.buy, amnj.sell)
            if roi:
                amnj.profitable = roi
            else:
                amnj.profitable = 0
        amnj.save()
        #print result.cssselect('.condition')[0].text_content()
        break

def fetchPage(url, add):
    content = retrievePage(url)
    # f = open('/home/seocortex/dropbox/web/static/testing-' + add + '.html', 'w')
    # f.write(content)
    # f.close()
<|endoftext|>


<commit_before># All the functions here are translations of ESRI Terraformer library from JS and are used for converting
# ESRI geometry JSON to GeoJSON
# TODO look for optimization possibilities
import json
import re

WKT_GEOM_REGEX = re.compile('((?:-?\d+(?:.\d+(?:[eE][-+]?\d+)?)?) (?:-?\d+(?:.\d+(?:[eE][-+]?\d+)?)?))')
ESRI_GEOM_REGEX = re.compile('(\[)(?:-?\d+(?:.\d+)?)(, ?)(?:-?\d+(?:.\d+)?)(\])')

def to_esri_feature(wkt):
    geom_type = wkt[:wkt.find('(')]

    def _geom_repl():
        bracket_multiplier = 1
        if geom_type == 'LINESTRING':
            bracket_multiplier = 2
        return json.loads(WKT_GEOM_REGEX.sub(
            lambda m: '[{}]'.format(m.group().replace(' ', ',')),
            wkt.replace(geom_type, '')
        ).replace('(', '[' * bracket_multiplier).replace(')', ']' * bracket_multiplier))

    if geom_type == 'POINT' or geom_type == 'MULTIPOINT':
        match = WKT_GEOM_REGEX.findall(wkt)
        if len(match) != 1:
            raise ValueError('Invalid Point Geometry: {0}'.format(wkt))
        try:
            x, y = map(float, match[0].split())
        except ValueError:
            raise ValueError('Invalid Point Geometry: {0}'.format(wkt))
        return {'x': x, 'y': y}
    elif geom_type in ['LINESTRING', 'MULTILINESTRING']:
        return {'paths': _geom_repl()}
    elif geom_type == 'POLYGON' or geom_type == 'MULTIPOLYGON':
        return {'rings': _geom_repl()[0]}

def from_esri_feature(feature, geom_type):

    def _get_geom_text(geom_text):
        return ESRI_GEOM_REGEX.sub(
            lambda m: m.group().replace('[', '').replace(']', '').replace(',', ' '),
            geom_text
        ).replace('[', '(').replace(']', ')')

    srid = 'SRID={0};'.format(feature.get('spatialReference', {}).get('wkid', 3857))
    if geom_type == 'esriGeometryPoint':
        return '{srid}POINT({x} {y})'.format(srid=srid, x=feature['x'], y=feature['y'])
    elif geom_type == 'esriGeometryPolyline':
        paths_text = json.dumps(feature['paths'])
        return '{srid}MULTILINESTRING{lines}'.format(srid=srid, lines=_get_geom_text(paths_text.replace(' ', '')))
    elif geom_type == 'esriGeometryPolygon':
        rings_text = json.dumps(from_rings(feature['rings'])['coordinates'])
        return '{srid}MULTIPOLYGON{polygons}'.format(srid=srid, polygons=_get_geom_text(rings_text.replace(' ', '')))
    raise ValueError('Unsupported geometry type')

def from_rings(rings):
    """
    Converts an ESRI polygon to GeoJSON.
    :param rings: a collection of rings from an ESRI geometry JSON similar to the following:
        .. code-block:: json

            {
                geometry: {
                    rings: [
                        [[5, 2], [6, 5], [2, 9], [-2, 5], [1, 2], [2, 3], [5, 2]],
                        [[0, 0], [-5, 5], [0, 10], [5, 10], [10, 5], [5, 0], [0, 0]],
                        [[1, 7], [3, 7], [3, 4], [1, 4], [1, 7]]
                    ],
                    'spatialReference': {'wkid': 4326}
                }
            }
        see here for more info: http://help.arcgis.com/en/arcgisserver/10.0/apis/rest/geometry.html
    :return: A GeoJSON object
    """
    outer_rings = []
    holes = []

    for r in rings:
        ring = close_ring(r)
        if len(ring) < 4:
            # A ring with length of less than 4 is not a polygon and can be ignored.
            # close_ring makes sure the first and the last points are the same; that's why minimum length is 4 and not 3
            continue

        if is_clockwise(ring):
            # a clockwise ring means its an outer ring
            outer_rings.append([ring])
        else:
            # otherwise it's a hole!
            holes.append(ring)

    uncontained_holes = []
    reversed_outer_rings = reversed(outer_rings)  # TODO why esri loops over a reversed rings?

    while holes:
        hole = holes.pop()

        # loop over all outer rings and see if they contain our hole.
        contained = False
        for r in reversed_outer_rings:
            outer_ring = r[0]
            if coordinates_contain_coordinates(outer_ring, hole):
                r.append(hole)
                contained = True
                break

        # sometimes a ring may not be contained in any outer ring (https://github.com/Esri/esri-leaflet/issues/320)
        if not contained:
            uncontained_holes.append(hole)

    # check and see if uncontained holes intersect with any outer rings
    while uncontained_holes:
        hole = uncontained_holes.pop()

        intersects = False
        for r in reversed_outer_rings:
            outer_ring = r[0]
            if arrays_intersect(outer_ring, hole):
                outer_rings[r].append(hole)
                intersects = True
                break

        # hole does not intersect ANY outer ring at this point; make it an outer ring.
        if not intersects:
            hole.reverse()
            outer_rings.append([hole])

    return {
        'type': 'MultiPolygon',
        'coordinates': outer_rings
    }

def close_ring(coordinates):
    """
    Makes sure the ring is closed; if it is not, then closes the ring.
    :param coordinates: a collection of coordinates representing a polygon
    :return: a closed ring
    """
    if coordinates[0] != coordinates[-1]:
        coordinates.append(coordinates[0])

    return coordinates

def is_clockwise(ring_to_test):
    """
    A clockwise ring means it is an outer ring; counter-clockwise means the opposite.
    For determining the direction of a ring, we need to calculate the sum of the areas under each side.
    A positive sum indicates a clockwise polygon. See here for more info:
    http://stackoverflow.com/questions/1165647/how-to-determine-if-a-list-of-polygon-points-are-in-clockwise-order
    :param ring_to_test:
    :return: a boolean
    """
    total = 0
    r_length = len(ring_to_test)
    for idx, pt1 in enumerate(ring_to_test):
        # pt1 and pt1 are arrays in form of [x, y] representing the beginning and end points of sides of the ring.
        if idx + 1 != r_length:
            pt2 = ring_to_test[idx + 1]
            total += (pt2[0] - pt1[0]) * (pt2[1] + pt1[1])
    return total >= 0

def edges_intersect(a1, a2, b1, b2):
    """
    The algorithm for determining whether two lines intersect or not:
    https://www.geometrictools.com/Documentation/IntersectionLine2Circle2.pdf
    This one is the same, with visualization: http://geomalgorithms.com/a05-_intersect-1.html
    a and b are vector representation of the received edges and u is the vector from point a1 to point b1
    """

    a = {'x': a2[0] - a1[0], 'y': a2[1] - a1[1]}
    b = {'x': b2[0] - b1[0], 'y': b2[1] - b1[1]}
    u = {'x': a1[0] - b1[0], 'y': a1[1] - b1[1]}
    a_dot_b = (b['y'] * a['x']) - (b['x'] * a['y'])

    if a_dot_b == 0:
        return False

    b_dot_u = (b['x'] * u['y']) - (b['y'] * u['x'])
    a_dot_u = (a['x'] * u['y']) - (a['y'] * u['x'])

    ua = b_dot_u / a_dot_b
    ub = a_dot_u / a_dot_b

    return 0 <= ua <= 1 and 0 <= ub <= 1

def arrays_intersect(a, b):
    try:
        # test for numbers
        int(a[0][0])
        try:
            int(b[0][0])
            for i, _ in enumerate(a[:-1]):
                for j, __ in enumerate(b[:-1]):
                    if edges_intersect(a[i], a[i + 1], b[j], b[j + 1]):
                        return True
        except TypeError:
            return any(arrays_intersect(a, k) for k in b)
    except TypeError:
        return any(arrays_intersect(l, b) for l in a)
    return False

def coordinates_contain_point(coordinates, point):
    """
    This function uses `Crossing number method` (http:#geomalgorithms.com/a03-_inclusion.html) to check whether
    a polygon contains a point or not.
    For each segment with the coordinates `pt1` and `pt2`, we check to see if y coordinate of `point` is
    between y of `pt1` and `pt2`.
    If so, then we check to see if `point` is to the left of the edge; i.e. a line drawn from `point` to the right
    will intersect the edge. If the line intersects the polygon an odd number of times, it is inside.
    """

    contains = False
    j = - 1
    for i, pt1 in enumerate(coordinates):
        pt2 = coordinates[j]
        check_y = lambda: (pt1[1] <= point[1] < pt2[1]) or (pt2[1] <= point[1] < pt1[1])
        # The following checks if `point` is to the left of the current segment
        check_x = lambda: point[0] < (point[1] - pt1[1]) * (pt2[0] - pt1[0]) / (pt2[1] - pt1[1]) + pt1[0]

        if check_y() and check_x():
            contains = not contains
        j = i

    return contains

def coordinates_contain_coordinates(outer, inner):
    intersects = arrays_intersect(outer, inner)
    contains = coordinates_contain_point(outer, inner[0])
    return not intersects and contains
<commit_msg>Add conditional for POLYGON vs MULTIPOLYGON<commit_after># All the functions here are translations of ESRI Terraformer library from JS and are used for converting
# ESRI geometry JSON to GeoJSON
# TODO look for optimization possibilities
import json
import re

WKT_GEOM_REGEX = re.compile('((?:-?\d+(?:.\d+(?:[eE][-+]?\d+)?)?) (?:-?\d+(?:.\d+(?:[eE][-+]?\d+)?)?))')
ESRI_GEOM_REGEX = re.compile('(\[)(?:-?\d+(?:.\d+)?)(, ?)(?:-?\d+(?:.\d+)?)(\])')

def to_esri_feature(wkt):
    geom_type = wkt[:wkt.find('(')]

    def _geom_repl():
        bracket_multiplier = 1
        if geom_type == 'LINESTRING':
            bracket_multiplier = 2
        return json.loads(WKT_GEOM_REGEX.sub(
            lambda m: '[{}]'.format(m.group().replace(' ', ',')),
            wkt.replace(geom_type, '')
        ).replace('(', '[' * bracket_multiplier).replace(')', ']' * bracket_multiplier))

    if geom_type == 'POINT' or geom_type == 'MULTIPOINT':
        match = WKT_GEOM_REGEX.findall(wkt)
        if len(match) != 1:
            raise ValueError('Invalid Point Geometry: {0}'.format(wkt))
        try:
            x, y = map(float, match[0].split())
        except ValueError:
            raise ValueError('Invalid Point Geometry: {0}'.format(wkt))
        return {'x': x, 'y': y}
    elif geom_type in ['LINESTRING', 'MULTILINESTRING']:
        return {'paths': _geom_repl()}
    elif geom_type == 'POLYGON':
        return {'rings': _geom_repl()}
    elif geom_type == 'MULTIPOLYGON':
        return {'rings': _geom_repl()[0]}

def from_esri_feature(feature, geom_type):

    def _get_geom_text(geom_text):
        return ESRI_GEOM_REGEX.sub(
            lambda m: m.group().replace('[', '').replace(']', '').replace(',', ' '),
            geom_text
        ).replace('[', '(').replace(']', ')')

    srid = 'SRID={0};'.format(feature.get('spatialReference', {}).get('wkid', 3857))
    if geom_type == 'esriGeometryPoint':
        return '{srid}POINT({x} {y})'.format(srid=srid, x=feature['x'], y=feature['y'])
    elif geom_type == 'esriGeometryPolyline':
        paths_text = json.dumps(feature['paths'])
        return '{srid}MULTILINESTRING{lines}'.format(srid=srid, lines=_get_geom_text(paths_text.replace(' ', '')))
    elif geom_type == 'esriGeometryPolygon':
        rings_text = json.dumps(from_rings(feature['rings'])['coordinates'])
        return '{srid}MULTIPOLYGON{polygons}'.format(srid=srid, polygons=_get_geom_text(rings_text.replace(' ', '')))
    raise ValueError('Unsupported geometry type')

def from_rings(rings):
    """
    Converts an ESRI polygon to GeoJSON.
    :param rings: a collection of rings from an ESRI geometry JSON similar to the following:
        .. code-block:: json

            {
                geometry: {
                    rings: [
                        [[5, 2], [6, 5], [2, 9], [-2, 5], [1, 2], [2, 3], [5, 2]],
                        [[0, 0], [-5, 5], [0, 10], [5, 10], [10, 5], [5, 0], [0, 0]],
                        [[1, 7], [3, 7], [3, 4], [1, 4], [1, 7]]
                    ],
                    'spatialReference': {'wkid': 4326}
                }
            }
        see here for more info: http://help.arcgis.com/en/arcgisserver/10.0/apis/rest/geometry.html
    :return: A GeoJSON object
    """
    outer_rings = []
    holes = []

    for r in rings:
        ring = close_ring(r)
        if len(ring) < 4:
            # A ring with length of less than 4 is not a polygon and can be ignored.
            # close_ring makes sure the first and the last points are the same; that's why minimum length is 4 and not 3
            continue

        if is_clockwise(ring):
            # a clockwise ring means its an outer ring
            outer_rings.append([ring])
        else:
            # otherwise it's a hole!
            holes.append(ring)

    uncontained_holes = []
    reversed_outer_rings = reversed(outer_rings)  # TODO why esri loops over a reversed rings?

    while holes:
        hole = holes.pop()

        # loop over all outer rings and see if they contain our hole.
        contained = False
        for r in reversed_outer_rings:
            outer_ring = r[0]
            if coordinates_contain_coordinates(outer_ring, hole):
                r.append(hole)
                contained = True
                break

        # sometimes a ring may not be contained in any outer ring (https://github.com/Esri/esri-leaflet/issues/320)
        if not contained:
            uncontained_holes.append(hole)

    # check and see if uncontained holes intersect with any outer rings
    while uncontained_holes:
        hole = uncontained_holes.pop()

        intersects = False
        for r in reversed_outer_rings:
            outer_ring = r[0]
            if arrays_intersect(outer_ring, hole):
                outer_rings[r].append(hole)
                intersects = True
                break

        # hole does not intersect ANY outer ring at this point; make it an outer ring.
        if not intersects:
            hole.reverse()
            outer_rings.append([hole])

    return {
        'type': 'MultiPolygon',
        'coordinates': outer_rings
    }

def close_ring(coordinates):
    """
    Makes sure the ring is closed; if it is not, then closes the ring.
    :param coordinates: a collection of coordinates representing a polygon
    :return: a closed ring
    """
    if coordinates[0] != coordinates[-1]:
        coordinates.append(coordinates[0])

    return coordinates

def is_clockwise(ring_to_test):
    """
    A clockwise ring means it is an outer ring; counter-clockwise means the opposite.
    For determining the direction of a ring, we need to calculate the sum of the areas under each side.
    A positive sum indicates a clockwise polygon. See here for more info:
    http://stackoverflow.com/questions/1165647/how-to-determine-if-a-list-of-polygon-points-are-in-clockwise-order
    :param ring_to_test:
    :return: a boolean
    """
    total = 0
    r_length = len(ring_to_test)
    for idx, pt1 in enumerate(ring_to_test):
        # pt1 and pt1 are arrays in form of [x, y] representing the beginning and end points of sides of the ring.
        if idx + 1 != r_length:
            pt2 = ring_to_test[idx + 1]
            total += (pt2[0] - pt1[0]) * (pt2[1] + pt1[1])
    return total >= 0

def edges_intersect(a1, a2, b1, b2):
    """
    The algorithm for determining whether two lines intersect or not:
    https://www.geometrictools.com/Documentation/IntersectionLine2Circle2.pdf
    This one is the same, with visualization: http://geomalgorithms.com/a05-_intersect-1.html
    a and b are vector representation of the received edges and u is the vector from point a1 to point b1
    """

    a = {'x': a2[0] - a1[0], 'y': a2[1] - a1[1]}
    b = {'x': b2[0] - b1[0], 'y': b2[1] - b1[1]}
    u = {'x': a1[0] - b1[0], 'y': a1[1] - b1[1]}
    a_dot_b = (b['y'] * a['x']) - (b['x'] * a['y'])

    if a_dot_b == 0:
        return False

    b_dot_u = (b['x'] * u['y']) - (b['y'] * u['x'])
    a_dot_u = (a['x'] * u['y']) - (a['y'] * u['x'])

    ua = b_dot_u / a_dot_b
    ub = a_dot_u / a_dot_b

    return 0 <= ua <= 1 and 0 <= ub <= 1

def arrays_intersect(a, b):
    try:
        # test for numbers
        int(a[0][0])
        try:
            int(b[0][0])
            for i, _ in enumerate(a[:-1]):
                for j, __ in enumerate(b[:-1]):
                    if edges_intersect(a[i], a[i + 1], b[j], b[j + 1]):
                        return True
        except TypeError:
            return any(arrays_intersect(a, k) for k in b)
    except TypeError:
        return any(arrays_intersect(l, b) for l in a)
    return False

def coordinates_contain_point(coordinates, point):
    """
    This function uses `Crossing number method` (http:#geomalgorithms.com/a03-_inclusion.html) to check whether
    a polygon contains a point or not.
    For each segment with the coordinates `pt1` and `pt2`, we check to see if y coordinate of `point` is
    between y of `pt1` and `pt2`.
    If so, then we check to see if `point` is to the left of the edge; i.e. a line drawn from `point` to the right
    will intersect the edge. If the line intersects the polygon an odd number of times, it is inside.
    """

    contains = False
    j = - 1
    for i, pt1 in enumerate(coordinates):
        pt2 = coordinates[j]
        check_y = lambda: (pt1[1] <= point[1] < pt2[1]) or (pt2[1] <= point[1] < pt1[1])
        # The following checks if `point` is to the left of the current segment
        check_x = lambda: point[0] < (point[1] - pt1[1]) * (pt2[0] - pt1[0]) / (pt2[1] - pt1[1]) + pt1[0]

        if check_y() and check_x():
            contains = not contains
        j = i

    return contains

def coordinates_contain_coordinates(outer, inner):
    intersects = arrays_intersect(outer, inner)
    contains = coordinates_contain_point(outer, inner[0])
    return not intersects and contains
<|endoftext|>


<commit_before>import os.path
import re
from csv import reader
from collections import namedtuple, Counter
from catalog.utils import is_cyr
from translitua import translit

class Phrase():
    __slots__ = ["term", "translation", "source", "quality", "usages"]

    def __init__(self, term, translation, source, quality, usages=0):
        self.term = term
        self.translation = translation
        self.source = source
        self.quality = quality
        self.usages = usages

    def inc_usages(self):
        self.usages += 1

    def __repr__(self):
        return "<{}> ({})".format(self.translation, self.source)

class Translator():
    def __init__(self, debug=False):
        self.inner_dict = {}
        self.loose_dict = {}
        self.unseen = Counter()
        self.debug = debug

    def get_id(self, term):
        return term.replace("\xa0", " ").replace("\u200b", "").lower().strip(" ,.;")

    def get_loose_id(self, term):
        return re.sub(
            "[.,\/#!$%\^&\*;:{}=\-_`~()\s]",
            "",
            term.replace("\xa0", " ").replace("\u200b", "").lower()
        )

    def load_dict_from_csv(self, fname, source, quality, ignore_header=True):
        with open(fname) as fp:
            r = reader(fp)
            if ignore_header:
                next(r)

            for l in r:
                if len(l) >= 2:
                    translation = Phrase(
                        term=l[0],
                        translation=l[1],
                        source=source,
                        quality=quality
                    )

                    term_id = self.get_id(l[0])
                    if term_id in self.inner_dict:
                        if quality < self.inner_dict[term_id].quality:
                            continue

                    if term_id:
                        self.inner_dict[term_id] = translation

                    loose_term_id = self.get_loose_id(l[0])
                    if loose_term_id in self.loose_dict:
                        if quality < self.loose_dict[loose_term_id].quality:
                            continue

                    if loose_term_id:
                        self.loose_dict[loose_term_id] = translation

    def translate(self, phrase):
        if not is_cyr(phrase):
            return Phrase(
                term=phrase,
                translation=phrase,
                source="not_required",
                quality=10
            ), "translation_not_required"

        term_id = self.get_id(phrase)
        if term_id in self.inner_dict:
            tr = self.inner_dict[term_id]
            tr.inc_usages()
            return tr, "exact"

        loose_term_id = self.get_loose_id(phrase)

        if loose_term_id in self.loose_dict:
            tr = self.loose_dict[loose_term_id]
            tr.inc_usages()
            return tr, "loose"

        if self.debug and phrase.strip():
            self.unseen.update([phrase.strip()])

        return Phrase(
            term=phrase,
            translation=translit(phrase),
            source="translit",
            quality=1
        ), "poor"

    def find_unused_strict(self):
        return [p for p in self.inner_dict.values() if p.usages==0]

    def find_unused_loose(self):
        return [p for p in self.loose_dict.values() if p.usages==0]

TRANSLATOR_SINGLETON = Translator()

CURR_DIR = os.path.dirname(__file__)
DICT_DIR = os.path.join(CURR_DIR, "data/dictionaries/")

TRANSLATOR_SINGLETON.load_dict_from_csv(os.path.join(DICT_DIR, "decl_translations.csv"), "translator", 10)
TRANSLATOR_SINGLETON.load_dict_from_csv(os.path.join(DICT_DIR, "pep_translations.csv"), "pep", 9)
TRANSLATOR_SINGLETON.load_dict_from_csv(os.path.join(DICT_DIR, "decl_translations.csv"), "google", 3)
<commit_msg>Goddamit<commit_after>import os.path
import re
from csv import reader
from collections import namedtuple, Counter
from catalog.utils import is_cyr
from translitua import translit

class Phrase():
    __slots__ = ["term", "translation", "source", "quality", "usages"]

    def __init__(self, term, translation, source, quality, usages=0):
        self.term = term
        self.translation = translation
        self.source = source
        self.quality = quality
        self.usages = usages

    def inc_usages(self):
        self.usages += 1

    def __repr__(self):
        return "<{}> ({})".format(self.translation, self.source)

class Translator():
    def __init__(self, debug=False):
        self.inner_dict = {}
        self.loose_dict = {}
        self.unseen = Counter()
        self.debug = debug

    def get_id(self, term):
        return term.replace("\xa0", " ").replace("\u200b", "").lower().strip(" ,.;")

    def get_loose_id(self, term):
        return re.sub(
            "[.,\/#!$%\^&\*;:{}=\-_`~()\s]",
            "",
            term.replace("\xa0", " ").replace("\u200b", "").lower()
        )

    def load_dict_from_csv(self, fname, source, quality, ignore_header=True):
        with open(fname) as fp:
            r = reader(fp)
            if ignore_header:
                next(r)

            for l in r:
                if len(l) >= 2:
                    translation = Phrase(
                        term=l[0],
                        translation=l[1],
                        source=source,
                        quality=quality
                    )

                    term_id = self.get_id(l[0])
                    if term_id in self.inner_dict:
                        if quality < self.inner_dict[term_id].quality:
                            continue

                    if term_id:
                        self.inner_dict[term_id] = translation

                    loose_term_id = self.get_loose_id(l[0])
                    if loose_term_id in self.loose_dict:
                        if quality < self.loose_dict[loose_term_id].quality:
                            continue

                    if loose_term_id:
                        self.loose_dict[loose_term_id] = translation

    def translate(self, phrase):
        if not is_cyr(phrase):
            return Phrase(
                term=phrase,
                translation=phrase,
                source="not_required",
                quality=10
            ), "translation_not_required"

        term_id = self.get_id(phrase)
        if term_id in self.inner_dict:
            tr = self.inner_dict[term_id]
            tr.inc_usages()
            return tr, "exact"

        loose_term_id = self.get_loose_id(phrase)

        if loose_term_id in self.loose_dict:
            tr = self.loose_dict[loose_term_id]
            tr.inc_usages()
            return tr, "loose"

        if self.debug and phrase.strip():
            self.unseen.update([phrase.strip()])

        return Phrase(
            term=phrase,
            translation=translit(phrase),
            source="translit",
            quality=1
        ), "poor"

    def find_unused_strict(self):
        return [p for p in self.inner_dict.values() if p.usages==0]

    def find_unused_loose(self):
        return [p for p in self.loose_dict.values() if p.usages==0]

TRANSLATOR_SINGLETON = Translator()

CURR_DIR = os.path.dirname(__file__)
DICT_DIR = os.path.join(CURR_DIR, "data/dictionaries/")

TRANSLATOR_SINGLETON.load_dict_from_csv(os.path.join(DICT_DIR, "decl_translations.csv"), "translator", 10)
TRANSLATOR_SINGLETON.load_dict_from_csv(os.path.join(DICT_DIR, "pep_translations.csv"), "pep", 9)
TRANSLATOR_SINGLETON.load_dict_from_csv(os.path.join(DICT_DIR, "google_dictionary.csv"), "google", 3)
<|endoftext|>