<commit_before>import operator as op
from functools import partial
import logging

import lib.const as C
import lib.visit as v

from .. import add_artifacts
from .. import util
from ..meta.template import Template
from ..meta.clazz import Clazz
from ..meta.method import Method
from ..meta.field import Field
from ..meta.statement import Statement, to_statements
from ..meta.expression import Expression, to_expression, gen_E_gen

class Singleton(object):

  def __init__(self, smpls, sng_conf=[]):
    self._smpls = smpls
    self._sng_conf = sng_conf

    self._clss = []
    self._aux_name = C.SNG.AUX
    self._aux = None

  @property
  def aux_name(self):
    return self._aux_name

  @property
  def aux(self):
    return self._aux

  @aux.setter
  def aux(self, v):
    self._aux = v

  @v.on("node")
  def visit(self, node):
    """
    This is the generic method to initialize the dynamic dispatcher
    """

  # find candidate classes that have candidate getter
  def find_clss_involved(self, tmpl):
    clss = util.flatten_classes(tmpl.classes, "inners")
    self._clss = filter(Singleton.is_candidate_cls, clss)
    logging.debug("{} candidates: {}".format(self._aux_name, self._clss))

  @staticmethod
  def is_candidate_cls(cls):
    mtds = Singleton.get_candidate_mtds(cls)
    return cls.is_class and any(mtds)

  @staticmethod
  def is_candidate_mtd(mtd):
    return not mtd.is_init and mtd.is_static and \
        len(mtd.params) == 0 and mtd.typ == mtd.clazz.name

  @staticmethod
  def get_candidate_mtds(cls):
    mtds = cls.mtds
    if cls.is_itf and cls.subs:
      mtds = util.flatten(map(Singleton.get_candidate_mtds, cls.subs))
    return filter(Singleton.is_candidate_mtd, mtds)

  def getter(self, aux, conf):
    params = [ (C.J.i, u"cls_id") ]
    getr = Method(clazz=aux, mods=C.PBST, typ=C.J.OBJ, params=params, name=u"getInstance")
    def setter_switch_whole(c):
      ins = u'_'.join([C.SNG.INS, c])
      v = getattr(aux, '_'.join([C.SNG.SNG, c]))
      def setter_switch(ins, v, cls):
        return u"if ({v} == {cls.id}) {ins} = new {cls.name}();".format(**locals())
      init = u"\nelse ".join(map(partial(setter_switch, ins, v), self._clss))
      return u"""
        if (cls_id == {v}) {{
          if ({ins} == null) {{
            {init}
          }}
          return {ins};
        }}
      """.format(**locals())
    getr_body = u"\nelse ".join(map(setter_switch_whole, conf)+[u"return null;"])
    getr.body = to_statements(getr, getr_body)
    aux.add_mtds([getr])
    setattr(aux, "gttr", getr)

  # getter will be invoked here
  @staticmethod
  def getter_in_one(aux, conf):
    params = [ (C.J.i, u"mtd_id") ]
    one = Method(clazz=aux, mods=C.PBST, typ=C.J.OBJ, params=params, name=u"getterInOne")
    def getter_switch(c):
      aname = aux.name
      v = getattr(aux, '_'.join([C.SNG.GET, c]))
      f = getattr(aux, "gttr").name
      args = getattr(aux, '_'.join([C.SNG.SNG, c]))
      return u"if (mtd_id == {v}) return {aname}.{f}({args});".format(**locals())
    one_body = "\nelse ".join(map(getter_switch, conf)+[u"return null;"])
    one.body = to_statements(one, one_body)
    aux.add_mtds([one])

  @staticmethod
  def add_fld(cls, ty, nm):
    logging.debug("adding field {}.{} of type {}".format(cls.name, nm, ty))
    fld = Field(clazz=cls, mods=[C.mod.ST], typ=ty, name=nm)
    cls.add_flds([fld])
    return fld

  ##
  ## generate an aux type
  ##
  def gen_aux_cls(self, conf, tmpl):
    aux = Clazz(name=self._aux_name, mods=[C.mod.PB], subs=self._clss)
    self.aux = aux
    tmpl.sng_auxs.append(self.aux_name)

    rv_sngs = map(lambda c: '_'.join([C.SNG.SNG, c]), conf)
    rv_gtts = map(lambda c: '_'.join([C.SNG.GET, c]), conf)

    # set role variables
    def set_role(role):
      setattr(aux, role, '_'.join([role, aux.name]))
    map(set_role, rv_sngs)
    map(set_role, rv_gtts)

    # add fields that stand for non-deterministic role choices
    def aux_fld(init, ty, nm):
      if hasattr(aux, nm): nm = getattr(aux, nm)
      return Field(clazz=aux, mods=[C.mod.ST], typ=ty, name=nm, init=init)
    hole = to_expression(C.T.HOLE)
    aux_int = partial(aux_fld, hole, C.J.i)

    c_to_e = lambda c: to_expression(unicode(c))

    ## range check
    rg_chk = Method(clazz=aux, mods=[C.mod.ST, C.mod.HN], name=u"checkRange")
    checkers = []
    gen_range = lambda ids: gen_E_gen(map(c_to_e, ids))
    get_id = op.attrgetter("id")

    # range check for singleton classes
    cls_ids = map(get_id, self._clss)
    cls_init = gen_range(cls_ids)
    aux_int_cls = partial(aux_fld, cls_init, C.J.i)
    aux.add_flds(map(aux_int_cls, rv_sngs))

    # range check for getter
    mtds = util.flatten(map(Singleton.get_candidate_mtds, self._clss))

    mtd_ids = map(get_id, mtds)
    mtd_init = gen_range(mtd_ids)
    aux_int_mtd = partial(aux_fld, mtd_init, C.J.i)
    aux.add_flds(map(aux_int_mtd, rv_gtts))

    # other semantics checks
    # such as ownership and signature types
    def owner_range(c):
      return u"assert "+getattr(aux, '_'.join([C.SNG.SNG, c]))+" == belongsTo("+getattr(aux, '_'.join([C.SNG.GET, c]))+");"
    checkers.extend(map(owner_range, conf))

    def getter_sig(c):
      return u"assert (argNum("+getattr(aux, '_'.join([C.SNG.GET, c]))+")) == 0;"
    checkers.extend(map(getter_sig, conf))

    rg_chk.body += to_statements(rg_chk, u'\n'.join(checkers))
    aux.add_mtds([rg_chk])

    ## a singleton holder
    def add_holder(c):
      Singleton.add_fld(aux, C.J.OBJ, '_'.join([C.SNG.INS, c]))
    map(add_holder, conf)

    ## getter
    self.getter(aux, conf)
    Singleton.getter_in_one(aux, conf)

    add_artifacts([aux.name])
    return aux

  @v.when(Template)
  def visit(self, node):
    self.find_clss_involved(node)
    aux = self.gen_aux_cls(self._sng_conf, node)
    node.add_classes([aux])

  @v.when(Clazz)
  def visit(self, node): pass

  @v.when(Field)
  def visit(self, node): pass

  @v.when(Method)
  def visit(self, node):
    if node.annos: return
    if node.clazz.pkg in ["java.lang"]: return
    if node.clazz.client: return
    cname = node.clazz.name

    # getter candidate
    if Singleton.is_candidate_mtd(node):
      mname = u"getterInOne"
      args = u", ".join([unicode(node.id)])
      call = u"return {}({});".format(u".".join([self._aux_name, mname]), args)
      node.body += to_statements(node, call)
      logging.debug("{}.{} => {}.{}".format(cname, node.name, self._aux_name, mname))

  @v.when(Statement)
  def visit(self, node): return [node]

  @v.when(Expression)
  def visit(self, node): return node

<commit_msg>unique Singleton pattern instantiations for the given configurations<commit_after>import operator as op
from functools import partial
from itertools import permutations
import logging

import lib.const as C
import lib.visit as v

from .. import add_artifacts
from .. import util
from ..meta.template import Template
from ..meta.clazz import Clazz
from ..meta.method import Method
from ..meta.field import Field
from ..meta.statement import Statement, to_statements
from ..meta.expression import Expression, to_expression, gen_E_gen

class Singleton(object):

  def __init__(self, smpls, sng_conf=[]):
    self._smpls = smpls
    self._sng_conf = sng_conf

    self._clss = []
    self._aux_name = C.SNG.AUX
    self._aux = None

  @property
  def aux_name(self):
    return self._aux_name

  @property
  def aux(self):
    return self._aux

  @aux.setter
  def aux(self, v):
    self._aux = v

  @v.on("node")
  def visit(self, node):
    """
    This is the generic method to initialize the dynamic dispatcher
    """

  # find candidate classes that have candidate getter
  def find_clss_involved(self, tmpl):
    clss = util.flatten_classes(tmpl.classes, "inners")
    self._clss = filter(Singleton.is_candidate_cls, clss)
    logging.debug("{} candidates: {}".format(self._aux_name, self._clss))

  @staticmethod
  def is_candidate_cls(cls):
    mtds = Singleton.get_candidate_mtds(cls)
    return cls.is_class and any(mtds)

  @staticmethod
  def is_candidate_mtd(mtd):
    return not mtd.is_init and mtd.is_static and \
        len(mtd.params) == 0 and mtd.typ == mtd.clazz.name

  @staticmethod
  def get_candidate_mtds(cls):
    mtds = cls.mtds
    if cls.is_itf and cls.subs:
      mtds = util.flatten(map(Singleton.get_candidate_mtds, cls.subs))
    return filter(Singleton.is_candidate_mtd, mtds)

  def getter(self, aux, conf):
    params = [ (C.J.i, u"cls_id") ]
    getr = Method(clazz=aux, mods=C.PBST, typ=C.J.OBJ, params=params, name=u"getInstance")
    def setter_switch_whole(c):
      ins = u'_'.join([C.SNG.INS, c])
      v = getattr(aux, '_'.join([C.SNG.SNG, c]))
      def setter_switch(ins, v, cls):
        return u"if ({v} == {cls.id}) {ins} = new {cls.name}();".format(**locals())
      init = u"\nelse ".join(map(partial(setter_switch, ins, v), self._clss))
      return u"""
        if (cls_id == {v}) {{
          if ({ins} == null) {{
            {init}
          }}
          return {ins};
        }}
      """.format(**locals())
    getr_body = u"\nelse ".join(map(setter_switch_whole, conf)+[u"return null;"])
    getr.body = to_statements(getr, getr_body)
    aux.add_mtds([getr])
    setattr(aux, "gttr", getr)

  # getter will be invoked here
  @staticmethod
  def getter_in_one(aux, conf):
    params = [ (C.J.i, u"mtd_id") ]
    one = Method(clazz=aux, mods=C.PBST, typ=C.J.OBJ, params=params, name=u"getterInOne")
    def getter_switch(c):
      aname = aux.name
      v = getattr(aux, '_'.join([C.SNG.GET, c]))
      f = getattr(aux, "gttr").name
      args = getattr(aux, '_'.join([C.SNG.SNG, c]))
      return u"if (mtd_id == {v}) return {aname}.{f}({args});".format(**locals())
    one_body = "\nelse ".join(map(getter_switch, conf)+[u"return null;"])
    one.body = to_statements(one, one_body)
    aux.add_mtds([one])

  @staticmethod
  def add_fld(cls, ty, nm):
    logging.debug("adding field {}.{} of type {}".format(cls.name, nm, ty))
    fld = Field(clazz=cls, mods=[C.mod.ST], typ=ty, name=nm)
    cls.add_flds([fld])
    return fld

  ##
  ## generate an aux type
  ##
  def gen_aux_cls(self, conf, tmpl):
    aux = Clazz(name=self._aux_name, mods=[C.mod.PB], subs=self._clss)
    self.aux = aux
    tmpl.sng_auxs.append(self.aux_name)

    rv_sngs = map(lambda c: '_'.join([C.SNG.SNG, c]), conf)
    rv_gtts = map(lambda c: '_'.join([C.SNG.GET, c]), conf)

    # set role variables
    def set_role(role):
      setattr(aux, role, '_'.join([role, aux.name]))
    map(set_role, rv_sngs)
    map(set_role, rv_gtts)

    # add fields that stand for non-deterministic role choices
    def aux_fld(init, ty, nm):
      if hasattr(aux, nm): nm = getattr(aux, nm)
      return Field(clazz=aux, mods=[C.mod.ST], typ=ty, name=nm, init=init)
    hole = to_expression(C.T.HOLE)
    aux_int = partial(aux_fld, hole, C.J.i)

    c_to_e = lambda c: to_expression(unicode(c))

    ## range check
    rg_chk = Method(clazz=aux, mods=[C.mod.ST, C.mod.HN], name=u"checkRange")
    checkers = []
    gen_range = lambda ids: gen_E_gen(map(c_to_e, ids))
    get_id = op.attrgetter("id")

    # range check for singleton classes
    cls_ids = map(get_id, self._clss)
    cls_init = gen_range(cls_ids)
    aux_int_cls = partial(aux_fld, cls_init, C.J.i)
    aux.add_flds(map(aux_int_cls, rv_sngs))

    # range check for getter
    mtds = util.flatten(map(Singleton.get_candidate_mtds, self._clss))

    mtd_ids = map(get_id, mtds)
    mtd_init = gen_range(mtd_ids)
    aux_int_mtd = partial(aux_fld, mtd_init, C.J.i)
    aux.add_flds(map(aux_int_mtd, rv_gtts))

    # other semantics checks
    # such as ownership, signature types, and uniqueness
    def owner_range(c):
      return u"assert "+getattr(aux, '_'.join([C.SNG.SNG, c]))+" == belongsTo("+getattr(aux, '_'.join([C.SNG.GET, c]))+");"
    checkers.extend(map(owner_range, conf))

    def getter_sig(c):
      return u"assert (argNum("+getattr(aux, '_'.join([C.SNG.GET, c]))+")) == 0;"
    checkers.extend(map(getter_sig, conf))

    def uniqueness(c1, c2):
      _c1, _c2 = map(lambda c: getattr(aux, '_'.join([C.SNG.SNG, c])), [c1, c2])
      return u"assert {} != {};".format(_c1, _c2)
    checkers.extend(map(uniqueness, permutations(conf, 2)))

    rg_chk.body += to_statements(rg_chk, u'\n'.join(checkers))
    aux.add_mtds([rg_chk])

    ## a singleton holder
    def add_holder(c):
      Singleton.add_fld(aux, C.J.OBJ, '_'.join([C.SNG.INS, c]))
    map(add_holder, conf)

    ## getter
    self.getter(aux, conf)
    Singleton.getter_in_one(aux, conf)

    add_artifacts([aux.name])
    return aux

  @v.when(Template)
  def visit(self, node):
    self.find_clss_involved(node)
    aux = self.gen_aux_cls(self._sng_conf, node)
    node.add_classes([aux])

  @v.when(Clazz)
  def visit(self, node): pass

  @v.when(Field)
  def visit(self, node): pass

  @v.when(Method)
  def visit(self, node):
    if node.annos: return
    if node.clazz.pkg in ["java.lang"]: return
    if node.clazz.client: return
    cname = node.clazz.name

    # getter candidate
    if Singleton.is_candidate_mtd(node):
      mname = u"getterInOne"
      args = u", ".join([unicode(node.id)])
      call = u"return {}({});".format(u".".join([self._aux_name, mname]), args)
      node.body += to_statements(node, call)
      logging.debug("{}.{} => {}.{}".format(cname, node.name, self._aux_name, mname))

  @v.when(Statement)
  def visit(self, node): return [node]

  @v.when(Expression)
  def visit(self, node): return node

<|endoftext|>


<commit_before>"""
pi-scanner
Barcode scanner project for the Raspberry Pi.
"""

import sys, argparse
import json
import gspread
from gspread.exceptions import CellNotFound
from oauth2client.client import SignedJwtAssertionCredentials

def _check_negative(value):
    ivalue = int(value)
    if ivalue < 1:
         raise argparse.ArgumentTypeError("%s is an invalid positive int value" % value)
    return ivalue

def _filterByRow(seq, row):
	for element in seq:
		if element.row == row:
			return element
	raise CellNotFound

def _filterByCol(seq, col):
	for element in seq:
		if element.col == col:
			return element
	raise CellNotFound

def main():
	# Parse the command line arguments - https://docs.python.org/2/library/argparse.html
	parser = argparse.ArgumentParser(description='pi-scanner - Barcode scanner project for the Raspberry Pi.', prefix_chars='-')
	parser.add_argument('-i', dest='oauthFile', action='store', required=True, help='OAuth file.')
	parser.add_argument('-sn', dest='sheetName', action='store', required=True, help='The name of the excel sheet.')
	parser.add_argument('-ws', dest='worksheet', action='store', required=True, help='The name of the work sheet.')
	search_group = parser.add_mutually_exclusive_group()
	search_group.add_argument('-rf', dest='rowFilter', action='store', help='Row number to filter results by.', type=_check_negative)
	search_group.add_argument('-cf', dest='colFilter', action='store', help='Column number to filter results by.', type=_check_negative)

	args = parser.parse_args()

	oauthFile = args.oauthFile
	sheetName = args.sheetName
	worksheet = args.worksheet
	rowFilter = args.rowFilter
	colFilter = args.colFilter

	print('\n========================================')
	print('Input file is [%s].' % oauthFile)
	print('Excel sheet name is [%s].' % sheetName)
	print('Work sheet name is [%s].' % worksheet)
	if rowFilter is not None:
		print ('Filtering on row [%s].' % rowFilter)
	if colFilter is not None:
		print ('Filtering on column [%s].' % colFilter)
	print('========================================\n')

	# Login through oauth (#6) - http://gspread.readthedocs.org/en/latest/oauth2.html
	json_key = json.load(open(oauthFile))
	scope = ['https://spreadsheets.google.com/feeds']

	credentials = SignedJwtAssertionCredentials(json_key['client_email'], json_key['private_key'].encode(), scope)

	# Open the specified excel sheet and worksheet - http://gspread.readthedocs.org/en/latest/index.html
	gc = gspread.authorize(credentials)
	wks = gc.open(sheetName).worksheet(worksheet)

	print('Cell A1 is [%s].' % wks.acell('A1').value)
	print('Cell A2 is [%s].' % wks.acell('A2').value)
	print('Cell B1 is [%s].' % wks.acell('B1').value)
	print('Cell B2 is [%s].' % wks.acell('B2').value)

	while True:    # Read input forever
		barcode = raw_input("\nEnter the barcode: ")
		if barcode == "quit":
			break  # Exit the program

		print('Barcode is [%s].' % barcode)
		try:
			if rowFilter is not None:
				cell = _filterByRow(wks.findall(barcode), rowFilter)
			elif colFilter is not None:
				cell = _filterByCol(wks.findall(barcode), colFilter)
			else:
				cell = wks.find(barcode)
			print('Barcode found at row [%s] column [%s].' % (cell.row, cell.col))
		except CellNotFound:
			wks.add_rows(1)
			row_count = wks.row_count
			wks.update_cell(row_count, 1, barcode)
			cell = wks.find(barcode)
			print('Barcode added at row [%s] column [%s].' % (cell.row, cell.col))

if __name__ == "__main__":
    main()
<commit_msg>Added row/col value filtering.<commit_after>"""
pi-scanner
Barcode scanner project for the Raspberry Pi.
"""

import sys, argparse
import json
import gspread
from gspread.exceptions import CellNotFound
from oauth2client.client import SignedJwtAssertionCredentials

def _check_positive(value):
    ivalue = int(value)
    if ivalue < 1:
         raise argparse.ArgumentTypeError("[%s] must be a positive integer." % value)
    return ivalue

def _filterByRow(seq, row):
	for element in seq:
		if element.row == row:
			return element
	raise CellNotFound

def _filterByCol(seq, col):
	for element in seq:
		if element.col == col:
			return element
	raise CellNotFound

def main():
	# Parse the command line arguments - https://docs.python.org/2/library/argparse.html
	parser = argparse.ArgumentParser(description='pi-scanner - Barcode scanner project for the Raspberry Pi.', prefix_chars='-')
	parser.add_argument('-i', dest='oauthFile', action='store', required=True, help='OAuth file.')
	parser.add_argument('-sn', dest='sheetName', action='store', required=True, help='The name of the excel sheet.')
	parser.add_argument('-ws', dest='worksheet', action='store', required=True, help='The name of the work sheet.')
	search_group = parser.add_mutually_exclusive_group()
	search_group.add_argument('-sfr', dest='searchFilterRow', action='store', help='Row number for search filtering results.', type=_check_positive)
	search_group.add_argument('-sfc', dest='searchFilterCol', action='store', help='Column number for search filtering results.', type=_check_positive)
	value_group = parser.add_mutually_exclusive_group()
	value_group.add_argument('-vfr', dest='valueFilterRow', action='store', help='Row number for value filtering results.', type=_check_positive)
	value_group.add_argument('-vfc', dest='valueFilterCol', action='store', help='Column number for value filtering results.', type=_check_positive)

	args = parser.parse_args()

	oauthFile = args.oauthFile
	sheetName = args.sheetName
	worksheet = args.worksheet
	searchFilterRow = args.searchFilterRow
	searchFilterCol = args.searchFilterCol
	valueFilterRow = args.valueFilterRow
	valueFilterCol = args.valueFilterCol

	print('\n========================================')
	print('Input file is [%s].' % oauthFile)
	print('Excel sheet name is [%s].' % sheetName)
	print('Work sheet name is [%s].' % worksheet)
	if searchFilterRow is not None:
		print ('Search filtering on row [%s].' % searchFilterRow)
	if searchFilterCol is not None:
		print ('Search filtering on column [%s].' % searchFilterCol)
	if valueFilterRow is not None:
		print ('Value filtering on row [%s].' % valueFilterRow)
	if valueFilterCol is not None:
		print ('Value filtering on column [%s].' % valueFilterCol)
	print('========================================\n')

	# Login through oauth (#6) - http://gspread.readthedocs.org/en/latest/oauth2.html
	json_key = json.load(open(oauthFile))
	scope = ['https://spreadsheets.google.com/feeds']

	credentials = SignedJwtAssertionCredentials(json_key['client_email'], json_key['private_key'].encode(), scope)

	# Open the specified excel sheet and worksheet - http://gspread.readthedocs.org/en/latest/index.html
	gc = gspread.authorize(credentials)
	wks = gc.open(sheetName).worksheet(worksheet)

	while True:    # Read input forever
		barcode = raw_input("\nEnter the barcode: ")
		if barcode == "quit":
			break  # Exit the program

		print('Barcode is [%s].' % barcode)
		try:
			# Search filter by row/col/or nothing
			if searchFilterRow is not None:
				cell = _filterByRow(wks.findall(barcode), searchFilterRow)
			elif searchFilterCol is not None:
				cell = _filterByCol(wks.findall(barcode), searchFilterCol)
			else:
				cell = wks.find(barcode)
			print('Barcode found at row [%s] column [%s].' % (cell.row, cell.col))

			# Value filter by row/col/or nothing
			if valueFilterRow is not None:
				valueCell = wks.cell(valueFilterRow, cell.col)
			elif valueFilterCol is not None:
				valueCell = wks.cell(cell.row, valueFilterCol)
			else:
				valueCell = cell
			print('Value found is [%s].' % (valueCell.value))
		except CellNotFound:
			wks.add_rows(1)
			row_count = wks.row_count
			# Put the value in the value filter row
			if valueFilterCol is not None:
				col = valueFilterCol
			else:
				col = 1
			wks.update_cell(row_count, col, barcode)
			# TODO: Determine if we want to print out where the cell was added
			# cell = wks.find(barcode)
			# print('Barcode added at row [%s] column [%s].' % (cell.row, cell.col))

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""Movies package.

Top-level package of movies library. This package contains catalog of movies
module components - ``MoviesModule``. It is recommended to use movies library
functionality by fetching required instances from ``MoviesModule`` providers.

Each of ``MoviesModule`` providers could be overridden.
"""

from . import finders
from . import listers
from . import models

from dependency_injector import catalogs
from dependency_injector import providers

class MoviesModule(catalogs.DeclarativeCatalog):
    """Catalog of movies module components."""

    movie_model = providers.DelegatedFactory(models.Movie)

    movie_finder = providers.Factory(finders.MovieFinder,
                                     movie_model=movie_model)

    movie_lister = providers.Factory(listers.MovieLister,
                                     movie_finder=movie_finder)
<commit_msg>Update imports for MovieLister standrard module<commit_after>"""Movies package.

Top-level package of movies library. This package contains catalog of movies
module components - ``MoviesModule``. It is recommended to use movies library
functionality by fetching required instances from ``MoviesModule`` providers.

Each of ``MoviesModule`` providers could be overridden.
"""

from dependency_injector import catalogs
from dependency_injector import providers

from . import finders
from . import listers
from . import models

class MoviesModule(catalogs.DeclarativeCatalog):
    """Catalog of movies module components."""

    movie_model = providers.DelegatedFactory(models.Movie)

    movie_finder = providers.Factory(finders.MovieFinder,
                                     movie_model=movie_model)

    movie_lister = providers.Factory(listers.MovieLister,
                                     movie_finder=movie_finder)
<|endoftext|>


<commit_before>from __future__ import absolute_import
import datetime
import logging
import os
import sys
import socket
from socket import error as SocketError, timeout as SocketTimeout
import warnings
from .packages import six

try:  # Python 3
    from http.client import HTTPConnection as _HTTPConnection
    from http.client import HTTPException  # noqa: unused in this module
except ImportError:
    from httplib import HTTPConnection as _HTTPConnection
    from httplib import HTTPException  # noqa: unused in this module

try:  # Compiled with SSL?
    import ssl
    BaseSSLError = ssl.SSLError
except (ImportError, AttributeError):  # Platform-specific: No SSL.
    ssl = None

    class BaseSSLError(BaseException):
        pass

try:  # Python 3:
    # Not a no-op, we're adding this to the namespace so it can be imported.
    ConnectionError = ConnectionError
except NameError:  # Python 2:
    class ConnectionError(Exception):
        pass

from .exceptions import (
    NewConnectionError,
    ConnectTimeoutError,
    SubjectAltNameWarning,
    SystemTimeWarning,
)
from .packages.ssl_match_hostname import match_hostname, CertificateError

from .util.ssl_ import (
    HAS_SNI,
    resolve_cert_reqs,
    resolve_ssl_version,
    assert_fingerprint,
    create_urllib3_context,
)

from .util import connection

from ._collections import HTTPHeaderDict

log = logging.getLogger(__name__)

port_by_scheme = {
    'http': 80,
    'https': 443,
}

RECENT_DATE = datetime.date(2014, 1, 1)

class DummyConnection(object):
    """Used to detect a failed ConnectionCls import."""
    pass

class HTTPConnection(_HTTPConnection, object):
    """
    Based on httplib.HTTPConnection but provides an extra constructor
    backwards-compatibility layer between older and newer Pythons.

    Additional keyword parameters are used to configure attributes of the connection.
    Accepted parameters include:

      - ``strict``: See the documentation on :class:`urllib3.connectionpool.HTTPConnectionPool`
      - ``source_address``: Set the source address for the current connection.

        .. note:: This is ignored for Python 2.6. It is only applied for 2.7 and 3.x

      - ``socket_options``: Set specific options on the underlying socket. If not specified, then
        defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling
        Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy.

        For example, if you wish to enable TCP Keep Alive in addition to the defaults,
        you might pass::

            HTTPConnection.default_socket_options + [
                (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),
            ]

        Or you may want to disable the defaults by passing an empty list (e.g., ``[]``).
    """

    default_port = port_by_scheme['http']

    #: Disable Nagle's algorithm by default.
    #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]``
    default_socket_options = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]

    #: Whether this connection verifies the host's certificate.
    is_verified = False

    def __init__(self, *args, **kw):
        if six.PY3:  # Python 3
            kw.pop('strict', None)

        # Pre-set source_address in case we have an older Python like 2.6.
        self.source_address = kw.get('source_address')

        if sys.version_info < (2, 7):  # Python 2.6
            # _HTTPConnection on Python 2.6 will balk at this keyword arg, but
            # not newer versions. We can still use it when creating a
            # connection though, so we pop it *after* we have saved it as
            # self.source_address.
            kw.pop('source_address', None)

        #: The socket options provided by the user. If no options are
        #: provided, we use the default options.
        self.socket_options = kw.pop('socket_options', self.default_socket_options)

        # Superclass also sets self.source_address in Python 2.7+.
        _HTTPConnection.__init__(self, *args, **kw)

    def _new_conn(self):
        """ Establish a socket connection and set nodelay settings on it.

        :return: New socket connection.
        """
        extra_kw = {}
        if self.source_address:
            extra_kw['source_address'] = self.source_address

        if self.socket_options:
            extra_kw['socket_options'] = self.socket_options

        try:
            conn = connection.create_connection(
                (self.host, self.port), self.timeout, **extra_kw)

        except SocketTimeout as e:
            raise ConnectTimeoutError(
                self, "Connection to %s timed out. (connect timeout=%s)" %
                (self.host, self.timeout))

        except SocketError as e:
            raise NewConnectionError(
                self, "Failed to establish a new connection: %s" % e)

        return conn

    def _prepare_conn(self, conn):
        self.sock = conn
        # the _tunnel_host attribute was added in python 2.6.3 (via
        # http://hg.python.org/cpython/rev/0f57b30a152f) so pythons 2.6(0-2) do
        # not have them.
        if getattr(self, '_tunnel_host', None):
            # TODO: Fix tunnel so it doesn't depend on self.sock state.
            self._tunnel()
            # Mark this connection as not reusable
            self.auto_open = 0

    def connect(self):
        conn = self._new_conn()
        self._prepare_conn(conn)

    def request_chunked(self, method, url, body=None, headers=None):
        """
        Alternative to the common request method, which sends the
        body with chunked encoding and not as one block
        """
        headers = HTTPHeaderDict(headers if headers is not None else {})
        skip_accept_encoding = 'accept-encoding' in headers
        self.putrequest(method, url, skip_accept_encoding=skip_accept_encoding)
        for header, value in headers.items():
            self.putheader(header, value)
        if 'transfer-encoding' not in headers:
            self.putheader('Transfer-Encoding', 'chunked')
        self.endheaders()

        if body is not None:
            stringish_types = six.string_types + (six.binary_type,)
            if isinstance(body, stringish_types):
                body = (body,)
            for chunk in body:
                if not chunk:
                    continue
                if not isinstance(chunk, six.binary_type):
                    chunk = chunk.encode('utf8')
                len_str = hex(len(chunk))[2:]
                self.send(len_str.encode('utf-8'))
                self.send(b'\r\n')
                self.send(chunk)
                self.send(b'\r\n')

        # After the if clause, to always have a closed body
        self.send(b'0\r\n\r\n')

class HTTPSConnection(HTTPConnection):
    default_port = port_by_scheme['https']

    def __init__(self, host, port=None, key_file=None, cert_file=None,
                 strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
                 ssl_context=None, ssl_version=None, **kw):

        HTTPConnection.__init__(self, host, port, strict=strict,
                                timeout=timeout, **kw)

        if ssl_context is None:
            ssl_context = create_urllib3_context(
                ssl_version=resolve_ssl_version(ssl_version),
                cert_reqs=ssl.CERT_NONE,
            )

            if cert_file:
                ssl_context.load_cert_chain(cert_file, key_file)

        self.key_file = key_file
        self.cert_file = cert_file
        self.ssl_context = ssl_context

        # Required property for Google AppEngine 1.9.0 which otherwise causes
        # HTTPS requests to go out as HTTP. (See Issue #356)
        self._protocol = 'https'

    def connect(self):
        conn = self._new_conn()
        self._prepare_conn(conn)
        self.sock = ssl.wrap_socket(conn, self.key_file, self.cert_file)

class VerifiedHTTPSConnection(HTTPSConnection):
    """
    Based on httplib.HTTPSConnection but wraps the socket with
    SSL certification.
    """
    cert_reqs = None
    ca_certs = None
    ca_cert_dir = None
    ssl_version = None
    assert_fingerprint = None

    def set_cert(self, key_file=None, cert_file=None,
                 cert_reqs=None, ca_certs=None,
                 assert_hostname=None, assert_fingerprint=None,
                 ca_cert_dir=None):

        if (ca_certs or ca_cert_dir) and cert_reqs is None:
            cert_reqs = 'CERT_REQUIRED'

        self.key_file = key_file
        self.cert_file = cert_file
        self.cert_reqs = cert_reqs
        self.assert_hostname = assert_hostname
        self.assert_fingerprint = assert_fingerprint
        self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
        self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)

        context = self.ssl_context
        context.verify_mode = resolve_cert_reqs(cert_reqs)

        if cert_file is not None:
            context.load_cert_chain(cert_file, key_file)

        if ca_certs is not None:
            self.ssl_context.load_verify_locations(ca_certs)

    def connect(self):
        # Add certificate verification
        conn = self._new_conn()

        hostname = self.host
        if getattr(self, '_tunnel_host', None):
            # _tunnel_host was added in Python 2.6.3
            # (See: http://hg.python.org/cpython/rev/0f57b30a152f)

            self.sock = conn
            # Calls self._set_hostport(), so self.host is
            # self._tunnel_host below.
            self._tunnel()
            # Mark this connection as not reusable
            self.auto_open = 0

            # Override the host with the one we're requesting data from.
            hostname = self._tunnel_host

        is_time_off = datetime.date.today() < RECENT_DATE
        if is_time_off:
            warnings.warn((
                'System time is way off (before {0}). This will probably '
                'lead to SSL verification errors').format(RECENT_DATE),
                SystemTimeWarning
            )

        # Wrap socket using verification with the root certs in
        # trusted_root_certs
        if HAS_SNI:
            self.sock = self.ssl_context.wrap_socket(conn,
                                                     server_hostname=hostname)
        else:
            self.sock = self.ssl_context.wrap_socket(conn)

        if self.assert_fingerprint:
            assert_fingerprint(self.sock.getpeercert(binary_form=True),
                               self.assert_fingerprint)
        elif self.ssl_context.verify_mode != ssl.CERT_NONE \
                and self.assert_hostname is not False:
            cert = self.sock.getpeercert()
            if not cert.get('subjectAltName', ()):
                warnings.warn((
                    'Certificate for {0} has no `subjectAltName`, falling back to check for a '
                    '`commonName` for now. This feature is being removed by major browsers and '
                    'deprecated by RFC 2818. (See https://github.com/shazow/urllib3/issues/497 '
                    'for details.)'.format(hostname)),
                    SubjectAltNameWarning
                )
            _match_hostname(cert, self.assert_hostname or hostname)

        self.is_verified = (self.ssl_context.verify_mode == ssl.CERT_REQUIRED
                            or self.assert_fingerprint is not None)

def _match_hostname(cert, asserted_hostname):
    try:
        match_hostname(cert, asserted_hostname)
    except CertificateError as e:
        log.error(
            'Certificate did not match expected hostname: %s. '
            'Certificate: %s', asserted_hostname, cert
        )
        # Add cert to exception and reraise so client code can inspect
        # the cert when catching the exception, if they want to
        e._peer_cert = cert
        raise

if ssl:
    # Make a copy for testing.
    UnverifiedHTTPSConnection = HTTPSConnection
    HTTPSConnection = VerifiedHTTPSConnection
else:
    HTTPSConnection = DummyConnection
<commit_msg>Add missing support for ca_cert_dir<commit_after>from __future__ import absolute_import
import datetime
import errno
import logging
import os
import sys
import socket
from socket import error as SocketError, timeout as SocketTimeout
import warnings
from .packages import six

try:  # Python 3
    from http.client import HTTPConnection as _HTTPConnection
    from http.client import HTTPException  # noqa: unused in this module
except ImportError:
    from httplib import HTTPConnection as _HTTPConnection
    from httplib import HTTPException  # noqa: unused in this module

try:  # Compiled with SSL?
    import ssl
    BaseSSLError = ssl.SSLError
except (ImportError, AttributeError):  # Platform-specific: No SSL.
    ssl = None

    class BaseSSLError(BaseException):
        pass

try:  # Python 3:
    # Not a no-op, we're adding this to the namespace so it can be imported.
    ConnectionError = ConnectionError
except NameError:  # Python 2:
    class ConnectionError(Exception):
        pass

from .exceptions import (
    NewConnectionError,
    ConnectTimeoutError,
    SubjectAltNameWarning,
    SystemTimeWarning,
    SSLError
)
from .packages.ssl_match_hostname import match_hostname, CertificateError

from .util.ssl_ import (
    HAS_SNI,
    resolve_cert_reqs,
    resolve_ssl_version,
    assert_fingerprint,
    create_urllib3_context,
    ssl_wrap_socket,
)

from .util import connection

from ._collections import HTTPHeaderDict

log = logging.getLogger(__name__)

port_by_scheme = {
    'http': 80,
    'https': 443,
}

RECENT_DATE = datetime.date(2014, 1, 1)

class DummyConnection(object):
    """Used to detect a failed ConnectionCls import."""
    pass

class HTTPConnection(_HTTPConnection, object):
    """
    Based on httplib.HTTPConnection but provides an extra constructor
    backwards-compatibility layer between older and newer Pythons.

    Additional keyword parameters are used to configure attributes of the connection.
    Accepted parameters include:

      - ``strict``: See the documentation on :class:`urllib3.connectionpool.HTTPConnectionPool`
      - ``source_address``: Set the source address for the current connection.

        .. note:: This is ignored for Python 2.6. It is only applied for 2.7 and 3.x

      - ``socket_options``: Set specific options on the underlying socket. If not specified, then
        defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling
        Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy.

        For example, if you wish to enable TCP Keep Alive in addition to the defaults,
        you might pass::

            HTTPConnection.default_socket_options + [
                (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),
            ]

        Or you may want to disable the defaults by passing an empty list (e.g., ``[]``).
    """

    default_port = port_by_scheme['http']

    #: Disable Nagle's algorithm by default.
    #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]``
    default_socket_options = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]

    #: Whether this connection verifies the host's certificate.
    is_verified = False

    def __init__(self, *args, **kw):
        if six.PY3:  # Python 3
            kw.pop('strict', None)

        # Pre-set source_address in case we have an older Python like 2.6.
        self.source_address = kw.get('source_address')

        if sys.version_info < (2, 7):  # Python 2.6
            # _HTTPConnection on Python 2.6 will balk at this keyword arg, but
            # not newer versions. We can still use it when creating a
            # connection though, so we pop it *after* we have saved it as
            # self.source_address.
            kw.pop('source_address', None)

        #: The socket options provided by the user. If no options are
        #: provided, we use the default options.
        self.socket_options = kw.pop('socket_options', self.default_socket_options)

        # Superclass also sets self.source_address in Python 2.7+.
        _HTTPConnection.__init__(self, *args, **kw)

    def _new_conn(self):
        """ Establish a socket connection and set nodelay settings on it.

        :return: New socket connection.
        """
        extra_kw = {}
        if self.source_address:
            extra_kw['source_address'] = self.source_address

        if self.socket_options:
            extra_kw['socket_options'] = self.socket_options

        try:
            conn = connection.create_connection(
                (self.host, self.port), self.timeout, **extra_kw)

        except SocketTimeout as e:
            raise ConnectTimeoutError(
                self, "Connection to %s timed out. (connect timeout=%s)" %
                (self.host, self.timeout))

        except SocketError as e:
            raise NewConnectionError(
                self, "Failed to establish a new connection: %s" % e)

        return conn

    def _prepare_conn(self, conn):
        self.sock = conn
        # the _tunnel_host attribute was added in python 2.6.3 (via
        # http://hg.python.org/cpython/rev/0f57b30a152f) so pythons 2.6(0-2) do
        # not have them.
        if getattr(self, '_tunnel_host', None):
            # TODO: Fix tunnel so it doesn't depend on self.sock state.
            self._tunnel()
            # Mark this connection as not reusable
            self.auto_open = 0

    def connect(self):
        conn = self._new_conn()
        self._prepare_conn(conn)

    def request_chunked(self, method, url, body=None, headers=None):
        """
        Alternative to the common request method, which sends the
        body with chunked encoding and not as one block
        """
        headers = HTTPHeaderDict(headers if headers is not None else {})
        skip_accept_encoding = 'accept-encoding' in headers
        self.putrequest(method, url, skip_accept_encoding=skip_accept_encoding)
        for header, value in headers.items():
            self.putheader(header, value)
        if 'transfer-encoding' not in headers:
            self.putheader('Transfer-Encoding', 'chunked')
        self.endheaders()

        if body is not None:
            stringish_types = six.string_types + (six.binary_type,)
            if isinstance(body, stringish_types):
                body = (body,)
            for chunk in body:
                if not chunk:
                    continue
                if not isinstance(chunk, six.binary_type):
                    chunk = chunk.encode('utf8')
                len_str = hex(len(chunk))[2:]
                self.send(len_str.encode('utf-8'))
                self.send(b'\r\n')
                self.send(chunk)
                self.send(b'\r\n')

        # After the if clause, to always have a closed body
        self.send(b'0\r\n\r\n')

class HTTPSConnection(HTTPConnection):
    default_port = port_by_scheme['https']

    def __init__(self, host, port=None, key_file=None, cert_file=None,
                 strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
                 ssl_context=None, ssl_version=None, **kw):

        HTTPConnection.__init__(self, host, port, strict=strict,
                                timeout=timeout, **kw)

        if ssl_context is None:
            ssl_context = create_urllib3_context(
                ssl_version=resolve_ssl_version(ssl_version),
                cert_reqs=ssl.CERT_NONE,
            )

            if cert_file:
                ssl_context.load_cert_chain(cert_file, key_file)

        self.key_file = key_file
        self.cert_file = cert_file
        self.ssl_context = ssl_context

        # Required property for Google AppEngine 1.9.0 which otherwise causes
        # HTTPS requests to go out as HTTP. (See Issue #356)
        self._protocol = 'https'

    def connect(self):
        conn = self._new_conn()
        self._prepare_conn(conn)
        self.sock = ssl.wrap_socket(conn, self.key_file, self.cert_file)

class VerifiedHTTPSConnection(HTTPSConnection):
    """
    Based on httplib.HTTPSConnection but wraps the socket with
    SSL certification.
    """
    cert_reqs = None
    ca_certs = None
    ca_cert_dir = None
    ssl_version = None
    assert_fingerprint = None

    def set_cert(self, key_file=None, cert_file=None,
                 cert_reqs=None, ca_certs=None,
                 assert_hostname=None, assert_fingerprint=None,
                 ca_cert_dir=None):

        if (ca_certs or ca_cert_dir) and cert_reqs is None:
            cert_reqs = 'CERT_REQUIRED'

        self.key_file = key_file
        self.cert_file = cert_file
        self.cert_reqs = cert_reqs
        self.assert_hostname = assert_hostname
        self.assert_fingerprint = assert_fingerprint
        self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
        self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)

        context = self.ssl_context
        context.verify_mode = resolve_cert_reqs(cert_reqs)

        if cert_file is not None:
            context.load_cert_chain(cert_file, key_file)

        if ca_certs or ca_cert_dir:
            try:
                context.load_verify_locations(ca_certs, ca_cert_dir)
            except IOError as e:  # Platform-specific: Python 2.6, 2.7, 3.2
                raise SSLError(e)
            # Py33 raises FileNotFoundError which subclasses OSError
            # These are not equivalent unless we check the errno attribute
            except OSError as e:  # Platform-specific: Python 3.3 and beyond
                if e.errno == errno.ENOENT:
                    raise SSLError(e)
                raise

    def connect(self):
        # Add certificate verification
        conn = self._new_conn()

        hostname = self.host
        if getattr(self, '_tunnel_host', None):
            # _tunnel_host was added in Python 2.6.3
            # (See: http://hg.python.org/cpython/rev/0f57b30a152f)

            self.sock = conn
            # Calls self._set_hostport(), so self.host is
            # self._tunnel_host below.
            self._tunnel()
            # Mark this connection as not reusable
            self.auto_open = 0

            # Override the host with the one we're requesting data from.
            hostname = self._tunnel_host

        is_time_off = datetime.date.today() < RECENT_DATE
        if is_time_off:
            warnings.warn((
                'System time is way off (before {0}). This will probably '
                'lead to SSL verification errors').format(RECENT_DATE),
                SystemTimeWarning
            )

        # Wrap socket using verification with the root certs in
        # trusted_root_certs
        if HAS_SNI:
            self.sock = self.ssl_context.wrap_socket(conn,
                                                     server_hostname=hostname)
        else:
            self.sock = self.ssl_context.wrap_socket(conn)

        if self.assert_fingerprint:
            assert_fingerprint(self.sock.getpeercert(binary_form=True),
                               self.assert_fingerprint)
        elif self.ssl_context.verify_mode != ssl.CERT_NONE \
                and self.assert_hostname is not False:
            cert = self.sock.getpeercert()
            if not cert.get('subjectAltName', ()):
                warnings.warn((
                    'Certificate for {0} has no `subjectAltName`, falling back to check for a '
                    '`commonName` for now. This feature is being removed by major browsers and '
                    'deprecated by RFC 2818. (See https://github.com/shazow/urllib3/issues/497 '
                    'for details.)'.format(hostname)),
                    SubjectAltNameWarning
                )
            _match_hostname(cert, self.assert_hostname or hostname)

        self.is_verified = (self.ssl_context.verify_mode == ssl.CERT_REQUIRED
                            or self.assert_fingerprint is not None)

def _match_hostname(cert, asserted_hostname):
    try:
        match_hostname(cert, asserted_hostname)
    except CertificateError as e:
        log.error(
            'Certificate did not match expected hostname: %s. '
            'Certificate: %s', asserted_hostname, cert
        )
        # Add cert to exception and reraise so client code can inspect
        # the cert when catching the exception, if they want to
        e._peer_cert = cert
        raise

if ssl:
    # Make a copy for testing.
    UnverifiedHTTPSConnection = HTTPSConnection
    HTTPSConnection = VerifiedHTTPSConnection
else:
    HTTPSConnection = DummyConnection
<|endoftext|>


<commit_before>from datetime import datetime

NORTH = 0
EAST = 1
SOUTH = 2
WEST = 3

BLOCK_LENGTH_MM = 400

class Section:

    def __init__(self, direction):
        self.direction = direction
        self.measurements = []
        self.block_distance = None

    def add_distance_sample(self, distance):
        # TODO: Verify distance validity by checking if the delta distance is
        # reasonable.
        if distance > 50:
            self.measurements.append((distance, datetime.now()))

    def finish(self, offset=0):
        # If we're somehow detecting the next turn for a dead end corner before
        # any measurements have been received our normal algorithm won't work.
        # Handle this by explicitly setting the distance to zero.
        if len(self.measurements) == 0:
            self.block_distance = 0
            return

        # Takes the difference between max measurement and min measurement and divide by block length.
        self.block_distance = round((self.get_max() - (self.get_min()-offset)) / BLOCK_LENGTH_MM)

    def for_right_turn(self):
        return Section((self.direction + 1) % 4)

    def for_left_turn(self):
        return Section((self.direction - 1) % 4)

    def get_max(self):
        if len(self.measurements) == 0:
            return 0
        return max(self.measurements, key=lambda x: x[0])[0]

    def get_min(self):
        if len(self.measurements) == 0:
            return 0
        return min(self.measurements, key=lambda x: x[0])[0]<commit_msg>Fixed offset calculations<commit_after>from datetime import datetime

NORTH = 0
EAST = 1
SOUTH = 2
WEST = 3

BLOCK_LENGTH_MM = 400

class Section:

    def __init__(self, direction):
        self.direction = direction
        self.measurements = []
        self.block_distance = None

    def add_distance_sample(self, distance):
        # TODO: Verify distance validity by checking if the delta distance is
        # reasonable.
        if distance > 50:
            self.measurements.append((distance, datetime.now()))

    def finish(self, offset=0):
        # If we're somehow detecting the next turn for a dead end corner before
        # any measurements have been received our normal algorithm won't work.
        # Handle this by explicitly setting the distance to zero.
        if len(self.measurements) == 0:
            self.block_distance = 0
            return

        # Takes the difference between max measurement and min measurement and divide by block length.
        self.block_distance = round((self.get_max() - (self.get_min()+offset)) / BLOCK_LENGTH_MM)

    def for_right_turn(self):
        return Section((self.direction + 1) % 4)

    def for_left_turn(self):
        return Section((self.direction - 1) % 4)

    def get_max(self):
        if len(self.measurements) == 0:
            return 0
        return max(self.measurements, key=lambda x: x[0])[0]

    def get_min(self):
        if len(self.measurements) == 0:
            return 0
        return min(self.measurements, key=lambda x: x[0])[0]<|endoftext|>


<commit_before>#!/usr/bin/env python

# ===============================================================================
# Copyright (c)  2014 Geoscience Australia
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#     * Neither Geoscience Australia nor the names of its contributors may be
#       used to endorse or promote products derived from this software
#       without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# ===============================================================================

"""
Some tools used to work with GDAL.
"""

from __future__ import absolute_import
from __future__ import print_function
import os
import sys
import logging
import numpy
from osgeo import gdal, osr
from EOtools.execute import execute
from EOtools.meta import print_call

logger = logging.getLogger('root.' + __name__)

NUMPY_GDAL_TYPE_MAP = {
    # numpy dtype: GDAL dtype
    numpy.dtype('uint8'): gdal.GDT_Byte,
    numpy.dtype('uint16'): gdal.GDT_UInt16,
    numpy.dtype('int16'): gdal.GDT_Int16,
    numpy.dtype('float32'): gdal.GDT_Float32,
    numpy.dtype('float64'): gdal.GDT_Float64,
    numpy.uint8: gdal.GDT_Byte,
    numpy.uint16: gdal.GDT_UInt16,
    numpy.int16: gdal.GDT_Int16,
    numpy.float32: gdal.GDT_Float32,
    numpy.float64: gdal.GDT_Float64}

class Buffers(object):

    """
    Holds some value for each side of an image. This was initially created to hold buffer widths (in pixels) for
    a scene, but does not care about the type of the values passed to the constructer and can hence be used for
    any type.
    """

    def __init__(self, left, right=None, top=None, bottom=None):
        """
        Constructor.

        The arguments are copied directly to members of the same name with the restrictions that:

        - if right is None then top and bottom must also be None, and

        - if right is not None then top and bottom must not be none either.
        """
        if right is None:
            assert top is None and bottom is None, "if right is None then top and bottom must also be None"
            self.left = self.right = self.top = self.bottom = left
        else:
            assert top is not None and bottom is not None, "if right is not None then top and bottom must also not be None"
            self.left = left
            self.right = right
            self.top = top
            self.bottom = bottom

    def __str__(self):
        return "DatasetDrivers.gdal_tools.Buffers(%s)" % ", ".join(
            map(str, (self.left, self.right, self.top, self.bottom)))

class ImageShape(object):

    """
    Holds a few values that describe the shape of an image.
    """

    def __init__(self, origin_x, origin_y, dim_x, dim_y, pix_sz_x, pix_sz_y=None, proj=None):
        """
        Constructor

        :param origin_x:
            The X origin of the image (top left).
        :type origin_x:
            float

        :param origin_y:
            The Y origin of the image (top left).
        :type origin_y:
            float

        :param dim_x:
            The width of the image in pixels.
        :type dim_x:
            int

        :param dim_y:
            The height of the image in pixels.
        :type dim_y:
            int

        :param pix_sz_x:
            The width of image pixels (in the appropriate coordinate space).
        :type pix_sz_x:
            float

        :param pix_sz_y:
            The height of image pixels (in the appropriate coordinate space).
        :type pix_sz_y:
            float

        :param proj:
            The projection (WKT).
        :type proj:
            str
        """
        self.__origin = (origin_x, origin_y)
        self.__pix_sz = (pix_sz_x, pix_sz_y or pix_sz_x)
        self.__dim = (dim_y, dim_x)
        self.__spatial_ref = osr.SpatialReference()
        if proj:
            self.__spatial_ref.ImportFromWkt(proj)

    def __str__(self):
        return "DatasetDrivers.utils.ImageShape(%s)" % ", ".join(
            map(str, (self._ImageShape__origin, self._ImageShape__pix_sz, self._ImageShape__dim)))

    @property
    def RasterXSize(self):
        """
        The number of columns in the raster.
        """
        return self._ImageShape__dim[1]

    @property
    def RasterYSize(self):
        """
        The number of rows in the raster.
        """
        return self._ImageShape__dim[0]

    @property
    def RasterCellSize(self):
        """
        The cell size of the raster.

        This asserts that the cell size is the same in both dimensions. If the cell size is different, you must use the
        X and Y getters instead.
        """
        assert self._ImageShape__pix_sz[0] == abs(
            self._ImageShape__pix_sz[1]), "Cannot use method ImageShape.CellSize when X and Y cell sizes are different."
        return self._ImageShape__pix_sz[0]

    @property
    def RasterXCellSize(self):
        """
        The width of a cell in the raster.
        """
        return self._ImageShape__pix_sz[0]

    @property
    def RasterYCellSize(self):
        """
        The height of a cell in the raster.
        """
        return self._ImageShape__pix_sz[1]

    @property
    def RasterXOrigin(self):
        """
        The X element of the origin of the raster.
        """
        return self._ImageShape__origin[0]

    @property
    def RasterYOrigin(self):
        """
        The Y element of the origin of the raster.
        """
        return self._ImageShape__origin[1]

    @property
    def shape(self):
        """
        Tuple containing the number of columns and rows (in that order).
        """
        return (self.RasterYSize, self.RasterXSize)

    def GetProjection(self, as_proj4=False):
        if as_proj4:
            return self._ImageShape__spatial_ref.ExportToProj4()
        else:
            return self._ImageShape__spatial_ref.ExportToWkt()

    def write_header_slope_file(self, file_name, bounds):
        output = open(file_name, 'w')
        output.write("%i %i\n" % (self.RasterYSize, self.RasterXSize))
        output.write("%i %i\n%i %i\n" % (bounds.left, bounds.right, bounds.top, bounds.bottom))
        output.write("%f\n" % self.RasterCellSize)
        output.write("%f %f\n" % (self.RasterYOrigin, self.RasterXOrigin))
        output.close()

def default_bounds_getter(dataset):
    """
    Get the various bits and pieces needed to determine the 'shape' of an image (origin, cell size and
    number of rows and cols). This function aids in making access to a :py:class:`ImageShape` for an
    object easy and hence making the code that uses it more generic. The algorithm is as follows:

    - If ``dataset`` is an instance of :py:class:`ImageShape`, then it is simply returned.

    - Otherwise, if ``dataset`` has an attribute ``bounds_getter`` it is retrieved and passed ``dataset``
        (see :py:meth:`DatasetDrivers.dataset._scene_dataset.SceneDataset.bounds_getter` for an example).

    - Otherwise ``dataset`` must be an instance of :py:class:`gdal.Dataset` (a :py:class:`TypeError` is
        raised otherwise), and the shape is generated with:

        >>> g = dataset.GetGeoTransform()
        >>> return ImageShape(g[0], g[3], dataset.RasterXSize, dataset.RasterYSize, g[1], g[5], dataset.GetProjection())

    :param dataset:
        The dataset to extract the required elements from.

    :return:
        A :py:class:`ImageShape` for ``dataset``.
    """
    if isinstance(dataset, ImageShape):
        return dataset
    elif hasattr(dataset, 'bounds_getter'):
        return dataset.bounds_getter(dataset)
    elif isinstance(dataset, gdal.Dataset):
        g = dataset.GetGeoTransform()
        return ImageShape(g[0], g[3], dataset.RasterXSize, dataset.RasterYSize, g[1], g[5], dataset.GetProjection())
    else:
        raise TypeError("dataset must be instance of ImageShape, gdal.Dataset or DatasetDrivers.dataset.Dataset.")

@print_call(logger.info)
def warp(
        shape_dataset,
        master_dataset_path,
        output_filename,
        buffer_widths,
        output_format,
        resammple_method="bilinear",
        bounds_getter=default_bounds_getter):
    """
    Use the gdalwarp executable to clip (and potentially resample) a region.

    Preconditions on this method are:
        - the directory specified for the output (``output_filename``) exists,
        - that ``master_dataset_path`` exists, and
        - gdalwarp is on the path.

    :param shape_dataset:
        Object to extract the shape of the desired region from. This is done using ``bounds_getter``
        (see :py:func:`default_bounds_getter` for specification of the interface).

    :param master_dataset_path:
        The path to the dataset to clip from. This should be a valid argument to :py:func:`gdal.Open`.
    :type shape_dataset:
        str

    :param output_filename:
        The name of the output file (passed as the output argument to gdalwarp).
    :type shape_dataset:
        str

    :param buffer_widths:
        An object of type :py:class:`ImageShape` (or one that supports the same interface).
    :type buffer_widths:
        :py:class:`Buffers`

    :param output_format:
        The desired format of the clipped dataset. (passed as argument -of to gdalwarp).
    :type output_format:
        str

    :param resample_method:
        The resampling method to be used (passed as argument -r to gdalwarp).
    :type resample_method:
        str

    :param bounds_getter:
        Callable used to extract the bounds from ``shape_dataset``.

    :return:
        The name of the dataset written to disk.
    """
    assert not execute("which gdalwarp")["returncode"], "gdalwarp not available"
    output_dir = os.path.dirname(output_filename)

    assert os.stat(output_dir), "output_dir (%s) must exist." % output_dir
    assert os.stat(master_dataset_path), "master_dataset (%s) must exist" % master_dataset_path

    shape = bounds_getter(shape_dataset)

    xres = shape.RasterXCellSize
    yres = shape.RasterYCellSize
    xmin = shape.RasterXOrigin - xres * buffer_widths.left
    xmax = shape.RasterXOrigin + xres * (shape.RasterXSize + buffer_widths.right)
    ymax = shape.RasterYOrigin - yres * buffer_widths.top  # in the cases I've looked at, yres is negative.
    ymin = shape.RasterYOrigin + yres * (shape.RasterYSize + buffer_widths.bottom)

    command_string = 'gdalwarp -overwrite -of %s -t_srs "%s" -r %s -te %f %f %f %f -tr %f %f %s %s' % (
        output_format,
        shape.GetProjection(as_proj4=True),
        resammple_method,
        float(xmin), float(ymin), float(xmax), float(ymax),
        float(xres), float(yres),
        master_dataset_path, output_filename)

    result = execute(command_string)
    if result["returncode"]:
        print("error in executing %s\n\n\tstdout: %s\n\n\tstderr: %s\n" % (command_string, result['stdout'], result['stderr']))

    assert not result["returncode"], "error in executing %s\n\n\tstdout: %s\n\n\tstderr: %s\n" % (
        command_string, result['stdout'], result['stderr'])
    return output_filename
<commit_msg>Removed gdal_tools.py, too specific and never used.<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# * Copyright (c) 2009-2015. Authors: see NOTICE file.
# *
# * Licensed under the Apache License, Version 2.0 (the "License");
# * you may not use this file except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.

__author__ = "Stévens Benjamin <b.stevens@ulg.ac.be>"
__contributors__ = ["Marée Raphaël <raphael.maree@ulg.ac.be>", "Rollus Loïc <lrollus@ulg.ac.be"]
__copyright__ = "Copyright 2010-2015 University of Liège, Belgium, http://www.cytomine.be/"

from model import Model
from collection import Collection

class Project(Model):
    def __init__(self, name=None, id_ontology=None, **attributes):
        super(Project, self).__init__(**attributes)
        self.name = name
        self.ontology = id_ontology

class ProjectCollection(Collection):
    def __init__(self, filters=None, query_parameters=None, max=0, offset=0):
        super(ProjectCollection, self).__init__(Project, filters, query_parameters, max, offset)
        self._allowed_filters = ["user", "software", "ontology"]

class Discipline(Model):
    def __init__(self, name=None, **attributes):
        super(Discipline, self).__init__(**attributes)
        self.name = name

class DisciplineCollection(Collection):
    def __init__(self, filters=None, query_parameters=None, max=0, offset=0):
        super(DisciplineCollection, self).__init__(Discipline, filters, query_parameters, max, offset)

# class ProjectProperty(Model):
#
#     def __init__(self, params = None):
#         super(ProjectProperty, self).__init__(params)
#         self._callback_identifier = "property"
#
#     def to_url(self):
#         if hasattr(self, "domainIdent") and not hasattr(self, "id"):#new
#             return "project/%d/property.json" % self.domainIdent
#         elif hasattr(self, "domainIdent") and hasattr(self, "id"):
#             return "project/%d/property/%d.json" % (self.domainIdent, self.id)
#
#     def __str__( self ):
#         return "Project Property %d,%d " % (self.project, self.id)
<commit_msg>complete Project model<commit_after># -*- coding: utf-8 -*-

# * Copyright (c) 2009-2015. Authors: see NOTICE file.
# *
# * Licensed under the Apache License, Version 2.0 (the "License");
# * you may not use this file except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.

__author__ = "Stévens Benjamin <b.stevens@ulg.ac.be>"
__contributors__ = ["Marée Raphaël <raphael.maree@ulg.ac.be>", "Rollus Loïc <lrollus@ulg.ac.be"]
__copyright__ = "Copyright 2010-2015 University of Liège, Belgium, http://www.cytomine.be/"

from collection import Collection
from model import Model

class Project(Model):
    def __init__(self, name=None, id_ontology=None, **attributes):
        super(Project, self).__init__(**attributes)
        self.name = name
        self.ontology = id_ontology
        self.ontologyName = None
        self.discipline = None
        self.blindMode = None
        self.disciplineName = None
        self.numberOfSlides = None
        self.numberOfImages = None
        self.numberOfAnnotations = None
        self.numberOfJobAnnotations = None
        self.retrievalProjects = None
        self.numberOfReviewedAnnotations = None
        self.retrievalDisable = None
        self.retrievalAllOntology = None
        self.isClosed = None
        self.isReadOnly = None
        self.hideUsersLayers = None
        self.hideAdminsLayers = None

class ProjectCollection(Collection):
    def __init__(self, filters=None, query_parameters=None, max=0, offset=0):
        super(ProjectCollection, self).__init__(Project, filters, query_parameters, max, offset)
        self._allowed_filters = ["user", "software", "ontology"]

class Discipline(Model):
    def __init__(self, name=None, **attributes):
        super(Discipline, self).__init__(**attributes)
        self.name = name

class DisciplineCollection(Collection):
    def __init__(self, filters=None, query_parameters=None, max=0, offset=0):
        super(DisciplineCollection, self).__init__(Discipline, filters, query_parameters, max, offset)

# class ProjectProperty(Model):
#
#     def __init__(self, params = None):
#         super(ProjectProperty, self).__init__(params)
#         self._callback_identifier = "property"
#
#     def to_url(self):
#         if hasattr(self, "domainIdent") and not hasattr(self, "id"):#new
#             return "project/%d/property.json" % self.domainIdent
#         elif hasattr(self, "domainIdent") and hasattr(self, "id"):
#             return "project/%d/property/%d.json" % (self.domainIdent, self.id)
#
#     def __str__( self ):
#         return "Project Property %d,%d " % (self.project, self.id)
<|endoftext|>


<commit_before>from piece.base import Piece
from piece.bishop import Bishop
from piece.knight import Knight
from piece.queen import Queen
from piece.rook import Rook
from util.enums import Player
from board.move import Move

class Pawn(Piece):
    @property
    def en_passant_vulnerable(self):
        """ Check if a pawn is vulnerable to an en passant attack

        :return: boolean
        """
        last_move = self.board.last_move()
        # if the last move was this piece and it moved by two spaces then it is vulnerable
        return (last_move is not None and
                self == last_move.piece and
                abs(last_move.old_location.row - last_move.new_location.row) == 2)

    def attacked_locations(self):
        """ Get all locations under attack by this piece

        :return: Location generator
        """
        for col in [-1, 1]:
            loc = self.location.offset(self.direction, col)
            # pawn can only attack when opponent is diagonal
            if loc is not None and not self.board.empty(loc, player=self.player.opponent()):
                yield loc

    def moves(self):
        """ Get all possible moves by this piece (unvalidated moves)

        :return: Move generator
        """
        # check attacking moves for promotions
        for move in super().moves():
            if move.new_location.end_of_row:
                for t in [Queen, Rook, Bishop, Knight]:
                    yield Move.create_promotion(self, move.new_location, t, move.captured_piece)
            else:
                yield move

        up1 = self.location.offset(self.direction, 0)
        up2 = up1.offset(self.direction, 0)
        # if space ahead is empty then can move
        if self.board.empty(up1):
            # check move for promotions
            if up1.end_of_row:
                for t in [Queen, Rook, Bishop, Knight]:
                    yield Move.create_promotion(self, up1, t)
            else:
                # regular move
                yield Move(self, up1)

            # if on the first move the two spaces ahead is empty then can move
            if not self.moved and self.board.empty(up2):
                yield Move(self, up2)

        # en passant
        for col in [-1, 1]:
            loc = self.location.offset(0, col)
            p = self.board.piece(loc, self.player.opponent())
            if isinstance(p, Pawn) and p.en_passant_vulnerable:
                yield Move.create_en_passant(self, loc.offset(self.direction, 0), p)

    @property
    def direction(self):
        """ Get the direction that a pawn moves

        :return: -1 or 1
        """
        return -1 if self.player == Player.BLACK else 1<commit_msg>add missing value member<commit_after>from piece.base import Piece
from piece.bishop import Bishop
from piece.knight import Knight
from piece.queen import Queen
from piece.rook import Rook
from util.enums import Player
from board.move import Move

class Pawn(Piece):
    VALUE = 1

    @property
    def en_passant_vulnerable(self):
        """ Check if a pawn is vulnerable to an en passant attack

        :return: boolean
        """
        last_move = self.board.last_move()
        # if the last move was this piece and it moved by two spaces then it is vulnerable
        return (last_move is not None and
                self == last_move.piece and
                abs(last_move.old_location.row - last_move.new_location.row) == 2)

    def attacked_locations(self):
        """ Get all locations under attack by this piece

        :return: Location generator
        """
        for col in [-1, 1]:
            loc = self.location.offset(self.direction, col)
            # pawn can only attack when opponent is diagonal
            if loc is not None and not self.board.empty(loc, player=self.player.opponent()):
                yield loc

    def moves(self):
        """ Get all possible moves by this piece (unvalidated moves)

        :return: Move generator
        """
        # check attacking moves for promotions
        for move in super().moves():
            if move.new_location.end_of_row:
                for t in [Queen, Rook, Bishop, Knight]:
                    yield Move.create_promotion(self, move.new_location, t, move.captured_piece)
            else:
                yield move

        up1 = self.location.offset(self.direction, 0)
        up2 = up1.offset(self.direction, 0)
        # if space ahead is empty then can move
        if self.board.empty(up1):
            # check move for promotions
            if up1.end_of_row:
                for t in [Queen, Rook, Bishop, Knight]:
                    yield Move.create_promotion(self, up1, t)
            else:
                # regular move
                yield Move(self, up1)

            # if on the first move the two spaces ahead is empty then can move
            if not self.moved and self.board.empty(up2):
                yield Move(self, up2)

        # en passant
        for col in [-1, 1]:
            loc = self.location.offset(0, col)
            p = self.board.piece(loc, self.player.opponent())
            if isinstance(p, Pawn) and p.en_passant_vulnerable:
                yield Move.create_en_passant(self, loc.offset(self.direction, 0), p)

    @property
    def direction(self):
        """ Get the direction that a pawn moves

        :return: -1 or 1
        """
        return -1 if self.player == Player.BLACK else 1<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import print_function, division, absolute_import

import os
import sys
import fnmatch
import importlib
import imp
import ast
import doctest
import collections
try:
    from types import FileType  # py2
except ImportError:
    from io import IOBase as FileType  # py3

from .log import logger
from .utils import parse_git_config
from .modules import ImportedModules

def project_import_modules(project_path, ignores):
    """Get entire project all imported modules."""
    modules = ImportedModules()
    local_mods = list()
    cur_dir = os.getcwd()
    ignore_paths = collections.defaultdict(set)
    if not ignores:
        ignore_paths[project_path].add('.git')
    else:
        for path in ignores:
            parent_dir = os.path.dirname(path)
            ignore_paths[parent_dir].add(os.path.basename(path))

    logger.info('Extracting project: {0}'.format(project_path))
    for dirpath, dirnames, files in os.walk(project_path, followlinks=True):
        if dirpath in ignore_paths:
            dirnames[:] = [d for d in dirnames
                           if d not in ignore_paths[dirpath]]
        logger.info('Extracting directory: {0}'.format(dirpath))
        py_files = list()
        for fn in files:
            # C extension.
            if fn.endswith('.so'):
                local_mods.append(fn[:-3])
            # Normal Python file.
            if fn.endswith('.py'):
                local_mods.append(fn[:-3])
                py_files.append(fn)
        if '__init__.py' in files:
            local_mods.append(os.path.basename(dirpath))
        for file in py_files:
            fpath = os.path.join(dirpath, file)
            fake_path = fpath.split(cur_dir)[1][1:]
            logger.info('Extracting file: {0}'.format(fpath))
            with open(fpath, 'rb') as f:
                modules |= file_import_modules(fake_path, f.read())

    logger.info('Finish extracting in project: {0}'.format(project_path))
    return modules, local_mods

def file_import_modules(fpath, fdata):
    """Get single file all imported modules."""
    modules = ImportedModules()
    str_codes = collections.deque([(fdata, 1)])

    while str_codes:
        str_code, lineno = str_codes.popleft()
        ic = ImportChecker(fpath, lineno)
        try:
            parsed = ast.parse(str_code)
            ic.visit(parsed)
        # Ignore SyntaxError in Python code.
        except SyntaxError:
            pass
        modules |= ic.modules
        str_codes.extend(ic.str_codes)
        del ic

    return modules

class ImportChecker(object):

    def __init__(self, fpath, lineno):
        self._fpath = fpath
        self._lineno = lineno - 1
        self._modules = ImportedModules()
        self._str_codes = collections.deque()

    def visit_Import(self, node):
        """As we know: `import a [as b]`."""
        lineno = node.lineno + self._lineno
        for alias in node.names:
            self._modules.add(alias.name, self._fpath, lineno)

    def visit_ImportFrom(self, node):
        """As we know: `from a import b [as c]`."""
        self._modules.add(node.module, self._fpath, node.lineno + self._lineno)

    def visit_Exec(self, node):
        """
        Check `expression` of `exec(expression[, globals[, locals]])`.
        **Just available in python 2.**
        """
        if hasattr(node.body, 's'):
            self._str_codes.append((node.body.s, node.lineno + self._lineno))
        # PR#13: https://github.com/Damnever/pigar/pull/13
        # Sometimes exec statement may be called with tuple in Py2.7.6
        elif hasattr(node.body, 'elts') and len(node.body.elts) >= 1:
            self._str_codes.append(
                (node.body.elts[0].s, node.lineno + self._lineno))

    def visit_Expr(self, node):
        """
        Check `expressin` of `eval(expression[, globals[, locals]])`.
        """
        # Built-in functions
        value = node.value
        if isinstance(value, ast.Call):
            if hasattr(value.func, 'id'):
                if (value.func.id == 'eval' and
                        hasattr(node.value.args[0], 's')):
                    self._str_codes.append(
                        (node.value.args[0].s, node.lineno + self._lineno))
                # **`exec` function in Python 3.**
                elif (value.func.id == 'exec' and
                        hasattr(node.value.args[0], 's')):
                    self._str_codes.append(
                        (node.value.args[0].s, node.lineno + self._lineno))

    def visit_FunctionDef(self, node):
        """
        Check docstring of function, if docstring is used for doctest.
        """
        docstring = self._parse_docstring(node)
        if docstring:
            self._str_codes.append((docstring, node.lineno + self._lineno + 2))

    def visit_ClassDef(self, node):
        """
        Check docstring of class, if docstring is used for doctest.
        """
        docstring = self._parse_docstring(node)
        if docstring:
            self._str_codes.append((docstring, node.lineno + self._lineno + 2))

    def visit(self, node):
        """Visit a node, no recursively."""
        for node in ast.walk(node):
            method = 'visit_' + node.__class__.__name__
            getattr(self, method, lambda x: x)(node)

    @staticmethod
    def _parse_docstring(node):
        """Extract code from docstring."""
        docstring = ast.get_docstring(node)
        if docstring:
            parser = doctest.DocTestParser()
            try:
                dt = parser.get_doctest(docstring, {}, None, None, None)
            except ValueError:
                # >>> 'abc'
                pass
            else:
                examples = dt.examples
                return '\n'.join([example.source for example in examples])
        return None

    @property
    def modules(self):
        return self._modules

    @property
    def str_codes(self):
        return self._str_codes

# #
# Check whether it is stdlib module.
# #
_CHECKED = dict()

def is_stdlib(name):
    if name in _CHECKED:
        return _CHECKED[name]

    exist = True
    module_info = ('', '', '')
    try:
        module_info = imp.find_module(name)
    except ImportError:
        try:
            # __import__(name)
            importlib.import_module(name)
            module_info = imp.find_module(name)
            sys.modules.pop(name)
        except ImportError:
            exist = False
    # Testcase: ResourceWarning
    if isinstance(module_info[0], FileType):
        module_info[0].close()
    if exist and (module_info[1] is not None and
                  ('site-packages' in module_info[1] or
                   'dist-packages' in module_info[1])):
        exist = False
    _CHECKED[name] = exist
    return exist

def get_installed_pkgs_detail():
    """Get mapping for import top level name
    and install package name with version.
    """
    mapping = dict()
    search_path = None
    for path in sys.path:
        if os.path.isdir(path) and path.rstrip('/').endswith(
                ('site-packages', 'dist-packages')):
            search_path = path
            break

    if search_path is None:
        return mapping

    for file in os.listdir(search_path):
        # Install from PYPI.
        if fnmatch.fnmatch(file, '*-info'):
            top_level = os.path.join(search_path, file, 'top_level.txt')
            if not os.path.isfile(top_level):
                continue
            pkg_name, version = file.split('-')[:2]
            if version.endswith('dist'):
                version = version.rsplit('.', 1)[0]
            with open(top_level, 'r') as f:
                for line in f:
                    mapping[line.strip()] = (pkg_name, version)

        # Install from local and available in GitHub.
        elif fnmatch.fnmatch(file, '*-link'):
            link = os.path.join(search_path, file)
            if not os.path.isfile(link):
                continue
            # Link path.
            with open(link, 'r') as f:
                for line in f:
                    line = line.strip()
                    if line != '.':
                        dev_dir = line
            if not dev_dir:
                continue
            # Egg info path.
            info_dir = [_file for _file in os.listdir(dev_dir)
                        if _file.endswith('egg-info')]
            if not info_dir:
                continue
            info_dir = info_dir[0]
            top_level = os.path.join(dev_dir, info_dir, 'top_level.txt')
            # Check whether it can be imported.
            if not os.path.isfile(top_level):
                continue

            # Check .git dir.
            git_path = os.path.join(dev_dir, '.git')
            if os.path.isdir(git_path):
                config = parse_git_config(git_path)
                url = config.get('remote "origin"', {}).get('url')
                if not url:
                    continue
                branch = 'branch "master"'
                if branch not in config:
                    for section in config:
                        if 'branch' in section:
                            branch = section
                            break
                if not branch:
                    continue
                branch = branch.split()[1][1:-1]

                pkg_name = info_dir.split('.egg')[0]
                git_url = 'git+{0}@{1}#egg={2}'.format(url, branch, pkg_name)
                with open(top_level, 'r') as f:
                    for line in f:
                        mapping[line.strip()] = ('-e', git_url)

    return mapping
<commit_msg>reduce memory usage: use decorator instead of global variable<commit_after># -*- coding: utf-8 -*-

from __future__ import print_function, division, absolute_import

import os
import sys
import fnmatch
import importlib
import imp
import ast
import doctest
import collections
import functools
try:
    from types import FileType  # py2
except ImportError:
    from io import IOBase as FileType  # py3

from .log import logger
from .utils import parse_git_config
from .modules import ImportedModules

def project_import_modules(project_path, ignores):
    """Get entire project all imported modules."""
    modules = ImportedModules()
    local_mods = list()
    cur_dir = os.getcwd()
    ignore_paths = collections.defaultdict(set)
    if not ignores:
        ignore_paths[project_path].add('.git')
    else:
        for path in ignores:
            parent_dir = os.path.dirname(path)
            ignore_paths[parent_dir].add(os.path.basename(path))

    logger.info('Extracting project: {0}'.format(project_path))
    for dirpath, dirnames, files in os.walk(project_path, followlinks=True):
        if dirpath in ignore_paths:
            dirnames[:] = [d for d in dirnames
                           if d not in ignore_paths[dirpath]]
        logger.info('Extracting directory: {0}'.format(dirpath))
        py_files = list()
        for fn in files:
            # C extension.
            if fn.endswith('.so'):
                local_mods.append(fn[:-3])
            # Normal Python file.
            if fn.endswith('.py'):
                local_mods.append(fn[:-3])
                py_files.append(fn)
        if '__init__.py' in files:
            local_mods.append(os.path.basename(dirpath))
        for file in py_files:
            fpath = os.path.join(dirpath, file)
            fake_path = fpath.split(cur_dir)[1][1:]
            logger.info('Extracting file: {0}'.format(fpath))
            with open(fpath, 'rb') as f:
                modules |= file_import_modules(fake_path, f.read())

    logger.info('Finish extracting in project: {0}'.format(project_path))
    return modules, local_mods

def file_import_modules(fpath, fdata):
    """Get single file all imported modules."""
    modules = ImportedModules()
    str_codes = collections.deque([(fdata, 1)])

    while str_codes:
        str_code, lineno = str_codes.popleft()
        ic = ImportChecker(fpath, lineno)
        try:
            parsed = ast.parse(str_code)
            ic.visit(parsed)
        # Ignore SyntaxError in Python code.
        except SyntaxError:
            pass
        modules |= ic.modules
        str_codes.extend(ic.str_codes)
        del ic

    return modules

class ImportChecker(object):

    def __init__(self, fpath, lineno):
        self._fpath = fpath
        self._lineno = lineno - 1
        self._modules = ImportedModules()
        self._str_codes = collections.deque()

    def visit_Import(self, node):
        """As we know: `import a [as b]`."""
        lineno = node.lineno + self._lineno
        for alias in node.names:
            self._modules.add(alias.name, self._fpath, lineno)

    def visit_ImportFrom(self, node):
        """As we know: `from a import b [as c]`."""
        self._modules.add(node.module, self._fpath, node.lineno + self._lineno)

    def visit_Exec(self, node):
        """
        Check `expression` of `exec(expression[, globals[, locals]])`.
        **Just available in python 2.**
        """
        if hasattr(node.body, 's'):
            self._str_codes.append((node.body.s, node.lineno + self._lineno))
        # PR#13: https://github.com/Damnever/pigar/pull/13
        # Sometimes exec statement may be called with tuple in Py2.7.6
        elif hasattr(node.body, 'elts') and len(node.body.elts) >= 1:
            self._str_codes.append(
                (node.body.elts[0].s, node.lineno + self._lineno))

    def visit_Expr(self, node):
        """
        Check `expressin` of `eval(expression[, globals[, locals]])`.
        """
        # Built-in functions
        value = node.value
        if isinstance(value, ast.Call):
            if hasattr(value.func, 'id'):
                if (value.func.id == 'eval' and
                        hasattr(node.value.args[0], 's')):
                    self._str_codes.append(
                        (node.value.args[0].s, node.lineno + self._lineno))
                # **`exec` function in Python 3.**
                elif (value.func.id == 'exec' and
                        hasattr(node.value.args[0], 's')):
                    self._str_codes.append(
                        (node.value.args[0].s, node.lineno + self._lineno))

    def visit_FunctionDef(self, node):
        """
        Check docstring of function, if docstring is used for doctest.
        """
        docstring = self._parse_docstring(node)
        if docstring:
            self._str_codes.append((docstring, node.lineno + self._lineno + 2))

    def visit_ClassDef(self, node):
        """
        Check docstring of class, if docstring is used for doctest.
        """
        docstring = self._parse_docstring(node)
        if docstring:
            self._str_codes.append((docstring, node.lineno + self._lineno + 2))

    def visit(self, node):
        """Visit a node, no recursively."""
        for node in ast.walk(node):
            method = 'visit_' + node.__class__.__name__
            getattr(self, method, lambda x: x)(node)

    @staticmethod
    def _parse_docstring(node):
        """Extract code from docstring."""
        docstring = ast.get_docstring(node)
        if docstring:
            parser = doctest.DocTestParser()
            try:
                dt = parser.get_doctest(docstring, {}, None, None, None)
            except ValueError:
                # >>> 'abc'
                pass
            else:
                examples = dt.examples
                return '\n'.join([example.source for example in examples])
        return None

    @property
    def modules(self):
        return self._modules

    @property
    def str_codes(self):
        return self._str_codes

# #
# Check whether it is stdlib module.
# #
def _checked_cache(func):
    checked = dict()

    @functools.wraps(func)
    def _wrapper(name):
        if name not in checked:
            checked[name] = func(name)
        return checked[name]

    return _wrapper

@_checked_cache
def is_stdlib(name):
    exist = True
    module_info = ('', '', '')
    try:
        module_info = imp.find_module(name)
    except ImportError:
        try:
            # __import__(name)
            importlib.import_module(name)
            module_info = imp.find_module(name)
            sys.modules.pop(name)
        except ImportError:
            exist = False
    # Testcase: ResourceWarning
    if isinstance(module_info[0], FileType):
        module_info[0].close()
    if exist and (module_info[1] is not None and
                  ('site-packages' in module_info[1] or
                   'dist-packages' in module_info[1])):
        exist = False
    return exist

def get_installed_pkgs_detail():
    """Get mapping for import top level name
    and install package name with version.
    """
    mapping = dict()
    search_path = None
    for path in sys.path:
        if os.path.isdir(path) and path.rstrip('/').endswith(
                ('site-packages', 'dist-packages')):
            search_path = path
            break

    if search_path is None:
        return mapping

    for file in os.listdir(search_path):
        # Install from PYPI.
        if fnmatch.fnmatch(file, '*-info'):
            top_level = os.path.join(search_path, file, 'top_level.txt')
            if not os.path.isfile(top_level):
                continue
            pkg_name, version = file.split('-')[:2]
            if version.endswith('dist'):
                version = version.rsplit('.', 1)[0]
            with open(top_level, 'r') as f:
                for line in f:
                    mapping[line.strip()] = (pkg_name, version)

        # Install from local and available in GitHub.
        elif fnmatch.fnmatch(file, '*-link'):
            link = os.path.join(search_path, file)
            if not os.path.isfile(link):
                continue
            # Link path.
            with open(link, 'r') as f:
                for line in f:
                    line = line.strip()
                    if line != '.':
                        dev_dir = line
            if not dev_dir:
                continue
            # Egg info path.
            info_dir = [_file for _file in os.listdir(dev_dir)
                        if _file.endswith('egg-info')]
            if not info_dir:
                continue
            info_dir = info_dir[0]
            top_level = os.path.join(dev_dir, info_dir, 'top_level.txt')
            # Check whether it can be imported.
            if not os.path.isfile(top_level):
                continue

            # Check .git dir.
            git_path = os.path.join(dev_dir, '.git')
            if os.path.isdir(git_path):
                config = parse_git_config(git_path)
                url = config.get('remote "origin"', {}).get('url')
                if not url:
                    continue
                branch = 'branch "master"'
                if branch not in config:
                    for section in config:
                        if 'branch' in section:
                            branch = section
                            break
                if not branch:
                    continue
                branch = branch.split()[1][1:-1]

                pkg_name = info_dir.split('.egg')[0]
                git_url = 'git+{0}@{1}#egg={2}'.format(url, branch, pkg_name)
                with open(top_level, 'r') as f:
                    for line in f:
                        mapping[line.strip()] = ('-e', git_url)

    return mapping
<|endoftext|>