<commit_before># Python module for scheduling bazaar project
# Includes: TLE class
#           load_gs()
#           load_tles()
#           get_passes()
#           calc_access_time()
#           plot_access_time()

# Import necessary libraries
from collections import namedtuple
from datetime import datetime, timedelta, timezone
from itertools import islice, product
import math

import ephem
from iso8601 import parse_date
import intervaltree
import matplotlib.pyplot as plt
# import seaborn

# TLE class definition
class TLE:
    """TLE class used for various tle attributes.

    Args: a three element list containing tle lines
    """
    def __init__(self, tle):
        self.tle = tle
        self.tle0 = tle[0]
        self.tle1 = tle[1]
        self.tle2 = tle[2]
        self.name = tle[0].rstrip()
        self.noradid = tle[1][2:7]
        self.epoch = tle[1][18:32]
        self.inclination = tle[2][8:16]

    def __str__(self):
        return '%s\n%s\n%s' % (self.tle0, self.tle1, self.tle2)

# load_tles() function definition
def load_tles(file):
    """Returns a list of tle's from a file.

    Arguments:
    file -- file name string containing unparsed tles
    """
    data = []
    with open(file) as f:
        while True:
            # an iterator that returns the next N lines and stops
            tripleline = islice(f, 3)
            # loop over these N lines, removing trailing spaces and \n
            tle = [x.rstrip() for x in tripleline]

            # only accept complete data
            # the end of the file *should* have len(tle)==0 but
            # this also handles extra junk at the end
            if len(tle) == 3:
                data.append(tle)
            else:
                break
    return data

# load_gs() function definition
def load_gs(file):
    """Returns a list of gs from a file.

    Arguments:
    file -- file name string containing unparsed gs

    gs has four elements: name, lat, lon, alt
    """
    stations = []
    with open(file) as f:
        while True:
            # an iterator that returns the next N lines and stops
            fourline = islice(f, 4)
            # loop over these N lines, removing trailing spaces and \n
            gs = [x.rstrip() for x in fourline]

            # only accept complete data
            # the end of the file *should* have len(tle)==0 but
            # this also handles extra junk at the end
            if len(gs) == 4:
                stations.append(gs)
            else:
                break
    return stations

# get_passes() function definition
def get_passes(observer, tle, start_time,
               num_passes=None, duration=None, horizon='00:00'):
    """Config obs and sat, Return pass data for all passes in given interval.

    Arguments:
    observer -- 4 element list containing desired [name,lat,lon,alt]
    tle -- 3 element list containing desired tle [line0,line1,line2]
    start_time -- ephem.date string formatted 'yyyy/mm/dd hr:min:sec'
    num_passes -- integer number of desired passes (defualt None)
    duration -- float number of hours or fraction of hours (default None)
    horizon -- str optional specification for observer horizon (defualt 0 deg)

    Specify either num_passes or duration.
    If both, use min(num_passes, duration).
    If neither, find passes for next 24 hours.
    """

    obs_name, obs_lat, obs_lon, obs_alt = observer
    tle_line0, tle_line1, tle_line2 = tle

    # Set up location of observer
    ground_station = ephem.Observer()
    ground_station.name = obs_name                # name string
    ground_station.lon = obs_lon                  # in degrees (+E)
    ground_station.lat = obs_lat                  # in degrees (+N)
    ground_station.elevation = int(obs_alt)       # in meters
    ground_station.date = ephem.date(start_time)  # in UTC
    ground_station.horizon = horizon              # in degrees

    # Read in most recent satellite TLE data
    sat = ephem.readtle(tle_line0, tle_line1, tle_line2)

    contacts = []

    if duration is None and num_passes is None:
        # get passes for next 24 hrs
        duration = 24
        # set num_passes > max passes possible in duration.
        # duration is in hours, so 4 per hour is large
        # enough for duration to break out of loop.
        num_passes = 4 * int(duration)
        # set end_time longer than suggested length for tle's
        end_time = ephem.date(ground_station.date+5*365)
    if duration is not None and num_passes is None:
        # set num_passes > max passes possible in duration.
        # duration is in hours, so 4 per hour is large
        # enough for duration to break out of loop.
        num_passes = 4 * int(duration)
        end_time = ephem.date(ground_station.date+duration*ephem.hour)
    if duration is None and num_passes is not None:
        # set end_time longer than suggested length for tle's
        end_time = ephem.date(ground_station.date+5*365)
    if num_passes is not None and duration is not None:
        # if both are given, use minimum
        end_time = ephem.date(ground_station.date+duration*ephem.hour)

    try:
        for i in range(num_passes):
            if ground_station.date > end_time:
                break
            sat.compute(ground_station)  # compute all body attributes for sat
            # next pass command yields array with [0]=rise time,
            # [1]=rise azimuth, [2]=max alt time, [3]=max alt,
            # [4]=set time, [5]=set azimuth
            info = ground_station.next_pass(sat)
            rise_time, rise_az, max_alt_time, max_alt, set_time, set_az = info
            deg_per_rad = 180.0/math.pi           # use to conv azimuth to deg
            try:
                pass_duration = timedelta(days=set_time-rise_time)  # timedelta
                r_angle = (rise_az*deg_per_rad)
                s_angle = (set_az*deg_per_rad)
            except TypeError:
                # when no set or rise time
                pass
            try:
                rising = rise_time.datetime()
                setting = set_time.datetime()
                pass_seconds = timedelta.total_seconds(pass_duration)
                tca = max_alt_time.datetime()
                max_el = (max_alt * deg_per_rad)
            except AttributeError:
                # when no set or rise time
                pass

            pass_data = {
                'start': rising,
                'end': setting,
                'duration': pass_seconds,
                'rise_az': r_angle,
                'set_az': s_angle,
                'tca': tca,
                'max_el': max_el,
            }

            try:
                # only update if set time > rise time
                if set_time > rise_time:
                    # new obs time = prev set time
                    ground_station.date = set_time
                    if ground_station.date <= end_time:
                        contacts.append(pass_data)
            except TypeError:
                pass

            # increase by 1 min and look for next pass
            ground_station.date = ground_station.date + ephem.minute
    except ValueError:
        # No (more) visible passes
        pass
    return contacts

PassTuple = namedtuple('PassTuple',
                       'start end duration rise_az set_az tca max_el gs sat')

# TODO: should this instead use a package like
# ** python3-iso8601  https://bitbucket.org/micktwomey/pyiso8601
def rfc3339_to_dt(s):
    """Convert from an RFC3339 formatted string to a datetime object
    in UTC.  Assumes UTC unless the string otherwise specifies."""
    formats = ('%Y-%m-%dT%H:%M:%SZ',  # SatNOGS API format
               '%Y-%m-%d %H:%M:%S.%f',  # str(datetime)
               '%Y-%m-%d %H:%M:%S',  # ... integral seconds
               )
    for fmt in formats:
        try:
            d = datetime.strptime(s, fmt)
        except ValueError:
            # print('format(%s) str(%s)' % (fmt, s))
            pass
        else:
            d = d.replace(tzinfo=timezone.utc)
            return d
    raise ValueError('Unknown date string format: %s' % s)

def _compute(args):
    """Wrapper for get_passes() for use with map() and converts the list of
    dicttionaries to a list of namedtuples to save RAM.
    """
    gs, sat, start, npasses, dur, horizon = args
    print(gs[0], sat[0].strip(), flush=True)
    passes = get_passes(gs, sat, start,
                        num_passes=npasses,
                        duration=dur,
                        horizon=horizon)
    # convert to namedtuples since the info doesn't change
    data = []
    for p in passes:
        d = PassTuple(str(p['start']),
                      str(p['end']),
                      p['duration'],
                      p['rise_az'],
                      p['set_az'],
                      str(p['tca']),
                      p['max_el'],
                      gs[0],
                      sat[0].rstrip())
        data.append(d)
    return data

def compute_all_passes(stations, satellites, start_time,
                       dbfile='passes.db',
                       num_passes=None, duration=None, horizon='10:00'):
    """Finds passes for all combinations of stations and satellites.

    Saves the pass info as rows in an sqlite3 database and returns the data as
    an IntervalTree with each data member set to the pass info as a namedtuple.

    horizon is a string in degrees:minutes for pyephem
    """
    import multiprocessing
    import sqlite3

    conn = sqlite3.connect(dbfile)
    cur = conn.cursor()
    cur.execute('''DROP TABLE IF EXISTS passes;''')

    # column order needs to match PassTuple order
    cur.execute('''CREATE TABLE passes
              (start text,
              end text,
              duration real,
              rise_az real,
              set_az real,
              tca text,
              max_el real,
              gs text,
              sat text);''')

    tree = intervaltree.IntervalTree()

    with multiprocessing.Pool(4) as pool:
        jobargs = product(stations,
                          satellites,
                          (start_time,),
                          (num_passes,),
                          (duration,),
                          (horizon,))
        result = pool.map(_compute, jobargs)
        print('Computed', len(result), 'Sat--GS pairs')

    # for (gs, sat) in product(stations, satellites):
        # passdata = compute(gs, sat)

    for passdata in result:
        for d in passdata:
            try:
                start = parse_date(d.start)
                end = parse_date(d.end)
                tree.addi(start, end, d)
                cur.execute(
                        'INSERT INTO passes VALUES (?,?,?,?,?,?,?,?,?);', d)
            except ValueError:
                print('!!! Invalid pass !!!')
                print(start)
                print(end)
                print(d)
    conn.commit()
    conn.close()
    return tree

def load_all_passes(dbfile='passes.db'):
    """Loads pre-computed passes from the SQLite database into an IntervalTree
    whose data is a namedtuple PassTuple.
    """
    import sqlite3

    tree = intervaltree.IntervalTree()
    conn = sqlite3.connect('file:' + dbfile + '?mode=ro', uri=True)
    conn.row_factory = sqlite3.Row
    cur = conn.cursor()

    for p in cur.execute('''SELECT * FROM passes;'''):
        data = PassTuple(p['start'], p['end'], p['duration'],
                         p['rise_az'], p['set_az'],
                         p['tca'], p['max_el'],
                         p['gs'], p['sat'])
        start = parse_date(data.start)
        end = parse_date(data.end)
        tree.addi(start, end, data)
    conn.close()
    return tree

# calc_access_time() function definition
def calc_access_time(start, gs, tle, days, horizon='00:00'):
    """Calculates Access Time in seconds/day.

    Arguments:
    start -- string formatted 'yyyy/mm/dd HH:MM:SS'
    gs -- 4 element list containing desired [name,lat,lon,alt]
    tle -- 3 element list containing desired tle [line0,line1,line2]
    days -- num of day to calc/plot access time
    horizon -- str optional specification for observer horizon (defualt 0 deg)
    """
    time = days
    start_time = ephem.date(start)
    access_list = []
    day_list = []

    for days in range(time):
        num_passes = None
        duration = 24.0
        gs_passes = {}

        gs_passes[tle.noradid] = get_passes(gs, tle.tle, start_time,
                                            num_passes=num_passes,
                                            duration=duration,
                                            horizon=horizon)

        access_time = 0
        for sat, passes in gs_passes.items():
            for obs in passes:
                access_time = access_time + obs['duration']
        access_list.append(access_time)
        day_list.append(days)
        start_time = start_time + 1
    return day_list, access_list

# plot_access_time() function definition
def plot_access_time(start, gs, tle, days, horizon='00:00'):
    """Plots Access Time in seconds/day.

    Arguments:
    start -- string formatted 'yyyy/mm/dd HH:MM:SS'
    gs -- 4 element list containing desired [name,lat,lon,alt]
    tle -- 3 element list containing desired tle [line0,line1,line2]
    days -- num of day to calc/plot access time
    horizon -- str optional specification for observer horizon (defualt 0 deg)
    """
    tle = TLE(tle)

    day_list, access_list = calc_access_time(start, gs, tle, days,
                                             horizon=horizon)

    fig = plt.figure(1)
    fig.suptitle('%s Access time for %s GS' % (tle.name, gs[0]))

    s1 = plt.subplot(221)
    s1.plot(day_list, access_list, 'b.')
    plt.xlabel('Days from %s' % (start))
    plt.ylabel('Access time (sec/day)')

    plt.subplot(222)
    plt.plot(day_list, access_list, 'b-')
    plt.xlabel('Days from %s' % (start))

    plt.show()
<commit_msg>Imports to top of file<commit_after># Python module for scheduling bazaar project
# Includes: TLE class
#           load_gs()
#           load_tles()
#           get_passes()
#           calc_access_time()
#           plot_access_time()

# Import necessary libraries
from collections import namedtuple
from datetime import datetime, timedelta, timezone
from itertools import islice, product
import math
import multiprocessing
import sqlite3

import ephem
from iso8601 import parse_date
import intervaltree
import matplotlib.pyplot as plt
# import seaborn

# TLE class definition
class TLE:
    """TLE class used for various tle attributes.

    Args: a three element list containing tle lines
    """
    def __init__(self, tle):
        self.tle = tle
        self.tle0 = tle[0]
        self.tle1 = tle[1]
        self.tle2 = tle[2]
        self.name = tle[0].rstrip()
        self.noradid = tle[1][2:7]
        self.epoch = tle[1][18:32]
        self.inclination = tle[2][8:16]

    def __str__(self):
        return '%s\n%s\n%s' % (self.tle0, self.tle1, self.tle2)

# load_tles() function definition
def load_tles(file):
    """Returns a list of tle's from a file.

    Arguments:
    file -- file name string containing unparsed tles
    """
    data = []
    with open(file) as f:
        while True:
            # an iterator that returns the next N lines and stops
            tripleline = islice(f, 3)
            # loop over these N lines, removing trailing spaces and \n
            tle = [x.rstrip() for x in tripleline]

            # only accept complete data
            # the end of the file *should* have len(tle)==0 but
            # this also handles extra junk at the end
            if len(tle) == 3:
                data.append(tle)
            else:
                break
    return data

# load_gs() function definition
def load_gs(file):
    """Returns a list of gs from a file.

    Arguments:
    file -- file name string containing unparsed gs

    gs has four elements: name, lat, lon, alt
    """
    stations = []
    with open(file) as f:
        while True:
            # an iterator that returns the next N lines and stops
            fourline = islice(f, 4)
            # loop over these N lines, removing trailing spaces and \n
            gs = [x.rstrip() for x in fourline]

            # only accept complete data
            # the end of the file *should* have len(tle)==0 but
            # this also handles extra junk at the end
            if len(gs) == 4:
                stations.append(gs)
            else:
                break
    return stations

# get_passes() function definition
def get_passes(observer, tle, start_time,
               num_passes=None, duration=None, horizon='00:00'):
    """Config obs and sat, Return pass data for all passes in given interval.

    Arguments:
    observer -- 4 element list containing desired [name,lat,lon,alt]
    tle -- 3 element list containing desired tle [line0,line1,line2]
    start_time -- ephem.date string formatted 'yyyy/mm/dd hr:min:sec'
    num_passes -- integer number of desired passes (defualt None)
    duration -- float number of hours or fraction of hours (default None)
    horizon -- str optional specification for observer horizon (defualt 0 deg)

    Specify either num_passes or duration.
    If both, use min(num_passes, duration).
    If neither, find passes for next 24 hours.
    """

    obs_name, obs_lat, obs_lon, obs_alt = observer
    tle_line0, tle_line1, tle_line2 = tle

    # Set up location of observer
    ground_station = ephem.Observer()
    ground_station.name = obs_name                # name string
    ground_station.lon = obs_lon                  # in degrees (+E)
    ground_station.lat = obs_lat                  # in degrees (+N)
    ground_station.elevation = int(obs_alt)       # in meters
    ground_station.date = ephem.date(start_time)  # in UTC
    ground_station.horizon = horizon              # in degrees

    # Read in most recent satellite TLE data
    sat = ephem.readtle(tle_line0, tle_line1, tle_line2)

    contacts = []

    if duration is None and num_passes is None:
        # get passes for next 24 hrs
        duration = 24
        # set num_passes > max passes possible in duration.
        # duration is in hours, so 4 per hour is large
        # enough for duration to break out of loop.
        num_passes = 4 * int(duration)
        # set end_time longer than suggested length for tle's
        end_time = ephem.date(ground_station.date+5*365)
    if duration is not None and num_passes is None:
        # set num_passes > max passes possible in duration.
        # duration is in hours, so 4 per hour is large
        # enough for duration to break out of loop.
        num_passes = 4 * int(duration)
        end_time = ephem.date(ground_station.date+duration*ephem.hour)
    if duration is None and num_passes is not None:
        # set end_time longer than suggested length for tle's
        end_time = ephem.date(ground_station.date+5*365)
    if num_passes is not None and duration is not None:
        # if both are given, use minimum
        end_time = ephem.date(ground_station.date+duration*ephem.hour)

    try:
        for i in range(num_passes):
            if ground_station.date > end_time:
                break
            sat.compute(ground_station)  # compute all body attributes for sat
            # next pass command yields array with [0]=rise time,
            # [1]=rise azimuth, [2]=max alt time, [3]=max alt,
            # [4]=set time, [5]=set azimuth
            info = ground_station.next_pass(sat)
            rise_time, rise_az, max_alt_time, max_alt, set_time, set_az = info
            deg_per_rad = 180.0/math.pi           # use to conv azimuth to deg
            try:
                pass_duration = timedelta(days=set_time-rise_time)  # timedelta
                r_angle = (rise_az*deg_per_rad)
                s_angle = (set_az*deg_per_rad)
            except TypeError:
                # when no set or rise time
                pass
            try:
                rising = rise_time.datetime()
                setting = set_time.datetime()
                pass_seconds = timedelta.total_seconds(pass_duration)
                tca = max_alt_time.datetime()
                max_el = (max_alt * deg_per_rad)
            except AttributeError:
                # when no set or rise time
                pass

            pass_data = {
                'start': rising,
                'end': setting,
                'duration': pass_seconds,
                'rise_az': r_angle,
                'set_az': s_angle,
                'tca': tca,
                'max_el': max_el,
            }

            try:
                # only update if set time > rise time
                if set_time > rise_time:
                    # new obs time = prev set time
                    ground_station.date = set_time
                    if ground_station.date <= end_time:
                        contacts.append(pass_data)
            except TypeError:
                pass

            # increase by 1 min and look for next pass
            ground_station.date = ground_station.date + ephem.minute
    except ValueError:
        # No (more) visible passes
        pass
    return contacts

PassTuple = namedtuple('PassTuple',
                       'start end duration rise_az set_az tca max_el gs sat')

# TODO: should this instead use a package like
# ** python3-iso8601  https://bitbucket.org/micktwomey/pyiso8601
def rfc3339_to_dt(s):
    """Convert from an RFC3339 formatted string to a datetime object
    in UTC.  Assumes UTC unless the string otherwise specifies."""
    formats = ('%Y-%m-%dT%H:%M:%SZ',  # SatNOGS API format
               '%Y-%m-%d %H:%M:%S.%f',  # str(datetime)
               '%Y-%m-%d %H:%M:%S',  # ... integral seconds
               )
    for fmt in formats:
        try:
            d = datetime.strptime(s, fmt)
        except ValueError:
            # print('format(%s) str(%s)' % (fmt, s))
            pass
        else:
            d = d.replace(tzinfo=timezone.utc)
            return d
    raise ValueError('Unknown date string format: %s' % s)

def _compute(args):
    """Wrapper for get_passes() for use with map() and converts the list of
    dicttionaries to a list of namedtuples to save RAM.
    """
    gs, sat, start, npasses, dur, horizon = args
    print(gs[0], sat[0].strip(), flush=True)
    passes = get_passes(gs, sat, start,
                        num_passes=npasses,
                        duration=dur,
                        horizon=horizon)
    # convert to namedtuples since the info doesn't change
    data = []
    for p in passes:
        d = PassTuple(str(p['start']),
                      str(p['end']),
                      p['duration'],
                      p['rise_az'],
                      p['set_az'],
                      str(p['tca']),
                      p['max_el'],
                      gs[0],
                      sat[0].rstrip())
        data.append(d)
    return data

def compute_all_passes(stations, satellites, start_time,
                       dbfile='passes.db',
                       num_passes=None, duration=None, horizon='10:00'):
    """Finds passes for all combinations of stations and satellites.

    Saves the pass info as rows in an sqlite3 database and returns the data as
    an IntervalTree with each data member set to the pass info as a namedtuple.

    horizon is a string in degrees:minutes for pyephem
    """
    conn = sqlite3.connect(dbfile)
    cur = conn.cursor()
    cur.execute('''DROP TABLE IF EXISTS passes;''')

    # column order needs to match PassTuple order
    cur.execute('''CREATE TABLE passes
              (start text,
              end text,
              duration real,
              rise_az real,
              set_az real,
              tca text,
              max_el real,
              gs text,
              sat text);''')

    tree = intervaltree.IntervalTree()

    with multiprocessing.Pool(4) as pool:
        jobargs = product(stations,
                          satellites,
                          (start_time,),
                          (num_passes,),
                          (duration,),
                          (horizon,))
        result = pool.map(_compute, jobargs)
        print('Computed', len(result), 'Sat--GS pairs')

    # for (gs, sat) in product(stations, satellites):
        # passdata = compute(gs, sat)

    for passdata in result:
        for d in passdata:
            try:
                start = parse_date(d.start)
                end = parse_date(d.end)
                tree.addi(start, end, d)
                cur.execute(
                        'INSERT INTO passes VALUES (?,?,?,?,?,?,?,?,?);', d)
            except ValueError:
                print('!!! Invalid pass !!!')
                print(start)
                print(end)
                print(d)
    conn.commit()
    conn.close()
    return tree

def load_all_passes(dbfile='passes.db'):
    """Loads pre-computed passes from the SQLite database into an IntervalTree
    whose data is a namedtuple PassTuple.
    """
    tree = intervaltree.IntervalTree()
    conn = sqlite3.connect('file:' + dbfile + '?mode=ro', uri=True)
    conn.row_factory = sqlite3.Row
    cur = conn.cursor()

    for p in cur.execute('''SELECT * FROM passes;'''):
        data = PassTuple(p['start'], p['end'], p['duration'],
                         p['rise_az'], p['set_az'],
                         p['tca'], p['max_el'],
                         p['gs'], p['sat'])
        start = parse_date(data.start)
        end = parse_date(data.end)
        tree.addi(start, end, data)
    conn.close()
    return tree

# calc_access_time() function definition
def calc_access_time(start, gs, tle, days, horizon='00:00'):
    """Calculates Access Time in seconds/day.

    Arguments:
    start -- string formatted 'yyyy/mm/dd HH:MM:SS'
    gs -- 4 element list containing desired [name,lat,lon,alt]
    tle -- 3 element list containing desired tle [line0,line1,line2]
    days -- num of day to calc/plot access time
    horizon -- str optional specification for observer horizon (defualt 0 deg)
    """
    time = days
    start_time = ephem.date(start)
    access_list = []
    day_list = []

    for days in range(time):
        num_passes = None
        duration = 24.0
        gs_passes = {}

        gs_passes[tle.noradid] = get_passes(gs, tle.tle, start_time,
                                            num_passes=num_passes,
                                            duration=duration,
                                            horizon=horizon)

        access_time = 0
        for sat, passes in gs_passes.items():
            for obs in passes:
                access_time = access_time + obs['duration']
        access_list.append(access_time)
        day_list.append(days)
        start_time = start_time + 1
    return day_list, access_list

# plot_access_time() function definition
def plot_access_time(start, gs, tle, days, horizon='00:00'):
    """Plots Access Time in seconds/day.

    Arguments:
    start -- string formatted 'yyyy/mm/dd HH:MM:SS'
    gs -- 4 element list containing desired [name,lat,lon,alt]
    tle -- 3 element list containing desired tle [line0,line1,line2]
    days -- num of day to calc/plot access time
    horizon -- str optional specification for observer horizon (defualt 0 deg)
    """
    tle = TLE(tle)

    day_list, access_list = calc_access_time(start, gs, tle, days,
                                             horizon=horizon)

    fig = plt.figure(1)
    fig.suptitle('%s Access time for %s GS' % (tle.name, gs[0]))

    s1 = plt.subplot(221)
    s1.plot(day_list, access_list, 'b.')
    plt.xlabel('Days from %s' % (start))
    plt.ylabel('Access time (sec/day)')

    plt.subplot(222)
    plt.plot(day_list, access_list, 'b-')
    plt.xlabel('Days from %s' % (start))

    plt.show()
<|endoftext|>


<commit_before># pylint: disable= invalid-name,  unused-import
"""For compatibility and optional dependencies."""
import importlib.util
import logging
import sys
import types
from typing import Any, Dict, List, Optional, Sequence, Type, cast

import numpy as np

from ._typing import _T

assert sys.version_info[0] == 3, "Python 2 is no longer supported."

def py_str(x: bytes) -> str:
    """convert c string back to python string"""
    return x.decode("utf-8")  # type: ignore

def lazy_isinstance(instance: Any, module: str, name: str) -> bool:
    """Use string representation to identify a type."""

    # Notice, we use .__class__ as opposed to type() in order
    # to support object proxies such as weakref.proxy
    cls = instance.__class__
    is_same_module = cls.__module__ == module
    has_same_name = cls.__name__ == name
    return is_same_module and has_same_name

# pandas
try:
    from pandas import DataFrame, MultiIndex, Series
    from pandas import concat as pandas_concat

    PANDAS_INSTALLED = True
except ImportError:

    MultiIndex = object
    DataFrame = object
    Series = object
    pandas_concat = None
    PANDAS_INSTALLED = False

# sklearn
try:
    from sklearn.base import BaseEstimator as XGBModelBase
    from sklearn.base import ClassifierMixin as XGBClassifierBase
    from sklearn.base import RegressorMixin as XGBRegressorBase
    from sklearn.preprocessing import LabelEncoder

    try:
        from sklearn.model_selection import KFold as XGBKFold
        from sklearn.model_selection import StratifiedKFold as XGBStratifiedKFold
    except ImportError:
        from sklearn.cross_validation import KFold as XGBKFold
        from sklearn.cross_validation import StratifiedKFold as XGBStratifiedKFold

    SKLEARN_INSTALLED = True

except ImportError:
    SKLEARN_INSTALLED = False

    # used for compatibility without sklearn
    XGBModelBase = object
    XGBClassifierBase = object
    XGBRegressorBase = object
    LabelEncoder = object

    XGBKFold = None
    XGBStratifiedKFold = None

class XGBoostLabelEncoder(LabelEncoder):
    """Label encoder with JSON serialization methods."""

    def to_json(self) -> Dict:
        """Returns a JSON compatible dictionary"""
        meta = {}
        for k, v in self.__dict__.items():
            if isinstance(v, np.ndarray):
                meta[k] = v.tolist()
            else:
                meta[k] = v
        return meta

    def from_json(self, doc: Dict) -> None:
        # pylint: disable=attribute-defined-outside-init
        """Load the encoder back from a JSON compatible dict."""
        meta = {}
        for k, v in doc.items():
            if k == "classes_":
                self.classes_ = np.array(v)
                continue
            meta[k] = v
        self.__dict__.update(meta)

try:
    import scipy.sparse as scipy_sparse
    from scipy.sparse import csr_matrix as scipy_csr
except ImportError:
    scipy_sparse = False
    scipy_csr = object

def concat(value: Sequence[_T]) -> _T:  # pylint: disable=too-many-return-statements
    """Concatenate row-wise."""
    if isinstance(value[0], np.ndarray):
        value_arr = cast(Sequence[np.ndarray], value)
        return np.concatenate(value_arr, axis=0)
    if scipy_sparse and isinstance(value[0], scipy_sparse.csr_matrix):
        return scipy_sparse.vstack(value, format="csr")
    if scipy_sparse and isinstance(value[0], scipy_sparse.csc_matrix):
        return scipy_sparse.vstack(value, format="csc")
    if scipy_sparse and isinstance(value[0], scipy_sparse.spmatrix):
        # other sparse format will be converted to CSR.
        return scipy_sparse.vstack(value, format="csr")
    if PANDAS_INSTALLED and isinstance(value[0], (DataFrame, Series)):
        return pandas_concat(value, axis=0)
    if lazy_isinstance(value[0], "cudf.core.dataframe", "DataFrame") or lazy_isinstance(
        value[0], "cudf.core.series", "Series"
    ):
        from cudf import concat as CUDF_concat  # pylint: disable=import-error

        return CUDF_concat(value, axis=0)
    if lazy_isinstance(value[0], "cupy._core.core", "ndarray"):
        import cupy  # pylint: disable=import-error

        # pylint: disable=c-extension-no-member,no-member
        d = cupy.cuda.runtime.getDevice()
        for v in value:
            arr = cast(cupy.ndarray, v)
            d_v = arr.device.id
            assert d_v == d, "Concatenating arrays on different devices."
        return cupy.concatenate(value, axis=0)
    raise TypeError("Unknown type.")

# Modified from tensorflow with added caching.  There's a `LazyLoader` in
# `importlib.utils`, except it's unclear from its document on how to use it.  This one
# seems to be easy to understand and works out of box.

# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this
# file except in compliance with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under
# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the specific language governing
# permissions and limitations under the License.
class LazyLoader(types.ModuleType):
    """Lazily import a module, mainly to avoid pulling in large dependencies."""

    def __init__(
        self,
        local_name: str,
        parent_module_globals: Dict,
        name: str,
        warning: Optional[str] = None,
    ) -> None:
        self._local_name = local_name
        self._parent_module_globals = parent_module_globals
        self._warning = warning
        self.module: Optional[types.ModuleType] = None

        super().__init__(name)

    def _load(self) -> types.ModuleType:
        """Load the module and insert it into the parent's globals."""
        # Import the target module and insert it into the parent's namespace
        module = importlib.import_module(self.__name__)
        self._parent_module_globals[self._local_name] = module

        # Emit a warning if one was specified
        if self._warning:
            logging.warning(self._warning)
            # Make sure to only warn once.
        self._warning = None

        # Update this object's dict so that if someone keeps a reference to the
        #   LazyLoader, lookups are efficient (__getattr__ is only called on lookups
        #   that fail).
        self.__dict__.update(module.__dict__)

        return module

    def __getattr__(self, item: str) -> Any:
        if not self.module:
            self.module = self._load()
        return getattr(self.module, item)

    def __dir__(self) -> List[str]:
        if not self.module:
            self.module = self._load()
        return dir(self.module)
<commit_msg>Fix dask interface with latest cupy. (#8210)<commit_after># pylint: disable= invalid-name,  unused-import
"""For compatibility and optional dependencies."""
import importlib.util
import logging
import sys
import types
from typing import Any, Dict, List, Optional, Sequence, cast

import numpy as np

from ._typing import _T

assert sys.version_info[0] == 3, "Python 2 is no longer supported."

def py_str(x: bytes) -> str:
    """convert c string back to python string"""
    return x.decode("utf-8")  # type: ignore

def lazy_isinstance(instance: Any, module: str, name: str) -> bool:
    """Use string representation to identify a type."""

    # Notice, we use .__class__ as opposed to type() in order
    # to support object proxies such as weakref.proxy
    cls = instance.__class__
    is_same_module = cls.__module__ == module
    has_same_name = cls.__name__ == name
    return is_same_module and has_same_name

# pandas
try:
    from pandas import DataFrame, MultiIndex, Series
    from pandas import concat as pandas_concat

    PANDAS_INSTALLED = True
except ImportError:

    MultiIndex = object
    DataFrame = object
    Series = object
    pandas_concat = None
    PANDAS_INSTALLED = False

# sklearn
try:
    from sklearn.base import BaseEstimator as XGBModelBase
    from sklearn.base import ClassifierMixin as XGBClassifierBase
    from sklearn.base import RegressorMixin as XGBRegressorBase
    from sklearn.preprocessing import LabelEncoder

    try:
        from sklearn.model_selection import KFold as XGBKFold
        from sklearn.model_selection import StratifiedKFold as XGBStratifiedKFold
    except ImportError:
        from sklearn.cross_validation import KFold as XGBKFold
        from sklearn.cross_validation import StratifiedKFold as XGBStratifiedKFold

    SKLEARN_INSTALLED = True

except ImportError:
    SKLEARN_INSTALLED = False

    # used for compatibility without sklearn
    XGBModelBase = object
    XGBClassifierBase = object
    XGBRegressorBase = object
    LabelEncoder = object

    XGBKFold = None
    XGBStratifiedKFold = None

class XGBoostLabelEncoder(LabelEncoder):
    """Label encoder with JSON serialization methods."""

    def to_json(self) -> Dict:
        """Returns a JSON compatible dictionary"""
        meta = {}
        for k, v in self.__dict__.items():
            if isinstance(v, np.ndarray):
                meta[k] = v.tolist()
            else:
                meta[k] = v
        return meta

    def from_json(self, doc: Dict) -> None:
        # pylint: disable=attribute-defined-outside-init
        """Load the encoder back from a JSON compatible dict."""
        meta = {}
        for k, v in doc.items():
            if k == "classes_":
                self.classes_ = np.array(v)
                continue
            meta[k] = v
        self.__dict__.update(meta)

try:
    import scipy.sparse as scipy_sparse
    from scipy.sparse import csr_matrix as scipy_csr
except ImportError:
    scipy_sparse = False
    scipy_csr = object

def concat(value: Sequence[_T]) -> _T:  # pylint: disable=too-many-return-statements
    """Concatenate row-wise."""
    if isinstance(value[0], np.ndarray):
        value_arr = cast(Sequence[np.ndarray], value)
        return np.concatenate(value_arr, axis=0)
    if scipy_sparse and isinstance(value[0], scipy_sparse.csr_matrix):
        return scipy_sparse.vstack(value, format="csr")
    if scipy_sparse and isinstance(value[0], scipy_sparse.csc_matrix):
        return scipy_sparse.vstack(value, format="csc")
    if scipy_sparse and isinstance(value[0], scipy_sparse.spmatrix):
        # other sparse format will be converted to CSR.
        return scipy_sparse.vstack(value, format="csr")
    if PANDAS_INSTALLED and isinstance(value[0], (DataFrame, Series)):
        return pandas_concat(value, axis=0)
    if lazy_isinstance(value[0], "cudf.core.dataframe", "DataFrame") or lazy_isinstance(
        value[0], "cudf.core.series", "Series"
    ):
        from cudf import concat as CUDF_concat  # pylint: disable=import-error

        return CUDF_concat(value, axis=0)
    from .data import _is_cupy_array

    if _is_cupy_array(value[0]):
        import cupy  # pylint: disable=import-error

        # pylint: disable=c-extension-no-member,no-member
        d = cupy.cuda.runtime.getDevice()
        for v in value:
            arr = cast(cupy.ndarray, v)
            d_v = arr.device.id
            assert d_v == d, "Concatenating arrays on different devices."
        return cupy.concatenate(value, axis=0)
    raise TypeError("Unknown type.")

# Modified from tensorflow with added caching.  There's a `LazyLoader` in
# `importlib.utils`, except it's unclear from its document on how to use it.  This one
# seems to be easy to understand and works out of box.

# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this
# file except in compliance with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under
# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the specific language governing
# permissions and limitations under the License.
class LazyLoader(types.ModuleType):
    """Lazily import a module, mainly to avoid pulling in large dependencies."""

    def __init__(
        self,
        local_name: str,
        parent_module_globals: Dict,
        name: str,
        warning: Optional[str] = None,
    ) -> None:
        self._local_name = local_name
        self._parent_module_globals = parent_module_globals
        self._warning = warning
        self.module: Optional[types.ModuleType] = None

        super().__init__(name)

    def _load(self) -> types.ModuleType:
        """Load the module and insert it into the parent's globals."""
        # Import the target module and insert it into the parent's namespace
        module = importlib.import_module(self.__name__)
        self._parent_module_globals[self._local_name] = module

        # Emit a warning if one was specified
        if self._warning:
            logging.warning(self._warning)
            # Make sure to only warn once.
        self._warning = None

        # Update this object's dict so that if someone keeps a reference to the
        #   LazyLoader, lookups are efficient (__getattr__ is only called on lookups
        #   that fail).
        self.__dict__.update(module.__dict__)

        return module

    def __getattr__(self, item: str) -> Any:
        if not self.module:
            self.module = self._load()
        return getattr(self.module, item)

    def __dir__(self) -> List[str]:
        if not self.module:
            self.module = self._load()
        return dir(self.module)
<|endoftext|>


<commit_before>##########################################################################
#
#  Copyright (c) 2011-2012, John Haddon. All rights reserved.
#  Copyright (c) 2011-2013, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#      * Redistributions of source code must retain the above
#        copyright notice, this list of conditions and the following
#        disclaimer.
#
#      * Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided with
#        the distribution.
#
#      * Neither the name of John Haddon nor the names of
#        any other contributors to this software may be used to endorse or
#        promote products derived from this software without specific prior
#        written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

import IECore

import Gaffer
import GafferUI

QtCore = GafferUI._qtImport( "QtCore" )

## The NodeSetEditor is a base class for all Editors which focus their
# editing on a subset of nodes beneath a ScriptNode. This set defaults
# to the ScriptNode.selection() but can be modified to be any Set of nodes.
class NodeSetEditor( GafferUI.EditorWidget ) :

	def __init__( self, topLevelWidget, scriptNode, **kw ) :

		self.__nodeSet = Gaffer.StandardSet()
		self.__nodeSetChangedSignal = GafferUI.WidgetSignal()

		GafferUI.EditorWidget.__init__( self, topLevelWidget, scriptNode, **kw )

		self.__titleFormat = None

		self.__updateScheduled = False
		# allow derived classes to call _updateFromSet() themselves after construction,
		# to avoid being called when they're only half constructed.
		self.__setNodeSetInternal( self.scriptNode().selection(), callUpdateFromSet=False )

	def setNodeSet( self, nodeSet ) :

		self.__setNodeSetInternal( nodeSet, callUpdateFromSet=True )

	def getNodeSet( self ) :

		return self.__nodeSet

	def nodeSetChangedSignal( self ) :

		return self.__nodeSetChangedSignal

	## Overridden to display the names of the nodes being edited.
	# Derived classes should override _titleFormat() rather than
	# reimplement this again.
	def getTitle( self ) :

		t = GafferUI.EditorWidget.getTitle( self )
		if t :
			return t

		if self.__titleFormat is None :
			self.__titleFormat = self._titleFormat()
			self.__nameChangedConnections = []
			for n in self.__titleFormat :
				if isinstance( n, Gaffer.GraphComponent ) :
					self.__nameChangedConnections.append( n.nameChangedSignal().connect( Gaffer.WeakMethod( self.__nameChanged ) ) )

		result = ""
		for t in self.__titleFormat :
			if isinstance( t, basestring ) :
				result += t
			else :
				result += t.getName()

		return result

	def _lastAddedNode( self ) :

		if len( self.__nodeSet ) :
			return self.__nodeSet[-1]

		return None

	## Called whenever the contents of getNodeSet() have changed - must be
	# implemented by derived classes to update the UI appropriately. Implementations
	# must first call the base class implementation.
	def _updateFromSet( self ) :

		# flush information needed for making the title -
		# we'll update it lazily in getTitle().
		self.__nameChangedConnections = []
		self.__titleFormat = None

		self.titleChangedSignal()( self )

	## May be reimplemented by derived classes to specify a combination of
	# strings and node names to use in building the title. The NodeSetEditor
	# will take care of updating the title appropriately as the nodes are renamed.
	def _titleFormat( self, _prefix = None, _maxNodes = 2, _reverseNodes = False, _ellipsis = True ) :

		if _prefix is None :
			result = [ IECore.CamelCase.toSpaced( self.__class__.__name__ ) ]
		else :
			result = [ _prefix ]

		numNames = min( _maxNodes, len( self.__nodeSet ) )
		if numNames :

			result.append( " : " )

			if _reverseNodes :
				nodes = self.__nodeSet[len(self.__nodeSet)-numNames:]
				nodes.reverse()
			else :
				nodes = self.__nodeSet[:numNames]

			for i, node in enumerate( nodes ) :
				result.append( node )
				if i < numNames - 1 :
					result.append( ", " )

			if _ellipsis and len( self.__nodeSet ) > _maxNodes :
				result.append( "..." )

		return result

	def __setNodeSetInternal( self, nodeSet, callUpdateFromSet ) :

		prevSet = self.__nodeSet
		self.__nodeSet = nodeSet
		self.__memberAddedConnection = self.__nodeSet.memberAddedSignal().connect( Gaffer.WeakMethod( self.__membersChanged ) )
		self.__memberRemovedConnection = self.__nodeSet.memberRemovedSignal().connect( Gaffer.WeakMethod( self.__membersChanged ) )

		if callUpdateFromSet :
			# only update if the nodes being held have actually changed,
			# so we don't get unnecessary flicker in any of the uis.
			needsUpdate = len( prevSet ) != len( self.__nodeSet )
			if not needsUpdate :
				for i in range( 0, len( prevSet ) ) :
					if not prevSet[i].isSame( self.__nodeSet[i] ) :
						needsUpdate = True
						break
			if needsUpdate :
				self._updateFromSet()

		self.__nodeSetChangedSignal( self )

	def __nameChanged( self, node ) :

		self.titleChangedSignal()( self )

	def __membersChanged( self, set, member ) :

		if self.__updateScheduled :
			return

		QtCore.QTimer.singleShot( 0, self.__updateTimeout )
		self.__updateScheduled = True

	def __updateTimeout( self ) :

		self.__updateScheduled = False
		self._updateFromSet()
<commit_msg>Fixed bug where pinned nodes were still visible in the UI after being deleted.<commit_after>##########################################################################
#
#  Copyright (c) 2011-2012, John Haddon. All rights reserved.
#  Copyright (c) 2011-2013, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#      * Redistributions of source code must retain the above
#        copyright notice, this list of conditions and the following
#        disclaimer.
#
#      * Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided with
#        the distribution.
#
#      * Neither the name of John Haddon nor the names of
#        any other contributors to this software may be used to endorse or
#        promote products derived from this software without specific prior
#        written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

import IECore

import Gaffer
import GafferUI

QtCore = GafferUI._qtImport( "QtCore" )

## The NodeSetEditor is a base class for all Editors which focus their
# editing on a subset of nodes beneath a ScriptNode. This set defaults
# to the ScriptNode.selection() but can be modified to be any Set of nodes.
class NodeSetEditor( GafferUI.EditorWidget ) :

	def __init__( self, topLevelWidget, scriptNode, **kw ) :

		self.__nodeSet = Gaffer.StandardSet()
		self.__nodeSetChangedSignal = GafferUI.WidgetSignal()

		GafferUI.EditorWidget.__init__( self, topLevelWidget, scriptNode, **kw )

		self.__titleFormat = None

		self.__updateScheduled = False
		# allow derived classes to call _updateFromSet() themselves after construction,
		# to avoid being called when they're only half constructed.
		self.__setNodeSetInternal( self.scriptNode().selection(), callUpdateFromSet=False )

	## Sets the nodes that will be displayed by this editor. As members are
	# added to and removed from the set, the UI will be updated automatically
	# to show them. If the set is not scriptNode.selection(), then an OrphanRemover
	# will be applied automatically to the set so that deleted nodes are not
	# visible in the UI.
	def setNodeSet( self, nodeSet ) :

		self.__setNodeSetInternal( nodeSet, callUpdateFromSet=True )

	def getNodeSet( self ) :

		return self.__nodeSet

	def nodeSetChangedSignal( self ) :

		return self.__nodeSetChangedSignal

	## Overridden to display the names of the nodes being edited.
	# Derived classes should override _titleFormat() rather than
	# reimplement this again.
	def getTitle( self ) :

		t = GafferUI.EditorWidget.getTitle( self )
		if t :
			return t

		if self.__titleFormat is None :
			self.__titleFormat = self._titleFormat()
			self.__nameChangedConnections = []
			for n in self.__titleFormat :
				if isinstance( n, Gaffer.GraphComponent ) :
					self.__nameChangedConnections.append( n.nameChangedSignal().connect( Gaffer.WeakMethod( self.__nameChanged ) ) )

		result = ""
		for t in self.__titleFormat :
			if isinstance( t, basestring ) :
				result += t
			else :
				result += t.getName()

		return result

	def _lastAddedNode( self ) :

		if len( self.__nodeSet ) :
			return self.__nodeSet[-1]

		return None

	## Called whenever the contents of getNodeSet() have changed - must be
	# implemented by derived classes to update the UI appropriately. Implementations
	# must first call the base class implementation.
	def _updateFromSet( self ) :

		# flush information needed for making the title -
		# we'll update it lazily in getTitle().
		self.__nameChangedConnections = []
		self.__titleFormat = None

		self.titleChangedSignal()( self )

	## May be reimplemented by derived classes to specify a combination of
	# strings and node names to use in building the title. The NodeSetEditor
	# will take care of updating the title appropriately as the nodes are renamed.
	def _titleFormat( self, _prefix = None, _maxNodes = 2, _reverseNodes = False, _ellipsis = True ) :

		if _prefix is None :
			result = [ IECore.CamelCase.toSpaced( self.__class__.__name__ ) ]
		else :
			result = [ _prefix ]

		numNames = min( _maxNodes, len( self.__nodeSet ) )
		if numNames :

			result.append( " : " )

			if _reverseNodes :
				nodes = self.__nodeSet[len(self.__nodeSet)-numNames:]
				nodes.reverse()
			else :
				nodes = self.__nodeSet[:numNames]

			for i, node in enumerate( nodes ) :
				result.append( node )
				if i < numNames - 1 :
					result.append( ", " )

			if _ellipsis and len( self.__nodeSet ) > _maxNodes :
				result.append( "..." )

		return result

	def __setNodeSetInternal( self, nodeSet, callUpdateFromSet ) :

		if self.__nodeSet.isSame( nodeSet ) :
			return

		prevSet = self.__nodeSet
		self.__nodeSet = nodeSet
		self.__memberAddedConnection = self.__nodeSet.memberAddedSignal().connect( Gaffer.WeakMethod( self.__membersChanged ) )
		self.__memberRemovedConnection = self.__nodeSet.memberRemovedSignal().connect( Gaffer.WeakMethod( self.__membersChanged ) )

		if not self.__nodeSet.isSame( self.scriptNode().selection() ) :
			self.__orphanRemover = Gaffer.Behaviours.OrphanRemover( self.__nodeSet )
		else :
			self.__orphanRemover = None

		if callUpdateFromSet :
			# only update if the nodes being held have actually changed,
			# so we don't get unnecessary flicker in any of the uis.
			needsUpdate = len( prevSet ) != len( self.__nodeSet )
			if not needsUpdate :
				for i in range( 0, len( prevSet ) ) :
					if not prevSet[i].isSame( self.__nodeSet[i] ) :
						needsUpdate = True
						break
			if needsUpdate :
				self._updateFromSet()

		self.__nodeSetChangedSignal( self )

	def __nameChanged( self, node ) :

		self.titleChangedSignal()( self )

	def __membersChanged( self, set, member ) :

		if self.__updateScheduled :
			return

		QtCore.QTimer.singleShot( 0, self.__updateTimeout )
		self.__updateScheduled = True

	def __updateTimeout( self ) :

		self.__updateScheduled = False
		self._updateFromSet()
<|endoftext|>


<commit_before>import numpy as np

from opensfm import context
from opensfm import types

"""
Trying to imitate the following structure

reconstruction = {
    "cameras": {
        "theta": {
            "projection_type": "equirectangular"
        }
    },

    "shots" : {
        'im1': {
            "camera": "theta",
            "rotation": [0.0, 0.0, 0.0],
            "translation": [0.0, 0.0, 0.0],
        },
        'im2': {
            "camera": "theta",
            "rotation": [0, 0, 0.0],
            "translation": [-1, 0, 0.0],
        },
    },

    "points" : {
    },
}
"""

def test_reconstruction_class_initialization():

    # Instantiate Reconstruction
    reconstruction = types.Reconstruction()

    # Instantiate camera instrinsics
    camera = types.PerspectiveCamera()
    camera.id = 'apple iphone 4s back camera 4.28mm f/2.4'
    camera.focal = 0.9722222222222222
    camera.k1 = 0.006094395128698237
    camera.k2 = -0.0004952058188617129
    camera.height = 2448
    camera.width = 3264

    # Instantiate GPS data
    metadata = types.ShotMetadata()
    metadata.orientation = 1
    metadata.capture_time = 0.0
    metadata.gps_dop = 5.0
    metadata.gps_position = [1.0815875281451939,
                             -0.96510451436708888,
                             1.2042133903991235]

    # Instantiate shots
    pose0 = types.Pose([0.0, 0.0, 0.0], [0.0, 0.0, 0.0])

    shot0 = types.Shot()
    shot0.id = 0
    shot0.camera = camera
    shot0.pose = pose0
    shot0.metadata = metadata

    pose1 = types.Pose([0.0, 0.0, 0.0], [-1.0, 0.0, 0.0])

    shot1 = types.Shot()
    shot1.id = 1
    shot1.camera = camera
    shot1.pose = pose1
    shot1.metadata = metadata

    # Add info to current reconstruction
    reconstruction.add_camera(camera)
    reconstruction.add_shot(shot0)
    reconstruction.add_shot(shot1)

    # TEST
    assert len(reconstruction.cameras) == 1
    assert len(reconstruction.shots) == 2
    assert len(reconstruction.points) == 0
    assert reconstruction.get_camera(camera.id) == camera
    assert reconstruction.get_camera(1) is None
    assert reconstruction.get_shot(shot0.id) == shot0
    assert reconstruction.get_shot(shot1.id) == shot1
    assert reconstruction.get_shot(2) is None

def test_perspective_camera_projection():
    """Test perspectiive projection--backprojection loop."""
    camera = _get_perspective_camera()
    pixel = [0.1, 0.2]
    bearing = camera.pixel_bearing(pixel)
    projected = camera.project(bearing)
    assert np.allclose(pixel, projected)

def test_fisheye_camera_projection():
    """Test fisheye projection--backprojection loop."""
    if not context.OPENCV3:
        return
    camera = _get_fisheye_camera()
    pixel = [0.1, 0.2]
    bearing = camera.pixel_bearing(pixel)
    projected = camera.project(bearing)
    assert np.allclose(pixel, projected)

def test_spherical_camera_projection():
    """Test spherical projection--backprojection loop."""
    camera = _get_spherical_camera()
    pixel = [0.1, 0.2]
    bearing = camera.pixel_bearing(pixel)
    projected = camera.project(bearing)
    assert np.allclose(pixel, projected)

def test_pose_properties():
    """Test pose constructor, getters and setters."""
    p = types.Pose([1, 2, 3], [4, 5, 6])
    assert np.allclose(p.rotation, [1, 2, 3])
    assert type(p.rotation) == np.ndarray
    assert p.rotation.dtype == float
    assert np.allclose(p.translation, [4, 5, 6])
    assert type(p.translation) == np.ndarray
    assert p.translation.dtype == float

def test_pose_inverse():
    p = types.Pose([1, 2, 3], [4, 5, 6])
    inverse = p.inverse()
    identity = p.compose(inverse)
    assert np.allclose(identity.rotation, [0, 0, 0])
    assert np.allclose(identity.translation, [0, 0, 0])

def test_single_vs_many():
    points = np.array([[1, 2, 3], [4, 5, 6]], dtype=float)
    pixels = np.array([[0.1, 0.2], [0.3, 0.4]], dtype=float)
    depths = np.array([1, 2], dtype=float)

    pose = types.Pose([1, 2, 3], [4, 5, 6])
    t_single = [pose.transform(p) for p in points]
    t_many = pose.transform_many(points)
    assert np.allclose(t_single, t_many)

    t_single = [pose.transform_inverse(p) for p in points]
    t_many = pose.transform_inverse_many(points)
    assert np.allclose(t_single, t_many)

    cameras = [
        _get_perspective_camera(),
        _get_brown_perspective_camera(),
        _get_fisheye_camera(),
        _get_spherical_camera(),
    ]
    for camera in cameras:
        p_single = [camera.project(p) for p in points]
        p_many = camera.project_many(points)
        assert np.allclose(p_single, p_many)

        b_single = [camera.pixel_bearing(p) for p in pixels]
        b_many = camera.pixel_bearing_many(pixels)
        assert np.allclose(b_single, b_many)

        if hasattr(camera, 'back_project'):
            q_single = [camera.back_project(p, d)
                        for p, d in zip(pixels, depths)]
            q_many = camera.back_project_many(pixels, depths)
            assert np.allclose(q_single, q_many)

def _get_perspective_camera():
    camera = types.PerspectiveCamera()
    camera.width = 800
    camera.height = 600
    camera.focal = 0.6
    camera.k1 = -0.1
    camera.k2 = 0.01
    return camera

def _get_brown_perspective_camera():
    camera = types.BrownPerspectiveCamera()
    camera.width = 800
    camera.height = 600
    camera.focal_x = 0.6
    camera.focal_y = 0.7
    camera.c_x = 0.1
    camera.c_y = -0.05
    camera.k1 = -0.1
    camera.k2 = 0.01
    camera.p1 = 0.001
    camera.p2 = 0.002
    camera.k3 = 0.01
    return camera

def _get_fisheye_camera():
    camera = types.FisheyeCamera()
    camera.width = 800
    camera.height = 600
    camera.focal = 0.6
    camera.k1 = -0.1
    camera.k2 = 0.01
    return camera

def _get_spherical_camera():
    camera = types.SphericalCamera()
    camera.width = 800
    camera.height = 600
    return camera
<commit_msg>test: disable fisheye camera testing when using opencv 2<commit_after>import numpy as np

from opensfm import context
from opensfm import types

"""
Trying to imitate the following structure

reconstruction = {
    "cameras": {
        "theta": {
            "projection_type": "equirectangular"
        }
    },

    "shots" : {
        'im1': {
            "camera": "theta",
            "rotation": [0.0, 0.0, 0.0],
            "translation": [0.0, 0.0, 0.0],
        },
        'im2': {
            "camera": "theta",
            "rotation": [0, 0, 0.0],
            "translation": [-1, 0, 0.0],
        },
    },

    "points" : {
    },
}
"""

def test_reconstruction_class_initialization():

    # Instantiate Reconstruction
    reconstruction = types.Reconstruction()

    # Instantiate camera instrinsics
    camera = types.PerspectiveCamera()
    camera.id = 'apple iphone 4s back camera 4.28mm f/2.4'
    camera.focal = 0.9722222222222222
    camera.k1 = 0.006094395128698237
    camera.k2 = -0.0004952058188617129
    camera.height = 2448
    camera.width = 3264

    # Instantiate GPS data
    metadata = types.ShotMetadata()
    metadata.orientation = 1
    metadata.capture_time = 0.0
    metadata.gps_dop = 5.0
    metadata.gps_position = [1.0815875281451939,
                             -0.96510451436708888,
                             1.2042133903991235]

    # Instantiate shots
    pose0 = types.Pose([0.0, 0.0, 0.0], [0.0, 0.0, 0.0])

    shot0 = types.Shot()
    shot0.id = 0
    shot0.camera = camera
    shot0.pose = pose0
    shot0.metadata = metadata

    pose1 = types.Pose([0.0, 0.0, 0.0], [-1.0, 0.0, 0.0])

    shot1 = types.Shot()
    shot1.id = 1
    shot1.camera = camera
    shot1.pose = pose1
    shot1.metadata = metadata

    # Add info to current reconstruction
    reconstruction.add_camera(camera)
    reconstruction.add_shot(shot0)
    reconstruction.add_shot(shot1)

    # TEST
    assert len(reconstruction.cameras) == 1
    assert len(reconstruction.shots) == 2
    assert len(reconstruction.points) == 0
    assert reconstruction.get_camera(camera.id) == camera
    assert reconstruction.get_camera(1) is None
    assert reconstruction.get_shot(shot0.id) == shot0
    assert reconstruction.get_shot(shot1.id) == shot1
    assert reconstruction.get_shot(2) is None

def test_perspective_camera_projection():
    """Test perspectiive projection--backprojection loop."""
    camera = _get_perspective_camera()
    pixel = [0.1, 0.2]
    bearing = camera.pixel_bearing(pixel)
    projected = camera.project(bearing)
    assert np.allclose(pixel, projected)

def test_fisheye_camera_projection():
    """Test fisheye projection--backprojection loop."""
    if not context.OPENCV3:
        return
    camera = _get_fisheye_camera()
    pixel = [0.1, 0.2]
    bearing = camera.pixel_bearing(pixel)
    projected = camera.project(bearing)
    assert np.allclose(pixel, projected)

def test_spherical_camera_projection():
    """Test spherical projection--backprojection loop."""
    camera = _get_spherical_camera()
    pixel = [0.1, 0.2]
    bearing = camera.pixel_bearing(pixel)
    projected = camera.project(bearing)
    assert np.allclose(pixel, projected)

def test_pose_properties():
    """Test pose constructor, getters and setters."""
    p = types.Pose([1, 2, 3], [4, 5, 6])
    assert np.allclose(p.rotation, [1, 2, 3])
    assert type(p.rotation) == np.ndarray
    assert p.rotation.dtype == float
    assert np.allclose(p.translation, [4, 5, 6])
    assert type(p.translation) == np.ndarray
    assert p.translation.dtype == float

def test_pose_inverse():
    p = types.Pose([1, 2, 3], [4, 5, 6])
    inverse = p.inverse()
    identity = p.compose(inverse)
    assert np.allclose(identity.rotation, [0, 0, 0])
    assert np.allclose(identity.translation, [0, 0, 0])

def test_single_vs_many():
    points = np.array([[1, 2, 3], [4, 5, 6]], dtype=float)
    pixels = np.array([[0.1, 0.2], [0.3, 0.4]], dtype=float)
    depths = np.array([1, 2], dtype=float)

    pose = types.Pose([1, 2, 3], [4, 5, 6])
    t_single = [pose.transform(p) for p in points]
    t_many = pose.transform_many(points)
    assert np.allclose(t_single, t_many)

    t_single = [pose.transform_inverse(p) for p in points]
    t_many = pose.transform_inverse_many(points)
    assert np.allclose(t_single, t_many)

    cameras = [
        _get_perspective_camera(),
        _get_brown_perspective_camera(),
        _get_spherical_camera(),
    ]
    if context.OPENCV3:
        cameras.append(_get_fisheye_camera())

    for camera in cameras:
        p_single = [camera.project(p) for p in points]
        p_many = camera.project_many(points)
        assert np.allclose(p_single, p_many)

        b_single = [camera.pixel_bearing(p) for p in pixels]
        b_many = camera.pixel_bearing_many(pixels)
        assert np.allclose(b_single, b_many)

        if hasattr(camera, 'back_project'):
            q_single = [camera.back_project(p, d)
                        for p, d in zip(pixels, depths)]
            q_many = camera.back_project_many(pixels, depths)
            assert np.allclose(q_single, q_many)

def _get_perspective_camera():
    camera = types.PerspectiveCamera()
    camera.width = 800
    camera.height = 600
    camera.focal = 0.6
    camera.k1 = -0.1
    camera.k2 = 0.01
    return camera

def _get_brown_perspective_camera():
    camera = types.BrownPerspectiveCamera()
    camera.width = 800
    camera.height = 600
    camera.focal_x = 0.6
    camera.focal_y = 0.7
    camera.c_x = 0.1
    camera.c_y = -0.05
    camera.k1 = -0.1
    camera.k2 = 0.01
    camera.p1 = 0.001
    camera.p2 = 0.002
    camera.k3 = 0.01
    return camera

def _get_fisheye_camera():
    camera = types.FisheyeCamera()
    camera.width = 800
    camera.height = 600
    camera.focal = 0.6
    camera.k1 = -0.1
    camera.k2 = 0.01
    return camera

def _get_spherical_camera():
    camera = types.SphericalCamera()
    camera.width = 800
    camera.height = 600
    return camera
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from copy import deepcopy
import os
from os import linesep
from flask import Flask, jsonify, abort, request, make_response
from werkzeug.contrib.fixers import ProxyFix
from flask_cors import CORS
import dpath

from openfisca_core.simulations import Simulation, SituationParsingError
from openfisca_core.commons import to_unicode
from openfisca_core.indexed_enums import Enum
from openfisca_web_api_preview.loader import build_data
import traceback
import logging

log = logging.getLogger('gunicorn.error')

def init_tracker(url, idsite, tracker_token):
    try:
        from openfisca_tracker.piwik import PiwikTracker
        tracker = PiwikTracker(url, idsite, tracker_token)

        info = linesep.join([u'You chose to activate the `tracker` module. ',
                             u'Tracking data will be sent to: ' + url,
                             u'For more information, see <https://github.com/openfisca/openfisca-core#tracker-configuration>.'])
        log.info(info)
        return tracker

    except ImportError:
        message = linesep.join([traceback.format_exc(),
                                u'You chose to activate the `tracker` module, but it is not installed.',
                                u'For more information, see <https://github.com/openfisca/openfisca-core#tracker-installation>.'])
        log.warn(message)

def create_app(tax_benefit_system,
               tracker_url = None,
               tracker_idsite = None,
               tracker_token = None
               ):

    if not tracker_url or not tracker_idsite:
        tracker = None
    else:
        tracker = init_tracker(tracker_url, tracker_idsite, tracker_token)

    app = Flask(__name__)
    app.wsgi_app = ProxyFix(app.wsgi_app, num_proxies = 1)  # Fix request.remote_addr to get the real client IP address
    CORS(app, origins = '*')

    app.config['JSON_AS_ASCII'] = False  # When False, lets jsonify encode to utf-8
    app.url_map.strict_slashes = False  # Accept url like /parameters/

    data = build_data(tax_benefit_system)

    @app.route('/parameters')
    def get_parameters():
        return jsonify(data['parameters_description'])

    @app.route('/parameter/<id>')
    def get_parameter(id):
        parameter = data['parameters'].get(id)
        if parameter is None:
            raise abort(404)
        return jsonify(parameter)

    @app.route('/variables')
    def get_variables():
        return jsonify(data['variables_description'])

    @app.route('/variable/<id>')
    def get_variable(id):
        variable = data['variables'].get(id)
        if variable is None:
            raise abort(404)
        return jsonify(variable)

    @app.route('/spec')
    def get_spec():
        return jsonify(data['openAPI_spec'])

    def handle_invalid_json(error):
        json_response = jsonify({
            'error': 'Invalid JSON: {}'.format(error.args[0]),
            })

        abort(make_response(json_response, 400))

    @app.route('/calculate', methods=['POST'])
    def calculate():
        tax_benefit_system = data['tax_benefit_system']
        request.on_json_loading_failed = handle_invalid_json
        input_data = request.get_json()
        try:
            simulation = Simulation(tax_benefit_system = tax_benefit_system, simulation_json = input_data)
        except SituationParsingError as e:
            abort(make_response(jsonify(e.error), e.code or 400))
        except UnicodeEncodeError as e:
            abort(make_response(jsonify({"error": "'" + e[1] + "' is not a valid ASCII value."}), 400))

        requested_computations = dpath.util.search(input_data, '*/*/*/*', afilter = lambda t: t is None, yielded = True)
        results = {}

        try:
            for computation in requested_computations:
                path = computation[0]
                entity_plural, entity_id, variable_name, period = path.split('/')
                variable = tax_benefit_system.get_variable(variable_name)
                result = simulation.calculate(variable_name, period)
                entity = simulation.get_entity(plural = entity_plural)
                entity_index = entity.ids.index(entity_id)

                if variable.value_type == Enum:
                    entity_result = result.decode()[entity_index].name
                else:
                    entity_result = result.tolist()[entity_index]

                dpath.util.new(results, path, entity_result)

        except UnicodeEncodeError as e:
            abort(make_response(jsonify({"error": "'" + e[1] + "' is not a valid ASCII value."}), 400))

        dpath.util.merge(input_data, results)

        return jsonify(input_data)

    @app.route('/trace', methods=['POST'])
    def trace():
        tax_benefit_system = data['tax_benefit_system']
        request.on_json_loading_failed = handle_invalid_json
        input_data = request.get_json()
        try:
            simulation = Simulation(tax_benefit_system = tax_benefit_system, simulation_json = input_data, trace = True)
        except SituationParsingError as e:
            abort(make_response(jsonify(e.error), e.code or 400))

        requested_computations = dpath.util.search(input_data, '*/*/*/*', afilter = lambda t: t is None, yielded = True)
        for computation in requested_computations:
            path = computation[0]
            entity_plural, entity_id, variable_name, period = path.split('/')
            simulation.calculate(variable_name, period)

        trace = deepcopy(simulation.tracer.trace)
        for vector_key, vector_trace in trace.items():
            value = vector_trace['value'].tolist()
            if isinstance(value[0], Enum):
                value = [item.name for item in value]
            vector_trace['value'] = value

        return jsonify({
            "trace": trace,
            "entitiesDescription": {entity.plural: entity.ids for entity in simulation.entities.values()},
            "requestedCalculations": list(simulation.tracer.requested_calculations)
            })

    @app.after_request
    def apply_headers(response):
        response.headers.extend({
            'Country-Package': data['country_package_metadata']['name'],
            'Country-Package-Version': data['country_package_metadata']['version']
            })
        return response

    @app.after_request
    def track_requests(response):
        if tracker:
            if request.headers.get('dnt'):
                tracker.track(request.url)
            else:
                api_version = "{}-{}".format(data['country_package_metadata']['name'], data['country_package_metadata']['version'])
                tracker.track(request.url, request.remote_addr, api_version)
        return response

    @app.errorhandler(500)
    def internal_server_error(e):
        message = to_unicode(e.args[0])
        if type(e) == UnicodeEncodeError or type(e) == UnicodeDecodeError:
            response = jsonify({"error": "Internal server error: '" + e[1] + "' is not a valid ASCII value."})
        elif message:
            response = jsonify({"error": "Internal server error: " + message.strip(os.linesep).replace(os.linesep, ' ')})
        else:
            response = jsonify({"error": "Internal server error: " + str(e)})
        response.status_code = 500
        return response

    return app
<commit_msg>Take the domain out of the action description<commit_after># -*- coding: utf-8 -*-

from copy import deepcopy
import os
from os import linesep
from flask import Flask, jsonify, abort, request, make_response
from werkzeug.contrib.fixers import ProxyFix
from flask_cors import CORS
import dpath

from openfisca_core.simulations import Simulation, SituationParsingError
from openfisca_core.commons import to_unicode
from openfisca_core.indexed_enums import Enum
from openfisca_web_api_preview.loader import build_data
import traceback
import logging

log = logging.getLogger('gunicorn.error')

def init_tracker(url, idsite, tracker_token):
    try:
        from openfisca_tracker.piwik import PiwikTracker
        tracker = PiwikTracker(url, idsite, tracker_token)

        info = linesep.join([u'You chose to activate the `tracker` module. ',
                             u'Tracking data will be sent to: ' + url,
                             u'For more information, see <https://github.com/openfisca/openfisca-core#tracker-configuration>.'])
        log.info(info)
        return tracker

    except ImportError:
        message = linesep.join([traceback.format_exc(),
                                u'You chose to activate the `tracker` module, but it is not installed.',
                                u'For more information, see <https://github.com/openfisca/openfisca-core#tracker-installation>.'])
        log.warn(message)

def create_app(tax_benefit_system,
               tracker_url = None,
               tracker_idsite = None,
               tracker_token = None
               ):

    if not tracker_url or not tracker_idsite:
        tracker = None
    else:
        tracker = init_tracker(tracker_url, tracker_idsite, tracker_token)

    app = Flask(__name__)
    app.wsgi_app = ProxyFix(app.wsgi_app, num_proxies = 1)  # Fix request.remote_addr to get the real client IP address
    CORS(app, origins = '*')

    app.config['JSON_AS_ASCII'] = False  # When False, lets jsonify encode to utf-8
    app.url_map.strict_slashes = False  # Accept url like /parameters/

    data = build_data(tax_benefit_system)

    @app.route('/parameters')
    def get_parameters():
        return jsonify(data['parameters_description'])

    @app.route('/parameter/<id>')
    def get_parameter(id):
        parameter = data['parameters'].get(id)
        if parameter is None:
            raise abort(404)
        return jsonify(parameter)

    @app.route('/variables')
    def get_variables():
        return jsonify(data['variables_description'])

    @app.route('/variable/<id>')
    def get_variable(id):
        variable = data['variables'].get(id)
        if variable is None:
            raise abort(404)
        return jsonify(variable)

    @app.route('/spec')
    def get_spec():
        return jsonify(data['openAPI_spec'])

    def handle_invalid_json(error):
        json_response = jsonify({
            'error': 'Invalid JSON: {}'.format(error.args[0]),
            })

        abort(make_response(json_response, 400))

    @app.route('/calculate', methods=['POST'])
    def calculate():
        tax_benefit_system = data['tax_benefit_system']
        request.on_json_loading_failed = handle_invalid_json
        input_data = request.get_json()
        try:
            simulation = Simulation(tax_benefit_system = tax_benefit_system, simulation_json = input_data)
        except SituationParsingError as e:
            abort(make_response(jsonify(e.error), e.code or 400))
        except UnicodeEncodeError as e:
            abort(make_response(jsonify({"error": "'" + e[1] + "' is not a valid ASCII value."}), 400))

        requested_computations = dpath.util.search(input_data, '*/*/*/*', afilter = lambda t: t is None, yielded = True)
        results = {}

        try:
            for computation in requested_computations:
                path = computation[0]
                entity_plural, entity_id, variable_name, period = path.split('/')
                variable = tax_benefit_system.get_variable(variable_name)
                result = simulation.calculate(variable_name, period)
                entity = simulation.get_entity(plural = entity_plural)
                entity_index = entity.ids.index(entity_id)

                if variable.value_type == Enum:
                    entity_result = result.decode()[entity_index].name
                else:
                    entity_result = result.tolist()[entity_index]

                dpath.util.new(results, path, entity_result)

        except UnicodeEncodeError as e:
            abort(make_response(jsonify({"error": "'" + e[1] + "' is not a valid ASCII value."}), 400))

        dpath.util.merge(input_data, results)

        return jsonify(input_data)

    @app.route('/trace', methods=['POST'])
    def trace():
        tax_benefit_system = data['tax_benefit_system']
        request.on_json_loading_failed = handle_invalid_json
        input_data = request.get_json()
        try:
            simulation = Simulation(tax_benefit_system = tax_benefit_system, simulation_json = input_data, trace = True)
        except SituationParsingError as e:
            abort(make_response(jsonify(e.error), e.code or 400))

        requested_computations = dpath.util.search(input_data, '*/*/*/*', afilter = lambda t: t is None, yielded = True)
        for computation in requested_computations:
            path = computation[0]
            entity_plural, entity_id, variable_name, period = path.split('/')
            simulation.calculate(variable_name, period)

        trace = deepcopy(simulation.tracer.trace)
        for vector_key, vector_trace in trace.items():
            value = vector_trace['value'].tolist()
            if isinstance(value[0], Enum):
                value = [item.name for item in value]
            vector_trace['value'] = value

        return jsonify({
            "trace": trace,
            "entitiesDescription": {entity.plural: entity.ids for entity in simulation.entities.values()},
            "requestedCalculations": list(simulation.tracer.requested_calculations)
            })

    @app.after_request
    def apply_headers(response):
        response.headers.extend({
            'Country-Package': data['country_package_metadata']['name'],
            'Country-Package-Version': data['country_package_metadata']['version']
            })
        return response

    @app.after_request
    def track_requests(response):
        if tracker:
            if request.headers.get('dnt'):
                tracker.track(request.url)
            else:
                api_version = "{}-{}".format(data['country_package_metadata']['name'], data['country_package_metadata']['version'])
                tracker.track(request.url, request.remote_addr, api_version, url.path)
        return response

    @app.errorhandler(500)
    def internal_server_error(e):
        message = to_unicode(e.args[0])
        if type(e) == UnicodeEncodeError or type(e) == UnicodeDecodeError:
            response = jsonify({"error": "Internal server error: '" + e[1] + "' is not a valid ASCII value."})
        elif message:
            response = jsonify({"error": "Internal server error: " + message.strip(os.linesep).replace(os.linesep, ' ')})
        else:
            response = jsonify({"error": "Internal server error: " + str(e)})
        response.status_code = 500
        return response

    return app
<|endoftext|>


<commit_before>import sys
import traceback

ERROR_WRONG_USAGE = 1
ERROR_NO_PACKAGING_TOOLS = 2
ERROR_EXCEPTION = 3

def exit(retcode):
    major, minor, micro, release, serial = sys.version_info
    version = major * 10 + minor
    if version < 25:
        import os

        os._exit(retcode)
    else:
        sys.exit(retcode)

def usage():
    sys.stderr.write('Usage: packaging_tool.py <list|install|uninstall>\n')
    sys.stderr.flush()
    exit(ERROR_WRONG_USAGE)

def error(message, retcode):
    sys.stderr.write('Error: %s\n' % message)
    sys.stderr.flush()
    exit(retcode)

def error_no_pip():
    error("Python package management tool 'pip' not found. Install 'pip'", ERROR_NO_PACKAGING_TOOLS)

def do_list():
    try:
        import pkg_resources
    except ImportError:
        error("Python package management tools not found. Install 'distribute'", ERROR_NO_PACKAGING_TOOLS)
    for pkg in pkg_resources.working_set:
        requires = ':'.join([str(x) for x in pkg.requires()])
        sys.stdout.write('\t'.join([pkg.project_name, pkg.version, pkg.location, requires])+chr(10))
    sys.stdout.flush()

def do_install(pkgs):
    try:
        import pip
    except ImportError:
        error_no_pip()
    return pip.main(['install'] + pkgs)

def do_uninstall(pkgs):
    try:
        import pip
    except ImportError:
        error_no_pip()
    return pip.main(['uninstall', '-y'] + pkgs)

def untarDirectory(name):
    import os
    import tempfile

    directory_name = tempfile.mkdtemp("pycharm-management")

    import tarfile

    filename = name + ".tar.gz"
    tar = tarfile.open(filename)
    for item in tar:
        tar.extract(item, directory_name)

    sys.stdout.write(directory_name+chr(10))
    sys.stdout.flush()

def mkdtemp_ifneeded():
    try:
        ind = sys.argv.index('--build-dir')
        if not os.path.exists(sys.argv[ind + 1]):
            import tempfile

            sys.argv[ind + 1] = tempfile.mkdtemp('pycharm-packaging')
            return sys.argv[ind + 1]
    except:
        pass

    return None

def main():
    retcode = 0
    try:
        if len(sys.argv) < 2:
            usage()
        cmd = sys.argv[1]
        if cmd == 'list':
            if len(sys.argv) != 2:
                usage()
            do_list()
        elif cmd == 'install':
            if len(sys.argv) < 2:
                usage()

            rmdir = mkdtemp_ifneeded()

            pkgs = sys.argv[2:]
            retcode = do_install(pkgs)

            if rmdir is not None:
                import shutil
                shutil.rmtree(rmdir)

        elif cmd == 'untar':
            if len(sys.argv) < 2:
                usage()
            name = sys.argv[2]
            retcode = untarDirectory(name)
        elif cmd == 'uninstall':
            if len(sys.argv) < 2:
                usage()
            pkgs = sys.argv[2:]
            retcode = do_uninstall(pkgs)
        else:
            usage()
    except Exception:
        traceback.print_exc()
        exit(ERROR_EXCEPTION)
    exit(retcode)

if __name__ == '__main__':
    main()
<commit_msg>fixed PY-6285 Install pip fails for python 2.4: TypeError: an integer is required<commit_after>import sys
import traceback

ERROR_WRONG_USAGE = 1
ERROR_NO_PACKAGING_TOOLS = 2
ERROR_EXCEPTION = 3

def exit(retcode):
    major, minor, micro, release, serial = sys.version_info
    version = major * 10 + minor
    if version < 25:
        import os
        os._exit(retcode)
    else:
        sys.exit(retcode)

def usage():
    sys.stderr.write('Usage: packaging_tool.py <list|install|uninstall>\n')
    sys.stderr.flush()
    exit(ERROR_WRONG_USAGE)

def error(message, retcode):
    sys.stderr.write('Error: %s\n' % message)
    sys.stderr.flush()
    exit(retcode)

def error_no_pip():
    error("Python package management tool 'pip' not found. Install 'pip'", ERROR_NO_PACKAGING_TOOLS)

def do_list():
    try:
        import pkg_resources
    except ImportError:
        error("Python package management tools not found. Install 'distribute'", ERROR_NO_PACKAGING_TOOLS)
    for pkg in pkg_resources.working_set:
        requires = ':'.join([str(x) for x in pkg.requires()])
        sys.stdout.write('\t'.join([pkg.project_name, pkg.version, pkg.location, requires])+chr(10))
    sys.stdout.flush()

def do_install(pkgs):
    try:
        import pip
    except ImportError:
        error_no_pip()
    return pip.main(['install'] + pkgs)

def do_uninstall(pkgs):
    try:
        import pip
    except ImportError:
        error_no_pip()
    return pip.main(['uninstall', '-y'] + pkgs)

def untarDirectory(name):
    import os
    import tempfile

    directory_name = tempfile.mkdtemp("pycharm-management")

    import tarfile

    filename = name + ".tar.gz"
    tar = tarfile.open(filename)
    for item in tar:
        tar.extract(item, directory_name)

    sys.stdout.write(directory_name+chr(10))
    sys.stdout.flush()
    return 0

def mkdtemp_ifneeded():
    try:
        ind = sys.argv.index('--build-dir')
        if not os.path.exists(sys.argv[ind + 1]):
            import tempfile

            sys.argv[ind + 1] = tempfile.mkdtemp('pycharm-packaging')
            return sys.argv[ind + 1]
    except:
        pass

    return None

def main():
    retcode = 0
    try:
        if len(sys.argv) < 2:
            usage()
        cmd = sys.argv[1]
        if cmd == 'list':
            if len(sys.argv) != 2:
                usage()
            do_list()
        elif cmd == 'install':
            if len(sys.argv) < 2:
                usage()

            rmdir = mkdtemp_ifneeded()

            pkgs = sys.argv[2:]
            retcode = do_install(pkgs)

            if rmdir is not None:
                import shutil
                shutil.rmtree(rmdir)

        elif cmd == 'untar':
            if len(sys.argv) < 2:
                usage()
            name = sys.argv[2]
            retcode = untarDirectory(name)
        elif cmd == 'uninstall':
            if len(sys.argv) < 2:
                usage()
            pkgs = sys.argv[2:]
            retcode = do_uninstall(pkgs)
        else:
            usage()
    except Exception:
        traceback.print_exc()
        exit(ERROR_EXCEPTION)
    exit(retcode)

if __name__ == '__main__':
    main()
<|endoftext|>