<commit_before>import gym
import os
from d4rl import offline_env
from gym.envs.registration import register

from copy import deepcopy

import flow
import flow.envs
from flow.networks.ring import RingNetwork
from flow.networks.minicity import MiniCityNetwork
from flow.networks.bay_bridge_toll import BayBridgeTollNetwork
from flow.core.params import NetParams, VehicleParams, EnvParams, InFlows
from flow.core.params import SumoLaneChangeParams, SumoCarFollowingParams
from flow.networks.ring import ADDITIONAL_NET_PARAMS
from flow.controllers.car_following_models import IDMController
from flow.controllers.routing_controllers import ContinuousRouter
from flow.controllers import SimCarFollowingController, BayBridgeRouter, SimLaneChangeController
from flow.controllers import RLController
from flow.core.params import InitialConfig
from flow.core.params import TrafficLightParams
from flow.envs.ring.accel import AccelEnv
from flow.core.params import SumoParams
from flow.utils.registry import make_create_env
from flow.envs import WaveAttenuationPOEnv
from flow.envs import BayBridgeEnv, TrafficLightGridPOEnv
from flow.envs import BottleneckDesiredVelocityEnv
from flow.networks import BottleneckNetwork

import d4rl.flow.traffic_light_grid as traffic_light_grid
import d4rl.flow.merge as merge

def flow_register(flow_params, render=None, **kwargs):
    exp_tag = flow_params["exp_tag"]
    env_params = flow_params['env']
    net_params = flow_params['net']
    env_class = flow_params['env_name']
    initial_config = flow_params.get('initial', InitialConfig())
    traffic_lights = flow_params.get("tls", TrafficLightParams())
    sim_params = deepcopy(flow_params['sim'])
    vehicles = deepcopy(flow_params['veh'])

    sim_params.render = render or sim_params.render

    if isinstance(flow_params["network"], str):
        print("""Passing of strings for network will be deprecated.
        Please pass the Network instance instead.""")
        module = __import__("flow.networks", fromlist=[flow_params["network"]])
        network_class = getattr(module, flow_params["network"])
    else:
        network_class = flow_params["network"]

    network = network_class(
        name=exp_tag,
        vehicles=vehicles,
        net_params=net_params,
        initial_config=initial_config,
        traffic_lights=traffic_lights,
    )

    flow_env = env_class(
        env_params= env_params,
        sim_params= sim_params,
        network= network,
        simulator= flow_params['simulator']
    )

    env = offline_env.OfflineEnvWrapper(flow_env,
        **kwargs
    )
    return env

def ring_env(render='drgb'):
    name = "ring"
    network_name = RingNetwork
    env_name = WaveAttenuationPOEnv

    net_params = NetParams(additional_params=ADDITIONAL_NET_PARAMS)
    initial_config = InitialConfig(spacing="uniform", shuffle=False)

    vehicles = VehicleParams()
    vehicles.add("human",
                 acceleration_controller=(IDMController, {}),
                 routing_controller=(ContinuousRouter, {}),
                 num_vehicles=21)
    vehicles.add(veh_id="rl",
                 acceleration_controller=(RLController, {}),
                 routing_controller=(ContinuousRouter, {}),
                 num_vehicles=1)

    sim_params = SumoParams(sim_step=0.5, render=render, save_render=True)
    HORIZON=100
    env_params = EnvParams(
        # length of one rollout
        horizon=HORIZON,
        additional_params={
            # maximum acceleration of autonomous vehicles
            "max_accel": 1,
            # maximum deceleration of autonomous vehicles
            "max_decel": 1,
            # bounds on the ranges of ring road lengths the autonomous vehicle
            # is trained on
            "ring_length": [220, 270],
        },
    )

    flow_params = dict(
        exp_tag=name,
        env_name=env_name,
        network=network_name,
        simulator='traci',
        sim=sim_params,
        env=env_params,
        net=net_params,
        veh=vehicles,
        initial=initial_config
    )
    return flow_params

def bottleneck(render='drgb', SCALING=1):
    HORIZON=1000
    DISABLE_TB = True
    DISABLE_RAMP_METER = True
    AV_FRAC = 0.10

    vehicles = VehicleParams()
    vehicles.add(
        veh_id="human",
        lane_change_controller=(SimLaneChangeController, {}),
        routing_controller=(ContinuousRouter, {}),
        car_following_params=SumoCarFollowingParams(
            speed_mode="all_checks",
        ),
        lane_change_params=SumoLaneChangeParams(
            lane_change_mode=0,
        ),
        num_vehicles=1 * SCALING)
    vehicles.add(
        veh_id="rl", #"followerstopper",
        acceleration_controller=(RLController, {}),
        lane_change_controller=(SimLaneChangeController, {}),
        routing_controller=(ContinuousRouter, {}),
        car_following_params=SumoCarFollowingParams(
            speed_mode=9,
        ),
        lane_change_params=SumoLaneChangeParams(
            lane_change_mode=0,
        ),
    num_vehicles=1 * SCALING)

    controlled_segments = [("1", 1, False), ("2", 2, True), ("3", 2, True),
                           ("4", 2, True), ("5", 1, False)]
    num_observed_segments = [("1", 1), ("2", 3), ("3", 3), ("4", 3), ("5", 1)]
    additional_env_params = {
        "target_velocity": 40,
        "disable_tb": True,
        "disable_ramp_metering": True,
        "controlled_segments": controlled_segments,
        "symmetric": False,
        "observed_segments": num_observed_segments,
        "reset_inflow": False,
        "lane_change_duration": 5,
        "max_accel": 3,
        "max_decel": 3,
        "inflow_range": [1000, 2000]
    }

    # flow rate
    flow_rate = 2300 * SCALING

    # percentage of flow coming out of each lane
    inflow = InFlows()
    inflow.add(
        veh_type="human",
        edge="1",
        vehs_per_hour=flow_rate * (1 - AV_FRAC),
        depart_lane="random",
        depart_speed=10)
    inflow.add(
        veh_type="rl", #"followerstopper",
        edge="1",
        vehs_per_hour=flow_rate * AV_FRAC,
        depart_lane="random",
        depart_speed=10)

    traffic_lights = TrafficLightParams()
    if not DISABLE_TB:
        traffic_lights.add(node_id="2")
    if not DISABLE_RAMP_METER:
        traffic_lights.add(node_id="3")

    additional_net_params = {"scaling": SCALING, "speed_limit": 23}
    net_params = NetParams(
        inflows=inflow,
        additional_params=additional_net_params)

    flow_params = dict(
        # name of the experiment
        exp_tag="bottleneck",

        # name of the flow environment the experiment is running on
        env_name=BottleneckDesiredVelocityEnv,

        # name of the network class the experiment is running on
        network=BottleneckNetwork,

        # simulator that is used by the experiment
        simulator='traci',

        # sumo-related parameters (see flow.core.params.SumoParams)
        sim=SumoParams(
            sim_step=0.5,
            render=render,
            save_render=True,
            print_warnings=False,
            restart_instance=True,
        ),

        # environment related parameters (see flow.core.params.EnvParams)
        env=EnvParams(
            warmup_steps=40,
            sims_per_step=1,
            horizon=HORIZON,
            additional_params=additional_env_params,
        ),

        # network-related parameters (see flow.core.params.NetParams and the
        # network's documentation or ADDITIONAL_NET_PARAMS component)
        net=NetParams(
            inflows=inflow,
            additional_params=additional_net_params,
        ),

        # vehicles to be placed in the network at the start of a rollout (see
        # flow.core.params.VehicleParams)
        veh=vehicles,

        # parameters specifying the positioning of vehicles upon initialization/
        # reset (see flow.core.params.InitialConfig)
        initial=InitialConfig(
            spacing="uniform",
            min_gap=5,
            lanes_distribution=float("inf"),
            edges_distribution=["2", "3", "4", "5"],
        ),

        # traffic lights to be introduced to specific nodes (see
        # flow.core.params.TrafficLightParams)
        tls=traffic_lights,
    )

    return flow_params

register(
    id='flow-ring-v0',
    entry_point='d4rl.flow:flow_register',
    max_episode_steps=500,
    kwargs={
        'flow_params': ring_env(render=False),
        'dataset_url': None,
        'ref_min_score': None,
        'ref_max_score': None
    }
)

register(
    id='flow-ring-render-v0',
    entry_point='d4rl.flow:flow_register',
    max_episode_steps=500,
    kwargs={
        'flow_params': ring_env(render='drgb'),
        'dataset_url': None,
        'ref_min_score': None,
        'ref_max_score': None
    }
)

register(
    id='flow-grid-v0',
    entry_point='d4rl.flow:flow_register',
    max_episode_steps=200,
    kwargs={
        'flow_params': traffic_light_grid.gen_env(render=False),
        'dataset_url': None,
        'ref_min_score': None,
        'ref_max_score': None
    }
)

register(
    id='flow-grid-render-v0',
    entry_point='d4rl.flow:flow_register',
    max_episode_steps=200,
    kwargs={
        'flow_params': traffic_light_grid.gen_env(render='drgb'),
        'dataset_url': None,
        'ref_min_score': None,
        'ref_max_score': None
    }
)

register(
    id='flow-merge-v0',
    entry_point='d4rl.flow:flow_register',
    max_episode_steps=600,
    kwargs={
        'flow_params': merge.gen_env(render=False),
        'dataset_url': None,
        'ref_min_score': None,
        'ref_max_score': None
    }
)

register(
    id='flow-merge-render-v0',
    entry_point='d4rl.flow:flow_register',
    max_episode_steps=600,
    kwargs={
        'flow_params': merge.gen_env(render='drgb'),
        'dataset_url': None,
        'ref_min_score': None,
        'ref_max_score': None
    }
)

register(
    id='flow-bottleneck-v0',
    entry_point='d4rl.flow:flow_register',
    max_episode_steps=1000,
    kwargs={
        'flow_params': bottleneck(render=False),
        'dataset_url': None,
        'ref_min_score': None,
        'ref_max_score': None
    }
)

register(
    id='flow-bottleneck-render-v0',
    entry_point='d4rl.flow:flow_register',
    max_episode_steps=1000,
    kwargs={
        'flow_params': bottleneck(render='drgb'),
        'dataset_url': None,
        'ref_min_score': None,
        'ref_max_score': None
    }
)
<commit_msg>Add flow scores<commit_after>import gym
import os
from d4rl import offline_env
from gym.envs.registration import register

from copy import deepcopy

import flow
import flow.envs
from flow.networks.ring import RingNetwork
from flow.networks.minicity import MiniCityNetwork
from flow.networks.bay_bridge_toll import BayBridgeTollNetwork
from flow.core.params import NetParams, VehicleParams, EnvParams, InFlows
from flow.core.params import SumoLaneChangeParams, SumoCarFollowingParams
from flow.networks.ring import ADDITIONAL_NET_PARAMS
from flow.controllers.car_following_models import IDMController
from flow.controllers.routing_controllers import ContinuousRouter
from flow.controllers import SimCarFollowingController, BayBridgeRouter, SimLaneChangeController
from flow.controllers import RLController
from flow.core.params import InitialConfig
from flow.core.params import TrafficLightParams
from flow.envs.ring.accel import AccelEnv
from flow.core.params import SumoParams
from flow.utils.registry import make_create_env
from flow.envs import WaveAttenuationPOEnv
from flow.envs import BayBridgeEnv, TrafficLightGridPOEnv
from flow.envs import BottleneckDesiredVelocityEnv
from flow.networks import BottleneckNetwork

import d4rl.flow.traffic_light_grid as traffic_light_grid
import d4rl.flow.merge as merge

def flow_register(flow_params, render=None, **kwargs):
    exp_tag = flow_params["exp_tag"]
    env_params = flow_params['env']
    net_params = flow_params['net']
    env_class = flow_params['env_name']
    initial_config = flow_params.get('initial', InitialConfig())
    traffic_lights = flow_params.get("tls", TrafficLightParams())
    sim_params = deepcopy(flow_params['sim'])
    vehicles = deepcopy(flow_params['veh'])

    sim_params.render = render or sim_params.render

    if isinstance(flow_params["network"], str):
        print("""Passing of strings for network will be deprecated.
        Please pass the Network instance instead.""")
        module = __import__("flow.networks", fromlist=[flow_params["network"]])
        network_class = getattr(module, flow_params["network"])
    else:
        network_class = flow_params["network"]

    network = network_class(
        name=exp_tag,
        vehicles=vehicles,
        net_params=net_params,
        initial_config=initial_config,
        traffic_lights=traffic_lights,
    )

    flow_env = env_class(
        env_params= env_params,
        sim_params= sim_params,
        network= network,
        simulator= flow_params['simulator']
    )

    env = offline_env.OfflineEnvWrapper(flow_env,
        **kwargs
    )
    return env

def ring_env(render='drgb'):
    name = "ring"
    network_name = RingNetwork
    env_name = WaveAttenuationPOEnv

    net_params = NetParams(additional_params=ADDITIONAL_NET_PARAMS)
    initial_config = InitialConfig(spacing="uniform", shuffle=False)

    vehicles = VehicleParams()
    vehicles.add("human",
                 acceleration_controller=(IDMController, {}),
                 routing_controller=(ContinuousRouter, {}),
                 num_vehicles=21)
    vehicles.add(veh_id="rl",
                 acceleration_controller=(RLController, {}),
                 routing_controller=(ContinuousRouter, {}),
                 num_vehicles=1)

    sim_params = SumoParams(sim_step=0.5, render=render, save_render=True)
    HORIZON=100
    env_params = EnvParams(
        # length of one rollout
        horizon=HORIZON,
        additional_params={
            # maximum acceleration of autonomous vehicles
            "max_accel": 1,
            # maximum deceleration of autonomous vehicles
            "max_decel": 1,
            # bounds on the ranges of ring road lengths the autonomous vehicle
            # is trained on
            "ring_length": [220, 270],
        },
    )

    flow_params = dict(
        exp_tag=name,
        env_name=env_name,
        network=network_name,
        simulator='traci',
        sim=sim_params,
        env=env_params,
        net=net_params,
        veh=vehicles,
        initial=initial_config
    )
    return flow_params

def bottleneck(render='drgb', SCALING=1):
    HORIZON=1000
    DISABLE_TB = True
    DISABLE_RAMP_METER = True
    AV_FRAC = 0.10

    vehicles = VehicleParams()
    vehicles.add(
        veh_id="human",
        lane_change_controller=(SimLaneChangeController, {}),
        routing_controller=(ContinuousRouter, {}),
        car_following_params=SumoCarFollowingParams(
            speed_mode="all_checks",
        ),
        lane_change_params=SumoLaneChangeParams(
            lane_change_mode=0,
        ),
        num_vehicles=1 * SCALING)
    vehicles.add(
        veh_id="rl", #"followerstopper",
        acceleration_controller=(RLController, {}),
        lane_change_controller=(SimLaneChangeController, {}),
        routing_controller=(ContinuousRouter, {}),
        car_following_params=SumoCarFollowingParams(
            speed_mode=9,
        ),
        lane_change_params=SumoLaneChangeParams(
            lane_change_mode=0,
        ),
    num_vehicles=1 * SCALING)

    controlled_segments = [("1", 1, False), ("2", 2, True), ("3", 2, True),
                           ("4", 2, True), ("5", 1, False)]
    num_observed_segments = [("1", 1), ("2", 3), ("3", 3), ("4", 3), ("5", 1)]
    additional_env_params = {
        "target_velocity": 40,
        "disable_tb": True,
        "disable_ramp_metering": True,
        "controlled_segments": controlled_segments,
        "symmetric": False,
        "observed_segments": num_observed_segments,
        "reset_inflow": False,
        "lane_change_duration": 5,
        "max_accel": 3,
        "max_decel": 3,
        "inflow_range": [1000, 2000]
    }

    # flow rate
    flow_rate = 2300 * SCALING

    # percentage of flow coming out of each lane
    inflow = InFlows()
    inflow.add(
        veh_type="human",
        edge="1",
        vehs_per_hour=flow_rate * (1 - AV_FRAC),
        depart_lane="random",
        depart_speed=10)
    inflow.add(
        veh_type="rl", #"followerstopper",
        edge="1",
        vehs_per_hour=flow_rate * AV_FRAC,
        depart_lane="random",
        depart_speed=10)

    traffic_lights = TrafficLightParams()
    if not DISABLE_TB:
        traffic_lights.add(node_id="2")
    if not DISABLE_RAMP_METER:
        traffic_lights.add(node_id="3")

    additional_net_params = {"scaling": SCALING, "speed_limit": 23}
    net_params = NetParams(
        inflows=inflow,
        additional_params=additional_net_params)

    flow_params = dict(
        # name of the experiment
        exp_tag="bottleneck",

        # name of the flow environment the experiment is running on
        env_name=BottleneckDesiredVelocityEnv,

        # name of the network class the experiment is running on
        network=BottleneckNetwork,

        # simulator that is used by the experiment
        simulator='traci',

        # sumo-related parameters (see flow.core.params.SumoParams)
        sim=SumoParams(
            sim_step=0.5,
            render=render,
            save_render=True,
            print_warnings=False,
            restart_instance=True,
        ),

        # environment related parameters (see flow.core.params.EnvParams)
        env=EnvParams(
            warmup_steps=40,
            sims_per_step=1,
            horizon=HORIZON,
            additional_params=additional_env_params,
        ),

        # network-related parameters (see flow.core.params.NetParams and the
        # network's documentation or ADDITIONAL_NET_PARAMS component)
        net=NetParams(
            inflows=inflow,
            additional_params=additional_net_params,
        ),

        # vehicles to be placed in the network at the start of a rollout (see
        # flow.core.params.VehicleParams)
        veh=vehicles,

        # parameters specifying the positioning of vehicles upon initialization/
        # reset (see flow.core.params.InitialConfig)
        initial=InitialConfig(
            spacing="uniform",
            min_gap=5,
            lanes_distribution=float("inf"),
            edges_distribution=["2", "3", "4", "5"],
        ),

        # traffic lights to be introduced to specific nodes (see
        # flow.core.params.TrafficLightParams)
        tls=traffic_lights,
    )

    return flow_params

register(
    id='flow-ring-v0',
    entry_point='d4rl.flow:flow_register',
    max_episode_steps=500,
    kwargs={
        'flow_params': ring_env(render=False),
        'dataset_url': None,
        'ref_min_score': -165.22,
        'ref_max_score': 24.42
    }
)

register(
    id='flow-ring-render-v0',
    entry_point='d4rl.flow:flow_register',
    max_episode_steps=500,
    kwargs={
        'flow_params': ring_env(render='drgb'),
        'dataset_url': None,
        'ref_min_score': -165.22,
        'ref_max_score': 24.42
    }
)

register(
    id='flow-ring-random-v0',
    entry_point='d4rl.flow:flow_register',
    max_episode_steps=500,
    kwargs={
        'flow_params': ring_env(render=False),
        'dataset_url':'http://rail.eecs.berkeley.edu/datasets/offline_rl/flow/flow-ring-v0-random.hdf5',
        'ref_min_score': -165.22,
        'ref_max_score': 24.42
    }
)

register(
    id='flow-ring-controller-v0',
    entry_point='d4rl.flow:flow_register',
    max_episode_steps=500,
    kwargs={
        'flow_params': ring_env(render=False),
        'dataset_url':'http://rail.eecs.berkeley.edu/datasets/offline_rl/flow/flow-ring-v0-idm.hdf5',
        'ref_min_score': -165.22,
        'ref_max_score': 24.42
    }
)

register(
    id='flow-grid-v0',
    entry_point='d4rl.flow:flow_register',
    max_episode_steps=200,
    kwargs={
        'flow_params': traffic_light_grid.gen_env(render=False),
        'dataset_url': None,
        'ref_min_score': -106.968754,
        'ref_max_score': -114.76
    }
)

register(
    id='flow-grid-render-v0',
    entry_point='d4rl.flow:flow_register',
    max_episode_steps=200,
    kwargs={
        'flow_params': traffic_light_grid.gen_env(render='drgb'),
        'dataset_url': None,
        'ref_min_score': -106.968754,
        'ref_max_score': -114.76
    }
)

register(
    id='flow-grid-random-v0',
    entry_point='d4rl.flow:flow_register',
    max_episode_steps=200,
    kwargs={
        'flow_params': traffic_light_grid.gen_env(render=False),
        'dataset_url':'http://rail.eecs.berkeley.edu/datasets/offline_rl/flow/flow-grid-v0-random.hdf5',
        'ref_min_score': -106.968754,
        'ref_max_score': -114.76
    }
)

register(
    id='flow-merge-v0',
    entry_point='d4rl.flow:flow_register',
    max_episode_steps=600,
    kwargs={
        'flow_params': merge.gen_env(render=False),
        'dataset_url': None,
        'ref_min_score': 97.53361,
        'ref_max_score': 266.07
    }
)

register(
    id='flow-merge-render-v0',
    entry_point='d4rl.flow:flow_register',
    max_episode_steps=600,
    kwargs={
        'flow_params': merge.gen_env(render='drgb'),
        'dataset_url': None,
        'ref_min_score': 97.53361,
        'ref_max_score': 266.07
    }
)

register(
    id='flow-merge-random-v0',
    entry_point='d4rl.flow:flow_register',
    max_episode_steps=600,
    kwargs={
        'flow_params': merge.gen_env(render=False),
        'dataset_url':'http://rail.eecs.berkeley.edu/datasets/offline_rl/flow/flow-merge-v0-random.hdf5',
        'ref_min_score': 97.53361,
        'ref_max_score': 266.07
    }
)

register(
    id='flow-merge-controller-v0',
    entry_point='d4rl.flow:flow_register',
    max_episode_steps=600,
    kwargs={
        'flow_params': merge.gen_env(render=False),
        'dataset_url':'http://rail.eecs.berkeley.edu/datasets/offline_rl/flow/flow-merge-v0-idm.hdf5',
        'ref_min_score': 97.53361,
        'ref_max_score': 266.07
    }
)

register(
    id='flow-bottleneck-v0',
    entry_point='d4rl.flow:flow_register',
    max_episode_steps=1000,
    kwargs={
        'flow_params': bottleneck(render=False),
        'dataset_url': None,
        'ref_min_score': 736.59240,
        'ref_max_score': 712.44
    }
)

register(
    id='flow-bottleneck-render-v0',
    entry_point='d4rl.flow:flow_register',
    max_episode_steps=1000,
    kwargs={
        'flow_params': bottleneck(render='drgb'),
        'dataset_url': None,
        'ref_min_score': 736.59240,
        'ref_max_score': 712.44
    }
)

register(
    id='flow-bottleneck-random-v0',
    entry_point='d4rl.flow:flow_register',
    max_episode_steps=1000,
    kwargs={
        'flow_params': bottleneck(render=False),
        'dataset_url':'http://rail.eecs.berkeley.edu/datasets/offline_rl/flow/flow-bottleneck-v0-random.hdf5',
        'ref_min_score': 736.59240,
        'ref_max_score': 712.44
    }
)

<|endoftext|>


<commit_before>"""

conversions.py

@author: derricw

Conversion functions.

"""
from bisect import bisect
import random
import os
import sys

from PIL import Image, ImageOps, ImageDraw, ImageFont
import numpy as np
import cv2
import imageio

from asciisciit.misc import *
from asciisciit.lut import LUM

RESOURCE_DIR = os.path.join(os.path.dirname(__file__),'res')

ASPECTCORRECTIONFACTOR = 6.0/11.0  # because text pixels are rectangular

PY2 = sys.version_info[0] < 3

def image_to_ascii(img, scalefactor=0.2, invert=False, equalize=True, lut='simple'):
    """
    Generates and ascii string from an image of some kind.

    Parameters
    ----------
    img : str, ndarray, PIL.Image
        Image to convert
    scalefactor : float
        ASCII chars per pixel
    invert : bool
        Invert luminance?
    equalize : bool
        Equalize histogram?

    Returns
    -------
    str

    Examples
    --------

    >>> ascii_img = image_to_ascii("http://i.imgur.com/l2FU2J0.jpg", scalefactor=0.3)
    >>> print(ascii_img)

    """
    if type(img) == str:
        img = open_pil_img(img)
    elif type(img) == np.ndarray:
        img = numpy_to_pil(img)
    try:
        text = pil_to_ascii(img, scalefactor, invert, equalize, lut)
    except:
        raise TypeError("That image type doesn't work.  Try PIL, Numpy, or file path...")
    return text

def pil_to_ascii(img,
                 scalefactor=0.2,
                 invert=False,
                 equalize=True,
                 lut='simple',
                 lookup_func=None
                 ):
    """
    Generates an ascii string from a PIL image.

    Parameters
    ----------
    img : PIL.Image
        PIL image to transform.
    scalefactor : float
        ASCII characters per pixel.
    invert : bool
        Invert luminance?
    equalize : bool
        equalize histogram (for best results do this).
    lut : str
        Name of the lookup table to use. Currently supports 'simple' and
        'binary'.
    lookup_func : function
        Method to use to perform lookup. The function should take an image and
        a lut string and return the converted ascii image. Defaults to
        `apply_lut_numpy`.

    Returns
    -------
    str

    Examples
    --------

    >>> from asciisciit.misc import open_pil_img
    >>> img = open_pil_img("http://i.imgur.com/l2FU2J0.jpg")
    >>> text_img = pil_to_ascii(img, scalefactor=0.3)
    >>> print(text_img)

    >>> from PIL import Image
    >>> img = Image.open("some_image.png")
    >>> text_img = pil_to_ascii(img)
    >>> print(text_img)

    """
    if lookup_func is None:
        lookup_func = apply_lut_numpy
    img = img.resize((int(img.size[0]*scalefactor),
        int(img.size[1]*scalefactor*ASPECTCORRECTIONFACTOR)),
        Image.BILINEAR)
    img = img.convert("L")  # convert to mono
    if equalize:
        img = ImageOps.equalize(img)

    if invert:
        img = ImageOps.invert(img)

    return lookup_func(img, lut)

def ascii_to_pil(text, font_size=10, bg_color=(20, 20, 20),
                 fg_color=(255, 255, 255), font_path=None):
    """
    Renders Ascii text to an Image of the appropriate size, using text of the
        specified font size.

    Parameters
    ----------
    text : str
        Ascii text to render.
    font_size : int (10)
        Font size for rendered image.
    bg_color : tuple (20,20,20)
        (R,G,B) values for image background.
    fg_color : tuple (255,255,255)
        (R,G,B) values for text color.
    font_path : str
        Use a custom font .ttf file.

    Returns
    -------
    PIL.Image

    Examples
    --------

    >>> ascii = AsciiImage("http://i.imgur.com/l2FU2J0.jpg", scalefactor=0.4)
    >>> pil = ascii_to_pil(ascii.data)
    >>> pil.show()

    """
    #font = ImageFont.load_default()
    if not font_path:
        font_path = os.path.join(RESOURCE_DIR, "Cousine-Regular.ttf")

    font = ImageFont.truetype(font_path, font_size)
    font_width, font_height = font.getsize(" ")  # shape of 1 char

    img_height, img_width = get_ascii_image_size(text)

    y_padding = 1

    out_img = np.zeros(((font_height+y_padding)*img_height,
                         font_width*img_width,
                         3),
                       dtype=np.uint8)
    out_img[:, :, 0] += bg_color[0]
    out_img[:, :, 1] += bg_color[1]
    out_img[:, :, 2] += bg_color[2]

    img = Image.fromarray(out_img)
    draw = ImageDraw.Draw(img)

    for index, line in enumerate(text.split("\n")):
        y = (font_height+y_padding)*index
        draw.text((0, y), line, fg_color, font=font)
    return img

def ascii_seq_to_gif(seq, output_path, fps=15.0, font_size=10):
    """ Creates a gif from a sequence of ascii images.

        Parameters
        ----------
        output_path : str
            Path for gif output.
        fps : float
            FPS for gif playback.
        font_size : int
            Font size for ascii.
    """
    images = []

    status = StatusBar(len(seq), text="Generating frames: ",)

    for index, ascii_img in enumerate(seq):
        if type(ascii_img) == str:
            #raw text
            text = ascii_img
        else:
            #AsciiImage instance
            text = ascii_img.data
        images.append(ascii_to_pil(text, font_size=font_size))
        status.update(index)

    status.complete()

    duration = 1.0/fps

    images_np = [np.array(img) for img in images]
    imageio.mimsave(output_path, images_np, duration=duration)

def numpy_to_ascii(img,
                   scalefactor=0.2,
                   invert=False,
                   equalize=True,
                   lut="simple"):
    """
    Generates an ascii string from a PIL image.

    Parameters
    ----------
    img : ndarray
        PIL image to transform.
    scalefactor : float
        ASCII characters per pixel.
    invert : bool
        Invert luminance?
    equalize : bool
        equalize histogram (for best results do this).
    lut : str
        Name of the lookup table to use. Currently supports 'simple' and
        'binary'.

    Returns
    -------
    str
    """
    h, w = img.shape

    img=cv2.resize(
        img, (int(w*scalefactor), int(h*scalefactor*ASPECTCORRECTIONFACTOR))
        )

    if img.ndim == 3: # weak check for RGB
        # works in opencv 3.4.3 but who knows, they keep moving/renaming stuff
        img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)

    if equalize:
        img=cv2.equalizeHist(img)

    if invert:
        img = 255-img

    return apply_lut_numpy(img, lut)

def image_to_numpy(path):
    """
    Image file to numpy matrix.
    """
    img = open_pil_img(path)
    return np.array(img, dtype=np.uint8)

def numpy_to_pil(nparray):
    """
    Numpy matrix to PIL Image.
    """
    return Image.fromarray(nparray)

def gif_to_numpy(gif_path):
    """
    Converts a GIF into a numpy movie.
    """
    gif = open_pil_img(gif_path)
    if hasattr(gif, 'info'):
        frame_duration = gif.info.get('duration', None)
    else:
        frame_duration = None
    length = get_length_of_gif(gif)

    size = gif.size

    status = StatusBar(length, "Reading frames: ")

    frames = []
    frame_count = 0
    while gif:
        new_img = Image.new("RGBA",size)
        new_img.paste(gif)
        frames.append(new_img)
        frame_count += 1
        try:
            gif.seek(frame_count)
        except EOFError:
            break
        status.update(frame_count)

    status.complete()

    assert(length==len(frames))

    final_frame_count = len(frames)
    frame1 = np.array(frames[0])
    shape = frame1.shape
    matrix = np.zeros((final_frame_count,shape[0],shape[1],shape[2]), dtype=np.uint8)
    for i, frame in enumerate(frames):
        img = np.asarray(frame)
        matrix[i] = img
    return matrix, frame_duration

def figure_to_numpy(mpl_figure):
    """
    Converts a matplotlib figure to numpy matrix.
    """
    mpl_figure.tight_layout(pad=0.1)
    mpl_figure.canvas.draw()
    data = np.fromstring(mpl_figure.canvas.tostring_rgb(), dtype=np.uint8, sep='')
    data = data.reshape(mpl_figure.canvas.get_width_height()[::-1]+ (3,))
    return data

def figure_to_ascii(mpl_figure):
    """
    Converts a matplotlib figure to ascii image.
    """
    npy_fig = figure_to_numpy(mpl_figure)
    return image_to_ascii(npy_fig, scalefactor=0.15, invert=False, equalize=False)

def apply_lut_pil(img, lut="simple"):
    """
    Apply an ascii lookup table to an image by looping over pixels.

    Parameters
    ----------
    img : ndarray, PIL.Image
        Greyscale image to directly apply LUT to.
    lut : str
        Name of the lookup table to use. Currently supports 'simple' and
        'binary'.

    Returns
    -------
    str
    """
    if isinstance(img, np.ndarray):
        img = numpy_to_pil(img)

    text = "\n"

    chars, lums = LUM[lut.upper()]
    chars = list(chars)

    #SLOW ##TODO: USE Image.point(lut) instead
    for y in range(0, img.size[1]):
        for x in range(0, img.size[0]):
            lum = img.getpixel((x, y))
            row = bisect(lums, lum)
            character = chars[row]
            text += character
        text += "\n"

    return text

def apply_lut_numpy(img, lut="simple"):
    """
    Apply an ascii lookup table to an image using numpy chararrays.

    Parameters
    ----------
    img : ndarray, PIL.Image
        Greyscale image to directly apply LUT to.
    lut : str
        Name of the lookup table to use. Currently supports 'simple' and
        'binary'.

    Returns
    -------
    str
    """
    if isinstance(img, Image.Image):
        img = np.array(img, dtype=np.uint8)

    chars, lums = LUM[lut.upper()]
    lums = np.array(lums)
    if PY2:
        chars = np.chararray(len(chars), buffer=chars)
    else:
        # all my attempts to read the character buffer as unicode failed to
        # correctly populate the chararray, so here we are...
        chars = np.chararray(len(chars), buffer=bytes(chars, "utf-8"))

    text = np.chararray((img.shape[0], img.shape[1]+1))
    text[:,-1] = "\n"
    text[:,:-1] = chars[np.digitize(img, lums)]

    return "\n" + text.tostring().decode("utf-8")

if __name__ == '__main__':
    pass
<commit_msg>Fix typo in docstring<commit_after>"""

conversions.py

@author: derricw

Conversion functions.

"""
from bisect import bisect
import random
import os
import sys

from PIL import Image, ImageOps, ImageDraw, ImageFont
import numpy as np
import cv2
import imageio

from asciisciit.misc import *
from asciisciit.lut import LUM

RESOURCE_DIR = os.path.join(os.path.dirname(__file__),'res')

ASPECTCORRECTIONFACTOR = 6.0/11.0  # because text pixels are rectangular

PY2 = sys.version_info[0] < 3

def image_to_ascii(img, scalefactor=0.2, invert=False, equalize=True, lut='simple'):
    """
    Generates and ascii string from an image of some kind.

    Parameters
    ----------
    img : str, ndarray, PIL.Image
        Image to convert
    scalefactor : float
        ASCII chars per pixel
    invert : bool
        Invert luminance?
    equalize : bool
        Equalize histogram?

    Returns
    -------
    str

    Examples
    --------

    >>> ascii_img = image_to_ascii("http://i.imgur.com/l2FU2J0.jpg", scalefactor=0.3)
    >>> print(ascii_img)

    """
    if type(img) == str:
        img = open_pil_img(img)
    elif type(img) == np.ndarray:
        img = numpy_to_pil(img)
    try:
        text = pil_to_ascii(img, scalefactor, invert, equalize, lut)
    except:
        raise TypeError("That image type doesn't work.  Try PIL, Numpy, or file path...")
    return text

def pil_to_ascii(img,
                 scalefactor=0.2,
                 invert=False,
                 equalize=True,
                 lut='simple',
                 lookup_func=None
                 ):
    """
    Generates an ascii string from a PIL image.

    Parameters
    ----------
    img : PIL.Image
        PIL image to transform.
    scalefactor : float
        ASCII characters per pixel.
    invert : bool
        Invert luminance?
    equalize : bool
        equalize histogram (for best results do this).
    lut : str
        Name of the lookup table to use. Currently supports 'simple' and
        'binary'.
    lookup_func : function
        Method to use to perform lookup. The function should take an image and
        a lut string and return the converted ascii image. Defaults to
        `apply_lut_numpy`.

    Returns
    -------
    str

    Examples
    --------

    >>> from asciisciit.misc import open_pil_img
    >>> img = open_pil_img("http://i.imgur.com/l2FU2J0.jpg")
    >>> text_img = pil_to_ascii(img, scalefactor=0.3)
    >>> print(text_img)

    >>> from PIL import Image
    >>> img = Image.open("some_image.png")
    >>> text_img = pil_to_ascii(img)
    >>> print(text_img)

    """
    if lookup_func is None:
        lookup_func = apply_lut_numpy
    img = img.resize((int(img.size[0]*scalefactor),
        int(img.size[1]*scalefactor*ASPECTCORRECTIONFACTOR)),
        Image.BILINEAR)
    img = img.convert("L")  # convert to mono
    if equalize:
        img = ImageOps.equalize(img)

    if invert:
        img = ImageOps.invert(img)

    return lookup_func(img, lut)

def ascii_to_pil(text, font_size=10, bg_color=(20, 20, 20),
                 fg_color=(255, 255, 255), font_path=None):
    """
    Renders Ascii text to an Image of the appropriate size, using text of the
        specified font size.

    Parameters
    ----------
    text : str
        Ascii text to render.
    font_size : int (10)
        Font size for rendered image.
    bg_color : tuple (20,20,20)
        (R,G,B) values for image background.
    fg_color : tuple (255,255,255)
        (R,G,B) values for text color.
    font_path : str
        Use a custom font .ttf file.

    Returns
    -------
    PIL.Image

    Examples
    --------

    >>> ascii = AsciiImage("http://i.imgur.com/l2FU2J0.jpg", scalefactor=0.4)
    >>> pil = ascii_to_pil(ascii.data)
    >>> pil.show()

    """
    #font = ImageFont.load_default()
    if not font_path:
        font_path = os.path.join(RESOURCE_DIR, "Cousine-Regular.ttf")

    font = ImageFont.truetype(font_path, font_size)
    font_width, font_height = font.getsize(" ")  # shape of 1 char

    img_height, img_width = get_ascii_image_size(text)

    y_padding = 1

    out_img = np.zeros(((font_height+y_padding)*img_height,
                         font_width*img_width,
                         3),
                       dtype=np.uint8)
    out_img[:, :, 0] += bg_color[0]
    out_img[:, :, 1] += bg_color[1]
    out_img[:, :, 2] += bg_color[2]

    img = Image.fromarray(out_img)
    draw = ImageDraw.Draw(img)

    for index, line in enumerate(text.split("\n")):
        y = (font_height+y_padding)*index
        draw.text((0, y), line, fg_color, font=font)
    return img

def ascii_seq_to_gif(seq, output_path, fps=15.0, font_size=10):
    """ Creates a gif from a sequence of ascii images.

        Parameters
        ----------
        output_path : str
            Path for gif output.
        fps : float
            FPS for gif playback.
        font_size : int
            Font size for ascii.
    """
    images = []

    status = StatusBar(len(seq), text="Generating frames: ",)

    for index, ascii_img in enumerate(seq):
        if type(ascii_img) == str:
            #raw text
            text = ascii_img
        else:
            #AsciiImage instance
            text = ascii_img.data
        images.append(ascii_to_pil(text, font_size=font_size))
        status.update(index)

    status.complete()

    duration = 1.0/fps

    images_np = [np.array(img) for img in images]
    imageio.mimsave(output_path, images_np, duration=duration)

def numpy_to_ascii(img,
                   scalefactor=0.2,
                   invert=False,
                   equalize=True,
                   lut="simple"):
    """
    Generates an ascii string from a numpy image.

    Parameters
    ----------
    img : ndarray
        PIL image to transform.
    scalefactor : float
        ASCII characters per pixel.
    invert : bool
        Invert luminance?
    equalize : bool
        equalize histogram (for best results do this).
    lut : str
        Name of the lookup table to use. Currently supports 'simple' and
        'binary'.

    Returns
    -------
    str
    """
    h, w = img.shape

    img=cv2.resize(
        img, (int(w*scalefactor), int(h*scalefactor*ASPECTCORRECTIONFACTOR))
        )

    if img.ndim == 3: # weak check for RGB
        # works in opencv 3.4.3 but who knows, they keep moving/renaming stuff
        img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)

    if equalize:
        img=cv2.equalizeHist(img)

    if invert:
        img = 255-img

    return apply_lut_numpy(img, lut)

def image_to_numpy(path):
    """
    Image file to numpy matrix.
    """
    img = open_pil_img(path)
    return np.array(img, dtype=np.uint8)

def numpy_to_pil(nparray):
    """
    Numpy matrix to PIL Image.
    """
    return Image.fromarray(nparray)

def gif_to_numpy(gif_path):
    """
    Converts a GIF into a numpy movie.
    """
    gif = open_pil_img(gif_path)
    if hasattr(gif, 'info'):
        frame_duration = gif.info.get('duration', None)
    else:
        frame_duration = None
    length = get_length_of_gif(gif)

    size = gif.size

    status = StatusBar(length, "Reading frames: ")

    frames = []
    frame_count = 0
    while gif:
        new_img = Image.new("RGBA",size)
        new_img.paste(gif)
        frames.append(new_img)
        frame_count += 1
        try:
            gif.seek(frame_count)
        except EOFError:
            break
        status.update(frame_count)

    status.complete()

    assert(length==len(frames))

    final_frame_count = len(frames)
    frame1 = np.array(frames[0])
    shape = frame1.shape
    matrix = np.zeros((final_frame_count,shape[0],shape[1],shape[2]), dtype=np.uint8)
    for i, frame in enumerate(frames):
        img = np.asarray(frame)
        matrix[i] = img
    return matrix, frame_duration

def figure_to_numpy(mpl_figure):
    """
    Converts a matplotlib figure to numpy matrix.
    """
    mpl_figure.tight_layout(pad=0.1)
    mpl_figure.canvas.draw()
    data = np.fromstring(mpl_figure.canvas.tostring_rgb(), dtype=np.uint8, sep='')
    data = data.reshape(mpl_figure.canvas.get_width_height()[::-1]+ (3,))
    return data

def figure_to_ascii(mpl_figure):
    """
    Converts a matplotlib figure to ascii image.
    """
    npy_fig = figure_to_numpy(mpl_figure)
    return image_to_ascii(npy_fig, scalefactor=0.15, invert=False, equalize=False)

def apply_lut_pil(img, lut="simple"):
    """
    Apply an ascii lookup table to an image by looping over pixels.

    Parameters
    ----------
    img : ndarray, PIL.Image
        Greyscale image to directly apply LUT to.
    lut : str
        Name of the lookup table to use. Currently supports 'simple' and
        'binary'.

    Returns
    -------
    str
    """
    if isinstance(img, np.ndarray):
        img = numpy_to_pil(img)

    text = "\n"

    chars, lums = LUM[lut.upper()]
    chars = list(chars)

    #SLOW ##TODO: USE Image.point(lut) instead
    for y in range(0, img.size[1]):
        for x in range(0, img.size[0]):
            lum = img.getpixel((x, y))
            row = bisect(lums, lum)
            character = chars[row]
            text += character
        text += "\n"

    return text

def apply_lut_numpy(img, lut="simple"):
    """
    Apply an ascii lookup table to an image using numpy chararrays.

    Parameters
    ----------
    img : ndarray, PIL.Image
        Greyscale image to directly apply LUT to.
    lut : str
        Name of the lookup table to use. Currently supports 'simple' and
        'binary'.

    Returns
    -------
    str
    """
    if isinstance(img, Image.Image):
        img = np.array(img, dtype=np.uint8)

    chars, lums = LUM[lut.upper()]
    lums = np.array(lums)
    if PY2:
        chars = np.chararray(len(chars), buffer=chars)
    else:
        # all my attempts to read the character buffer as unicode failed to
        # correctly populate the chararray, so here we are...
        chars = np.chararray(len(chars), buffer=bytes(chars, "utf-8"))

    text = np.chararray((img.shape[0], img.shape[1]+1))
    text[:,-1] = "\n"
    text[:,:-1] = chars[np.digitize(img, lums)]

    return "\n" + text.tostring().decode("utf-8")

if __name__ == '__main__':
    pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.conf import settings
from django.conf.urls import patterns, include, url
from django.conf.urls.static import static
from django.views.generic import TemplateView

# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    url(r'^media/$',
        TemplateView.as_view(template_name='pages/media.html'),
        name="media"),

    # Uncomment the next line to enable the admin:
    url(r'^admin/', include(admin.site.urls)),

    # User management
    # url(r'^users/', include("users.urls", namespace="users")),
    # url(r'^accounts/', include('allauth.urls')),

    # Uncomment the next line to enable avatars
    # url(r'^avatar/', include('avatar.urls')),

    # Your stuff: custom urls go here
    url(r'^', include('petition.urls', namespace='petition')),
    url('^', include('django.contrib.flatpages.urls')),
    url('^contact/', include('contact.urls', namespace="contact")),

) + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
<commit_msg>Fix contact page<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.conf import settings
from django.conf.urls import patterns, include, url
from django.conf.urls.static import static
from django.views.generic import TemplateView

# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    url(r'^media/$',
        TemplateView.as_view(template_name='pages/media.html'),
        name="media"),

    # Uncomment the next line to enable the admin:
    url(r'^admin/', include(admin.site.urls)),

    # User management
    # url(r'^users/', include("users.urls", namespace="users")),
    # url(r'^accounts/', include('allauth.urls')),

    # Uncomment the next line to enable avatars
    # url(r'^avatar/', include('avatar.urls')),

    # Your stuff: custom urls go here
    url(r'^', include('petition.urls', namespace='petition')),
    url('^contact', include('contact.urls', namespace="contact")),
    url('^', include('django.contrib.flatpages.urls')),

) + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
<|endoftext|>


<commit_before><commit_msg>New Dependency Manager (from github session)<commit_after><|endoftext|>


<commit_before>#
# Copyright (C) 2012 - 2017 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
"""Misc utility routines for anyconfig module.
"""
from __future__ import absolute_import

import collections
import glob
import os.path
import types

import anyconfig.compat

def get_file_extension(file_path):
    """
    >>> get_file_extension("/a/b/c")
    ''
    >>> get_file_extension("/a/b.txt")
    'txt'
    >>> get_file_extension("/a/b/c.tar.xz")
    'xz'
    """
    _ext = os.path.splitext(file_path)[-1]
    if _ext:
        return _ext[1:] if _ext.startswith('.') else _ext

    return ""

def sglob(files_pattern):
    """
    glob.glob alternative of which results sorted always.
    """
    return sorted(glob.glob(files_pattern))

def is_iterable(obj):
    """
    >>> is_iterable([])
    True
    >>> is_iterable(())
    True
    >>> is_iterable([x for x in range(10)])
    True
    >>> is_iterable((1, 2, 3))
    True
    >>> g = (x for x in range(10))
    >>> is_iterable(g)
    True
    >>> is_iterable("abc")
    False
    >>> is_iterable(0)
    False
    >>> is_iterable({})
    False
    """
    return isinstance(obj, (list, tuple, types.GeneratorType)) or \
        (not isinstance(obj, (int, str, dict)) and
         bool(getattr(obj, "next", False)))

def concat(xss):
    """
    Concatenates a list of lists.

    >>> concat([[]])
    []
    >>> concat((()))
    []
    >>> concat([[1,2,3],[4,5]])
    [1, 2, 3, 4, 5]
    >>> concat([[1,2,3],[4,5,[6,7]]])
    [1, 2, 3, 4, 5, [6, 7]]
    >>> concat(((1,2,3),(4,5,[6,7])))
    [1, 2, 3, 4, 5, [6, 7]]
    >>> concat(((1,2,3),(4,5,[6,7])))
    [1, 2, 3, 4, 5, [6, 7]]
    >>> concat((i, i*2) for i in range(3))
    [0, 0, 1, 2, 2, 4]
    """
    return list(anyconfig.compat.from_iterable(xs for xs in xss))

def normpath(path):
    """Normalize path.

    - eliminating double slashes, etc. (os.path.normpath)
    - ensure paths contain ~[user]/ expanded.

    :param path: Path string :: str
    """
    return os.path.normpath(os.path.expanduser(path) if '~' in path else path)

def is_path(path_or_stream):
    """
    Is given object `path_or_stream` a file path?

    :param path_or_stream: file path or stream, file/file-like object
    :return: True if `path_or_stream` is a file path
    """
    return isinstance(path_or_stream, anyconfig.compat.STR_TYPES)

def get_path_from_stream(maybe_stream):
    """
    Try to get file path from given stream `stream`.

    :param maybe_stream: A file or file-like object
    :return: Path of given file or file-like object or None

    >>> __file__ == get_path_from_stream(__file__)
    True
    >>> __file__ == get_path_from_stream(open(__file__, 'r'))
    True
    >>> strm = anyconfig.compat.StringIO()
    >>> get_path_from_stream(strm) is None
    True
    """
    if is_path(maybe_stream):
        return maybe_stream  # It's path.

    maybe_path = getattr(maybe_stream, "name", None)
    if maybe_path is not None:
        maybe_path = os.path.abspath(maybe_path)

    return maybe_path

def _try_to_get_extension(path_or_strm):
    """
    Try to get file extension from given path or file object.

    :return: File extension or None
    """
    path = get_path_from_stream(path_or_strm)
    if path is None:
        return None

    return get_file_extension(path) or None

def are_same_file_types(paths):
    """
    Are given (maybe) file paths same type (extension) ?

    :param paths: A list of file path or file(-like) objects

    >>> are_same_file_types([])
    False
    >>> are_same_file_types(["a.conf"])
    True
    >>> are_same_file_types(["a.conf", "b.conf"])
    True
    >>> are_same_file_types(["a.yml", "b.yml"])
    True
    >>> are_same_file_types(["a.yml", "b.json"])
    False
    >>> strm = anyconfig.compat.StringIO()
    >>> are_same_file_types(["a.yml", "b.yml", strm])
    False
    """
    if not paths:
        return False

    ext = _try_to_get_extension(paths[0])
    if ext is None:
        return False

    return all(_try_to_get_extension(p) == ext for p in paths[1:])

def _norm_paths_itr(paths, marker='*'):
    """Iterator version of :func:`norm_paths`.
    """
    for path in paths:
        if is_path(path):
            if marker in path:  # glob path pattern
                for ppath in sglob(path):
                    yield ppath
            else:
                yield path  # a simple file path
        else:  # A file or file-like object
            yield path

def norm_paths(paths, marker='*'):
    """
    :param paths:
        A glob path pattern string, or a list consists of path strings or glob
        path pattern strings or file objects
    :param marker: Glob marker character or string, e.g. '*'
    :return: List of path strings

    >>> norm_paths([])
    []
    >>> norm_paths("/usr/lib/a/b.conf /etc/a/b.conf /run/a/b.conf".split())
    ['/usr/lib/a/b.conf', '/etc/a/b.conf', '/run/a/b.conf']
    >>> paths_s = os.path.join(os.path.dirname(__file__), "u*.py")
    >>> ref = sglob(paths_s)
    >>> assert norm_paths(paths_s) == ref
    >>> ref = ["/etc/a.conf"] + ref
    >>> assert norm_paths(["/etc/a.conf", paths_s]) == ref
    >>> strm = anyconfig.compat.StringIO()
    >>> assert norm_paths(["/etc/a.conf", strm]) == ["/etc/a.conf", strm]
    """
    if is_path(paths) and marker in paths:
        return sglob(paths)

    return list(_norm_paths_itr(paths, marker=marker))

# pylint: disable=unused-argument
def noop(val, *args, **kwargs):
    """A function does nothing.

    >>> noop(1)
    1
    """
    # It means nothing but can suppress 'Unused argument' pylint warns.
    # (val, args, kwargs)[0]
    return val

_LIST_LIKE_TYPES = (collections.Iterable, collections.Sequence)

def is_dict_like(obj):
    """
    :param obj: Any object behaves like a dict.

    >>> is_dict_like("a string")
    False
    >>> is_dict_like({})
    True
    >>> is_dict_like(anyconfig.compat.OrderedDict((('a', 1), ('b', 2))))
    True
    """
    return isinstance(obj, (dict, collections.Mapping))  # any others?

def is_namedtuple(obj):
    """
    >>> p0 = collections.namedtuple("Point", "x y")(1, 2)
    >>> is_namedtuple(p0)
    True
    >>> is_namedtuple(tuple(p0))
    False
    """
    return isinstance(obj, tuple) and hasattr(obj, "_asdict")

def is_list_like(obj):
    """
    >>> is_list_like([])
    True
    >>> is_list_like(())
    True
    >>> is_list_like([x for x in range(10)])
    True
    >>> is_list_like((1, 2, 3))
    True
    >>> g = (x for x in range(10))
    >>> is_list_like(g)
    True
    >>> is_list_like("abc")
    False
    >>> is_list_like(0)
    False
    >>> is_list_like({})
    False
    """
    return isinstance(obj, _LIST_LIKE_TYPES) and \
        not (isinstance(obj, anyconfig.compat.STR_TYPES) or is_dict_like(obj))

def filter_options(keys, options):
    """
    Filter `options` with given `keys`.

    :param keys: key names of optional keyword arguments
    :param options: optional keyword arguments to filter with `keys`

    >>> filter_options(("aaa", ), dict(aaa=1, bbb=2))
    {'aaa': 1}
    >>> filter_options(("aaa", ), dict(bbb=2))
    {}
    """
    return dict((k, options[k]) for k in keys if k in options)

# vim:sw=4:ts=4:et:
<commit_msg>enhancement: add .utils.is_path_obj to check if given argument is pathlib.Path object<commit_after>#
# Copyright (C) 2012 - 2017 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
"""Misc utility routines for anyconfig module.
"""
from __future__ import absolute_import

import collections
import glob
import os.path
import types

import anyconfig.compat

def get_file_extension(file_path):
    """
    >>> get_file_extension("/a/b/c")
    ''
    >>> get_file_extension("/a/b.txt")
    'txt'
    >>> get_file_extension("/a/b/c.tar.xz")
    'xz'
    """
    _ext = os.path.splitext(file_path)[-1]
    if _ext:
        return _ext[1:] if _ext.startswith('.') else _ext

    return ""

def sglob(files_pattern):
    """
    glob.glob alternative of which results sorted always.
    """
    return sorted(glob.glob(files_pattern))

def is_iterable(obj):
    """
    >>> is_iterable([])
    True
    >>> is_iterable(())
    True
    >>> is_iterable([x for x in range(10)])
    True
    >>> is_iterable((1, 2, 3))
    True
    >>> g = (x for x in range(10))
    >>> is_iterable(g)
    True
    >>> is_iterable("abc")
    False
    >>> is_iterable(0)
    False
    >>> is_iterable({})
    False
    """
    return isinstance(obj, (list, tuple, types.GeneratorType)) or \
        (not isinstance(obj, (int, str, dict)) and
         bool(getattr(obj, "next", False)))

def concat(xss):
    """
    Concatenates a list of lists.

    >>> concat([[]])
    []
    >>> concat((()))
    []
    >>> concat([[1,2,3],[4,5]])
    [1, 2, 3, 4, 5]
    >>> concat([[1,2,3],[4,5,[6,7]]])
    [1, 2, 3, 4, 5, [6, 7]]
    >>> concat(((1,2,3),(4,5,[6,7])))
    [1, 2, 3, 4, 5, [6, 7]]
    >>> concat(((1,2,3),(4,5,[6,7])))
    [1, 2, 3, 4, 5, [6, 7]]
    >>> concat((i, i*2) for i in range(3))
    [0, 0, 1, 2, 2, 4]
    """
    return list(anyconfig.compat.from_iterable(xs for xs in xss))

def normpath(path):
    """Normalize path.

    - eliminating double slashes, etc. (os.path.normpath)
    - ensure paths contain ~[user]/ expanded.

    :param path: Path string :: str
    """
    return os.path.normpath(os.path.expanduser(path) if '~' in path else path)

def is_path(path_or_stream):
    """
    Is given object `path_or_stream` a file path?

    :param path_or_stream: file path or stream, file/file-like object
    :return: True if `path_or_stream` is a file path
    """
    return isinstance(path_or_stream, anyconfig.compat.STR_TYPES)

def is_path_obj(input_):
    """Is given object `input` a pathlib.Path object?

    :param input_: Input object may be pathlib.Path object or something
    :return: True if `input_` is a pathlib.Path object

    >>> from anyconfig.compat import pathlib
    >>> if pathlib is not None:
    ...      input_ = pathlib.Path(__file__)
    ...      assert is_path_obj(input_)
    >>>
    >>> assert not is_path_obj(__file__)
    """
    return (anyconfig.compat.pathlib is not None and
            isinstance(input_, anyconfig.compat.pathlib.Path))

def get_path_from_stream(maybe_stream):
    """
    Try to get file path from given stream `stream`.

    :param maybe_stream: A file or file-like object
    :return: Path of given file or file-like object or None

    >>> __file__ == get_path_from_stream(__file__)
    True
    >>> __file__ == get_path_from_stream(open(__file__, 'r'))
    True
    >>> strm = anyconfig.compat.StringIO()
    >>> get_path_from_stream(strm) is None
    True
    """
    if is_path(maybe_stream):
        return maybe_stream  # It's path.

    maybe_path = getattr(maybe_stream, "name", None)
    if maybe_path is not None:
        maybe_path = os.path.abspath(maybe_path)

    return maybe_path

def _try_to_get_extension(path_or_strm):
    """
    Try to get file extension from given path or file object.

    :return: File extension or None
    """
    path = get_path_from_stream(path_or_strm)
    if path is None:
        return None

    return get_file_extension(path) or None

def are_same_file_types(paths):
    """
    Are given (maybe) file paths same type (extension) ?

    :param paths: A list of file path or file(-like) objects

    >>> are_same_file_types([])
    False
    >>> are_same_file_types(["a.conf"])
    True
    >>> are_same_file_types(["a.conf", "b.conf"])
    True
    >>> are_same_file_types(["a.yml", "b.yml"])
    True
    >>> are_same_file_types(["a.yml", "b.json"])
    False
    >>> strm = anyconfig.compat.StringIO()
    >>> are_same_file_types(["a.yml", "b.yml", strm])
    False
    """
    if not paths:
        return False

    ext = _try_to_get_extension(paths[0])
    if ext is None:
        return False

    return all(_try_to_get_extension(p) == ext for p in paths[1:])

def _norm_paths_itr(paths, marker='*'):
    """Iterator version of :func:`norm_paths`.
    """
    for path in paths:
        if is_path(path):
            if marker in path:  # glob path pattern
                for ppath in sglob(path):
                    yield ppath
            else:
                yield path  # a simple file path
        else:  # A file or file-like object
            yield path

def norm_paths(paths, marker='*'):
    """
    :param paths:
        A glob path pattern string, or a list consists of path strings or glob
        path pattern strings or file objects
    :param marker: Glob marker character or string, e.g. '*'
    :return: List of path strings

    >>> norm_paths([])
    []
    >>> norm_paths("/usr/lib/a/b.conf /etc/a/b.conf /run/a/b.conf".split())
    ['/usr/lib/a/b.conf', '/etc/a/b.conf', '/run/a/b.conf']
    >>> paths_s = os.path.join(os.path.dirname(__file__), "u*.py")
    >>> ref = sglob(paths_s)
    >>> assert norm_paths(paths_s) == ref
    >>> ref = ["/etc/a.conf"] + ref
    >>> assert norm_paths(["/etc/a.conf", paths_s]) == ref
    >>> strm = anyconfig.compat.StringIO()
    >>> assert norm_paths(["/etc/a.conf", strm]) == ["/etc/a.conf", strm]
    """
    if is_path(paths) and marker in paths:
        return sglob(paths)

    return list(_norm_paths_itr(paths, marker=marker))

# pylint: disable=unused-argument
def noop(val, *args, **kwargs):
    """A function does nothing.

    >>> noop(1)
    1
    """
    # It means nothing but can suppress 'Unused argument' pylint warns.
    # (val, args, kwargs)[0]
    return val

_LIST_LIKE_TYPES = (collections.Iterable, collections.Sequence)

def is_dict_like(obj):
    """
    :param obj: Any object behaves like a dict.

    >>> is_dict_like("a string")
    False
    >>> is_dict_like({})
    True
    >>> is_dict_like(anyconfig.compat.OrderedDict((('a', 1), ('b', 2))))
    True
    """
    return isinstance(obj, (dict, collections.Mapping))  # any others?

def is_namedtuple(obj):
    """
    >>> p0 = collections.namedtuple("Point", "x y")(1, 2)
    >>> is_namedtuple(p0)
    True
    >>> is_namedtuple(tuple(p0))
    False
    """
    return isinstance(obj, tuple) and hasattr(obj, "_asdict")

def is_list_like(obj):
    """
    >>> is_list_like([])
    True
    >>> is_list_like(())
    True
    >>> is_list_like([x for x in range(10)])
    True
    >>> is_list_like((1, 2, 3))
    True
    >>> g = (x for x in range(10))
    >>> is_list_like(g)
    True
    >>> is_list_like("abc")
    False
    >>> is_list_like(0)
    False
    >>> is_list_like({})
    False
    """
    return isinstance(obj, _LIST_LIKE_TYPES) and \
        not (isinstance(obj, anyconfig.compat.STR_TYPES) or is_dict_like(obj))

def filter_options(keys, options):
    """
    Filter `options` with given `keys`.

    :param keys: key names of optional keyword arguments
    :param options: optional keyword arguments to filter with `keys`

    >>> filter_options(("aaa", ), dict(aaa=1, bbb=2))
    {'aaa': 1}
    >>> filter_options(("aaa", ), dict(bbb=2))
    {}
    """
    return dict((k, options[k]) for k in keys if k in options)

# vim:sw=4:ts=4:et:
<|endoftext|>


<commit_before>#
# Copyright (C) 2015 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
"""anytemplate.api - API of anytemplate module
"""
from __future__ import absolute_import

import os.path

import anytemplate.compat
import anytemplate.engine
import anytemplate.globals
import anytemplate.utils

# Aliases:
LOGGER = anytemplate.globals.LOGGER
TemplateNotFound = anytemplate.engine.TemplateNotFound

class TemplateEngineNotFound(Exception):
    """
    Raised if no any appropriate template engines were found.
    """
    pass

def list_engines():
    """
    List available template engines sorted by each priority.

    :return: A list of child classes of the class
        :class:`anytemplate.engines.base.Engine`
    """
    return anytemplate.engine.list_engines_by_priority()

def find_engine(filepath=None, name=None):
    """
    :param filepath: Template file path
    :param name: Specify the name of template engine to use explicitly or
        None; it will be selected automatically anyhow.

    :return: Template engine class found
    """
    if name is None:
        if filepath is None:
            engines = list_engines()
        else:
            engines = anytemplate.engine.find_by_filename(filepath)

        if not engines:
            raise TemplateEngineNotFound("filename=%s" % str(filepath))

        return engines[0]  # It should have highest priority.
    else:
        engine = anytemplate.engine.find_by_name(name)
        if engine is None:
            raise TemplateEngineNotFound("(template) name=%s" % name)

        return engine

def renders(template_content, context=None, at_paths=None,
            at_encoding=anytemplate.compat.ENCODING,
            at_engine=None, at_ask_missing=False,
            at_cls_args=None, **kwargs):
    """
    Compile and render given template string and return the result string.

    :param template_content: Template content string
    :param context: A dict or dict-like object to instantiate given
        template file
    :param at_paths: Template search paths
    :param at_encoding: Template encoding
    :param at_engine: Specify the name of template engine to use explicitly or
        None to find it automatically anyhow.
    :param at_cls_args: Arguments passed to instantiate template engine class
    :param kwargs: Keyword arguments passed to the template engine to
        render templates with specific features enabled.

    :return: Rendered string
    """
    ecls = find_engine(None, at_engine)
    LOGGER.info("Use the template engine: %s", ecls.name())
    engine = ecls() if at_cls_args is None else ecls(**at_cls_args)
    at_paths = anytemplate.utils.mk_template_paths(None, at_paths)

    try:
        return engine.renders(template_content, context=context,
                              at_paths=at_paths, at_encoding=at_encoding,
                              **kwargs)
    except anytemplate.engine.TemplateNotFound:
        if not at_ask_missing:
            raise

        usr_tmpl = anytemplate.compat.raw_input(
            "\n*** Missing template included. Please enter absolute or "
            "relative path starting from '.' to the template file: "
        )
        usr_tmpl = anytemplate.utils.normpath(usr_tmpl.strip())
        usr_tmpldir = os.path.dirname(usr_tmpl)

        return engine.renders(template_content, context=context,
                              at_paths=(at_paths + [usr_tmpldir]),
                              at_encoding=at_encoding, **kwargs)

def render(filepath, context=None, at_paths=None,
           at_encoding=anytemplate.compat.ENCODING,
           at_engine=None, at_ask_missing=False,
           at_cls_args=None, **kwargs):
    """
    Compile and render given template file and return the result string.

    :param template: Template file path
    :param context: A dict or dict-like object to instantiate given
        template file
    :param at_paths: Template search paths
    :param at_encoding: Template encoding
    :param at_engine: Specify the name of template engine to use explicitly or
        None to find it automatically anyhow.
    :param at_cls_args: Arguments passed to instantiate template engine class
    :param kwargs: Keyword arguments passed to the template engine to
        render templates with specific features enabled.

    :return: Rendered string
    """
    ecls = find_engine(filepath, at_engine)
    LOGGER.info("Use the template engine: %s", ecls.name())
    engine = ecls() if at_cls_args is None else ecls(**at_cls_args)

    if at_paths is None:
        at_paths = anytemplate.utils.mk_template_paths(filepath, at_paths)
    try:
        return engine.render(filepath, context=context, at_paths=at_paths,
                             at_encoding=at_encoding, **kwargs)
    except anytemplate.engine.TemplateNotFound:
        if not at_ask_missing:
            raise

        usr_tmpl = anytemplate.compat.raw_input(
            "\n*** Missing template '%s' or any/one of templates included "
            "from %s. Please enter absolute or relative path starting "
            "from '.' to the template file: " % (filepath, filepath)
        )
        usr_tmpl = anytemplate.utils.normpath(usr_tmpl.strip())
        usr_tmpldir = os.path.dirname(usr_tmpl)

        if os.path.basename(filepath) == os.path.basename(usr_tmpl):
            tmpl = usr_tmpl
        else:
            tmpl = filepath

        return engine.render(tmpl, context=context,
                             at_paths=(at_paths + [usr_tmpldir]),
                             at_encoding=at_encoding, **kwargs)

def render_to(filepath, context=None, output=None, at_paths=None,
              at_encoding=anytemplate.compat.ENCODING,
              at_engine=None, at_ask_missing=False,
              at_cls_args=None, **kwargs):
    """
    Render given template file and write the result string to given `output`.
    The result string will be printed to sys.stdout if output is None or '-'.

    :param template: Template file path
    :param context: A dict or dict-like object to instantiate given
        template file
    :param output: File path to write the rendered result string to or None/'-'
        to print it to stdout
    :param at_paths: Template search paths
    :param at_encoding: Template encoding
    :param at_engine: Specify the name of template engine to use explicitly or
        None to find it automatically anyhow.
    :param at_cls_args: Arguments passed to instantiate template engine class
    :param kwargs: Keyword arguments passed to the template engine to
        render templates with specific features enabled.
    """
    res = render(filepath, context=context, at_paths=at_paths,
                 at_encoding=at_encoding, at_engine=at_engine,
                 at_ask_missing=at_ask_missing, at_cls_args=at_cls_args,
                 **kwargs)
    anytemplate.utils.write_to_output(res, output, at_encoding)

# vim:sw=4:ts=4:et:
<commit_msg>refactor: extract common parts in anytemplate.api.render{s,} to anytemplate.api.__render<commit_after>#
# Copyright (C) 2015 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
"""anytemplate.api - API of anytemplate module
"""
from __future__ import absolute_import

import os.path

import anytemplate.compat
import anytemplate.engine
import anytemplate.globals
import anytemplate.utils

# Aliases:
LOGGER = anytemplate.globals.LOGGER
TemplateNotFound = anytemplate.engine.TemplateNotFound

class TemplateEngineNotFound(Exception):
    """
    Raised if no any appropriate template engines were found.
    """
    pass

def list_engines():
    """
    List available template engines sorted by each priority.

    :return: A list of child classes of the class
        :class:`anytemplate.engines.base.Engine`
    """
    return anytemplate.engine.list_engines_by_priority()

def find_engine(filepath=None, name=None):
    """
    :param filepath: Template file path
    :param name: Specify the name of template engine to use explicitly or
        None; it will be selected automatically anyhow.

    :return: Template engine class found
    """
    if name is None:
        if filepath is None:
            engines = list_engines()
        else:
            engines = anytemplate.engine.find_by_filename(filepath)

        if not engines:
            raise TemplateEngineNotFound("filename=%s" % str(filepath))

        return engines[0]  # It should have highest priority.
    else:
        engine = anytemplate.engine.find_by_name(name)
        if engine is None:
            raise TemplateEngineNotFound("(template) name=%s" % name)

        return engine

def __render(template=None, filepath=None, context=None, at_paths=None,
             at_encoding=anytemplate.compat.ENCODING, at_engine=None,
             at_ask_missing=False, at_cls_args=None, **kwargs):
    """
    Compile and render given template string and return the result string.

    :param template: Template content string or None
    :param filepath: Template file path or None
    :param context: A dict or dict-like object to instantiate given
        template file
    :param at_paths: Template search paths
    :param at_encoding: Template encoding
    :param at_engine: Specify the name of template engine to use explicitly or
        None to find it automatically anyhow.
    :param at_cls_args: Arguments passed to instantiate template engine class
    :param kwargs: Keyword arguments passed to the template engine to
        render templates with specific features enabled.

    :return: Rendered string
    """
    ecls = find_engine(filepath, at_engine)
    LOGGER.info("Use the template engine: %s", ecls.name())
    engine = ecls() if at_cls_args is None else ecls(**at_cls_args)
    at_paths = anytemplate.utils.mk_template_paths(filepath, at_paths)

    if filepath is None:
        (render_fn, target) = (engine.renders, template)
    else:
        (render_fn, target) = (engine.render, filepath)

    try:
        return render_fn(target, context=context, at_paths=at_paths,
                         at_encoding=at_encoding, **kwargs)
    except anytemplate.engine.TemplateNotFound:
        if not at_ask_missing:
            raise

        usr_tmpl = anytemplate.compat.raw_input(
            "\n*** Missing template included. Please enter absolute or "
            "relative path starting from '.' to the template file: "
        )
        usr_tmpl = anytemplate.utils.normpath(usr_tmpl.strip())
        usr_tmpldir = os.path.dirname(usr_tmpl)

        return render_fn(target, context=context,
                         at_paths=(at_paths + [usr_tmpldir]),
                         at_encoding=at_encoding, **kwargs)

def renders(template_content, context=None, at_paths=None,
            at_encoding=anytemplate.compat.ENCODING,
            at_engine=None, at_ask_missing=False,
            at_cls_args=None, **kwargs):
    """
    Compile and render given template string and return the result string.

    :param template_content: Template content string
    :param context: A dict or dict-like object to instantiate given
        template file
    :param at_paths: Template search paths
    :param at_encoding: Template encoding
    :param at_engine: Specify the name of template engine to use explicitly or
        None to find it automatically anyhow.
    :param at_cls_args: Arguments passed to instantiate template engine class
    :param kwargs: Keyword arguments passed to the template engine to
        render templates with specific features enabled.

    :return: Rendered string
    """
    return __render(template_content, context=context, at_paths=at_paths,
                    at_encoding=at_encoding, at_engine=at_engine,
                    at_ask_missing=at_ask_missing, at_cls_args=at_cls_args,
                    **kwargs)

def render(filepath, context=None, at_paths=None,
           at_encoding=anytemplate.compat.ENCODING,
           at_engine=None, at_ask_missing=False,
           at_cls_args=None, **kwargs):
    """
    Compile and render given template file and return the result string.

    :param template: Template file path
    :param context: A dict or dict-like object to instantiate given
        template file
    :param at_paths: Template search paths
    :param at_encoding: Template encoding
    :param at_engine: Specify the name of template engine to use explicitly or
        None to find it automatically anyhow.
    :param at_cls_args: Arguments passed to instantiate template engine class
    :param kwargs: Keyword arguments passed to the template engine to
        render templates with specific features enabled.

    :return: Rendered string
    """
    return __render(filepath=filepath, context=context, at_paths=at_paths,
                    at_encoding=at_encoding, at_engine=at_engine,
                    at_ask_missing=at_ask_missing, at_cls_args=at_cls_args,
                    **kwargs)

def render_to(filepath, context=None, output=None, at_paths=None,
              at_encoding=anytemplate.compat.ENCODING,
              at_engine=None, at_ask_missing=False,
              at_cls_args=None, **kwargs):
    """
    Render given template file and write the result string to given `output`.
    The result string will be printed to sys.stdout if output is None or '-'.

    :param template: Template file path
    :param context: A dict or dict-like object to instantiate given
        template file
    :param output: File path to write the rendered result string to or None/'-'
        to print it to stdout
    :param at_paths: Template search paths
    :param at_encoding: Template encoding
    :param at_engine: Specify the name of template engine to use explicitly or
        None to find it automatically anyhow.
    :param at_cls_args: Arguments passed to instantiate template engine class
    :param kwargs: Keyword arguments passed to the template engine to
        render templates with specific features enabled.
    """
    res = render(filepath, context=context, at_paths=at_paths,
                 at_encoding=at_encoding, at_engine=at_engine,
                 at_ask_missing=at_ask_missing, at_cls_args=at_cls_args,
                 **kwargs)
    anytemplate.utils.write_to_output(res, output, at_encoding)

# vim:sw=4:ts=4:et:
<|endoftext|>


<commit_before>'''
Created by:
Dori Dick
College of Earth, Ocean and Atmospheric Sciences
Oregon State Univeristy

Created on: 29 July 2012
Last modified: 15 May 2013

Description:
This script exports a feature class containing spatially referenced genetic into the text file
format required by Genepop. See http://genepop.curtin.edu.au/ for more information on Genepop.

Required Inputs:
An existing Feature Class containing spatially referenced genetic data (i.e. co-dominant microsatellite data).
An Attribute Field within the Feature Class that can be used to designate different populations.
An Output File Location  and file name for the text file

Optional Inputs:
A Where Clause to identify only those rows with genetic data in the Feature Class.  Uses a SQL Expression.
NOTE: This parameter is optional and was included because some data sets may have individual IDs based
on more than just genetics (i.e. photo-id).

Script Outputs:
A text file formatted to meet the required input file format for Genepop.

NOTE:  Keep a record of the Loci names and their order as this script uses generic locus names
(e.g., Locus1_A1, Locus1_A2, Locus2_A1, Locus2_A2 etc.)

This script was developed to work with ArcGIS 10.1 and Python 2.7 installed during ArcGIS installation.
'''

import arcpy
import re
import sys
import time
from collections import OrderedDict
from datetime import datetime

# local imports
import utils
import config

def normalize(allele):
    val = str(allele)
    if allele == '' or int(allele) == 0:
        val = "000"
    return val

def main(input_features=None, where_clause=None, order_by=None,  output_name=None,
         mode=config.settings.mode):

    utils.msg("Executing ExportToGenepop.")

    # Create and open the Output text file to which the data will be written
    with open(output_name, "w") as output_file:
        utils.msg("Opened `%s` for writing." % output_name)

        # Create the Genepop header information required for the text file.
        comment_line = "Export to Genepop from the feature class `{input_features}`. Export occurred on {datetime}.\n".format(input_features=input_features, datetime=datetime.now())
        output_file.write(comment_line)
        utils.msg("comment: %s" % comment_line)

        # Find our Loci columns.
        loci = OrderedDict()
        loci_columns = []
        genetic_columns = config.settings.genetic_columns.split(";")
        loci_expr = '^l_(.*)_[0-9]+'
        for field in [f.name for f in arcpy.ListFields(input_features)]:
            if field in genetic_columns:
                match = re.match(loci_expr, field, re.IGNORECASE)
                if match:
                    name = match.groups()[0]
                    if loci.has_key(name):
                        loci[name].append(field)
                    else:
                        loci[name] = [field]
                    loci_columns.append(field)

        output_file.write("\n".join(loci.keys()) + "\n")
        utils.msg("loci set: %s" % loci.keys())
        # sql clause can be prefix or suffix; set up ORDER BY
        sql_clause = (None, "ORDER BY {0} ASC".format(order_by))
        # query the input_features in ascending order; filtering as needed
        selected_columns = loci_columns + [config.settings.id_field, order_by]
        rows = arcpy.da.SearchCursor(input_features, selected_columns, where_clause, "", "", sql_clause)
        current_group = ""
        for row in rows:
            group = row[-1] # last column is 'order_by', or key column
            id_field = row[-2] # as set on import
            if group != current_group:
                output_file.write("Pop\n")
                current_group = group

            label = "{0}-{1},".format(id_field, group).replace(" ", "_")
            result_row = [label]

            # GENEPOP only supports haploid and diploid data.
            for (key, cols) in loci.items():
                loci_val = ""
                for col in cols:
                    col_pos = selected_columns.index(col)
                    loci_val += normalize(row[col_pos])
                result_row.append(loci_val)
            output_file.write(" ".join(result_row) + "\n")
    utils.msg("Exported results saved to %s." % output_name)
    time.sleep(4)
<commit_msg>Finish genepop rewrite. related to #9.<commit_after>'''
Created by:
Dori Dick
College of Earth, Ocean and Atmospheric Sciences
Oregon State Univeristy

Created on: 29 July 2012
Last modified: 15 May 2013

Description:
This script exports a feature class containing spatially referenced genetic into the text file
format required by Genepop. See http://genepop.curtin.edu.au/ for more information on Genepop.

Required Inputs:
An existing Feature Class containing spatially referenced genetic data (i.e. co-dominant microsatellite data).
An Attribute Field within the Feature Class that can be used to designate different populations.
An Output File Location  and file name for the text file

Optional Inputs:
A Where Clause to identify only those rows with genetic data in the Feature Class.  Uses a SQL Expression.
NOTE: This parameter is optional and was included because some data sets may have individual IDs based
on more than just genetics (i.e. photo-id).

Script Outputs:
A text file formatted to meet the required input file format for Genepop.

This script was developed to work with ArcGIS 10.1 and Python 2.7 installed during ArcGIS installation.
'''

import arcpy
import re
import sys
import time
from collections import OrderedDict
from datetime import datetime

# local imports
import utils
import config

def normalize(allele):
    val = str(allele)
    if allele == '' or int(allele) == 0:
        val = "000"
    return val

def main(input_features=None, where_clause=None, order_by=None,  output_name=None,
         mode=config.settings.mode):

    utils.msg("Executing ExportToGenepop.")

    # Create and open the Output text file to which the data will be written
    with open(output_name, "w") as output_file:
        utils.msg("Opened `%s` for writing." % output_name)

        # Create the Genepop header information required for the text file.
        comment_line = "Export to Genepop from the feature class `{input_features}`. Export occurred on {datetime}.\n".format(input_features=input_features, datetime=datetime.now())
        output_file.write(comment_line)
        utils.msg("comment: %s" % comment_line)

        # Find our Loci columns.
        loci = OrderedDict()
        loci_columns = []
        genetic_columns = config.settings.genetic_columns.split(";")
        loci_expr = '^l_(.*)_[0-9]+'
        for field in [f.name for f in arcpy.ListFields(input_features)]:
            if field in genetic_columns:
                match = re.match(loci_expr, field, re.IGNORECASE)
                if match:
                    name = match.groups()[0]
                    if loci.has_key(name):
                        loci[name].append(field)
                    else:
                        loci[name] = [field]
                    loci_columns.append(field)

        output_file.write("\n".join(loci.keys()) + "\n")
        utils.msg("loci set: %s" % loci.keys())
        # sql clause can be prefix or suffix; set up ORDER BY
        sql_clause = (None, "ORDER BY {0} ASC".format(order_by))
        # query the input_features in ascending order; filtering as needed
        selected_columns = loci_columns + [config.settings.id_field, order_by]
        rows = arcpy.da.SearchCursor(input_features, selected_columns, where_clause, "", "", sql_clause)
        current_group = ""
        for row in rows:
            group = row[-1] # last column is 'order_by', or key column
            id_field = row[-2] # as set on import
            if group != current_group:
                output_file.write("Pop\n")
                current_group = group

            label = "{0}-{1},".format(id_field, group).replace(" ", "_")
            result_row = [label]

            # GENEPOP only supports haploid and diploid data.
            for (key, cols) in loci.items():
                loci_val = ""
                for col in cols:
                    col_pos = selected_columns.index(col)
                    loci_val += normalize(row[col_pos])
                result_row.append(loci_val)
            output_file.write(" ".join(result_row) + "\n")
    utils.msg("Exported results saved to %s." % output_name)
    time.sleep(4)
<|endoftext|>