<commit_before>from typing import Union, Optional, List, Tuple, Text, BinaryIO
import pathlib
import torch
import math
import warnings
import numpy as np
from PIL import Image, ImageDraw, ImageFont, ImageColor

__all__ = ["make_grid", "save_image", "draw_bounding_boxes", "draw_segmentation_masks"]

@torch.no_grad()
def make_grid(
    tensor: Union[torch.Tensor, List[torch.Tensor]],
    nrow: int = 8,
    padding: int = 2,
    normalize: bool = False,
    value_range: Optional[Tuple[int, int]] = None,
    scale_each: bool = False,
    pad_value: int = 0,
    **kwargs
) -> torch.Tensor:
    """
    Make a grid of images.

    Args:
        tensor (Tensor or list): 4D mini-batch Tensor of shape (B x C x H x W)
            or a list of images all of the same size.
        nrow (int, optional): Number of images displayed in each row of the grid.
            The final grid size is ``(B / nrow, nrow)``. Default: ``8``.
        padding (int, optional): amount of padding. Default: ``2``.
        normalize (bool, optional): If True, shift the image to the range (0, 1),
            by the min and max values specified by ``value_range``. Default: ``False``.
        value_range (tuple, optional): tuple (min, max) where min and max are numbers,
            then these numbers are used to normalize the image. By default, min and max
            are computed from the tensor.
        scale_each (bool, optional): If ``True``, scale each image in the batch of
            images separately rather than the (min, max) over all images. Default: ``False``.
        pad_value (float, optional): Value for the padded pixels. Default: ``0``.

    Returns:
        grid (Tensor): the tensor containing grid of images.
    """
    if not (torch.is_tensor(tensor) or
            (isinstance(tensor, list) and all(torch.is_tensor(t) for t in tensor))):
        raise TypeError(f'tensor or list of tensors expected, got {type(tensor)}')

    if "range" in kwargs.keys():
        warning = "range will be deprecated, please use value_range instead."
        warnings.warn(warning)
        value_range = kwargs["range"]

    # if list of tensors, convert to a 4D mini-batch Tensor
    if isinstance(tensor, list):
        tensor = torch.stack(tensor, dim=0)

    if tensor.dim() == 2:  # single image H x W
        tensor = tensor.unsqueeze(0)
    if tensor.dim() == 3:  # single image
        if tensor.size(0) == 1:  # if single-channel, convert to 3-channel
            tensor = torch.cat((tensor, tensor, tensor), 0)
        tensor = tensor.unsqueeze(0)

    if tensor.dim() == 4 and tensor.size(1) == 1:  # single-channel images
        tensor = torch.cat((tensor, tensor, tensor), 1)

    if normalize is True:
        tensor = tensor.clone()  # avoid modifying tensor in-place
        if value_range is not None:
            assert isinstance(value_range, tuple), \
                "value_range has to be a tuple (min, max) if specified. min and max are numbers"

        def norm_ip(img, low, high):
            img.clamp_(min=low, max=high)
            img.sub_(low).div_(max(high - low, 1e-5))

        def norm_range(t, value_range):
            if value_range is not None:
                norm_ip(t, value_range[0], value_range[1])
            else:
                norm_ip(t, float(t.min()), float(t.max()))

        if scale_each is True:
            for t in tensor:  # loop over mini-batch dimension
                norm_range(t, value_range)
        else:
            norm_range(tensor, value_range)

    if tensor.size(0) == 1:
        return tensor.squeeze(0)

    # make the mini-batch of images into a grid
    nmaps = tensor.size(0)
    xmaps = min(nrow, nmaps)
    ymaps = int(math.ceil(float(nmaps) / xmaps))
    height, width = int(tensor.size(2) + padding), int(tensor.size(3) + padding)
    num_channels = tensor.size(1)
    grid = tensor.new_full((num_channels, height * ymaps + padding, width * xmaps + padding), pad_value)
    k = 0
    for y in range(ymaps):
        for x in range(xmaps):
            if k >= nmaps:
                break
            # Tensor.copy_() is a valid method but seems to be missing from the stubs
            # https://pytorch.org/docs/stable/tensors.html#torch.Tensor.copy_
            grid.narrow(1, y * height + padding, height - padding).narrow(  # type: ignore[attr-defined]
                2, x * width + padding, width - padding
            ).copy_(tensor[k])
            k = k + 1
    return grid

@torch.no_grad()
def save_image(
    tensor: Union[torch.Tensor, List[torch.Tensor]],
    fp: Union[Text, pathlib.Path, BinaryIO],
    format: Optional[str] = None,
    **kwargs
) -> None:
    """
    Save a given Tensor into an image file.

    Args:
        tensor (Tensor or list): Image to be saved. If given a mini-batch tensor,
            saves the tensor as a grid of images by calling ``make_grid``.
        fp (string or file object): A filename or a file object
        format(Optional):  If omitted, the format to use is determined from the filename extension.
            If a file object was used instead of a filename, this parameter should always be used.
        **kwargs: Other arguments are documented in ``make_grid``.
    """

    grid = make_grid(tensor, **kwargs)
    # Add 0.5 after unnormalizing to [0, 255] to round to nearest integer
    ndarr = grid.mul(255).add_(0.5).clamp_(0, 255).permute(1, 2, 0).to('cpu', torch.uint8).numpy()
    im = Image.fromarray(ndarr)
    im.save(fp, format=format)

@torch.no_grad()
def draw_bounding_boxes(
    image: torch.Tensor,
    boxes: torch.Tensor,
    labels: Optional[List[str]] = None,
    colors: Optional[List[Union[str, Tuple[int, int, int]]]] = None,
    fill: Optional[bool] = False,
    width: int = 1,
    font: Optional[str] = None,
    font_size: int = 10
) -> torch.Tensor:

    """
    Draws bounding boxes on given image.
    The values of the input image should be uint8 between 0 and 255.
    If fill is True, Resulting Tensor should be saved as PNG image.

    Args:
        image (Tensor): Tensor of shape (C x H x W) and dtype uint8.
        boxes (Tensor): Tensor of size (N, 4) containing bounding boxes in (xmin, ymin, xmax, ymax) format. Note that
            the boxes are absolute coordinates with respect to the image. In other words: `0 <= xmin < xmax < W` and
            `0 <= ymin < ymax < H`.
        labels (List[str]): List containing the labels of bounding boxes.
        colors (List[Union[str, Tuple[int, int, int]]]): List containing the colors of bounding boxes. The colors can
            be represented as `str` or `Tuple[int, int, int]`.
        fill (bool): If `True` fills the bounding box with specified color.
        width (int): Width of bounding box.
        font (str): A filename containing a TrueType font. If the file is not found in this filename, the loader may
            also search in other directories, such as the `fonts/` directory on Windows or `/Library/Fonts/`,
            `/System/Library/Fonts/` and `~/Library/Fonts/` on macOS.
        font_size (int): The requested font size in points.

    Returns:
        img (Tensor[C, H, W]): Image Tensor of dtype uint8 with bounding boxes plotted.
    """

    if not isinstance(image, torch.Tensor):
        raise TypeError(f"Tensor expected, got {type(image)}")
    elif image.dtype != torch.uint8:
        raise ValueError(f"Tensor uint8 expected, got {image.dtype}")
    elif image.dim() != 3:
        raise ValueError("Pass individual images, not batches")

    ndarr = image.permute(1, 2, 0).numpy()
    img_to_draw = Image.fromarray(ndarr)

    img_boxes = boxes.to(torch.int64).tolist()

    if fill:
        draw = ImageDraw.Draw(img_to_draw, "RGBA")

    else:
        draw = ImageDraw.Draw(img_to_draw)

    txt_font = ImageFont.load_default() if font is None else ImageFont.truetype(font=font, size=font_size)

    for i, bbox in enumerate(img_boxes):
        if colors is None:
            color = None
        else:
            color = colors[i]

        if fill:
            if color is None:
                fill_color = (255, 255, 255, 100)
            elif isinstance(color, str):
                # This will automatically raise Error if rgb cannot be parsed.
                fill_color = ImageColor.getrgb(color) + (100,)
            elif isinstance(color, tuple):
                fill_color = color + (100,)
            draw.rectangle(bbox, width=width, outline=color, fill=fill_color)
        else:
            draw.rectangle(bbox, width=width, outline=color)

        if labels is not None:
            margin = width + 1
            draw.text((bbox[0] + margin, bbox[1] + margin), labels[i], fill=color, font=txt_font)

    return torch.from_numpy(np.array(img_to_draw)).permute(2, 0, 1).to(dtype=torch.uint8)

@torch.no_grad()
def draw_segmentation_masks(
    image: torch.Tensor,
    masks: torch.Tensor,
    alpha: float = 0.8,
    colors: Optional[List[Union[str, Tuple[int, int, int]]]] = None,
) -> torch.Tensor:

    """
    Draws segmentation masks on given RGB image.
    The values of the input image should be uint8 between 0 and 255.

    Args:
        image (Tensor): Tensor of shape (3, H, W) and dtype uint8.
        masks (Tensor): Tensor of shape (num_masks, H, W) or (H, W) and dtype bool.
        alpha (float): Float number between 0 and 1 denoting the transparency of the masks.
            0 means full transparency, 1 means no transparency.
        colors (list or None): List containing the colors of the masks. The colors can
            be represented as PIL strings e.g. "red" or "#FF00FF", or as RGB tuples e.g. ``(240, 10, 157)``.
            When ``masks`` has a single entry of shape (H, W), you can pass a single color instead of a list
            with one element. By default, random colors are generated for each mask.

    Returns:
        img (Tensor[C, H, W]): Image Tensor, with segmentation masks drawn on top.
    """

    if not isinstance(image, torch.Tensor):
        raise TypeError(f"The image must be a tensor, got {type(image)}")
    elif image.dtype != torch.uint8:
        raise ValueError(f"The image dtype must be uint8, got {image.dtype}")
    elif image.dim() != 3:
        raise ValueError("Pass individual images, not batches")
    elif image.size()[0] != 3:
        raise ValueError("Pass an RGB image. Other Image formats are not supported")
    if masks.ndim == 2:
        masks = masks[None, :, :]
    if masks.ndim != 3:
        raise ValueError("masks must be of shape (H, W) or (batch_size, H, W)")
    if masks.dtype != torch.bool:
        raise ValueError(f"The masks must be of dtype bool. Got {masks.dtype}")
    if masks.shape[-2:] != image.shape[-2:]:
        raise ValueError("The image and the masks must have the same height and width")

    num_masks = masks.size()[0]
    if colors is not None and num_masks > len(colors):
        raise ValueError(f"There are more masks ({num_masks}) than colors ({len(colors)})")

    if colors is None:
        colors = _generate_color_palette(num_masks)

    if not isinstance(colors, list):
        colors = [colors]
    if not isinstance(colors[0], (tuple, str)):
        raise ValueError("colors must be a tuple or a string, or a list thereof")
    if isinstance(colors[0], tuple) and len(colors[0]) != 3:
        raise ValueError("It seems that you passed a tuple of colors instead of a list of colors")

    out_dtype = torch.uint8

    colors_ = []
    for color in colors:
        if isinstance(color, str):
            color = ImageColor.getrgb(color)
        color = torch.tensor(color, dtype=out_dtype)
        colors_.append(color)

    img_to_draw = image.detach().clone()
    # TODO: There might be a way to vectorize this
    for mask, color in zip(masks, colors_):
        img_to_draw[:, mask] = color[:, None]

    out = image * (1 - alpha) + img_to_draw * alpha
    return out.to(out_dtype)

def _generate_color_palette(num_masks):
    palette = torch.tensor([2 ** 25 - 1, 2 ** 15 - 1, 2 ** 21 - 1])
    return [tuple((i * palette) % 255) for i in range(num_masks)]
<commit_msg>bug-fix to issue #4042 (#4043)<commit_after>from typing import Union, Optional, List, Tuple, Text, BinaryIO
import pathlib
import torch
import math
import warnings
import numpy as np
from PIL import Image, ImageDraw, ImageFont, ImageColor

__all__ = ["make_grid", "save_image", "draw_bounding_boxes", "draw_segmentation_masks"]

@torch.no_grad()
def make_grid(
    tensor: Union[torch.Tensor, List[torch.Tensor]],
    nrow: int = 8,
    padding: int = 2,
    normalize: bool = False,
    value_range: Optional[Tuple[int, int]] = None,
    scale_each: bool = False,
    pad_value: int = 0,
    **kwargs
) -> torch.Tensor:
    """
    Make a grid of images.

    Args:
        tensor (Tensor or list): 4D mini-batch Tensor of shape (B x C x H x W)
            or a list of images all of the same size.
        nrow (int, optional): Number of images displayed in each row of the grid.
            The final grid size is ``(B / nrow, nrow)``. Default: ``8``.
        padding (int, optional): amount of padding. Default: ``2``.
        normalize (bool, optional): If True, shift the image to the range (0, 1),
            by the min and max values specified by ``value_range``. Default: ``False``.
        value_range (tuple, optional): tuple (min, max) where min and max are numbers,
            then these numbers are used to normalize the image. By default, min and max
            are computed from the tensor.
        scale_each (bool, optional): If ``True``, scale each image in the batch of
            images separately rather than the (min, max) over all images. Default: ``False``.
        pad_value (float, optional): Value for the padded pixels. Default: ``0``.

    Returns:
        grid (Tensor): the tensor containing grid of images.
    """
    if not (torch.is_tensor(tensor) or
            (isinstance(tensor, list) and all(torch.is_tensor(t) for t in tensor))):
        raise TypeError(f'tensor or list of tensors expected, got {type(tensor)}')

    if "range" in kwargs.keys():
        warning = "range will be deprecated, please use value_range instead."
        warnings.warn(warning)
        value_range = kwargs["range"]

    # if list of tensors, convert to a 4D mini-batch Tensor
    if isinstance(tensor, list):
        tensor = torch.stack(tensor, dim=0)

    if tensor.dim() == 2:  # single image H x W
        tensor = tensor.unsqueeze(0)
    if tensor.dim() == 3:  # single image
        if tensor.size(0) == 1:  # if single-channel, convert to 3-channel
            tensor = torch.cat((tensor, tensor, tensor), 0)
        tensor = tensor.unsqueeze(0)

    if tensor.dim() == 4 and tensor.size(1) == 1:  # single-channel images
        tensor = torch.cat((tensor, tensor, tensor), 1)

    if normalize is True:
        tensor = tensor.clone()  # avoid modifying tensor in-place
        if value_range is not None:
            assert isinstance(value_range, tuple), \
                "value_range has to be a tuple (min, max) if specified. min and max are numbers"

        def norm_ip(img, low, high):
            img.clamp_(min=low, max=high)
            img.sub_(low).div_(max(high - low, 1e-5))

        def norm_range(t, value_range):
            if value_range is not None:
                norm_ip(t, value_range[0], value_range[1])
            else:
                norm_ip(t, float(t.min()), float(t.max()))

        if scale_each is True:
            for t in tensor:  # loop over mini-batch dimension
                norm_range(t, value_range)
        else:
            norm_range(tensor, value_range)

    if tensor.size(0) == 1:
        return tensor.squeeze(0)

    # make the mini-batch of images into a grid
    nmaps = tensor.size(0)
    xmaps = min(nrow, nmaps)
    ymaps = int(math.ceil(float(nmaps) / xmaps))
    height, width = int(tensor.size(2) + padding), int(tensor.size(3) + padding)
    num_channels = tensor.size(1)
    grid = tensor.new_full((num_channels, height * ymaps + padding, width * xmaps + padding), pad_value)
    k = 0
    for y in range(ymaps):
        for x in range(xmaps):
            if k >= nmaps:
                break
            # Tensor.copy_() is a valid method but seems to be missing from the stubs
            # https://pytorch.org/docs/stable/tensors.html#torch.Tensor.copy_
            grid.narrow(1, y * height + padding, height - padding).narrow(  # type: ignore[attr-defined]
                2, x * width + padding, width - padding
            ).copy_(tensor[k])
            k = k + 1
    return grid

@torch.no_grad()
def save_image(
    tensor: Union[torch.Tensor, List[torch.Tensor]],
    fp: Union[Text, pathlib.Path, BinaryIO],
    format: Optional[str] = None,
    **kwargs
) -> None:
    """
    Save a given Tensor into an image file.

    Args:
        tensor (Tensor or list): Image to be saved. If given a mini-batch tensor,
            saves the tensor as a grid of images by calling ``make_grid``.
        fp (string or file object): A filename or a file object
        format(Optional):  If omitted, the format to use is determined from the filename extension.
            If a file object was used instead of a filename, this parameter should always be used.
        **kwargs: Other arguments are documented in ``make_grid``.
    """

    grid = make_grid(tensor, **kwargs)
    # Add 0.5 after unnormalizing to [0, 255] to round to nearest integer
    ndarr = grid.mul(255).add_(0.5).clamp_(0, 255).permute(1, 2, 0).to('cpu', torch.uint8).numpy()
    im = Image.fromarray(ndarr)
    im.save(fp, format=format)

@torch.no_grad()
def draw_bounding_boxes(
    image: torch.Tensor,
    boxes: torch.Tensor,
    labels: Optional[List[str]] = None,
    colors: Optional[List[Union[str, Tuple[int, int, int]]]] = None,
    fill: Optional[bool] = False,
    width: int = 1,
    font: Optional[str] = None,
    font_size: int = 10
) -> torch.Tensor:

    """
    Draws bounding boxes on given image.
    The values of the input image should be uint8 between 0 and 255.
    If fill is True, Resulting Tensor should be saved as PNG image.

    Args:
        image (Tensor): Tensor of shape (C x H x W) and dtype uint8.
        boxes (Tensor): Tensor of size (N, 4) containing bounding boxes in (xmin, ymin, xmax, ymax) format. Note that
            the boxes are absolute coordinates with respect to the image. In other words: `0 <= xmin < xmax < W` and
            `0 <= ymin < ymax < H`.
        labels (List[str]): List containing the labels of bounding boxes.
        colors (List[Union[str, Tuple[int, int, int]]]): List containing the colors of bounding boxes. The colors can
            be represented as `str` or `Tuple[int, int, int]`.
        fill (bool): If `True` fills the bounding box with specified color.
        width (int): Width of bounding box.
        font (str): A filename containing a TrueType font. If the file is not found in this filename, the loader may
            also search in other directories, such as the `fonts/` directory on Windows or `/Library/Fonts/`,
            `/System/Library/Fonts/` and `~/Library/Fonts/` on macOS.
        font_size (int): The requested font size in points.

    Returns:
        img (Tensor[C, H, W]): Image Tensor of dtype uint8 with bounding boxes plotted.
    """

    if not isinstance(image, torch.Tensor):
        raise TypeError(f"Tensor expected, got {type(image)}")
    elif image.dtype != torch.uint8:
        raise ValueError(f"Tensor uint8 expected, got {image.dtype}")
    elif image.dim() != 3:
        raise ValueError("Pass individual images, not batches")

    ndarr = image.permute(1, 2, 0).numpy()
    # allow single-channel-images
    # shape: (1, H, W) with C = 1
    if ndarr.shape[-1] == 1:
        ndarr = np.tile(ndarr, (1, 1, 3))
    img_to_draw = Image.fromarray(ndarr)

    img_boxes = boxes.to(torch.int64).tolist()

    if fill:
        draw = ImageDraw.Draw(img_to_draw, "RGBA")

    else:
        draw = ImageDraw.Draw(img_to_draw)

    txt_font = ImageFont.load_default() if font is None else ImageFont.truetype(font=font, size=font_size)

    for i, bbox in enumerate(img_boxes):
        if colors is None:
            color = None
        else:
            color = colors[i]

        if fill:
            if color is None:
                fill_color = (255, 255, 255, 100)
            elif isinstance(color, str):
                # This will automatically raise Error if rgb cannot be parsed.
                fill_color = ImageColor.getrgb(color) + (100,)
            elif isinstance(color, tuple):
                fill_color = color + (100,)
            draw.rectangle(bbox, width=width, outline=color, fill=fill_color)
        else:
            draw.rectangle(bbox, width=width, outline=color)

        if labels is not None:
            margin = width + 1
            draw.text((bbox[0] + margin, bbox[1] + margin), labels[i], fill=color, font=txt_font)

    return torch.from_numpy(np.array(img_to_draw)).permute(2, 0, 1).to(dtype=torch.uint8)

@torch.no_grad()
def draw_segmentation_masks(
    image: torch.Tensor,
    masks: torch.Tensor,
    alpha: float = 0.8,
    colors: Optional[List[Union[str, Tuple[int, int, int]]]] = None,
) -> torch.Tensor:

    """
    Draws segmentation masks on given RGB image.
    The values of the input image should be uint8 between 0 and 255.

    Args:
        image (Tensor): Tensor of shape (3, H, W) and dtype uint8.
        masks (Tensor): Tensor of shape (num_masks, H, W) or (H, W) and dtype bool.
        alpha (float): Float number between 0 and 1 denoting the transparency of the masks.
            0 means full transparency, 1 means no transparency.
        colors (list or None): List containing the colors of the masks. The colors can
            be represented as PIL strings e.g. "red" or "#FF00FF", or as RGB tuples e.g. ``(240, 10, 157)``.
            When ``masks`` has a single entry of shape (H, W), you can pass a single color instead of a list
            with one element. By default, random colors are generated for each mask.

    Returns:
        img (Tensor[C, H, W]): Image Tensor, with segmentation masks drawn on top.
    """

    if not isinstance(image, torch.Tensor):
        raise TypeError(f"The image must be a tensor, got {type(image)}")
    elif image.dtype != torch.uint8:
        raise ValueError(f"The image dtype must be uint8, got {image.dtype}")
    elif image.dim() != 3:
        raise ValueError("Pass individual images, not batches")
    elif image.size()[0] != 3:
        raise ValueError("Pass an RGB image. Other Image formats are not supported")
    if masks.ndim == 2:
        masks = masks[None, :, :]
    if masks.ndim != 3:
        raise ValueError("masks must be of shape (H, W) or (batch_size, H, W)")
    if masks.dtype != torch.bool:
        raise ValueError(f"The masks must be of dtype bool. Got {masks.dtype}")
    if masks.shape[-2:] != image.shape[-2:]:
        raise ValueError("The image and the masks must have the same height and width")

    num_masks = masks.size()[0]
    if colors is not None and num_masks > len(colors):
        raise ValueError(f"There are more masks ({num_masks}) than colors ({len(colors)})")

    if colors is None:
        colors = _generate_color_palette(num_masks)

    if not isinstance(colors, list):
        colors = [colors]
    if not isinstance(colors[0], (tuple, str)):
        raise ValueError("colors must be a tuple or a string, or a list thereof")
    if isinstance(colors[0], tuple) and len(colors[0]) != 3:
        raise ValueError("It seems that you passed a tuple of colors instead of a list of colors")

    out_dtype = torch.uint8

    colors_ = []
    for color in colors:
        if isinstance(color, str):
            color = ImageColor.getrgb(color)
        color = torch.tensor(color, dtype=out_dtype)
        colors_.append(color)

    img_to_draw = image.detach().clone()
    # TODO: There might be a way to vectorize this
    for mask, color in zip(masks, colors_):
        img_to_draw[:, mask] = color[:, None]

    out = image * (1 - alpha) + img_to_draw * alpha
    return out.to(out_dtype)

def _generate_color_palette(num_masks):
    palette = torch.tensor([2 ** 25 - 1, 2 ** 15 - 1, 2 ** 21 - 1])
    return [tuple((i * palette) % 255) for i in range(num_masks)]
<|endoftext|>


<commit_before>import logging
import urllib
import pytz

from datetime import datetime, timedelta

from api.models import RoadClosure, RoadClosureStatus
from bs4 import BeautifulSoup
from dateutil import tz

logger = logging.getLogger('bot.digest')

def get_road_closure():
    tzTex = pytz.timezone('US/Central')
    url = "http://www.cameroncounty.us/spacex/"
    user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36'
    request = urllib.request.Request(url, headers={'User-Agent': user_agent})
    try:
        response = urllib.request.urlopen(request)
    except Exception as e:
        logger.error(e)
        return
    if response.code != 200:
        logger.error("Received bad response code %s" % response.code)
        return
    html_content = response.read()

    # Parse the html content
    soup = BeautifulSoup(html_content, "lxml")
    tableRows = []
    gdp_table = soup.find("table")
    gdp_table_data = gdp_table.tbody.find_all("tr")  # contains 2 rows
    for row in gdp_table_data:
        aux = ''
        for cell in row.find_all("td"):
            aux += str(cell.text.strip()) + '|'
        tableRows.append(aux[:-1])

    closures = []
    for row in tableRows:
        row = row.replace('a.m.', 'AM')
        row = row.replace('p.m.', 'PM')

        dtP1 = row.split('|')[1]
        dtP2 = row.split('|')[2]
        dtP2B = dtP2.split('–')[0].strip()
        dtP2E = dtP2.split('–')[1].strip()

        dtSta = dtP1 + ' ' + dtP2B
        staTex = tzTex.localize(datetime.strptime(dtSta, "%B %d, %Y %I:%M %p"))

        dtEnd = dtP1 + ' ' + dtP2E
        endTex = tzTex.localize(datetime.strptime(dtEnd, "%B %d, %Y %I:%M %p"))
        if 'PM' in dtP2B and 'AM' in dtP2E:
            endTex += timedelta(days=1)

        nowTex = datetime.now(tz=pytz.utc)

        if endTex < nowTex:
            continue

        name = row.split('|')[0]
        status = row.split('|')[3].replace('Closure ', '')
        closures.append([staTex, endTex, name, status])

    logger.info("Found %s closures" % len(closures))
    for closure in closures:
        window_start = closure[0].astimezone(tz.tzutc())
        window_end = closure[1].astimezone(tz.tzutc())
        status_text = closure[3]
        status, created = RoadClosureStatus.objects.get_or_create(name=status_text)
        try:
            obj = RoadClosure.objects.get(window_start__exact=window_start,
                                          window_end__exact=window_end)
            obj.status = status
        except RoadClosure.DoesNotExist:
            obj = RoadClosure.objects.create(title=closure[2],
                                             window_start=window_start,
                                             window_end=window_end,
                                             status=status)
            logger.info("Creating new Road Closure %s" % obj)
        obj.save()
<commit_msg>Fix for updated Cameron date format.<commit_after>import logging
import urllib
import pytz

from datetime import datetime, timedelta

from api.models import RoadClosure, RoadClosureStatus
from bs4 import BeautifulSoup
from dateutil import tz

logger = logging.getLogger('bot.digest')

def get_road_closure():
    tzTex = pytz.timezone('US/Central')
    url = "http://www.cameroncounty.us/spacex/"
    user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36'
    request = urllib.request.Request(url, headers={'User-Agent': user_agent})
    try:
        response = urllib.request.urlopen(request)
    except Exception as e:
        logger.error(e)
        return
    if response.code != 200:
        logger.error("Received bad response code %s" % response.code)
        return
    html_content = response.read()

    # Parse the html content
    soup = BeautifulSoup(html_content, "lxml")
    tableRows = []
    gdp_table = soup.find("table")
    gdp_table_data = gdp_table.tbody.find_all("tr")  # contains 2 rows
    for row in gdp_table_data:
        aux = ''
        for cell in row.find_all("td"):
            aux += str(cell.text.strip()) + '|'
        tableRows.append(aux[:-1])

    closures = []
    for row in tableRows:
        row = row.replace('a.m.', 'AM')
        row = row.replace('p.m.', 'PM')

        dtP1 = row.split('|')[1]
        dtP2 = row.split('|')[2]
        dtP2B = dtP2.split(' to ')[0].strip()
        dtP2E = dtP2.split(' to ')[1].strip()

        dtSta = dtP1 + ' ' + dtP2B
        staTex = tzTex.localize(datetime.strptime(dtSta, "%b %d, %Y %I:%M %p"))

        dtEnd = dtP1 + ' ' + dtP2E
        endTex = tzTex.localize(datetime.strptime(dtEnd, "%b %d, %Y %I:%M %p"))
        if 'PM' in dtP2B and 'AM' in dtP2E:
            endTex += timedelta(days=1)

        nowTex = datetime.now(tz=pytz.utc)

        if endTex < nowTex:
            continue

        name = row.split('|')[0]
        status = row.split('|')[3].replace('Closure ', '')
        closures.append([staTex, endTex, name, status])

    logger.info("Found %s closures" % len(closures))
    for closure in closures:
        window_start = closure[0].astimezone(tz.tzutc())
        window_end = closure[1].astimezone(tz.tzutc())
        status_text = closure[3]
        status, created = RoadClosureStatus.objects.get_or_create(name=status_text)
        try:
            obj = RoadClosure.objects.get(window_start__exact=window_start,
                                          window_end__exact=window_end)
            obj.status = status
        except RoadClosure.DoesNotExist:
            obj = RoadClosure.objects.create(title=closure[2],
                                             window_start=window_start,
                                             window_end=window_end,
                                             status=status)
            logger.info("Creating new Road Closure %s" % obj)
        obj.save()
<|endoftext|>


<commit_before>###############################################################################
# Copyright 2014 OCLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################

import BaseHTTPServer
import ssl
from SimpleHTTPServer import SimpleHTTPRequestHandler
from authliboclc import wskey
from urlparse import urlparse, parse_qs
from session_handler import SessionHandler
from access_token_formatter import AccessTokenFormatter
from bibliographic_record import BibRecord

PORT = 8000

"""Authentication parameters."""
KEY = '{clientID}'
SECRET = '{secret}'
AUTHENTICATING_INSTITUTION_ID = '128807'  # default value for Sandbox institution
CONTEXT_INSTITUTION_ID = '128807'  # default value for Sandbox institution
SERVICES = ['WorldCatMetadataAPI', 'refresh_token']
REDIRECT_URI = 'https://localhost:8000/auth/'

class Request(SimpleHTTPRequestHandler):
    """This is a general purpose request handler. We focus on /auth/ and managing access tokens."""

    def do_GET(self):

        if KEY == '{clientID}':
            """The developer forgot to insert authentication parameters into the example."""
            self.send_response(200)
            self.send_header('Content-Type', 'text/html')
            self.end_headers()
            self.wfile.write('
Please set the authentication parameters in ' +
                             'access_token.py, ' +
                             'lines 29 & 30.
')
            return

        if (self.path[:6] != '/auth/'):
            """Handle other, non authentication requests here. For example, loading the favicon.ico"""
            return

        print "\n-- Handling a Request --"

        html = ("""<!doctype html>

        <meta charset="utf-8">
        <meta name="viewport" content="width=device-width, initial-scale=1.0">
        <link rel="stylesheet" href="https://static1.worldcat.org/yui/combo?pure/0.4.2/pure-min.css">

Authentication Token & Bibliographic Record

            """)

        """Populate the WSKey object's parameters"""
        my_wskey = wskey.Wskey(key=KEY, secret=SECRET, options={'services': SERVICES, 'redirect_uri': REDIRECT_URI})

        session_handler = SessionHandler(headers=self.headers)
        access_token = session_handler.get_access_token()

        """If there is a code parameter on the current URL, load it."""
        code = None
        if (self.path is not None):
            params = parse_qs(urlparse(self.path).query)
            if 'code' in params:
                code = params['code'][0]

        """If there are error parameters on the current URL, load them."""
        error = self.headers.get('error', None)
        error_description = self.headers.get('error_description', None)

        if access_token is None and code is None:
            """There is no access token and no authentication code. Initiate user authentication."""

            """Get the user authentication url"""
            login_url = my_wskey.get_login_url(
                authenticating_institution_id=AUTHENTICATING_INSTITUTION_ID,
                context_institution_id=CONTEXT_INSTITUTION_ID
            )

            """Redirect the browser to the login_url"""
            self.send_response(303)
            self.send_header('Location', login_url)
            self.end_headers()

            print "Requiring user to authenticate."

        else:

            if error is not None:
                """If an error was returned, display it."""
                html = ''.join([html, '
Error: ', error, '
', error_description, '
'])

            if access_token is None and code is not None:
                """Request an access token using the user authentication code returned after the user authenticated"""
                """Then request a bibliographic record"""

                print "I now have an authentication code. I will request an access token."
                access_token = my_wskey.get_access_token_with_auth_code(
                    code=code,
                    authenticating_institution_id=AUTHENTICATING_INSTITUTION_ID,
                    context_institution_id=CONTEXT_INSTITUTION_ID
                )

                if access_token.error_code is None:
                    session_handler.save_access_token(access_token)
                    html = ''.join([html, '
Access Token saved to session database.
'])

            if access_token is not None:
                """Display the token and request a Bibliographic Record"""
                print "Displaying access token parameters."
                if access_token.error_code is None and code is None:
                    html = ''.join([html, '
Access Token retrieved from session database.
'])

                access_token_formatter = AccessTokenFormatter(access_token=access_token)
                html = ''.join([html, access_token_formatter.format()])

                if access_token.error_code is None:
                    print "Using Access Token to request a Bibliographic Record."
                    bib_record = BibRecord(access_token=access_token, wskey=my_wskey)
                    html = ''.join([html, bib_record.read()])

            html = ''.join([html, ''])

            self.send_response(200)
            self.send_header('Content-Type', 'text/html')
            self.wfile.write(''.join([session_handler.cookie_header_output(), '\n']))
            self.end_headers()
            self.wfile.write(html)

        return

httpd = BaseHTTPServer.HTTPServer(('localhost', PORT), Request)
httpd.socket = ssl.wrap_socket(httpd.socket, certfile='server.pem', server_side=True)
print "\n\n\nStarting https server.\nNavigate your browser to " + \
      REDIRECT_URI + \
      "\nPress Ctrl-C to abort.\n---------------------------------------------------------"
httpd.serve_forever()<commit_msg>Correct error message for missing parameters.<commit_after>###############################################################################
# Copyright 2014 OCLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################

import BaseHTTPServer
import ssl
from SimpleHTTPServer import SimpleHTTPRequestHandler
from authliboclc import wskey
from urlparse import urlparse, parse_qs
from session_handler import SessionHandler
from access_token_formatter import AccessTokenFormatter
from bibliographic_record import BibRecord

PORT = 8000

"""Authentication parameters."""
KEY = '{clientID}'
SECRET = '{secret}'
AUTHENTICATING_INSTITUTION_ID = '128807'  # default value for Sandbox institution
CONTEXT_INSTITUTION_ID = '128807'  # default value for Sandbox institution
SERVICES = ['WorldCatMetadataAPI', 'refresh_token']
REDIRECT_URI = 'https://localhost:8000/auth/'

class Request(SimpleHTTPRequestHandler):
    """This is a general purpose request handler. We focus on /auth/ and managing access tokens."""

    def do_GET(self):

        if KEY == '{clientID}':
            """The developer forgot to insert authentication parameters into the example."""
            self.send_response(200)
            self.send_header('Content-Type', 'text/html')
            self.end_headers()
            self.wfile.write('
Please set the authentication parameters in ' +
                             'examples/authentication_token/server.py, ' +
                             'lines 29 & 30.
')
            return

        if (self.path[:6] != '/auth/'):
            """Handle other, non authentication requests here. For example, loading the favicon.ico"""
            return

        print "\n-- Handling a Request --"

        html = ("""<!doctype html>

        <meta charset="utf-8">
        <meta name="viewport" content="width=device-width, initial-scale=1.0">
        <link rel="stylesheet" href="https://static1.worldcat.org/yui/combo?pure/0.4.2/pure-min.css">

Authentication Token & Bibliographic Record

            """)

        """Populate the WSKey object's parameters"""
        my_wskey = wskey.Wskey(key=KEY, secret=SECRET, options={'services': SERVICES, 'redirect_uri': REDIRECT_URI})

        session_handler = SessionHandler(headers=self.headers)
        access_token = session_handler.get_access_token()

        """If there is a code parameter on the current URL, load it."""
        code = None
        if (self.path is not None):
            params = parse_qs(urlparse(self.path).query)
            if 'code' in params:
                code = params['code'][0]

        """If there are error parameters on the current URL, load them."""
        error = self.headers.get('error', None)
        error_description = self.headers.get('error_description', None)

        if access_token is None and code is None:
            """There is no access token and no authentication code. Initiate user authentication."""

            """Get the user authentication url"""
            login_url = my_wskey.get_login_url(
                authenticating_institution_id=AUTHENTICATING_INSTITUTION_ID,
                context_institution_id=CONTEXT_INSTITUTION_ID
            )

            """Redirect the browser to the login_url"""
            self.send_response(303)
            self.send_header('Location', login_url)
            self.end_headers()

            print "Requiring user to authenticate."

        else:

            if error is not None:
                """If an error was returned, display it."""
                html = ''.join([html, '
Error: ', error, '
', error_description, '
'])

            if access_token is None and code is not None:
                """Request an access token using the user authentication code returned after the user authenticated"""
                """Then request a bibliographic record"""

                print "I now have an authentication code. I will request an access token."
                access_token = my_wskey.get_access_token_with_auth_code(
                    code=code,
                    authenticating_institution_id=AUTHENTICATING_INSTITUTION_ID,
                    context_institution_id=CONTEXT_INSTITUTION_ID
                )

                if access_token.error_code is None:
                    session_handler.save_access_token(access_token)
                    html = ''.join([html, '
Access Token saved to session database.
'])

            if access_token is not None:
                """Display the token and request a Bibliographic Record"""
                print "Displaying access token parameters."
                if access_token.error_code is None and code is None:
                    html = ''.join([html, '
Access Token retrieved from session database.
'])

                access_token_formatter = AccessTokenFormatter(access_token=access_token)
                html = ''.join([html, access_token_formatter.format()])

                if access_token.error_code is None:
                    print "Using Access Token to request a Bibliographic Record."
                    bib_record = BibRecord(access_token=access_token, wskey=my_wskey)
                    html = ''.join([html, bib_record.read()])

            html = ''.join([html, ''])

            self.send_response(200)
            self.send_header('Content-Type', 'text/html')
            self.wfile.write(''.join([session_handler.cookie_header_output(), '\n']))
            self.end_headers()
            self.wfile.write(html)

        return

httpd = BaseHTTPServer.HTTPServer(('localhost', PORT), Request)
httpd.socket = ssl.wrap_socket(httpd.socket, certfile='server.pem', server_side=True)
print "\n\n\nStarting https server.\nNavigate your browser to " + \
      REDIRECT_URI + \
      "\nPress Ctrl-C to abort.\n---------------------------------------------------------"
httpd.serve_forever()
<|endoftext|>


<commit_before># A python script to generate graphs used by the
# Visual Cross Reference Documentation (DOX) pages.
#---------------------------------------------------------------------------
# Copyright 2018 The Open Source Electronic Health Record Agent
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import subprocess

from LogManager import logger, initLogging

from CrossReferenceBuilder import CrossReferenceBuilder
from CrossReferenceBuilder import createCrossReferenceLogArgumentParser
from CrossReference import PlatformDependentGenericRoutine

from UtilityFunctions import *

# Do not generate the graph if have more than 30 nodes
MAX_DEPENDENCY_LIST_SIZE = 30

class GraphGenerator:
    def __init__(self, crossReference, outDir, docRepDir, dot):
        self._crossRef = crossReference
        self._allPackages = crossReference.getAllPackages()
        self._outDir = outDir
        self._docRepDir = docRepDir
        self._dot = dot

    def generateGraphs(self):
        self.generatePackageDependenciesGraph()
        self.generatePackageDependentsGraph()
        self.generateRoutineCallGraph()
        self.generateRoutineCallerGraph()
        self.generateColorLegend()

    #==========================================================================
    #
    #==========================================================================
    def generatePackageDependenciesGraph(self, isDependency=True):
        # generate all dot file and use dot to generated the image file format
        if isDependency:
            name = "dependencies"
        else:
            name = "dependents"
        logger.info("Start generating package %s......" % name)
        logger.info("Total Packages: %d" % len(self._allPackages))
        for package in self._allPackages.values():
            self.generatePackageDependencyGraph(package, isDependency)
        logger.info("End of generating package %s......" % name)

    #==========================================================================
    #
    #==========================================================================
    def generatePackageDependentsGraph(self):
        self.generatePackageDependenciesGraph(False)

    #==========================================================================
    ## Method to generate the package dependency/dependent graph
    #==========================================================================
    def generatePackageDependencyGraph(self, package, dependencyList=True):
        # merge the routine and package list
        depPackages, depPackageMerged = mergeAndSortDependencyListByPackage(
                                                                      package,
                                                                      dependencyList)
        if dependencyList:
            packageSuffix = "_dependency"
        else:
            packageSuffix = "_dependent"
        packageName = package.getName()
        normalizedName = normalizePackageName(packageName)
        totalPackage = len(depPackageMerged)
        if  (totalPackage == 0) or (totalPackage > MAX_DEPENDENCY_LIST_SIZE):
            logger.info("Nothing to do exiting... Package: %s Total: %d " %
                         (packageName, totalPackage))
            return
        try:
            dirName = os.path.join(self._outDir, packageName)
            if not os.path.exists(dirName):
                os.makedirs(dirName)
        except OSError, e:
            logger.error("Error making dir %s : Error: %s" % (dirName, e))
            return
        output = open(os.path.join(dirName, normalizedName + packageSuffix + ".dot"), 'w')
        output.write("digraph %s {\n" % (normalizedName + packageSuffix))
        output.write("\tnode [shape=box fontsize=14];\n") # set the node shape to be box
        output.write("\tnodesep=0.35;\n") # set the node sep to be 0.35
        output.write("\transsep=0.55;\n") # set the rank sep to be 0.75
        output.write("\tedge [fontsize=12];\n") # set the edge label and size props
        output.write("\t%s [style=filled fillcolor=orange label=\"%s\"];\n" % (normalizedName,
                                                                               packageName))
        for depPackage in depPackages:
            depPackageName = depPackage.getName()
            normalizedDepPackName = normalizePackageName(depPackageName)
            output.write("\t%s [label=\"%s\" URL=\"%s\"];\n" % (normalizedDepPackName,
                                                                depPackageName,
                                                                getPackageHtmlFileName(depPackageName)))
            depMetricsList = depPackageMerged[depPackage]
            edgeWeight = sum(depMetricsList[0:7:2])
            edgeLinkURL = getPackageDependencyHtmlFile(normalizedName, normalizedDepPackName)
            edgeStartNode = normalizedName
            edgeEndNode = normalizedDepPackName
            edgeLinkArch = packageName
            toolTipStartPackage = packageName
            toolTipEndPackage = depPackageName
            if not dependencyList:
                edgeStartNode = normalizedDepPackName
                edgeEndNode = normalizedName
                edgeLinkArch = depPackageName
                toolTipStartPackage = depPackageName
                toolTipEndPackage = packageName
            (edgeLabel, edgeToolTip, edgeStyle) = getPackageGraphEdgePropsByMetrics(depMetricsList,
                                                                                toolTipStartPackage,
                                                                                toolTipEndPackage)
            output.write("\t%s->%s [label=\"%s\" weight=%d URL=\"%s#%s\" style=\"%s\" labeltooltip=\"%s\" edgetooltip=\"%s\"];\n" % (edgeStartNode,
                                                     edgeEndNode,
                                                     edgeLabel,
                                                     edgeWeight,
                                                     edgeLinkURL,
                                                     edgeLinkArch,
                                                     edgeStyle,
                                                     edgeToolTip,
                                                     edgeToolTip))
        output.write("}\n")
        output.close()
        # use dot tools to generated the image and client side mapping
        outputName = os.path.join(dirName, normalizedName + packageSuffix + ".png")
        outputmap = os.path.join(dirName, normalizedName + packageSuffix + ".cmapx")
        inputName = os.path.join(dirName, normalizedName + packageSuffix + ".dot")
        # this is to generated the image in gif format and also cmapx (client side map) to make sure link
        # embeded in the graph is clickable
        command = "\"%s\" -Tpng -o\"%s\" -Tcmapx -o\"%s\" \"%s\"" % (self._dot,
                                                               outputName,
                                                               outputmap,
                                                               inputName)
        retCode = subprocess.call(command, shell=True)
        if retCode != 0:
            logger.error("calling dot with command[%s] returns %d" % (command, retCode))

    #===============================================================================
    #
    #===============================================================================
    def generateRoutineCallGraph(self, isCalled=True):
        logger.info("Start Routine generating call graph......")
        for package in self._allPackages.itervalues():
            routines = package.getAllRoutines()
            for routine in routines.itervalues():
                isPlatformGenericRoutine = self._crossRef.isPlatformGenericRoutineByName(routine.getName())
                if isCalled and isPlatformGenericRoutine:
                    self.generatePlatformGenericDependencyGraph(routine, isCalled)
                else:
                    self.generateRoutineDependencyGraph(routine, isCalled)
        logger.info("End of generating call graph......")

    #==========================================================================
    # Method to generate routine caller graph for platform dependent routines
    #==========================================================================
    def generatePlatformGenericDependencyGraph(self, genericRoutine, isDependency):
        assert genericRoutine
        assert isinstance(genericRoutine, PlatformDependentGenericRoutine)
        if not isDependency:
            return
        platformRoutines = genericRoutine.getAllPlatformDepRoutines()
        for routineInfo in platformRoutines.itervalues():
            self.generateRoutineDependencyGraph(routineInfo[0], isDependency)

    #==========================================================================
    #
    #==========================================================================
    def generateRoutineCallerGraph(self):
        self.generateRoutineCallGraph(False)

    #==========================================================================
    ## generate all dot file and use dot to generated the image file format
    #==========================================================================
    def generateRoutineDependencyGraph(self, routine, isDependency=True):
        if not routine.getPackage():
            return
        routineName = routine.getName()
        escapedName = routineName.replace("%","\%")
        packageName = routine.getPackage().getName()
        if isDependency:
            depRoutines = routine.getCalledRoutines()
            routineSuffix = "_called"
            totalDep = routine.getTotalCalled()
        else:
            depRoutines = routine.getCallerRoutines()
            routineSuffix = "_caller"
            totalDep = routine.getTotalCaller()
        # do not generate graph if no dep routines or
        # totalDep routines > max_dependency_list
        if (not depRoutines or  totalDep > MAX_DEPENDENCY_LIST_SIZE):
            logger.debug("No called Routines found! for routine:%s package:%s" % (routineName, packageName))
            return
        try:
            dirName = os.path.join(self._outDir, packageName)
            if not os.path.exists(dirName):
                os.makedirs(dirName)
        except OSError, e:
            logger.error("Error making dir %s : Error: %s" % (dirName, e))
            return
        output = open(os.path.join(dirName, routineName + routineSuffix + ".dot"), 'wb')
        output.write("digraph \"%s\" {\n" % (routineName + routineSuffix))
        output.write("\tnode [shape=box fontsize=14];\n") # set the node shape to be box
        output.write("\tnodesep=0.45;\n") # set the node sep to be 0.15
        output.write("\transsep=0.45;\n") # set the rank sep to be 0.75
#        output.write("\tedge [fontsize=12];\n") # set the edge label and size props
        if routine.getPackage() not in depRoutines:
            output.write("\tsubgraph \"cluster_%s\"{\n" % (routine.getPackage()))
            output.write("\t\t\"%s\" [style=filled fillcolor=orange];\n" % escapedName)
            output.write("\t}\n")
        for (package, callDict) in depRoutines.iteritems():
            output.write("\tsubgraph \"cluster_%s\"{\n" % (package))
            for routine in callDict.keys():
                output.write("\t\t\"%s\" [penwidth=2 %s URL=\"%s\" tooltip=\"%s\"];\n" % (routine.getName().replace("%","\%"),
                                                         findDotColor(routine),
                                                         getPackageObjHtmlFileName(routine),
                                                         getPackageObjHtmlFileName(routine)
                                                        ))
                if str(package) == packageName:
                    output.write("\t\t\"%s\" [style=filled fillcolor=orange];\n" % escapedName)
            output.write("\t\tlabel=\"%s\";\n" % package)
            output.write("\t}\n")
            for (routine, tags) in callDict.iteritems():
                if isDependency:
                    output.write("\t\t\"%s\"->\"%s\"" % (escapedName, routine.getName().replace("%","\%")))
                else:
                    output.write("\t\t\"%s\"->\"%s\"" % (routine.getName().replace("%","\%"), escapedName))
                output.write(";\n")
        output.write("}\n")
        output.close()
        outputName = os.path.join(dirName, routineName + routineSuffix + ".png")
        outputmap = os.path.join(dirName, routineName + routineSuffix + ".cmapx")
        inputName = os.path.join(dirName, routineName + routineSuffix + ".dot")
        # this is to generated the image in png format and also cmapx (client side map) to make sure link
        # embeded in the graph is clickable
        # @TODO this should be able to run in parallel
        command = "\"%s\" -Tpng -o\"%s\" -Tcmapx -o\"%s\" \"%s\"" % (self._dot,
                                                               outputName,
                                                               outputmap,
                                                               inputName)
        retCode = subprocess.call(command, shell=True)
        if retCode != 0:
            logger.error("calling dot with command[%s] returns %d" % (command, retCode))

    #==========================================================================
    #  Generate Color legend image
    #==========================================================================
    def generateColorLegend(self, isCalled=True):
        command = "\"%s\" -Tpng -o\"%s\" -Tcmapx -o\"%s\" \"%s\"" % (self._dot,
                                                    os.path.join(self._outDir,"colorLegend.png"),
                                                    os.path.join(self._outDir,"colorLegend.cmapx"),
                                                    os.path.join(self._docRepDir,'callerGraph_color_legend.dot'))
        retCode = subprocess.call(command, shell=True)
        if retCode != 0:
            logger.error("calling dot with command[%s] returns %d" % (command, retCode))

#===============================================================================
# main
#===============================================================================
def run(args):
  logger.info ("Parsing ICR JSON file....")
  icrJsonFile = os.path.abspath(args.icrJsonFile)
  parsedICRJSON = parseICRJson(icrJsonFile)
  logger.info ("Building cross reference....")
  doxDir = os.path.join(args.patchRepositDir, 'Utilities/Dox')
  crossRef = CrossReferenceBuilder().buildCrossReferenceWithArgs(args,
                                                                 icrJson=parsedICRJSON,
                                                                 inputTemplateDeps=readIntoDictionary(args.inputTemplateDep),
                                                                 sortTemplateDeps=readIntoDictionary(args.sortTemplateDep),
                                                                 printTemplateDeps=readIntoDictionary(args.printTemplateDep)
                                                                 )
  logger.info ("Starting generating graphs....")
  graphGenerator = GraphGenerator(crossRef,
                                  args.outDir,
                                  doxDir,
                                  args.dot)
  graphGenerator.generateGraphs()
  logger.info ("End of generating graphs")

if __name__ == '__main__':
    crossRefArgParse = createCrossReferenceLogArgumentParser()
    parser = argparse.ArgumentParser(
        description='VistA Visual Cross-Reference Graph Generator',
        parents=[crossRefArgParse])
    parser.add_argument('-dot', required=True,
                        help='path to the folder containing dot excecutable')
    parser.add_argument('-icr','--icrJsonFile', required=True,
                        help='JSON formatted information of DBIA/ICR')
    parser.add_argument('-st','--sortTemplateDep', required=True,
                        help='CSV formatted "Relational Jump" field data for Sort Templates')
    parser.add_argument('-it','--inputTemplateDep', required=True,
                        help='CSV formatted "Relational Jump" field data for Input Templates')
    parser.add_argument('-pt','--printTemplateDep', required=True,
                        help='CSV formatted "Relational Jump" field data for Print Templates')
    result = parser.parse_args();

    initLogging(result.logFileDir, "GraphGen.log")
    logger.debug(result)

    run(result)
<commit_msg>Reduce runtime of GraphGenerator<commit_after># A python script to generate graphs used by the
# Visual Cross Reference Documentation (DOX) pages.
#---------------------------------------------------------------------------
# Copyright 2018 The Open Source Electronic Health Record Agent
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import subprocess

from LogManager import logger, initLogging

from CrossReferenceBuilder import CrossReferenceBuilder
from CrossReferenceBuilder import createCrossReferenceLogArgumentParser
from CrossReference import PlatformDependentGenericRoutine

from UtilityFunctions import *

from multiprocessing import Pool
from multiprocessing.dummy import Pool as ThreadPool

# Do not generate the graph if have more than 30 nodes
MAX_DEPENDENCY_LIST_SIZE = 30

class GraphGenerator:
    def __init__(self, crossReference, outDir, docRepDir, dot):
        self._crossRef = crossReference
        self._allPackages = crossReference.getAllPackages()
        self._outDir = outDir
        self._docRepDir = docRepDir
        self._dot = dot
        self._isDependency = False

    def generateGraphs(self):
        self.generatePackageDependenciesGraph()
        self.generatePackageDependentsGraph()
        self.generateRoutineCallGraph()
        self.generateRoutineCallerGraph()
        self.generateColorLegend()

    #==========================================================================
    #
    #==========================================================================
    def generatePackageDependenciesGraph(self, isDependency=True):
        # generate all dot file and use dot to generated the image file format
        if self._isDependency:
            name = "dependencies"
        else:
            name = "dependents"
        logger.info("Start generating package %s......" % name)
        logger.info("Total Packages: %d" % len(self._allPackages))
        for package in self._allPackages.values():
            self.generatePackageDependencyGraph(package)
        logger.info("End of generating package %s......" % name)

    #==========================================================================
    #
    #==========================================================================
    def generatePackageDependentsGraph(self):
        self.generatePackageDependenciesGraph(False)

    #==========================================================================
    ## Method to generate the package dependency/dependent graph
    #==========================================================================
    def generatePackageDependencyGraph(self, package):
        # merge the routine and package list
        depPackages, depPackageMerged = mergeAndSortDependencyListByPackage(package, self._isDependency)
        if self._isDependency:
            packageSuffix = "_dependency"
        else:
            packageSuffix = "_dependent"
        packageName = package.getName()
        normalizedName = normalizePackageName(packageName)
        totalPackage = len(depPackageMerged)
        if  (totalPackage == 0) or (totalPackage > MAX_DEPENDENCY_LIST_SIZE):
            logger.info("Nothing to do exiting... Package: %s Total: %d " %
                         (packageName, totalPackage))
            return
        try:
            dirName = os.path.join(self._outDir, packageName)
            if not os.path.exists(dirName):
                os.makedirs(dirName)
        except OSError, e:
            logger.error("Error making dir %s : Error: %s" % (dirName, e))
            return
        output = open(os.path.join(dirName, normalizedName + packageSuffix + ".dot"), 'w')
        output.write("digraph %s {\n" % (normalizedName + packageSuffix))
        output.write("\tnode [shape=box fontsize=14];\n") # set the node shape to be box
        output.write("\tnodesep=0.35;\n") # set the node sep to be 0.35
        output.write("\transsep=0.55;\n") # set the rank sep to be 0.75
        output.write("\tedge [fontsize=12];\n") # set the edge label and size props
        output.write("\t%s [style=filled fillcolor=orange label=\"%s\"];\n" % (normalizedName,
                                                                               packageName))
        for depPackage in depPackages:
            depPackageName = depPackage.getName()
            normalizedDepPackName = normalizePackageName(depPackageName)
            output.write("\t%s [label=\"%s\" URL=\"%s\"];\n" % (normalizedDepPackName,
                                                                depPackageName,
                                                                getPackageHtmlFileName(depPackageName)))
            depMetricsList = depPackageMerged[depPackage]
            edgeWeight = sum(depMetricsList[0:7:2])
            edgeLinkURL = getPackageDependencyHtmlFile(normalizedName, normalizedDepPackName)
            edgeStartNode = normalizedName
            edgeEndNode = normalizedDepPackName
            edgeLinkArch = packageName
            toolTipStartPackage = packageName
            toolTipEndPackage = depPackageName
            if not self._isDependency:
                edgeStartNode = normalizedDepPackName
                edgeEndNode = normalizedName
                edgeLinkArch = depPackageName
                toolTipStartPackage = depPackageName
                toolTipEndPackage = packageName
            (edgeLabel, edgeToolTip, edgeStyle) = getPackageGraphEdgePropsByMetrics(depMetricsList,
                                                                                toolTipStartPackage,
                                                                                toolTipEndPackage)
            output.write("\t%s->%s [label=\"%s\" weight=%d URL=\"%s#%s\" style=\"%s\" labeltooltip=\"%s\" edgetooltip=\"%s\"];\n" % (edgeStartNode,
                                                     edgeEndNode,
                                                     edgeLabel,
                                                     edgeWeight,
                                                     edgeLinkURL,
                                                     edgeLinkArch,
                                                     edgeStyle,
                                                     edgeToolTip,
                                                     edgeToolTip))
        output.write("}\n")
        output.close()
        # use dot tools to generated the image and client side mapping
        outputName = os.path.join(dirName, normalizedName + packageSuffix + ".png")
        outputmap = os.path.join(dirName, normalizedName + packageSuffix + ".cmapx")
        inputName = os.path.join(dirName, normalizedName + packageSuffix + ".dot")
        # this is to generated the image in gif format and also cmapx (client side map) to make sure link
        # embeded in the graph is clickable
        command = "\"%s\" -Tpng -o\"%s\" -Tcmapx -o\"%s\" \"%s\"" % (self._dot,
                                                               outputName,
                                                               outputmap,
                                                               inputName)
        retCode = subprocess.call(command, shell=True)
        if retCode != 0:
            logger.error("calling dot with command[%s] returns %d" % (command, retCode))

    #===============================================================================
    #
    #===============================================================================
    def generateRoutineCallGraph(self, isCalled=True):
        logger.info("Start Routine generating call graph......")
        self._isDependency = isCalled

        # Make a list of all routines we want to process
        allRoutines = []
        for package in self._allPackages.itervalues():
            routines = package.getAllRoutines()
            for routine in routines.itervalues():
                isPlatformGenericRoutine = self._crossRef.isPlatformGenericRoutineByName(routine.getName())
                if self._isDependency and isPlatformGenericRoutine:
                    platformRoutines = routine.getAllPlatformDepRoutines()
                    for routineInfo in platformRoutines.itervalues():
                        allRoutines.append(routineInfo[0])
                else:
                    allRoutines.append(routine)

        # Make the Pool of workers
        pool = ThreadPool(4)
        # Create graphs in their own threads
        # and return the results
        results = pool.map(self.generateRoutineDependencyGraph, allRoutines)
        # close the pool and wait for the work to finish
        pool.close()
        pool.join()

        logger.info("End of generating call graph......")

    #==========================================================================
    #
    #==========================================================================
    def generateRoutineCallerGraph(self):
        self.generateRoutineCallGraph(False)

    #==========================================================================
    ## generate all dot file and use dot to generated the image file format
    #==========================================================================
    def generateRoutineDependencyGraph(self, routine):
        if not routine.getPackage():
            return
        routineName = routine.getName()
        escapedName = routineName.replace("%","\%")
        packageName = routine.getPackage().getName()
        if self._isDependency:
            depRoutines = routine.getCalledRoutines()
            routineSuffix = "_called"
            totalDep = routine.getTotalCalled()
        else:
            depRoutines = routine.getCallerRoutines()
            routineSuffix = "_caller"
            totalDep = routine.getTotalCaller()
        # do not generate graph if no dep routines or
        # totalDep routines > max_dependency_list
        if (not depRoutines or  totalDep > MAX_DEPENDENCY_LIST_SIZE):
            logger.debug("No called Routines found! for routine:%s package:%s" % (routineName, packageName))
            return
        try:
            dirName = os.path.join(self._outDir, packageName)
            if not os.path.exists(dirName):
                os.makedirs(dirName)
        except OSError, e:
            logger.error("Error making dir %s : Error: %s" % (dirName, e))
            return
        output = open(os.path.join(dirName, routineName + routineSuffix + ".dot"), 'wb')
        output.write("digraph \"%s\" {\n" % (routineName + routineSuffix))
        output.write("\tnode [shape=box fontsize=14];\n") # set the node shape to be box
        output.write("\tnodesep=0.45;\n") # set the node sep to be 0.15
        output.write("\transsep=0.45;\n") # set the rank sep to be 0.75
#        output.write("\tedge [fontsize=12];\n") # set the edge label and size props
        if routine.getPackage() not in depRoutines:
            output.write("\tsubgraph \"cluster_%s\"{\n" % (routine.getPackage()))
            output.write("\t\t\"%s\" [style=filled fillcolor=orange];\n" % escapedName)
            output.write("\t}\n")
        for (package, callDict) in depRoutines.iteritems():
            output.write("\tsubgraph \"cluster_%s\"{\n" % (package))
            for routine in callDict.keys():
                output.write("\t\t\"%s\" [penwidth=2 color=black URL=\"%s\" tooltip=\"%s\"];\n" % (routine.getName().replace("%","\%"),
                                                         getPackageObjHtmlFileName(routine),
                                                         getPackageObjHtmlFileName(routine)
                                                        ))
                if str(package) == packageName:
                    output.write("\t\t\"%s\" [style=filled fillcolor=orange];\n" % escapedName)
            output.write("\t\tlabel=\"%s\";\n" % package)
            output.write("\t}\n")
            for (routine, tags) in callDict.iteritems():
                if self._isDependency:
                    output.write("\t\t\"%s\"->\"%s\"" % (escapedName, routine.getName().replace("%","\%")))
                else:
                    output.write("\t\t\"%s\"->\"%s\"" % (routine.getName().replace("%","\%"), escapedName))
                output.write(";\n")
        output.write("}\n")
        output.close()
        outputName = os.path.join(dirName, routineName + routineSuffix + ".png")
        outputmap = os.path.join(dirName, routineName + routineSuffix + ".cmapx")
        inputName = os.path.join(dirName, routineName + routineSuffix + ".dot")
        # this is to generated the image in png format and also cmapx (client side map) to make sure link
        # embeded in the graph is clickable
        # @TODO this should be able to run in parallel
        command = "\"%s\" -Tpng -o\"%s\" -Tcmapx -o\"%s\" \"%s\"" % (self._dot,
                                                               outputName,
                                                               outputmap,
                                                               inputName)
        retCode = subprocess.call(command, shell=True)
        if retCode != 0:
            logger.error("calling dot with command[%s] returns %d" % (command, retCode))

    #==========================================================================
    #  Generate Color legend image
    #==========================================================================
    def generateColorLegend(self, isCalled=True):
        command = "\"%s\" -Tpng -o\"%s\" -Tcmapx -o\"%s\" \"%s\"" % (self._dot,
                                                    os.path.join(self._outDir,"colorLegend.png"),
                                                    os.path.join(self._outDir,"colorLegend.cmapx"),
                                                    os.path.join(self._docRepDir,'callerGraph_color_legend.dot'))
        retCode = subprocess.call(command, shell=True)
        if retCode != 0:
            logger.error("calling dot with command[%s] returns %d" % (command, retCode))

#===============================================================================
# main
#===============================================================================
def run(args):
  logger.info ("Parsing ICR JSON file....")
  icrJsonFile = os.path.abspath(args.icrJsonFile)
  parsedICRJSON = parseICRJson(icrJsonFile)
  logger.info ("Building cross reference....")
  doxDir = os.path.join(args.patchRepositDir, 'Utilities/Dox')
  crossRef = CrossReferenceBuilder().buildCrossReferenceWithArgs(args,
                                                                 icrJson=parsedICRJSON,
                                                                 inputTemplateDeps=readIntoDictionary(args.inputTemplateDep),
                                                                 sortTemplateDeps=readIntoDictionary(args.sortTemplateDep),
                                                                 printTemplateDeps=readIntoDictionary(args.printTemplateDep)
                                                                 )
  logger.info ("Starting generating graphs....")
  graphGenerator = GraphGenerator(crossRef,
                                  args.outDir,
                                  doxDir,
                                  args.dot)

  graphGenerator.generateGraphs()

  logger.info ("End of generating graphs")

if __name__ == '__main__':
    crossRefArgParse = createCrossReferenceLogArgumentParser()
    parser = argparse.ArgumentParser(
        description='VistA Visual Cross-Reference Graph Generator',
        parents=[crossRefArgParse])
    parser.add_argument('-dot', required=True,
                        help='path to the folder containing dot excecutable')
    parser.add_argument('-icr','--icrJsonFile', required=True,
                        help='JSON formatted information of DBIA/ICR')
    parser.add_argument('-st','--sortTemplateDep', required=True,
                        help='CSV formatted "Relational Jump" field data for Sort Templates')
    parser.add_argument('-it','--inputTemplateDep', required=True,
                        help='CSV formatted "Relational Jump" field data for Input Templates')
    parser.add_argument('-pt','--printTemplateDep', required=True,
                        help='CSV formatted "Relational Jump" field data for Print Templates')
    result = parser.parse_args();

    initLogging(result.logFileDir, "GraphGen.log")
    logger.debug(result)

    run(result)
<|endoftext|>


<commit_before>from totalimpact.models import Item
import totalimpact.dao as dao
import time

# some data useful for testing
# d = {"DOI" : ["10.1371/journal.pcbi.1000361", "10.1016/j.meegid.2011.02.004"], "URL" : ["http://cottagelabs.com"]}

class Queue(dao.Dao):
    __type__ = None

    # TODO:
    # return next item from this queue (e.g. whatever is on the top of the list
    # does NOT remove item from tip of queue
    def first(self):
        # turn this into an instantiation of an item based on the query result
        #if len(self.queue) > 0:
        #    return self.queue[0]
        #else:
        #    return None
        return Item(**{'_rev': '4-a3e3574c44c95b86bb2247fe49e171c8', '_id': 'test', '_deleted_conflicts': ['3-2b27cebd890ff56e616f3d7dadc69c74'], 'hello': 'world', 'aliases': {'URL': ['http://cottagelabs.com'], 'DOI': ['10.1371/journal.pcbi.1000361', "10.1016/j.meegid.2011.02.004"]}})

    # implement this in inheriting classes if needs to be different
    def save_and_unqueue(self,item):
        # alter to use aliases method once exists
        item.data[self.__type__]['last_modified'] = time.time()
        item.save()

class AliasQueue(Queue):
    __type__ = 'aliases'

class MetricsQueue(Queue):
    __type__ = 'metrics'

    @property
    def provider(self):
        try:
            return self._provider
        except:
            self._provider = None
            return self._provider

    @provider.setter
    def provider(self, _provider):
        self._provider = _provider

    @property
    def queue(self):
        # change this for live
        addr = 'queues/' + self.__type__
        if self.provider:
            addr += '?key=["' + self.provider + '"]'
        items = self.view(addr)
        return [Item(**i['key']) for i in items.rows]

    def save_and_unqueue(self,item):
        # alter to use aliases method once exists
        if self.provider:
            item.data[self.__type__][self.provider]['last_updated'] = datetime.datetime.now()
            item.save()
        else:
            return 'No! you have not set a provider'
<commit_msg>updating queue to call db - got a problem with dao though<commit_after>from totalimpact.models import Item
import totalimpact.dao as dao
import time

# some data useful for testing
# d = {"DOI" : ["10.1371/journal.pcbi.1000361", "10.1016/j.meegid.2011.02.004"], "URL" : ["http://cottagelabs.com"]}

class Queue(dao.Dao):
    __type__ = None

    # TODO:
    # return next item from this queue (e.g. whatever is on the top of the list
    # does NOT remove item from tip of queue
    def first(self):
        # turn this into an instantiation of an item based on the query result
        if len(self.queue) > 0:
            return self.queue[0]
        else:
            return None
        #return Item(**{'_rev': '4-a3e3574c44c95b86bb2247fe49e171c8', '_id': 'test', '_deleted_conflicts': ['3-2b27cebd890ff56e616f3d7dadc69c74'], 'hello': 'world', 'aliases': {'URL': ['http://cottagelabs.com'], 'DOI': ['10.1371/journal.pcbi.1000361', "10.1016/j.meegid.2011.02.004"]}})

    # implement this in inheriting classes if needs to be different
    def save_and_unqueue(self,item):
        # alter to use aliases method once exists
        item.data[self.__type__]['last_modified'] = time.time()
        item.save()

class AliasQueue(Queue):
    __type__ = 'aliases'

    def queue(self):
        addr = 'queues/' + self.__type__
        items = self.view(addr)
        return [Item(**i['value']) for i in items.rows]

class MetricsQueue(Queue):
    __type__ = 'metrics'

    def __init__(self,prov=None):
        self._provider = prov

    @property
    def provider(self):
        try:
            return self._provider
        except:
            self._provider = None
            return self._provider

    @provider.setter
    def provider(self, _provider):
        self._provider = _provider

    @property
    def queue(self):
        # change this for live
        addr = 'queues/' + self.__type__
        if self.provider:
            addr += '?key=["' + self.provider + '"]'
        items = self.view(addr)
        return [Item(**i['value']) for i in items.rows]

    def save_and_unqueue(self,item):
        # alter to use aliases method once exists
        if self.provider:
            item.data['meta'][self.provider]['last_updated'] = datetime.datetime.now()
            item.save()
        else:
            return 'No! you have not set a provider'
<|endoftext|>


<commit_before>#!/usr/bin/env python
#coding:utf-8
import os
import functools
from sqlalchemy import create_engine
ISOLATION_LEVEL = {
    1 : 'READ COMMITTED',
    2 : 'READ UNCOMMITTED',
    3 : 'REPEATABLE READ',
    4 : 'SERIALIZABLE'
}

class DBEngine(object):

    def __init__(self,config, **kwargs):
        self.config = config
        self.dbtype = os.environ.get("DB_TYPE", self.config.database.dbtype)
        self.dburl = os.environ.get("DB_URL", self.config.database.dburl)
        self.dbinit = os.environ.get("DB_INIT", 1)
        self.pool_size = kwargs.pop('pool_size',self.config.database.pool_size)

    def __call__(self):
        return self.get_engine()

    def get_engine(self):
        if self.dbtype == 'mysql':
            return create_engine(
                self.dburl,
                echo=bool(self.config.database.echo),
                pool_size = int(self.pool_size),
                pool_recycle=int(self.config.database.pool_recycle)
            )
        elif self.dbtype == 'postgresql':
            return create_engine(
                self.dburl,
                echo=bool(self.config.database.echo),
                pool_size = int(self.pool_size),
                isolation_level = int(ISOLATION_LEVEL.get(self.config.database.isolation_level, 'READ COMMITTED')),
                pool_recycle=int(self.config.database.pool_recycle)
            )
        elif self.dbtype == 'sqlite':
            def my_con_func():
                import sqlite3.dbapi2 as sqlite
                con = sqlite.connect(self.dburl.replace('sqlite:///',''))
                con.text_factory=str
                # con.execute("PRAGMA synchronous=OFF;")
                # con.isolation_level = 'IMMEDIATE'
                return con
            return create_engine(
                "sqlite+pysqlite:///",
                creator=my_con_func,
                echo=bool(self.config.database.echo)
            )
        else:
            return create_engine(
                self.dburl,
                echo=bool(self.config.database.echo),
                pool_size = int(self.pool_size)
            )

def get_engine(config, pool_size=0):
    if pool_size > 0
        return DBEngine(config,pool_size=pool_size)()
    else:
        return DBEngine(config)()

<commit_msg>dbengine update<commit_after>#!/usr/bin/env python
#coding:utf-8
import os
import functools
from sqlalchemy import create_engine
ISOLATION_LEVEL = {
    1 : 'READ COMMITTED',
    2 : 'READ UNCOMMITTED',
    3 : 'REPEATABLE READ',
    4 : 'SERIALIZABLE'
}

class DBEngine(object):

    def __init__(self,config, **kwargs):
        self.config = config
        self.dbtype = os.environ.get("DB_TYPE", self.config.database.dbtype)
        self.dburl = os.environ.get("DB_URL", self.config.database.dburl)
        self.dbinit = os.environ.get("DB_INIT", 1)
        self.pool_size = kwargs.pop('pool_size',self.config.database.pool_size)

    def __call__(self):
        return self.get_engine()

    def get_engine(self):
        if self.dbtype == 'mysql':
            return create_engine(
                self.dburl,
                echo=bool(self.config.database.echo),
                pool_size = int(self.pool_size),
                pool_recycle=int(self.config.database.pool_recycle)
            )
        elif self.dbtype == 'postgresql':
            return create_engine(
                self.dburl,
                echo=bool(self.config.database.echo),
                pool_size = int(self.pool_size),
                isolation_level = int(ISOLATION_LEVEL.get(self.config.database.isolation_level, 'READ COMMITTED')),
                pool_recycle=int(self.config.database.pool_recycle)
            )
        elif self.dbtype == 'sqlite':
            def my_con_func():
                import sqlite3.dbapi2 as sqlite
                con = sqlite.connect(self.dburl.replace('sqlite:///',''))
                con.text_factory=str
                # con.execute("PRAGMA synchronous=OFF;")
                # con.isolation_level = 'IMMEDIATE'
                return con
            return create_engine(
                "sqlite+pysqlite:///",
                creator=my_con_func,
                echo=bool(self.config.database.echo)
            )
        else:
            return create_engine(
                self.dburl,
                echo=bool(self.config.database.echo),
                pool_size = int(self.pool_size)
            )

def get_engine(config, pool_size=0):
    if pool_size > 0:
        return DBEngine(config,pool_size=pool_size)()
    else:
        return DBEngine(config)()

<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys, os
from optparse import OptionParser
from string import maketrans
from subprocess import call

def revc(s):
    '''reverse complement a string'''
    t = " " * len(s)
    l = len(s)
    intab = "AaCcGgTt"
    outtab = "TtGgCcAa"
    trantab = maketrans(intab, outtab)
    t = s.translate(trantab)[::-1]
    return t

def write_adapter_fasta(filename, namesandadapters):
    fh = open(filename, 'w')
    assert len(namesandadapters) % 2 == 0
    for i in range(len(namesandadapters)/2):
        if len(namesandadapters[2*i+1]) > 0:
            fh.write(">"+namesandadapters[2*i]+"\n" +
                     namesandadapters[2*i+1] + "\n")
    fh.close()

def read_fasta_to_table(filen):
    table = {}
    try:
        FILE = open(filen)
    except IOError:
        sys.stderr.write("Can't open file"+ filen+ "!\n")
        sys.exit(1)
    for line in FILE:
        if line[0] == ">":
            header = line.strip()[1:].split()[0]
        else:
            sequence = line.strip()
            table[header] = sequence
    return table

def grab_first_field(filen):
    try:
        contents = open(filen).readlines()
    except IOError:
        sys.stderr.write("Can't open file"+ filen+ "!\n")
        sys.exit(1)
    try:
        firstfield = contents[0].strip().split("\t")[0]
        return firstfield
    except IndexError:
        return ""

if __name__ == '__main__':
    usage = "usage: %prog <filestem>\nExamines already-calculated .P5.csv and .P7.csv and invokes skew.sh"
    parser = OptionParser(usage)
#    parser.add_option("-i", "--input",  dest="input", default=None, help="Input sequence file.")
#    parser.add_option("-o", "--output", dest="output", default=None, help="Output file.")
    parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=True, help="Verbose [default off]")

    (opts, args) = parser.parse_args()
    dirname = os.path.dirname(os.path.realpath(sys.argv[0]))
    if not len(args) == 1:
        parser.error("Missing input filename")
    filename = args[0]
    if filename[-6:] == ".fastq":
        filestem = filename[:-6]
    else:
        filestem = filename
    if not (filename and os.path.isfile(filename)):
        parser.error("Missing input file")
    if not (os.path.isfile(filename+".P5.csv")):
        call([dirname+"/src/idvector.sh", filename])
    P5table = read_fasta_to_table(dirname + "/data/vectors-P5.fa")
    P7table = read_fasta_to_table(dirname + "/data/vectors-P7.fa")
    P5table[""] = ""
    P7table[""] = ""
    P5adaptername = grab_first_field(filestem + ".P5.csv")
    P7adaptername = grab_first_field(filestem + ".P7.csv")
    P5adapter = P5table[P5adaptername]
    P7adapter = P7table[P7adaptername]
    P5r = revc(P5adapter)
    P7r = revc(P7adapter)
    print P5adapter
    print P5r
    print P7adapter
    print P7r
    write_adapter_fasta(filestem+".adapter.fa",
                        [P5adaptername, P5adapter, P5adaptername+"_R",
                        P5r, P7adaptername, P7adapter,
                        P7adaptername+"_R", P7r])
    options = "-k 5 -l 0 --quiet -t 4 -r .2 -m any"

    skewcmd = "skewer -x " + filestem + ".adapter.fa "+ options + " " + filename +" -o " + filestem + ".4"
    print skewcmd
    call(skewcmd.split(" "))
    os.rename(filestem + ".4-trimmed.fastq",
              filestem + ".scrubbed.fastq")
    os.rename(filestem + ".4-trimmed.log",
              filestem + ".scrubbed.log")
#    call([dirname+"/src/skew1.sh", filename, P7adapter, P7r, P5adapter, P5r])
<commit_msg>Test for wrong file<commit_after>#!/usr/bin/env python

import sys, os
from optparse import OptionParser
from string import maketrans
from subprocess import call

def revc(s):
    '''reverse complement a string'''
    t = " " * len(s)
    l = len(s)
    intab = "AaCcGgTt"
    outtab = "TtGgCcAa"
    trantab = maketrans(intab, outtab)
    t = s.translate(trantab)[::-1]
    return t

def write_adapter_fasta(filename, namesandadapters):
    fh = open(filename, 'w')
    assert len(namesandadapters) % 2 == 0
    for i in range(len(namesandadapters)/2):
        if len(namesandadapters[2*i+1]) > 0:
            fh.write(">"+namesandadapters[2*i]+"\n" +
                     namesandadapters[2*i+1] + "\n")
    fh.close()

def read_fasta_to_table(filen):
    table = {}
    try:
        FILE = open(filen)
    except IOError:
        sys.stderr.write("Can't open file"+ filen+ "!\n")
        sys.exit(1)
    for line in FILE:
        if line[0] == ">":
            header = line.strip()[1:].split()[0]
        else:
            sequence = line.strip()
            table[header] = sequence
    return table

def grab_first_field(filen):
    try:
        contents = open(filen).readlines()
    except IOError:
        sys.stderr.write("Can't open file"+ filen+ "!\n")
        sys.exit(1)
    try:
        firstfield = contents[0].strip().split("\t")[0]
        return firstfield
    except IndexError:
        return ""

if __name__ == '__main__':
    usage = "usage: %prog <filestem>\nExamines already-calculated .P5.csv and .P7.csv and invokes skew.sh"
    parser = OptionParser(usage)
#    parser.add_option("-i", "--input",  dest="input", default=None, help="Input sequence file.")
#    parser.add_option("-o", "--output", dest="output", default=None, help="Output file.")
    parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=True, help="Verbose [default off]")

    (opts, args) = parser.parse_args()
    dirname = os.path.dirname(os.path.realpath(sys.argv[0]))
    if not len(args) == 1:
        parser.error("Missing input filename")
    filename = args[0]
    if filename[-6:] == ".fastq":
        filestem = filename[:-6]
    else:
        filestem = filename
    if not (filename and os.path.isfile(filename)):
        parser.error("Missing input file")
    if not (os.path.isfile(filestem+".P5.csv")):
        call([dirname+"/src/idvector.sh", filename])
    P5table = read_fasta_to_table(dirname + "/data/vectors-P5.fa")
    P7table = read_fasta_to_table(dirname + "/data/vectors-P7.fa")
    P5table[""] = ""
    P7table[""] = ""
    P5adaptername = grab_first_field(filestem + ".P5.csv")
    P7adaptername = grab_first_field(filestem + ".P7.csv")
    P5adapter = P5table[P5adaptername]
    P7adapter = P7table[P7adaptername]
    P5r = revc(P5adapter)
    P7r = revc(P7adapter)
    print P5adapter
    print P5r
    print P7adapter
    print P7r
    write_adapter_fasta(filestem+".adapter.fa",
                        [P5adaptername, P5adapter, P5adaptername+"_R",
                        P5r, P7adaptername, P7adapter,
                        P7adaptername+"_R", P7r])
    options = "-k 5 -l 0 --quiet -t 4 -r .2 -m any"

    skewcmd = "skewer -x " + filestem + ".adapter.fa "+ options + " " + filename +" -o " + filestem + ".4"
    print skewcmd
    call(skewcmd.split(" "))
    os.rename(filestem + ".4-trimmed.fastq",
              filestem + ".scrubbed.fastq")
    os.rename(filestem + ".4-trimmed.log",
              filestem + ".scrubbed.log")
#    call([dirname+"/src/skew1.sh", filename, P7adapter, P7r, P5adapter, P5r])
<|endoftext|>


<commit_before># init<commit_msg>removed autoupdater.py<commit_after><|endoftext|>