<commit_before>"""Class for Pixel Painter resource generator."""

from PIL import Image, ImageDraw, ImageFont
from utils.retrieve_query_parameter import retrieve_query_parameter
from math import ceil
from yattag import Doc
import string
from utils.BaseResourceGenerator import BaseResourceGenerator
from django.utils.translation import ugettext as _

class PixelPainterResourceGenerator(BaseResourceGenerator):
    """Class for Pixel Painter resource generator."""

    additional_valid_options = {
        "method": ["binary", "run-length-encoding"],
        "image": ["boat", "fish", "hot-air-balloon"],
    }

    method_strings = {
        "binary": _("Binary"),
        "run-length-encoding": _("Run length encoding"),
    }

    image_strings = {
        "boat": _("Boat"),
        "fish": _("Fish"),
        "hot-air-balloon": _("Hot air balloon"),
    }

    def data(self):
        """Create data for a copy of the Pixel Painter resource.

        Returns:
            A dictionary of the one page for the resource.
            """
        STATIC_PATH = "static/img/resources/pixel-painter/{}.png"
        FONT_PATH = "static/fonts/PatrickHand-Regular.ttf"
        FONT = ImageFont.truetype(FONT_PATH, 80)
        FONT_SMALL = ImageFont.truetype(FONT_PATH, 50)
        TEXT_COLOUR = "#888"

        method = self.requested_options["method"]
        image_name = self.requested_options["image"]

        image = Image.open(STATIC_PATH.format(image_name))
        (image_width, image_height) = image.size

        COLUMNS_PER_PAGE = 15
        ROWS_PER_PAGE = 20
        BOX_SIZE = 200
        IMAGE_SIZE_X = BOX_SIZE * COLUMNS_PER_PAGE
        IMAGE_SIZE_Y = BOX_SIZE * ROWS_PER_PAGE
        LINE_COLOUR = "#666"
        LINE_WIDTH = 1

        pages = []
        if method == "run-length-encoding":
            pages_encoding = dict()

        number_column_pages = ceil(image_width / COLUMNS_PER_PAGE)
        number_row_pages = ceil(image_height / ROWS_PER_PAGE)
        page_grid_coords = self.create_page_grid_coords(number_column_pages, number_row_pages)

        grid_page = self.grid_reference_page(page_grid_coords, image_name)
        pages.append({"type": "html", "data": grid_page})

        # For each page row
        for number_row_page in range(0, number_row_pages):
            page_start_row = (number_row_page) * ROWS_PER_PAGE
            # For each page column
            for number_column_page in range(0, number_column_pages):
                page_start_column = (number_column_page) * COLUMNS_PER_PAGE
                # Create page
                page = Image.new("RGB", (IMAGE_SIZE_X, IMAGE_SIZE_Y), "#fff")
                draw = ImageDraw.Draw(page)
                page_columns = min(COLUMNS_PER_PAGE, image_width - page_start_column)
                page_rows = min(ROWS_PER_PAGE, image_height - page_start_row)
                page_reference = page_grid_coords[number_row_page][number_column_page]
                page_reference_added = False

                if method == "run-length-encoding":
                    page_encoding = []

                # Draw grid
                grid_width = page_columns * BOX_SIZE
                grid_height = page_rows * BOX_SIZE

                for x_coord in range(0, page_columns * BOX_SIZE, BOX_SIZE):
                    draw.line(
                        [(x_coord, 0), (x_coord, grid_height)],
                        fill=LINE_COLOUR,
                        width=LINE_WIDTH
                    )
                draw.line(
                    [(page_columns * BOX_SIZE - 1, 0), (page_columns * BOX_SIZE - 1, grid_height)],
                    fill=LINE_COLOUR,
                    width=LINE_WIDTH
                )

                for y_coord in range(0, grid_height, BOX_SIZE):
                    draw.line(
                        [(0, y_coord), (grid_width, y_coord)],
                        fill=LINE_COLOUR,
                        width=LINE_WIDTH
                    )
                draw.line(
                    [(0, grid_height - 1), (grid_width, grid_height - 1)],
                    fill=LINE_COLOUR,
                    width=LINE_WIDTH
                )

                for row in range(0, page_rows):
                    if method == "run-length-encoding":
                        row_encoding = []
                        encoding_colour = "0"
                        encoding_count = 0

                    for column in range(0, page_columns):
                        pixel_value = image.getpixel((page_start_column + column, page_start_row + row))
                        text = str(1 - int(pixel_value / 255))

                        # Draw text
                        if method == "binary":
                            text_width, text_height = draw.textsize(text, font=FONT)
                            text_coord_x = (column * BOX_SIZE) + (BOX_SIZE / 2) - (text_width / 2)
                            text_coord_y = (row * BOX_SIZE) + (BOX_SIZE / 2) - (text_height / 2)
                            draw.text(
                                (text_coord_x, text_coord_y),
                                text,
                                font=FONT,
                                fill=TEXT_COLOUR
                            )
                        else:
                            if text != encoding_colour:
                                row_encoding.append(encoding_count)
                                if text != encoding_colour:
                                    if encoding_colour == "0":
                                        encoding_colour = "1"
                                    else:
                                        encoding_colour = "0"
                                    encoding_count = 0
                            encoding_count += 1
                            if page_columns - 1 == column:
                                row_encoding.append(encoding_count)

                        # Add page grid reference
                        if not page_reference_added and text == "0":
                            draw.text(
                                ((column * BOX_SIZE) + LINE_WIDTH * 4, (row * BOX_SIZE) + -4),
                                page_reference,
                                font=FONT_SMALL,
                                fill=TEXT_COLOUR
                            )
                            page_reference_added = True
                    if method == "run-length-encoding":
                        page_encoding.append(row_encoding)
                if method == "run-length-encoding":
                    pages_encoding[page_reference] = page_encoding
                pages.append({"type": "image", "data": page})

        # Set first page of grid as thumbnail
        pages[1]["thumbnail"] = True

        if method == "run-length-encoding":
            encoding_html = self.create_run_length_encoding_html(page_grid_coords, pages_encoding)
            pages.insert(1, {"type": "html", "data": encoding_html})
        return pages

    def create_page_grid_coords(self, columns, rows):
        """Create a grid of page coordinates in a 2D array.

        Example:
            When asked for a grid for a 3x2 page grid, the result is:
            grid = [
             ["A1", "A2", "A3"],
             ["B1", "B2", "B3"]
            ]
            grid[0][0] = "A1"
            grid[0][1] = "A2"
            grid[2][1] = "C2"

        Args:
            columns: Number of page columns (int).
            rows: Number of page rows (int).

        Returns:
            A 2D list containing page grid references as strings (list).
        """
        LETTERS = string.ascii_uppercase
        page_grid_coords = [[""] * columns for i in range(rows)]
        for row in range(0, rows):
            for column in range(0, columns):
                page_grid_coords[row][column] = "{}{}".format(LETTERS[row], column + 1)
        return page_grid_coords

    def grid_reference_page(self, page_grid_coords, image_name):
        """Create page grid reference HTML.

        Args:
            page_grid_coords: A 2D list containing the page grid as strings (list).
            image_name: The name of the image used in this resource (str).

        Returns:
            HTML string for the grid reference page.
        """
        doc, tag, text, line = Doc().ttl()
        line("style", "#grid-table td {border:1px solid black;padding:1rem 0.5rem;}")
        with tag("h1"):
            text("Pixel Painter")
        with tag("h2"):
            text("Page grid reference for {} image".format(image_name))
        with tag("p"):
            text(
                "Once pixels on each page are filled in correctly, ",
                "cut each grid out and arrange in the following layout ",
                "(page names are in the top right corner)."
            )
        with tag("table", id="grid-table"):
            with tag("tbody"):
                for row_num in range(0, len(page_grid_coords)):
                    with tag("tr"):
                        for column_num in range(0, len(page_grid_coords[row_num])):
                            line("td", page_grid_coords[row_num][column_num])
        return doc.getvalue()

    def create_run_length_encoding_html(self, page_grid_coords, pages_encoding):
        doc, tag, text, line = Doc().ttl()
        line("style", ".avoid-page-break {page-break-inside:avoid;}")
        with tag("h1"):
            text("Run Length Encodings")
        for index, row_of_page_references in enumerate(page_grid_coords):
            for page_reference in row_of_page_references:
                with tag("div", klass="avoid-page-break"):
                    with tag("h2"):
                        text("Encoding for page {}".format(page_reference))
                    page_encoding = pages_encoding[page_reference]
                    with tag("ul", klass="list-unstyled"):
                        for line_values in page_encoding:
                            line("li", ", ".join(str(number) for number in line_values))
        return doc.getvalue()

    @property
    def subtitle(self):
        """Return the subtitle string of the resource.

        Used after the resource name in the filename, and
        also on the resource image.

        Returns:
            text for subtitle (str).
        """
        text = "{} - {} - {}".format(
            self.image_strings[self.requested_options["image"]],
            self.method_strings[self.requested_options["method"]],
            super().subtitle
        )
        return text
<commit_msg>Add image reference to page reference for pixel painter resource (fixes #663)<commit_after>"""Class for Pixel Painter resource generator."""

from PIL import Image, ImageDraw, ImageFont
from utils.retrieve_query_parameter import retrieve_query_parameter
from math import ceil
from yattag import Doc
import string
from utils.BaseResourceGenerator import BaseResourceGenerator
from django.utils.translation import ugettext as _

class PixelPainterResourceGenerator(BaseResourceGenerator):
    """Class for Pixel Painter resource generator."""

    additional_valid_options = {
        "method": ["binary", "run-length-encoding"],
        "image": ["boat", "fish", "hot-air-balloon"],
    }

    method_strings = {
        "binary": _("Binary"),
        "run-length-encoding": _("Run length encoding"),
    }

    image_strings = {
        "boat": _("Boat"),
        "fish": _("Fish"),
        "hot-air-balloon": _("Hot air balloon"),
    }

    def data(self):
        """Create data for a copy of the Pixel Painter resource.

        Returns:
            A dictionary of the one page for the resource.
            """
        STATIC_PATH = "static/img/resources/pixel-painter/{}.png"
        FONT_PATH = "static/fonts/PatrickHand-Regular.ttf"
        FONT = ImageFont.truetype(FONT_PATH, 80)
        FONT_SMALL = ImageFont.truetype(FONT_PATH, 50)
        TEXT_COLOUR = "#888"

        method = self.requested_options["method"]
        image_name = self.requested_options["image"]

        image = Image.open(STATIC_PATH.format(image_name))
        (image_width, image_height) = image.size

        COLUMNS_PER_PAGE = 15
        ROWS_PER_PAGE = 20
        BOX_SIZE = 200
        IMAGE_SIZE_X = BOX_SIZE * COLUMNS_PER_PAGE
        IMAGE_SIZE_Y = BOX_SIZE * ROWS_PER_PAGE
        LINE_COLOUR = "#666"
        LINE_WIDTH = 1

        pages = []
        if method == "run-length-encoding":
            pages_encoding = dict()

        number_column_pages = ceil(image_width / COLUMNS_PER_PAGE)
        number_row_pages = ceil(image_height / ROWS_PER_PAGE)
        page_grid_coords = self.create_page_grid_coords(number_column_pages, number_row_pages, image_name)

        grid_page = self.grid_reference_page(page_grid_coords, image_name)
        pages.append({"type": "html", "data": grid_page})

        # For each page row
        for number_row_page in range(0, number_row_pages):
            page_start_row = (number_row_page) * ROWS_PER_PAGE
            # For each page column
            for number_column_page in range(0, number_column_pages):
                page_start_column = (number_column_page) * COLUMNS_PER_PAGE
                # Create page
                page = Image.new("RGB", (IMAGE_SIZE_X, IMAGE_SIZE_Y), "#fff")
                draw = ImageDraw.Draw(page)
                page_columns = min(COLUMNS_PER_PAGE, image_width - page_start_column)
                page_rows = min(ROWS_PER_PAGE, image_height - page_start_row)
                page_reference = page_grid_coords[number_row_page][number_column_page]
                page_reference_added = False

                if method == "run-length-encoding":
                    page_encoding = []

                # Draw grid
                grid_width = page_columns * BOX_SIZE
                grid_height = page_rows * BOX_SIZE

                for x_coord in range(0, page_columns * BOX_SIZE, BOX_SIZE):
                    draw.line(
                        [(x_coord, 0), (x_coord, grid_height)],
                        fill=LINE_COLOUR,
                        width=LINE_WIDTH
                    )
                draw.line(
                    [(page_columns * BOX_SIZE - 1, 0), (page_columns * BOX_SIZE - 1, grid_height)],
                    fill=LINE_COLOUR,
                    width=LINE_WIDTH
                )

                for y_coord in range(0, grid_height, BOX_SIZE):
                    draw.line(
                        [(0, y_coord), (grid_width, y_coord)],
                        fill=LINE_COLOUR,
                        width=LINE_WIDTH
                    )
                draw.line(
                    [(0, grid_height - 1), (grid_width, grid_height - 1)],
                    fill=LINE_COLOUR,
                    width=LINE_WIDTH
                )

                for row in range(0, page_rows):
                    if method == "run-length-encoding":
                        row_encoding = []
                        encoding_colour = "0"
                        encoding_count = 0

                    for column in range(0, page_columns):
                        pixel_value = image.getpixel((page_start_column + column, page_start_row + row))
                        text = str(1 - int(pixel_value / 255))

                        # Draw text
                        if method == "binary":
                            text_width, text_height = draw.textsize(text, font=FONT)
                            text_coord_x = (column * BOX_SIZE) + (BOX_SIZE / 2) - (text_width / 2)
                            text_coord_y = (row * BOX_SIZE) + (BOX_SIZE / 2) - (text_height / 2)
                            draw.text(
                                (text_coord_x, text_coord_y),
                                text,
                                font=FONT,
                                fill=TEXT_COLOUR
                            )
                        else:
                            if text != encoding_colour:
                                row_encoding.append(encoding_count)
                                if text != encoding_colour:
                                    if encoding_colour == "0":
                                        encoding_colour = "1"
                                    else:
                                        encoding_colour = "0"
                                    encoding_count = 0
                            encoding_count += 1
                            if page_columns - 1 == column:
                                row_encoding.append(encoding_count)

                        # Add page grid reference
                        if not page_reference_added and text == "0":
                            draw.text(
                                ((column * BOX_SIZE) + LINE_WIDTH * 4, (row * BOX_SIZE) + -4),
                                page_reference,
                                font=FONT_SMALL,
                                fill=TEXT_COLOUR
                            )
                            page_reference_added = True
                    if method == "run-length-encoding":
                        page_encoding.append(row_encoding)
                if method == "run-length-encoding":
                    pages_encoding[page_reference] = page_encoding
                pages.append({"type": "image", "data": page})

        # Set first page of grid as thumbnail
        pages[1]["thumbnail"] = True

        if method == "run-length-encoding":
            encoding_html = self.create_run_length_encoding_html(page_grid_coords, pages_encoding)
            pages.insert(1, {"type": "html", "data": encoding_html})
        return pages

    def create_page_grid_coords(self, columns, rows, image):
        """Create a grid of page coordinates in a 2D array.

        Example:
            When asked for a grid for a 3x2 page grid, the result is:
            grid = [
             ["A1", "A2", "A3"],
             ["B1", "B2", "B3"]
            ]
            grid[0][0] = "A1"
            grid[0][1] = "A2"
            grid[2][1] = "C2"

            The first value refers to the image used.

        Args:
            columns: Number of page columns (int).
            rows: Number of page rows (int).
            image: Name of the image used (str).

        Returns:
            A 2D list containing page grid references as strings (list).
        """
        image_number = self.additional_valid_options["image"].index(image)
        LETTERS = string.ascii_uppercase
        page_grid_coords = [[""] * columns for i in range(rows)]
        for row in range(0, rows):
            for column in range(0, columns):
                page_grid_coords[row][column] = "{}{}{}".format(image_number, LETTERS[row], column + 1)
        return page_grid_coords

    def grid_reference_page(self, page_grid_coords, image_name):
        """Create page grid reference HTML.

        Args:
            page_grid_coords: A 2D list containing the page grid as strings (list).
            image_name: The name of the image used in this resource (str).

        Returns:
            HTML string for the grid reference page.
        """
        doc, tag, text, line = Doc().ttl()
        line("style", "#grid-table td {border:1px solid black;padding:1rem 0.5rem;}")
        with tag("h1"):
            text("Pixel Painter")
        with tag("h2"):
            text("Page grid reference for {} image".format(image_name))
        with tag("p"):
            text(
                "Once pixels on each page are filled in correctly, ",
                "cut each grid out and arrange in the following layout ",
                "(page names are in the top right corner)."
            )
        with tag("table", id="grid-table"):
            with tag("tbody"):
                for row_num in range(0, len(page_grid_coords)):
                    with tag("tr"):
                        for column_num in range(0, len(page_grid_coords[row_num])):
                            line("td", page_grid_coords[row_num][column_num])
        return doc.getvalue()

    def create_run_length_encoding_html(self, page_grid_coords, pages_encoding):
        doc, tag, text, line = Doc().ttl()
        line("style", ".avoid-page-break {page-break-inside:avoid;}")
        with tag("h1"):
            text("Run Length Encodings")
        for index, row_of_page_references in enumerate(page_grid_coords):
            for page_reference in row_of_page_references:
                with tag("div", klass="avoid-page-break"):
                    with tag("h2"):
                        text("Encoding for page {}".format(page_reference))
                    page_encoding = pages_encoding[page_reference]
                    with tag("ul", klass="list-unstyled"):
                        for line_values in page_encoding:
                            line("li", ", ".join(str(number) for number in line_values))
        return doc.getvalue()

    @property
    def subtitle(self):
        """Return the subtitle string of the resource.

        Used after the resource name in the filename, and
        also on the resource image.

        Returns:
            text for subtitle (str).
        """
        text = "{} - {} - {}".format(
            self.image_strings[self.requested_options["image"]],
            self.method_strings[self.requested_options["method"]],
            super().subtitle
        )
        return text
<|endoftext|>


<commit_before>import json
import os
import subprocess
from django.core.management import BaseCommand

from custom.icds_reports.utils.topojson_util.topojson_util import get_topojson_file_for_level, \
    get_topojson_directory, copy_custom_metadata

class Command(BaseCommand):
    help = "Downsize a topojson file by a given percentage."

    def add_arguments(self, parser):
        parser.add_argument('level')
        parser.add_argument('output_file')
        parser.add_argument('percent', type=int)

    def handle(self, level, output_file, percent, *args, **kwargs):
        topojson_file = get_topojson_file_for_level(level)
        output_dir = get_topojson_directory()
        output_file_path = os.path.join(output_dir, output_file)
        tmp_input_filename = os.path.join(output_dir, 'topojson_tmp_in.topojson')
        with open(tmp_input_filename , 'w+') as f:
            f.write(topojson_file.get_formatted_topojson())

        tmp_output_filename = os.path.join(output_dir, 'topojson_tmp_out.topojson')
        mapshaper_command = f"mapshaper {tmp_input_filename} -simplify {percent}% -o {tmp_output_filename}"
        print(f'calling: {mapshaper_command}')
        subprocess.call(mapshaper_command, shell=True)

        # now add metadata back
        with open(tmp_output_filename) as f:
            output_topojson = json.loads(f.read())

        copy_custom_metadata(topojson_file.topojson, output_topojson)
        with open(output_file_path, 'w+') as f:
            f.write(json.dumps(output_topojson))

        print(f'Topojson output to {output_file_path}')
<commit_msg>more lint<commit_after>import json
import os
import subprocess
from django.core.management import BaseCommand

from custom.icds_reports.utils.topojson_util.topojson_util import get_topojson_file_for_level, \
    get_topojson_directory, copy_custom_metadata

class Command(BaseCommand):
    help = "Downsize a topojson file by a given percentage."

    def add_arguments(self, parser):
        parser.add_argument('level')
        parser.add_argument('output_file')
        parser.add_argument('percent', type=int)

    def handle(self, level, output_file, percent, *args, **kwargs):
        topojson_file = get_topojson_file_for_level(level)
        output_dir = get_topojson_directory()
        output_file_path = os.path.join(output_dir, output_file)
        tmp_input_filename = os.path.join(output_dir, 'topojson_tmp_in.topojson')
        with open(tmp_input_filename, 'w+') as f:
            f.write(topojson_file.get_formatted_topojson())

        tmp_output_filename = os.path.join(output_dir, 'topojson_tmp_out.topojson')
        mapshaper_command = f"mapshaper {tmp_input_filename} -simplify {percent}% -o {tmp_output_filename}"
        print(f'calling: {mapshaper_command}')
        subprocess.call(mapshaper_command, shell=True)

        # now add metadata back
        with open(tmp_output_filename) as f:
            output_topojson = json.loads(f.read())

        copy_custom_metadata(topojson_file.topojson, output_topojson)
        with open(output_file_path, 'w+') as f:
            f.write(json.dumps(output_topojson))

        print(f'Topojson output to {output_file_path}')
<|endoftext|>


<commit_before># Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.from nose.tools import assert_equal

import StringIO
import logging

from nose.tools import assert_equal, assert_true
from nose.plugins.skip import SkipTest

from django.contrib.auth.models import User

from desktop.lib.django_test_util import make_logged_in_client
from desktop.lib.test_utils import grant_access, add_to_group
from hadoop.pseudo_hdfs4 import is_live_cluster, shared_cluster

from indexer.conf import ENABLE_NEW_INDEXER
from indexer.controller import CollectionManagerController
from indexer.file_format import ApacheCombinedFormat, RubyLogFormat, HueLogFormat
from indexer.fields import Field
from indexer.indexers.morphline_operations import get_operator
from indexer.indexers.morphline import MorphlineIndexer
from indexer.solr_client import SolrClient
from indexer.solr_client_tests import MockSolrCdhCloudHdfsApi

LOG = logging.getLogger(__name__)

class TestIndexer():

  simpleCSVString = """id,Rating,Location,Name,Time
1,5,San Francisco,Good Restaurant,8:30pm
2,4,San Mateo,Cafe,11:30am
3,3,Berkeley,Sauls,2:30pm
"""
  simpleCSVFields = [
    {
      "name": "id",
      "type": "long",
      "operations": [],
      "keep": True,
      "required": False
    },
    {
      "name": "Rating",
      "type": "long",
      "operations": [],
      "keep": True,
      "required": False
    },
    {
      "name": "Location",
      "type": "string",
      "operations": [],
      "keep": True,
      "required": False
    },
    {
      "name": "Name",
      "type": "string",
      "operations": [],
      "keep": True,
      "required": False
    },
    {
      "name": "Time",
      "type": "string",
      "operations": [],
      "keep": True,
      "required": False
    }
  ]
  simpleCSVFormat = {
    'type': 'csv',
    'fieldSeparator': ',',
    'recordSeparator': '\n',
    'hasHeader': True,
    'quoteChar': '"'
  }

  def setUp(self):
    self.c = make_logged_in_client(is_superuser=False)
    grant_access("test", "test", "indexer")
    add_to_group("test")
    self.user = User.objects.get(username='test')
    self.solr_client = SolrClient(self.user, api=MockSolrCdhCloudHdfsApi())

    self.finish = ENABLE_NEW_INDEXER.set_for_testing(True)

  def tearDown(self):
    self.finish()

  def test_guess_csv_format(self):
    stream = StringIO.StringIO(TestIndexer.simpleCSVString)
    indexer = MorphlineIndexer("test", solr_client=self.solr_client)

    guessed_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}})

    fields = indexer.guess_field_types({"file": {"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns']
    # test format
    expected_format = self.simpleCSVFormat

    assert_equal(expected_format, guessed_format)

    # test fields
    expected_fields = self.simpleCSVFields

    for expected, actual in zip(expected_fields, fields):
      for key in ("name", "type"):
        assert_equal(expected[key], actual[key])

  def test_guess_format_invalid_csv_format(self):
    indexer = MorphlineIndexer("test", solr_client=self.solr_client)
    stream = StringIO.StringIO(TestIndexer.simpleCSVString)

    guessed_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}})

    guessed_format["fieldSeparator"] = "invalid separator"

    fields = indexer.guess_field_types({"file": {"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns']
    assert_equal(fields, [])

    stream.seek(0)
    guessed_format = indexer.guess_format({'file':  {"stream": stream, "name": "test.csv"}})

    guessed_format["recordSeparator"] = "invalid separator"

    fields = indexer.guess_field_types({"file": {"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns']
    assert_equal(fields, [])

    stream.seek(0)
    guessed_format = indexer.guess_format({'file':  {"stream": stream, "name": "test.csv"}})

    guessed_format["quoteChar"] = "invalid quoteChar"

    fields = indexer.guess_field_types({"file": {"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns']
    assert_equal(fields, [])

  def test_generate_csv_morphline(self):
    indexer = MorphlineIndexer("test", solr_client=self.solr_client)
    morphline =indexer.generate_morphline_config("test_collection", {
        "columns": self.simpleCSVFields,
        "format": self.simpleCSVFormat
      })

    assert_true(isinstance(morphline, basestring))

  def test_generate_apache_combined_morphline(self):
    self._test_fixed_type_format_generate_morphline(ApacheCombinedFormat)

  def test_generate_ruby_logs_morphline(self):
    raise SkipTest
    self._test_fixed_type_format_generate_morphline(RubyLogFormat)

  def test_generate_hue_log_morphline(self):
    self._test_fixed_type_format_generate_morphline(HueLogFormat)

  def test_generate_split_operation_morphline(self):
    split_dict = get_operator('split').get_default_operation()

    split_dict['fields'] = [
        Field("test_field_1", "string").to_dict(),
        Field("test_field_2", "string").to_dict()
      ]

    self._test_generate_field_operation_morphline(split_dict)

  def test_generate_extract_uri_components_operation_morphline(self):
    extract_uri_dict = get_operator('extract_uri_components').get_default_operation()

    extract_uri_dict['fields'] = [
        Field("test_field_1", "string").to_dict(),
        Field("test_field_2", "string").to_dict()
      ]

    self._test_generate_field_operation_morphline(extract_uri_dict)

  def test_generate_grok_operation_morphline(self):
    grok_dict = get_operator('grok').get_default_operation()

    grok_dict['fields'] = [
        Field("test_field_1", "string").to_dict(),
        Field("test_field_2", "string").to_dict()
      ]

    self._test_generate_field_operation_morphline(grok_dict)

  def test_generate_convert_date_morphline(self):
    convert_date_dict = get_operator('convert_date').get_default_operation()

    self._test_generate_field_operation_morphline(convert_date_dict)

  def test_generate_geo_ip_morphline(self):
    geo_ip_dict = get_operator('geo_ip').get_default_operation()

    geo_ip_dict['fields'] = [
        Field("test_field_1", "string").to_dict(),
        Field("test_field_2", "string").to_dict()
      ]

    self._test_generate_field_operation_morphline(geo_ip_dict)

  def test_generate_translate_morphline(self):
    translate_dict = get_operator('translate').get_default_operation()

    translate_dict['fields'] = [
      Field("test_field_1", "string").to_dict(),
      Field("test_field_2", "string").to_dict()
    ]

    translate_dict['settings']['mapping'].append({"key":"key","value":"value"})

    self._test_generate_field_operation_morphline(translate_dict)

  def test_generate_find_replace_morphline(self):
    find_replace_dict = get_operator('find_replace').get_default_operation()

    self._test_generate_field_operation_morphline(find_replace_dict)

  def test_end_to_end(self):
    if not is_live_cluster() or True: # Skipping as requires morplines libs to be setup
      raise SkipTest()

    cluster = shared_cluster()
    fs = cluster.fs
    make_logged_in_client(username="test", groupname="default", recreate=True, is_superuser=False)
    user = User.objects.get(username="test")
    collection_name = "test_collection"
    indexer = MorphlineIndexer("test", fs=fs, jt=cluster.jt, solr_client=self.solr_client)
    input_loc = "/tmp/test.csv"

    # upload the test file to hdfs
    fs.create(input_loc, data=TestIndexer.simpleCSVString, overwrite=True)

    # open a filestream for the file on hdfs
    stream = fs.open(input_loc)

    # guess the format of the file
    file_type_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}})

    field_types = indexer.guess_field_types({"file":{"stream": stream, "name": "test.csv"}, "format": file_type_format})

    format_ = field_types.copy()
    format_['format'] = file_type_format

    # find a field name available to use for the record's uuid
    unique_field = indexer.get_unique_field(format_)
    is_unique_generated = indexer.is_unique_generated(format_)

    # generate morphline
    morphline = indexer.generate_morphline_config(collection_name, format_, unique_field)

    schema_fields = indexer.get_kept_field_list(format_['columns'])
    if is_unique_generated:
      schema_fields += [{"name": unique_field, "type": "string"}]

    # create the collection from the specified fields
    collection_manager = CollectionManagerController("test")
    if collection_manager.collection_exists(collection_name):
      collection_manager.delete_collection(collection_name, None)
    collection_manager.create_collection(collection_name, schema_fields, unique_key_field=unique_field)

    # index the file
    indexer.run_morphline(MockedRequest(user=user, fs=cluster.fs, jt=cluster.jt), collection_name, morphline, input_loc)

  def _test_fixed_type_format_generate_morphline(self, format_):
    indexer = MorphlineIndexer("test", solr_client=self.solr_client)
    format_instance = format_()

    morphline = indexer.generate_morphline_config("test_collection", {
        "columns": [field.to_dict() for field in format_instance.fields],
        "format": format_instance.get_format()
      })

    assert_true(isinstance(morphline, basestring))

  def _test_generate_field_operation_morphline(self, operation_format):
    fields = TestIndexer.simpleCSVFields[:]
    fields[0]['operations'].append(operation_format)

    indexer = MorphlineIndexer("test", solr_client=self.solr_client)
    morphline =indexer.generate_morphline_config("test_collection", {
        "columns": fields,
        "format": TestIndexer.simpleCSVFormat
      })

    assert_true(isinstance(morphline, basestring))

class MockedRequest():
  def __init__(self, user, fs, jt):
    self.user = user
    self.fs = fs
    self.jt = jt
<commit_msg>HUE-7587 [indexer] Fix csv format type guessing test<commit_after># Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.from nose.tools import assert_equal

from copy import deepcopy

import StringIO
import logging

from nose.tools import assert_equal, assert_true
from nose.plugins.skip import SkipTest

from django.contrib.auth.models import User

from desktop.lib.django_test_util import make_logged_in_client
from desktop.lib.test_utils import grant_access, add_to_group
from hadoop.pseudo_hdfs4 import is_live_cluster, shared_cluster

from indexer.conf import ENABLE_NEW_INDEXER
from indexer.controller import CollectionManagerController
from indexer.file_format import ApacheCombinedFormat, RubyLogFormat, HueLogFormat
from indexer.fields import Field
from indexer.indexers.morphline_operations import get_operator
from indexer.indexers.morphline import MorphlineIndexer
from indexer.solr_client import SolrClient
from indexer.solr_client_tests import MockSolrCdhCloudHdfsApi

LOG = logging.getLogger(__name__)

class TestIndexer():

  simpleCSVString = """id,Rating,Location,Name,Time
1,5,San Francisco,Good Restaurant,8:30pm
2,4,San Mateo,Cafe,11:30am
3,3,Berkeley,Sauls,2:30pm
"""
  simpleCSVFields = [
    {
      "name": "id",
      "type": "long",
      "operations": [],
      "keep": True,
      "required": False
    },
    {
      "name": "Rating",
      "type": "long",
      "operations": [],
      "keep": True,
      "required": False
    },
    {
      "name": "Location",
      "type": "string",
      "operations": [],
      "keep": True,
      "required": False
    },
    {
      "name": "Name",
      "type": "string",
      "operations": [],
      "keep": True,
      "required": False
    },
    {
      "name": "Time",
      "type": "string",
      "operations": [],
      "keep": True,
      "required": False
    }
  ]
  simpleCSVFormat = {
    'type': 'csv',
    'fieldSeparator': ',',
    'recordSeparator': '\n',
    'hasHeader': True,
    'quoteChar': '"'
  }

  def setUp(self):
    self.c = make_logged_in_client(is_superuser=False)
    grant_access("test", "test", "indexer")
    add_to_group("test")
    self.user = User.objects.get(username='test')
    self.solr_client = SolrClient(self.user, api=MockSolrCdhCloudHdfsApi())

    self.finish = ENABLE_NEW_INDEXER.set_for_testing(True)

  def tearDown(self):
    self.finish()

  def test_guess_csv_format(self):
    stream = StringIO.StringIO(TestIndexer.simpleCSVString)
    indexer = MorphlineIndexer("test", solr_client=self.solr_client)

    guessed_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}})

    fields = indexer.guess_field_types({"file": {"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns']
    # test format
    expected_format = self.simpleCSVFormat

    assert_equal(expected_format, guessed_format)

    # test fields
    expected_fields = self.simpleCSVFields

    for expected, actual in zip(expected_fields, fields):
      for key in ("name", "type"):
        assert_equal(expected[key], actual[key])

  def test_guess_format_invalid_csv_format(self):
    indexer = MorphlineIndexer("test", solr_client=self.solr_client)
    stream = StringIO.StringIO(TestIndexer.simpleCSVString)

    guessed_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}})

    guessed_format["fieldSeparator"] = "invalid separator"

    fields = indexer.guess_field_types({"file": {"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns']
    assert_equal(fields, [])

    stream.seek(0)
    guessed_format = indexer.guess_format({'file':  {"stream": stream, "name": "test.csv"}})

    guessed_format["recordSeparator"] = "invalid separator"

    fields = indexer.guess_field_types({"file": {"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns']
    assert_equal(fields, [])

    stream.seek(0)
    guessed_format = indexer.guess_format({'file':  {"stream": stream, "name": "test.csv"}})

    guessed_format["quoteChar"] = "invalid quoteChar"

    fields = indexer.guess_field_types({"file": {"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns']
    assert_equal(fields, [])

  def test_generate_csv_morphline(self):
    indexer = MorphlineIndexer("test", solr_client=self.solr_client)
    morphline = indexer.generate_morphline_config("test_collection", {
        "columns": deepcopy(self.simpleCSVFields),
        "format": self.simpleCSVFormat
      })

    assert_true(isinstance(morphline, basestring))

  def test_generate_apache_combined_morphline(self):
    self._test_fixed_type_format_generate_morphline(ApacheCombinedFormat)

  def test_generate_ruby_logs_morphline(self):
    raise SkipTest
    self._test_fixed_type_format_generate_morphline(RubyLogFormat)

  def test_generate_hue_log_morphline(self):
    self._test_fixed_type_format_generate_morphline(HueLogFormat)

  def test_generate_split_operation_morphline(self):
    split_dict = get_operator('split').get_default_operation()

    split_dict['fields'] = [
        Field("test_field_1", "string").to_dict(),
        Field("test_field_2", "string").to_dict()
      ]

    self._test_generate_field_operation_morphline(split_dict)

  def test_generate_extract_uri_components_operation_morphline(self):
    extract_uri_dict = get_operator('extract_uri_components').get_default_operation()

    extract_uri_dict['fields'] = [
        Field("test_field_1", "string").to_dict(),
        Field("test_field_2", "string").to_dict()
      ]

    self._test_generate_field_operation_morphline(extract_uri_dict)

  def test_generate_grok_operation_morphline(self):
    grok_dict = get_operator('grok').get_default_operation()

    grok_dict['fields'] = [
        Field("test_field_1", "string").to_dict(),
        Field("test_field_2", "string").to_dict()
      ]

    self._test_generate_field_operation_morphline(grok_dict)

  def test_generate_convert_date_morphline(self):
    convert_date_dict = get_operator('convert_date').get_default_operation()

    self._test_generate_field_operation_morphline(convert_date_dict)

  def test_generate_geo_ip_morphline(self):
    geo_ip_dict = get_operator('geo_ip').get_default_operation()

    geo_ip_dict['fields'] = [
        Field("test_field_1", "string").to_dict(),
        Field("test_field_2", "string").to_dict()
      ]

    self._test_generate_field_operation_morphline(geo_ip_dict)

  def test_generate_translate_morphline(self):
    translate_dict = get_operator('translate').get_default_operation()

    translate_dict['fields'] = [
      Field("test_field_1", "string").to_dict(),
      Field("test_field_2", "string").to_dict()
    ]

    translate_dict['settings']['mapping'].append({"key":"key","value":"value"})

    self._test_generate_field_operation_morphline(translate_dict)

  def test_generate_find_replace_morphline(self):
    find_replace_dict = get_operator('find_replace').get_default_operation()

    self._test_generate_field_operation_morphline(find_replace_dict)

  def test_end_to_end(self):
    if not is_live_cluster() or True: # Skipping as requires morplines libs to be setup
      raise SkipTest()

    cluster = shared_cluster()
    fs = cluster.fs
    make_logged_in_client(username="test", groupname="default", recreate=True, is_superuser=False)
    user = User.objects.get(username="test")
    collection_name = "test_collection"
    indexer = MorphlineIndexer("test", fs=fs, jt=cluster.jt, solr_client=self.solr_client)
    input_loc = "/tmp/test.csv"

    # upload the test file to hdfs
    fs.create(input_loc, data=TestIndexer.simpleCSVString, overwrite=True)

    # open a filestream for the file on hdfs
    stream = fs.open(input_loc)

    # guess the format of the file
    file_type_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}})

    field_types = indexer.guess_field_types({"file":{"stream": stream, "name": "test.csv"}, "format": file_type_format})

    format_ = field_types.copy()
    format_['format'] = file_type_format

    # find a field name available to use for the record's uuid
    unique_field = indexer.get_unique_field(format_)
    is_unique_generated = indexer.is_unique_generated(format_)

    # generate morphline
    morphline = indexer.generate_morphline_config(collection_name, format_, unique_field)

    schema_fields = indexer.get_kept_field_list(format_['columns'])
    if is_unique_generated:
      schema_fields += [{"name": unique_field, "type": "string"}]

    # create the collection from the specified fields
    collection_manager = CollectionManagerController("test")
    if collection_manager.collection_exists(collection_name):
      collection_manager.delete_collection(collection_name, None)
    collection_manager.create_collection(collection_name, schema_fields, unique_key_field=unique_field)

    # index the file
    indexer.run_morphline(MockedRequest(user=user, fs=cluster.fs, jt=cluster.jt), collection_name, morphline, input_loc)

  def _test_fixed_type_format_generate_morphline(self, format_):
    indexer = MorphlineIndexer("test", solr_client=self.solr_client)
    format_instance = format_()

    morphline = indexer.generate_morphline_config("test_collection", {
        "columns": [field.to_dict() for field in format_instance.fields],
        "format": format_instance.get_format()
      })

    assert_true(isinstance(morphline, basestring))

  def _test_generate_field_operation_morphline(self, operation_format):
    fields = deepcopy(TestIndexer.simpleCSVFields)
    fields[0]['operations'].append(operation_format)

    indexer = MorphlineIndexer("test", solr_client=self.solr_client)
    morphline =indexer.generate_morphline_config("test_collection", {
        "columns": fields,
        "format": TestIndexer.simpleCSVFormat
      })

    assert_true(isinstance(morphline, basestring))

class MockedRequest():
  def __init__(self, user, fs, jt):
    self.user = user
    self.fs = fs
    self.jt = jt
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.core.management.base import BaseCommand
from django.core.management.utils import get_random_secret_key

from django_extensions.management.utils import signalcommand

class Command(BaseCommand):
    help = "Generates a new SECRET_KEY that can be used in a project settings file."

    requires_system_checks = False

    @signalcommand
    def handle(self, *args, **options):
        return get_random_secret_key()
<commit_msg>Django 1.8+ compatiblity<commit_after># -*- coding: utf-8 -*-
from django.core.management.base import BaseCommand
from django.utils.crypto import get_random_string

from django_extensions.management.utils import signalcommand

class Command(BaseCommand):
    help = "Generates a new SECRET_KEY that can be used in a project settings file."

    requires_system_checks = False

    @signalcommand
    def handle(self, *args, **options):
        chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
        return get_random_string(50, chars)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Generated by Django 1.10.8 on 2018-02-01 19:50
from __future__ import unicode_literals

from django.db import migrations, models
import jsonfield.fields

class Migration(migrations.Migration):

    dependencies = [
        ('djangocms_slick_slider', '0002_slicksliderimage_link_target'),
    ]

    operations = [
        migrations.AlterModelOptions(
            name='slickslider',
            options={'verbose_name': 'slick slider', 'verbose_name_plural': 'slick sliders'},
        ),
        migrations.AlterField(
            model_name='slickslider',
            name='arrow_color',
            field=models.CharField(default='#666', help_text='Define the color of slider arrows here. All CSS color values work (e.g. #efefef).', max_length=255, verbose_name='arrow color'),
        ),
        migrations.AlterField(
            model_name='slickslider',
            name='settings',
            field=jsonfield.fields.JSONField(blank=True, help_text='Check Slick Documentation for possible settings
Use JSON format and check the errors in the editor
You can also use online JSON validators', verbose_name='slick settings'),
        ),
        migrations.AlterField(
            model_name='slickslider',
            name='title',
            field=models.CharField(blank=True, max_length=255, null=True, verbose_name='slider title'),
        ),
    ]
<commit_msg>debugging<commit_after># -*- coding: utf-8 -*-
# Generated by Django 1.10.8 on 2018-02-01 19:50
from __future__ import unicode_literals

from django.db import migrations, models
import jsonfield.fields

class Migration(migrations.Migration):

    dependencies = [
        ('djangocms_slick_slider', '0002_slicksliderimage_link_target'),
    ]

    operations = [
        migrations.AlterModelOptions(
            name='slickslider',
            options={'verbose_name': 'slick slider', 'verbose_name_plural': 'slick sliders'},
        ),
        migrations.AlterField(
            model_name='slickslider',
            name='arrow_color',
            field=models.CharField(default='#666', help_text='Define the color of slider arrows here. All CSS color values work (e.g. #efefef).', max_length=255, verbose_name='arrow color'),
        ),
        migrations.AlterField(
            model_name='slickslider',
            name='settings',
            field=jsonfield.fields.JSONField(blank=True, null=True, help_text='Check Slick Documentation for possible settings
Use JSON format and check the errors in the editor
You can also use online JSON validators', verbose_name='slick settings'),
        ),
        migrations.AlterField(
            model_name='slickslider',
            name='title',
            field=models.CharField(blank=True, max_length=255, null=True, verbose_name='slider title'),
        ),
    ]
<|endoftext|>


<commit_before># Standard imports
import unittest
import datetime as pydt
import logging
import json
import uuid
import attrdict as ad
import time
import geojson as gj

# Our imports
import emission.tests.common

import emission.core.get_database as edb
import emission.net.usercache.abstract_usercache as enua
import emission.storage.timeseries.abstract_timeseries as esta
import emission.net.usercache.abstract_usercache_handler as enuah
import emission.net.api.usercache as mauc
import emission.core.wrapper.trip as ecwt

# These are the current formatters, so they are included here for testing.
# However, it is unclear whether or not we need to add other tests as we add other formatters,
# specially if they follow the same pattern.

class TestBuiltinUserCacheHandlerOutput(unittest.TestCase):
    def setUp(self):
        emission.tests.common.dropAllCollections(edb.get_db())
        self.testUserUUID1 = uuid.uuid4()
        self.testUserUUID2 = uuid.uuid4()
        self.testUserUUIDios = uuid.uuid4()

        self.activity_entry = json.load(open("emission/tests/data/netTests/android.activity.txt"))
        self.location_entry = json.load(open("emission/tests/data/netTests/android.location.raw.txt"))
        self.transition_entry = json.load(open("emission/tests/data/netTests/android.transition.txt"))
        self.entry_list = [self.activity_entry, self.location_entry, self.transition_entry]

        self.uc1 = enua.UserCache.getUserCache(self.testUserUUID1)
        self.uc2 = enua.UserCache.getUserCache(self.testUserUUID2)
        self.ucios = enua.UserCache.getUserCache(self.testUserUUIDios)

        self.ts1 = esta.TimeSeries.get_time_series(self.testUserUUID1)
        self.ts2 = esta.TimeSeries.get_time_series(self.testUserUUID2)
        self.tsios = esta.TimeSeries.get_time_series(self.testUserUUIDios)

        for entry in self.entry_list:
            # Needed because otherwise we get a DuplicateKeyError while
            # inserting the mutiple copies
            del entry["_id"]

        self.curr_ts = int(time.time())
        for offset in range(self.curr_ts - 5 * 60, self.curr_ts, 30):
            for entry in self.entry_list:
                entry["metadata"]["write_ts"] = offset * 1000
            mauc.sync_phone_to_server(self.testUserUUID1, self.entry_list)

        for offset in range(self.curr_ts - 7 * 60 + 1, self.curr_ts - 2 * 60 + 1, 30):
            for entry in self.entry_list:
                entry["metadata"]["write_ts"] = offset * 1000
            mauc.sync_phone_to_server(self.testUserUUID2, self.entry_list)

        self.ios_activity_entry = json.load(open("emission/tests/data/netTests/ios.activity.txt"))
        self.ios_location_entry = json.load(open("emission/tests/data/netTests/ios.location.txt"))
        self.ios_transition_entry = json.load(open("emission/tests/data/netTests/ios.transition.txt"))
        self.ios_entry_list = [self.ios_activity_entry, self.ios_location_entry, self.ios_transition_entry]
        for entry in self.ios_entry_list:
            # Needed because otherwise we get a DuplicateKeyError while
            # inserting the mutiple copies
            del entry["_id"]

        for offset in range(self.curr_ts - 5 * 60, self.curr_ts, 30):
            for entry in self.ios_entry_list:
                entry["metadata"]["write_ts"] = offset
            mauc.sync_phone_to_server(self.testUserUUIDios, self.ios_entry_list)

    # The first query for every platform is likely to work
    # because startTs = None and endTs, at least for iOS, is way out there
    # But on the next call, if we are multiplying by 1000, it won't work any more.
    # Let's add a new test for this
    def testGetLocalDay(self):
        test_dt = pydt.datetime(2016, 1, 1, 9, 46, 0, 0)
        test_trip = ecwt.Trip({'start_local_dt': test_dt, 'start_fmt_time': test_dt.isoformat()})
        test_handler = enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1)
        self.assertEqual(test_handler.get_local_day_from_fmt_time(test_trip), "2016-01-01")
        self.assertEqual(test_handler.get_local_day_from_local_dt(test_trip), "2016-01-01")

    def testGetTripListForSevenDays(self):
        test_handler = enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1)

    def testGetObsoleteEntries(self):
        valid_entries = ["2015-12-30", "2015-12-29", "2015-12-31", "2015-01-01"]
        uc = enua.UserCache.getUserCache(self.testUserUUID1)
        uch = enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1)
        uc.putDocument("2015-12-30", {"a": 1})
        uc.putDocument("2015-12-29", {"a": 1})
        uc.putDocument("2015-12-28", {"a": 1})
        uc.putDocument("2015-12-27", {"a": 1})
        uc.putDocument("2015-12-26", {"a": 1})
        obsolete_entries = uch.get_obsolete_entries(uc, valid_entries)
        # the result should include entries that are in the past (28,27,26), but should
        # NOT include newly added entries
        self.assertEqual(obsolete_entries, set(["2015-12-28", "2015-12-27", "2015-12-26"]))

    def testDeleteObsoleteEntries(self):
        valid_bins = {"2015-12-30":[{"b": 2}],
                      "2015-12-29":[{"b": 2}],
                      "2015-12-31":[{"b": 2}]}
        uc = enua.UserCache.getUserCache(self.testUserUUID1)
        uch = enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1)
        uc.putDocument("2015-12-30", {"a": 1})
        uc.putDocument("2015-12-29", {"a": 1})
        uc.putDocument("2015-12-28", {"a": 1})
        uc.putDocument("2015-12-27", {"a": 1})
        uc.putDocument("2015-12-26", {"a": 1})
        uch.delete_obsolete_entries(uc, list(valid_bins.iterkeys()))
        # the result should include entries that are in the past (28,27,26), but should
        # NOT include newly added entries
        self.assertEqual(uc.getDocumentKeyList(), ["2015-12-30", "2015-12-29"])

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    unittest.main()
<commit_msg>Add a test for the last behavior - that we always retain the sensor_config<commit_after># Standard imports
import unittest
import datetime as pydt
import logging
import json
import uuid
import attrdict as ad
import time
import geojson as gj

# Our imports
import emission.tests.common

import emission.core.get_database as edb
import emission.net.usercache.abstract_usercache as enua
import emission.storage.timeseries.abstract_timeseries as esta
import emission.net.usercache.abstract_usercache_handler as enuah
import emission.net.api.usercache as mauc
import emission.core.wrapper.trip as ecwt

# These are the current formatters, so they are included here for testing.
# However, it is unclear whether or not we need to add other tests as we add other formatters,
# specially if they follow the same pattern.

class TestBuiltinUserCacheHandlerOutput(unittest.TestCase):
    def setUp(self):
        emission.tests.common.dropAllCollections(edb.get_db())
        self.testUserUUID1 = uuid.uuid4()
        self.testUserUUID2 = uuid.uuid4()
        self.testUserUUIDios = uuid.uuid4()

        self.activity_entry = json.load(open("emission/tests/data/netTests/android.activity.txt"))
        self.location_entry = json.load(open("emission/tests/data/netTests/android.location.raw.txt"))
        self.transition_entry = json.load(open("emission/tests/data/netTests/android.transition.txt"))
        self.entry_list = [self.activity_entry, self.location_entry, self.transition_entry]

        self.uc1 = enua.UserCache.getUserCache(self.testUserUUID1)
        self.uc2 = enua.UserCache.getUserCache(self.testUserUUID2)
        self.ucios = enua.UserCache.getUserCache(self.testUserUUIDios)

        self.ts1 = esta.TimeSeries.get_time_series(self.testUserUUID1)
        self.ts2 = esta.TimeSeries.get_time_series(self.testUserUUID2)
        self.tsios = esta.TimeSeries.get_time_series(self.testUserUUIDios)

        for entry in self.entry_list:
            # Needed because otherwise we get a DuplicateKeyError while
            # inserting the mutiple copies
            del entry["_id"]

        self.curr_ts = int(time.time())
        for offset in range(self.curr_ts - 5 * 60, self.curr_ts, 30):
            for entry in self.entry_list:
                entry["metadata"]["write_ts"] = offset * 1000
            mauc.sync_phone_to_server(self.testUserUUID1, self.entry_list)

        for offset in range(self.curr_ts - 7 * 60 + 1, self.curr_ts - 2 * 60 + 1, 30):
            for entry in self.entry_list:
                entry["metadata"]["write_ts"] = offset * 1000
            mauc.sync_phone_to_server(self.testUserUUID2, self.entry_list)

        self.ios_activity_entry = json.load(open("emission/tests/data/netTests/ios.activity.txt"))
        self.ios_location_entry = json.load(open("emission/tests/data/netTests/ios.location.txt"))
        self.ios_transition_entry = json.load(open("emission/tests/data/netTests/ios.transition.txt"))
        self.ios_entry_list = [self.ios_activity_entry, self.ios_location_entry, self.ios_transition_entry]
        for entry in self.ios_entry_list:
            # Needed because otherwise we get a DuplicateKeyError while
            # inserting the mutiple copies
            del entry["_id"]

        for offset in range(self.curr_ts - 5 * 60, self.curr_ts, 30):
            for entry in self.ios_entry_list:
                entry["metadata"]["write_ts"] = offset
            mauc.sync_phone_to_server(self.testUserUUIDios, self.ios_entry_list)

    # The first query for every platform is likely to work
    # because startTs = None and endTs, at least for iOS, is way out there
    # But on the next call, if we are multiplying by 1000, it won't work any more.
    # Let's add a new test for this
    def testGetLocalDay(self):
        test_dt = pydt.datetime(2016, 1, 1, 9, 46, 0, 0)
        test_trip = ecwt.Trip({'start_local_dt': test_dt, 'start_fmt_time': test_dt.isoformat()})
        test_handler = enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1)
        self.assertEqual(test_handler.get_local_day_from_fmt_time(test_trip), "2016-01-01")
        self.assertEqual(test_handler.get_local_day_from_local_dt(test_trip), "2016-01-01")

    def testGetTripListForSevenDays(self):
        test_handler = enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1)

    def testGetObsoleteEntries(self):
        valid_entries = ["2015-12-30", "2015-12-29", "2015-12-31", "2015-01-01"]
        uc = enua.UserCache.getUserCache(self.testUserUUID1)
        uch = enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1)
        uc.putDocument("2015-12-30", {"a": 1})
        uc.putDocument("2015-12-29", {"a": 1})
        uc.putDocument("2015-12-28", {"a": 1})
        uc.putDocument("2015-12-27", {"a": 1})
        uc.putDocument("2015-12-26", {"a": 1})
        obsolete_entries = uch.get_obsolete_entries(uc, valid_entries)
        # the result should include entries that are in the past (28,27,26), but should
        # NOT include newly added entries
        self.assertEqual(obsolete_entries, set(["2015-12-28", "2015-12-27", "2015-12-26"]))

    def testDeleteObsoleteEntries(self):
        valid_bins = {"2015-12-30":[{"b": 2}],
                      "2015-12-29":[{"b": 2}],
                      "2015-12-31":[{"b": 2}]}
        uc = enua.UserCache.getUserCache(self.testUserUUID1)
        uch = enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1)
        uc.putDocument("2015-12-30", {"a": 1})
        uc.putDocument("2015-12-29", {"a": 1})
        uc.putDocument("2015-12-28", {"a": 1})
        uc.putDocument("2015-12-27", {"a": 1})
        uc.putDocument("2015-12-26", {"a": 1})
        uch.delete_obsolete_entries(uc, list(valid_bins.iterkeys()))
        # the result should include entries that are in the past (28,27,26), but should
        # NOT include newly added entries
        self.assertEqual(uc.getDocumentKeyList(), ["2015-12-30", "2015-12-29"])

    def testRetainSetConfig(self):
        valid_bins = {"2015-12-30":[{"b": 2}],
                      "2015-12-29":[{"b": 2}],
                      "2015-12-31":[{"b": 2}]}
        uc = enua.UserCache.getUserCache(self.testUserUUID1)
        uch = enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1)
        uc.putDocument("2015-12-30", {"a": 1})
        uc.putDocument("2015-12-29", {"a": 1})
        uc.putDocument("2015-12-28", {"a": 1})
        uc.putDocument("2015-12-27", {"a": 1})
        uc.putDocument("2015-12-26", {"a": 1})
        uc.putDocument("config/sensor_config", {"a": 1})
        uch.delete_obsolete_entries(uc, list(valid_bins.iterkeys()))
        # the result should include entries that are in the past (28,27,26), but should
        # NOT include newly added entries
        self.assertEqual(uc.getDocumentKeyList(), ["2015-12-30", "2015-12-29",
            "config/sensor_config"])

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    unittest.main()
<|endoftext|>


<commit_before># modified from http://djangosnippets.org/snippets/656/

from django.template import Library
from django.utils.safestring import mark_safe
from django.core.urlresolvers import resolve, Resolver404
from django.conf import settings
import re

register = Library()

url_name_mappings = settings.BREADCRUMB_URL_NAME_MAPPINGS

separator = ' » ';
hansard_part = 'hansard/'

@register.filter
def breadcrumbs(url):
    query_pos = url.find("?")
    bare_url = url
    if query_pos >= 0:
        bare_url = bare_url[0:query_pos]
    bare_url = bare_url.strip('/')
    if bare_url.startswith(hansard_part):
        bare_url = hansard_part + bare_url[len(hansard_part):].replace('/',' : ')
    links = bare_url.split('/')
    bread = []
    total = len(links)-1
    if total == 0 and links[0] == "":
        bcrumb = '
Home
'
    else:
        if links[total] == 'all': # if links ends with 'all', drop it
          links = links[0:total]
          total -= 1
        home = ['
Home %s
' % separator]

        seen_links = {}

        for i, link in enumerate(links):

            if seen_links.get(link):
                continue
            else:
                seen_links[link] = True

            if not link == '':
                bread.append(link)
                if link in url_name_mappings:
                    (sub_link, this_url) = url_name_mappings[link]
                elif re.match(r'^[\d\-\.,]+$', link):
                    # eg '-1.23,4.56'
                    sub_link = link
                    sub_link = re.sub(r',\s*', ', ', sub_link)
                else:
                    sub_link = re.sub('[_\-]', ' ', link).title()
                    sub_link = re.sub('\\bFaq\\b', 'FAQ', sub_link)
                    this_url = "/{0}/".format("/".join(bread))
                if not i == total:
                    try:
                        resolve(this_url)
                        tlink = '
%s %s
' % (this_url, sub_link, sub_link, separator)
                    except Resolver404:
                        tlink = '
%s %s
' % (sub_link, separator)

                else:
                    tlink = '
%s
' % sub_link
                home.append(tlink)
        bcrumb = "".join(home)
    return mark_safe(bcrumb)

<commit_msg>Remove /*/is/* fragment from breadcrumbs<commit_after># modified from http://djangosnippets.org/snippets/656/

from django.template import Library
from django.utils.safestring import mark_safe
from django.core.urlresolvers import resolve, Resolver404
from django.conf import settings
import re

register = Library()

url_name_mappings = settings.BREADCRUMB_URL_NAME_MAPPINGS

separator = ' » ';
hansard_part = 'hansard/'

@register.filter
def breadcrumbs(url):
    query_pos = url.find("?")
    bare_url = url
    if query_pos >= 0:
        bare_url = bare_url[0:query_pos]
    bare_url = bare_url.strip('/')
    if bare_url.startswith(hansard_part):
        bare_url = hansard_part + bare_url[len(hansard_part):].replace('/',' : ')
    links = bare_url.split('/')
    bread = []
    total = len(links)-1
    if total == 0 and links[0] == "":
        bcrumb = '
Home
'
    else:

        if links[1] == 'is':
          # (Organisation|Place|etc.)Kind links like /organization/is/house/
          # (drop it)
          links[1:2] = []
          total -= 1
        if links[total] == 'all': # if links ends with 'all', drop it
          links = links[0:total]
          total -= 1

        home = ['
Home %s
' % separator]

        seen_links = {}

        for i, link in enumerate(links):

            if seen_links.get(link):
                continue
            else:
                seen_links[link] = True

            if not link == '':
                bread.append(link)
                if link in url_name_mappings:
                    (sub_link, this_url) = url_name_mappings[link]
                elif re.match(r'^[\d\-\.,]+$', link):
                    # eg '-1.23,4.56'
                    sub_link = link
                    sub_link = re.sub(r',\s*', ', ', sub_link)
                else:
                    sub_link = re.sub('[_\-]', ' ', link).title()
                    sub_link = re.sub('\\bFaq\\b', 'FAQ', sub_link)
                    this_url = "/{0}/".format("/".join(bread))
                if not i == total:
                    try:
                        resolve(this_url)
                        tlink = '
%s %s
' % (this_url, sub_link, sub_link, separator)
                    except Resolver404:
                        tlink = '
%s %s
' % (sub_link, separator)

                else:
                    tlink = '
%s
' % sub_link
                home.append(tlink)
        bcrumb = "".join(home)
    return mark_safe(bcrumb)

<|endoftext|>


<commit_before>import unittest
from pony.orm import *

class TestObjectFlatMonad(unittest.TestCase):
    def setUp(self):
        _diagram_ = Diagram()
        self.diagram = _diagram_
        class Student(Entity):
            name = Required(unicode)
            scholarship = Optional(int)
            group = Required("Group")
            marks = Set("Mark")
        class Group(Entity):
            number = PrimaryKey(int)
            department = Required(int)
            students = Set(Student)
            subjects = Set("Subject")
        class Subject(Entity):
            name = PrimaryKey(unicode)
            groups = Set(Group)
            marks = Set("Mark")
        class Mark(Entity):
            value = Required(int)
            student = Required(Student)
            subject = Required(Subject)
            PrimaryKey(student, subject)
        self.Student = Student
        self.Group = Group
        self.Subject = Subject
        self.Mark = Mark
        self.db = Database('sqlite', ':memory:')
        con = self.db.get_connection()
        con.executescript("""
        drop table if exists Student;
        create table Student(
            id integer primary key,
            name varchar(20),
            scholarship integer,
            [group] integer
        );
        drop table if exists [Group];
        create table [Group](
            number integer primary key,
            department integer
        );
        drop table if exists Subject;
        create table Subject(
            name varchar(20) primary key
        );
        drop table if exists Group_Subject;
        create table Group_Subject(
            [group] integer,
            subject varchar(20),
            primary key ([group], subject)
        );
        drop table if exists Mark;
        create table Mark(
            value integer,
            student integer,
            subject varchar(20),
            primary key (student, subject)
        );
        insert into Student values (1, "Joe", null, 41);
        insert into Student values (2, "Bob", 100, 41);
        insert into Student values (3, "Beth", 500, 41);
        insert into Student values (4, "Jon", 500, 42);
        insert into Student values (5, "Pete", 700, 42);
        insert into [Group] values (41, 101);
        insert into [Group] values (42, 102);
        insert into [Group] values (43, 102);
        insert into Group_Subject values (41, "Math");
        insert into Group_Subject values (41, "Physics");
        insert into Group_Subject values (41, "History");
        insert into Group_Subject values (42, "Math");
        insert into Group_Subject values (42, "Physics");
        insert into Group_Subject values (43, "Physics");
        insert into Subject values ("Math");
        insert into Subject values ("Physics");
        insert into Subject values ("History");
        insert into Mark values (5, 1, "Math");
        insert into Mark values (4, 2, "Physics");
        insert into Mark values (3, 2, "Math");
        insert into Mark values (2, 2, "History");
        insert into Mark values (1, 3, "History");
        insert into Mark values (2, 3, "Math");
        insert into Mark values (2, 4, "Math");
        """)
        self.db.generate_mapping(check_tables=True)
    def test1(self):
        Group = self.Group
        Subject = self.Subject
        result = set(select(s.groups for s in Subject if len(s.name) == 4))
        self.assertEquals(result, set([Group(41), Group(42)]))
    def test2(self):
        Group = self.Group
        Student = self.Student
        result = set(select(g.students for g in Group if g.department == 102))
        self.assertEquals(result, set([Student(5), Student(4)]))

if __name__ == '__main__':
    unittest.main()
        <commit_msg>Refactoring of test_object_flat_monad.py<commit_after>import unittest
from pony.orm import *

class Student(Entity):
    name = Required(unicode)
    scholarship = Optional(int)
    group = Required("Group")
    marks = Set("Mark")

class Group(Entity):
    number = PrimaryKey(int)
    department = Required(int)
    students = Set(Student)
    subjects = Set("Subject")

class Subject(Entity):
    name = PrimaryKey(unicode)
    groups = Set(Group)
    marks = Set("Mark")

class Mark(Entity):
    value = Required(int)
    student = Required(Student)
    subject = Required(Subject)
    PrimaryKey(student, subject)

db = Database('sqlite', ':memory:')
db.generate_mapping(create_tables=True)

@with_transaction
def populate_db():
    Math = Subject.create("Math")
    Physics = Subject.create("Physics")
    History = Subject.create("History")

    g41 = Group.create(41, department=101, subjects=[ Math, Physics, History ])
    g42 = Group.create(42, department=102, subjects=[ Math, Physics ])
    g43 = Group.create(43, department=102, subjects=[ Physics ])

    s1 = Student.create(1, name="Joe", scholarship=None, group=g41)
    s2 = Student.create(2, name="Bob", scholarship=100, group=g41)
    s3 = Student.create(3, name="Beth", scholarship=500, group=g41)
    s4 = Student.create(4, name="Jon", scholarship=500, group=g42)
    s5 = Student.create(5, name="Pete", scholarship=700, group=g42)

    Mark.create(value=5, student=s1, subject=Math)
    Mark.create(value=4, student=s2, subject=Physics)
    Mark.create(value=3, student=s2, subject=Math)
    Mark.create(value=2, student=s2, subject=History)
    Mark.create(value=1, student=s3, subject=History)
    Mark.create(value=2, student=s3, subject=Math)
    Mark.create(value=2, student=s4, subject=Math)
populate_db()

class TestObjectFlatMonad(unittest.TestCase):
    def setUp(self):
        rollback()

    def tearDown(Self):
        rollback()

    def test1(self):
        result = set(select(s.groups for s in Subject if len(s.name) == 4))
        self.assertEquals(result, set([Group(41), Group(42)]))

    def test2(self):
        result = set(select(g.students for g in Group if g.department == 102))
        self.assertEquals(result, set([Student(5), Student(4)]))

if __name__ == '__main__':
    unittest.main()
        <|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf8 -*-
"""
Copyright (c) 2011 Tyler Kenendy <tk@tkte.ch>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

from .topping import Topping
import types

from jawa.constants import *
from jawa.cf import ClassFile

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

class BiomeTopping(Topping):
    """Gets most biome types."""

    PROVIDES = [
        "biomes"
    ]

    DEPENDS = [
        "identify.biome.superclass"
    ]

    @staticmethod
    def act(aggregate, jar, verbose=False):
        biomes = aggregate.setdefault("biomes", {})
        if "biome.superclass" not in aggregate["classes"]:
            return
        superclass = aggregate["classes"]["biome.superclass"]
        cf = ClassFile(StringIO(jar.read(superclass + ".class")))
        method = cf.methods.find_one(name="<clinit>")
        tmp = None
        stack = None
        for ins in method.code.disassemble():
            if ins.mnemonic == "new":
                if tmp is not None and tmp.has_key("name") and tmp["name"] != " and ":
                    biomes[tmp["name"]] = tmp
                stack = []
                const = cf.constants.get(ins.operands[0].value)
                tmp = {
                    "calls": {},
                    "rainfall": 0.5,
                    "height": [0.1, 0.3],
                    "temperature": 0.5,
                    "class": cf.constants.get(ins.operands[0].value).name.value
                }
            elif tmp is None:
                continue
            elif ins.mnemonic == "invokespecial":
                const = cf.constants.get(ins.operands[0].value)
                name = const.name_and_type.name.value
                if len(stack) == 2 and (type(stack[1]) == types.FloatType or type(stack[0]) == types.FloatType):
                    tmp["calls"][name] = [stack.pop(), stack.pop()]
                elif len(stack) >= 1 and type(stack[0]) == types.IntType: # 1, 2, 3-argument beginning with int = id
                    tmp["id"] = stack[0]
                    stack = []
                elif name != "<init>":
                    tmp["rainfall"] = 0
            elif ins.opcode == "invokevirtual":
                if len(stack) == 1 and "color" not in tmp:
                    tmp["color"] = stack.pop()
                if len(stack) == 2:
                    tmp["rainfall"] = stack.pop()
                    tmp["temperature"] = stack.pop()

            # numeric values & constants
            elif ins.mnemonic in ("ldc", "ldc_w"):
                const = cf.constants.get(ins.operands[0].value)
                if isinstance(const, ConstantString):
                    tmp["name"] = const.string.value
                if isinstance(const, (ConstantInteger, ConstantFloat)):
                    stack.append(const.value)

            elif ins.opcode <= 8 and ins.opcode >= 2:  # iconst
                stack.append(ins.opcode - 3)
            elif ins.opcode >= 0xb and ins.opcode <= 0xd:  # fconst
                stack.append(ins.opcode - 0xb)
            elif ins.mnemonic == "bipush":
                stack.append(ins.operands[0].value)

        if tmp is not None and tmp.has_key("name") and tmp["name"] != " and ":
            biomes[tmp["name"]] = tmp

        weather, height = BiomeTopping.map_methods(biomes)

        for biome in biomes.itervalues():
            calls = biome.pop("calls")
            if height in calls:
                biome["height"] = calls[height]
                biome["height"].reverse()
            if weather in calls:
                biome["temperature"] = calls[weather][1]
                biome["rainfall"] = calls[weather][0]

    @staticmethod
    def map_methods(biomes):
        for biome in biomes.itervalues():
            for call in biome["calls"]:
                if biome["calls"][call][1] > 1 and len(biome["calls"]) > 1:
                    keys = biome["calls"].keys()
                    keys.remove(call)
                    return (call, keys[0])
        return (None, None)
<commit_msg>Partially fix biomes.py for 1.9<commit_after>#!/usr/bin/env python
# -*- coding: utf8 -*-
"""
Copyright (c) 2011 Tyler Kenendy <tk@tkte.ch>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

from .topping import Topping
import types

from jawa.constants import *
from jawa.cf import ClassFile

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

class BiomeTopping(Topping):
    """Gets most biome types."""

    PROVIDES = [
        "biomes"
    ]

    DEPENDS = [
        "identify.biome.superclass"
    ]

    @staticmethod
    def act(aggregate, jar, verbose=False):
        biomes = aggregate.setdefault("biomes", {})
        if "biome.superclass" not in aggregate["classes"]:
            return
        superclass = aggregate["classes"]["biome.superclass"]
        cf = ClassFile(StringIO(jar.read(superclass + ".class")))
        method = cf.methods.find_one(returns="V", args="", f=lambda m: m.access_flags.acc_static)
        tmp = None
        stack = None
        for ins in method.code.disassemble():
            if ins.mnemonic == "new":
                if tmp is not None and tmp.has_key("name") and tmp["name"] != " and ":
                    biomes[tmp["name"]] = tmp
                stack = []
                const = cf.constants.get(ins.operands[0].value)
                tmp = {
                    "calls": {},
                    "rainfall": 0.5,
                    "height": [0.1, 0.3],
                    "temperature": 0.5,
                    "class": cf.constants.get(ins.operands[0].value).name.value
                }
            elif tmp is None:
                continue
            elif ins.mnemonic == "invokespecial":
                const = cf.constants.get(ins.operands[0].value)
                name = const.name_and_type.name.value
                if len(stack) == 2 and (type(stack[1]) == types.FloatType or type(stack[0]) == types.FloatType):
                    tmp["calls"][name] = [stack.pop(), stack.pop()]
                elif len(stack) >= 1 and type(stack[0]) == types.IntType: # 1, 2, 3-argument beginning with int = id
                    tmp["id"] = stack[0]
                    stack = []
                elif name != "<init>":
                    tmp["rainfall"] = 0
            elif ins.opcode == "invokevirtual":
                if len(stack) == 1 and "color" not in tmp:
                    tmp["color"] = stack.pop()
                if len(stack) == 2:
                    tmp["rainfall"] = stack.pop()
                    tmp["temperature"] = stack.pop()

            # numeric values & constants
            elif ins.mnemonic in ("ldc", "ldc_w"):
                const = cf.constants.get(ins.operands[0].value)
                if isinstance(const, ConstantString):
                    tmp["name"] = const.string.value
                if isinstance(const, (ConstantInteger, ConstantFloat)):
                    stack.append(const.value)

            elif ins.opcode <= 8 and ins.opcode >= 2:  # iconst
                stack.append(ins.opcode - 3)
            elif ins.opcode >= 0xb and ins.opcode <= 0xd:  # fconst
                stack.append(ins.opcode - 0xb)
            elif ins.mnemonic == "bipush":
                stack.append(ins.operands[0].value)

        if tmp is not None and tmp.has_key("name") and tmp["name"] != " and ":
            biomes[tmp["name"]] = tmp

        weather, height = BiomeTopping.map_methods(biomes)

        for biome in biomes.itervalues():
            calls = biome.pop("calls")
            if height in calls:
                biome["height"] = calls[height]
                biome["height"].reverse()
            if weather in calls:
                biome["temperature"] = calls[weather][1]
                biome["rainfall"] = calls[weather][0]

    @staticmethod
    def map_methods(biomes):
        for biome in biomes.itervalues():
            for call in biome["calls"]:
                if biome["calls"][call][1] > 1 and len(biome["calls"]) > 1:
                    keys = biome["calls"].keys()
                    keys.remove(call)
                    return (call, keys[0])
        return (None, None)
<|endoftext|>


<commit_before>import click
import cooler

from . import cli
from .. import insulation

@cli.command()
@click.argument("in_path", metavar="IN_PATH", type=str, nargs=1)
@click.argument("window", nargs=-1, metavar="WINDOW", type=int)
@click.option(
    "--output",
    "-o",
    help="Specify output file name to store the insulation in a tsv format.",
    type=str,
    required=False,
)
@click.option(
    "--ignore-diags",
    help="The number of diagonals to ignore. By default, equals"
    " the number of diagonals ignored during IC balancing.",
    type=int,
    default=None,
    show_default=True,
)
@click.option(
    "--min-frac-valid-pixels",
    help="The minimal fraction of valid pixels in a sliding diamond. "
    "Used to mask bins during boundary detection.",
    type=float,
    default=0.66,
    show_default=True,
)
@click.option(
    "--min-dist-bad-bin",
    help="The minimal allowed distance to a bad bin. "
    "Used to mask bins during boundary detection.",
    type=int,
    default=0,
    show_default=True,
)
@click.option(
    "--window-pixels",
    help="If set then the window sizes are provided in units of pixels.",
    is_flag=True,
)
@click.option(
    "--append-raw-scores",
    help="Append columns with raw scores (sum_counts, sum_balanced, n_pixels) "
    "to the output table.",
    is_flag=True,
)
@click.option("--chunksize", help="", type=int, default=20000000, show_default=True)
@click.option("--verbose", help="Report real-time progress.", is_flag=True)
def diamond_insulation(
    in_path,
    window,
    output,
    ignore_diags,
    min_frac_valid_pixels,
    min_dist_bad_bin,
    window_pixels,
    append_raw_scores,
    chunksize,
    verbose,
):
    """
    Calculate the diamond insulation scores and call insulating boundaries.

    IN_PATH : The paths to a .cool file with a balanced Hi-C map.

    WINDOW : The window size for the insulation score calculations.
             Multiple space-separated values can be provided.
             By default, the window size must be provided in units of bp.
             When the flag --window-pixels is set, the window sizes must
             be provided in units of pixels instead.
    """

    clr = cooler.Cooler(in_path)
    if window_pixels:
        window = [win * clr.info["bin-size"] for win in window]

    ins_table = insulation.calculate_insulation_score(
        clr,
        window_bp=window,
        ignore_diags=ignore_diags,
        append_raw_scores=append_raw_scores,
        chunksize=chunksize,
        verbose=verbose,
    )

    ins_table = insulation.find_boundaries(
        ins_table,
        min_frac_valid_pixels=min_frac_valid_pixels,
        min_dist_bad_bin=min_dist_bad_bin,
    )

    # output to file if specified:
    if output:
        ins_table.to_csv(output, sep="\t", index=False, na_rep="nan")
    # or print into stdout otherwise:
    else:
        print(ins_table.to_csv(sep="\t", index=False, na_rep="nan"))
<commit_msg>enhancement(diamond_insualtion): --bigwig output.<commit_after>import click
import cooler

from . import cli
from .. import insulation
import bioframe

@cli.command()
@click.argument("in_path", metavar="IN_PATH", type=str, nargs=1)
@click.argument("window", nargs=-1, metavar="WINDOW", type=int)
@click.option(
    "--output",
    "-o",
    help="Specify output file name to store the insulation in a tsv format.",
    type=str,
    required=False,
)
@click.option(
    "--ignore-diags",
    help="The number of diagonals to ignore. By default, equals"
    " the number of diagonals ignored during IC balancing.",
    type=int,
    default=None,
    show_default=True,
)
@click.option(
    "--min-frac-valid-pixels",
    help="The minimal fraction of valid pixels in a sliding diamond. "
    "Used to mask bins during boundary detection.",
    type=float,
    default=0.66,
    show_default=True,
)
@click.option(
    "--min-dist-bad-bin",
    help="The minimal allowed distance to a bad bin. "
    "Used to mask bins during boundary detection.",
    type=int,
    default=0,
    show_default=True,
)
@click.option(
    "--window-pixels",
    help="If set then the window sizes are provided in units of pixels.",
    is_flag=True,
)
@click.option(
    "--append-raw-scores",
    help="Append columns with raw scores (sum_counts, sum_balanced, n_pixels) "
    "to the output table.",
    is_flag=True,
)
@click.option("--chunksize", help="", type=int, default=20000000, show_default=True)
@click.option("--verbose", help="Report real-time progress.", is_flag=True)
@click.option(
    "--bigwig",
    help="Also save insulation tracks as a bigWig files for different window sizes"
    " with the names output.<window-size>.bw",
    is_flag=True,
    default=False,
)
def diamond_insulation(
    in_path,
    window,
    output,
    ignore_diags,
    min_frac_valid_pixels,
    min_dist_bad_bin,
    window_pixels,
    append_raw_scores,
    chunksize,
    verbose,
    bigwig
):
    """
    Calculate the diamond insulation scores and call insulating boundaries.

    IN_PATH : The paths to a .cool file with a balanced Hi-C map.

    WINDOW : The window size for the insulation score calculations.
             Multiple space-separated values can be provided.
             By default, the window size must be provided in units of bp.
             When the flag --window-pixels is set, the window sizes must
             be provided in units of pixels instead.
    """

    clr = cooler.Cooler(in_path)
    if window_pixels:
        window = [win * clr.info["bin-size"] for win in window]

    ins_table = insulation.calculate_insulation_score(
        clr,
        window_bp=window,
        ignore_diags=ignore_diags,
        append_raw_scores=append_raw_scores,
        chunksize=chunksize,
        verbose=verbose,
    )

    ins_table = insulation.find_boundaries(
        ins_table,
        min_frac_valid_pixels=min_frac_valid_pixels,
        min_dist_bad_bin=min_dist_bad_bin,
    )

    # output to file if specified:
    if output:
        ins_table.to_csv(output, sep="\t", index=False, na_rep="nan")
    # or print into stdout otherwise:
    else:
        print(ins_table.to_csv(sep="\t", index=False, na_rep="nan"))

    # Write the insulation track as a bigwig:

    if bigwig:
        for w in window:
            bioframe.to_bigwig(
                ins_table,
                clr.chromsizes,
                output + "." + str(w) + ".bw",
                value_field=f"log2_insulation_score_{w}",
            )<|endoftext|>