<commit_before># -*- coding: utf-8 -*-
import urllib.request
import urllib.parse
import json

class weather:

    def __init__(self, core, client):
        try:
            self.apikey = core.readConf("config.wundergroundapikey")
            if self.apikey == "":
                return None
        except:
            return None

        core.addCommandHandler("tiempo", self, chelp="Muestra el pronóstico" +
        " del tiempo para una ciudad. Sintaxis: tiempo <ciudad>",
        alias=['weather', 'comoestaen'])

    def tiempo(self, bot, cli, event):
        if len(event.splitd) > 0:
            ts = urllib.parse.quote_plus(" ".join(event.splitd)).replace("+",
                "%20")
        else:
            cli.msg(event.target, "\00304Error\003: Faltan parametros")
        if ts == "Polsaker":
            cli.msg(event.target, "Húmedo")
            return

        r = urllib.request.urlopen("http://api.wunderground.com/api/%s/conditi"
        "ons/forecast/lang:es/q/%s.json" %
         (self.apikey, ts)).read()
        w = json.loads(r.decode('utf-8'))
        try:
            w['current_observation']
        except KeyError:
            resp = "\00304Error\003: No se ha encontrado la ciudad. "
            try:
                w['response']['results']
                resp = resp + "Quizás quiso decir: "
                j = 0
                for val in w['response']['results']:
                    j = j + 1
                    resp = resp + "\2\"%s, %s\"\2 (zmw:%s), " % \
                    (val['city'], val["country_name"], val['zmw'])
                    if j >= 2:
                        break
            except:
                pass
            cli.msg(event.target, resp)
            return 1
        resp = "El tiempo en \00310\2%s\003\2: Viento a \2%s\2 km/h (\2%s" \
        "\2), " % (w['current_observation']['display_location']['full'],
        w['current_observation']['wind_kph'],
        w['current_observation']['wind_dir'])
        resp = resp + "presión \2%s\2 hPa. Temperatura: \2%s\2ºC, Sensació"\
        "n térmica: \2%s\2ºC [\2%s\2]" % (w['current_observation']['pressu'
        're_mb'], w['current_observation']['temp_c'], w['current_observati'
        'on']['feelslike_c'], self._conv(w['current_observation']['icon']))
        fc = w['forecast']['simpleforecast']['forecastday']
        resp = resp + "Pronóstico: \00303Hoy\003 [\2%s\2], máxima de " \
        "\2%s\2ºC, mínima de \2%s\2ºC, " % (self._conv(fc[0]['icon']),
        fc[0]['high']['celsius'], fc[0]['low']['celsius'])

        resp = resp + "Pronóstico: \00303%s\003 [\2%s\2], máxima de " \
        "\2%s\2ºC, mínima de \2%s\2ºC, " % (self._convday(fc[1]['date']
        ['weekday']), self._conv(fc[1]['icon']), fc[1]['high']['celsius'],
        fc[1]['low']['celsius'])

        resp = resp + "Pronóstico: \00303%s\003 [\2%s\2], máxima de " \
        "\2%s\2ºC, mínima de \2%s\2ºC" % (self._convday(fc[2]['date']
        ['weekday']), self._conv(fc[2]['icon']), fc[2]['high']['celsius'],
        fc[2]['low']['celsius'])

        cli.msg(event.target, resp)

    def _convday(self, c):
        if c == "Monday":
            return "Lunes"
        elif c == "Sunday":
            return "Domingo"
        elif c == "Tuesday":
            return "Martes"
        elif c == "Wednesday":
            return "Miercoles"
        elif c == "Thursday":
            return "Jueves"
        elif c == "Friday":
            return "Viernes"
        elif c == "Saturday":
            return "Sábado"

    def _conv(self, c):
        if c == "clear":
            return "Despejado"
        elif c == "mostlysunny":
            return "Parcialmente despejado"
        elif c == "sunny":
            return "Soleado"
        elif c == "partlycloudy" or c == "mostlycloudy":
            return "Parcialmente nublado"
        elif c == "mist" or c == "foggy" or c == "fog":
            return "Niebla"
        elif c == "chancerain":
            return "Posibles precipitaciones"
        elif c == "rain":
            return "Lluvia"
        elif c == "chancestorms":
            return "Posibles tormentas"
        elif c == "storm":
            return "Tormentas"
        elif c == "snow":
            return "Nevadas"
        elif c == "cloudy":
            return "Nublado"
        elif c == "showers":
            return "Chubascos"
        elif c == "thunderstorm" or c == "tstorms":
            return "Tormentas eléctricas"
        elif c == "icy":
            return "Helado"
        elif c == "chancetstorms":
            return "Posibles tormentas eléctricas"
        elif c == "sleet":
            return "Aguanieve"
<commit_msg>10 results is better.<commit_after># -*- coding: utf-8 -*-
import urllib.request
import urllib.parse
import json

class weather:

    def __init__(self, core, client):
        try:
            self.apikey = core.readConf("config.wundergroundapikey")
            if self.apikey == "":
                return None
        except:
            return None

        core.addCommandHandler("tiempo", self, chelp="Muestra el pronóstico" +
        " del tiempo para una ciudad. Sintaxis: tiempo <ciudad>",
        alias=['weather', 'comoestaen'])

    def tiempo(self, bot, cli, event):
        if len(event.splitd) > 0:
            ts = urllib.parse.quote_plus(" ".join(event.splitd)).replace("+",
                "%20")
        else:
            cli.msg(event.target, "\00304Error\003: Faltan parametros")
        if ts == "Polsaker":
            cli.msg(event.target, "Húmedo")
            return

        r = urllib.request.urlopen("http://api.wunderground.com/api/%s/conditi"
        "ons/forecast/lang:es/q/%s.json" %
         (self.apikey, ts)).read()
        w = json.loads(r.decode('utf-8'))
        try:
            w['current_observation']
        except KeyError:
            resp = "\00304Error\003: No se ha encontrado la ciudad. "
            try:
                w['response']['results']
                resp = resp + "Quizás quiso decir: "
                j = 0
                for val in w['response']['results']:
                    j = j + 1
                    resp = resp + "\2\"%s, %s\"\2 (zmw:%s), " % \
                    (val['city'], val["country_name"], val['zmw'])
                    if j >= 10:
                        break
            except:
                pass
            cli.msg(event.target, resp)
            return 1
        resp = "El tiempo en \00310\2%s\003\2: Viento a \2%s\2 km/h (\2%s" \
        "\2), " % (w['current_observation']['display_location']['full'],
        w['current_observation']['wind_kph'],
        w['current_observation']['wind_dir'])
        resp = resp + "presión \2%s\2 hPa. Temperatura: \2%s\2ºC, Sensació"\
        "n térmica: \2%s\2ºC [\2%s\2]" % (w['current_observation']['pressu'
        're_mb'], w['current_observation']['temp_c'], w['current_observati'
        'on']['feelslike_c'], self._conv(w['current_observation']['icon']))
        fc = w['forecast']['simpleforecast']['forecastday']
        resp = resp + "Pronóstico: \00303Hoy\003 [\2%s\2], máxima de " \
        "\2%s\2ºC, mínima de \2%s\2ºC, " % (self._conv(fc[0]['icon']),
        fc[0]['high']['celsius'], fc[0]['low']['celsius'])

        resp = resp + "Pronóstico: \00303%s\003 [\2%s\2], máxima de " \
        "\2%s\2ºC, mínima de \2%s\2ºC, " % (self._convday(fc[1]['date']
        ['weekday']), self._conv(fc[1]['icon']), fc[1]['high']['celsius'],
        fc[1]['low']['celsius'])

        resp = resp + "Pronóstico: \00303%s\003 [\2%s\2], máxima de " \
        "\2%s\2ºC, mínima de \2%s\2ºC" % (self._convday(fc[2]['date']
        ['weekday']), self._conv(fc[2]['icon']), fc[2]['high']['celsius'],
        fc[2]['low']['celsius'])

        cli.msg(event.target, resp)

    def _convday(self, c):
        if c == "Monday":
            return "Lunes"
        elif c == "Sunday":
            return "Domingo"
        elif c == "Tuesday":
            return "Martes"
        elif c == "Wednesday":
            return "Miercoles"
        elif c == "Thursday":
            return "Jueves"
        elif c == "Friday":
            return "Viernes"
        elif c == "Saturday":
            return "Sábado"

    def _conv(self, c):
        if c == "clear":
            return "Despejado"
        elif c == "mostlysunny":
            return "Parcialmente despejado"
        elif c == "sunny":
            return "Soleado"
        elif c == "partlycloudy" or c == "mostlycloudy":
            return "Parcialmente nublado"
        elif c == "mist" or c == "foggy" or c == "fog":
            return "Niebla"
        elif c == "chancerain":
            return "Posibles precipitaciones"
        elif c == "rain":
            return "Lluvia"
        elif c == "chancestorms":
            return "Posibles tormentas"
        elif c == "storm":
            return "Tormentas"
        elif c == "snow":
            return "Nevadas"
        elif c == "cloudy":
            return "Nublado"
        elif c == "showers":
            return "Chubascos"
        elif c == "thunderstorm" or c == "tstorms":
            return "Tormentas eléctricas"
        elif c == "icy":
            return "Helado"
        elif c == "chancetstorms":
            return "Posibles tormentas eléctricas"
        elif c == "sleet":
            return "Aguanieve"
<|endoftext|>


<commit_before>"""
Various importers for the different content types
"""
import json
import requests
from io import BytesIO

from django.core.files.images import ImageFile

from wagtail.wagtailcore.models import Page
from wagtail.wagtailimages.models import Image

from molo.core.models import (
    Languages,
    SiteLanguageRelation,
    ArticlePage,
    SectionPage,
)
from molo.core.api.constants import (
    API_IMAGES_ENDPOINT, API_PAGES_ENDPOINT, KEYS_TO_EXCLUDE,
)

# functions used to find images
def get_image_attributes(base_url, image_id):
    image_attrs = requests.get(
        base_url + API_IMAGES_ENDPOINT + str(image_id)
    ).json()
    return image_attrs

def get_image(base_url, image_id):
    # TODO: guard against non-existent images
    image_attributes = get_image_attributes(base_url, image_id)
    image_file = requests.get(image_attributes["file"])
    image = Image(
        title=image_attributes["title"],
        file=ImageFile(
            BytesIO(image_file.content), name=image_attributes["title"]
        )
    )
    image.save()
    return image

def separate_fields(fields):
    """
    Non-foreign key fields can be mapped to new article instances
    directly. Foreign key fields require a bit more work.
    This method returns a tuple, of the same format:
    (flat fields, nested fields)
    """
    flat_fields = {}
    nested_fields = {}

    # exclude "id" and "meta" elements
    for k, v in fields.items():
        if k not in KEYS_TO_EXCLUDE:
            if type(v) not in [type({}), type([])]:
                flat_fields.update({k: v})
            else:
                nested_fields.update({k: v})

    return flat_fields, nested_fields

class PageImporter(object):

    def __init__(self, base_url=None, content=None, content_type=None):
        self._content_type = content_type
        self._fields = []
        self._content = content
        self._base_url = base_url

    def get_content_from_url(self, base_url):
        """
        Sections can have SectionPage and ArticlePage child objects.
        These have different fields, and thus have to be treated
        differently.
        """
        # assemble url
        base_url = base_url.rstrip("/")
        url = base_url + API_PAGES_ENDPOINT + "?type=" + self._content_type + \
            "&fields=" + ",".join(self._fields) + \
            "&order=latest_revision_created_at"

        # make request
        try:
            response = requests.get(url)
            self._base_url = base_url
            self._content = response.json()
            self._content = self._content["items"]
            return self._content
        except requests.exceptions.ConnectionError:
            return "No content could be found from {}. " \
                "Are you sure this is the correct URL?".format(base_url)
        except requests.exceptions.RequestException:
            return "Content could not be imported at this time. " \
                   "Please try again later."

    def content(self):
        if self._content:
            return self._content
        return []

    def save(self, indexes, parent_id):
        pass

class ArticlePageImporter(PageImporter):

    def __init__(self, base_url=None, content=None, content_type=None):
        super(ArticlePageImporter, self).__init__(
            base_url=base_url, content=content, content_type=content_type
        )
        self._content_type = "core.ArticlePage"
        self._fields = ArticlePage.get_api_fields()

    def save(self, indexes, parent_id):
        if self.content():
            parent = Page.objects.get(id=parent_id)
            for index in indexes:
                # Remove "id" and "meta" fields
                selected_article = self.content()[index]
                fields, nested_fields = separate_fields(selected_article)
                article = ArticlePage(**fields)

                # TODO: u'related_sections'
                # process the nested fields
                if ("tags" in nested_fields) and nested_fields["tags"]:
                    article.tags.add(", ".join(nested_fields["tags"]))

                if ("metadata_tags" in nested_fields) and \
                        nested_fields["metadata_tags"]:
                    article.metadata_tags.add(
                        ", ".join(nested_fields["metadata_tags"])
                    )

                if ("body" in nested_fields) and nested_fields["body"]:
                    article.body = json.dumps(nested_fields["body"])

                if ("image" in nested_fields) and nested_fields["image"]:
                    article.image = get_image(
                        self._base_url, nested_fields["image"]["id"]
                    )

                parent.add_child(instance=article)
                parent.save_revision().publish()

class SectionPageImporter(PageImporter):

    def __init__(self, base_url=None, content=None, content_type=None):
        super(SectionPageImporter, self).__init__(
            base_url=base_url, content=content, content_type=content_type
        )
        self._content_type = "core.SectionPage"
        self._fields = SectionPage.get_api_fields()

    def _save_item(self, item, parent):
        if item["meta"]["type"] == "core.SectionPage":
            flat_fields, nested_fields = separate_fields(item)
            child_section = SectionPage(**flat_fields)

            if ("image" in nested_fields) and nested_fields["image"]:
                child_section.image = get_image(
                    self._base_url, nested_fields["image"]["id"]
                )

            parent.add_child(instance=child_section)
            parent.save_revision().publish()
            return child_section

        elif item["meta"]["type"] == "core.ArticlePage":
            flat_fields, nested_fields = separate_fields(item)
            child_article = ArticlePage(**flat_fields)
            if ("tags" in nested_fields) and nested_fields["tags"]:
                child_article.tags.add(
                    ", ".join(nested_fields["tags"])
                )

            if ("metadata_tags" in nested_fields) and \
                    nested_fields["metadata_tags"]:
                child_article.metadata_tags.add(
                    ", ".join(nested_fields["metadata_tags"])
                )

            if ("body" in nested_fields) and nested_fields["body"]:
                child_article.body = json.dumps(nested_fields["body"])

            if ("image" in nested_fields) and nested_fields["image"]:
                child_article.image = get_image(
                    self._base_url, nested_fields["image"]["id"]
                )

            parent.add_child(instance=child_article)
            parent.save_revision().publish()
            return child_article

    def process_child_section(self, id, parent):
        response = requests.get(
            self._base_url + API_PAGES_ENDPOINT + str(id)
        ).json()
        ancestor = self._save_item(response, parent)
        if response["meta"]["children"]:
            for child_id in response["meta"]["children"]["items"]:
                self.process_child_section(child_id, ancestor)
        return ancestor

    def recursive_children(self, node):
        results = [node['id']]
        if len(node['children']) > 0:
            for child in node['children']:
                results.extend(self.recursive_children(child))
        return results

    def save(self, indexes, parent_id):
        """
        Save the selected section. This will save the selected section
        as well as its direct child pages obtained through the ?child_of
        query parameter. The ?descendant_of query parameter is probably
         better suited because it all pages under that part of the tree will
         be obtained. The problem , however, is that that will require being
         able to traverse the tree and recreate parent-child relationships
         after they are imported
        """
        if self.content():
            parent = Page.objects.get(id=parent_id)

            # Save the selected section page
            response = requests.get(
                self._base_url + API_PAGES_ENDPOINT + str(indexes[0]) + "/"
            )

            section_page = response.json()
            self.process_child_section(section_page["id"], parent)

class SiteImporter(object):

    def __init__(self, site_pk, base_url=''):
        self.base_url = base_url
        self.api_url = self.base_url + '/api/v2/'
        self.site_pk = site_pk
        self.content = None
        # maps foreign IDs to local page IDs
        self.id_map = {}
        # maps foreign image IDs to local IDs
        self.image_map = {}
        # maps local id to list of foreign section page ids
        self.related_sections = {}
        # maps local id to list of foreign page ids
        self.recommended_articles = {}
        # maps local pages to list of foreign reaction question IDs
        self.reaction_questions = {}
        # maps local pages to list of foreign nav_tag IDs
        self.nav_tags = {}
        # maps local pages to list of foreign section_tag IDs
        self.section_tags = {}

    def get_language_ids(self):
        language_url = "{}{}/".format(self.api_url, "languages")
        response = requests.get(language_url)

        language_ids = []
        # TODO: handle broken case
        for language in json.loads(response.content)["items"]:
            language_ids.append(language["id"])
        return language_ids

    def copy_site_languages(self):
        language_foreign_ids = self.get_language_ids()
        language_setting, created = Languages.objects.get_or_create(
            site_id=self.site_pk)
        for foreign_id in language_foreign_ids:
            language_url = "{}{}/{}/".format(self.api_url,
                                             "languages",
                                             foreign_id)
            response = requests.get(language_url)
            content = json.loads(response.content)

            # TODO: review whether this works with Multi-site
            # TODO: handle case where Main lang is not first
            sle, created = SiteLanguageRelation.objects.get_or_create(
                locale=content['locale'],
                is_active=content['is_active'],
                language_setting=language_setting)

    def fetch_and_create_image(self):
        # create image object
        # update self.image_map
        # return image
        pass

    def attach_image(self):
        # if not (image has already been imported)
        #   get_image()
        # attach image
        pass

    def create_page(self, parent, content):
        fields, nested_fields = separate_fields(content)

        # handle the unwanted fields
        foreign_id = content.pop('id')

        page = None
        if content["meta"]["type"] == "core.SectionPage":
            page = SectionPage(**fields)
        elif content["meta"]["type"] == "core.ArticlePage":
            page = ArticlePage(**fields)
        # TODO: handle other Page types

        parent.add_child(instance=page)

        # TODO: handle live/published
        # Need to review this line:
        #   handle drafts and 'not live' content
        parent.save_revision().publish()

        self.id_map[foreign_id] = page.id

        # time
        if (("time" in nested_fields) and
                nested_fields["time"]):
            page.time = json.dumps(nested_fields["time"])

        # section_tags/nav_tags
        #  list -> ["tag"]["id"]
        # -> Need to fetch and create the nav tags
        #  THEN create the link between page and nav_tag
        if (("section_tags" in nested_fields) and
                nested_fields["section_tags"]):
            self.section_tags[page.id] = []
            for section_tag in nested_fields["section_tags"]:
                self.section_tags[page.id].append(
                    section_tag["tag"]["id"])

        # nav_tags
        #  list -> ["tag"]["id"]
        # -> Need to fetch and create the nav tags
        #  THEN create the link between page and nav_tag
        if (("nav_tags" in nested_fields) and
                nested_fields["nav_tags"]):
            self.nav_tags[page.id] = []
            for nav_tag in nested_fields["nav_tags"]:
                self.nav_tags[page.id].append(
                    nav_tag["tag"]["id"])

        # reaction_questions
        #  list -> ["reaction_question"]["id"]
        # -> Need to fetch and create the reaction questions
        #  THEN create the link between page and reaction question
        if (("reaction_questions" in nested_fields) and
                nested_fields["reaction_questions"]):
            self.reaction_questions[page.id] = []
            for reaction_question in nested_fields["reaction_questions"]:
                self.reaction_questions[page.id].append(
                    reaction_question["reaction_question"]["id"])

        # related_sections
        #  list -> ["section"]["id"]
        # -> Only need to create the relationship
        if (("related_sections" in nested_fields) and
                nested_fields["related_sections"]):
            self.related_sections[page.id] = []
            for related_section in nested_fields["related_sections"]:
                self.related_sections[page.id].append(
                    related_section["section"]["id"])

        if ("body" in nested_fields) and nested_fields["body"]:
            page.body = json.dumps(nested_fields["body"])

        if ("tags" in nested_fields) and nested_fields["tags"]:
            for tag in nested_fields["tags"]:
                page.tags.add(tag)

        if (("metadata_tags" in nested_fields) and
                nested_fields["metadata_tags"]):
            for tag in nested_fields["metadata_tags"]:
                page.metadata_tags.add(tag)

        if (("social_media_image" in nested_fields) and
                nested_fields["social_media_image"]):
            self.attach_image()

        if ("image" in nested_fields) and nested_fields["image"]:
            self.attach_image()

        # update the state of the page ?
        page.save()
        return page
<commit_msg>Refactor recording recommended article relations with higher order func<commit_after>"""
Various importers for the different content types
"""
import json
import requests
from io import BytesIO

from django.core.files.images import ImageFile

from wagtail.wagtailcore.models import Page
from wagtail.wagtailimages.models import Image

from molo.core.models import (
    Languages,
    SiteLanguageRelation,
    ArticlePage,
    SectionPage,
)
from molo.core.api.constants import (
    API_IMAGES_ENDPOINT, API_PAGES_ENDPOINT, KEYS_TO_EXCLUDE,
)

# functions used to find images
def get_image_attributes(base_url, image_id):
    image_attrs = requests.get(
        base_url + API_IMAGES_ENDPOINT + str(image_id)
    ).json()
    return image_attrs

def get_image(base_url, image_id):
    # TODO: guard against non-existent images
    image_attributes = get_image_attributes(base_url, image_id)
    image_file = requests.get(image_attributes["file"])
    image = Image(
        title=image_attributes["title"],
        file=ImageFile(
            BytesIO(image_file.content), name=image_attributes["title"]
        )
    )
    image.save()
    return image

def separate_fields(fields):
    """
    Non-foreign key fields can be mapped to new article instances
    directly. Foreign key fields require a bit more work.
    This method returns a tuple, of the same format:
    (flat fields, nested fields)
    """
    flat_fields = {}
    nested_fields = {}

    # exclude "id" and "meta" elements
    for k, v in fields.items():
        if k not in KEYS_TO_EXCLUDE:
            if type(v) not in [type({}), type([])]:
                flat_fields.update({k: v})
            else:
                nested_fields.update({k: v})

    return flat_fields, nested_fields

def record_foreign_relation(field, key, record_keeper, id_key="id"):
    '''
    returns a function with the attributes necessary to
    correctly reference the necessary objects, to correctly record
    the relationship between a page and its foreign foreign-key pages
    '''
    def record_relationship(nested_fields, page_id):
        if ((field in nested_fields) and nested_fields[field]):
            record_keeper[page_id] = []
            for thing in nested_fields[field]:
                record_keeper[page_id].append(thing[key][id_key])
    return record_relationship

class PageImporter(object):

    def __init__(self, base_url=None, content=None, content_type=None):
        self._content_type = content_type
        self._fields = []
        self._content = content
        self._base_url = base_url

    def get_content_from_url(self, base_url):
        """
        Sections can have SectionPage and ArticlePage child objects.
        These have different fields, and thus have to be treated
        differently.
        """
        # assemble url
        base_url = base_url.rstrip("/")
        url = base_url + API_PAGES_ENDPOINT + "?type=" + self._content_type + \
            "&fields=" + ",".join(self._fields) + \
            "&order=latest_revision_created_at"

        # make request
        try:
            response = requests.get(url)
            self._base_url = base_url
            self._content = response.json()
            self._content = self._content["items"]
            return self._content
        except requests.exceptions.ConnectionError:
            return "No content could be found from {}. " \
                "Are you sure this is the correct URL?".format(base_url)
        except requests.exceptions.RequestException:
            return "Content could not be imported at this time. " \
                   "Please try again later."

    def content(self):
        if self._content:
            return self._content
        return []

    def save(self, indexes, parent_id):
        pass

class ArticlePageImporter(PageImporter):

    def __init__(self, base_url=None, content=None, content_type=None):
        super(ArticlePageImporter, self).__init__(
            base_url=base_url, content=content, content_type=content_type
        )
        self._content_type = "core.ArticlePage"
        self._fields = ArticlePage.get_api_fields()

    def save(self, indexes, parent_id):
        if self.content():
            parent = Page.objects.get(id=parent_id)
            for index in indexes:
                # Remove "id" and "meta" fields
                selected_article = self.content()[index]
                fields, nested_fields = separate_fields(selected_article)
                article = ArticlePage(**fields)

                # TODO: u'related_sections'
                # process the nested fields
                if ("tags" in nested_fields) and nested_fields["tags"]:
                    article.tags.add(", ".join(nested_fields["tags"]))

                if ("metadata_tags" in nested_fields) and \
                        nested_fields["metadata_tags"]:
                    article.metadata_tags.add(
                        ", ".join(nested_fields["metadata_tags"])
                    )

                if ("body" in nested_fields) and nested_fields["body"]:
                    article.body = json.dumps(nested_fields["body"])

                if ("image" in nested_fields) and nested_fields["image"]:
                    article.image = get_image(
                        self._base_url, nested_fields["image"]["id"]
                    )

                parent.add_child(instance=article)
                parent.save_revision().publish()

class SectionPageImporter(PageImporter):

    def __init__(self, base_url=None, content=None, content_type=None):
        super(SectionPageImporter, self).__init__(
            base_url=base_url, content=content, content_type=content_type
        )
        self._content_type = "core.SectionPage"
        self._fields = SectionPage.get_api_fields()

    def _save_item(self, item, parent):
        if item["meta"]["type"] == "core.SectionPage":
            flat_fields, nested_fields = separate_fields(item)
            child_section = SectionPage(**flat_fields)

            if ("image" in nested_fields) and nested_fields["image"]:
                child_section.image = get_image(
                    self._base_url, nested_fields["image"]["id"]
                )

            parent.add_child(instance=child_section)
            parent.save_revision().publish()
            return child_section

        elif item["meta"]["type"] == "core.ArticlePage":
            flat_fields, nested_fields = separate_fields(item)
            child_article = ArticlePage(**flat_fields)
            if ("tags" in nested_fields) and nested_fields["tags"]:
                child_article.tags.add(
                    ", ".join(nested_fields["tags"])
                )

            if ("metadata_tags" in nested_fields) and \
                    nested_fields["metadata_tags"]:
                child_article.metadata_tags.add(
                    ", ".join(nested_fields["metadata_tags"])
                )

            if ("body" in nested_fields) and nested_fields["body"]:
                child_article.body = json.dumps(nested_fields["body"])

            if ("image" in nested_fields) and nested_fields["image"]:
                child_article.image = get_image(
                    self._base_url, nested_fields["image"]["id"]
                )

            parent.add_child(instance=child_article)
            parent.save_revision().publish()
            return child_article

    def process_child_section(self, id, parent):
        response = requests.get(
            self._base_url + API_PAGES_ENDPOINT + str(id)
        ).json()
        ancestor = self._save_item(response, parent)
        if response["meta"]["children"]:
            for child_id in response["meta"]["children"]["items"]:
                self.process_child_section(child_id, ancestor)
        return ancestor

    def recursive_children(self, node):
        results = [node['id']]
        if len(node['children']) > 0:
            for child in node['children']:
                results.extend(self.recursive_children(child))
        return results

    def save(self, indexes, parent_id):
        """
        Save the selected section. This will save the selected section
        as well as its direct child pages obtained through the ?child_of
        query parameter. The ?descendant_of query parameter is probably
         better suited because it all pages under that part of the tree will
         be obtained. The problem , however, is that that will require being
         able to traverse the tree and recreate parent-child relationships
         after they are imported
        """
        if self.content():
            parent = Page.objects.get(id=parent_id)

            # Save the selected section page
            response = requests.get(
                self._base_url + API_PAGES_ENDPOINT + str(indexes[0]) + "/"
            )

            section_page = response.json()
            self.process_child_section(section_page["id"], parent)

class SiteImporter(object):

    def __init__(self, site_pk, base_url=''):
        self.base_url = base_url
        self.api_url = self.base_url + '/api/v2/'
        self.site_pk = site_pk
        self.content = None
        # maps foreign IDs to local page IDs
        self.id_map = {}
        # maps foreign image IDs to local IDs
        self.image_map = {}
        # maps local id to list of foreign section page ids
        self.related_sections = {}
        # maps local id to list of foreign page ids
        self.recommended_articles = {}
        # maps local pages to list of foreign reaction question IDs
        self.reaction_questions = {}
        # maps local pages to list of foreign nav_tag IDs
        self.nav_tags = {}
        # maps local pages to list of foreign section_tag IDs
        self.section_tags = {}
        self.record_recommended_articles = record_foreign_relation(
            "recommended_articles", "recommended_article",
            self.recommended_articles)

    def get_language_ids(self):
        language_url = "{}{}/".format(self.api_url, "languages")
        response = requests.get(language_url)

        language_ids = []
        # TODO: handle broken case
        for language in json.loads(response.content)["items"]:
            language_ids.append(language["id"])
        return language_ids

    def copy_site_languages(self):
        language_foreign_ids = self.get_language_ids()
        language_setting, created = Languages.objects.get_or_create(
            site_id=self.site_pk)
        for foreign_id in language_foreign_ids:
            language_url = "{}{}/{}/".format(self.api_url,
                                             "languages",
                                             foreign_id)
            response = requests.get(language_url)
            content = json.loads(response.content)

            # TODO: review whether this works with Multi-site
            # TODO: handle case where Main lang is not first
            sle, created = SiteLanguageRelation.objects.get_or_create(
                locale=content['locale'],
                is_active=content['is_active'],
                language_setting=language_setting)

    def fetch_and_create_image(self):
        # create image object
        # update self.image_map
        # return image
        pass

    def attach_image(self):
        # if not (image has already been imported)
        #   get_image()
        # attach image
        pass

    def create_page(self, parent, content):
        fields, nested_fields = separate_fields(content)

        # handle the unwanted fields
        foreign_id = content.pop('id')

        page = None
        if content["meta"]["type"] == "core.SectionPage":
            page = SectionPage(**fields)
        elif content["meta"]["type"] == "core.ArticlePage":
            page = ArticlePage(**fields)
        # TODO: handle other Page types

        parent.add_child(instance=page)

        # TODO: handle live/published
        # Need to review this line:
        #   handle drafts and 'not live' content
        parent.save_revision().publish()

        self.id_map[foreign_id] = page.id

        # time
        if (("time" in nested_fields) and
                nested_fields["time"]):
            page.time = json.dumps(nested_fields["time"])

        # section_tags/nav_tags
        #  list -> ["tag"]["id"]
        # -> Need to fetch and create the nav tags
        #  THEN create the link between page and nav_tag
        if (("section_tags" in nested_fields) and
                nested_fields["section_tags"]):
            self.section_tags[page.id] = []
            for section_tag in nested_fields["section_tags"]:
                self.section_tags[page.id].append(
                    section_tag["tag"]["id"])

        # nav_tags
        #  list -> ["tag"]["id"]
        # -> Need to fetch and create the nav tags
        #  THEN create the link between page and nav_tag
        if (("nav_tags" in nested_fields) and
                nested_fields["nav_tags"]):
            self.nav_tags[page.id] = []
            for nav_tag in nested_fields["nav_tags"]:
                self.nav_tags[page.id].append(
                    nav_tag["tag"]["id"])

        # reaction_questions
        #  list -> ["reaction_question"]["id"]
        # -> Need to fetch and create the reaction questions
        #  THEN create the link between page and reaction question
        if (("reaction_questions" in nested_fields) and
                nested_fields["reaction_questions"]):
            self.reaction_questions[page.id] = []
            for reaction_question in nested_fields["reaction_questions"]:
                self.reaction_questions[page.id].append(
                    reaction_question["reaction_question"]["id"])

        self.record_recommended_articles(nested_fields, page.id)

        # related_sections
        #  list -> ["section"]["id"]
        # -> Only need to create the relationship
        if (("related_sections" in nested_fields) and
                nested_fields["related_sections"]):
            self.related_sections[page.id] = []
            for related_section in nested_fields["related_sections"]:
                self.related_sections[page.id].append(
                    related_section["section"]["id"])

        if ("body" in nested_fields) and nested_fields["body"]:
            page.body = json.dumps(nested_fields["body"])

        if ("tags" in nested_fields) and nested_fields["tags"]:
            for tag in nested_fields["tags"]:
                page.tags.add(tag)

        if (("metadata_tags" in nested_fields) and
                nested_fields["metadata_tags"]):
            for tag in nested_fields["metadata_tags"]:
                page.metadata_tags.add(tag)

        if (("social_media_image" in nested_fields) and
                nested_fields["social_media_image"]):
            self.attach_image()

        if ("image" in nested_fields) and nested_fields["image"]:
            self.attach_image()

        # update the state of the page ?
        page.save()
        return page
<|endoftext|>


<commit_before>import argparse
import boto3
import subprocess
import getpass
import time
import os
import zipfile

def _get_client(aws_region):
    return boto3.client('emr', region_name=aws_region)

def add_step_to_job_flow(job_flow_id=None,
                         python_path=None,
                         spark_main=None,
                         py_files=None,
                         num_of_steps=1,
                         use_mysql=False,
                         spark_main_args=None,
                         s3_work_bucket=None,
                         aws_region=None,
                         send_success_email_to=None):
    assert(job_flow_id)
    assert(aws_region)

    job_flow_name = _create_job_flow_name(spark_main)
    steps = _create_steps(job_flow_name=job_flow_name,
                          python_path=python_path,
                          spark_main=spark_main,
                          py_files=py_files,
                          num_of_steps=num_of_steps,
                          use_mysql=use_mysql,
                          spark_main_args=spark_main_args,
                          s3_work_bucket=s3_work_bucket,
                          send_success_email_to=send_success_email_to)
    client = _get_client(aws_region)
    step_response = client.add_job_flow_steps(
      JobFlowId=job_flow_id,
      Steps=steps
    )
    step_ids = step_response['StepIds']
    print "Created steps: {}".format(step_ids)
    _wait_for_job_flow(aws_region, job_flow_id, step_ids)

def _create_job_flow_name(spark_main):
    return '{}.{}.{}'.format(getpass.getuser(),
                             spark_main,
                             time.strftime("%H%M%S", time.gmtime()))

def _ls_recursive(dir, suffix=None):
    files = [os.path.join(dp, f) for dp, dn, fn in os.walk(os.path.expanduser(dir)) for f in fn]
    if suffix:
        files = filter(lambda f: f.endswith(suffix), files)
    return files

def _create_steps(job_flow_name=None,
                  python_path=None,
                  spark_main=None,
                  py_files=[],
                  num_of_steps=1,
                  spark_main_args=None,
                  s3_work_bucket=None,
                  use_mysql=False,
                  send_success_email_to=None):
    assert(python_path)
    assert(spark_main)
    assert(s3_work_bucket)

    zip_file = 'spark_zip.zip'
    sources_rel_path = job_flow_name
    sources_on_host = '/home/hadoop/{}'.format(sources_rel_path)
    local_zip_file = '/tmp/{}'.format(zip_file)
    python_path_files = _ls_recursive(python_path, '.py')
    with zipfile.ZipFile(local_zip_file, 'w') as myzip:
        for f in python_path_files:
            myzip.write(f)
        if py_files:
          for py_file in py_files:
              if py_file.endswith('.zip'):  # Currently only support sip files
                  with zipfile.ZipFile(py_file, 'r') as openzip:
                      [myzip.writestr(t[0], t[1].read())
                       for t in ((n, openzip.open(n))
                       for n in openzip.namelist())]
    s3sources = 's3://{}/sources/{}'.format(s3_work_bucket, sources_rel_path)
    zip_file_on_s3 = '{}/{}'.format(s3sources, zip_file)
    print 'Storing python sources on {}'.format(s3sources)
    # TODO: Change these subprocess calls to use python native API instead of shell
    subprocess.check_call('aws s3 cp {} {}'.format(local_zip_file, zip_file_on_s3), shell=True)
    zip_file_on_host = '{}/{}'.format(sources_on_host, zip_file)
    spark_main_on_host = '{}/{}'.format(sources_on_host, spark_main)
    # spark_main_args = spark_main_args.split() if spark_main_args else ['']
    packages_to_add = []
    if use_mysql:
        packages_to_add.append('mysql:mysql-connector-java:5.1.39')
    packages = ['--packages'] + packages_to_add if packages_to_add else []

    steps = []
    steps.append({
      'Name': 'setup - copy files',
      'ActionOnFailure': 'CANCEL_AND_WAIT',
      'HadoopJarStep': {
        'Jar': 'command-runner.jar',
        'Args': ['aws', 's3', 'cp', zip_file_on_s3, sources_on_host + '/']
      }
    })
    steps.append({
      'Name': 'setup - extract files',
      'ActionOnFailure': 'CANCEL_AND_WAIT',
      'HadoopJarStep': {
        'Jar': 'command-runner.jar',
        'Args': ['unzip', zip_file_on_host, '-d', sources_on_host]
      }
    })
    for i in range(num_of_steps):
        steps.append({
          'Name': 'run spark {}'.format(spark_main),
          'ActionOnFailure': 'CANCEL_AND_WAIT',
          'HadoopJarStep': {
            'Jar': 'command-runner.jar',
            'Args': (['spark-submit'] +
                     packages +
                     ['--py-files', zip_file_on_host, spark_main_on_host] +
                     spark_main_args.format(i).split())
          }
        })

    if send_success_email_to is not None:
        steps.append({
          'Name': 'Send success email to {}'.format(send_success_email_to),
          'ActionOnFailure': 'CANCEL_AND_WAIT',
          'HadoopJarStep': {
            'Jar': 'command-runner.jar',
            'Args': ['aws', 'ses', 'send-email', '--from', 'ops@yodas.com',
                     '--to', send_success_email_to, '--subject',
                     'EMR COMPLETED SUCCESSFULY', '--text', 'Life is good']
          }
        })

    return steps

def _create_debug_steps(setup_debug):
    if setup_debug:
        return [
          {
            'Name': 'Setup Debugging',
            'ActionOnFailure': 'TERMINATE_CLUSTER',
            'HadoopJarStep': {
                'Jar': 'command-runner.jar',
                'Args': ['state-pusher-script']
            }
          },
        ]
    else:
        return []

def create_cluster_and_run_job_flow(create_cluster_master_type=None,
                                    create_cluster_slave_type=None,
                                    create_cluster_num_hosts=1,
                                    create_cluster_ec2_key_name=None,
                                    create_cluster_ec2_subnet_id=None,
                                    create_cluster_setup_debug=None,
                                    create_cluster_keep_alive_when_done=None,
                                    python_path=None,
                                    num_of_steps=1,
                                    spark_main=None,
                                    py_files=None,
                                    spark_main_args=None,
                                    s3_work_bucket=None,
                                    use_mysql=False,
                                    aws_region=None,
                                    send_success_email_to=None):
    assert(create_cluster_master_type)
    assert(create_cluster_slave_type)
    assert(aws_region)

    s3_logs_uri = 's3n://{}/logs/{}/'.format(s3_work_bucket, getpass.getuser())
    job_flow_name = _create_job_flow_name(spark_main)
    steps = _create_steps(job_flow_name=job_flow_name,
                          python_path=python_path,
                          spark_main=spark_main,
                          py_files=py_files,
                          num_of_steps=num_of_steps,
                          spark_main_args=spark_main_args,
                          s3_work_bucket=s3_work_bucket,
                          use_mysql=use_mysql,
                          send_success_email_to=send_success_email_to)
    client = _get_client(aws_region)
    debug_steps = _create_debug_steps(create_cluster_setup_debug)
    response = client.run_job_flow(
        Name=job_flow_name,
        LogUri=s3_logs_uri,
        ReleaseLabel='emr-4.6.0',
        Instances={
          'MasterInstanceType': create_cluster_master_type,
          'SlaveInstanceType': create_cluster_slave_type,
          'InstanceCount': create_cluster_num_hosts,
          'Ec2KeyName': create_cluster_ec2_key_name,
          'KeepJobFlowAliveWhenNoSteps': create_cluster_keep_alive_when_done,
          'TerminationProtected': False,
          'Ec2SubnetId': create_cluster_ec2_subnet_id,
        },
        Steps=debug_steps + steps,
        Applications=[{'Name': 'Ganglia'}, {'Name': 'Spark'}],
        Configurations=[
          {
            'Classification': 'spark',
            'Properties': {
               'maximizeResourceAllocation': 'true'
            }
          },
          {
            "Classification": "spark-defaults",
            "Properties": {
               "spark.dynamicAllocation.enabled": "true",
               "spark.executor.instances": "0"
            }
          }
        ],
        VisibleToAllUsers=True,
        JobFlowRole='EMR_EC2_DefaultRole',
        ServiceRole='EMR_DefaultRole',
        Tags=[{'Key': 'Name', 'Value': spark_main}]
    )
    job_flow_id = response['JobFlowId']
    print 'Created Job Flow: {}'.format(job_flow_id)
    step_ids = _get_step_ids_for_job_flow(job_flow_id, client)
    print 'Created Job steps: {}'.format(step_ids)
    print '''Waiting for steps to finish. Visit on aws portal:
        https://{0}.console.aws.amazon.com/elasticmapreduce/home?region={0}#cluster-details:{1}'''.format(aws_region, job_flow_id)
    print "Find logs here: {0}{1}/".format(s3_logs_uri, job_flow_id)
    _wait_for_job_flow(aws_region, job_flow_id, step_ids)

def _get_step_ids_for_job_flow(job_flow_id, client):
    steps = client.list_steps(ClusterId=job_flow_id)
    step_ids = map(lambda s: s['Id'], steps['Steps'])
    return step_ids

def _wait_for_job_flow(aws_region, job_flow_id, step_ids=[]):
    while True:
        time.sleep(5)
        client = _get_client(aws_region)
        cluster = client.describe_cluster(ClusterId=job_flow_id)
        state = cluster['Cluster']['Status']['State']
        state_failed = state in ['TERMINATED_WITH_ERRORS']
        p = []
        p.append('Cluster: {}'.format(state))
        all_done = True
        for step_id in step_ids:
            step = client.describe_step(ClusterId=job_flow_id, StepId=step_id)
            step_state = step['Step']['Status']['State']
            step_failed = step_state in ['FAILED', 'CANCELLED']
            step_success = step_state in ['COMPLETED']
            step_done = step_success or step_failed
            step_name = step['Step']['Name']
            if not step_success:
                p.append('{} ({}) - {}'.format(step_name, step_id, step_state))
            all_done = all_done and step_done
            if step_failed:
                print '!!! STEP FAILED: {} ({})'.format(step_name, step_id)
        print '\t'.join(p)
        if all_done:
            print "All done"
            break
        if state_failed:
            print ">>>>>>>>>>>>>>>> FAILED <<<<<<<<<<<<<<<<<<"
            print "Error message: {}".format(cluster['Cluster']
                                             ['Status']['Message'])

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--create_cluster',
                        help='Create a new cluster (and destroy it when it ' +
                             'is done',
                        action='store_true')
    parser.add_argument('--create_cluster_master_type', help='Number of hosts',
                        default='m1.medium')
    parser.add_argument('--create_cluster_slave_type', help='Number of hosts',
                        default='m3.xlarge')
    parser.add_argument('--create_cluster_num_hosts', help='Number of hosts',
                        type=int, default=1)
    parser.add_argument('--create_cluster_ec2_key_name', help='Keyfile when ' +
                        'you want to create a new cluster and connect to it')
    parser.add_argument('--create_cluster_ec2_subnet_id', help='')
    parser.add_argument('--create_cluster_keep_alive_when_done', default=False,
                        action='store_true',
                        help='Terminate the cluster when execution is done')
    parser.add_argument('--create_cluster_setup_debug', default=True,
                        help='Whether to setup the cluster for debugging',
                        action='store_true')

    parser.add_argument('--aws_region', help='AWS region', required=True)

    parser.add_argument('--job_flow_id',
                        help='Job flow ID (EMR cluster) to submit to')
    parser.add_argument('--python_path', required=True,
                        help='Path to python files to zip and upload to the' +
                        ' server and add to the python path. This should ' +
                        'include the python_main file`')
    parser.add_argument('--spark_main', required=True,
                        help='Main python file for spark')
    parser.add_argument('--spark_main_args',
                        help='Arguments passed to your spark script')
    parser.add_argument('--s3_work_bucket', required=True,
                        help='Name of s3 bucket where sources and logs are ' +
                        'uploaded')
    parser.add_argument('--py-files', nargs='*', dest='py_files',
                        help='A list of py or zip or egg files to pass over ' +
                        'to spark-submit')
    parser.add_argument('--use_mysql', default=False,
                        help='Whether to setup mysql dataframes jar',
                        action='store_true')
    parser.add_argument('--send_success_email_to', default=None,
                        help='Email address to send on success')
    parser.add_argument('--num_of_steps', default=1, type=int)

    args = parser.parse_args()

    if args.job_flow_id:
        add_step_to_job_flow(job_flow_id=args.job_flow_id,
                             python_path=args.python_path,
                             spark_main=args.spark_main,
                             spark_main_args=args.spark_main_args,
                             num_of_steps=args.num_of_steps,
                             py_files=args.py_files,
                             use_mysql=args.use_mysql,
                             s3_work_bucket=args.s3_work_bucket,
                             aws_region=args.aws_region,
                             send_success_email_to=args.send_success_email_to)
    elif args.create_cluster:
        create_cluster_and_run_job_flow(
            create_cluster_master_type=args.create_cluster_master_type,
            create_cluster_slave_type=args.create_cluster_slave_type,
            create_cluster_num_hosts=args.create_cluster_num_hosts,
            create_cluster_ec2_key_name=args.create_cluster_ec2_key_name,
            create_cluster_ec2_subnet_id=args.create_cluster_ec2_subnet_id,
            create_cluster_setup_debug=args.create_cluster_setup_debug,
            create_cluster_keep_alive_when_done=args.create_cluster_keep_alive_when_done,
            python_path=args.python_path,
            spark_main=args.spark_main,
            py_files=args.py_files,
            use_mysql=args.use_mysql,
            spark_main_args=args.spark_main_args,
            num_of_steps=args.num_of_steps,
            s3_work_bucket=args.s3_work_bucket,
            aws_region=args.aws_region,
            send_success_email_to=args.send_success_email_to)
    else:
        print "Nothing to do"
        parser.print_help()
<commit_msg>spots<commit_after>import argparse
import boto3
import subprocess
import getpass
import time
import os
import zipfile

def _get_client(aws_region):
    return boto3.client('emr', region_name=aws_region)

def add_step_to_job_flow(job_flow_id=None,
                         python_path=None,
                         spark_main=None,
                         py_files=None,
                         num_of_steps=1,
                         use_mysql=False,
                         spark_main_args=None,
                         s3_work_bucket=None,
                         aws_region=None,
                         send_success_email_to=None):
    assert(job_flow_id)
    assert(aws_region)

    job_flow_name = _create_job_flow_name(spark_main)
    steps = _create_steps(job_flow_name=job_flow_name,
                          python_path=python_path,
                          spark_main=spark_main,
                          py_files=py_files,
                          num_of_steps=num_of_steps,
                          use_mysql=use_mysql,
                          spark_main_args=spark_main_args,
                          s3_work_bucket=s3_work_bucket,
                          send_success_email_to=send_success_email_to)
    client = _get_client(aws_region)
    step_response = client.add_job_flow_steps(
      JobFlowId=job_flow_id,
      Steps=steps
    )
    step_ids = step_response['StepIds']
    print "Created steps: {}".format(step_ids)
    _wait_for_job_flow(aws_region, job_flow_id, step_ids)

def _create_job_flow_name(spark_main):
    return '{}.{}.{}'.format(getpass.getuser(),
                             spark_main,
                             time.strftime("%H%M%S", time.gmtime()))

def _ls_recursive(dir, suffix=None):
    files = [os.path.join(dp, f) for dp, dn, fn in os.walk(os.path.expanduser(dir)) for f in fn]
    if suffix:
        files = filter(lambda f: f.endswith(suffix), files)
    return files

def _create_steps(job_flow_name=None,
                  python_path=None,
                  spark_main=None,
                  py_files=[],
                  num_of_steps=1,
                  spark_main_args=None,
                  s3_work_bucket=None,
                  use_mysql=False,
                  send_success_email_to=None):
    assert(python_path)
    assert(spark_main)
    assert(s3_work_bucket)

    zip_file = 'spark_zip.zip'
    sources_rel_path = job_flow_name
    sources_on_host = '/home/hadoop/{}'.format(sources_rel_path)
    local_zip_file = '/tmp/{}'.format(zip_file)
    python_path_files = _ls_recursive(python_path, '.py')
    with zipfile.ZipFile(local_zip_file, 'w') as myzip:
        for f in python_path_files:
            myzip.write(f)
        if py_files:
          for py_file in py_files:
              if py_file.endswith('.zip'):  # Currently only support sip files
                  with zipfile.ZipFile(py_file, 'r') as openzip:
                      [myzip.writestr(t[0], t[1].read())
                       for t in ((n, openzip.open(n))
                       for n in openzip.namelist())]
    s3sources = 's3://{}/sources/{}'.format(s3_work_bucket, sources_rel_path)
    zip_file_on_s3 = '{}/{}'.format(s3sources, zip_file)
    print 'Storing python sources on {}'.format(s3sources)
    # TODO: Change these subprocess calls to use python native API instead of shell
    subprocess.check_call('aws s3 cp {} {}'.format(local_zip_file, zip_file_on_s3), shell=True)
    zip_file_on_host = '{}/{}'.format(sources_on_host, zip_file)
    spark_main_on_host = '{}/{}'.format(sources_on_host, spark_main)
    # spark_main_args = spark_main_args.split() if spark_main_args else ['']
    packages_to_add = []
    if use_mysql:
        packages_to_add.append('mysql:mysql-connector-java:5.1.39')
    packages = ['--packages'] + packages_to_add if packages_to_add else []

    steps = []
    steps.append({
      'Name': 'setup - copy files',
      'ActionOnFailure': 'CANCEL_AND_WAIT',
      'HadoopJarStep': {
        'Jar': 'command-runner.jar',
        'Args': ['aws', 's3', 'cp', zip_file_on_s3, sources_on_host + '/']
      }
    })
    steps.append({
      'Name': 'setup - extract files',
      'ActionOnFailure': 'CANCEL_AND_WAIT',
      'HadoopJarStep': {
        'Jar': 'command-runner.jar',
        'Args': ['unzip', zip_file_on_host, '-d', sources_on_host]
      }
    })
    for i in range(num_of_steps):
        steps.append({
          'Name': 'run spark {}'.format(spark_main),
          'ActionOnFailure': 'CANCEL_AND_WAIT',
          'HadoopJarStep': {
            'Jar': 'command-runner.jar',
            'Args': (['spark-submit'] +
                     packages +
                     ['--py-files', zip_file_on_host, spark_main_on_host] +
                     spark_main_args.format(i).split())
          }
        })

    if send_success_email_to is not None:
        steps.append({
          'Name': 'Send success email to {}'.format(send_success_email_to),
          'ActionOnFailure': 'CANCEL_AND_WAIT',
          'HadoopJarStep': {
            'Jar': 'command-runner.jar',
            'Args': ['aws', 'ses', 'send-email', '--from', 'ops@yodas.com',
                     '--to', send_success_email_to, '--subject',
                     'EMR COMPLETED SUCCESSFULY', '--text', 'Life is good']
          }
        })

    return steps

def _create_debug_steps(setup_debug):
    if setup_debug:
        return [
          {
            'Name': 'Setup Debugging',
            'ActionOnFailure': 'TERMINATE_CLUSTER',
            'HadoopJarStep': {
                'Jar': 'command-runner.jar',
                'Args': ['state-pusher-script']
            }
          },
        ]
    else:
        return []

def create_cluster_and_run_job_flow(create_cluster_master_type=None,
                                    create_cluster_slave_type=None,
                                    create_cluster_num_hosts=1,
                                    create_cluster_ec2_key_name=None,
                                    create_cluster_ec2_subnet_id=None,
                                    create_cluster_setup_debug=None,
                                    create_cluster_keep_alive_when_done=None,
                                    bid_price=None,
                                    python_path=None,
                                    num_of_steps=1,
                                    spark_main=None,
                                    py_files=None,
                                    spark_main_args=None,
                                    s3_work_bucket=None,
                                    use_mysql=False,
                                    aws_region=None,
                                    send_success_email_to=None):
    assert(create_cluster_master_type)
    assert(create_cluster_slave_type)
    assert(aws_region)

    s3_logs_uri = 's3n://{}/logs/{}/'.format(s3_work_bucket, getpass.getuser())
    job_flow_name = _create_job_flow_name(spark_main)
    steps = _create_steps(job_flow_name=job_flow_name,
                          python_path=python_path,
                          spark_main=spark_main,
                          py_files=py_files,
                          num_of_steps=num_of_steps,
                          spark_main_args=spark_main_args,
                          s3_work_bucket=s3_work_bucket,
                          use_mysql=use_mysql,
                          send_success_email_to=send_success_email_to)
    client = _get_client(aws_region)
    debug_steps = _create_debug_steps(create_cluster_setup_debug)
    if bid_price:
        instances = {
                'InstanceGroups': [
                    {
                        'Name': 'EmrMaster',
                        'Market': 'SPOT',
                        'InstanceRole': 'MASTER',
                        'BidPrice': bid_price,
                        'InstanceType': create_cluster_master_type,
                        'InstanceCount': 1,
                        },
                    {
                        'Name': 'EmrCore',
                        'Market': 'SPOT',
                        'InstanceRole': 'CORE',
                        'BidPrice': bid_price,
                        'InstanceType': create_cluster_slave_type,
                        'InstanceCount': create_cluster_num_hosts,
                        },
                    ],
                'Ec2KeyName': create_cluster_ec2_key_name,
                'KeepJobFlowAliveWhenNoSteps': create_cluster_keep_alive_when_done,
                'TerminationProtected': False,
                'Ec2SubnetId': create_cluster_ec2_subnet_id
                }
    else:
        instances = {
          'MasterInstanceType': create_cluster_master_type,
          'SlaveInstanceType': create_cluster_slave_type,
          'InstanceCount': create_cluster_num_hosts,
          'Ec2KeyName': create_cluster_ec2_key_name,
          'KeepJobFlowAliveWhenNoSteps': create_cluster_keep_alive_when_done,
          'TerminationProtected': False,
          'Ec2SubnetId': create_cluster_ec2_subnet_id
        }
    response = client.run_job_flow(
        Name=job_flow_name,
        LogUri=s3_logs_uri,
        ReleaseLabel='emr-4.6.0',
        Instances=instances,
        Steps=debug_steps + steps,
        Applications=[{'Name': 'Ganglia'}, {'Name': 'Spark'}],
        Configurations=[
          {
            'Classification': 'spark',
            'Properties': {
               'maximizeResourceAllocation': 'true'
            }
          },
          {
            "Classification": "spark-defaults",
            "Properties": {
               "spark.dynamicAllocation.enabled": "true",
               "spark.executor.instances": "0"
            }
          }
        ],
        VisibleToAllUsers=True,
        JobFlowRole='EMR_EC2_DefaultRole',
        ServiceRole='EMR_DefaultRole',
        Tags=[{'Key': 'Name', 'Value': spark_main}]
    )
    job_flow_id = response['JobFlowId']
    print 'Created Job Flow: {}'.format(job_flow_id)
    step_ids = _get_step_ids_for_job_flow(job_flow_id, client)
    print 'Created Job steps: {}'.format(step_ids)
    print '''Waiting for steps to finish. Visit on aws portal:
        https://{0}.console.aws.amazon.com/elasticmapreduce/home?region={0}#cluster-details:{1}'''.format(aws_region, job_flow_id)
    print "Find logs here: {0}{1}/".format(s3_logs_uri, job_flow_id)
    _wait_for_job_flow(aws_region, job_flow_id, step_ids)

def _get_step_ids_for_job_flow(job_flow_id, client):
    steps = client.list_steps(ClusterId=job_flow_id)
    step_ids = map(lambda s: s['Id'], steps['Steps'])
    return step_ids

def _wait_for_job_flow(aws_region, job_flow_id, step_ids=[]):
    while True:
        time.sleep(5)
        client = _get_client(aws_region)
        cluster = client.describe_cluster(ClusterId=job_flow_id)
        state = cluster['Cluster']['Status']['State']
        state_failed = state in ['TERMINATED_WITH_ERRORS']
        p = []
        p.append('Cluster: {}'.format(state))
        all_done = True
        for step_id in step_ids:
            step = client.describe_step(ClusterId=job_flow_id, StepId=step_id)
            step_state = step['Step']['Status']['State']
            step_failed = step_state in ['FAILED', 'CANCELLED']
            step_success = step_state in ['COMPLETED']
            step_done = step_success or step_failed
            step_name = step['Step']['Name']
            if not step_success:
                p.append('{} ({}) - {}'.format(step_name, step_id, step_state))
            all_done = all_done and step_done
            if step_failed:
                print '!!! STEP FAILED: {} ({})'.format(step_name, step_id)
        print '\t'.join(p)
        if all_done:
            print "All done"
            break
        if state_failed:
            print ">>>>>>>>>>>>>>>> FAILED <<<<<<<<<<<<<<<<<<"
            print "Error message: {}".format(cluster['Cluster']
                                             ['Status']['Message'])

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--create_cluster',
                        help='Create a new cluster (and destroy it when it ' +
                             'is done',
                        action='store_true')
    parser.add_argument('--create_cluster_master_type', help='Number of hosts',
                        default='m1.medium')
    parser.add_argument('--create_cluster_slave_type', help='Number of hosts',
                        default='m3.xlarge')
    parser.add_argument('--create_cluster_num_hosts', help='Number of hosts',
                        type=int, default=1)
    parser.add_argument('--create_cluster_ec2_key_name', help='Keyfile when ' +
                        'you want to create a new cluster and connect to it')
    parser.add_argument('--create_cluster_ec2_subnet_id', help='')
    parser.add_argument('--create_cluster_keep_alive_when_done', default=False,
                        action='store_true',
                        help='Terminate the cluster when execution is done')
    parser.add_argument('--create_cluster_setup_debug', default=True,
                        help='Whether to setup the cluster for debugging',
                        action='store_true')

    parser.add_argument('--aws_region', help='AWS region', required=True)

    parser.add_argument('--job_flow_id',
                        help='Job flow ID (EMR cluster) to submit to')
    parser.add_argument('--python_path', required=True,
                        help='Path to python files to zip and upload to the' +
                        ' server and add to the python path. This should ' +
                        'include the python_main file`')
    parser.add_argument('--spark_main', required=True,
                        help='Main python file for spark')
    parser.add_argument('--spark_main_args',
                        help='Arguments passed to your spark script')
    parser.add_argument('--s3_work_bucket', required=True,
                        help='Name of s3 bucket where sources and logs are ' +
                        'uploaded')
    parser.add_argument('--py-files', nargs='*', dest='py_files',
                        help='A list of py or zip or egg files to pass over ' +
                        'to spark-submit')
    parser.add_argument('--use_mysql', default=False,
                        help='Whether to setup mysql dataframes jar',
                        action='store_true')
    parser.add_argument('--send_success_email_to', default=None,
                        help='Email address to send on success')
    parser.add_argument('--num_of_steps', default=1, type=int)
    parser.add_argument('--bid_price', default=None)

    args = parser.parse_args()

    if args.job_flow_id:
        add_step_to_job_flow(job_flow_id=args.job_flow_id,
                             python_path=args.python_path,
                             spark_main=args.spark_main,
                             spark_main_args=args.spark_main_args,
                             num_of_steps=args.num_of_steps,
                             py_files=args.py_files,
                             use_mysql=args.use_mysql,
                             s3_work_bucket=args.s3_work_bucket,
                             aws_region=args.aws_region,
                             send_success_email_to=args.send_success_email_to)
    elif args.create_cluster:
        create_cluster_and_run_job_flow(
            create_cluster_master_type=args.create_cluster_master_type,
            create_cluster_slave_type=args.create_cluster_slave_type,
            create_cluster_num_hosts=args.create_cluster_num_hosts,
            create_cluster_ec2_key_name=args.create_cluster_ec2_key_name,
            create_cluster_ec2_subnet_id=args.create_cluster_ec2_subnet_id,
            create_cluster_setup_debug=args.create_cluster_setup_debug,
            create_cluster_keep_alive_when_done=args.create_cluster_keep_alive_when_done,
            bid_price=args.bid_price,
            python_path=args.python_path,
            spark_main=args.spark_main,
            py_files=args.py_files,
            use_mysql=args.use_mysql,
            spark_main_args=args.spark_main_args,
            num_of_steps=args.num_of_steps,
            s3_work_bucket=args.s3_work_bucket,
            aws_region=args.aws_region,
            send_success_email_to=args.send_success_email_to)
    else:
        print "Nothing to do"
        parser.print_help()
<|endoftext|>


<commit_before>'''
Created on Mar 11, 2016

@author: enikher
'''
from flash_test.utils.node import Node
from flash_test.utils.ssh_util import SshUtil
from tempest.api.compute import base as compute_base
import os
import yaml
from novaclient.client import Client as nova
from novaclient import api_versions

FLASH_TEST_CONFIG_YAML = "/etc/flash_test.yaml"

class OpenstackEnv(object):

    # These clients are only used to create the environment
    # In general the tempest clients shall be used.
    _novacl = nova(api_versions.APIVersion("2.0"),
                   os.environ['OS_USERNAME'],
                   os.environ['OS_PASSWORD'],
                   os.environ['OS_TENANT_NAME'],
                   auth_url=os.environ['OS_AUTH_URL'])
    hypervisors = {}
    env_nodes = []
    main_controller = None

    @classmethod
    def resource_setup(cls):
        if os.path.isfile(FLASH_TEST_CONFIG_YAML):
            with open(FLASH_TEST_CONFIG_YAML) as f:
                cls.config = yaml.load(f)
        else:
            raise Exception("Config: %s is not found."
                            % FLASH_TEST_CONFIG_YAML)
        # add all nodes
        for (name, node) in cls.config['env']['nodes'].iteritems():
            new_node = Node(name, dict=node)
            cls.env_nodes.append(new_node)
            if name == "main_controller":
                cls.main_controller = new_node

        # Jump: string to real objects
        for (name, node) in cls.config['env']['nodes'].iteritems():
            jump = node.get('jump')
            if jump:
                for node_jumb in cls.env_nodes:
                    if node_jumb == jump:
                        node['jump'] = node_jumb

        # All openstack hypervisors
        nova_hypervisors = cls._novacl.hypervisors.list()
        for nova_hypervisor in nova_hypervisors:
            new_hypervisor = Node(
                name=nova_hypervisor.hypervisor_hostname,
                address=nova_hypervisor.host_ip,
                user=cls.config['env']['hypervisor_ssh_user'],
                jump=cls.main_controller)
            cls.hypervisors[nova_hypervisor.hypervisor_hostname] =\
                new_hypervisor
            cls.env_nodes.append(new_hypervisor)

    @classmethod
    def gen_ssh_config(cls, node):
        if node not in cls.env_nodes:
            cls.env_nodes.append(node)
        SshUtil.gen_ssh_config(cls.env_nodes)

    @classmethod
    def delete_server(cls, serverid):
        cls._novacl.servers.delete(serverid)

<commit_msg>fixup! Bugfix dict<commit_after>'''
Created on Mar 11, 2016

@author: enikher
'''
from flash_test.utils.node import Node
from flash_test.utils.ssh_util import SshUtil
from tempest.api.compute import base as compute_base
import os
import yaml
from novaclient.client import Client as nova
from novaclient import api_versions

FLASH_TEST_CONFIG_YAML = "/etc/flash_test.yaml"

class OpenstackEnv(object):

    # These clients are only used to create the environment
    # In general the tempest clients shall be used.
    _novacl = nova(api_versions.APIVersion("2.0"),
                   os.environ['OS_USERNAME'],
                   os.environ['OS_PASSWORD'],
                   os.environ['OS_TENANT_NAME'],
                   auth_url=os.environ['OS_AUTH_URL'])
    hypervisors = {}
    env_nodes = []
    main_controller = None

    @classmethod
    def resource_setup(cls):
        if os.path.isfile(FLASH_TEST_CONFIG_YAML):
            with open(FLASH_TEST_CONFIG_YAML) as f:
                cls.config = yaml.load(f)
        else:
            raise Exception("Config: %s is not found."
                            % FLASH_TEST_CONFIG_YAML)
        # add all nodes
        for (name, node) in cls.config['env']['nodes'].iteritems():
            new_node = Node(name, dict=node)
            cls.env_nodes.append(new_node)
            if name == "main_controller":
                cls.main_controller = new_node

        # Jump: string to real objects
        for (name, node) in cls.config['env']['nodes'].iteritems():
            jump = node.get('jump')
            if jump:
                for node_jumb in cls.env_nodes:
                    if node_jumb.name == jump:
                        node['jump'] = node_jumb

        # All openstack hypervisors
        nova_hypervisors = cls._novacl.hypervisors.list()
        for nova_hypervisor in nova_hypervisors:
            new_hypervisor = Node(
                name=nova_hypervisor.hypervisor_hostname,
                address=nova_hypervisor.host_ip,
                user=cls.config['env']['hypervisor_ssh_user'],
                jump=cls.main_controller)
            cls.hypervisors[nova_hypervisor.hypervisor_hostname] =\
                new_hypervisor
            cls.env_nodes.append(new_hypervisor)

    @classmethod
    def gen_ssh_config(cls, node):
        if node not in cls.env_nodes:
            cls.env_nodes.append(node)
        SshUtil.gen_ssh_config(cls.env_nodes)

    @classmethod
    def delete_server(cls, serverid):
        cls._novacl.servers.delete(serverid)

<|endoftext|>


<commit_before># coding: utf-8
import datetime
from django.core.urlresolvers import reverse
import django.db.models

from division_perm import models

class CreateMainEntityTestMixin(object): # todo: from division_perm.generic.CreateMainEntityMixin -используется только в этом месте , но никогда не выполняется
   pass

class FuncAccessTestMixin(object):

    def test_without_func_code(self): # todo: избыточный код, везде в функциях есть func_code
        pass

    def test_no_is_authenticated(self):
        self.client.logout()
        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 403)  # todo: избыточный код, везде есть LoginRequiredMixin зачем доп. проверка?

    def test_without_level(self):
        self.user.employee.roles.all().delete()
        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 403)

    def test_priority_func_level(self):
        self.user.employee.roles.all().update(level=5)
        models.Func.objects.filter(code=self.func_code).update(level=10)
        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 403)

    def test_priority_user_level(self):
        self.user.employee.roles.all().update(level=10)
        models.Func.objects.filter(code=self.func_code).update(level=5)
        response = self.client.get(self.get_url(), follow=True)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(
            response.redirect_chain[0],
            (self.success_url, 302)
        )

class ModifyAccessTestMixin(object):

    def test_without_passed(self):
        models.Func.objects.all().update(level=10)
        self.user.employee.roles.all().update(level=5)
        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 403)

    def test_with_have_access(self):
        self.user.employee.roles.all().update(level=10)
        models.Func.objects.all().update(level=10)
        division = self.user.employee.divisions.all()[0]
        self.get_instance().full_access.add(division)

        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 200)

    def test_without_have_access(self):
        self.user.employee.roles.all().update(level=10)
        models.Func.objects.all().update(level=10)
        self.get_instance().full_access.clear()

        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 403)

class ReadAccessTestMixin(object):

    def test_access_divisions(self):
        division = models.Division.objects.all()[0]
        self.get_instance().read_access.add(division)

        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 200)
        self.assertEqual(self.get_instance(), response.context['object'])

    def test_without_access_divisions(self):
        self.get_instance().read_access.clear()
        self.get_instance().full_access.clear()
        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 403)

class ListAccessTestMixin(object): # todo: вынесла отдельно, т.к func нет такого поведения

    def test_get_queryset(self):
        response = self.client.get(self.get_url(), follow=True)
        self.assertEqual(response.status_code, 200)
        self.assertListEqual(
            list(response.context['object_list'].values_list('id', flat=True)),
            list(self.model_access.GetAccessible(self.user).values_list('id', flat=True))
        )

class FormAccessTestMixin(object): # todo: надо подумать
    pass

class RestTestMixin(object): # todo: используется?
    pass

class RestNestedTestMixin(object): # todo: используется?
    pass

class LoginRequiredTestMixin(object):

    def test_is_not_authenticated(self):
        self.client.logout()
        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 403) # todo: страннова-то логичнее бы было редиректить на страницу логина

class ListTestMixin(LoginRequiredTestMixin):
    success_url = None
    model_access = None
    func_code = None

    def test_200(self):
        response = self.client.get(self.get_url(), follow=True)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(
            response.redirect_chain[0],
            (self.success_url, 302)
        )

    def test_object(self):
        response = self.client.get(self.get_url(), follow=True)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(
            response.context['object_list'].count(),
            self.model_access.objects.all().count()
        )

class DetailTestMixin(LoginRequiredTestMixin):
    model_access = None

    def test_200(self):
        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 200)
        self.assertEqual(self.get_instance(), response.context['object'])

class CreateTestMixin(object):
    view_path = 'perm_employee_create'
    models = None
    success_path = None

    def test_is_not_authenticated(self): # todo: здесь правильно редиректится, LoginRequiredTestMixin - сюда не подходит, но может стоит все же доп. проверку засунуть в LoginRequiredTestMixin
        self.client.logout()
        response = self.client.get(self.get_url(), follow=True)
        self.assertIn('login/?next', response.redirect_chain[0][0])

    def test_200(self):
        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 200)

    def test_create(self):
        params = self.get_params()
        response = self.client.post(self.get_url(), params, follow=True)
        if 'form' in response.context:
            self.assertFalse(response.context['form'].errors)
        self.assertEqual(len(response.redirect_chain), 1)
        created_obj = self.model.objects.filter(**self.get_ident_param())[0]
        success_url = reverse(self.success_path, args=[created_obj.id])
        self.assertEqual(response.redirect_chain[0], (success_url, 302))

class DeleteTestMixin(LoginRequiredTestMixin, ModifyAccessTestMixin):
    model = None

    def test_200(self):
        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 200)

    def test_delete(self):
        response = self.client.post(self.get_url(), follow=True)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(self.model.objects.filter(id=self.get_instance().id).count(), 0)<commit_msg>#7 - доработка миксинов для создания и обновления<commit_after># coding: utf-8
import datetime
from django.core.urlresolvers import reverse
import django.db.models

from division_perm import models

class CreateMainEntityTestMixin(object): # todo: from division_perm.generic.CreateMainEntityMixin -используется только в этом месте , но никогда не выполняется
   pass

class FuncAccessTestMixin(object):

    def test_without_func_code(self): # todo: избыточный код, везде в функциях есть func_code
        pass

    def test_no_is_authenticated(self):
        self.client.logout()
        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 403)  # todo: избыточный код, везде есть LoginRequiredMixin зачем доп. проверка?

    def test_without_level(self):
        self.user.employee.roles.all().delete()
        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 403)

    def test_priority_func_level(self):
        self.user.employee.roles.all().update(level=5)
        models.Func.objects.filter(code=self.func_code).update(level=10)
        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 403)

    def test_priority_user_level(self):
        self.user.employee.roles.all().update(level=10)
        models.Func.objects.filter(code=self.func_code).update(level=5)
        response = self.client.get(self.get_url(), follow=True)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(
            response.redirect_chain[0],
            (self.success_url, 302)
        )

class ModifyAccessTestMixin(object):

    def test_without_passed(self):
        models.Func.objects.all().update(level=10)
        self.user.employee.roles.all().update(level=5)
        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 403)

    def test_with_have_access(self):
        self.user.employee.roles.all().update(level=10)
        models.Func.objects.all().update(level=10)
        division = self.user.employee.divisions.all()[0]
        self.get_instance().full_access.add(division)

        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 200)

    def test_without_have_access(self):
        self.user.employee.roles.all().update(level=10)
        models.Func.objects.all().update(level=10)
        self.get_instance().full_access.clear()

        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 403)

class ReadAccessTestMixin(object):

    def test_access_divisions(self):
        division = models.Division.objects.all()[0]
        self.get_instance().read_access.add(division)

        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 200)
        self.assertEqual(self.get_instance(), response.context['object'])

    def test_without_access_divisions(self):
        self.get_instance().read_access.clear()
        self.get_instance().full_access.clear()
        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 403)

class ListAccessTestMixin(object):

    def test_get_queryset(self):
        response = self.client.get(self.get_url(), follow=True)
        self.assertEqual(response.status_code, 200)
        self.assertListEqual(
            list(response.context['object_list'].values_list('id', flat=True)),
            list(self.model_access.GetAccessible(self.user).values_list('id', flat=True))
        )

class FormAccessTestMixin(object): # todo: надо подумать
    pass

class RestTestMixin(object): # todo: используется?
    pass

class RestNestedTestMixin(object): # todo: используется?
    pass

class LoginRequiredTestMixin(object):

    def test_is_not_authenticated(self):
        self.client.logout()
        response = self.client.get(self.get_url(),follow=True)
        if response.status_code == 302:
            self.assertIn('login/?next', response.redirect_chain[0][0]) # todo:  в случае с create
        else:
            self.assertEqual(response.status_code, 403) # todo: страннова-то логичнее бы было редиректить на страницу логина

class ListTestMixin(LoginRequiredTestMixin):
    success_url = None
    model_access = None
    func_code = None

    def test_200(self):
        response = self.client.get(self.get_url(), follow=True)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(
            response.redirect_chain[0],
            (self.success_url, 302)
        )

    def test_object(self):
        response = self.client.get(self.get_url(), follow=True)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(
            response.context['object_list'].count(),
            self.model_access.objects.all().count()
        )

class DetailTestMixin(LoginRequiredTestMixin):
    model_access = None

    def test_200(self):
        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 200)
        self.assertEqual(self.get_instance(), response.context['object'])

class CreateTestMixin(object):
    view_path = 'perm_employee_create'
    models = None
    success_path = None

    def test_200(self):
        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 200)

    def test_update(self):
        params = self.get_params()
        response = self.client.post(self.get_url(), params, follow=True)
        if 'form' in response.context:
            self.assertFalse(response.context['form'].errors)
        self.assertEqual(len(response.redirect_chain), 1)
        created_obj = self.model.objects.filter(**self.get_ident_param())[0]
        success_url = reverse(self.success_path, args=[created_obj.id])
        self.assertEqual(response.redirect_chain[0], (success_url, 302))

class UpdateTestMixin(CreateTestMixin):

    def test_update(self):
        params = self.get_params()
        response = self.client.post(self.get_url(), params, follow=True)
        if 'form' in response.context:
            self.assertFalse(response.context['form'].errors)
        self.assertEqual(len(response.redirect_chain), 1)
        update_obj = self.model.objects.get(id=self.get_instance().id, **self.get_ident_param())
        success_url = reverse(self.success_path, args=[update_obj.id])
        self.assertEqual(response.redirect_chain[0], (success_url, 302))

class DeleteTestMixin(LoginRequiredTestMixin, ModifyAccessTestMixin):
    model = None

    def test_200(self):
        response = self.client.get(self.get_url())
        self.assertEqual(response.status_code, 200)

    def test_delete(self):
        response = self.client.post(self.get_url(), follow=True)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(self.model.objects.filter(id=self.get_instance().id).count(), 0)<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import sys

def compile_messages():
    basedir = None

    if os.path.isdir(os.path.join('conf', 'locale')):
        basedir = os.path.abspath(os.path.join('conf', 'locale'))
    elif os.path.isdir('locale'):
        basedir = os.path.abspath('locale')
    else:
        print "This script should be run from the Django SVN tree or your project or app tree."
        sys.exit(1)

    for dirpath, dirnames, filenames in os.walk(basedir):
        for f in filenames:
            if f.endswith('.po'):
                sys.stderr.write('processing file %s in %s\n' % (f, dirpath))
                pf = os.path.splitext(os.path.join(dirpath, f))[0]
                # Store the names of the .mo and .po files in an environment
                # variable, rather than doing a string replacement into the
                # command, so that we can take advantage of shell quoting, to
                # quote any malicious characters/escaping.
                # See http://cyberelk.net/tim/articles/cmdline/ar01s02.html
                os.environ['djangocompilemo'] = pf + '.mo'
                os.environ['djangocompilepo'] = pf + '.po'
                if sys.platform == 'win32': # Different shell-variable syntax
                    cmd = 'msgfmt -o "%djangocompilemo%" "%djangocompilepo%"'
                else:
                    cmd = 'msgfmt -o "$djangocompilemo" "$djangocompilepo"'
                os.system(cmd)

if __name__ == "__main__":
    compile_messages()
<commit_msg>Fixed #3679 -- Added an option to compile localisation messages for a single locale.<commit_after>#!/usr/bin/env python

import optparse
import os
import sys

def compile_messages(locale=None):
    basedir = None

    if os.path.isdir(os.path.join('conf', 'locale')):
        basedir = os.path.abspath(os.path.join('conf', 'locale'))
    elif os.path.isdir('locale'):
        basedir = os.path.abspath('locale')
    else:
        print "This script should be run from the Django SVN tree or your project or app tree."
        sys.exit(1)

    if locale is not None:
        basedir = os.path.join(basedir, locale, 'LC_MESSAGES')

    for dirpath, dirnames, filenames in os.walk(basedir):
        for f in filenames:
            if f.endswith('.po'):
                sys.stderr.write('processing file %s in %s\n' % (f, dirpath))
                pf = os.path.splitext(os.path.join(dirpath, f))[0]
                # Store the names of the .mo and .po files in an environment
                # variable, rather than doing a string replacement into the
                # command, so that we can take advantage of shell quoting, to
                # quote any malicious characters/escaping.
                # See http://cyberelk.net/tim/articles/cmdline/ar01s02.html
                os.environ['djangocompilemo'] = pf + '.mo'
                os.environ['djangocompilepo'] = pf + '.po'
                if sys.platform == 'win32': # Different shell-variable syntax
                    cmd = 'msgfmt -o "%djangocompilemo%" "%djangocompilepo%"'
                else:
                    cmd = 'msgfmt -o "$djangocompilemo" "$djangocompilepo"'
                os.system(cmd)

def main():
    parser = optparse.OptionParser()
    parser.add_option('-l', '--locale', dest='locale',
            help="The locale to process. Default is to process all.")
    options, args = parser.parse_args()
    if len(args):
        parser.error("This program takes no arguments")
    compile_messages(options.locale)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import sys

def compile_messages():
    basedir = None

    if os.path.isdir(os.path.join('conf', 'locale')):
        basedir = os.path.abspath(os.path.join('conf', 'locale'))
    elif os.path.isdir('locale'):
        basedir = os.path.abspath('locale')
    else:
        print "this script should be run from the django svn tree or your project or app tree"
        sys.exit(1)

    for dirpath, dirnames, filenames in os.walk(basedir):
        for f in filenames:
            if f.endswith('.po'):
                sys.stderr.write('processing file %s in %s\n' % (f, dirpath))
                pf = os.path.splitext(os.path.join(dirpath, f))[0]
                cmd = 'msgfmt -o "%s.mo" "%s.po"' % (pf, pf)
                os.system(cmd)

if __name__ == "__main__":
    compile_messages()
<commit_msg>Small error-message formatting change to compile-messages.py<commit_after>#!/usr/bin/env python

import os
import sys

def compile_messages():
    basedir = None

    if os.path.isdir(os.path.join('conf', 'locale')):
        basedir = os.path.abspath(os.path.join('conf', 'locale'))
    elif os.path.isdir('locale'):
        basedir = os.path.abspath('locale')
    else:
        print "This script should be run from the Django SVN tree or your project or app tree."
        sys.exit(1)

    for dirpath, dirnames, filenames in os.walk(basedir):
        for f in filenames:
            if f.endswith('.po'):
                sys.stderr.write('processing file %s in %s\n' % (f, dirpath))
                pf = os.path.splitext(os.path.join(dirpath, f))[0]
                cmd = 'msgfmt -o "%s.mo" "%s.po"' % (pf, pf)
                os.system(cmd)

if __name__ == "__main__":
    compile_messages()
<|endoftext|>