<commit_before>__author__ = 'msivak'

from ..lib.fsm import BaseFSM, BaseState
from ..lib import util
from ..lib.util import engine_status_score
from . import constants
import time
from states import ReinitializeFSM
from state_data import HostedEngineData, StatsData
from ..lib import monotonic

class StartState(BaseState):
    """
    :transition ReinitializeFSM:
    """
    def consume(self, fsm, new_data, logger):
        return ReinitializeFSM(new_data)

    def score(self, logger):
        return 0

    def collect(self, fsm, new_data, logger):
        return new_data

    def metadata(self):
        return {}

class EngineStateMachine(BaseFSM):
    def __init__(self, he, logger, actions):
        """
        :type he: ovirt_hosted_engine_ha.agent.hosted_engine.HostedEngine
        :type logger: logging.Logger
        :type actions: dict[str, T]
        """
        initial_data = HostedEngineData(
            host_id=None,
            history=(),
            stats=None,
            timeout_start_time=None,
            engine_vm_retry_count=0,
            engine_vm_retry_time=None,
            engine_vm_shutdown_time=None,
            unexpected_shutdown_time=None,
            last_metadata_log_time=0,
            migration_host_id=None,
            migration_result=None,
            score_cfg=he.score_config,
            min_memory_threshold=he.min_memory_threshold,
            best_engine_status=None,
            best_engine_host_id=None,
            best_score_host=None,
            alive_hosts=[],
        )

        super(EngineStateMachine, self).__init__(StartState, initial_data,
                                                 logger, actions)
        self.hosted_engine = he

    def refresh(self, old_data):
        """
        Ask hosted engine to provide fresh state information and
        pre-compute all the values we need to use it easily.

        :type old_data: HostedEngineData
        :rtype: HostedEngineData
        """
        # Collect stats
        mtime = monotonic.time()
        stats = {
            "collect_start": int(mtime),
            "time_epoch": int(time.time() - mtime)
        }

        # Do not refresh if the time has not changed
        if (old_data.stats is not None and
                stats["collect_start"] == old_data.stats.collect_finish):
            return old_data

        stats.update(self.hosted_engine.collect_stats())
        stats.update({
            "collect_finish": int(monotonic.time())
        })

        # Convert to read only structure to prevent accidental changes
        stats = StatsData(**stats)

        # Check if hosts are alive using the stat history and local time
        # do not use host-ts directly as it might be wrong, only check
        # if it changed in the last HOST_ALIVE_TIMEOUT_SECS period.

        # 1) Find the first point in history that has to have older update
        # from all hosts
        historic = None
        for h in old_data.history:
            if (h.collect_finish + constants.HOST_ALIVE_TIMEOUT_SECS <
                    stats.collect_start):
                historic = h
                break

        # 2) If a currently available host is not present in the historic
        # record or its timestamp is different then it is alive
        if historic:
            for hid, st in stats.hosts.iteritems():
                st["alive"] = (hid not in historic.hosts or
                               historic.hosts[hid].get("host-ts", None)
                               != st.get("host-ts", None))

            alive_hosts = [st for hid, st in stats.hosts.iteritems()
                           if st["alive"] if "engine-status" in st]
        else:
            alive_hosts = []

        # Prepare changes to the data structure
        new_data = {}
        # save the copy of alive_hosts right after we compute it to avoid
        # any changes that might be done to the list
        new_data["alive_hosts"] = [host['host-id'] for host in alive_hosts]

        if alive_hosts:
            # Pre-compute the best remote engine (skip old metadata
            # for local host)
            best_engine = min(alive_hosts,
                              key=lambda st:
                              engine_status_score(st['engine-status']))

            # Pre-compute best remote score (skip old metadata for local host)
            best_score = min(alive_hosts,
                             key=lambda st: st['score'])

        # Compare the best engine remote values with the local state
        lm = stats.local
        if (not alive_hosts or engine_status_score(lm["engine-health"])
                >= engine_status_score(best_engine["engine-status"])):
            new_data["best_engine_status"] = lm["engine-health"]
            new_data["best_engine_host_id"] = self.hosted_engine.host_id
        else:
            new_data["best_engine_status"] = best_engine["engine_status"]
            new_data["best_engine_host_id"] = best_engine["host_id"]

        # Save the best score remote values
        # we can't compare them because we first need to fully initialize
        # the current state
        new_data["best_score_host"] = best_score if alive_hosts else None

        # Re-initialize retry status variables if the retry window
        # has expired.
        if util.has_elapsed(old_data.engine_vm_retry_time,
                            constants.ENGINE_RETRY_EXPIRATION_SECS):
            new_data['engine_vm_retry_time'] = None
            new_data['engine_vm_retry_count'] = 0
            self.logger.debug("Cleared retry status")

        # log the global status every now and then
        if util.has_elapsed(old_data.last_metadata_log_time,
                            constants.METADATA_LOG_PERIOD_SECS):
            new_data['last_metadata_log_time'] = int(time.time())
            self.logger.info('Global metadata: %s',
                             str(stats.cluster))
            for host_id, attr in stats.hosts.iteritems():
                info_str = "{0}".format(attr)
                self.logger.info("Host %s (id %d): %s",
                                 attr.get('hostname', "<unknown hostname"),
                                 host_id, info_str)

            self.logger.info("Local (id %d): %s",
                             self.hosted_engine.host_id, str(stats.local))

        # create new data object with the newest stats at the
        # beginning, we are not interested in more than
        # 15 second history snapshots
        # namedtuple._replace makes a copy with updated values
        hlimit = stats.collect_start - constants.STATS_HISTORY_SECS
        new_data = old_data._replace(
            history=(stats,) + self.trim_history(old_data.history, hlimit),
            stats=stats,
            host_id=self.hosted_engine.host_id,

            # The rest of changes as computed by the code above
            **new_data)

        return new_data

    @staticmethod
    def trim_history(history, time_limit):
        return tuple(stat for stat in history
                     if stat.collect_start > time_limit)
<commit_msg>fixed typo when accessing the metadata dict<commit_after>__author__ = 'msivak'

from ..lib.fsm import BaseFSM, BaseState
from ..lib import util
from ..lib.util import engine_status_score
from . import constants
import time
from states import ReinitializeFSM
from state_data import HostedEngineData, StatsData
from ..lib import monotonic

class StartState(BaseState):
    """
    :transition ReinitializeFSM:
    """
    def consume(self, fsm, new_data, logger):
        return ReinitializeFSM(new_data)

    def score(self, logger):
        return 0

    def collect(self, fsm, new_data, logger):
        return new_data

    def metadata(self):
        return {}

class EngineStateMachine(BaseFSM):
    def __init__(self, he, logger, actions):
        """
        :type he: ovirt_hosted_engine_ha.agent.hosted_engine.HostedEngine
        :type logger: logging.Logger
        :type actions: dict[str, T]
        """
        initial_data = HostedEngineData(
            host_id=None,
            history=(),
            stats=None,
            timeout_start_time=None,
            engine_vm_retry_count=0,
            engine_vm_retry_time=None,
            engine_vm_shutdown_time=None,
            unexpected_shutdown_time=None,
            last_metadata_log_time=0,
            migration_host_id=None,
            migration_result=None,
            score_cfg=he.score_config,
            min_memory_threshold=he.min_memory_threshold,
            best_engine_status=None,
            best_engine_host_id=None,
            best_score_host=None,
            alive_hosts=[],
        )

        super(EngineStateMachine, self).__init__(StartState, initial_data,
                                                 logger, actions)
        self.hosted_engine = he

    def refresh(self, old_data):
        """
        Ask hosted engine to provide fresh state information and
        pre-compute all the values we need to use it easily.

        :type old_data: HostedEngineData
        :rtype: HostedEngineData
        """
        # Collect stats
        mtime = monotonic.time()
        stats = {
            "collect_start": int(mtime),
            "time_epoch": int(time.time() - mtime)
        }

        # Do not refresh if the time has not changed
        if (old_data.stats is not None and
                stats["collect_start"] == old_data.stats.collect_finish):
            return old_data

        stats.update(self.hosted_engine.collect_stats())
        stats.update({
            "collect_finish": int(monotonic.time())
        })

        # Convert to read only structure to prevent accidental changes
        stats = StatsData(**stats)

        # Check if hosts are alive using the stat history and local time
        # do not use host-ts directly as it might be wrong, only check
        # if it changed in the last HOST_ALIVE_TIMEOUT_SECS period.

        # 1) Find the first point in history that has to have older update
        # from all hosts
        historic = None
        for h in old_data.history:
            if (h.collect_finish + constants.HOST_ALIVE_TIMEOUT_SECS <
                    stats.collect_start):
                historic = h
                break

        # 2) If a currently available host is not present in the historic
        # record or its timestamp is different then it is alive
        if historic:
            for hid, st in stats.hosts.iteritems():
                st["alive"] = (hid not in historic.hosts or
                               historic.hosts[hid].get("host-ts", None)
                               != st.get("host-ts", None))

            alive_hosts = [st for hid, st in stats.hosts.iteritems()
                           if st["alive"] if "engine-status" in st]
        else:
            alive_hosts = []

        # Prepare changes to the data structure
        new_data = {}
        # save the copy of alive_hosts right after we compute it to avoid
        # any changes that might be done to the list
        new_data["alive_hosts"] = [host['host-id'] for host in alive_hosts]

        if alive_hosts:
            # Pre-compute the best remote engine (skip old metadata
            # for local host)
            best_engine = min(alive_hosts,
                              key=lambda st:
                              engine_status_score(st['engine-status']))

            # Pre-compute best remote score (skip old metadata for local host)
            best_score = min(alive_hosts,
                             key=lambda st: st['score'])

        # Compare the best engine remote values with the local state
        lm = stats.local
        if (not alive_hosts or engine_status_score(lm["engine-health"])
                >= engine_status_score(best_engine["engine-status"])):
            new_data["best_engine_status"] = lm["engine-health"]
            new_data["best_engine_host_id"] = self.hosted_engine.host_id
        else:
            new_data["best_engine_status"] = best_engine["engine-status"]
            new_data["best_engine_host_id"] = best_engine["host-id"]

        # Save the best score remote values
        # we can't compare them because we first need to fully initialize
        # the current state
        new_data["best_score_host"] = best_score if alive_hosts else None

        # Re-initialize retry status variables if the retry window
        # has expired.
        if util.has_elapsed(old_data.engine_vm_retry_time,
                            constants.ENGINE_RETRY_EXPIRATION_SECS):
            new_data['engine_vm_retry_time'] = None
            new_data['engine_vm_retry_count'] = 0
            self.logger.debug("Cleared retry status")

        # log the global status every now and then
        if util.has_elapsed(old_data.last_metadata_log_time,
                            constants.METADATA_LOG_PERIOD_SECS):
            new_data['last_metadata_log_time'] = int(time.time())
            self.logger.info('Global metadata: %s',
                             str(stats.cluster))
            for host_id, attr in stats.hosts.iteritems():
                info_str = "{0}".format(attr)
                self.logger.info("Host %s (id %d): %s",
                                 attr.get('hostname', "<unknown hostname"),
                                 host_id, info_str)

            self.logger.info("Local (id %d): %s",
                             self.hosted_engine.host_id, str(stats.local))

        # create new data object with the newest stats at the
        # beginning, we are not interested in more than
        # 15 second history snapshots
        # namedtuple._replace makes a copy with updated values
        hlimit = stats.collect_start - constants.STATS_HISTORY_SECS
        new_data = old_data._replace(
            history=(stats,) + self.trim_history(old_data.history, hlimit),
            stats=stats,
            host_id=self.hosted_engine.host_id,

            # The rest of changes as computed by the code above
            **new_data)

        return new_data

    @staticmethod
    def trim_history(history, time_limit):
        return tuple(stat for stat in history
                     if stat.collect_start > time_limit)
<|endoftext|>


<commit_before>from tasks.util import (Shp2TempTableTask, TempTableTask, TableTask, TagsTask, ColumnsTask,
                        DownloadUnzipTask, CSV2TempTableTask,
                        underscore_slugify, shell, classpath)
from tasks.meta import current_session, DENOMINATOR, GEOM_REF, UNIVERSE
from collections import OrderedDict
from luigi import IntParameter, Parameter, WrapperTask, Task, LocalTarget, ListParameter
import os
from tasks.meta import OBSTable, OBSColumn, OBSTag
from tasks.tags import SectionTags, SubsectionTags, UnitTags
from tasks.fr.communes import OutputCommuneColumns
import csv
import pandas as pd
#
# dl_code_list = "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&downfile=dic%2Fen%2F{code}.dic".format(code=code)
# flag_explanation = "http://ec.europa.eu/eurostat/data/database/information"
# database = "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?dir=data&sort=1&sort=2&start={}".format(first_letter)
# dl_data = "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&downfile=data%2F{}.tsv.gz".format(table_code)
# dl_data = "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&file=data%2Fdemo_r_pjangrp3.tsv.gz

class DownloadEurostat(Task):

    table_code = "demo_r_pjangrp3"
    URL = "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&downfile=data%2F{}.tsv.gz".format(table_code)

    def download(self):
        shell('wget -O {output}.gz "{url}"'.format(
        output=self.output().path,
        url=self.URL))

    def run(self):
        self.output().makedirs()
        self.download()
        shell('gunzip {output}.gz'.format(output=self.output().path))

    def output(self):
        return LocalTarget(os.path.join('tmp', classpath(self), self.task_id))

class EUTempTable(CSV2TempTableTask):
    delimiter = Parameter(default='\t', significant=False)
    def version(self):
        return 2

    def requires(self):
        return DownloadEurostat()

    def input_csv(self):
        shell("cat {} | tr ',' '\t' > {}.csv".format(self.input().path,self.input().path))
        return self.input().path + '.csv'

class EUFormatTable(TempTableTask):

    def requires(self):
        return EUTempTable()

    def run(self):
        pass

class FlexEurostatColumns(ColumnsTask):

    subsection = Parameter()
    tablename = Parameter()  # Ex. "DEMO_R_PJANAGGR3"

    # From tablename, determine basis of name for columns from table_dic.dic
    # Then, look at metabase.txt to find relevant dimensions (exclude "geo" and "time", utilize "unit")
    # Finally, look up definitions for dimensions from their .dic files, and use that to complete the metadata definition

    def requires(self):
        return {
            'units': UnitTags(),
            'subsection': SubsectionTags(),
            'section': SectionTags()
        }

    def columns(self):
        columns = OrderedDict()
        session = current_session()
        resp = session.execute('''
            SELECT DISTINCT {dimnames}, UNIT FROM {input}
        '''.format(
            input=self.input()['data'].table,
            dimnames=', '.join(self.dimnames)
        ))
        for row in resp:
            dimvalues = row[0:-1]
            unit = row[-1]

            input_ = self.input()

            subsectiontags = input_['subsection']
            unittags = input_['units']
            eu = input_['section']['eu']

            tags = [eu] # ?? figure out appropriate tags here from our unit info
            aggregate = 'sum' # ?? does not work for Fertility Rate, Business Demographics, Transport...
            if unit == 'NR': # ?? does not work for Euros, Fertility Rate, Business Demographics, Transport...
                tags.append(unittags['people'])
            tags.append(subsectiontags[self.subsection])

            columns['_'.join(dimvalues)] = generate_eurostat_age_gender_column('DEMO_R_PJANGRP3',
                tags, aggregate,
                OrderedDict(zip(self.dimnames, dimvalues)))
        return columns
<commit_msg>added columns task<commit_after>from tasks.util import (Shp2TempTableTask, TempTableTask, TableTask, TagsTask, ColumnsTask,
                        DownloadUnzipTask, CSV2TempTableTask,
                        underscore_slugify, shell, classpath)
from tasks.meta import current_session, DENOMINATOR, GEOM_REF, UNIVERSE
from collections import OrderedDict
from luigi import IntParameter, Parameter, WrapperTask, Task, LocalTarget, ListParameter
import os
from tasks.meta import OBSTable, OBSColumn, OBSTag
from tasks.tags import SectionTags, SubsectionTags, UnitTags
from tasks.fr.communes import OutputCommuneColumns
import csv
import pandas as pd
import itertools
#
# dl_code_list = "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&downfile=dic%2Fen%2F{code}.dic".format(code=code)
# flag_explanation = "http://ec.europa.eu/eurostat/data/database/information"
# database = "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?dir=data&sort=1&sort=2&start={}".format(first_letter)
# dl_data = "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&downfile=data%2F{}.tsv.gz".format(table_code)
# dl_data = "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&file=data%2Fdemo_r_pjangrp3.tsv.gz

class DownloadEurostat(Task):

    table_code =  Parameter()
    URL = "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&downfile=data%2F{}.tsv.gz".format(table_code)

    def download(self):
        shell('wget -O {output}.gz "{url}"'.format(
        output=self.output().path,
        url=self.URL))

    def run(self):
        self.output().makedirs()
        self.download()
        shell('gunzip {output}.gz'.format(output=self.output().path))

    def output(self):
        return LocalTarget(os.path.join('tmp', classpath(self), self.task_id))

class EUTempTable(CSV2TempTableTask):
    delimiter = Parameter(default='\t', significant=False)
    def version(self):
        return 2

    def requires(self):
        return DownloadEurostat()

    def input_csv(self):
        shell("cat {} | tr ',' '\t' > {}.csv".format(self.input().path,self.input().path))
        return self.input().path + '.csv'

class EUFormatTable(TempTableTask):

    def requires(self):
        return EUTempTable()

    def run(self):
        pass

class FlexEurostatColumns(ColumnsTask):

    subsection = Parameter()
    tablename = Parameter()  # Ex. "DEMO_R_PJANAGGR3"

    # From tablename, determine basis of name for columns from table_dic.dic
    # Then, look at metabase.txt to find relevant dimensions (exclude "geo" and "time", utilize "unit")
    # Finally, look up definitions for dimensions from their .dic files, and use that to complete the metadata definition

    def requires(self):
        return {
            'units': UnitTags(),
            'subsection': SubsectionTags(),
            'section': SectionTags()
        }

    def columns(self):
        columns = OrderedDict()

        input_ = self.input()

        subsectiontags = input_['subsection']
        unittags = input_['units'] #???
        eu = input_['section']['eu']

        tags=[eu,subsectiontags[self.subsection]]

        current_code = None
        codes = {}
        current_list = []
        with open(os.path.join(os.path.dirname(__file__),'metabase.txt')) as metabase:
            reader = csv.reader(metabase,delimiter='\t')
            for possible_tablenames, code, code_value in reader:
                if self.tablename.lower() == possible_tablenames.lower():
                    if code != "geo" and code != "unit" and code != "time":
                        if current_code == code:
                            current_list.append(code_value)
                        elif current_code:
                            codes[current_code] = current_list
                            current_list = []
                            current_list.append(code_value)
                        else:
                            current_list.append(code_value)
                        current_code = code
                    codes[current_code] = current_list

        product = [x for x in apply(itertools.product, codes.values())]
        cross_prod = [dict(zip(codes.keys(), p)) for p in product]

        with open(os.path.join(os.path.dirname(__file__),'table_dic.dic')) as tabledicfile:
            tablereader = csv.reader(tabledicfile,delimiter='\t')
            for possible_tablenames, table_description in tablereader:
                if self.tablename.lower() == possible_tablenames.lower():
                    table_desc = table_description
                    variable_name = table_description.split('by')[0].strip()
                    break

        if cross_prod:
            for i in cross_prod:
                var_code = underscore_slugify(variable_name+"_".join(i.values()))
                dimdefs = []
                for dimname, value in i.iteritems():
                    with open(os.path.join(os.path.dirname(__file__),'dic_lists/{dimension}.dic'.format(dimension=dimname))) as dimfile:
                        reader = csv.reader(dimfile, delimiter='\t')
                        for possible_dimvalue, dimdef in reader:
                            if value == possible_dimvalue:
                                dimdefs.append(dimdef)
                description = "{} ".format(variable_name)+", ".join([str(x) for x in dimdefs])

                columns[var_code] = OBSColumn(
                    id=var_code,
                    name=description,
                    type='Numeric',
                    description=table_desc,
                    weight=1,
                    aggregate=None, #???
                    targets={}, #???
                    tags=tags,
                )
        else:
            var_code = underscore_slugify(variable_name)
            description = variable_name
            columns[var_code] = OBSColumn(
                id=var_code,
                name=variable_name,
                type='Numeric',
                description= table_desc,
                weight=1,
                aggregate=None, #???
                targets={}, #???
                tags=tags,
            )

        return columns

class FranceCensus(TableTask):

    table_name = Parameter()
    subsection = Parameter()

    def version(self):
        return 8

    def timespan(self):
        return '2012'

    def requires(self):
        requirements = {
            'data': RawFRData(table_name=self.table_name),
=            'meta': FlexEurostatColumns(table_name=self.table_theme, subsection = self.subsection),
            'geometa': OutputAreaColumns(),
        }
        return requirements

    def columns(self):
        cols = OrderedDict()
        cols['IRIS'] = self.input()['geometa']['dcomiris']
        cols.update(self.input()['meta'])
        return cols

    def populate(self):
        session = current_session()

        column_targets = self.columns()
        colnames = ', '.join(column_targets.keys())
        colnames_typed = ','.join(['{}::{}'.format(colname, ct.get(session).type)
                              for colname, ct in column_targets.iteritems()])
        session.execute('INSERT INTO {output} ({ids}) '
                        'SELECT {ids_typed} '
                        'FROM {input} '.format(
                            ids=colnames,
                            ids_typed=colnames_typed,
                            output=self.output().table,
                            input=self.input()['iris_data'].table
                        ))
        session.execute('INSERT INTO {output} ({ids}) '
                        'SELECT {ids_typed} '
                        'FROM {input} '.format(
                            ids=colnames,
                            ids_typed=colnames_typed,
                            output=self.output().table,
                            input=self.input()['overseas_data'].table
                        ))
<|endoftext|>


<commit_before>import mysql.connector
import os

from ... import config

# FIXME: use proper config files
if os.environ.get('PAAS_MANAGER_ENV') == 'test':
    config['mysql']['database'] += '_test'

def db_action(fn):
    def wrapped(*args, **kwargs):
        res = fn(*args, **kwargs)
        DatabaseConnector.connect.commit()
        return res
    return wrapped

class DatabaseConnector():
    connect = mysql.connector.connect(**config['mysql'])
    cursor = connect.cursor()

    def __init__(self, **kwargs):
        self.__dict__ = kwargs
        if not hasattr(self, 'id'):
            self.id = None

    @classmethod
    def hydrate_obj(cls, values):
        if not values:
            return None
        fields = [i[0] for i in cls.cursor.description]
        obj = cls()
        for i, f in enumerate(fields):
            setattr(obj, f, values[i])
        return obj

    @classmethod
    def _val_to_cond(cls, v):
        return str(v[0]) + "=%s"

    @classmethod
    def _conditions_to_str(cls, conditions):
        return ' AND '.join(map(cls._val_to_cond, conditions.items()))

    @classmethod
    @db_action
    def _make_query(cls, conditions, options):
        if 'fields' in options:
            fields = ', '.join(options['fields'])
        else:
            fields = '*'

        if 'one' in options:
            options['limit'] = 1
        query_template = "select {fields} from {table}"
        args = ()
        if conditions:
            query_template += " where {conditions}"
            args += tuple(conditions.values())

        if 'order' in options:
            # FIXME: strange behavior with prepared query
            query_template += " order by " + options['order']

        if 'limit' in options:
            query_template += " limit %s"
            args += (options['limit'],)

        query = query_template.format(
            fields=fields,
            table=cls.table,
            conditions=cls._conditions_to_str(conditions)
        )
        cls.cursor.execute(query, args)
        if 'one' in options:
            return cls.hydrate_obj(cls.cursor.fetchone())
        else:
            return [cls.hydrate_obj(obj) for obj in cls.cursor.fetchall()]

    @classmethod
    def query(cls, **kwargs):
        options = kwargs.pop('_options', {})
        return cls._make_query(kwargs, options)

    @classmethod
    def count(cls, **kwargs):
        kwargs['_options'] = {'fields': ['id']}
        result = cls.query(**kwargs)
        return len(result)

    @classmethod
    def exists(cls, **kwargs):
        return cls.count(**kwargs) > 0

    @classmethod
    def create(cls, **kwargs):
        obj = cls(**kwargs)
        return obj.save()

    @classmethod
    def find(cls, id):
        return cls.find_by(id=id)

    @classmethod
    def find_by(cls, **kwargs):
        options = {'one': True}
        return cls._make_query(kwargs, options)

    @classmethod
    @db_action
    def remove_all(cls):
        query_template = "delete from {table}"
        query = query_template.format(table=cls.table)
        cls.cursor.execute(query)

    @classmethod
    def update_entity(cls, id, **kwargs):
        entity = cls.find(id)
        if entity:
            entity.update(**kwargs)
            return entity
        return None

    def update(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self, k, v)
        self.save()

    def is_new(self):
        return self.id is None

    def before_save(self):
        pass

    def _insert(self):
        query_template = "insert into {table} ({fields}) values ({values})"
        fields = ', '.join([str(key) for key in self.__dict__.keys()])
        values = ', '.join(['%s' for _ in self.__dict__.values()])
        query = query_template.format(
            table=self.table, fields=fields, values=values)
        self.cursor.execute(query, tuple(self.__dict__.values()))
        self.id = self.cursor.lastrowid

    def _update(self):
        query_template = "update {table} set {fields} where id=%s"
        items = self.__dict__.copy()
        items.pop('id')
        fields = ', '.join(map(self._val_to_cond, items.items()))
        query = query_template.format(table=self.table, fields=fields)
        self.cursor.execute(query, tuple(items.values()) + (self.id,))

    @db_action
    def save(self):
        self.before_save()
        if self.is_new():
            self._insert()
        else:
            self._update()
        return self

    @db_action
    def remove(self):
        query_template = "delete from {table} where id=%s"
        query = query_template.format(table=self.table)
        self.cursor.execute(query, (self.id,))

    def upload_dir(self):
        base_path = os.path.expanduser(config['app']['upload_folder'])
        class_name = self.__class__.__name__.lower()
        return os.path.join(base_path, class_name, str(self.id))
<commit_msg>Add reconnect<commit_after>import mysql.connector
import os

from ... import config

# FIXME: use proper config files
if os.environ.get('PAAS_MANAGER_ENV') == 'test':
    config['mysql']['database'] += '_test'

def db_action(fn):
    def wrapped(*args, **kwargs):
        try:
            res = fn(*args, **kwargs)
            DatabaseConnector.connect.commit()
        except (mysql.connector.InterfaceError, mysql.connector.OperationalError):
            DatabaseConnector.connect.reconnect()
            res = fn(*args, **kwargs)
            DatabaseConnector.connect.commit()
        return res
    return wrapped

class DatabaseConnector():
    connect = mysql.connector.connect(**config['mysql'])
    cursor = connect.cursor()

    def __init__(self, **kwargs):
        self.__dict__ = kwargs
        if not hasattr(self, 'id'):
            self.id = None

    @classmethod
    def hydrate_obj(cls, values):
        if not values:
            return None
        fields = [i[0] for i in cls.cursor.description]
        obj = cls()
        for i, f in enumerate(fields):
            setattr(obj, f, values[i])
        return obj

    @classmethod
    def _val_to_cond(cls, v):
        return str(v[0]) + "=%s"

    @classmethod
    def _conditions_to_str(cls, conditions):
        return ' AND '.join(map(cls._val_to_cond, conditions.items()))

    @classmethod
    @db_action
    def _make_query(cls, conditions, options):
        if 'fields' in options:
            fields = ', '.join(options['fields'])
        else:
            fields = '*'

        if 'one' in options:
            options['limit'] = 1
        query_template = "select {fields} from {table}"
        args = ()
        if conditions:
            query_template += " where {conditions}"
            args += tuple(conditions.values())

        if 'order' in options:
            # FIXME: strange behavior with prepared query
            query_template += " order by " + options['order']

        if 'limit' in options:
            query_template += " limit %s"
            args += (options['limit'],)

        query = query_template.format(
            fields=fields,
            table=cls.table,
            conditions=cls._conditions_to_str(conditions)
        )
        cls.cursor.execute(query, args)
        if 'one' in options:
            return cls.hydrate_obj(cls.cursor.fetchone())
        else:
            return [cls.hydrate_obj(obj) for obj in cls.cursor.fetchall()]

    @classmethod
    def query(cls, **kwargs):
        options = kwargs.pop('_options', {})
        return cls._make_query(kwargs, options)

    @classmethod
    def count(cls, **kwargs):
        kwargs['_options'] = {'fields': ['id']}
        result = cls.query(**kwargs)
        return len(result)

    @classmethod
    def exists(cls, **kwargs):
        return cls.count(**kwargs) > 0

    @classmethod
    def create(cls, **kwargs):
        obj = cls(**kwargs)
        return obj.save()

    @classmethod
    def find(cls, id):
        return cls.find_by(id=id)

    @classmethod
    def find_by(cls, **kwargs):
        options = {'one': True}
        return cls._make_query(kwargs, options)

    @classmethod
    @db_action
    def remove_all(cls):
        query_template = "delete from {table}"
        query = query_template.format(table=cls.table)
        cls.cursor.execute(query)

    @classmethod
    def update_entity(cls, id, **kwargs):
        entity = cls.find(id)
        if entity:
            entity.update(**kwargs)
            return entity
        return None

    def update(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self, k, v)
        self.save()

    def is_new(self):
        return self.id is None

    def before_save(self):
        pass

    def _insert(self):
        query_template = "insert into {table} ({fields}) values ({values})"
        fields = ', '.join([str(key) for key in self.__dict__.keys()])
        values = ', '.join(['%s' for _ in self.__dict__.values()])
        query = query_template.format(
            table=self.table, fields=fields, values=values)
        self.cursor.execute(query, tuple(self.__dict__.values()))
        self.id = self.cursor.lastrowid

    def _update(self):
        query_template = "update {table} set {fields} where id=%s"
        items = self.__dict__.copy()
        items.pop('id')
        fields = ', '.join(map(self._val_to_cond, items.items()))
        query = query_template.format(table=self.table, fields=fields)
        self.cursor.execute(query, tuple(items.values()) + (self.id,))

    @db_action
    def save(self):
        self.before_save()
        if self.is_new():
            self._insert()
        else:
            self._update()
        return self

    @db_action
    def remove(self):
        query_template = "delete from {table} where id=%s"
        query = query_template.format(table=self.table)
        self.cursor.execute(query, (self.id,))

    def upload_dir(self):
        base_path = os.path.expanduser(config['app']['upload_folder'])
        class_name = self.__class__.__name__.lower()
        return os.path.join(base_path, class_name, str(self.id))
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import os

from django.conf import settings
from sorl.thumbnail import get_thumbnail
from tastypie.fields import FileField

try:
    from tastypie import VERSION
except ImportError:
    from tastypie import __version__ as VERSION

class ThumbnailField(FileField):
    """
    A image thumbnail field.
    """
    dehydrated_type = 'string'
    help_text = 'A image thumbnail URL as a string. Ex: ' \
                '"http://media.example.com/media/photos/my_photo.jpg"'

    def __init__(self, attribute, geometry_string, null=False, help_text=None,
                 use_in='all', verbose_name=None, **sorl_options):
        field_options = dict(attribute=attribute, null=null, readonly=True,
                             help_text=help_text, use_in=use_in)
        if VERSION[:2] >= (0, 13):
            field_options.update({'verbose_name': verbose_name})
        super(ThumbnailField, self).__init__(**field_options)
        self.geometry_string = geometry_string
        self.sorl_options = sorl_options

    def convert(self, value):
        value = super(ThumbnailField, self).convert(value)

        if value is None:
            return None

        if value.startswith(settings.MEDIA_URL):
            value = value[len(settings.MEDIA_URL):]

        media_root = settings.MEDIA_ROOT \
            if settings.MEDIA_ROOT[-1] != os.path.sep \
            else settings.MEDIA_ROOT[:-1]
        image_path = '%s%s%s' % (media_root, os.path.sep, value)

        try:
            thumbnail = get_thumbnail(image_path, self.geometry_string,
                                      **self.sorl_options)
        except Exception:
            return None
        else:
            return thumbnail.url
<commit_msg>simplify<commit_after>from __future__ import unicode_literals

import os

from django.conf import settings
from sorl.thumbnail import get_thumbnail
from tastypie.fields import FileField

try:
    from tastypie import VERSION
except ImportError:
    from tastypie import __version__ as VERSION

class ThumbnailField(FileField):
    """
    A image thumbnail field.
    """
    dehydrated_type = 'string'
    help_text = 'A image thumbnail URL as a string. Ex: ' \
                '"http://media.example.com/media/photos/my_photo.jpg"'

    def __init__(self, attribute, geometry_string, null=False, help_text=None,
                 use_in='all', verbose_name=None, **sorl_options):
        field_options = dict(attribute=attribute, null=null, readonly=True,
                             help_text=help_text, use_in=use_in)
        if VERSION[:2] >= (0, 13):
            field_options.update({'verbose_name': verbose_name})
        super(ThumbnailField, self).__init__(**field_options)
        self.geometry_string = geometry_string
        self.sorl_options = sorl_options

    def convert(self, value):
        value = super(ThumbnailField, self).convert(value)

        if value is None:
            return None

        if value.startswith(settings.MEDIA_URL):
            value = value[len(settings.MEDIA_URL):]

        media_root = settings.MEDIA_ROOT.rstrip(os.path.sep)
        image_path = '%s%s%s' % (media_root, os.path.sep, value)

        try:
            thumbnail = get_thumbnail(image_path, self.geometry_string,
                                      **self.sorl_options)
        except Exception:
            return None
        else:
            return thumbnail.url
<|endoftext|>


<commit_before>import logging
import sys
import unittest

from tonalmodel.diatonic_tone_cache import DiatonicToneCache
from transformation.functions.tonalfunctions.tonal_permutation import TonalPermutation

class TestTonalPermutation(unittest.TestCase):
    logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_basic_tonal_permutation(self):
        domain = {'C', 'D', 'Eb', 'F#'}
        cycles = [['C', 'D'], ['Eb', 'F#']]
        p = TonalPermutation(cycles, domain)

        assert DiatonicToneCache.get_tone('C') in p.tone_domain
        assert DiatonicToneCache.get_tone('D') in p.tone_domain
        assert DiatonicToneCache.get_tone('Eb') in p.tone_domain
        assert DiatonicToneCache.get_tone('F#') in p.tone_domain

        assert DiatonicToneCache.get_tone('D') == p['C']
        assert DiatonicToneCache.get_tone('C') == p['D']
        assert DiatonicToneCache.get_tone('Eb') == p['F#']
        assert DiatonicToneCache.get_tone('F#') == p['Eb']

    def test_empty_tonal_permutation(self):
        p = TonalPermutation(None)

        assert isinstance(p.tone_domain, set)
        assert len(p.tone_domain) == 0
        assert isinstance(p.cycles, list)
        assert len(p.cycles) == 0

    def test_only_cycles(self):
        cycles = [['C', 'D'], ['Eb', 'F#']]
        p = TonalPermutation(cycles)

        assert DiatonicToneCache.get_tone('C') in p.tone_domain
        assert DiatonicToneCache.get_tone('D') in p.tone_domain
        assert DiatonicToneCache.get_tone('Eb') in p.tone_domain
        assert DiatonicToneCache.get_tone('F#') in p.tone_domain

        assert DiatonicToneCache.get_tone('D') == p['C']
        assert DiatonicToneCache.get_tone('C') == p['D']
        assert DiatonicToneCache.get_tone('Eb') == p['F#']
        assert DiatonicToneCache.get_tone('F#') == p['Eb']
<commit_msg>Take out spurious print().<commit_after>import logging
import sys
import unittest

from tonalmodel.diatonic_tone_cache import DiatonicToneCache
from transformation.functions.tonalfunctions.tonal_permutation import TonalPermutation

class TestTonalPermutation(unittest.TestCase):
    logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_basic_tonal_permutation(self):
        domain = {'C', 'D', 'Eb', 'F#'}
        cycles = [['C', 'D'], ['Eb', 'F#']]
        p = TonalPermutation(cycles, domain)
        print(p)

        assert DiatonicToneCache.get_tone('C') in p.tone_domain
        assert DiatonicToneCache.get_tone('D') in p.tone_domain
        assert DiatonicToneCache.get_tone('Eb') in p.tone_domain
        assert DiatonicToneCache.get_tone('F#') in p.tone_domain

        assert DiatonicToneCache.get_tone('D') == p['C']
        assert DiatonicToneCache.get_tone('C') == p['D']
        assert DiatonicToneCache.get_tone('Eb') == p['F#']
        assert DiatonicToneCache.get_tone('F#') == p['Eb']

    def test_empty_tonal_permutation(self):
        p = TonalPermutation(None)

        assert isinstance(p.tone_domain, set)
        assert len(p.tone_domain) == 0
        assert isinstance(p.cycles, list)
        assert len(p.cycles) == 0

    def test_only_cycles(self):
        cycles = [['C', 'D'], ['Eb', 'F#']]
        p = TonalPermutation(cycles)

        assert DiatonicToneCache.get_tone('C') in p.tone_domain
        assert DiatonicToneCache.get_tone('D') in p.tone_domain
        assert DiatonicToneCache.get_tone('Eb') in p.tone_domain
        assert DiatonicToneCache.get_tone('F#') in p.tone_domain

        assert DiatonicToneCache.get_tone('D') == p['C']
        assert DiatonicToneCache.get_tone('C') == p['D']
        assert DiatonicToneCache.get_tone('Eb') == p['F#']
        assert DiatonicToneCache.get_tone('F#') == p['Eb']
<|endoftext|>


<commit_before>"""EJDB driver for Magpy.
Currently does not do much except support certain command line scripts.
"""
import os
import pyejdb

class Database(object):
    """Simple database connection for use in serverside scripts etc."""
    def __init__(self,
                 database_name=None,
                 config_file=None):
        self.config_file = config_file
        if not database_name:
            database_name = os.path.expanduser('~/magpy.tct')
        self._database_name = database_name
        self.database = pyejdb.EJDB(database_name,
                                    pyejdb.DEFAULT_OPEN_MODE |
                                    pyejdb.JBOTRUNC)

    def get_collection(self, collection):
        """Get a collection by name."""
        return Collection(collection, self)

    def drop_collections(self, collections):
        """Drop a named tuple or list of collections."""
        for collection in collections:
            self.database.dropCollection(collection)

class Collection(object):
    """A mongodb style collection object."""
    def __init__(self, name, database):
        self.database = database
        self.name = name

    def find(self):
        """Find instances from a collection."""
        return self.database.database.find(self.name)

    def find_one(self):
        """Find a single instance from a collection."""
        raise NotImplementedError

    def save(self, instance):
        """Save an instance."""
        instance['id'] = instance.pop('_id')
        self.database.database.save(self.name, instance)
<commit_msg>Fix for ejdb _ids.<commit_after>"""EJDB driver for Magpy.
Currently does not do much except support certain command line scripts.
"""
import os
import pyejdb
from pyejdb.bson import BSON_ObjectId
from pyejdb.typecheck import InputParameterError

class Database(object):
    """Simple database connection for use in serverside scripts etc."""
    def __init__(self,
                 database_name=None,
                 config_file=None):
        self.config_file = config_file
        if not database_name:
            database_name = os.path.expanduser('~/magpy.tct')
        self._database_name = database_name
        self.database = pyejdb.EJDB(database_name,
                                    pyejdb.DEFAULT_OPEN_MODE |
                                    pyejdb.JBOTRUNC)

    def get_collection(self, collection):
        """Get a collection by name."""
        return Collection(collection, self)

    def drop_collections(self, collections):
        """Drop a named tuple or list of collections."""
        for collection in collections:
            self.database.dropCollection(collection)

class Collection(object):
    """A mongodb style collection object."""
    def __init__(self, name, database):
        self.database = database
        self.name = name

    def find(self):
        """Find instances from a collection."""
        return self.database.database.find(self.name)

    def find_one(self):
        """Find a single instance from a collection."""
        raise NotImplementedError

    def save(self, instance):
        """Save an instance."""
        # Ejdb is more specific about ids than mongodb
        if instance['_id']:
            try:
                # Check it is a valid objectid
                BSON_ObjectId(instance['_id'])
            except InputParameterError:
                # Move the _id into id
                instance['id'] = instance.pop('_id')
        self.database.database.save(self.name, instance)
<|endoftext|>


<commit_before># License: https://github.com/RobFisher/mailshare/blob/master/LICENSE

import datetime
from django.template import RequestContext, loader
from django.http import HttpResponse
from django.db.models import Q
from mailshareapp.models import Mail, Contact, Tag
import search
import email_utils
import tags

def index_view(request):
    week_search = search.get_days_search(70)
    last_week_emails = week_search.get_query_set()
    month_search = search.get_days_search(30)
    tag_cloud = tags.search_results_to_tag_cloud_html(last_week_emails, month_search)
    t = loader.get_template('index.html')
    c = RequestContext(request, {
            'tag_cloud': tag_cloud,
            'hidden_form': month_search.get_hidden_form_html(),
    })
    return HttpResponse(t.render(c))

def get_string(request, parameter):
    """Returns a string from a GET request if available, otherwise returns ''"""
    result = ''
    if parameter in request.GET:
        result = request.GET[parameter]
    return result

def get_int(request, parameter):
    """Returns an int from a GET request if available, otherwise returns -1"""
    result = -1
    if parameter in request.GET:
        try:
            result = int(request.GET[parameter])
        except ValueError:
            pass
    return result

def get_expanded_html(mail, current_search=None):
    """Return the HTML representing the expanded email."""
    expanded_html = email_utils.mail_contacts_bar_html(mail, current_search)
    expanded_html += tags.mail_tags_bar_html(mail, current_search)
    expanded_html += email_utils.mail_body_html(mail)
    return expanded_html

def search_view(request):
    search_query = get_string(request, 'query')
    tag_cloud = ''
    expanded_html = ''

    s = search.Search(request.GET.items())

    if len(s.get_query_set()) == 1:
        expanded_html = get_expanded_html(s.get_query_set()[0], s)
    elif len(s.get_query_set()) != 0:
        tag_cloud = tags.search_results_to_tag_cloud_html(s.get_query_set(), s)

    t = loader.get_template('search.html')
    c = RequestContext(request, {
        'query_name': s.get_form_query_name(),
        'hidden_form': s.get_hidden_form_html(),
        'search_html': s.get_html(),
        'tag_cloud': tag_cloud,
        'expanded_html': expanded_html,
        'results' : s.get_query_set(),
    })
    return HttpResponse(t.render(c))
<commit_msg>Fix bug introduced in previous commit: weekly tag cloud should show 7 days, not 70<commit_after># License: https://github.com/RobFisher/mailshare/blob/master/LICENSE

import datetime
from django.template import RequestContext, loader
from django.http import HttpResponse
from django.db.models import Q
from mailshareapp.models import Mail, Contact, Tag
import search
import email_utils
import tags

def index_view(request):
    week_search = search.get_days_search(7)
    last_week_emails = week_search.get_query_set()
    month_search = search.get_days_search(30)
    tag_cloud = tags.search_results_to_tag_cloud_html(last_week_emails, month_search)
    t = loader.get_template('index.html')
    c = RequestContext(request, {
            'tag_cloud': tag_cloud,
            'hidden_form': month_search.get_hidden_form_html(),
    })
    return HttpResponse(t.render(c))

def get_string(request, parameter):
    """Returns a string from a GET request if available, otherwise returns ''"""
    result = ''
    if parameter in request.GET:
        result = request.GET[parameter]
    return result

def get_int(request, parameter):
    """Returns an int from a GET request if available, otherwise returns -1"""
    result = -1
    if parameter in request.GET:
        try:
            result = int(request.GET[parameter])
        except ValueError:
            pass
    return result

def get_expanded_html(mail, current_search=None):
    """Return the HTML representing the expanded email."""
    expanded_html = email_utils.mail_contacts_bar_html(mail, current_search)
    expanded_html += tags.mail_tags_bar_html(mail, current_search)
    expanded_html += email_utils.mail_body_html(mail)
    return expanded_html

def search_view(request):
    search_query = get_string(request, 'query')
    tag_cloud = ''
    expanded_html = ''

    s = search.Search(request.GET.items())

    if len(s.get_query_set()) == 1:
        expanded_html = get_expanded_html(s.get_query_set()[0], s)
    elif len(s.get_query_set()) != 0:
        tag_cloud = tags.search_results_to_tag_cloud_html(s.get_query_set(), s)

    t = loader.get_template('search.html')
    c = RequestContext(request, {
        'query_name': s.get_form_query_name(),
        'hidden_form': s.get_hidden_form_html(),
        'search_html': s.get_html(),
        'tag_cloud': tag_cloud,
        'expanded_html': expanded_html,
        'results' : s.get_query_set(),
    })
    return HttpResponse(t.render(c))
<|endoftext|>


<commit_before>#coding: utf8

# The MIT License (MIT)
#
# Copyright (c) 2013 PengYi
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
# the Software, and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import re

class Inflection(object):
    """
    Inflector for pluralize and singularize English nouns.
    This is the default Inflector for the Inflector obj
    """
    uncountable_words = [
        'equipment', 'information', 'rice', 'money',
        'species', 'series', 'fish', 'sheep', 'sms'
    ]
    pluralize_irregular_words = [
        ['person', 'people'],
        ['man', 'men'],
        ['child', 'children'],
        ['sex', 'sexes'],
        ['move', 'moves']
    ]
    singularize_irregular_words = [ [ irregular[1], irregular[0] ] for irregular in pluralize_irregular_words]

    pluralize_rules = [
        ['(?i)(quiz)$' , '\\1zes'],
        ['^(?i)(ox)$' , '\\1en'],
        ['(?i)([m|l])ouse$' , '\\1ice'],
        ['(?i)(matr|vert|ind)ix|ex$' , '\\1ices'],
        ['(?i)(x|ch|ss|sh)$' , '\\1es'],
        ['(?i)([^aeiouy]|qu)ies$' , '\\1y'],
        ['(?i)([^aeiouy]|qu)y$' , '\\1ies'],
        ['(?i)(hive)$' , '\\1s'],
        ['(?i)(?:([^f])fe|([lr])f)$' , '\\1\\2ves'],
        ['(?i)sis$' , 'ses'],
        ['(?i)([ti])um$' , '\\1a'],
        ['(?i)(buffal|tomat)o$' , '\\1oes'],
        ['(?i)(bu)s$' , '\\1ses'],
        ['(?i)(alias|status)' , '\\1es'],
        ['(?i)(octop|vir)us$' , '\\1i'],
        ['(?i)(ax|test)is$' , '\\1es'],
        ['(?i)s$' , 's'],
        ['(?i)$' , 's']
    ]
    singularize_rules = [
        ['(?i)(quiz)zes$' , '\\1'],
        ['(?i)(matr)ices$' , '\\1ix'],
        ['(?i)(vert|ind)ices$' , '\\1ex'],
        ['(?i)^(ox)en' , '\\1'],
        ['(?i)(alias|status)es$' , '\\1'],
        ['(?i)([octop|vir])i$' , '\\1us'],
        ['(?i)(cris|ax|test)es$' , '\\1is'],
        ['(?i)(shoe)s$' , '\\1'],
        ['(?i)(o)es$' , '\\1'],
        ['(?i)(bus)es$' , '\\1'],
        ['(?i)([m|l])ice$' , '\\1ouse'],
        ['(?i)(x|ch|ss|sh)es$' , '\\1'],
        ['(?i)(m)ovies$' , '\\1ovie'],
        ['(?i)(s)eries$' , '\\1eries'],
        ['(?i)([^aeiouy]|qu)ies$' , '\\1y'],
        ['(?i)([lr])ves$' , '\\1f'],
        ['(?i)(tive)s$' , '\\1'],
        ['(?i)(hive)s$' , '\\1'],
        ['(?i)([^f])ves$' , '\\1fe'],
        ['(?i)(^analy)ses$' , '\\1sis'],
        ['(?i)((a)naly|(b)a|(d)iagno|(p)arenthe|(p)rogno|(s)ynop|(t)he)ses$' , '\\1\\2sis'],
        ['(?i)([ti])a$' , '\\1um'],
        ['(?i)(n)ews$' , '\\1ews'],
        ['(?i)s$' , ''],
    ]

    @classmethod
    def singularize(cls, word):
        '''Singularizes English nouns.'''
        return cls.__singularize_or_pluralize(word, cls.singularize_rules, cls.singularize_irregular_words)

    @classmethod
    def pluralize(cls, word):
        '''Pluralizes English nouns.'''
        return cls.__singularize_or_pluralize(word, cls.pluralize_rules, cls.pluralize_irregular_words)

    @classmethod
    def __singularize_or_pluralize(cls, word, rules, irregular_words):
        if cls.__is_uncountable(word):
            return word
        result = cls.__irregular(word, irregular_words)
        if result is not None:
            return result
        return cls.__core_deal(word, rules)

    @classmethod
    def __is_uncountable(cls, word):
        lower_cased_word = word.lower()
        for uncountable_word in cls.uncountable_words:
            if lower_cased_word.endswith(uncountable_word):
                return True
        return False

    @classmethod
    def __irregular(cls, word, irregular_words):
        for irregular, dest in irregular_words:
            match = re.search('(' + irregular + ')$', word, re.IGNORECASE)
            if match:
                return re.sub('(?i)' + irregular + '$', match.expand('\\1')[0] + dest[1:], word)
        return None

    @classmethod
    def __core_deal(cls, word, rules):
        for patten, replacement in rules:
            match = re.search(patten, word, re.IGNORECASE)
            if match :
                for k, group in enumerate(match.groups()):
                    if group == None :
                        replacement = replacement.replace('\\%s' % str(k + 1), '')
                return re.sub(patten, replacement, word)
        return word

    @classmethod
    def pascal_of(cls, name):
        """
        a_b_c       => ABC
        ab_ab_ab    => AbAbAb
        """
        def is_letter(ch):
            return 'a' <= ch <= 'z' or 'A' <= ch <= 'Z'

        def is_char(ch):
            return 'a' <= ch <= 'z' or 'A' <= ch <= 'Z' or '0' <= ch <= '9'

        name = cls.hungarian_of(name)
        pos = 0
        for i, ch in enumerate(name):
            if is_letter(ch):
                pos = i
                break

        pascal_name = []
        upper_flag = True
        for ch in name[pos:]:
            if upper_flag:
                pascal_name.append(ch.upper())
                upper_flag = False
            elif is_char(ch):
                pascal_name.append(ch)
            else:
                upper_flag = True

        return ''.join(pascal_name)

    @classmethod
    def hungarian_of(cls, name):
        """
        UserAddress  => user_address
        userAddress  => user_address
        User_Address => user_address
        user_Address => user_address
        User_address => user_address
        Useraddress  => useraddress
        """
        def append_under_line_to(hungarian_name):
            if hungarian_name[-1] != '_':
                hungarian_name.append('_')

        hungarian_name = [] # the first charactor
        for i, ch in enumerate(name):
            if i == 0: # the first charactor
                hungarian_name.append(ch.lower())
            elif ch.isupper():
                append_under_line_to(hungarian_name)
                hungarian_name.append(ch.lower())
            elif ch == '_':
                append_under_line_to(hungarian_name)
            else:
                hungarian_name.append(ch.lower())

        return ''.join(hungarian_name)

<commit_msg>rename Inflection.singularize to Inflection.singularize_of<commit_after>#coding: utf8

# The MIT License (MIT)
#
# Copyright (c) 2013 PengYi
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
# the Software, and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import re

class Inflection(object):
    """
    Inflector for pluralize and singularize English nouns.
    This is the default Inflector for the Inflector obj
    """
    uncountable_words = [
        'equipment', 'information', 'rice', 'money',
        'species', 'series', 'fish', 'sheep', 'sms'
    ]
    pluralize_irregular_words = [
        ['person', 'people'],
        ['man', 'men'],
        ['child', 'children'],
        ['sex', 'sexes'],
        ['move', 'moves']
    ]
    singularize_irregular_words = [ [ irregular[1], irregular[0] ] for irregular in pluralize_irregular_words]

    pluralize_rules = [
        ['(?i)(quiz)$' , '\\1zes'],
        ['^(?i)(ox)$' , '\\1en'],
        ['(?i)([m|l])ouse$' , '\\1ice'],
        ['(?i)(matr|vert|ind)ix|ex$' , '\\1ices'],
        ['(?i)(x|ch|ss|sh)$' , '\\1es'],
        ['(?i)([^aeiouy]|qu)ies$' , '\\1y'],
        ['(?i)([^aeiouy]|qu)y$' , '\\1ies'],
        ['(?i)(hive)$' , '\\1s'],
        ['(?i)(?:([^f])fe|([lr])f)$' , '\\1\\2ves'],
        ['(?i)sis$' , 'ses'],
        ['(?i)([ti])um$' , '\\1a'],
        ['(?i)(buffal|tomat)o$' , '\\1oes'],
        ['(?i)(bu)s$' , '\\1ses'],
        ['(?i)(alias|status)' , '\\1es'],
        ['(?i)(octop|vir)us$' , '\\1i'],
        ['(?i)(ax|test)is$' , '\\1es'],
        ['(?i)s$' , 's'],
        ['(?i)$' , 's']
    ]
    singularize_rules = [
        ['(?i)(quiz)zes$' , '\\1'],
        ['(?i)(matr)ices$' , '\\1ix'],
        ['(?i)(vert|ind)ices$' , '\\1ex'],
        ['(?i)^(ox)en' , '\\1'],
        ['(?i)(alias|status)es$' , '\\1'],
        ['(?i)([octop|vir])i$' , '\\1us'],
        ['(?i)(cris|ax|test)es$' , '\\1is'],
        ['(?i)(shoe)s$' , '\\1'],
        ['(?i)(o)es$' , '\\1'],
        ['(?i)(bus)es$' , '\\1'],
        ['(?i)([m|l])ice$' , '\\1ouse'],
        ['(?i)(x|ch|ss|sh)es$' , '\\1'],
        ['(?i)(m)ovies$' , '\\1ovie'],
        ['(?i)(s)eries$' , '\\1eries'],
        ['(?i)([^aeiouy]|qu)ies$' , '\\1y'],
        ['(?i)([lr])ves$' , '\\1f'],
        ['(?i)(tive)s$' , '\\1'],
        ['(?i)(hive)s$' , '\\1'],
        ['(?i)([^f])ves$' , '\\1fe'],
        ['(?i)(^analy)ses$' , '\\1sis'],
        ['(?i)((a)naly|(b)a|(d)iagno|(p)arenthe|(p)rogno|(s)ynop|(t)he)ses$' , '\\1\\2sis'],
        ['(?i)([ti])a$' , '\\1um'],
        ['(?i)(n)ews$' , '\\1ews'],
        ['(?i)s$' , ''],
    ]

    @classmethod
    def singularize_of(cls, word):
        '''Singularizes English nouns.'''
        return cls.__singularize_or_pluralize(word, cls.singularize_rules, cls.singularize_irregular_words)

    @classmethod
    def pluralize(cls, word):
        '''Pluralizes English nouns.'''
        return cls.__singularize_or_pluralize(word, cls.pluralize_rules, cls.pluralize_irregular_words)

    @classmethod
    def __singularize_or_pluralize(cls, word, rules, irregular_words):
        if cls.__is_uncountable(word):
            return word
        result = cls.__irregular(word, irregular_words)
        if result is not None:
            return result
        return cls.__core_deal(word, rules)

    @classmethod
    def __is_uncountable(cls, word):
        lower_cased_word = word.lower()
        for uncountable_word in cls.uncountable_words:
            if lower_cased_word.endswith(uncountable_word):
                return True
        return False

    @classmethod
    def __irregular(cls, word, irregular_words):
        for irregular, dest in irregular_words:
            match = re.search('(' + irregular + ')$', word, re.IGNORECASE)
            if match:
                return re.sub('(?i)' + irregular + '$', match.expand('\\1')[0] + dest[1:], word)
        return None

    @classmethod
    def __core_deal(cls, word, rules):
        for patten, replacement in rules:
            match = re.search(patten, word, re.IGNORECASE)
            if match :
                for k, group in enumerate(match.groups()):
                    if group == None :
                        replacement = replacement.replace('\\%s' % str(k + 1), '')
                return re.sub(patten, replacement, word)
        return word

    @classmethod
    def pascal_of(cls, name):
        """
        a_b_c       => ABC
        ab_ab_ab    => AbAbAb
        """
        def is_letter(ch):
            return 'a' <= ch <= 'z' or 'A' <= ch <= 'Z'

        def is_char(ch):
            return 'a' <= ch <= 'z' or 'A' <= ch <= 'Z' or '0' <= ch <= '9'

        name = cls.hungarian_of(name)
        pos = 0
        for i, ch in enumerate(name):
            if is_letter(ch):
                pos = i
                break

        pascal_name = []
        upper_flag = True
        for ch in name[pos:]:
            if upper_flag:
                pascal_name.append(ch.upper())
                upper_flag = False
            elif is_char(ch):
                pascal_name.append(ch)
            else:
                upper_flag = True

        return ''.join(pascal_name)

    @classmethod
    def hungarian_of(cls, name):
        """
        UserAddress  => user_address
        userAddress  => user_address
        User_Address => user_address
        user_Address => user_address
        User_address => user_address
        Useraddress  => useraddress
        """
        def append_under_line_to(hungarian_name):
            if hungarian_name[-1] != '_':
                hungarian_name.append('_')

        hungarian_name = [] # the first charactor
        for i, ch in enumerate(name):
            if i == 0: # the first charactor
                hungarian_name.append(ch.lower())
            elif ch.isupper():
                append_under_line_to(hungarian_name)
                hungarian_name.append(ch.lower())
            elif ch == '_':
                append_under_line_to(hungarian_name)
            else:
                hungarian_name.append(ch.lower())

        return ''.join(hungarian_name)

<|endoftext|>


<commit_before>from django.core.management.base import BaseCommand

import gevent

from corehq.apps.auditcare.tasks import copy_events_to_sql
from corehq.apps.auditcare.utils.migration import AuditCareMigrationUtil

class Command(BaseCommand):
    help = """Copy audit data from couch to sql"""

    def add_arguments(self, parser):
        parser.add_argument(
            '--workers',
            default=1,
            type=int,
            help="Number of concurrent workers, defaults to 1"
        )
        parser.add_argument(
            '--batch_by',
            default='h',
            choices=['h', 'd'],
            help="Execution can be batched in hours or days. Defaults to hour"
        )
        parser.add_argument(
            '--only_errored',
            default=False,
            type=bool,
            choices=[True, False],
            help="Will try to process batches that have been errored"
        )

    def handle(self, **options):
        workers = options['workers']
        batch_by = options['batch_by']
        util = AuditCareMigrationUtil()
        batches = []
        while True:
            if options['only_errored']:
                if util.total_errored_keys() > 0:
                    batches = util.get_errored_keys(5)
                else:
                    print("No errored keys present")
                    exit(1)
            else:
                batches = self.generate_batches(workers, batch_by)
            batched_processes = [gevent.spawn(copy_events_to_sql, *batch) for batch in batches]
            gevent.joinall([*batched_processes])

        #count = copy_events_to_sql(int(limit))

    def generate_batches(self, worker_count, batch_by):
        migration_util = AuditCareMigrationUtil()
        start_datetime = migration_util.get_next_batch_start()
        migration_util.acquire_read_lock()
        if not start_datetime:
            # for the first call
            start_datetime = INITIAL_START_DATE

        start_time = _get_formatted_start_time(start_datetime, batch_by)
        end_time = None
        batches = []
        for index in range(worker_count):
            end_time = _get_end_time(start_time, batch_by)
            batches.append([start_time, end_time])
            start_time = end_time
        migration_util.set_next_batch_start(end_time)
        migration_util.release_read_lock()
        if end_time > datetime.now():
            print("Migration successfully done")
            exit(1)
        return batches

def _get_end_time(start_time, batch_by):
    delta = timedelta(hours=1) if batch_by == 'h' else timedelta(days=1)
    return start_time + delta

def _get_formatted_start_time(start_time, batch_by):
    if batch_by == 'h':
        return start_time.replace(minute=0, second=0, microsecond=0)
    else:
        return start_time.replace(hour=0, minute=0, second=0, microsecond=0)
<commit_msg>use generate_batches from util<commit_after>from django.core.management.base import BaseCommand

import gevent

from corehq.apps.auditcare.tasks import copy_events_to_sql
from corehq.apps.auditcare.utils.migration import AuditCareMigrationUtil

class Command(BaseCommand):
    help = """Copy audit data from couch to sql"""

    def add_arguments(self, parser):
        parser.add_argument(
            '--workers',
            default=1,
            type=int,
            help="Number of concurrent workers, defaults to 1"
        )
        parser.add_argument(
            '--batch_by',
            default='h',
            choices=['h', 'd'],
            help="Execution can be batched in hours or days. Defaults to hour"
        )
        parser.add_argument(
            '--only_errored',
            default=False,
            type=bool,
            choices=[True, False],
            help="Will try to process batches that have been errored"
        )

    def handle(self, **options):
        workers = options['workers']
        batch_by = options['batch_by']
        util = AuditCareMigrationUtil()
        batches = []
        while True:
            if options['only_errored']:
                if util.total_errored_keys() > 0:
                    batches = util.get_errored_keys(5)
                else:
                    print("No errored keys present")
                    exit(1)
            else:
                batches = util.generate_batches(workers, batch_by)
            batched_processes = [gevent.spawn(copy_events_to_sql, *batch) for batch in batches]
            gevent.joinall([*batched_processes])

        #count = copy_events_to_sql(int(limit))

    def generate_batches(self, worker_count, batch_by):
        migration_util = AuditCareMigrationUtil()
        start_datetime = migration_util.get_next_batch_start()
        migration_util.acquire_read_lock()
        if not start_datetime:
            # for the first call
            start_datetime = INITIAL_START_DATE

        start_time = _get_formatted_start_time(start_datetime, batch_by)
        end_time = None
        batches = []
        for index in range(worker_count):
            end_time = _get_end_time(start_time, batch_by)
            batches.append([start_time, end_time])
            start_time = end_time
        migration_util.set_next_batch_start(end_time)
        migration_util.release_read_lock()
        if end_time > datetime.now():
            print("Migration successfully done")
            exit(1)
        return batches

def _get_end_time(start_time, batch_by):
    delta = timedelta(hours=1) if batch_by == 'h' else timedelta(days=1)
    return start_time + delta

def _get_formatted_start_time(start_time, batch_by):
    if batch_by == 'h':
        return start_time.replace(minute=0, second=0, microsecond=0)
    else:
        return start_time.replace(hour=0, minute=0, second=0, microsecond=0)
<|endoftext|>


<commit_before>import json
import os
import zipfile

from django.core.management.base import BaseCommand, CommandError

from corehq.apps.dump_reload.couch.load import (
    CouchDataLoader,
    DomainLoader,
    ToggleLoader,
)
from corehq.apps.dump_reload.exceptions import DataExistsException
from corehq.apps.dump_reload.sql import SqlDataLoader

# Domain loader should be first
LOADERS = [DomainLoader, SqlDataLoader, CouchDataLoader, ToggleLoader]

class Command(BaseCommand):
    help = 'Loads data from the give file into the database.'

    def add_arguments(self, parser):
        parser.add_argument('dump_file_path')
        parser.add_argument('--use-extracted', action='store_true', default=False, dest='use_extracted',
                            help="Use already extracted dump if it exists.")
        parser.add_argument('--force', action='store_true', default=False, dest='force',
                            help="Load data for domain that already exists.")
        parser.add_argument('--dry-run', action='store_true', default=False, dest='dry_run',
                            help="Skip saving data to the DB")
        parser.add_argument('--loader', dest='loaders', action='append', default=[],
                            help='loader slug to run (use multiple --loader to run multiple loaders).'
                                 'domain should always be the first loader to be invoked in case of '
                                 'very first import',
                            choices=[loader.slug for loader in LOADERS])
        parser.add_argument('--skip', dest='skip', action='append', default=[],
                            help='Skip over the first n objects for the specified loader.  '
                            '`--skip=sql:1000` skips the first 1000 objects in the sql dump')
        parser.add_argument('--object-filter',
                            help="Regular expression to use to selectively load data. Will be matched"
                                 " against a CouchDB 'doc_type' or Django model name: 'app_label.ModelName'."
                                 "Use 'print_domain_stats' command to get a list of available types.")

    def handle(self, dump_file_path, **options):
        self.force = options.get('force')
        self.dry_run = options.get('dry_run')
        self.use_extracted = options.get('use_extracted')
        skip = {slug: int(count) for slug, count in
                (s.split(':') for s in options.get('skip'))}

        if not os.path.isfile(dump_file_path):
            raise CommandError("Dump file not found: {}".format(dump_file_path))

        self.stdout.write("Loading data from %s." % dump_file_path)
        extracted_dir = self.extract_dump_archive(dump_file_path)

        loaded_meta = {}
        loaders = options.get('loaders')
        object_filter = options.get('object_filter')
        if loaders:
            loaders = [loader for loader in LOADERS if loader.slug in loaders]
        else:
            loaders = LOADERS

        dump_meta = _get_dump_meta(extracted_dir)
        for loader in loaders:
            loaded_meta.update(self._load_data(
                loader, extracted_dir, object_filter, skip.get(loader.slug), dump_meta
            ))

        self._print_stats(loaded_meta, dump_meta)
        return json.dumps(loaded_meta)

    def _print_stats(self, loaded_meta, dump_meta):
        self.stdout.write('{0} Load Stats {0}'.format('-' * 40))
        for loader, models in sorted(loaded_meta.items()):
            self.stdout.write(loader)
            for model, count in sorted(models.items()):
                expected = dump_meta[loader].get(model, 0)
                self.stdout.write(f"  {model:<50}: {count} / {expected}")
        self.stdout.write('{0}{0}'.format('-' * 46))
        loaded_object_count = sum(count for model in loaded_meta.values() for count in model.values())
        total_object_count = sum(count for model in dump_meta.values() for count in model.values())
        self.stdout.write(f'Loaded {loaded_object_count}/{total_object_count} objects')
        self.stdout.write('{0}{0}'.format('-' * 46))

    def extract_dump_archive(self, dump_file_path):
        target_dir = '_tmp_load_{}'.format(dump_file_path)
        if not os.path.exists(target_dir):
            with zipfile.ZipFile(dump_file_path, 'r') as archive:
                archive.extractall(target_dir)
        elif not self.use_extracted:
            raise CommandError(
                "Extracted dump already exists at {}. Delete it or use --use-extracted".format(target_dir))
        return target_dir

    def _load_data(self, loader_class, extracted_dump_path, object_filter, skip, dump_meta):
        try:
            loader = loader_class(object_filter, skip, self.stdout, self.stderr)
            return loader.load_from_path(extracted_dump_path, dump_meta, force=self.force, dry_run=self.dry_run)
        except DataExistsException as e:
            raise CommandError('Some data already exists. Use --force to load anyway: {}'.format(str(e)))
        except Exception as e:
            if not isinstance(e, CommandError):
                e.args = ("Problem loading data '%s': %s" % (extracted_dump_path, e),)
            raise

def _get_dump_meta(extracted_dir):
    # The dump command should have a metadata json file of the form
    # {dumper_slug: {model_name: count}}
    meta_path = os.path.join(extracted_dir, 'meta.json')
    with open(meta_path) as f:
        return json.loads(f.read())
<commit_msg>argument to output JSON<commit_after>import json
import os
import zipfile

from django.core.management.base import BaseCommand, CommandError

from corehq.apps.dump_reload.couch.load import (
    CouchDataLoader,
    DomainLoader,
    ToggleLoader,
)
from corehq.apps.dump_reload.exceptions import DataExistsException
from corehq.apps.dump_reload.sql import SqlDataLoader

# Domain loader should be first
LOADERS = [DomainLoader, SqlDataLoader, CouchDataLoader, ToggleLoader]

class Command(BaseCommand):
    help = 'Loads data from the give file into the database.'

    def add_arguments(self, parser):
        parser.add_argument('dump_file_path')
        parser.add_argument('--use-extracted', action='store_true', default=False, dest='use_extracted',
                            help="Use already extracted dump if it exists.")
        parser.add_argument('--force', action='store_true', default=False, dest='force',
                            help="Load data for domain that already exists.")
        parser.add_argument('--dry-run', action='store_true', default=False, dest='dry_run',
                            help="Skip saving data to the DB")
        parser.add_argument('--loader', dest='loaders', action='append', default=[],
                            help='loader slug to run (use multiple --loader to run multiple loaders).'
                                 'domain should always be the first loader to be invoked in case of '
                                 'very first import',
                            choices=[loader.slug for loader in LOADERS])
        parser.add_argument('--skip', dest='skip', action='append', default=[],
                            help='Skip over the first n objects for the specified loader.  '
                            '`--skip=sql:1000` skips the first 1000 objects in the sql dump')
        parser.add_argument('--object-filter',
                            help="Regular expression to use to selectively load data. Will be matched"
                                 " against a CouchDB 'doc_type' or Django model name: 'app_label.ModelName'."
                                 "Use 'print_domain_stats' command to get a list of available types.")
        parser.add_argument('--json-output', action="store_true", help="Produce JSON output for use in tests")

    def handle(self, dump_file_path, **options):
        self.force = options.get('force')
        self.dry_run = options.get('dry_run')
        self.use_extracted = options.get('use_extracted')
        skip = {slug: int(count) for slug, count in
                (s.split(':') for s in options.get('skip'))}

        if not os.path.isfile(dump_file_path):
            raise CommandError("Dump file not found: {}".format(dump_file_path))

        self.stdout.write("Loading data from %s." % dump_file_path)
        extracted_dir = self.extract_dump_archive(dump_file_path)

        loaded_meta = {}
        loaders = options.get('loaders')
        object_filter = options.get('object_filter')
        if loaders:
            loaders = [loader for loader in LOADERS if loader.slug in loaders]
        else:
            loaders = LOADERS

        dump_meta = _get_dump_meta(extracted_dir)
        for loader in loaders:
            loaded_meta.update(self._load_data(
                loader, extracted_dir, object_filter, skip.get(loader.slug), dump_meta
            ))

        if options.get("json_output"):
            return json.dumps(loaded_meta)
        else:
            self._print_stats(loaded_meta, dump_meta)

    def _print_stats(self, loaded_meta, dump_meta):
        self.stdout.write('{0} Load Stats {0}'.format('-' * 40))
        for loader, models in sorted(loaded_meta.items()):
            self.stdout.write(loader)
            for model, count in sorted(models.items()):
                expected = dump_meta[loader].get(model, 0)
                self.stdout.write(f"  {model:<50}: {count} / {expected}")
        self.stdout.write('{0}{0}'.format('-' * 46))
        loaded_object_count = sum(count for model in loaded_meta.values() for count in model.values())
        total_object_count = sum(count for model in dump_meta.values() for count in model.values())
        self.stdout.write(f'Loaded {loaded_object_count}/{total_object_count} objects')
        self.stdout.write('{0}{0}'.format('-' * 46))

    def extract_dump_archive(self, dump_file_path):
        target_dir = '_tmp_load_{}'.format(dump_file_path)
        if not os.path.exists(target_dir):
            with zipfile.ZipFile(dump_file_path, 'r') as archive:
                archive.extractall(target_dir)
        elif not self.use_extracted:
            raise CommandError(
                "Extracted dump already exists at {}. Delete it or use --use-extracted".format(target_dir))
        return target_dir

    def _load_data(self, loader_class, extracted_dump_path, object_filter, skip, dump_meta):
        try:
            loader = loader_class(object_filter, skip, self.stdout, self.stderr)
            return loader.load_from_path(extracted_dump_path, dump_meta, force=self.force, dry_run=self.dry_run)
        except DataExistsException as e:
            raise CommandError('Some data already exists. Use --force to load anyway: {}'.format(str(e)))
        except Exception as e:
            if not isinstance(e, CommandError):
                e.args = ("Problem loading data '%s': %s" % (extracted_dump_path, e),)
            raise

def _get_dump_meta(extracted_dir):
    # The dump command should have a metadata json file of the form
    # {dumper_slug: {model_name: count}}
    meta_path = os.path.join(extracted_dir, 'meta.json')
    with open(meta_path) as f:
        return json.loads(f.read())
<|endoftext|>