<commit_before>from math import ceil
import simplejson as json
import uuid
from time import gmtime, strftime

from celery.task import task
from celery.contrib.methods import task as class_task
from pandas import concat

from bamboo.config.settings import DB_READ_BATCH_SIZE
from bamboo.core.calculator import Calculator
from bamboo.core.frame import BambooFrame, BAMBOO_RESERVED_KEY_PREFIX,\
    DATASET_ID, DATASET_OBSERVATION_ID, PARENT_DATASET_ID
from bamboo.core.summary import summarize
from bamboo.lib.exceptions import ArgumentError
from bamboo.lib.mongo import reserve_encoded
from bamboo.lib.schema_builder import Schema, schema_from_data_and_dtypes
from bamboo.lib.utils import call_async, split_groups
from bamboo.models.abstract_model import AbstractModel
from bamboo.models.calculation import Calculation
from bamboo.models.observation import Observation

@task
def delete_task(dataset):
    """Background task to delete dataset and its associated observations."""
    Observation.delete_all(dataset)
    super(dataset.__class__, dataset).delete({DATASET_ID: dataset.dataset_id})

class Dataset(AbstractModel):

    __collectionname__ = 'datasets'

    # caching keys
    STATS = '_stats'
    ALL = '_all'

    # metadata
    AGGREGATED_DATASETS = BAMBOO_RESERVED_KEY_PREFIX + 'linked_datasets'
    ATTRIBUTION = 'attribution'
    CARDINALITY = 'cardinality'
    CREATED_AT = 'created_at'
    DESCRIPTION = 'description'
    ID = 'id'
    JOINED_DATASETS = 'joined_datasets'
    LABEL = 'label'
    LICENSE = 'license'
    NUM_COLUMNS = 'num_columns'
    NUM_ROWS = 'num_rows'
    MERGED_DATASETS = 'merged_datasets'
    SCHEMA = 'schema'
    UPDATED_AT = 'updated_at'

    # commonly accessed variables
    @property
    def dataset_observation_id(self):
        return self.record[DATASET_OBSERVATION_ID]

    @property
    def dataset_id(self):
        return self.record[DATASET_ID]

    @property
    def num_columns(self):
        return self.record.get(self.NUM_COLUMNS, 0)

    @property
    def num_rows(self):
        return self.record.get(self.NUM_ROWS, 0)

    @property
    def schema(self):
        schema_dict = self.record.get(self.SCHEMA)
        return Schema(schema_dict) if schema_dict else None

    @property
    def stats(self):
        return self.record.get(self.STATS, {})

    @property
    def aggregated_datasets_dict(self):
        return self.record.get(self.AGGREGATED_DATASETS, {})

    @property
    def aggregated_datasets(self):
        return {group: self.find_one(_id) for (group, _id) in
                self.aggregated_datasets_dict.items()}

    @property
    def joined_dataset_ids(self):
        return [
            tuple(_list) for _list in self.record.get(self.JOINED_DATASETS, [])
        ]

    @property
    def joined_datasets(self):
        result = []
        # TODO: fetch all datasets in single DB call
        return [
            (direction, self.find_one(other_dataset_id), on,
             self.find_one(joined_dataset_id))
            for direction, other_dataset_id, on, joined_dataset_id in
            self.joined_dataset_ids]

    @property
    def merged_dataset_ids(self):
        return self.record.get(self.MERGED_DATASETS, [])

    @property
    def merged_datasets(self):
        return self._linked_datasets(self.merged_dataset_ids)

    def _linked_datasets(self, ids):
        return [self.find_one(_id) for _id in ids]

    def is_factor(self, col):
        return self.schema.is_dimension(col)

    def cardinality(self, col):
        if self.is_factor(col):
            return self.schema[col][self.CARDINALITY]

    def dframe(self, query=None, select=None, keep_parent_ids=False,
               limit=0, order_by=None, padded=False):
        """Fetch the dframe for this dataset.

        :param select: An optional select to limit the fields in the dframe.
        :param keep_parent_ids: Do not remove parent IDs from the dframe,
            default False.
        :param limit: Limit on the number of rows in the returned dframe.
        :param order_by: Sort resulting rows according to a column value and
            sign indicating ascending or descending.

        Example of `order_by`:

          - ``order_by='mycolumn'``
          - ``order_by='-mycolumn'``

        :returns: Return BambooFrame with contents based on query parameters
            passed to MongoDB. BambooFrame will not have parent ids if
            `keep_parent_ids` is False.
        """
        observations = self.observations(
            query=query, select=select, limit=limit, order_by=order_by,
            as_cursor=True)

        batches = int(ceil(float(observations.count(with_limit_and_skip=True))
                      / DB_READ_BATCH_SIZE))
        dframes = []

        for batch in xrange(0, batches):
            start = batch * DB_READ_BATCH_SIZE
            end = (batch + 1) * DB_READ_BATCH_SIZE
            if limit > 0 and end > limit:
                end = limit
            dframes.append(BambooFrame([ob for ob in observations[start:end]]))
            observations.rewind()

        dframe = BambooFrame(concat(dframes) if len(dframes) else [])
        dframe.decode_mongo_reserved_keys()
        dframe.remove_bamboo_reserved_keys(keep_parent_ids)

        if padded:
            if len(dframe.columns):
                on = dframe.columns[0]
                place_holder = self.place_holder_dframe(dframe).set_index(on)
                dframe = BambooFrame(dframe.join(place_holder, on=on))
            else:
                dframe = self.place_holder_dframe()

        return dframe

    def add_joined_dataset(self, new_data):
        """Add the ID of `new_dataset` to the list of joined datasets."""
        self._add_linked_data(self.JOINED_DATASETS, self.joined_dataset_ids,
                              new_data)

    def add_merged_dataset(self, new_dataset):
        """Add the ID of `new_dataset` to the list of merged datasets."""
        self._add_linked_data(self.MERGED_DATASETS, self.merged_dataset_ids,
                              new_dataset.dataset_id)

    def _add_linked_data(self, link_key, existing_data, new_data):
        self.update({link_key: existing_data + [new_data]})

    def clear_summary_stats(self, field=ALL):
        """Remove summary stats for `field`."""
        stats = self.stats

        if stats:
            stats.pop(field, None)
            self.update({self.STATS: stats})

    def save(self, dataset_id=None):
        """Store dataset with `dataset_id` as the unique internal ID.

        Store a new dataset with an ID given by `dataset_id` is exists,
        otherwise reate a random UUID for this dataset. Additionally, set the
        created at time to the current time and the state to pending.

        :param dataset_id: The ID to store for this dataset, default is None.

        :returns: A dict representing this dataset.
        """
        if dataset_id is None:
            dataset_id = uuid.uuid4().hex

        record = {
            DATASET_ID: dataset_id,
            DATASET_OBSERVATION_ID: uuid.uuid4().hex,
            self.AGGREGATED_DATASETS: {},
            self.CREATED_AT: strftime("%Y-%m-%d %H:%M:%S", gmtime()),
            self.STATE: self.STATE_PENDING,
        }

        return super(self.__class__, self).save(record)

    def delete(self):
        """Delete this dataset."""
        call_async(delete_task, self)

    @class_task
    def summarize(self, query=None, select=None,
                  group_str=None, limit=0, order_by=None):
        """Build and return a summary of the data in this dataset.

        Return a summary of the rows/values filtered by `query` and `select`
        and grouped by `group_str`, or the overall summary if no group is
        specified.

        :param query: An optional MongoDB query to limit the data summarized.
        :param select: An optional select to limit the columns summarized.
        :param group_str: A column in the dataset as a string or a list comma
          separated columns to group on.

        :returns: A JSON summary of the dataset. Numeric columns will be
            summarized by the arithmetic mean, standard deviation, and
            percentiles. Dimensional columns will be summarized by counts.
        """
        # interpret none as all
        if not group_str:
            group_str = self.ALL

        # split group in case of multigroups
        groups = split_groups(group_str)

        # if select append groups to select
        if select:
            select = json.loads(select)
            if not isinstance(select, dict):
                raise ArgumentError('select argument must be a JSON dictionary'
                                    ', found: %s.' % select)
            select.update(dict(zip(groups, [1] * len(groups))))
            select = json.dumps(select)

        dframe = self.dframe(query=query, select=select,
                             limit=limit, order_by=order_by)

        return summarize(self, dframe, groups, group_str, query or select)

    @classmethod
    def find_one(cls, dataset_id):
        """Return dataset for `dataset_id`."""
        return super(cls, cls).find_one({DATASET_ID: dataset_id})

    @classmethod
    def find(cls, dataset_id):
        """Return datasets for `dataset_id`."""
        return super(cls, cls).find({DATASET_ID: dataset_id})

    def update(self, record):
        """Update dataset `dataset` with `record`."""
        record[self.UPDATED_AT] = strftime("%Y-%m-%d %H:%M:%S", gmtime())
        super(self.__class__, self).update(record)

    def build_schema(self, dframe, overwrite=False, set_num_columns=True):
        """Build schema for a dataset.

        If no schema exists, build a schema from the passed `dframe` and store
        that schema for this dataset.  Otherwise, if a schema does exist, build
        a schema for the passed `dframe` and merge this schema with the current
        schema.  Keys in the new schema replace keys in the current schema but
        keys in the current schema not in the new schema are retained.

        If `set_num_columns` is True the number of columns will be set to the
        number of keys (columns) in the new schema.

        :param dframe: The DataFrame whose schema to merge with the current
            schema.
        :param set_num_columns: If True also set the number of columns.
        """
        current_schema = self.schema
        new_schema = schema_from_data_and_dtypes(self, dframe)
        if current_schema and not overwrite:
            # merge new schema with existing schema
            current_schema.update(new_schema)
            new_schema = current_schema
        self.set_schema(new_schema,
                        set_num_columns=(set_num_columns or overwrite))

    def set_schema(self, schema, set_num_columns=True):
        """Set the schema from an existing one."""
        update_dict = {self.SCHEMA: schema}
        if set_num_columns:
            update_dict.update({self.NUM_COLUMNS: len(schema.keys())})
        self.update(update_dict)

    def info(self):
        """Return meta-data for this dataset."""
        return {
            self.ID: self.dataset_id,
            self.LABEL: '',
            self.DESCRIPTION: '',
            self.SCHEMA: self.schema,
            self.LICENSE: '',
            self.ATTRIBUTION: '',
            self.CREATED_AT: self.record.get(self.CREATED_AT),
            self.UPDATED_AT: self.record.get(self.UPDATED_AT),
            self.NUM_COLUMNS: self.num_columns,
            self.NUM_ROWS: self.num_rows,
        }

    def build_labels_to_slugs(self):
        """Build dict from column labels to slugs."""
        return {
            column_attrs[self.LABEL]: reserve_encoded(column_name) for
            (column_name, column_attrs) in self.schema.items()}

    def observations(self, query=None, select=None, limit=0, order_by=None,
                     as_cursor=False):
        """Return observations for this dataset.

        :param query: Optional query for MongoDB to limit rows returned.
        :param select: Optional select for MongoDB to limit columns.
        :param limit: If greater than 0, limit number of observations returned
            to this maximum.
        :param order_by: Order the returned observations.
        """
        return Observation.find(self, query, select, limit=limit,
                                order_by=order_by, as_cursor=as_cursor)

    def calculations(self):
        """Return the calculations for this dataset."""
        return Calculation.find(self)

    def remove_parent_observations(self, parent_id):
        """Remove obervations for this dataset with the passed `parent_id`.

        Args:

        :param parent_id: Remove observations with this ID as their parent
            dataset ID.
        """
        Observation.delete_all(self, {PARENT_DATASET_ID: parent_id})

    def add_observations(self, json_data):
        """Update `dataset` with new `data`."""
        calculator = Calculator(self)
        call_async(calculator.calculate_updates, calculator,
                   json.loads(json_data))

    def save_observations(self, dframe):
        """Save rows in `dframe` for this dataset."""
        Observation().save(dframe, self)
        return self.dframe()

    def replace_observations(self, dframe, overwrite=False,
                             set_num_columns=True):
        """Remove all rows for this dataset and save the rows in `dframe`.

        :param dframe: Replace rows in this dataset with this DataFrame's rows.

        :returns: BambooFrame equivalent to the passed in `dframe`.
        """
        self.build_schema(dframe, overwrite=overwrite,
                          set_num_columns=set_num_columns)
        Observation.delete_all(self)
        return self.save_observations(dframe)

    def drop_columns(self, columns):
        """Remove columns from this dataset's observations.

        :param columns: List of columns to remove from this dataset.
        """
        dframe = self.dframe(keep_parent_ids=True)
        self.replace_observations(dframe.drop(columns, axis=1))

    def place_holder_dframe(self, dframe=None):
        columns = self.schema.keys()
        if dframe is not None:
            columns = [col for col in columns if col not in dframe.columns[1:]]

        return BambooFrame([[''] * len(columns)], columns=columns)

    def join(self, other, on):
        merged_dframe = self.dframe()

        if not len(merged_dframe.columns):
            # Empty dataset, simulate columns
            merged_dframe = self.place_holder_dframe()

        merged_dframe = merged_dframe.join_dataset(other, on)
        merged_dataset = self.__class__()
        merged_dataset.save()
        if self.num_rows and other.num_rows:
            merged_dataset.save_observations(merged_dframe)
        else:
            merged_dataset.build_schema(merged_dframe, set_num_columns=True)
            merged_dataset.ready()

        self.add_joined_dataset(
            ('right', other.dataset_id, on, merged_dataset.dataset_id))
        other.add_joined_dataset(
            ('left', self.dataset_id, on, merged_dataset.dataset_id))

        return merged_dataset

    def reload(self):
        self.record = Dataset.find_one(self.dataset_id).record
        return self
<commit_msg>check that {} is not None<commit_after>from math import ceil
import simplejson as json
import uuid
from time import gmtime, strftime

from celery.task import task
from celery.contrib.methods import task as class_task
from pandas import concat

from bamboo.config.settings import DB_READ_BATCH_SIZE
from bamboo.core.calculator import Calculator
from bamboo.core.frame import BambooFrame, BAMBOO_RESERVED_KEY_PREFIX,\
    DATASET_ID, DATASET_OBSERVATION_ID, PARENT_DATASET_ID
from bamboo.core.summary import summarize
from bamboo.lib.exceptions import ArgumentError
from bamboo.lib.mongo import reserve_encoded
from bamboo.lib.schema_builder import Schema, schema_from_data_and_dtypes
from bamboo.lib.utils import call_async, split_groups
from bamboo.models.abstract_model import AbstractModel
from bamboo.models.calculation import Calculation
from bamboo.models.observation import Observation

@task
def delete_task(dataset):
    """Background task to delete dataset and its associated observations."""
    Observation.delete_all(dataset)
    super(dataset.__class__, dataset).delete({DATASET_ID: dataset.dataset_id})

class Dataset(AbstractModel):

    __collectionname__ = 'datasets'

    # caching keys
    STATS = '_stats'
    ALL = '_all'

    # metadata
    AGGREGATED_DATASETS = BAMBOO_RESERVED_KEY_PREFIX + 'linked_datasets'
    ATTRIBUTION = 'attribution'
    CARDINALITY = 'cardinality'
    CREATED_AT = 'created_at'
    DESCRIPTION = 'description'
    ID = 'id'
    JOINED_DATASETS = 'joined_datasets'
    LABEL = 'label'
    LICENSE = 'license'
    NUM_COLUMNS = 'num_columns'
    NUM_ROWS = 'num_rows'
    MERGED_DATASETS = 'merged_datasets'
    SCHEMA = 'schema'
    UPDATED_AT = 'updated_at'

    # commonly accessed variables
    @property
    def dataset_observation_id(self):
        return self.record[DATASET_OBSERVATION_ID]

    @property
    def dataset_id(self):
        return self.record[DATASET_ID]

    @property
    def num_columns(self):
        return self.record.get(self.NUM_COLUMNS, 0)

    @property
    def num_rows(self):
        return self.record.get(self.NUM_ROWS, 0)

    @property
    def schema(self):
        schema_dict = self.record.get(self.SCHEMA)
        return None if schema_dict is None else Schema(schema_dict)

    @property
    def stats(self):
        return self.record.get(self.STATS, {})

    @property
    def aggregated_datasets_dict(self):
        return self.record.get(self.AGGREGATED_DATASETS, {})

    @property
    def aggregated_datasets(self):
        return {group: self.find_one(_id) for (group, _id) in
                self.aggregated_datasets_dict.items()}

    @property
    def joined_dataset_ids(self):
        return [
            tuple(_list) for _list in self.record.get(self.JOINED_DATASETS, [])
        ]

    @property
    def joined_datasets(self):
        result = []
        # TODO: fetch all datasets in single DB call
        return [
            (direction, self.find_one(other_dataset_id), on,
             self.find_one(joined_dataset_id))
            for direction, other_dataset_id, on, joined_dataset_id in
            self.joined_dataset_ids]

    @property
    def merged_dataset_ids(self):
        return self.record.get(self.MERGED_DATASETS, [])

    @property
    def merged_datasets(self):
        return self._linked_datasets(self.merged_dataset_ids)

    def _linked_datasets(self, ids):
        return [self.find_one(_id) for _id in ids]

    def is_factor(self, col):
        return self.schema.is_dimension(col)

    def cardinality(self, col):
        if self.is_factor(col):
            return self.schema[col][self.CARDINALITY]

    def dframe(self, query=None, select=None, keep_parent_ids=False,
               limit=0, order_by=None, padded=False):
        """Fetch the dframe for this dataset.

        :param select: An optional select to limit the fields in the dframe.
        :param keep_parent_ids: Do not remove parent IDs from the dframe,
            default False.
        :param limit: Limit on the number of rows in the returned dframe.
        :param order_by: Sort resulting rows according to a column value and
            sign indicating ascending or descending.

        Example of `order_by`:

          - ``order_by='mycolumn'``
          - ``order_by='-mycolumn'``

        :returns: Return BambooFrame with contents based on query parameters
            passed to MongoDB. BambooFrame will not have parent ids if
            `keep_parent_ids` is False.
        """
        observations = self.observations(
            query=query, select=select, limit=limit, order_by=order_by,
            as_cursor=True)

        batches = int(ceil(float(observations.count(with_limit_and_skip=True))
                      / DB_READ_BATCH_SIZE))
        dframes = []

        for batch in xrange(0, batches):
            start = batch * DB_READ_BATCH_SIZE
            end = (batch + 1) * DB_READ_BATCH_SIZE
            if limit > 0 and end > limit:
                end = limit
            dframes.append(BambooFrame([ob for ob in observations[start:end]]))
            observations.rewind()

        dframe = BambooFrame(concat(dframes) if len(dframes) else [])
        dframe.decode_mongo_reserved_keys()
        dframe.remove_bamboo_reserved_keys(keep_parent_ids)

        if padded:
            if len(dframe.columns):
                on = dframe.columns[0]
                place_holder = self.place_holder_dframe(dframe).set_index(on)
                dframe = BambooFrame(dframe.join(place_holder, on=on))
            else:
                dframe = self.place_holder_dframe()

        return dframe

    def add_joined_dataset(self, new_data):
        """Add the ID of `new_dataset` to the list of joined datasets."""
        self._add_linked_data(self.JOINED_DATASETS, self.joined_dataset_ids,
                              new_data)

    def add_merged_dataset(self, new_dataset):
        """Add the ID of `new_dataset` to the list of merged datasets."""
        self._add_linked_data(self.MERGED_DATASETS, self.merged_dataset_ids,
                              new_dataset.dataset_id)

    def _add_linked_data(self, link_key, existing_data, new_data):
        self.update({link_key: existing_data + [new_data]})

    def clear_summary_stats(self, field=ALL):
        """Remove summary stats for `field`."""
        stats = self.stats

        if stats:
            stats.pop(field, None)
            self.update({self.STATS: stats})

    def save(self, dataset_id=None):
        """Store dataset with `dataset_id` as the unique internal ID.

        Store a new dataset with an ID given by `dataset_id` is exists,
        otherwise reate a random UUID for this dataset. Additionally, set the
        created at time to the current time and the state to pending.

        :param dataset_id: The ID to store for this dataset, default is None.

        :returns: A dict representing this dataset.
        """
        if dataset_id is None:
            dataset_id = uuid.uuid4().hex

        record = {
            DATASET_ID: dataset_id,
            DATASET_OBSERVATION_ID: uuid.uuid4().hex,
            self.AGGREGATED_DATASETS: {},
            self.CREATED_AT: strftime("%Y-%m-%d %H:%M:%S", gmtime()),
            self.STATE: self.STATE_PENDING,
        }

        return super(self.__class__, self).save(record)

    def delete(self):
        """Delete this dataset."""
        call_async(delete_task, self)

    @class_task
    def summarize(self, query=None, select=None,
                  group_str=None, limit=0, order_by=None):
        """Build and return a summary of the data in this dataset.

        Return a summary of the rows/values filtered by `query` and `select`
        and grouped by `group_str`, or the overall summary if no group is
        specified.

        :param query: An optional MongoDB query to limit the data summarized.
        :param select: An optional select to limit the columns summarized.
        :param group_str: A column in the dataset as a string or a list comma
          separated columns to group on.

        :returns: A JSON summary of the dataset. Numeric columns will be
            summarized by the arithmetic mean, standard deviation, and
            percentiles. Dimensional columns will be summarized by counts.
        """
        # interpret none as all
        if not group_str:
            group_str = self.ALL

        # split group in case of multigroups
        groups = split_groups(group_str)

        # if select append groups to select
        if select:
            select = json.loads(select)
            if not isinstance(select, dict):
                raise ArgumentError('select argument must be a JSON dictionary'
                                    ', found: %s.' % select)
            select.update(dict(zip(groups, [1] * len(groups))))
            select = json.dumps(select)

        dframe = self.dframe(query=query, select=select,
                             limit=limit, order_by=order_by)

        return summarize(self, dframe, groups, group_str, query or select)

    @classmethod
    def find_one(cls, dataset_id):
        """Return dataset for `dataset_id`."""
        return super(cls, cls).find_one({DATASET_ID: dataset_id})

    @classmethod
    def find(cls, dataset_id):
        """Return datasets for `dataset_id`."""
        return super(cls, cls).find({DATASET_ID: dataset_id})

    def update(self, record):
        """Update dataset `dataset` with `record`."""
        record[self.UPDATED_AT] = strftime("%Y-%m-%d %H:%M:%S", gmtime())
        super(self.__class__, self).update(record)

    def build_schema(self, dframe, overwrite=False, set_num_columns=True):
        """Build schema for a dataset.

        If no schema exists, build a schema from the passed `dframe` and store
        that schema for this dataset.  Otherwise, if a schema does exist, build
        a schema for the passed `dframe` and merge this schema with the current
        schema.  Keys in the new schema replace keys in the current schema but
        keys in the current schema not in the new schema are retained.

        If `set_num_columns` is True the number of columns will be set to the
        number of keys (columns) in the new schema.

        :param dframe: The DataFrame whose schema to merge with the current
            schema.
        :param set_num_columns: If True also set the number of columns.
        """
        current_schema = self.schema
        new_schema = schema_from_data_and_dtypes(self, dframe)
        if current_schema and not overwrite:
            # merge new schema with existing schema
            current_schema.update(new_schema)
            new_schema = current_schema
        self.set_schema(new_schema,
                        set_num_columns=(set_num_columns or overwrite))

    def set_schema(self, schema, set_num_columns=True):
        """Set the schema from an existing one."""
        update_dict = {self.SCHEMA: schema}
        if set_num_columns:
            update_dict.update({self.NUM_COLUMNS: len(schema.keys())})
        self.update(update_dict)

    def info(self):
        """Return meta-data for this dataset."""
        return {
            self.ID: self.dataset_id,
            self.LABEL: '',
            self.DESCRIPTION: '',
            self.SCHEMA: self.schema,
            self.LICENSE: '',
            self.ATTRIBUTION: '',
            self.CREATED_AT: self.record.get(self.CREATED_AT),
            self.UPDATED_AT: self.record.get(self.UPDATED_AT),
            self.NUM_COLUMNS: self.num_columns,
            self.NUM_ROWS: self.num_rows,
        }

    def build_labels_to_slugs(self):
        """Build dict from column labels to slugs."""
        return {
            column_attrs[self.LABEL]: reserve_encoded(column_name) for
            (column_name, column_attrs) in self.schema.items()}

    def observations(self, query=None, select=None, limit=0, order_by=None,
                     as_cursor=False):
        """Return observations for this dataset.

        :param query: Optional query for MongoDB to limit rows returned.
        :param select: Optional select for MongoDB to limit columns.
        :param limit: If greater than 0, limit number of observations returned
            to this maximum.
        :param order_by: Order the returned observations.
        """
        return Observation.find(self, query, select, limit=limit,
                                order_by=order_by, as_cursor=as_cursor)

    def calculations(self):
        """Return the calculations for this dataset."""
        return Calculation.find(self)

    def remove_parent_observations(self, parent_id):
        """Remove obervations for this dataset with the passed `parent_id`.

        Args:

        :param parent_id: Remove observations with this ID as their parent
            dataset ID.
        """
        Observation.delete_all(self, {PARENT_DATASET_ID: parent_id})

    def add_observations(self, json_data):
        """Update `dataset` with new `data`."""
        calculator = Calculator(self)
        call_async(calculator.calculate_updates, calculator,
                   json.loads(json_data))

    def save_observations(self, dframe):
        """Save rows in `dframe` for this dataset."""
        Observation().save(dframe, self)
        return self.dframe()

    def replace_observations(self, dframe, overwrite=False,
                             set_num_columns=True):
        """Remove all rows for this dataset and save the rows in `dframe`.

        :param dframe: Replace rows in this dataset with this DataFrame's rows.

        :returns: BambooFrame equivalent to the passed in `dframe`.
        """
        self.build_schema(dframe, overwrite=overwrite,
                          set_num_columns=set_num_columns)
        Observation.delete_all(self)
        return self.save_observations(dframe)

    def drop_columns(self, columns):
        """Remove columns from this dataset's observations.

        :param columns: List of columns to remove from this dataset.
        """
        dframe = self.dframe(keep_parent_ids=True)
        self.replace_observations(dframe.drop(columns, axis=1))

    def place_holder_dframe(self, dframe=None):
        columns = self.schema.keys()
        if dframe is not None:
            columns = [col for col in columns if col not in dframe.columns[1:]]

        return BambooFrame([[''] * len(columns)], columns=columns)

    def join(self, other, on):
        merged_dframe = self.dframe()

        if not len(merged_dframe.columns):
            # Empty dataset, simulate columns
            merged_dframe = self.place_holder_dframe()

        merged_dframe = merged_dframe.join_dataset(other, on)
        merged_dataset = self.__class__()
        merged_dataset.save()
        if self.num_rows and other.num_rows:
            merged_dataset.save_observations(merged_dframe)
        else:
            merged_dataset.build_schema(merged_dframe, set_num_columns=True)
            merged_dataset.ready()

        self.add_joined_dataset(
            ('right', other.dataset_id, on, merged_dataset.dataset_id))
        other.add_joined_dataset(
            ('left', self.dataset_id, on, merged_dataset.dataset_id))

        return merged_dataset

    def reload(self):
        self.record = Dataset.find_one(self.dataset_id).record
        return self
<|endoftext|>


<commit_before><commit_msg>Blocked level validation on inactive sessions<commit_after><|endoftext|>


<commit_before>import config
import logging
import time
from modules.loggingFunctions import initialize_logging
from modules.turtleGrapher.turtle_utils import generate_uri as gu
from modules.groupComparisons.sparql_utils import generate_prefixes
from modules.groupComparisons.decorators import toset, tolist, tostring, prefix, submit

# logging
log_file = initialize_logging()
log = logging.getLogger(__name__)

@tostring
@submit
@prefix
def query_single_objectid(relation, attribute):
    '''
    Grabs a single object id having the relation.
    '''
    query = """
    SELECT ?s WHERE {{
        ?s <{relation}> '{attribute}' .
    }}
    LIMIT 1
    """.format(relation=relation,attribute=attribute)
    return query

@tolist
@submit
@prefix
def query_objecttypes(uri):
    '''
    Grabs the types of a given uri.
    '''
    query = """
    SELECT ?s WHERE {{
        <{uri}> a ?s .
    }}
    """.format(uri=uri)
    return query

def directlink_spfyid(relation, attribute):
    '''
    Tells you if a given relation-attribute pair has a direct link to a given spfyId.
    This is required for generating more complex SPARQL queries.
    '''
    objectid = query_single_objectid(relation, attribute)
    print objectid
    objectypes = query_objecttypes(objectid)
    print objectypes
    return unicode(gu(':spfyId')) in objectypes

def resolve_spfyids(relation, attribute):
    '''
    Args:
        relation: ex. "http://purl.obolibrary.org/obo/GENEPIO_0001076"
        attribute: ex. "O136"
    Ret:
    '''
    print directlink_spfyid(relation, attribute)

if __name__ == "__main__":
    resolve_spfyids(gu('ge:0001076'), 'O157')
<commit_msg>DEBUG: spfyids?<commit_after>import config
import logging
import time
from modules.loggingFunctions import initialize_logging
from modules.turtleGrapher.turtle_utils import generate_uri as gu
from modules.groupComparisons.sparql_utils import generate_prefixes
from modules.groupComparisons.decorators import toset, tolist, tostring, prefix, submit

# logging
log_file = initialize_logging()
log = logging.getLogger(__name__)

@tostring
@submit
@prefix
def query_objectids(relation, attribute):
    '''
    Grabs all objectids having the relation.
    '''
    query = """
    SELECT ?s WHERE {{
        ?s <{relation}> '{attribute}' .
    }}
    """.format(relation=relation,attribute=attribute)
    return query

@tostring
@submit
@prefix
def query_single_objectid(relation, attribute):
    '''
    Grabs a single object id having the relation.
    '''
    query = """
    SELECT ?s WHERE {{
        ?s <{relation}> '{attribute}' .
    }}
    LIMIT 1
    """.format(relation=relation,attribute=attribute)
    return query

@tolist
@submit
@prefix
def query_objecttypes(uri):
    '''
    Grabs the types of a given uri.
    '''
    query = """
    SELECT ?s WHERE {{
        <{uri}> a ?s .
    }}
    """.format(uri=uri)
    return query

def directlink_spfyid(relation, attribute):
    '''
    Tells you if a given relation-attribute pair has a direct link to a given spfyId.
    This is required for generating more complex SPARQL queries.
    '''
    objectid = query_single_objectid(relation, attribute)
    print objectid
    objectypes = query_objecttypes(objectid)
    print objectypes
    return unicode(gu(':spfyId')) in objectypes

def resolve_spfyids(relation, attribute):
    '''
    Args:
        relation: ex. "http://purl.obolibrary.org/obo/GENEPIO_0001076"
        attribute: ex. "O136"
    Ret:
    '''
    print directlink_spfyid(relation, attribute)
    if directlink_spfyid(relation, attribute):
        # if we have a direct link to a spfyid, we can generate automatically.
        spfyids = query_objectids(relation, attribute)
        return spfyids

if __name__ == "__main__":
    resolve_spfyids(gu('ge:0001076'), 'O157')
<|endoftext|>


<commit_before>from flask import Blueprint
from flask import current_app
from flask import json
from flask import request, jsonify

from app.errors import InvalidRequest, register_errors
from app.notifications.process_client_response import validate_callback_data, process_sms_client_response

sms_callback_blueprint = Blueprint("sms_callback", __name__, url_prefix="/notifications/sms")
register_errors(sms_callback_blueprint)

@sms_callback_blueprint.route('/mmg', methods=['POST'])
def process_mmg_response():
    client_name = 'MMG'
    data = json.loads(request.data)
    errors = validate_callback_data(data=data,
                                    fields=['status', 'CID'],
                                    client_name=client_name)
    if errors:
        raise InvalidRequest(errors, status_code=400)

    success, errors = process_sms_client_response(status=str(data.get('status')),
                                                  reference=data.get('CID'),
                                                  client_name=client_name)

    safe_to_log = data.copy()
    safe_to_log.pop("MSISDN")
    current_app.logger.info(
        "Full delivery response from {} for notification: {}\n{}".format(client_name, request.form.get('CID'),
                                                                        safe_to_log))
    if errors:
        raise InvalidRequest(errors, status_code=400)
    else:
        return jsonify(result='success', message=success), 200

@sms_callback_blueprint.route('/firetext', methods=['POST'])
def process_firetext_response():
    client_name = 'Firetext'
    errors = validate_callback_data(data=request.form,
                                    fields=['status', 'reference'],
                                    client_name=client_name)
    if errors:
        raise InvalidRequest(errors, status_code=400)
    safe_to_log = dict(request.form).copy()
    safe_to_log.pop('mobile')
    current_app.logger.info(
        "Full delivery response from {} for notification: {}\n{}".format(client_name, request.form.get('reference'),
                                                                         safe_to_log))
    success, errors = process_sms_client_response(status=request.form.get('status'),
                                                  reference=request.form.get('reference'),
                                                  client_name=client_name)
    if errors:
        raise InvalidRequest(errors, status_code=400)
    else:
        return jsonify(result='success', message=success), 200
<commit_msg>Fix code style<commit_after>from flask import Blueprint
from flask import current_app
from flask import json
from flask import request, jsonify

from app.errors import InvalidRequest, register_errors
from app.notifications.process_client_response import validate_callback_data, process_sms_client_response

sms_callback_blueprint = Blueprint("sms_callback", __name__, url_prefix="/notifications/sms")
register_errors(sms_callback_blueprint)

@sms_callback_blueprint.route('/mmg', methods=['POST'])
def process_mmg_response():
    client_name = 'MMG'
    data = json.loads(request.data)
    errors = validate_callback_data(data=data,
                                    fields=['status', 'CID'],
                                    client_name=client_name)
    if errors:
        raise InvalidRequest(errors, status_code=400)

    success, errors = process_sms_client_response(status=str(data.get('status')),
                                                  reference=data.get('CID'),
                                                  client_name=client_name)

    safe_to_log = data.copy()
    safe_to_log.pop("MSISDN")
    current_app.logger.info(
        "Full delivery response from {} for notification: {}\n{}".format(client_name, request.form.get('CID'),
                                                                         safe_to_log))
    if errors:
        raise InvalidRequest(errors, status_code=400)
    else:
        return jsonify(result='success', message=success), 200

@sms_callback_blueprint.route('/firetext', methods=['POST'])
def process_firetext_response():
    client_name = 'Firetext'
    errors = validate_callback_data(data=request.form,
                                    fields=['status', 'reference'],
                                    client_name=client_name)
    if errors:
        raise InvalidRequest(errors, status_code=400)
    safe_to_log = dict(request.form).copy()
    safe_to_log.pop('mobile')
    current_app.logger.info(
        "Full delivery response from {} for notification: {}\n{}".format(client_name, request.form.get('reference'),
                                                                         safe_to_log))
    success, errors = process_sms_client_response(status=request.form.get('status'),
                                                  reference=request.form.get('reference'),
                                                  client_name=client_name)
    if errors:
        raise InvalidRequest(errors, status_code=400)
    else:
        return jsonify(result='success', message=success), 200
<|endoftext|>


<commit_before>from level import FakeChunk
import logging
from materials import pocketMaterials
from mclevelbase import ChunkNotPresent, notclosing
from nbt import TAG_List
from numpy import array, fromstring, zeros
import os
import struct

# values are usually little-endian, unlike Minecraft PC

logger = logging.getLogger(__file__)

class PocketChunksFile(object):
    holdFileOpen = False  # if False, reopens and recloses the file on each access
    SECTOR_BYTES = 4096
    CHUNK_HEADER_SIZE = 4

    @property
    def file(self):
        openfile = lambda: file(self.path, "rb+")
        if PocketChunksFile.holdFileOpen:
            if self._file is None:
                self._file = openfile()
            return notclosing(self._file)
        else:
            return openfile()

    def close(self):
        if PocketChunksFile.holdFileOpen:
            self._file.close()
            self._file = None

    def __init__(self, path):
        self.path = path
        self._file = None
        if not os.path.exists(path):
            file(path, "w").close()

        with self.file as f:

            filesize = os.path.getsize(path)
            if filesize & 0xfff:
                filesize = (filesize | 0xfff) + 1
                f.truncate(filesize)

            if filesize == 0:
                filesize = self.SECTOR_BYTES
                f.truncate(filesize)

            f.seek(0)
            offsetsData = f.read(self.SECTOR_BYTES)

            self.freeSectors = [True] * (filesize / self.SECTOR_BYTES)
            self.freeSectors[0] = False

            self.offsets = fromstring(offsetsData, dtype='<u4')

        needsRepair = False

        for index, offset in enumerate(self.offsets):
            sector = offset >> 8
            count = offset & 0xff

            for i in xrange(sector, sector + count):
                if i >= len(self.freeSectors):
                    # raise RegionMalformed("Region file offset table points to sector {0} (past the end of the file)".format(i))
                    print  "Region file offset table points to sector {0} (past the end of the file)".format(i)
                    needsRepair = True
                    break
                if self.freeSectors[i] is False:
                    logger.debug("Double-allocated sector number %s (offset %s @ %s)", i, offset, index)
                    needsRepair = True
                self.freeSectors[i] = False

        if needsRepair:
            self.repair()

        logger.info("Found region file {file} with {used}/{total} sectors used and {chunks} chunks present".format(
             file=os.path.basename(path), used=self.usedSectors, total=self.sectorCount, chunks=self.chunkCount))

    @property
    def usedSectors(self):
        return len(self.freeSectors) - sum(self.freeSectors)

    @property
    def sectorCount(self):
        return len(self.freeSectors)

    @property
    def chunkCount(self):
        return sum(self.offsets > 0)

    def repair(self):
        pass
#        lostAndFound = {}
#        _freeSectors = [True] * len(self.freeSectors)
#        _freeSectors[0] = _freeSectors[1] = False
#        deleted = 0
#        recovered = 0
#        logger.info("Beginning repairs on {file} ({chunks} chunks)".format(file=os.path.basename(self.path), chunks=sum(self.offsets > 0)))
#        rx, rz = self.regionCoords
#        for index, offset in enumerate(self.offsets):
#            if offset:
#                cx = index & 0x1f
#                cz = index >> 5
#                cx += rx << 5
#                cz += rz << 5
#                sectorStart = offset >> 8
#                sectorCount = offset & 0xff
#                try:
#
#                    if sectorStart + sectorCount > len(self.freeSectors):
#                        raise RegionMalformed("Offset {start}:{end} ({offset}) at index {index} pointed outside of the file".format()
#                            start=sectorStart, end=sectorStart + sectorCount, index=index, offset=offset)
#
#                    compressedData = self._readChunk(cx, cz)
#                    if compressedData is None:
#                        raise RegionMalformed("Failed to read chunk data for {0}".format((cx, cz)))
#
#                    format, data = self.decompressSectors(compressedData)
#                    chunkTag = nbt.load(buf=data)
#                    lev = chunkTag["Level"]
#                    xPos = lev["xPos"].value
#                    zPos = lev["zPos"].value
#                    overlaps = False
#
#                    for i in xrange(sectorStart, sectorStart + sectorCount):
#                        if _freeSectors[i] is False:
#                            overlaps = True
#                        _freeSectors[i] = False
#
#
#                    if xPos != cx or zPos != cz or overlaps:
#                        lostAndFound[xPos, zPos] = (format, compressedData)
#
#                        if (xPos, zPos) != (cx, cz):
#                            raise RegionMalformed("Chunk {found} was found in the slot reserved for {expected}".format(found=(xPos, zPos), expected=(cx, cz)))
#                        else:
#                            raise RegionMalformed("Chunk {found} (in slot {expected}) has overlapping sectors with another chunk!".format(found=(xPos, zPos), expected=(cx, cz)))
#
#
#
#                except Exception, e:
#                    logger.info("Unexpected chunk data at sector {sector} ({exc})".format(sector=sectorStart, exc=e))
#                    self.setOffset(cx, cz, 0)
#                    deleted += 1
#
#        for cPos, (format, foundData) in lostAndFound.iteritems():
#            cx, cz = cPos
#            if self.getOffset(cx, cz) == 0:
#                logger.info("Found chunk {found} and its slot is empty, recovering it".format(found=cPos))
#                self._saveChunk(cx, cz, foundData[5:], format)
#                recovered += 1
#
#        logger.info("Repair complete. Removed {0} chunks, recovered {1} chunks, net {2}".format(deleted, recovered, recovered - deleted))
#

    def _readChunk(self, cx, cz):
        cx &= 0x1f
        cz &= 0x1f
        offset = self.getOffset(cx, cz)
        if offset == 0:
            return None

        sectorStart = offset >> 8
        numSectors = offset & 0xff
        if numSectors == 0:
            return None

        if sectorStart + numSectors > len(self.freeSectors):
            return None

        with self.file as f:
            f.seek(sectorStart * self.SECTOR_BYTES)
            data = f.read(numSectors * self.SECTOR_BYTES)
        assert(len(data) > 0)
        logger.debug("REGION LOAD %s,%s sector %s", cx, cz, sectorStart)
        return data

    def loadChunk(self, cx, cz, world):
        data = self._readChunk(cx, cz)
        if data is None:
            raise ChunkNotPresent((cx, cz, self))

        chunk = PocketChunk(cx, cz, data[4:], world)
        return chunk

    def saveChunk(self, chunk):
        cx, cz = chunk.chunkPosition

        cx &= 0x1f
        cz &= 0x1f
        offset = self.getOffset(cx, cz)
        sectorNumber = offset >> 8
        sectorsAllocated = offset & 0xff

        data = chunk._savedData()

        sectorsNeeded = (len(data) + self.CHUNK_HEADER_SIZE) / self.SECTOR_BYTES + 1
        if sectorsNeeded >= 256:
            return

        if sectorNumber != 0 and sectorsAllocated >= sectorsNeeded:
            logger.debug("REGION SAVE {0},{1} rewriting {2}b".format(cx, cz, len(data)))
            self.writeSector(sectorNumber, data, format)
        else:
            # we need to allocate new sectors

            # mark the sectors previously used for this chunk as free
            for i in xrange(sectorNumber, sectorNumber + sectorsAllocated):
                self.freeSectors[i] = True

            runLength = 0
            try:
                runStart = self.freeSectors.index(True)

                for i in range(runStart, len(self.freeSectors)):
                    if runLength:
                        if self.freeSectors[i]:
                            runLength += 1
                        else:
                            runLength = 0
                    elif self.freeSectors[i]:
                        runStart = i
                        runLength = 1

                    if runLength >= sectorsNeeded:
                        break
            except ValueError:
                pass

            # we found a free space large enough
            if runLength >= sectorsNeeded:
                logger.debug("REGION SAVE {0},{1}, reusing {2}b".format(cx, cz, len(data)))
                sectorNumber = runStart
                self.setOffset(cx, cz, sectorNumber << 8 | sectorsNeeded)
                self.writeSector(sectorNumber, data, format)
                self.freeSectors[sectorNumber:sectorNumber + sectorsNeeded] = [False] * sectorsNeeded

            else:
                # no free space large enough found -- we need to grow the
                # file

                logger.debug("REGION SAVE {0},{1}, growing by {2}b".format(cx, cz, len(data)))

                with self.file as f:
                    f.seek(0, 2)
                    filesize = f.tell()

                    sectorNumber = len(self.freeSectors)

                    assert sectorNumber * self.SECTOR_BYTES == filesize

                    filesize += sectorsNeeded * self.SECTOR_BYTES
                    f.truncate(filesize)

                self.freeSectors += [False] * sectorsNeeded

                self.setOffset(cx, cz, sectorNumber << 8 | sectorsNeeded)
                self.writeSector(sectorNumber, data, format)

    def writeSector(self, sectorNumber, data, format):
        with self.file as f:
            logger.debug("REGION: Writing sector {0}".format(sectorNumber))

            f.seek(sectorNumber * self.SECTOR_BYTES)
            f.write(struct.pack("<I", len(data) + self.CHUNK_HEADER_SIZE))  # // chunk length
            f.write(data)  # // chunk data
            # f.flush()

    def containsChunk(self, cx, cz):
        return self.getOffset(cx, cz) != 0

    def getOffset(self, cx, cz):
        cx &= 0x1f
        cz &= 0x1f
        return self.offsets[cx + cz * 32]

    def setOffset(self, cx, cz, offset):
        cx &= 0x1f
        cz &= 0x1f
        self.offsets[cx + cz * 32] = offset
        with self.file as f:
            f.seek(0)
            f.write(self.offsets.tostring())

    def chunkCoords(self):
        indexes = (i for (i, offset) in enumerate(self.offsets) if offset)
        coords = ((i % 32, i // 32) for i in indexes)
        return coords

from infiniteworld import ChunkedLevelMixin
from level import MCLevel, LightedChunk

class PocketWorld(ChunkedLevelMixin, MCLevel):
    Height = 128
    Length = 512
    Width = 512

    isInfinite = True  # Wrong. isInfinite actually means 'isChunked' and should be changed
    materials = pocketMaterials

    @property
    def allChunks(self):
        return list(self.chunkFile.chunkCoords())

    def __init__(self, filename):
        if not os.path.isdir(filename):
            filename = os.path.dirname(filename)
        self.filename = filename
        self.dimensions = {}

        self.chunkFile = PocketChunksFile(os.path.join(filename, "chunks.dat"))
        self._loadedChunks = {}

    def getChunk(self, cx, cz):
        for p in cx, cz:
            if not 0 <= p <= 31:
                raise ChunkNotPresent((cx, cz, self))

        c = self._loadedChunks.get((cx, cz))
        if c is None:
            c = self.chunkFile.loadChunk(cx, cz, self)
            self._loadedChunks[cx, cz] = c
        return c

    @classmethod
    def _isLevel(cls, filename):
        clp = ("chunks.dat", "level.dat")

        if not os.path.isdir(filename):
            f = os.path.basename(filename)
            if f not in clp:
                return False
            filename = os.path.dirname(filename)

        return all([os.path.exists(os.path.join(filename, f)) for f in clp])

    def saveInPlace(self):
        for chunk in self._loadedChunks.itervalues():
            if chunk.dirty:
                self.chunkFile.saveChunk(chunk)
                chunk.dirty = False

    def containsChunk(self, cx, cz):
        if cx > 31 or cz > 31 or cx < 0 or cz < 0:
            return False
        return self.chunkFile.getOffset(cx, cz) != 0

class PocketChunk(LightedChunk):
    Blocks = Data = SkyLight = BlockLight = None

    HeightMap = FakeChunk.HeightMap

    Entities = TileEntities = property(lambda self: TAG_List())

    dirty = False
    filename = "chunks.dat"

    def __init__(self, cx, cz, data, world):
        self.chunkPosition = (cx, cz)
        self.world = world
        data = fromstring(data, dtype='uint8')

        self.Blocks, data = data[:32768], data[32768:]
        self.Data, data = data[:16384], data[16384:]
        self.SkyLight, data = data[:16384], data[16384:]
        self.BlockLight, data = data[:16384], data[16384:]
        self.DirtyColumns = data[:256]

        self.unpackChunkData()
        self.shapeChunkData()

    def unpackChunkData(self):
        for key in ('SkyLight', 'BlockLight', 'Data'):
            dataArray = getattr(self, key)
            dataArray.shape = (16, 16, 64)
            s = dataArray.shape
            # assert s[2] == self.world.Height / 2
            # unpackedData = insert(dataArray[...,newaxis], 0, 0, 3)

            unpackedData = zeros((s[0], s[1], s[2] * 2), dtype='uint8')

            unpackedData[:, :, ::2] = dataArray
            unpackedData[:, :, ::2] &= 0xf
            unpackedData[:, :, 1::2] = dataArray
            unpackedData[:, :, 1::2] >>= 4
            setattr(self, key, unpackedData)

    def shapeChunkData(self):
        chunkSize = 16
        self.Blocks.shape = (chunkSize, chunkSize, self.world.Height)
        self.SkyLight.shape = (chunkSize, chunkSize, self.world.Height)
        self.BlockLight.shape = (chunkSize, chunkSize, self.world.Height)
        self.Data.shape = (chunkSize, chunkSize, self.world.Height)
        self.DirtyColumns.shape = chunkSize, chunkSize

    def _savedData(self):
        def packData(dataArray):
            assert dataArray.shape[2] == self.world.Height

            data = array(dataArray).reshape(16, 16, self.world.Height / 2, 2)
            data[..., 1] <<= 4
            data[..., 1] |= data[..., 0]
            return array(data[:, :, :, 1])

        if self.dirty:
            # elements of DirtyColumns are bitfields. Each bit corresponds to a
            # 16-block segment of the column. We set all of the bits because
            # we only track modifications at the chunk level.
            self.DirtyColumns[:] = 255

        return "".join([self.Blocks.tostring(),
                       packData(self.Data).tostring(),
                       packData(self.SkyLight).tostring(),
                       packData(self.BlockLight).tostring(),
                       self.DirtyColumns.tostring(),
                       ])
<commit_msg>Refactor: Remove dead class members.<commit_after>from level import FakeChunk
import logging
from materials import pocketMaterials
from mclevelbase import ChunkNotPresent, notclosing
from nbt import TAG_List
from numpy import array, fromstring, zeros
import os
import struct

# values are usually little-endian, unlike Minecraft PC

logger = logging.getLogger(__file__)

class PocketChunksFile(object):
    holdFileOpen = False  # if False, reopens and recloses the file on each access
    SECTOR_BYTES = 4096
    CHUNK_HEADER_SIZE = 4

    @property
    def file(self):
        openfile = lambda: file(self.path, "rb+")
        if PocketChunksFile.holdFileOpen:
            if self._file is None:
                self._file = openfile()
            return notclosing(self._file)
        else:
            return openfile()

    def close(self):
        if PocketChunksFile.holdFileOpen:
            self._file.close()
            self._file = None

    def __init__(self, path):
        self.path = path
        self._file = None
        if not os.path.exists(path):
            file(path, "w").close()

        with self.file as f:

            filesize = os.path.getsize(path)
            if filesize & 0xfff:
                filesize = (filesize | 0xfff) + 1
                f.truncate(filesize)

            if filesize == 0:
                filesize = self.SECTOR_BYTES
                f.truncate(filesize)

            f.seek(0)
            offsetsData = f.read(self.SECTOR_BYTES)

            self.freeSectors = [True] * (filesize / self.SECTOR_BYTES)
            self.freeSectors[0] = False

            self.offsets = fromstring(offsetsData, dtype='<u4')

        needsRepair = False

        for index, offset in enumerate(self.offsets):
            sector = offset >> 8
            count = offset & 0xff

            for i in xrange(sector, sector + count):
                if i >= len(self.freeSectors):
                    # raise RegionMalformed("Region file offset table points to sector {0} (past the end of the file)".format(i))
                    print  "Region file offset table points to sector {0} (past the end of the file)".format(i)
                    needsRepair = True
                    break
                if self.freeSectors[i] is False:
                    logger.debug("Double-allocated sector number %s (offset %s @ %s)", i, offset, index)
                    needsRepair = True
                self.freeSectors[i] = False

        if needsRepair:
            self.repair()

        logger.info("Found region file {file} with {used}/{total} sectors used and {chunks} chunks present".format(
             file=os.path.basename(path), used=self.usedSectors, total=self.sectorCount, chunks=self.chunkCount))

    @property
    def usedSectors(self):
        return len(self.freeSectors) - sum(self.freeSectors)

    @property
    def sectorCount(self):
        return len(self.freeSectors)

    @property
    def chunkCount(self):
        return sum(self.offsets > 0)

    def repair(self):
        pass
#        lostAndFound = {}
#        _freeSectors = [True] * len(self.freeSectors)
#        _freeSectors[0] = _freeSectors[1] = False
#        deleted = 0
#        recovered = 0
#        logger.info("Beginning repairs on {file} ({chunks} chunks)".format(file=os.path.basename(self.path), chunks=sum(self.offsets > 0)))
#        rx, rz = self.regionCoords
#        for index, offset in enumerate(self.offsets):
#            if offset:
#                cx = index & 0x1f
#                cz = index >> 5
#                cx += rx << 5
#                cz += rz << 5
#                sectorStart = offset >> 8
#                sectorCount = offset & 0xff
#                try:
#
#                    if sectorStart + sectorCount > len(self.freeSectors):
#                        raise RegionMalformed("Offset {start}:{end} ({offset}) at index {index} pointed outside of the file".format()
#                            start=sectorStart, end=sectorStart + sectorCount, index=index, offset=offset)
#
#                    compressedData = self._readChunk(cx, cz)
#                    if compressedData is None:
#                        raise RegionMalformed("Failed to read chunk data for {0}".format((cx, cz)))
#
#                    format, data = self.decompressSectors(compressedData)
#                    chunkTag = nbt.load(buf=data)
#                    lev = chunkTag["Level"]
#                    xPos = lev["xPos"].value
#                    zPos = lev["zPos"].value
#                    overlaps = False
#
#                    for i in xrange(sectorStart, sectorStart + sectorCount):
#                        if _freeSectors[i] is False:
#                            overlaps = True
#                        _freeSectors[i] = False
#
#
#                    if xPos != cx or zPos != cz or overlaps:
#                        lostAndFound[xPos, zPos] = (format, compressedData)
#
#                        if (xPos, zPos) != (cx, cz):
#                            raise RegionMalformed("Chunk {found} was found in the slot reserved for {expected}".format(found=(xPos, zPos), expected=(cx, cz)))
#                        else:
#                            raise RegionMalformed("Chunk {found} (in slot {expected}) has overlapping sectors with another chunk!".format(found=(xPos, zPos), expected=(cx, cz)))
#
#
#
#                except Exception, e:
#                    logger.info("Unexpected chunk data at sector {sector} ({exc})".format(sector=sectorStart, exc=e))
#                    self.setOffset(cx, cz, 0)
#                    deleted += 1
#
#        for cPos, (format, foundData) in lostAndFound.iteritems():
#            cx, cz = cPos
#            if self.getOffset(cx, cz) == 0:
#                logger.info("Found chunk {found} and its slot is empty, recovering it".format(found=cPos))
#                self._saveChunk(cx, cz, foundData[5:], format)
#                recovered += 1
#
#        logger.info("Repair complete. Removed {0} chunks, recovered {1} chunks, net {2}".format(deleted, recovered, recovered - deleted))
#

    def _readChunk(self, cx, cz):
        cx &= 0x1f
        cz &= 0x1f
        offset = self.getOffset(cx, cz)
        if offset == 0:
            return None

        sectorStart = offset >> 8
        numSectors = offset & 0xff
        if numSectors == 0:
            return None

        if sectorStart + numSectors > len(self.freeSectors):
            return None

        with self.file as f:
            f.seek(sectorStart * self.SECTOR_BYTES)
            data = f.read(numSectors * self.SECTOR_BYTES)
        assert(len(data) > 0)
        logger.debug("REGION LOAD %s,%s sector %s", cx, cz, sectorStart)
        return data

    def loadChunk(self, cx, cz, world):
        data = self._readChunk(cx, cz)
        if data is None:
            raise ChunkNotPresent((cx, cz, self))

        chunk = PocketChunk(cx, cz, data[4:], world)
        return chunk

    def saveChunk(self, chunk):
        cx, cz = chunk.chunkPosition

        cx &= 0x1f
        cz &= 0x1f
        offset = self.getOffset(cx, cz)
        sectorNumber = offset >> 8
        sectorsAllocated = offset & 0xff

        data = chunk._savedData()

        sectorsNeeded = (len(data) + self.CHUNK_HEADER_SIZE) / self.SECTOR_BYTES + 1
        if sectorsNeeded >= 256:
            return

        if sectorNumber != 0 and sectorsAllocated >= sectorsNeeded:
            logger.debug("REGION SAVE {0},{1} rewriting {2}b".format(cx, cz, len(data)))
            self.writeSector(sectorNumber, data, format)
        else:
            # we need to allocate new sectors

            # mark the sectors previously used for this chunk as free
            for i in xrange(sectorNumber, sectorNumber + sectorsAllocated):
                self.freeSectors[i] = True

            runLength = 0
            try:
                runStart = self.freeSectors.index(True)

                for i in range(runStart, len(self.freeSectors)):
                    if runLength:
                        if self.freeSectors[i]:
                            runLength += 1
                        else:
                            runLength = 0
                    elif self.freeSectors[i]:
                        runStart = i
                        runLength = 1

                    if runLength >= sectorsNeeded:
                        break
            except ValueError:
                pass

            # we found a free space large enough
            if runLength >= sectorsNeeded:
                logger.debug("REGION SAVE {0},{1}, reusing {2}b".format(cx, cz, len(data)))
                sectorNumber = runStart
                self.setOffset(cx, cz, sectorNumber << 8 | sectorsNeeded)
                self.writeSector(sectorNumber, data, format)
                self.freeSectors[sectorNumber:sectorNumber + sectorsNeeded] = [False] * sectorsNeeded

            else:
                # no free space large enough found -- we need to grow the
                # file

                logger.debug("REGION SAVE {0},{1}, growing by {2}b".format(cx, cz, len(data)))

                with self.file as f:
                    f.seek(0, 2)
                    filesize = f.tell()

                    sectorNumber = len(self.freeSectors)

                    assert sectorNumber * self.SECTOR_BYTES == filesize

                    filesize += sectorsNeeded * self.SECTOR_BYTES
                    f.truncate(filesize)

                self.freeSectors += [False] * sectorsNeeded

                self.setOffset(cx, cz, sectorNumber << 8 | sectorsNeeded)
                self.writeSector(sectorNumber, data, format)

    def writeSector(self, sectorNumber, data, format):
        with self.file as f:
            logger.debug("REGION: Writing sector {0}".format(sectorNumber))

            f.seek(sectorNumber * self.SECTOR_BYTES)
            f.write(struct.pack("<I", len(data) + self.CHUNK_HEADER_SIZE))  # // chunk length
            f.write(data)  # // chunk data
            # f.flush()

    def containsChunk(self, cx, cz):
        return self.getOffset(cx, cz) != 0

    def getOffset(self, cx, cz):
        cx &= 0x1f
        cz &= 0x1f
        return self.offsets[cx + cz * 32]

    def setOffset(self, cx, cz, offset):
        cx &= 0x1f
        cz &= 0x1f
        self.offsets[cx + cz * 32] = offset
        with self.file as f:
            f.seek(0)
            f.write(self.offsets.tostring())

    def chunkCoords(self):
        indexes = (i for (i, offset) in enumerate(self.offsets) if offset)
        coords = ((i % 32, i // 32) for i in indexes)
        return coords

from infiniteworld import ChunkedLevelMixin
from level import MCLevel, LightedChunk

class PocketWorld(ChunkedLevelMixin, MCLevel):
    Height = 128
    Length = 512
    Width = 512

    isInfinite = True  # Wrong. isInfinite actually means 'isChunked' and should be changed
    materials = pocketMaterials

    @property
    def allChunks(self):
        return list(self.chunkFile.chunkCoords())

    def __init__(self, filename):
        if not os.path.isdir(filename):
            filename = os.path.dirname(filename)
        self.filename = filename
        self.dimensions = {}

        self.chunkFile = PocketChunksFile(os.path.join(filename, "chunks.dat"))
        self._loadedChunks = {}

    def getChunk(self, cx, cz):
        for p in cx, cz:
            if not 0 <= p <= 31:
                raise ChunkNotPresent((cx, cz, self))

        c = self._loadedChunks.get((cx, cz))
        if c is None:
            c = self.chunkFile.loadChunk(cx, cz, self)
            self._loadedChunks[cx, cz] = c
        return c

    @classmethod
    def _isLevel(cls, filename):
        clp = ("chunks.dat", "level.dat")

        if not os.path.isdir(filename):
            f = os.path.basename(filename)
            if f not in clp:
                return False
            filename = os.path.dirname(filename)

        return all([os.path.exists(os.path.join(filename, f)) for f in clp])

    def saveInPlace(self):
        for chunk in self._loadedChunks.itervalues():
            if chunk.dirty:
                self.chunkFile.saveChunk(chunk)
                chunk.dirty = False

    def containsChunk(self, cx, cz):
        if cx > 31 or cz > 31 or cx < 0 or cz < 0:
            return False
        return self.chunkFile.getOffset(cx, cz) != 0

class PocketChunk(LightedChunk):
    HeightMap = FakeChunk.HeightMap

    Entities = TileEntities = property(lambda self: TAG_List())

    dirty = False
    filename = "chunks.dat"

    def __init__(self, cx, cz, data, world):
        self.chunkPosition = (cx, cz)
        self.world = world
        data = fromstring(data, dtype='uint8')

        self.Blocks, data = data[:32768], data[32768:]
        self.Data, data = data[:16384], data[16384:]
        self.SkyLight, data = data[:16384], data[16384:]
        self.BlockLight, data = data[:16384], data[16384:]
        self.DirtyColumns = data[:256]

        self.unpackChunkData()
        self.shapeChunkData()

    def unpackChunkData(self):
        for key in ('SkyLight', 'BlockLight', 'Data'):
            dataArray = getattr(self, key)
            dataArray.shape = (16, 16, 64)
            s = dataArray.shape
            # assert s[2] == self.world.Height / 2
            # unpackedData = insert(dataArray[...,newaxis], 0, 0, 3)

            unpackedData = zeros((s[0], s[1], s[2] * 2), dtype='uint8')

            unpackedData[:, :, ::2] = dataArray
            unpackedData[:, :, ::2] &= 0xf
            unpackedData[:, :, 1::2] = dataArray
            unpackedData[:, :, 1::2] >>= 4
            setattr(self, key, unpackedData)

    def shapeChunkData(self):
        chunkSize = 16
        self.Blocks.shape = (chunkSize, chunkSize, self.world.Height)
        self.SkyLight.shape = (chunkSize, chunkSize, self.world.Height)
        self.BlockLight.shape = (chunkSize, chunkSize, self.world.Height)
        self.Data.shape = (chunkSize, chunkSize, self.world.Height)
        self.DirtyColumns.shape = chunkSize, chunkSize

    def _savedData(self):
        def packData(dataArray):
            assert dataArray.shape[2] == self.world.Height

            data = array(dataArray).reshape(16, 16, self.world.Height / 2, 2)
            data[..., 1] <<= 4
            data[..., 1] |= data[..., 0]
            return array(data[:, :, :, 1])

        if self.dirty:
            # elements of DirtyColumns are bitfields. Each bit corresponds to a
            # 16-block segment of the column. We set all of the bits because
            # we only track modifications at the chunk level.
            self.DirtyColumns[:] = 255

        return "".join([self.Blocks.tostring(),
                       packData(self.Data).tostring(),
                       packData(self.SkyLight).tostring(),
                       packData(self.BlockLight).tostring(),
                       self.DirtyColumns.tostring(),
                       ])
<|endoftext|>


<commit_before>import glob
import logging
import os
import subprocess
from plugins import BaseAssembler
from yapsy.IPlugin import IPlugin

class A5Assembler(BaseAssembler, IPlugin):
    OUTPUT = 'scaffolds'
    def run(self, reads):
        """
        Build the command and run.
        Return list of contig file(s)
        """

        cmd_args = [os.path.join(os.getcwd(),self.executable)]
        libfile =  open(os.path.join(self.outpath, 'a5lib.out'), 'w')
        for d in reads:
            libfile.write('[LIB]\n')
            if d['type'] == 'paired':
                if len(d['files']) == 1:
                    libfile.write('shuf={}\n'.format(d['files'][0]))
                elif len(d['files']) == 2:
                    libfile.write('p1={}\n'.format(d['files'][0]))
                    libfile.write('p2={}\n'.format(d['files'][1]))
            elif d['type'] == 'single':
                for up in d['files']:
                    libfile.write('up={}\n'.format(up))
            try:
                libfile.write('ins={}\n'.format(d['insert']))
            except:
                logging.info('No Insert Info Given')
        cmd_args.append(libfile.name)
        libfile.close()
        cmd_args.append('a5')
        self.arast_popen(cmd_args, cwd=self.outpath)

        #contigs = glob.glob(self.outpath + '/*.contigs.
        contigs = glob.glob(self.outpath + '/*.final.scaffolds.fasta')

        if not contigs:
            #raise Exception("No contigs")
            print "No contigs"
        #return contigs, scaffolds
        return contigs

<commit_msg>update a5 plugin<commit_after>import glob
import logging
import os
import subprocess
from plugins import BaseAssembler
from yapsy.IPlugin import IPlugin

class A5Assembler(BaseAssembler, IPlugin):
    OUTPUT = 'scaffolds'
    def run(self):
        """
        Build the command and run.
        Return list of contig file(s)
        """

        cmd_args = [os.path.join(os.getcwd(),self.executable)]
        libfile =  open(os.path.join(self.outpath, 'a5lib.out'), 'w')
        reads = self.data.readsets
        for d in reads:
            libfile.write('[LIB]\n')
            if d.type == 'paired':
                if len(d.files) == 1:
                    libfile.write('shuf={}\n'.format(d.files[0]))
                elif len(d.files) == 2:
                    libfile.write('p1={}\n'.format(d.files[0]))
                    libfile.write('p2={}\n'.format(d.files[1]))
            elif d.type == 'single':
                for up in d.files:
                    libfile.write('up={}\n'.format(up))
            try:
                libfile.write('ins={}\n'.format(d['insert']))
            except:
                logging.info('No Insert Info Given')
        cmd_args.append(libfile.name)
        libfile.close()
        cmd_args.append('a5')
        self.arast_popen(cmd_args, cwd=self.outpath)

        contigs = glob.glob(self.outpath + '/*.contigs.fasta')
        scaffolds = glob.glob(self.outpath + '/*.final.scaffolds.fasta')

        output = {}
        if contigs:
            output['contigs'] = contigs
        if scaffolds:
            output['scaffolds'] = scaffolds
        return output

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import json
import socket
import struct
import select
import os
import hashlib
import re
from sys import exit as exit

# we import PollingObserver instead of Observer because the deleted event
# is not capturing https://github.com/gorakhargosh/watchdog/issues/46
from watchdog.observers.polling import PollingObserver as Observer
from watchdog.events import RegexMatchingEventHandler

from connection_manager import ConnectionManager

class Daemon(RegexMatchingEventHandler):
    # TODO : if conf_path doesn't exist create it
    DEFAULT_CONFIG = {'sharing_path': './sharing_folder',
                      'cmd_address': 'localhost',
                      'cmd_port': 50001,
                      'api_suffix': '/API/V1/',
                      'server_address': 'http://localhost:5000',
                      'user': 'default_user',
                      'pass': 'default_pass',
                      'timeout_listener_sock': 0.5,
                      'backlog_listener_sock': 5,
                      }

    IGNORED_REGEX = ['.*\.[a-zA-z]+?#',  # Libreoffice suite temporary file ignored
                      '.*\.[a-zA-Z]+?~',  # gedit issue solved ignoring this pattern:
                      # gedit first delete file, create, and move to dest_path *.txt~
                      '.*\/(\..*)',  # hidden files TODO: improve
                      ]

    PATH_CONFIG = 'config.json'
    INT_SIZE = struct.calcsize('!i')

    def __init__(self):
        RegexMatchingEventHandler.__init__(self, ignore_regexes=Daemon.IGNORED_REGEX, ignore_directories=True)
        # Initialize variable
        self.daemon_state = 'down'  # TODO implement the daemon state (disconnected, connected, syncronizing, ready...)
        self.running = 0
        self.client_snapshot = {}
        self.listener_socket = None
        self.observer = None
        self.cfg = self.load_cfg(Daemon.PATH_CONFIG)
        # We call os.path.abspath to unrelativize the sharing path(now is relative for development purpose)
        # TODO: Allow the setting of sharing path by user
        self.cfg['sharing_path'] = os.path.abspath(self.cfg['sharing_path'])

        self.conn_mng = ConnectionManager(self.cfg)
        # Operations necessary to start the daemon
        self.connect_to_server()
        self.build_client_snapshot()
        self.sync_with_server()
        self.create_observer()

    def load_cfg(self, config_path):
        """
        Load config, if impossible to find it or config file is corrupted restore it and load default configuration
        :param config_path: Path of config
        :return: dictionary containing configuration
        """
        def build_default_cfg():
            """
            Restore default config file by writing on file
            :return: default configuration contained in the dictionary DEFAULT_CONFIG
            """
            with open(Daemon.PATH_CONFIG, 'wb') as fo:
                json.dump(Daemon.DEFAULT_CONFIG, fo, skipkeys=True, ensure_ascii= True, indent=4)
            return Daemon.DEFAULT_CONFIG

        if os.path.isfile(config_path):
            try:
                with open(config_path, 'r') as fo:
                    loaded_config = json.load(fo)
            except ValueError:
                print 'Impossible to read "{0}"! "{0}" overwrited and loaded default config!'.format(config_path)
                return build_default_cfg()
            corrupted_config = False
            for k in Daemon.DEFAULT_CONFIG:
                if k not in loaded_config:
                    corrupted_config = True
            if not corrupted_config:
                return loaded_config
            else:
                print '"{0}" corrupted! "{0}" overwrited and loaded default config!'.format(config_path)
                return build_default_cfg()
        else:
            print '{0} doesn\'t exist, "{0}" overwrited and loaded default config!'.format(config_path)
            return build_default_cfg()

    def connect_to_server(self):
        # self.cfg['server_address']
        pass

    def build_client_snapshot(self):
        self.client_snapshot = {}
        for dirpath, dirs, files in os.walk(self.cfg['sharing_path']):
                for filename in files:
                    file_path = os.path.join(dirpath, filename)
                    matched_regex = False
                    for r in Daemon.IGNORED_REGEX:
                        if re.match(r, file_path) is not None:
                            matched_regex = True
                            print 'Ignored Path:', file_path
                            break
                    if not matched_regex:
                        relative_path = self.relativize_path(file_path)
                        with open(file_path, 'rb') as f:
                            self.client_snapshot[relative_path] = hashlib.md5(f.read()).hexdigest()

    def sync_with_server(self):
        """
        Download from server the files state and find the difference from actual state.
        """
        server_snapshot = self.conn_mng.dispatch_request('get_server_snapshot')
        if server_snapshot is None:
            self.stop(1, '\nReceived bad snapshot. Server down?\n')
        else:
            server_snapshot = server_snapshot['files']

        for file_path in server_snapshot:
            if file_path not in self.client_snapshot:
                # TODO: check if download succeed, if so update client_snapshot with the new file
                self.conn_mng.dispatch_request('download', {'filepath': file_path})
                self.client_snapshot[file_path] = server_snapshot[file_path]
            elif server_snapshot[file_path] != self.client_snapshot[file_path]:
                self.conn_mng.dispatch_request('modify', {'filepath': file_path})
        for file_path in self.client_snapshot:
            if file_path not in server_snapshot:
                self.conn_mng.dispatch_request('upload', {'filepath': file_path})

    def relativize_path(self, abs_path):
        """
        This function relativize the path watched by daemon:
        for example: /home/user/watched/subfolder/ will be subfolder/
        """
        folder_watched_abs = os.path.abspath(self.cfg['sharing_path'])
        relative_path = abs_path.split(folder_watched_abs)[-1]
        return relative_path[1:]

    def create_observer(self):
        """
        Create an instance of the watchdog Observer thread class.
        """
        self.observer = Observer()
        self.observer.schedule(self, path=self.cfg['sharing_path'], recursive=True)

    # TODO GESTIRE ERRORI DEL DICTIONARY NEL CASO client_dispatcher NON ABBIA I DATI RICHIESTI!!
    # TODO update struct with new more performance data structure
    # TODO verify what happen if the server return a error message
    ####################################
    # In client_snapshot the structure are {'<filepath>' : '<md5>'} so you have to convert!!!!
    ####################################

    def on_created(self, e):
        def build_data(cmd, e, new_md5=None):
            """
            Prepares the data from event handler to be delivered to connection_manager.
            """
            data = {'cmd': cmd}
            if cmd == 'copy':
                for path in self.client_snapshot:
                    if new_md5 in self.client_snapshot[path]:
                        path_with_searched_md5 = path
                # TODO check what happen when i find more than 1 path with the new_md5
                data['file'] = {'src': path_with_searched_md5,
                                'dst': self.relativize_path(e.src_path),
                                'md5': self.client_snapshot[path_with_searched_md5],
                                }
            else:
                f = open(e.src_path, 'rb')
                data['file'] = {'filepath': self.relativize_path(e.src_path),
                                'md5': hashlib.md5(f.read()).hexdigest(),
                                }
            return data

        with open(e.src_path, 'rb') as f:
            new_md5 = hashlib.md5(f.read()).hexdigest()
        relative_path = self.relativize_path(e.src_path)
        # with this check i found the copy events
        if new_md5 in self.client_snapshot.values():
            print 'start copy'
            data = build_data('copy', e, new_md5)
            self.client_snapshot[data['file']['dst']] = data['file']['md5']
        # this elif check that this created aren't modified event
        elif relative_path in self.client_snapshot:
            print 'start modified DA UN CREATE!!!!!'
            data = build_data('modify', e)
            self.client_snapshot[data['file']['filepath']] = data['file']['md5']
        else:
            print 'start create'
            data = build_data('upload', e)
            self.client_snapshot[data['file']['filepath']] = data['file']['md5']
        self.conn_mng.dispatch_request(data['cmd'], data['file'])

    def on_moved(self, e):

        print 'start move'
        with open(e.dest_path, 'rb') as f:
            dest_md5 = hashlib.md5(f.read()).hexdigest()
        data = {'cmd': 'move',
                'file': {'src': self.relativize_path(e.src_path),
                         'dst': self.relativize_path(e.dest_path),
                         'md5': dest_md5,
                         }
                }
        self.client_snapshot[data['file']['dst']] = data['file']['md5']
        self.client_snapshot.pop(data['file']['src'], 'NOTHING TO POP')
        self.conn_mng.dispatch_request(data['cmd'], data['file'])

    def on_deleted(self, e):

        print 'start delete'
        data = {'cmd': 'delete',
                'file': {'filepath': self.relativize_path(e.src_path),
                         }
                }
        self.client_snapshot.pop(data['file']['filepath'], 'NOTHING TO POP')
        self.conn_mng.dispatch_request(data['cmd'], data['file'])

    def on_modified(self, e):

        print 'start modified'
        with open(e.src_path, 'rb') as f:
            data = {'cmd': 'modify',
                    'file': {'filepath': self.relativize_path(e.src_path),
                             'md5': hashlib.md5(f.read()).hexdigest(),
                             }
                    }
        self.client_snapshot[data['file']['filepath']] = data['file']['md5']
        self.conn_mng.dispatch_request(data['cmd'], data['file'])

    def start(self):
        """
        Starts the communication with the command_manager.
        """

        self.listener_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.listener_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.listener_socket.bind((self.cfg['cmd_address'], self.cfg['cmd_port']))
        self.listener_socket.listen(self.cfg['backlog_listener_sock'])
        r_list = [self.listener_socket]
        self.observer.start()
        self.daemon_state = 'started'
        self.running = 1
        try:
            while self.running:
                r_ready, w_ready, e_ready = select.select(r_list, [], [], self.cfg['timeout_listener_sock'])

                for s in r_ready:

                    if s == self.listener_socket:
                        # handle the server socket
                        client_socket, client_address = self.listener_socket.accept()
                        r_list.append(client_socket)
                    else:
                        # handle all other sockets
                        length = s.recv(Daemon.INT_SIZE)
                        if length:
                            length = int(struct.unpack('!i', length)[0])
                            message = json.loads(s.recv(length))
                            for cmd, data in message.items():
                                if cmd == 'shutdown':
                                    raise KeyboardInterrupt
                                self.conn_mng.dispatch_request(cmd, data)
                        else:
                            s.close()
                            r_list.remove(s)
        except KeyboardInterrupt:
            self.stop(0)

    def stop(self, exit_status, exit_message=None):
        """
        Stop the Daemon components (observer and communication with command_manager).
        """
        if self.daemon_state == 'started':
            self.observer.stop()
            self.observer.join()
            self.listener_socket.close()
        self.running = 0
        if exit_message:
            print exit_message
        exit(exit_status)

if __name__ == '__main__':
    daemon = Daemon()
    daemon.start()
<commit_msg>DEFAULT_CONFIG transformed in OrderedDict to maintain the order of the key if i need to restore the config.json file<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import json
import socket
import struct
import select
import os
import hashlib
import re
from sys import exit as exit
from collections import OrderedDict

# we import PollingObserver instead of Observer because the deleted event
# is not capturing https://github.com/gorakhargosh/watchdog/issues/46
from watchdog.observers.polling import PollingObserver as Observer
from watchdog.events import RegexMatchingEventHandler

from connection_manager import ConnectionManager

class Daemon(RegexMatchingEventHandler):
    # Default configuration for Daemon, loaded if fail to load from file in PATH_CONFIG
    DEFAULT_CONFIG = OrderedDict()
    DEFAULT_CONFIG['sharing_path'] = './sharing_folder'
    DEFAULT_CONFIG['cmd_address'] = 'localhost'
    DEFAULT_CONFIG['cmd_port'] = 50001
    DEFAULT_CONFIG['api_suffix'] = '/API/V1/'
    DEFAULT_CONFIG['server_address'] = 'http://localhost:5000'
    DEFAULT_CONFIG['user'] = 'pasquale'
    DEFAULT_CONFIG['pass'] = 'secretpass'
    DEFAULT_CONFIG['timeout_listener_sock'] = 0.5
    DEFAULT_CONFIG['backlog_listener_sock'] = 1

    IGNORED_REGEX = ['.*\.[a-zA-z]+?#',  # Libreoffice suite temporary file ignored
                      '.*\.[a-zA-Z]+?~',  # gedit issue solved ignoring this pattern:
                      # gedit first delete file, create, and move to dest_path *.txt~
                      '.*\/(\..*)',  # hidden files TODO: improve
                      ]

    PATH_CONFIG = 'config.json'
    INT_SIZE = struct.calcsize('!i')

    def __init__(self):
        RegexMatchingEventHandler.__init__(self, ignore_regexes=Daemon.IGNORED_REGEX, ignore_directories=True)
        # Initialize variable
        self.daemon_state = 'down'  # TODO implement the daemon state (disconnected, connected, syncronizing, ready...)
        self.running = 0
        self.client_snapshot = {}
        self.listener_socket = None
        self.observer = None
        self.cfg = self.load_cfg(Daemon.PATH_CONFIG)
        # We call os.path.abspath to unrelativize the sharing path(now is relative for development purpose)
        # TODO: Allow the setting of sharing path by user
        self.cfg['sharing_path'] = os.path.abspath(self.cfg['sharing_path'])

        self.conn_mng = ConnectionManager(self.cfg)
        # Operations necessary to start the daemon
        self.connect_to_server()
        self.build_client_snapshot()
        self.sync_with_server()
        self.create_observer()

    def load_cfg(self, config_path):
        """
        Load config, if impossible to find it or config file is corrupted restore it and load default configuration
        :param config_path: Path of config
        :return: dictionary containing configuration
        """
        def build_default_cfg():
            """
            Restore default config file by writing on file
            :return: default configuration contained in the dictionary DEFAULT_CONFIG
            """
            with open(Daemon.PATH_CONFIG, 'wb') as fo:
                json.dump(Daemon.DEFAULT_CONFIG, fo, skipkeys=True, ensure_ascii= True, indent=4)
            return Daemon.DEFAULT_CONFIG

        if os.path.isfile(config_path):
            try:
                with open(config_path, 'r') as fo:
                    loaded_config = json.load(fo)
            except ValueError:
                print 'Impossible to read "{0}"! "{0}" overwrited and loaded default config!'.format(config_path)
                return build_default_cfg()
            corrupted_config = False
            for k in Daemon.DEFAULT_CONFIG:
                if k not in loaded_config:
                    corrupted_config = True
            if not corrupted_config:
                return loaded_config
            else:
                print '"{0}" corrupted! "{0}" overwrited and loaded default config!'.format(config_path)
                return build_default_cfg()
        else:
            print '{0} doesn\'t exist, "{0}" overwrited and loaded default config!'.format(config_path)
            return build_default_cfg()

    def connect_to_server(self):
        # self.cfg['server_address']
        pass

    def build_client_snapshot(self):
        self.client_snapshot = {}
        for dirpath, dirs, files in os.walk(self.cfg['sharing_path']):
                for filename in files:
                    file_path = os.path.join(dirpath, filename)
                    matched_regex = False
                    for r in Daemon.IGNORED_REGEX:
                        if re.match(r, file_path) is not None:
                            matched_regex = True
                            print 'Ignored Path:', file_path
                            break
                    if not matched_regex:
                        relative_path = self.relativize_path(file_path)
                        with open(file_path, 'rb') as f:
                            self.client_snapshot[relative_path] = hashlib.md5(f.read()).hexdigest()

    def sync_with_server(self):
        """
        Download from server the files state and find the difference from actual state.
        """
        server_snapshot = self.conn_mng.dispatch_request('get_server_snapshot')
        if server_snapshot is None:
            self.stop(1, '\nReceived bad snapshot. Server down?\n')
        else:
            server_snapshot = server_snapshot['files']

        for file_path in server_snapshot:
            if file_path not in self.client_snapshot:
                # TODO: check if download succeed, if so update client_snapshot with the new file
                self.conn_mng.dispatch_request('download', {'filepath': file_path})
                self.client_snapshot[file_path] = server_snapshot[file_path]
            elif server_snapshot[file_path] != self.client_snapshot[file_path]:
                self.conn_mng.dispatch_request('modify', {'filepath': file_path})
        for file_path in self.client_snapshot:
            if file_path not in server_snapshot:
                self.conn_mng.dispatch_request('upload', {'filepath': file_path})

    def relativize_path(self, abs_path):
        """
        This function relativize the path watched by daemon:
        for example: /home/user/watched/subfolder/ will be subfolder/
        """
        folder_watched_abs = os.path.abspath(self.cfg['sharing_path'])
        relative_path = abs_path.split(folder_watched_abs)[-1]
        return relative_path[1:]

    def create_observer(self):
        """
        Create an instance of the watchdog Observer thread class.
        """
        self.observer = Observer()
        self.observer.schedule(self, path=self.cfg['sharing_path'], recursive=True)

    # TODO GESTIRE ERRORI DEL DICTIONARY NEL CASO client_dispatcher NON ABBIA I DATI RICHIESTI!!
    # TODO update struct with new more performance data structure
    # TODO verify what happen if the server return a error message
    ####################################
    # In client_snapshot the structure are {'<filepath>' : '<md5>'} so you have to convert!!!!
    ####################################

    def on_created(self, e):
        def build_data(cmd, e, new_md5=None):
            """
            Prepares the data from event handler to be delivered to connection_manager.
            """
            data = {'cmd': cmd}
            if cmd == 'copy':
                for path in self.client_snapshot:
                    if new_md5 in self.client_snapshot[path]:
                        path_with_searched_md5 = path
                # TODO check what happen when i find more than 1 path with the new_md5
                data['file'] = {'src': path_with_searched_md5,
                                'dst': self.relativize_path(e.src_path),
                                'md5': self.client_snapshot[path_with_searched_md5],
                                }
            else:
                f = open(e.src_path, 'rb')
                data['file'] = {'filepath': self.relativize_path(e.src_path),
                                'md5': hashlib.md5(f.read()).hexdigest(),
                                }
            return data

        with open(e.src_path, 'rb') as f:
            new_md5 = hashlib.md5(f.read()).hexdigest()
        relative_path = self.relativize_path(e.src_path)
        # with this check i found the copy events
        if new_md5 in self.client_snapshot.values():
            print 'start copy'
            data = build_data('copy', e, new_md5)
            self.client_snapshot[data['file']['dst']] = data['file']['md5']
        # this elif check that this created aren't modified event
        elif relative_path in self.client_snapshot:
            print 'start modified DA UN CREATE!!!!!'
            data = build_data('modify', e)
            self.client_snapshot[data['file']['filepath']] = data['file']['md5']
        else:
            print 'start create'
            data = build_data('upload', e)
            self.client_snapshot[data['file']['filepath']] = data['file']['md5']
        self.conn_mng.dispatch_request(data['cmd'], data['file'])

    def on_moved(self, e):

        print 'start move'
        with open(e.dest_path, 'rb') as f:
            dest_md5 = hashlib.md5(f.read()).hexdigest()
        data = {'cmd': 'move',
                'file': {'src': self.relativize_path(e.src_path),
                         'dst': self.relativize_path(e.dest_path),
                         'md5': dest_md5,
                         }
                }
        self.client_snapshot[data['file']['dst']] = data['file']['md5']
        self.client_snapshot.pop(data['file']['src'], 'NOTHING TO POP')
        self.conn_mng.dispatch_request(data['cmd'], data['file'])

    def on_deleted(self, e):

        print 'start delete'
        data = {'cmd': 'delete',
                'file': {'filepath': self.relativize_path(e.src_path),
                         }
                }
        self.client_snapshot.pop(data['file']['filepath'], 'NOTHING TO POP')
        self.conn_mng.dispatch_request(data['cmd'], data['file'])

    def on_modified(self, e):

        print 'start modified'
        with open(e.src_path, 'rb') as f:
            data = {'cmd': 'modify',
                    'file': {'filepath': self.relativize_path(e.src_path),
                             'md5': hashlib.md5(f.read()).hexdigest(),
                             }
                    }
        self.client_snapshot[data['file']['filepath']] = data['file']['md5']
        self.conn_mng.dispatch_request(data['cmd'], data['file'])

    def start(self):
        """
        Starts the communication with the command_manager.
        """

        self.listener_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.listener_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.listener_socket.bind((self.cfg['cmd_address'], self.cfg['cmd_port']))
        self.listener_socket.listen(self.cfg['backlog_listener_sock'])
        r_list = [self.listener_socket]
        self.observer.start()
        self.daemon_state = 'started'
        self.running = 1
        try:
            while self.running:
                r_ready, w_ready, e_ready = select.select(r_list, [], [], self.cfg['timeout_listener_sock'])

                for s in r_ready:

                    if s == self.listener_socket:
                        # handle the server socket
                        client_socket, client_address = self.listener_socket.accept()
                        r_list.append(client_socket)
                    else:
                        # handle all other sockets
                        length = s.recv(Daemon.INT_SIZE)
                        if length:
                            length = int(struct.unpack('!i', length)[0])
                            message = json.loads(s.recv(length))
                            for cmd, data in message.items():
                                if cmd == 'shutdown':
                                    raise KeyboardInterrupt
                                self.conn_mng.dispatch_request(cmd, data)
                        else:
                            s.close()
                            r_list.remove(s)
        except KeyboardInterrupt:
            self.stop(0)

    def stop(self, exit_status, exit_message=None):
        """
        Stop the Daemon components (observer and communication with command_manager).
        """
        if self.daemon_state == 'started':
            self.observer.stop()
            self.observer.join()
            self.listener_socket.close()
        self.running = 0
        if exit_message:
            print exit_message
        exit(exit_status)

if __name__ == '__main__':
    daemon = Daemon()
    daemon.start()
<|endoftext|>