<commit_before># -*- coding: utf-8 -*-
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import sys
import json
import time
import base64

from airflow.contrib.hooks.gcs_hook import GoogleCloudStorageHook
from airflow.hooks.mysql_hook import MySqlHook
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults
from datetime import date, datetime
from decimal import Decimal
from MySQLdb.constants import FIELD_TYPE
from tempfile import NamedTemporaryFile
from six import string_types

PY3 = sys.version_info[0] == 3

class MySqlToGoogleCloudStorageOperator(BaseOperator):
    """
    Copy data from MySQL to Google cloud storage in JSON format.
    """
    template_fields = ('sql', 'bucket', 'filename', 'schema_filename', 'schema')
    template_ext = ('.sql',)
    ui_color = '#a0e08c'

    @apply_defaults
    def __init__(self,
                 sql,
                 bucket,
                 filename,
                 schema_filename=None,
                 approx_max_file_size_bytes=1900000000,
                 mysql_conn_id='mysql_default',
                 google_cloud_storage_conn_id='google_cloud_default',
                 schema=None,
                 delegate_to=None,
                 *args,
                 **kwargs):
        """
        :param sql: The SQL to execute on the MySQL table.
        :type sql: string
        :param bucket: The bucket to upload to.
        :type bucket: string
        :param filename: The filename to use as the object name when uploading
            to Google cloud storage. A {} should be specified in the filename
            to allow the operator to inject file numbers in cases where the
            file is split due to size.
        :type filename: string
        :param schema_filename: If set, the filename to use as the object name
            when uploading a .json file containing the BigQuery schema fields
            for the table that was dumped from MySQL.
        :type schema_filename: string
        :param approx_max_file_size_bytes: This operator supports the ability
            to split large table dumps into multiple files (see notes in the
            filenamed param docs above). Google cloud storage allows for files
            to be a maximum of 4GB. This param allows developers to specify the
            file size of the splits.
        :type approx_max_file_size_bytes: long
        :param mysql_conn_id: Reference to a specific MySQL hook.
        :type mysql_conn_id: string
        :param google_cloud_storage_conn_id: Reference to a specific Google
            cloud storage hook.
        :type google_cloud_storage_conn_id: string
        :param schema: The schema to use, if any. Should be a list of dict or
            a str. Pass a string if using Jinja template, otherwise, pass a list of
            dict. Examples could be seen: https://cloud.google.com/bigquery/docs
            /schemas#specifying_a_json_schema_file
        :type schema: str or list
        :param delegate_to: The account to impersonate, if any. For this to
            work, the service account making the request must have domain-wide
            delegation enabled.
        """
        super(MySqlToGoogleCloudStorageOperator, self).__init__(*args, **kwargs)
        self.sql = sql
        self.bucket = bucket
        self.filename = filename
        self.schema_filename = schema_filename
        self.approx_max_file_size_bytes = approx_max_file_size_bytes
        self.mysql_conn_id = mysql_conn_id
        self.google_cloud_storage_conn_id = google_cloud_storage_conn_id
        self.schema = schema
        self.delegate_to = delegate_to

    def execute(self, context):
        cursor = self._query_mysql()
        files_to_upload = self._write_local_data_files(cursor)

        # If a schema is set, create a BQ schema JSON file.
        if self.schema_filename:
            files_to_upload.update(self._write_local_schema_file(cursor))

        # Flush all files before uploading.
        for file_handle in files_to_upload.values():
            file_handle.flush()

        self._upload_to_gcs(files_to_upload)

        # Close all temp file handles.
        for file_handle in files_to_upload.values():
            file_handle.close()

    def _query_mysql(self):
        """
        Queries mysql and returns a cursor to the results.
        """
        mysql = MySqlHook(mysql_conn_id=self.mysql_conn_id)
        conn = mysql.get_conn()
        cursor = conn.cursor()
        cursor.execute(self.sql)
        return cursor

    def _write_local_data_files(self, cursor):
        """
        Takes a cursor, and writes results to a local file.

        :return: A dictionary where keys are filenames to be used as object
            names in GCS, and values are file handles to local files that
            contain the data for the GCS objects.
        """
        schema = list(map(lambda schema_tuple: schema_tuple[0], cursor.description))
        col_type_dict = self._get_col_type_dict()
        file_no = 0
        tmp_file_handle = NamedTemporaryFile(delete=True)
        tmp_file_handles = {self.filename.format(file_no): tmp_file_handle}

        for row in cursor:
            # Convert datetime objects to utc seconds, and decimals to floats.
            # Convert binary type object to string encoded with base64.
            row = self._convert_types(schema, col_type_dict, row)
            row_dict = dict(zip(schema, row))

            # TODO validate that row isn't > 2MB. BQ enforces a hard row size of 2MB.
            s = json.dumps(row_dict)
            if PY3:
                s = s.encode('utf-8')
            tmp_file_handle.write(s)

            # Append newline to make dumps BigQuery compatible.
            tmp_file_handle.write(b'\n')

            # Stop if the file exceeds the file size limit.
            if tmp_file_handle.tell() >= self.approx_max_file_size_bytes:
                file_no += 1
                tmp_file_handle = NamedTemporaryFile(delete=True)
                tmp_file_handles[self.filename.format(file_no)] = tmp_file_handle

        return tmp_file_handles

    def _write_local_schema_file(self, cursor):
        """
        Takes a cursor, and writes the BigQuery schema for the results to a
        local file system.

        :return: A dictionary where key is a filename to be used as an object
            name in GCS, and values are file handles to local files that
            contains the BigQuery schema fields in .json format.
        """
        schema_str = None
        tmp_schema_file_handle = NamedTemporaryFile(delete=True)
        if self.schema is not None and isinstance(self.schema, string_types):
            schema_str = self.schema
        elif self.schema is not None and isinstance(self.schema, list):
            schema_str = json.dumps(self.schema)
        else:
            schema = []
            for field in cursor.description:
                # See PEP 249 for details about the description tuple.
                field_name = field[0]
                field_type = self.type_map(field[1])
                # Always allow TIMESTAMP to be nullable. MySQLdb returns None types
                # for required fields because some MySQL timestamps can't be
                # represented by Python's datetime (e.g. 0000-00-00 00:00:00).
                if field[6] or field_type == 'TIMESTAMP':
                    field_mode = 'NULLABLE'
                else:
                    field_mode = 'REQUIRED'
                schema.append({
                    'name': field_name,
                    'type': field_type,
                    'mode': field_mode,
                })
            schema_str = json.dumps(schema)
        if PY3:
            schema_str = schema_str.encode('utf-8')
        tmp_schema_file_handle.write(schema_str)

        self.log.info('Using schema for %s: %s', self.schema_filename, schema_str)
        return {self.schema_filename: tmp_schema_file_handle}

    def _upload_to_gcs(self, files_to_upload):
        """
        Upload all of the file splits (and optionally the schema .json file) to
        Google cloud storage.
        """
        hook = GoogleCloudStorageHook(
            google_cloud_storage_conn_id=self.google_cloud_storage_conn_id,
            delegate_to=self.delegate_to)
        for object, tmp_file_handle in files_to_upload.items():
            hook.upload(self.bucket, object, tmp_file_handle.name, 'application/json')

    def _convert_types(self, schema, col_type_dict, row):
        """
        Takes a value from MySQLdb, and converts it to a value that's safe for
        JSON/Google cloud storage/BigQuery. Dates are converted to UTC seconds.
        Decimals are converted to floats. Binary type fields are encoded with base64,
        as imported BYTES data must be base64-encoded according to Bigquery SQL
        date type documentation: https://cloud.google.com/bigquery/data-types
        """
        converted_row = []
        for col_name, col_val in zip(schema, row):
            if type(col_val) in (datetime, date):
                col_val = time.mktime(col_val.timetuple())
            elif isinstance(col_val, Decimal):
                col_val = float(col_val)
            elif col_type_dict.get(col_name) == "BYTES":
                col_val = base64.urlsafe_b64encode(col_val)
                if PY3:
                    col_val = col_val.decode('ascii')
            else:
                col_val = col_val
            converted_row.append(col_val)
        return converted_row

    def _get_col_type_dict(self):
        """
        Return a dict of column name and column type based on self.schema if not None.
        """
        schema = []
        if isinstance(self.schema, string_types):
            schema = json.loads(self.schema)
        elif isinstance(self.schema, list):
            schema = self.schema
        elif self.schema is not None:
            self.log.warn('Using default schema due to unexpected type.'
                          'Should be a string or list.')

        col_type_dict = {}
        try:
            col_type_dict = {col['name']: col['type'] for col in schema}
        except KeyError:
            self.log.warn('Using default schema due to missing name or type. Please '
                          'refer to: https://cloud.google.com/bigquery/docs/schemas'
                          '#specifying_a_json_schema_file')
        return col_type_dict

    @classmethod
    def type_map(cls, mysql_type):
        """
        Helper function that maps from MySQL fields to BigQuery fields. Used
        when a schema_filename is set.
        """
        d = {
            FIELD_TYPE.INT24: 'INTEGER',
            FIELD_TYPE.TINY: 'INTEGER',
            FIELD_TYPE.BIT: 'INTEGER',
            FIELD_TYPE.DATETIME: 'TIMESTAMP',
            FIELD_TYPE.DATE: 'TIMESTAMP',
            FIELD_TYPE.DECIMAL: 'FLOAT',
            FIELD_TYPE.NEWDECIMAL: 'FLOAT',
            FIELD_TYPE.DOUBLE: 'FLOAT',
            FIELD_TYPE.FLOAT: 'FLOAT',
            FIELD_TYPE.LONG: 'INTEGER',
            FIELD_TYPE.LONGLONG: 'INTEGER',
            FIELD_TYPE.SHORT: 'INTEGER',
            FIELD_TYPE.TIMESTAMP: 'TIMESTAMP',
            FIELD_TYPE.YEAR: 'INTEGER',
        }
        return d[mysql_type] if mysql_type in d else 'STRING'
<commit_msg>[AIRFLOW-2551] Encode binary data with base64 standard rather than base64 url<commit_after># -*- coding: utf-8 -*-
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import sys
import json
import time
import base64

from airflow.contrib.hooks.gcs_hook import GoogleCloudStorageHook
from airflow.hooks.mysql_hook import MySqlHook
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults
from datetime import date, datetime
from decimal import Decimal
from MySQLdb.constants import FIELD_TYPE
from tempfile import NamedTemporaryFile
from six import string_types

PY3 = sys.version_info[0] == 3

class MySqlToGoogleCloudStorageOperator(BaseOperator):
    """
    Copy data from MySQL to Google cloud storage in JSON format.
    """
    template_fields = ('sql', 'bucket', 'filename', 'schema_filename', 'schema')
    template_ext = ('.sql',)
    ui_color = '#a0e08c'

    @apply_defaults
    def __init__(self,
                 sql,
                 bucket,
                 filename,
                 schema_filename=None,
                 approx_max_file_size_bytes=1900000000,
                 mysql_conn_id='mysql_default',
                 google_cloud_storage_conn_id='google_cloud_default',
                 schema=None,
                 delegate_to=None,
                 *args,
                 **kwargs):
        """
        :param sql: The SQL to execute on the MySQL table.
        :type sql: string
        :param bucket: The bucket to upload to.
        :type bucket: string
        :param filename: The filename to use as the object name when uploading
            to Google cloud storage. A {} should be specified in the filename
            to allow the operator to inject file numbers in cases where the
            file is split due to size.
        :type filename: string
        :param schema_filename: If set, the filename to use as the object name
            when uploading a .json file containing the BigQuery schema fields
            for the table that was dumped from MySQL.
        :type schema_filename: string
        :param approx_max_file_size_bytes: This operator supports the ability
            to split large table dumps into multiple files (see notes in the
            filenamed param docs above). Google cloud storage allows for files
            to be a maximum of 4GB. This param allows developers to specify the
            file size of the splits.
        :type approx_max_file_size_bytes: long
        :param mysql_conn_id: Reference to a specific MySQL hook.
        :type mysql_conn_id: string
        :param google_cloud_storage_conn_id: Reference to a specific Google
            cloud storage hook.
        :type google_cloud_storage_conn_id: string
        :param schema: The schema to use, if any. Should be a list of dict or
            a str. Pass a string if using Jinja template, otherwise, pass a list of
            dict. Examples could be seen: https://cloud.google.com/bigquery/docs
            /schemas#specifying_a_json_schema_file
        :type schema: str or list
        :param delegate_to: The account to impersonate, if any. For this to
            work, the service account making the request must have domain-wide
            delegation enabled.
        """
        super(MySqlToGoogleCloudStorageOperator, self).__init__(*args, **kwargs)
        self.sql = sql
        self.bucket = bucket
        self.filename = filename
        self.schema_filename = schema_filename
        self.approx_max_file_size_bytes = approx_max_file_size_bytes
        self.mysql_conn_id = mysql_conn_id
        self.google_cloud_storage_conn_id = google_cloud_storage_conn_id
        self.schema = schema
        self.delegate_to = delegate_to

    def execute(self, context):
        cursor = self._query_mysql()
        files_to_upload = self._write_local_data_files(cursor)

        # If a schema is set, create a BQ schema JSON file.
        if self.schema_filename:
            files_to_upload.update(self._write_local_schema_file(cursor))

        # Flush all files before uploading.
        for file_handle in files_to_upload.values():
            file_handle.flush()

        self._upload_to_gcs(files_to_upload)

        # Close all temp file handles.
        for file_handle in files_to_upload.values():
            file_handle.close()

    def _query_mysql(self):
        """
        Queries mysql and returns a cursor to the results.
        """
        mysql = MySqlHook(mysql_conn_id=self.mysql_conn_id)
        conn = mysql.get_conn()
        cursor = conn.cursor()
        cursor.execute(self.sql)
        return cursor

    def _write_local_data_files(self, cursor):
        """
        Takes a cursor, and writes results to a local file.

        :return: A dictionary where keys are filenames to be used as object
            names in GCS, and values are file handles to local files that
            contain the data for the GCS objects.
        """
        schema = list(map(lambda schema_tuple: schema_tuple[0], cursor.description))
        col_type_dict = self._get_col_type_dict()
        file_no = 0
        tmp_file_handle = NamedTemporaryFile(delete=True)
        tmp_file_handles = {self.filename.format(file_no): tmp_file_handle}

        for row in cursor:
            # Convert datetime objects to utc seconds, and decimals to floats.
            # Convert binary type object to string encoded with base64.
            row = self._convert_types(schema, col_type_dict, row)
            row_dict = dict(zip(schema, row))

            # TODO validate that row isn't > 2MB. BQ enforces a hard row size of 2MB.
            s = json.dumps(row_dict)
            if PY3:
                s = s.encode('utf-8')
            tmp_file_handle.write(s)

            # Append newline to make dumps BigQuery compatible.
            tmp_file_handle.write(b'\n')

            # Stop if the file exceeds the file size limit.
            if tmp_file_handle.tell() >= self.approx_max_file_size_bytes:
                file_no += 1
                tmp_file_handle = NamedTemporaryFile(delete=True)
                tmp_file_handles[self.filename.format(file_no)] = tmp_file_handle

        return tmp_file_handles

    def _write_local_schema_file(self, cursor):
        """
        Takes a cursor, and writes the BigQuery schema for the results to a
        local file system.

        :return: A dictionary where key is a filename to be used as an object
            name in GCS, and values are file handles to local files that
            contains the BigQuery schema fields in .json format.
        """
        schema_str = None
        tmp_schema_file_handle = NamedTemporaryFile(delete=True)
        if self.schema is not None and isinstance(self.schema, string_types):
            schema_str = self.schema
        elif self.schema is not None and isinstance(self.schema, list):
            schema_str = json.dumps(self.schema)
        else:
            schema = []
            for field in cursor.description:
                # See PEP 249 for details about the description tuple.
                field_name = field[0]
                field_type = self.type_map(field[1])
                # Always allow TIMESTAMP to be nullable. MySQLdb returns None types
                # for required fields because some MySQL timestamps can't be
                # represented by Python's datetime (e.g. 0000-00-00 00:00:00).
                if field[6] or field_type == 'TIMESTAMP':
                    field_mode = 'NULLABLE'
                else:
                    field_mode = 'REQUIRED'
                schema.append({
                    'name': field_name,
                    'type': field_type,
                    'mode': field_mode,
                })
            schema_str = json.dumps(schema)
        if PY3:
            schema_str = schema_str.encode('utf-8')
        tmp_schema_file_handle.write(schema_str)

        self.log.info('Using schema for %s: %s', self.schema_filename, schema_str)
        return {self.schema_filename: tmp_schema_file_handle}

    def _upload_to_gcs(self, files_to_upload):
        """
        Upload all of the file splits (and optionally the schema .json file) to
        Google cloud storage.
        """
        hook = GoogleCloudStorageHook(
            google_cloud_storage_conn_id=self.google_cloud_storage_conn_id,
            delegate_to=self.delegate_to)
        for object, tmp_file_handle in files_to_upload.items():
            hook.upload(self.bucket, object, tmp_file_handle.name, 'application/json')

    def _convert_types(self, schema, col_type_dict, row):
        """
        Takes a value from MySQLdb, and converts it to a value that's safe for
        JSON/Google cloud storage/BigQuery. Dates are converted to UTC seconds.
        Decimals are converted to floats. Binary type fields are encoded with base64,
        as imported BYTES data must be base64-encoded according to Bigquery SQL
        date type documentation: https://cloud.google.com/bigquery/data-types
        """
        converted_row = []
        for col_name, col_val in zip(schema, row):
            if type(col_val) in (datetime, date):
                col_val = time.mktime(col_val.timetuple())
            elif isinstance(col_val, Decimal):
                col_val = float(col_val)
            elif col_type_dict.get(col_name) == "BYTES":
                col_val = base64.standard_b64encode(col_val)
                if PY3:
                    col_val = col_val.decode('ascii')
            else:
                col_val = col_val
            converted_row.append(col_val)
        return converted_row

    def _get_col_type_dict(self):
        """
        Return a dict of column name and column type based on self.schema if not None.
        """
        schema = []
        if isinstance(self.schema, string_types):
            schema = json.loads(self.schema)
        elif isinstance(self.schema, list):
            schema = self.schema
        elif self.schema is not None:
            self.log.warn('Using default schema due to unexpected type.'
                          'Should be a string or list.')

        col_type_dict = {}
        try:
            col_type_dict = {col['name']: col['type'] for col in schema}
        except KeyError:
            self.log.warn('Using default schema due to missing name or type. Please '
                          'refer to: https://cloud.google.com/bigquery/docs/schemas'
                          '#specifying_a_json_schema_file')
        return col_type_dict

    @classmethod
    def type_map(cls, mysql_type):
        """
        Helper function that maps from MySQL fields to BigQuery fields. Used
        when a schema_filename is set.
        """
        d = {
            FIELD_TYPE.INT24: 'INTEGER',
            FIELD_TYPE.TINY: 'INTEGER',
            FIELD_TYPE.BIT: 'INTEGER',
            FIELD_TYPE.DATETIME: 'TIMESTAMP',
            FIELD_TYPE.DATE: 'TIMESTAMP',
            FIELD_TYPE.DECIMAL: 'FLOAT',
            FIELD_TYPE.NEWDECIMAL: 'FLOAT',
            FIELD_TYPE.DOUBLE: 'FLOAT',
            FIELD_TYPE.FLOAT: 'FLOAT',
            FIELD_TYPE.LONG: 'INTEGER',
            FIELD_TYPE.LONGLONG: 'INTEGER',
            FIELD_TYPE.SHORT: 'INTEGER',
            FIELD_TYPE.TIMESTAMP: 'TIMESTAMP',
            FIELD_TYPE.YEAR: 'INTEGER',
        }
        return d[mysql_type] if mysql_type in d else 'STRING'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from django.db import models
from django.conf import settings
from django.template.defaultfilters import slugify
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType

from niwi.utils import get_url_data
from niwi.forms.paste import LEXER_CHOICES
from niwi.contrib.db.fields import CreationDateTimeField, ModificationDateTimeField

import datetime, uuid

STATUS_CHOICES = (
    ('public', u'Public'),
    ('private', u'Private'),
    ('draft', u'Draft'),
)

def slugify_uniquely(value, model, slugfield="slug"):
    suffix = 0
    potential = base = slugify(value)
    if len(potential) == 0:
        potential = 'null'
    while True:
        if suffix:
            potential = "-".join([base, str(suffix)])
        if not model.objects.filter(**{slugfield: potential}).count():
            return potential
        suffix += 1

class Page(models.Model):
    slug  = models.SlugField(unique=True, db_index=True, editable=True)
    title = models.CharField(max_length=500, db_index=True)
    content = models.TextField()
    markup = models.BooleanField(default=False)
    status = models.CharField(max_length=40, choices=STATUS_CHOICES, db_index=True, default='draft')
    owner = models.ForeignKey('auth.User', related_name='pages')
    created_date = CreationDateTimeField(editable=True)
    modified_date = ModificationDateTimeField(editable=True)

    class Meta:
        db_table = 'pages'

    @models.permalink
    def get_absolute_url(self):
        return ('web:show-page', (), {'slug': self.slug})

    def save(self, *args, **kwargs):
        if not self.slug:
            self.slug = slugify_uniquely(self.title, self.__class__)

        super(Page, self).save(*args, **kwargs)

class Post(models.Model):
    slug  = models.SlugField(unique=True, db_index=True, editable=True)
    title = models.CharField(max_length=500, db_index=True, blank=True)
    content = models.TextField()
    markup = models.BooleanField(default=False)
    owner = models.ForeignKey('auth.User', related_name='posts')
    status = models.CharField(max_length=40, choices=STATUS_CHOICES, db_index=True, default='draft')

    created_date = CreationDateTimeField(editable=True)
    modified_date = ModificationDateTimeField(editable=True)

    class Meta:
        db_table = 'posts'

    def save(self, *args, **kwargs):
        if not self.slug:
            self.slug = slugify_uniquely(self.title, self.__class__)

        super(Post, self).save(*args, **kwargs)

    @models.permalink
    def get_absolute_url(self):
        return ('web:show-post', (), {'slug': self.slug})

class Bookmark(models.Model):
    title = models.CharField(max_length=500, blank=True)
    slug = models.SlugField(unique=True, db_index=True, editable=True, blank=True)
    url = models.CharField(max_length=1000, unique=True, db_index=True)
    tags = models.CharField(max_length=1000, db_index=True, blank=True, default='')
    owner = models.ForeignKey('auth.User', related_name='bookmarks')

    created_date = models.DateTimeField(auto_now_add=True)
    modified_date = models.DateTimeField(auto_now_add=True)

    public = models.BooleanField(default=True)

    class Meta:
        db_table = 'bookmarks'

    def save(self, *args, **kwargs):
        if not self.title:
            self.title, body = get_url_data(self.url)
        if not self.slug:
            self.slug = slugify_uniquely(self.title, self.__class__)
        super(Bookmark, self).save(*args, **kwargs)

    def __unicode__(self):
        return self.slug

    @models.permalink
    def get_absolute_url(self):
        return ('web:show-bookmark', (), {'slug': self.slug})

class Config(models.Model):
    google_analytics_domain = models.CharField(max_length=200, blank=True)
    google_analytics_code = models.CharField(max_length=200, blank=True)
    core_homepage = models.CharField(max_length=200, blank=True)

    class Meta:
        db_table = 'config'

class Paste(models.Model):
    text = models.TextField()
    lexer = models.CharField(max_length=5)
    title = models.CharField(max_length=100, blank=True)
    group = models.CharField(max_length=100, blank=True)
    created = models.DateTimeField(auto_now_add=True)

    class Meta:
        db_table = 'paste'
<commit_msg>Fix on slug field.<commit_after># -*- coding: utf-8 -*-

from django.db import models
from django.conf import settings
from django.template.defaultfilters import slugify
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType

from niwi.utils import get_url_data
from niwi.forms.paste import LEXER_CHOICES
from niwi.contrib.db.fields import CreationDateTimeField, ModificationDateTimeField

import datetime, uuid

STATUS_CHOICES = (
    ('public', u'Public'),
    ('private', u'Private'),
    ('draft', u'Draft'),
)

def slugify_uniquely(value, model, slugfield="slug"):
    suffix = 0
    potential = base = slugify(value)
    if len(potential) == 0:
        potential = 'null'
    while True:
        if suffix:
            potential = "-".join([base, str(suffix)])
        if not model.objects.filter(**{slugfield: potential}).count():
            return potential
        suffix += 1

class Page(models.Model):
    slug  = models.SlugField(unique=True, db_index=True, editable=True, blank=True)
    title = models.CharField(max_length=500, db_index=True)
    content = models.TextField()
    markup = models.BooleanField(default=False)
    status = models.CharField(max_length=40, choices=STATUS_CHOICES, db_index=True, default='draft')
    owner = models.ForeignKey('auth.User', related_name='pages')
    created_date = CreationDateTimeField(editable=True)
    modified_date = ModificationDateTimeField(editable=True)

    class Meta:
        db_table = 'pages'

    @models.permalink
    def get_absolute_url(self):
        return ('web:show-page', (), {'slug': self.slug})

    def save(self, *args, **kwargs):
        if not self.slug:
            self.slug = slugify_uniquely(self.title, self.__class__)

        super(Page, self).save(*args, **kwargs)

class Post(models.Model):
    slug  = models.SlugField(unique=True, db_index=True, editable=True, blank=True)
    title = models.CharField(max_length=500, db_index=True, blank=True)
    content = models.TextField()
    markup = models.BooleanField(default=False)
    owner = models.ForeignKey('auth.User', related_name='posts')
    status = models.CharField(max_length=40, choices=STATUS_CHOICES, db_index=True, default='draft')

    created_date = CreationDateTimeField(editable=True)
    modified_date = ModificationDateTimeField(editable=True)

    class Meta:
        db_table = 'posts'

    def save(self, *args, **kwargs):
        if not self.slug:
            self.slug = slugify_uniquely(self.title, self.__class__)

        super(Post, self).save(*args, **kwargs)

    @models.permalink
    def get_absolute_url(self):
        return ('web:show-post', (), {'slug': self.slug})

class Bookmark(models.Model):
    title = models.CharField(max_length=500, blank=True)
    slug = models.SlugField(unique=True, db_index=True, editable=True, blank=True)
    url = models.CharField(max_length=1000, unique=True, db_index=True)
    tags = models.CharField(max_length=1000, db_index=True, blank=True, default='')
    owner = models.ForeignKey('auth.User', related_name='bookmarks')

    created_date = models.DateTimeField(auto_now_add=True)
    modified_date = models.DateTimeField(auto_now_add=True)

    public = models.BooleanField(default=True)

    class Meta:
        db_table = 'bookmarks'

    def save(self, *args, **kwargs):
        if not self.title:
            self.title, body = get_url_data(self.url)
        if not self.slug:
            self.slug = slugify_uniquely(self.title, self.__class__)
        super(Bookmark, self).save(*args, **kwargs)

    def __unicode__(self):
        return self.slug

    @models.permalink
    def get_absolute_url(self):
        return ('web:show-bookmark', (), {'slug': self.slug})

class Config(models.Model):
    google_analytics_domain = models.CharField(max_length=200, blank=True)
    google_analytics_code = models.CharField(max_length=200, blank=True)
    core_homepage = models.CharField(max_length=200, blank=True)

    class Meta:
        db_table = 'config'

class Paste(models.Model):
    text = models.TextField()
    lexer = models.CharField(max_length=5)
    title = models.CharField(max_length=100, blank=True)
    group = models.CharField(max_length=100, blank=True)
    created = models.DateTimeField(auto_now_add=True)

    class Meta:
        db_table = 'paste'
<|endoftext|>


<commit_before>import json

from django.http import HttpResponse
from django.shortcuts import get_object_or_404
from django.db import connection

from catmaid.control.authentication import *
from catmaid.control.common import *
from catmaid.models import ClassInstance, ClassInstanceClassInstance, Treenode

import operator
from collections import defaultdict

@requires_user_role([UserRole.Annotate, UserRole.Browse])
def get_all_skeletons_of_neuron(request, project_id=None, neuron_id=None):
    p = get_object_or_404(Project, pk=project_id)
    neuron = get_object_or_404(ClassInstance,
        pk=neuron_id,
        class_column__class_name='neuron',
        project=p)
    qs = ClassInstance.objects.filter(
        project=p,
        cici_via_a__relation__relation_name='model_of',
        cici_via_a__class_instance_b=neuron)
    return HttpResponse(json.dumps([x.id for x in qs]), mimetype="text/json")

def _delete_if_empty(neuron_id):
    """ Delete this neuron if no class_instance is a model_of it;
    which is to say, it contains no skeletons. """
    is_empty = not ClassInstanceClassInstance.objects.filter(
            class_instance_b=neuron_id,
            relation__relation_name='model_of').exists()
    if is_empty:
        ClassInstance.objects.filter(pk=neuron_id).delete()
    return is_empty

@requires_user_role(UserRole.Annotate)
def delete_neuron(request, project_id=None, neuron_id=None):
    """ Deletes a neuron if and only if two things are the case: 1. The user
    ownes all treenodes of the skeleton modeling the neuron in question and
    2. The neuron is not annotated by other users.
    """
    # Make sure the user can edit the neuron in general
    can_edit_class_instance_or_fail(request.user, neuron_id, 'neuron')

    # Create class and relation dictionaries
    classes = dict(Class.objects.filter(
            project_id=project_id).values_list('class_name', 'id'))
    relations = dict(Relation.objects.filter(
            project_id=project_id).values_list('relation_name', 'id'))

    # Make sure the user has permission to edit all treenodes of all skeletons
    skeleton_ids = ClassInstanceClassInstance.objects.filter(
            class_instance_b=neuron_id,
            relation_id=relations['model_of']).values_list(
                    'class_instance_a', flat=True)
    for skid in skeleton_ids:
        others_nodes = Treenode.objects.filter(skeleton_id=skid).exclude(
                user_id=request.user.id).values_list('id', flat=True)
        if others_nodes:
            try:
                can_edit_all_or_fail(request.user, others_nodes, 'treenode')
            except Exception:
                raise Exception("You don't have permission to remove all " \
                        "treenodes of skeleton %s modeling this neuron. The " \
                        "neuron won't be deleted." % skid)

    # Make sure the user has permission to edit all annotations of this neuron
    annotation_ids = set(ClassInstanceClassInstance.objects.filter(
            class_instance_a_id=neuron_id,
            relation_id=relations['annotated_with']).values_list(
                    'id', flat=True))
    if annotation_ids:
        try:
            can_edit_all_or_fail(request.user, annotation_ids,
                    'class_instance_class_instance')
        except Exception:
            raise Exception("You don't have permission to remove all " \
                    "annotations linked to this neuron. The neuron won't " \
                    "be deleted.")

    # Try to get the root node to have a valid location for a log entry
    if skeleton_ids:
        try:
            root_node = Treenode.objects.get(
                    skeleton_id=skeleton_ids[0], parent=None)
            root_location = (root_node.location_x, root_node.location_y,
                             root_node.location_z)
        except (Treenode.DoesNotExist, Treenode.MultipleObjectsReturned):
            root_location = None
    else:
        root_location = None

    # Delete neuron (and implicitely all annotation links due to Django's
    # cascading deletion)
    neuron = get_object_or_404(ClassInstance, pk=neuron_id)
    neuron.delete()

    # Delete all annotations that are not used anymore
    used_annotation_ids = set(ClassInstanceClassInstance.objects.filter(
            class_instance_b_id__in=annotation_ids,
            relation_id=relations['annotated_with']).values_list(
                    'id', flat=True))
    unused_annotation_ids = annotation_ids.difference(used_annotation_ids)
    ClassInstance.objects.filter(id__in=unused_annotation_ids).delete()

    # Delete the skeletons (and their treenodes through cascading delete)
    cursor = connection.cursor()
    for skid in skeleton_ids:
        # Because there are constraints used in the database that Django is not
        # aware of, it's emulation of cascading deletion doesn't work.
        # Therefore, raw SQL needs to be used to use true cascading deletion.
        cursor.execute('''
        BEGIN;
        DELETE FROM change_request WHERE treenode_id IN (
            SELECT id FROM treenode WHERE skeleton_id=%s AND project_id=%s);
        DELETE FROM change_request WHERE connector_id IN (
            SELECT id FROM treenode_connector WHERE skeleton_id=%s AND project_id=%s);
        DELETE FROM treenode_class_instance WHERE treenode_id IN (
            SELECT id FROM treenode WHERE skeleton_id=%s AND project_id=%s);
        DELETE FROM treenode WHERE skeleton_id=%s AND project_id=%s;
        DELETE FROM treenode_connector WHERE skeleton_id=%s AND project_id=%s;
        DELETE FROM class_instance WHERE id=%s AND project_id=%s;
        DELETE FROM review WHERE skeleton_id=%s AND project_id=%s;
        COMMIT;
        ''', (skid, project_id, skid, project_id, skid, project_id, skid,
              project_id, skid, project_id, skid, project_id, skid, project_id))

    # Insert log entry and refer to position of the first skeleton's root node
    insert_into_log(project_id, request.user.id, 'remove_neuron', root_location,
            'Deleted neuron %s and skeleton(s) %s.' % (neuron_id,
                    ', '.join([str(s) for s in skeleton_ids])))

    return HttpResponse(json.dumps({
        'success': "Deleted neuron #%s as well as it's skeletons and " \
                "annotations." % neuron_id}))

@requires_user_role(UserRole.Annotate)
def give_neuron_to_other_user(request, project_id=None, neuron_id=None):
    neuron_id = int(neuron_id)
    target_user = User.objects.get(pk=int(request.POST['target_user_id']))

    # 1. Check that the request.user is superuser
    #    or owns the neuron and the skeletons under it
    neuron = ClassInstance.objects.get(pk=neuron_id)
    if not request.user.is_superuser and neuron.user.id != request.user.id:
        return HttpResponse(json.dumps({'error': 'You don\'t own the neuron!'}))

    qs = ClassInstanceClassInstance.objects.filter(
            class_instance_b=neuron_id,
            relation__relation_name='model_of').values_list('class_instance_a__user_id', 'class_instance_a')
    skeletons = defaultdict(list) # user_id vs list of owned skeletons
    for row in qs:
        skeletons[row[0]].append(row[1])

    if not skeletons:
        return HttpResponse(json.dumps({'error': 'The neuron does not contain any skeletons!'}))

    sks = {k:v[:] for k,v in skeletons.iteritems()} # deep copy
    if request.user.id in sks:
        del sks[request.user.id]
    if not request.user.is_superuser and sks:
        return HttpResponse(json.dumps({'error': 'You don\'t own: %s' % reduce(operator.add, sks.values())}))

    # 2. Change neuron's and skeleton's and class_instance_class_instance relationship owner to target_user

    # Update user_id of the relation 'model_of' between any skeletons and the chosen neuron
    ClassInstanceClassInstance.objects.filter(
        relation__relation_name='model_of',
        class_instance_b=neuron_id).update(user=target_user)

    # Update user_id of the neuron
    ClassInstance.objects.filter(pk=neuron_id).update(user=target_user)

    # Update user_id of the skeleton(s)
    ClassInstance.objects.filter(pk__in=reduce(operator.add, skeletons.values())).update(user=target_user)

    return HttpResponse(json.dumps({'success':'Moved neuron #%s to %s staging area.'}))

<commit_msg>Fix typo<commit_after>import json

from django.http import HttpResponse
from django.shortcuts import get_object_or_404
from django.db import connection

from catmaid.control.authentication import *
from catmaid.control.common import *
from catmaid.models import ClassInstance, ClassInstanceClassInstance, Treenode

import operator
from collections import defaultdict

@requires_user_role([UserRole.Annotate, UserRole.Browse])
def get_all_skeletons_of_neuron(request, project_id=None, neuron_id=None):
    p = get_object_or_404(Project, pk=project_id)
    neuron = get_object_or_404(ClassInstance,
        pk=neuron_id,
        class_column__class_name='neuron',
        project=p)
    qs = ClassInstance.objects.filter(
        project=p,
        cici_via_a__relation__relation_name='model_of',
        cici_via_a__class_instance_b=neuron)
    return HttpResponse(json.dumps([x.id for x in qs]), mimetype="text/json")

def _delete_if_empty(neuron_id):
    """ Delete this neuron if no class_instance is a model_of it;
    which is to say, it contains no skeletons. """
    is_empty = not ClassInstanceClassInstance.objects.filter(
            class_instance_b=neuron_id,
            relation__relation_name='model_of').exists()
    if is_empty:
        ClassInstance.objects.filter(pk=neuron_id).delete()
    return is_empty

@requires_user_role(UserRole.Annotate)
def delete_neuron(request, project_id=None, neuron_id=None):
    """ Deletes a neuron if and only if two things are the case: 1. The user
    ownes all treenodes of the skeleton modeling the neuron in question and
    2. The neuron is not annotated by other users.
    """
    # Make sure the user can edit the neuron in general
    can_edit_class_instance_or_fail(request.user, neuron_id, 'neuron')

    # Create class and relation dictionaries
    classes = dict(Class.objects.filter(
            project_id=project_id).values_list('class_name', 'id'))
    relations = dict(Relation.objects.filter(
            project_id=project_id).values_list('relation_name', 'id'))

    # Make sure the user has permission to edit all treenodes of all skeletons
    skeleton_ids = ClassInstanceClassInstance.objects.filter(
            class_instance_b=neuron_id,
            relation_id=relations['model_of']).values_list(
                    'class_instance_a', flat=True)
    for skid in skeleton_ids:
        others_nodes = Treenode.objects.filter(skeleton_id=skid).exclude(
                user_id=request.user.id).values_list('id', flat=True)
        if others_nodes:
            try:
                can_edit_all_or_fail(request.user, others_nodes, 'treenode')
            except Exception:
                raise Exception("You don't have permission to remove all " \
                        "treenodes of skeleton %s modeling this neuron. The " \
                        "neuron won't be deleted." % skid)

    # Make sure the user has permission to edit all annotations of this neuron
    annotation_ids = set(ClassInstanceClassInstance.objects.filter(
            class_instance_a_id=neuron_id,
            relation_id=relations['annotated_with']).values_list(
                    'id', flat=True))
    if annotation_ids:
        try:
            can_edit_all_or_fail(request.user, annotation_ids,
                    'class_instance_class_instance')
        except Exception:
            raise Exception("You don't have permission to remove all " \
                    "annotations linked to this neuron. The neuron won't " \
                    "be deleted.")

    # Try to get the root node to have a valid location for a log entry
    if skeleton_ids:
        try:
            root_node = Treenode.objects.get(
                    skeleton_id=skeleton_ids[0], parent=None)
            root_location = (root_node.location_x, root_node.location_y,
                             root_node.location_z)
        except (Treenode.DoesNotExist, Treenode.MultipleObjectsReturned):
            root_location = None
    else:
        root_location = None

    # Delete neuron (and implicitely all annotation links due to Django's
    # cascading deletion)
    neuron = get_object_or_404(ClassInstance, pk=neuron_id)
    neuron.delete()

    # Delete all annotations that are not used anymore
    used_annotation_ids = set(ClassInstanceClassInstance.objects.filter(
            class_instance_b_id__in=annotation_ids,
            relation_id=relations['annotated_with']).values_list(
                    'id', flat=True))
    unused_annotation_ids = annotation_ids.difference(used_annotation_ids)
    ClassInstance.objects.filter(id__in=unused_annotation_ids).delete()

    # Delete the skeletons (and their treenodes through cascading delete)
    cursor = connection.cursor()
    for skid in skeleton_ids:
        # Because there are constraints used in the database that Django is not
        # aware of, it's emulation of cascading deletion doesn't work.
        # Therefore, raw SQL needs to be used to use true cascading deletion.
        cursor.execute('''
        BEGIN;
        DELETE FROM change_request WHERE treenode_id IN (
            SELECT id FROM treenode WHERE skeleton_id=%s AND project_id=%s);
        DELETE FROM change_request WHERE connector_id IN (
            SELECT id FROM treenode_connector WHERE skeleton_id=%s AND project_id=%s);
        DELETE FROM treenode_class_instance WHERE treenode_id IN (
            SELECT id FROM treenode WHERE skeleton_id=%s AND project_id=%s);
        DELETE FROM treenode WHERE skeleton_id=%s AND project_id=%s;
        DELETE FROM treenode_connector WHERE skeleton_id=%s AND project_id=%s;
        DELETE FROM class_instance WHERE id=%s AND project_id=%s;
        DELETE FROM review WHERE skeleton_id=%s AND project_id=%s;
        COMMIT;
        ''', (skid, project_id, skid, project_id, skid, project_id, skid,
              project_id, skid, project_id, skid, project_id, skid, project_id))

    # Insert log entry and refer to position of the first skeleton's root node
    insert_into_log(project_id, request.user.id, 'remove_neuron', root_location,
            'Deleted neuron %s and skeleton(s) %s.' % (neuron_id,
                    ', '.join([str(s) for s in skeleton_ids])))

    return HttpResponse(json.dumps({
        'success': "Deleted neuron #%s as well as its skeletons and " \
                "annotations." % neuron_id}))

@requires_user_role(UserRole.Annotate)
def give_neuron_to_other_user(request, project_id=None, neuron_id=None):
    neuron_id = int(neuron_id)
    target_user = User.objects.get(pk=int(request.POST['target_user_id']))

    # 1. Check that the request.user is superuser
    #    or owns the neuron and the skeletons under it
    neuron = ClassInstance.objects.get(pk=neuron_id)
    if not request.user.is_superuser and neuron.user.id != request.user.id:
        return HttpResponse(json.dumps({'error': 'You don\'t own the neuron!'}))

    qs = ClassInstanceClassInstance.objects.filter(
            class_instance_b=neuron_id,
            relation__relation_name='model_of').values_list('class_instance_a__user_id', 'class_instance_a')
    skeletons = defaultdict(list) # user_id vs list of owned skeletons
    for row in qs:
        skeletons[row[0]].append(row[1])

    if not skeletons:
        return HttpResponse(json.dumps({'error': 'The neuron does not contain any skeletons!'}))

    sks = {k:v[:] for k,v in skeletons.iteritems()} # deep copy
    if request.user.id in sks:
        del sks[request.user.id]
    if not request.user.is_superuser and sks:
        return HttpResponse(json.dumps({'error': 'You don\'t own: %s' % reduce(operator.add, sks.values())}))

    # 2. Change neuron's and skeleton's and class_instance_class_instance relationship owner to target_user

    # Update user_id of the relation 'model_of' between any skeletons and the chosen neuron
    ClassInstanceClassInstance.objects.filter(
        relation__relation_name='model_of',
        class_instance_b=neuron_id).update(user=target_user)

    # Update user_id of the neuron
    ClassInstance.objects.filter(pk=neuron_id).update(user=target_user)

    # Update user_id of the skeleton(s)
    ClassInstance.objects.filter(pk__in=reduce(operator.add, skeletons.values())).update(user=target_user)

    return HttpResponse(json.dumps({'success':'Moved neuron #%s to %s staging area.'}))

<|endoftext|>


<commit_before>from os.path import exists, join

import click
from giturlparse import parse
from plumbum.cmd import git
from plumbum import local
import yaml

CONFIG_FILE = '.changes'
DEFAULTS = {
    'changelog': 'CHANGELOG.md',
    'readme': 'README.md',
    'gh_token': None,
}

class CLI(object):
    test_command = None
    pypi = None
    skip_changelog = None
    changelog_content = None
    parsed_repo = None

    def __init__(self, module_name, dry_run, debug, no_input, requirements,
                 new_version, current_version, repo_url, version_prefix):
        self.module_name = module_name
        self.dry_run = dry_run
        self.debug = debug
        self.no_input = no_input
        self.requirements = requirements
        self.new_version = (
            version_prefix + new_version
            if version_prefix
            else new_version
        )
        self.current_version = current_version
        self.repo_url = repo_url

    @property
    def repo(self):
        return self.parsed_repo.repo

    @property
    def owner(self):
        return self.parsed_repo.owner

    @property
    def github(self):
        return self.parsed_repo.github

    @property
    def bitbucket(self):
        return self.parsed_repo.bitbucket

    @property
    def parsed_repo(self):
        with local.cwd(local.cwd / self.module_name):
            return parse(git('config --get remote.origin.url'.split(' ')))

def project_config(context):
    config = {}
    config_path = join(context.module_name, CONFIG_FILE)

    # initialise config with defaults
    if not exists(config_path):
        config = DEFAULTS.copy()

        with click.open_file(config_path, 'w') as f:
            config_yaml = yaml.dump(config, default_flow_style=False)
            f.write(config_yaml)

    config = yaml.safe_load(click.open_file(config_path))
    return config or {}

def store_settings(module_name, settings):
    config_path = path(join(module_name, CONFIG_FILE))
    with click.open_file(config_path, 'w') as f:
        f.write(yaml.dump(settings, default_flow_style=False))
<commit_msg>Store initial settings<commit_after>from os.path import exists, join
import io

import click
from giturlparse import parse
from path import path
from plumbum.cmd import git
from plumbum import local
import yaml

CONFIG_FILE = '.changes'
DEFAULTS = {
    'changelog': 'CHANGELOG.md',
    'readme': 'README.md',
    'gh_token': None,
}

class CLI(object):
    test_command = None
    pypi = None
    skip_changelog = None
    changelog_content = None
    parsed_repo = None

    def __init__(self, module_name, dry_run, debug, no_input, requirements,
                 new_version, current_version, repo_url, version_prefix):
        self.module_name = module_name
        self.dry_run = dry_run
        self.debug = debug
        self.no_input = no_input
        self.requirements = requirements
        self.new_version = (
            version_prefix + new_version
            if version_prefix
            else new_version
        )
        self.current_version = current_version
        self.repo_url = repo_url

    @property
    def repo(self):
        return self.parsed_repo.repo

    @property
    def owner(self):
        return self.parsed_repo.owner

    @property
    def github(self):
        return self.parsed_repo.github

    @property
    def bitbucket(self):
        return self.parsed_repo.bitbucket

    @property
    def parsed_repo(self):
        with local.cwd(local.cwd / self.module_name):
            return parse(git('config --get remote.origin.url'.split(' ')))

def project_config(module_name):
    config = {}
    config_path = path(join(module_name, CONFIG_FILE))

    if not exists(config_path):
        store_settings(module_name, DEFAULTS.copy())

    config = yaml.load(io.open(config_path))
    return config or {}

def store_settings(module_name, settings):
    config_path = path(join(module_name, CONFIG_FILE))
    with click.open_file(config_path, 'w') as f:
        f.write(yaml.dump(settings, default_flow_style=False))
<|endoftext|>


<commit_before>from .models import Alarm
from rest_framework import serializers
from measurement.serializers import AlarmMeasurementSerializer, PatientAlarmMeasurementSerializer

class AlarmSerializer(serializers.ModelSerializer):
    measurement = AlarmMeasurementSerializer()

    class Meta:
        model = Alarm
        fields = ('id', 'measurement', 'time_created', 'is_treated')

class PatientAlarmSerializer(serializers.ModelSerializer):
    measurement = PatientAlarmMeasurementSerializer()

    class Meta:
        model = Alarm
        fields = (
            'id',
            'measurement',
            'time_created',
            'is_treated',
            'treated_text',
            'search_tag',
            'reason'
        )
<commit_msg>Include reason in AlarmSerializer<commit_after>from .models import Alarm
from rest_framework import serializers
from measurement.serializers import AlarmMeasurementSerializer, PatientAlarmMeasurementSerializer

class AlarmSerializer(serializers.ModelSerializer):
    measurement = AlarmMeasurementSerializer()

    class Meta:
        model = Alarm
        fields = ('id', 'measurement', 'time_created', 'is_treated', 'reason')

class PatientAlarmSerializer(serializers.ModelSerializer):
    measurement = PatientAlarmMeasurementSerializer()

    class Meta:
        model = Alarm
        fields = (
            'id',
            'measurement',
            'time_created',
            'is_treated',
            'treated_text',
            'search_tag',
            'reason'
        )
<|endoftext|>


<commit_before>from django.http import HttpResponse

def index(request):
	return HttpResponse("Hello, world. You're at the core index ")
<commit_msg>suppression des apps<commit_after>from django.shortcuts import render

# Create your views here.
<|endoftext|>


<commit_before>from matplotlib import animation
from matplotlib import pyplot as plt
from IPython.display import HTML
from tempfile import NamedTemporaryFile
import base64

def image_stack_to_movie(images, frames=None):

    if frames is None:
        frames = images.shape[0]

    fig = plt.figure(figsize=(8, 6))
    ax = fig.add_subplot(111)
    im = plt.imshow(images[1], vmin=-10, vmax=10, cmap='CMRmap',
                    interpolation='none')
    fig.colorbar(im)
    for item in ([ax.xaxis.label, ax.yaxis.label] +
                 ax.get_xticklabels() + ax.get_yticklabels()):
        item.set_fontsize(20)

    def animate(i):
        im.set_array(images[i])
        ax.set_title('Frame {}'.format(i), fontsize=24, fontweight='bold')
        return im,

    anim = animation.FuncAnimation(fig, animate, frames=1000,
                                   interval=1, blit=True)
    plt.close(anim._fig)
    return HTML(anim_to_html(anim))

def anim_to_html(anim):
    VIDEO_TAG = """<video controls>
    <source src="data:video/x-m4v;base64,{0}" type="video/mp4">
    Your browser does not support the video tag.
    </video>"""

    if not hasattr(anim, '_encoded_video'):
        with NamedTemporaryFile(suffix='.mp4') as f:
            anim.save(f.name, fps=10, extra_args=['-vcodec', 'libx264',
                                                  '-pix_fmt', 'yuv420p'])
            video = open(f.name, "rb").read()
        anim._encoded_video = base64.b64encode(video)
    return VIDEO_TAG.format(anim._encoded_video.decode("utf-8"))
<commit_msg>[ENH] Added options<commit_after>from matplotlib import animation
from matplotlib import pyplot as plt
from IPython.display import HTML
from tempfile import NamedTemporaryFile
import base64

def image_stack_to_movie(images, frames=None, vmin=None, vmax=None,
                         figsize=(6, 5), cmap='CMRmap', fps=10):

    if frames is None:
        frames = images.shape[0]

    fig = plt.figure(figsize=figsize)
    ax = fig.add_subplot(111)
    im = plt.imshow(images[1], vmin=vmin, vmax=vmax, cmap=cmap,
                    interpolation='none')
    fig.colorbar(im)
    for item in ([ax.xaxis.label, ax.yaxis.label] +
                 ax.get_xticklabels() + ax.get_yticklabels()):
        item.set_fontsize(14)
        item.set_fontweight('bold')

    def animate(i):
        im.set_array(images[i])
        ax.set_title('Frame {}'.format(i), fontsize=16, fontweight='bold')
        return im,

    anim = animation.FuncAnimation(fig, animate, frames=frames,
                                   interval=1, blit=True)
    plt.close(anim._fig)
    return HTML(anim_to_html(anim, fps))

def anim_to_html(anim, fps):
    VIDEO_TAG = """<video controls>
    <source src="data:video/x-m4v;base64,{0}" type="video/mp4">
    Your browser does not support the video tag.
    </video>"""

    if not hasattr(anim, '_encoded_video'):
        with NamedTemporaryFile(suffix='.mp4') as f:
            anim.save(f.name, fps=fps, extra_args=['-vcodec', 'libx264',
                                                   '-pix_fmt', 'yuv420p'])
            video = open(f.name, "rb").read()
        anim._encoded_video = base64.b64encode(video)
    return VIDEO_TAG.format(anim._encoded_video.decode("utf-8"))
<|endoftext|>


<commit_before>from pprint import pprint

from django.db import connections
from django.shortcuts import render
from django.urls import reverse
from django.views import View

from utils.functions.views import (
    cleanned_fields_to_context,
    context_to_form_post,
    )

from estoque import forms
from estoque import models

class ListaMovimentos(View):

    Form_class = forms.ListaMovimentosForm
    template_name = 'estoque/lista_movs.html'
    title_name = 'Movimentações de um documento'

    cleanned_fields_to_context = cleanned_fields_to_context
    context_to_form_post = context_to_form_post

    def __init__(self):
        self.context = {'titulo': self.title_name}

    def mount_context(self):
        self.cursor = connections['so'].cursor()

        try:
            documento = models.DocMovStq.objects.get(
                num_doc=self.context['num_doc'])
        except models.DocMovStq.DoesNotExist:
            self.context.update({
                'erro_msg': 'Documento não encontrado'
            })
            return

        self.context.update({
            'infos': {
                'Data': documento.data,
                'Descrição': documento.descricao,
                'Usuário': documento.usuario.username,
            }
        })

        dados = models.MovStq.objects.filter(
            documento=documento)
        if len(dados) == 0:
            self.context.update({
                'erro_msg': 'Nenhum movimento encontrado'
            })
            return

        fields = [
            'tipo_mov__descricao',
            'item__produto__nivel',
            'item__produto__referencia',
            'item__cor__cor',
            'item__tamanho__tamanho__nome',
            'quantidade',
            'deposito_origem',
            'deposito_destino',
            'usuario__username',
        ]
        dados = dados.values(*fields)

        for row in dados:
            if row['deposito_origem'] == 0:
                row['deposito_origem'] = '-'
            else:
                row['deposito_origem|GLYPHICON'] = 'glyphicon-time'
                row['deposito_origem|TARGET'] = '_BLANK'
                row['deposito_origem|LINK'] = reverse(
                    'estoque:item_no_tempo__get', args=[
                        row['item__produto__referencia'],
                        row['item__cor__cor'],
                        row['item__tamanho__tamanho__nome'],
                        row['deposito_origem']])

            if row['deposito_destino'] == 0:
                row['deposito_destino'] = '-'
            else:
                row['deposito_destino|GLYPHICON'] = 'glyphicon-time'
                row['deposito_destino|TARGET'] = '_BLANK'
                row['deposito_destino|LINK'] = reverse(
                    'estoque:item_no_tempo__get', args=[
                        row['item__produto__referencia'],
                        row['item__cor__cor'],
                        row['item__tamanho__tamanho__nome'],
                        row['deposito_destino']])

        headers = [
            'Tipo de Movimento',
            'Nível',
            'Referência',
            'Cor',
            'Tamanho',
            'Quantidade',
            'Dep. origem',
            'Dep. destino',
            'Usuário',
        ]
        self.context.update({
            'headers': headers,
            'fields': fields,
            'dados': dados,
            })

    def get(self, request, *args, **kwargs):
        if 'num_doc' in kwargs:
            return self.post(request, *args, **kwargs)
        self.context['form'] = self.Form_class()
        return render(request, self.template_name, self.context)

    def post(self, request, *args, **kwargs):
        self.request = request
        if 'num_doc' in kwargs:
            self.context['post'] = True
            self.context['form'] = self.Form_class(kwargs)
        else:
            self.context['post'] = 'busca' in self.request.POST
            self.context['form'] = self.Form_class(self.request.POST)
        if self.context['form'].is_valid():
            self.cleanned_fields_to_context()
            self.mount_context()
            self.context_to_form_post()
            self.context['form'] = self.Form_class(self.context)
        return render(request, self.template_name, self.context)
<commit_msg>mostra novo item (renomeado)<commit_after>from pprint import pprint

from django.db import connections
from django.shortcuts import render
from django.urls import reverse
from django.views import View

from utils.functions.views import (
    cleanned_fields_to_context,
    context_to_form_post,
    )

from estoque import forms
from estoque import models

class ListaMovimentos(View):

    Form_class = forms.ListaMovimentosForm
    template_name = 'estoque/lista_movs.html'
    title_name = 'Movimentações de um documento'

    cleanned_fields_to_context = cleanned_fields_to_context
    context_to_form_post = context_to_form_post

    def __init__(self):
        self.context = {'titulo': self.title_name}

    def mount_context(self):
        self.cursor = connections['so'].cursor()

        try:
            documento = models.DocMovStq.objects.get(
                num_doc=self.context['num_doc'])
        except models.DocMovStq.DoesNotExist:
            self.context.update({
                'erro_msg': 'Documento não encontrado'
            })
            return

        self.context.update({
            'infos': {
                'Data': documento.data,
                'Descrição': documento.descricao,
                'Usuário': documento.usuario.username,
            }
        })

        dados = models.MovStq.objects.filter(
            documento=documento)
        if len(dados) == 0:
            self.context.update({
                'erro_msg': 'Nenhum movimento encontrado'
            })
            return

        fields = [
            'tipo_mov__descricao',
            'item__produto__nivel',
            'item__produto__referencia',
            'item__cor__cor',
            'item__tamanho__tamanho__nome',
            'quantidade',
            'deposito_origem',
            'deposito_destino',
            'novo_item__produto__nivel',
            'novo_item__produto__referencia',
            'novo_item__cor__cor',
            'novo_item__tamanho__tamanho__nome',
            'usuario__username',
        ]
        dados = dados.values(*fields)

        for row in dados:
            if row['novo_item__produto__nivel'] is None:
                row['novo_item__produto__nivel'] = '='
            if row['novo_item__produto__referencia'] is None:
                row['novo_item__produto__referencia'] = '='
            if row['novo_item__cor__cor'] is None:
                row['novo_item__cor__cor'] = '='
            if row['novo_item__tamanho__tamanho__nome'] is None:
                row['novo_item__tamanho__tamanho__nome'] = '='

            if row['deposito_origem'] == 0:
                row['deposito_origem'] = '-'
            else:
                row['deposito_origem|GLYPHICON'] = 'glyphicon-time'
                row['deposito_origem|TARGET'] = '_BLANK'
                row['deposito_origem|LINK'] = reverse(
                    'estoque:item_no_tempo__get', args=[
                        row['item__produto__referencia'],
                        row['item__cor__cor'],
                        row['item__tamanho__tamanho__nome'],
                        row['deposito_origem']])

            if row['deposito_destino'] == 0:
                row['deposito_destino'] = '-'
            else:
                row['deposito_destino|GLYPHICON'] = 'glyphicon-time'
                row['deposito_destino|TARGET'] = '_BLANK'
                row['deposito_destino|LINK'] = reverse(
                    'estoque:item_no_tempo__get', args=[
                        row['item__produto__referencia'],
                        row['item__cor__cor'],
                        row['item__tamanho__tamanho__nome'],
                        row['deposito_destino']])

        headers = [
            'Tipo de Movimento',
            'Nível',
            'Referência',
            'Cor',
            'Tamanho',
            'Quantidade',
            'Dep. origem',
            'Dep. destino',
            'Nível',
            'Referência',
            'Cor',
            'Tamanho',
            'Usuário',
        ]
        self.context.update({
            'headers': headers,
            'fields': fields,
            'dados': dados,
            })

    def get(self, request, *args, **kwargs):
        if 'num_doc' in kwargs:
            return self.post(request, *args, **kwargs)
        self.context['form'] = self.Form_class()
        return render(request, self.template_name, self.context)

    def post(self, request, *args, **kwargs):
        self.request = request
        if 'num_doc' in kwargs:
            self.context['post'] = True
            self.context['form'] = self.Form_class(kwargs)
        else:
            self.context['post'] = 'busca' in self.request.POST
            self.context['form'] = self.Form_class(self.request.POST)
        if self.context['form'].is_valid():
            self.cleanned_fields_to_context()
            self.mount_context()
            self.context_to_form_post()
            self.context['form'] = self.Form_class(self.context)
        return render(request, self.template_name, self.context)
<|endoftext|>


<commit_before># Copyright (C) 2010-2012 Cuckoo Sandbox Developers.
# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
# See the file 'docs/LICENSE' for copying permission.

from lib.common.abstracts import Package
from lib.api.process import Process

class XLS(Package):
    """Excel analysis package."""

    def get_path(self):
        paths = [
            os.path.join(os.getenv("ProgramFiles"), "Microsoft Office", "EXCEL.EXE"),
            os.path.join(os.getenv("ProgramFiles"), "Microsoft Office", "Office11", "EXCEL.EXE"),
            os.path.join(os.getenv("ProgramFiles"), "Microsoft Office", "Office12", "EXCEL.EXE"),
            os.path.join(os.getenv("ProgramFiles"), "Microsoft Office", "Office14", "EXCEL.EXE"),
            os.path.join(os.getenv("ProgramFiles"), "Microsoft Office", "Office15", "EXCEL.EXE")
        ]

        for path in paths:
            if os.path.exists(path):
                return path

        return None

    def start(self, path):
        excel = self.get_path()
        if not excel:
            raise CuckooPackageError("Unable to find any Microsoft Office Excel executable available")

        free = self.options.get("free", False)
        suspended = True
        if free:
            suspended = False

        p = Process()
        if not p.execute(path=excel, args="\"%s\"" % path, suspended=suspended):
            raise CuckooPackageError("Unable to execute initial Microsoft Office Excel process, analysis aborted")

        if not free and suspended:
            p.inject()
            p.resume()
            return p.pid
        else:
            return None

    def check(self):
        return True

    def finish(self):
        return True
<commit_msg>somebody forget to test the excel package<commit_after># Copyright (C) 2010-2012 Cuckoo Sandbox Developers.
# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
# See the file 'docs/LICENSE' for copying permission.

import os.path

from lib.common.abstracts import Package
from lib.api.process import Process

class XLS(Package):
    """Excel analysis package."""

    def get_path(self):
        paths = [
            os.path.join(os.getenv("ProgramFiles"), "Microsoft Office", "EXCEL.EXE"),
            os.path.join(os.getenv("ProgramFiles"), "Microsoft Office", "Office11", "EXCEL.EXE"),
            os.path.join(os.getenv("ProgramFiles"), "Microsoft Office", "Office12", "EXCEL.EXE"),
            os.path.join(os.getenv("ProgramFiles"), "Microsoft Office", "Office14", "EXCEL.EXE"),
            os.path.join(os.getenv("ProgramFiles"), "Microsoft Office", "Office15", "EXCEL.EXE")
        ]

        for path in paths:
            if os.path.exists(path):
                return path

        return None

    def start(self, path):
        excel = self.get_path()
        if not excel:
            raise CuckooPackageError("Unable to find any Microsoft Office Excel executable available")

        free = self.options.get("free", False)
        suspended = True
        if free:
            suspended = False

        p = Process()
        if not p.execute(path=excel, args="\"%s\"" % path, suspended=suspended):
            raise CuckooPackageError("Unable to execute initial Microsoft Office Excel process, analysis aborted")

        if not free and suspended:
            p.inject()
            p.resume()
            return p.pid
        else:
            return None

    def check(self):
        return True

    def finish(self):
        return True
<|endoftext|>


<commit_before>from django.contrib import admin
from reversion.admin import VersionAdmin
from ajax_select.admin import AjaxSelectAdmin
from ajax_select import make_ajax_form

# from cms.admin import JSONEditor

from django.forms.widgets import Textarea
try:
    from django.forms.util import flatatt
except ImportError:
    from django.forms.utils import flatatt
from django.utils.safestring import mark_safe
from django.conf import settings
import json

from jsonfield import JSONField
from .models import Discipline, TrainingTerms, Semester, Variant, Technology, Diagram, WorkingType

class JSONEditor(Textarea):
    class Media:
        js = (
            getattr(settings, "JQUERY", settings.STATIC_URL + 'jquery-1.7.2.min.js'),
            getattr(settings, "JQUERY_UI", settings.STATIC_URL + 'jquery-ui.min.js'),
            getattr(settings, "JQUERY_EVENT_DRAG", settings.STATIC_URL + 'jquery.event.drag.js'),

            getattr(settings, "SLICK_JS_CORE", settings.STATIC_URL+'slickgrid/slick.core.js'),
            getattr(settings, "SLICK_JS_FORMATTERS", settings.STATIC_URL + 'slickgrid/slick.formatters.js'),
            getattr(settings, "SLICK_JS_EDITORS", settings.STATIC_URL + 'slickgrid/slick.editors.js'),
            getattr(settings, "SLICK_JS_GRID", settings.STATIC_URL + 'slickgrid/slick.grid.js'),
        )
        css = {'all': (getattr(settings, "SLICK_CSS", settings.STATIC_URL+'slickgrid/slick.grid.css'),)}

    def render(self, name, value, attrs=None):
        data = []
        wts = WorkingType.objects.all()
        for wt in wts:
            data.append({
                "wt": wt.title,
            })

        try:
            value = json.loads(value)
        except TypeError:
            pass
        input_attrs = {'hidden': True}
        input_attrs.update(attrs)
        if 'class' not in input_attrs:
            input_attrs['class'] = 'for_grideditor'
        else:
            input_attrs['class'] += ' for_grideditor'
        r = super(JSONEditor, self).render(name, value, input_attrs)
        div_attrs = {}
        div_attrs.update(attrs)
        div_attrs.update({'id': (attrs['id']+'_jsoneditor')})
        final_attrs = self.build_attrs(div_attrs, extra_attrs={"name": name})
        r += '''

        '''.format(
            attrs=flatatt(final_attrs),
            name=name,
            data=data
        )
        return mark_safe(r)

@admin.register(Discipline)
class DisciplineAdmin(VersionAdmin):
    fields = ("title", "description", "module", "period",  "labor", "form", "status", "archived", "results", "results_text")
    list_display = ("id", "title", "description", "module", "period", "labor", "form", "status", "archived")
    filter_horizontal = ("results",)
    list_filter = ("archived", "created", "updated", "status", "form")
    search_fields = ['title', 'module__title', "module__uni_number"]

@admin.register(TrainingTerms)
class TrainingTermsAdmin(VersionAdmin):
    list_display = ("title", "limit")

@admin.register(Semester)
class SemesterAdmin(VersionAdmin):
    list_display = ("__str__", )
    search_fields = ("discipline__title", "discipline__module__title", "program__title")
    form = make_ajax_form(Semester, {'discipline': 'discipline'})

@admin.register(Variant)
class VariantAdmin(VersionAdmin, AjaxSelectAdmin):
    list_display = ("discipline", "program", "course", "semester", "parity", "link", "status", "archived")
    list_filter = ("parity", "archived", "created", "updated", "status")
    search_fields = ("discipline__title", "program__title", "course__title", "parity", "link")
    form = make_ajax_form(Variant, {'discipline': 'discipline'})

@admin.register(Technology)
class TechnologyAdmin(VersionAdmin):
    list_display = ("title", "description", "contact_work_category", "get_color")

@admin.register(Diagram)
class DiagramAdmin(VersionAdmin):
    list_display = ("title", "diagram")
    formfield_overrides = {
        JSONField: {'widget': JSONEditor},
    }

@admin.register(WorkingType)
class WorkingTypeAdmin(VersionAdmin):
    list_display = ("title", "color")
<commit_msg>slickgrid#36<commit_after>from django.contrib import admin
from reversion.admin import VersionAdmin
from ajax_select.admin import AjaxSelectAdmin
from ajax_select import make_ajax_form

# from cms.admin import JSONEditor

from django.forms.widgets import Textarea
try:
    from django.forms.util import flatatt
except ImportError:
    from django.forms.utils import flatatt
from django.utils.safestring import mark_safe
from django.conf import settings
import json

from jsonfield import JSONField
from .models import Discipline, TrainingTerms, Semester, Variant, Technology, Diagram, WorkingType

class JSONEditor(Textarea):
    class Media:
        js = (
            getattr(settings, "JQUERY", settings.STATIC_URL + 'jquery-1.7.2.min.js'),
            getattr(settings, "JQUERY_UI", settings.STATIC_URL + 'jquery-ui.min.js'),
            getattr(settings, "JQUERY_EVENT_DRAG", settings.STATIC_URL + 'jquery.event.drag.js'),

            getattr(settings, "SLICK_JS_CORE", settings.STATIC_URL+'slickgrid/slick.core.js'),
            getattr(settings, "SLICK_JS_FORMATTERS", settings.STATIC_URL + 'slickgrid/slick.formatters.js'),
            getattr(settings, "SLICK_JS_EDITORS", settings.STATIC_URL + 'slickgrid/slick.editors.js'),
            getattr(settings, "SLICK_JS_GRID", settings.STATIC_URL + 'slickgrid/slick.grid.js'),
        )
        css = {'all': (getattr(settings, "SLICK_CSS", settings.STATIC_URL+'slickgrid/slick.grid.css'),)}

    def render(self, name, value, attrs=None):
        data = []
        wts = WorkingType.objects.all()
        for wt in wts:
            data.append({
                "wt": wt.title,
            })

        try:
            value = json.loads(value)
        except TypeError:
            pass
        input_attrs = {'hidden': True}
        input_attrs.update(attrs)
        if 'class' not in input_attrs:
            input_attrs['class'] = 'for_grideditor'
        else:
            input_attrs['class'] += ' for_grideditor'
        r = super(JSONEditor, self).render(name, value, input_attrs)
        div_attrs = {}
        div_attrs.update(attrs)
        div_attrs.update({'id': (attrs['id']+'_jsoneditor')})
        final_attrs = self.build_attrs(div_attrs, extra_attrs={"name": name})
        r += '''

        '''.format(
            attrs=flatatt(final_attrs),
            name=name,
            data=data
        )
        return mark_safe(r)

@admin.register(Discipline)
class DisciplineAdmin(VersionAdmin):
    fields = ("title", "description", "module", "period",  "labor", "form", "status", "archived", "results", "results_text")
    list_display = ("id", "title", "description", "module", "period", "labor", "form", "status", "archived")
    filter_horizontal = ("results",)
    list_filter = ("archived", "created", "updated", "status", "form")
    search_fields = ['title', 'module__title', "module__uni_number"]

@admin.register(TrainingTerms)
class TrainingTermsAdmin(VersionAdmin):
    list_display = ("title", "limit")

@admin.register(Semester)
class SemesterAdmin(VersionAdmin):
    list_display = ("__str__", )
    search_fields = ("discipline__title", "discipline__module__title", "program__title")
    form = make_ajax_form(Semester, {'discipline': 'discipline'})

@admin.register(Variant)
class VariantAdmin(VersionAdmin, AjaxSelectAdmin):
    list_display = ("discipline", "program", "course", "semester", "parity", "link", "status", "archived")
    list_filter = ("parity", "archived", "created", "updated", "status")
    search_fields = ("discipline__title", "program__title", "course__title", "parity", "link")
    form = make_ajax_form(Variant, {'discipline': 'discipline'})

@admin.register(Technology)
class TechnologyAdmin(VersionAdmin):
    list_display = ("title", "description", "contact_work_category", "get_color")

@admin.register(Diagram)
class DiagramAdmin(VersionAdmin):
    list_display = ("title", "diagram")
    formfield_overrides = {
        JSONField: {'widget': JSONEditor},
    }

@admin.register(WorkingType)
class WorkingTypeAdmin(VersionAdmin):
    list_display = ("title", "color")
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import json
import pandas as pd
import numpy as np
import matplotlib
# from include import plotting
# When using Ipython within vim
matplotlib.use('TkAgg')

# When using within jupyter
# get_ipython().magic('matplotlib inline')  # Activat that line to use in Jupyter

import matplotlib.pyplot as plt

def get_type_question(input_location):
    """
    """
    with open(input_location, 'r') as f:
        return json.load(f)

def freq_table(df, colnames=False, columns='count', add_ratio=False, sort_order=False):
    """
    """
    if colnames:
        df_to_freq = df[colnames]
    else:
        df_to_freq = df
    if add_ratio:
        output = pd.concat([pd.crosstab(df_to_freq, columns='count', normalize=False),
                            pd.crosstab(df_to_freq, columns='ratio', normalize=True)],
                           axis=1)
    else:
        output = pd.crosstab(df_to_freq, colnames=[''], columns=columns)
    if sort_order:
        output = output.sort_values(by='count')
    return output

def freq_plotting(df, colnames='count', sort_order=False, stacked=False, horizontal=False):
    """
    Plot the others variables
    :params:
        :df pd.df(): dataframe containing the data, should be a df of frequencies
        created with crosstab
        :colname str(): string that have the column header to select the right column
    """
    type_plot = 'bar'
    # Call the freq_table function to create the count to plot
    # d = freq_table(df, colnames, columns)
    if sort_order:
        df = df.sort_values(by=colnames, ascending=False)
    if horizontal is True:
        type_plot='barh'

    df[colnames].plot(kind=type_plot, stacked=stacked)
#
#
# def save_freq_plotting(df, columns, colnames=False, sort_order=False, stacked=False, horizontal=False):
#     """
#     Plot the others variables
#     :params:
#         :df pd.df(): dataframe containing the data
#         :colname str(): string that have the column header to select the right column
#     """
#     type_plot = 'bar'
#     if colnames is False:
#         colnames = columns
#     # Call the freq_table function to create the count to plot
#     d = freq_table(df, colnames, columns)
#     if sort_order:
#         d = d.sort_values(by=colnames, ascending=False)
#     if horizontal is True:
#         type_plot='barh'
#
#     d.plot(kind=type_plot, stacked=stacked)
#     return d

def count_unique_value(df, colnames, rename_columns=False, dropna=False, normalize=False):
    """
    Count the values of different columns and transpose the count
    :params:
        :df pd.df(): dataframe containing the data
        :colnames list(): list of strings corresponding to the column header to select the right column
    :return:
        :result_df pd.df(): dataframe with the count of each answer for each columns
    """
    # Subset the columns
    colnames = [i for j in colnames for i in j]
    df_sub = df[colnames]

    if rename_columns is True:
        df_sub.columns = [s.split('[', 1)[1].split(']')[0] for s in colnames]

    # Calculate the counts for them
    df_sub = df_sub.apply(pd.Series.value_counts, dropna=dropna, normalize=normalize)
    # Transpose the column to row to be able to plot a stacked bar chart
    return df_sub.transpose()

def plot_likert():
    """
    """
    pass

def get_colors(df, colormap=plt.cm.RdBu, vmin=None, vmax=None):
    """
    Function to automatically gets a colormap for all the values passed in,
    Have the option to normalise the colormap.
    :params:
        values list(): list of int() or str() that have all the values that need a color to be map
        to. In case of a list() of str(), the try/except use the range(len()) to map a colour
        colormap cm(): type of colormap that need to be used. All can be found here:
            https://matplotlib.org/examples/color/colormaps_reference.html
        vmin, vmax int(): Number to normalise the return of the colourmap if needed a Normalised colourmap

    :return:
        colormap cm.colormap(): An array of RGBA values

    Original version found on stackerOverflow (w/o the try/except) but cannot find it back
    """
    values = df.columns
    norm = plt.Normalize(vmin, vmax)
    try:
        return colormap(norm(values))
    except (AttributeError, TypeError):  # May happen when gives a list of categorical values
        return colormap(norm(range(len(values))))

def create_bars(df, ax, y_pos, colors, left_gap=False):
    """
    Loop through the columns and create an horizontal bar for each.
    First it creates all the left bars, for all the columns, then the
    one on the right. Each time, it add the distance from the previous bar.
    If 'left_invisible_bar' is passed, it will create a empty gap on the left
    before the first bar to centred the plot in the middle

    :params:
        df df(): The dataframe containing the information
        ax plt(): The subplot to draw on
        y_pos np.array(): an array of the number of bars (likert items)
        colors np.array(): an array containing the colors for the different answers
        left_gap np.array(): the empty left gap needed to
            centre the stacked bar

    :return:
        patch_handles list(): A list containing the drawn horizontal stacked bars
    """
    patch_handles = []
    for i, c in enumerate(df.columns):
        d = np.array(df[c])
        if left_gap:
            new_bar = ax.barh(y_pos, d, color=colors[i], align='center', left=left_gap)
            # accumulate the left-hand offsets
            left_gap += d
        else:
            new_bar = ax.barh(y_pos, d, color=colors[i], align='center', left=left_gap)
        patch_handles.append(new_bar)
    return patch_handles

def plot_y_n(df, colnames='count', sort_order=False, stacked=False, horizontal=False, set_label=False):
    """
    Plot the others variables
    :params:
        :df pd.df(): dataframe containing the data, should be a df of frequencies
        created with crosstab
        :colname str(): string that have the column header to select the right column
    """
    df = df[['Yes', 'No']]
    fig, ax = plt.subplots()
    index = np.arange(len(df))
    bar_width = 0.35
    opacity = 0.7
    yes_bar = plt.bar(index, df['Yes'], width=bar_width, bottom=None, color='blue', label='Yes')
    no_bar = plt.bar(index, df['No'], width=bar_width, bottom=df['Yes'], color='red', label='No')
    return fig

def plot_discrete():
    """
    """
    # Calculate the average of all the time_activity questions and plotting them
    # Convert the different column to an int value to be able to calculate the mean after
    # The option 'coerce' is needed to force passing the NaN values
    # df[time_activity] = df[time_activity].apply(pd.to_numeric, errors='coerce')
    # mean_activity = df[time_activity].mean(axis=0)
    pass

def process_question(df, q, type_chart):
    """
    """
    if any(isinstance(el, list) for el in q):
        return count_unique_value(df, q)
    else:
        data_to_plot = df[q]
    if type_chart == 'barchart':
        result = freq_table(data_to_plot)
        return result

def main():

    #  When using this script with ipython and vim
    plt.ion()
    plt.show()
    pd.set_option('display.max_rows', 300)

    # load the dataframe
    df = pd.read_csv('./dataset/cleaned_data.csv')
    location_type_q = './to_plot.json'
    type_questions = get_type_question(location_type_q)

    data_to_plot = count_unique_value(df, type_questions['single_questions']['yes_no'], dropna=True)
    data_to_plot.sort_values(by='Yes').plot(kind='barh', stacked=True)

    for group in type_questions['grouped_questions']:
        pass

    for group in type_questions['single_questions']:
        if group == 'yes_no':  # aggregate in case of grouped_questions
            for q in type_questions['single_questions'][group]:
                process_question(df, q, 'yes_no')

        elif q == 'decision_job':  # ranking
            pass
        elif q.startswith('likert'):  # likert
            pass
        elif q == 'messy_data':  # wordcounts
            pass
        elif q == 'single_item':  # passing
            pass
        else:  # Should be easily plotted in bar chart for all other cases
            pass

if __name__ == "__main__":
    main()
<commit_msg>Working plot_y_n()<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import json
import pandas as pd
import numpy as np
import matplotlib
# from include import plotting
# When using Ipython within vim
matplotlib.use('TkAgg')

# When using within jupyter
# get_ipython().magic('matplotlib inline')  # Activat that line to use in Jupyter

import matplotlib.pyplot as plt

def get_type_question(input_location):
    """
    """
    with open(input_location, 'r') as f:
        return json.load(f)

def freq_table(df, colnames=False, columns='count', add_ratio=False, sort_order=False):
    """
    """
    if colnames:
        df_to_freq = df[colnames]
    else:
        df_to_freq = df
    if add_ratio:
        output = pd.concat([pd.crosstab(df_to_freq, columns='count', normalize=False),
                            pd.crosstab(df_to_freq, columns='ratio', normalize=True)],
                           axis=1)
    else:
        output = pd.crosstab(df_to_freq, colnames=[''], columns=columns)
    if sort_order:
        output = output.sort_values(by='count')
    return output

def freq_plotting(df, colnames='count', sort_order=False, stacked=False, horizontal=False):
    """
    Plot the others variables
    :params:
        :df pd.df(): dataframe containing the data, should be a df of frequencies
        created with crosstab
        :colname str(): string that have the column header to select the right column
    """
    type_plot = 'bar'
    # Call the freq_table function to create the count to plot
    # d = freq_table(df, colnames, columns)
    if sort_order:
        df = df.sort_values(by=colnames, ascending=False)
    if horizontal is True:
        type_plot='barh'

    df[colnames].plot(kind=type_plot, stacked=stacked)

# def save_freq_plotting(df, columns, colnames=False, sort_order=False, stacked=False, horizontal=False):
#     """
#     Plot the others variables
#     :params:
#         :df pd.df(): dataframe containing the data
#         :colname str(): string that have the column header to select the right column
#     """
#     type_plot = 'bar'
#     if colnames is False:
#         colnames = columns
#     # Call the freq_table function to create the count to plot
#     d = freq_table(df, colnames, columns)
#     if sort_order:
#         d = d.sort_values(by=colnames, ascending=False)
#     if horizontal is True:
#         type_plot='barh'
#
#     d.plot(kind=type_plot, stacked=stacked)
#     return d

def plot_discrete():
    """
    """
    # Calculate the average of all the time_activity questions and plotting them
    # Convert the different column to an int value to be able to calculate the mean after
    # The option 'coerce' is needed to force passing the NaN values
    # df[time_activity] = df[time_activity].apply(pd.to_numeric, errors='coerce')
    # mean_activity = df[time_activity].mean(axis=0)
    pass

def process_question(df, q, type_chart):
    """
    """
    if any(isinstance(el, list) for el in q):
        return count_unique_value(df, q)
    else:
        data_to_plot = df[q]
    if type_chart == 'barchart':
        result = freq_table(data_to_plot)
        return result

def count_unique_value(df, colnames, rename_columns=False, dropna=False, normalize=False):
    """
    Count the values of different columns and transpose the count
    :params:
        :df pd.df(): dataframe containing the data
        :colnames list(): list of strings corresponding to the column header to select the right column
    :return:
        :result_df pd.df(): dataframe with the count of each answer for each columns
    """
    # Subset the columns
    colnames = [i for j in colnames for i in j]
    df_sub = df[colnames]

    if rename_columns is True:
        df_sub.columns = [s.split('[', 1)[1].split(']')[0] for s in colnames]

    # Calculate the counts for them
    df_sub = df_sub.apply(pd.Series.value_counts, dropna=dropna, normalize=normalize)
    # Transpose the column to row to be able to plot a stacked bar chart
    return df_sub.transpose()

def get_colors(df, colormap=plt.cm.RdBu, vmin=None, vmax=None):
    """
    Function to automatically gets a colormap for all the values passed in,
    Have the option to normalise the colormap.
    :params:
        values list(): list of int() or str() that have all the values that need a color to be map
        to. In case of a list() of str(), the try/except use the range(len()) to map a colour
        colormap cm(): type of colormap that need to be used. All can be found here:
            https://matplotlib.org/examples/color/colormaps_reference.html
        vmin, vmax int(): Number to normalise the return of the colourmap if needed a Normalised colourmap

    :return:
        colormap cm.colormap(): An array of RGBA values

    Original version found on stackerOverflow (w/o the try/except) but cannot find it back
    """
    values = df.columns
    norm = plt.Normalize(vmin, vmax)
    try:
        return colormap(norm(values))
    except (AttributeError, TypeError):  # May happen when gives a list of categorical values
        return colormap(norm(range(len(values))))

def add_labels(df, ax, bars, rotation=0):
    """
    """
    # Create percentage for each cells to have the right annotation
    percentages = compute_percentage(df)
    # go through all of the bar segments and annotate
    for j in range(len(bars)):
        for i, bar in enumerate(bars[j].get_children()):
            bl = bar.get_xy()
            x = 0.5 *bar.get_width() +bl[0]
            y = 0.5 *bar.get_height() +bl[1]
            ax.text(x, y, "{}".format(percentages[i, j]), ha='center', rotation=rotation)

def plot_y_n(df, sort_order=False, horizontal=False,
             legend=True, set_label=False):
    """
    Plotting Y-N values as stacked bars
    :params:
        :df pd.df(): dataframe containing the data, should be a df of frequencies
        created with crosstab
        :sort_order bool(): to order the value by the number of yes
        :horizontal bool(): to plot the bar horizontal rather than vertical (Default behaviour)
        :legend bool(): to show the legend or not
        :set_labels bool(): to add labels on the individuals bars
        :set_n bool(): to show the total n for each items

    :return:
        :fig matplotlib.plt.plot(): Return the plot itself
    """
    df = df[['Yes', 'No']]
    fig, ax = plt.subplots()
    index = np.arange(len(df))
    colors = plt.cm.tab10
    bar_width = 0.9
    opacity = 0.7

    # To set up the label on x or y axis
    label_txt = df.index
    label_ticks = range(len(df.index))

    # Sorting the df with the Yes values
    if sort_order is True:
        df.sort_values(by='Yes', inplace=True, ascending=False)
    else:
        pass

    if horizontal is True:
        for i, d in enumerate(df.index):
            yes_bar = ax.barh(index[i], width=df['Yes'][i], height=bar_width, color=colors(0), label='Yes')
            no_bar = ax.barh(index[i], width=df['No'][i], height=bar_width, left=df['Yes'][i], color=colors(1), label='No')
    else:
        yes_bar = ax.bar(index, df['Yes'], width=bar_width, bottom=None, color=colors(0), label='Yes')
        no_bar = ax.bar(index, df['No'], width=bar_width, bottom=df['Yes'], color=colors(1), label='No')

    if set_label is True:
        pass

    # Add the legend
    ax.legend((yes_bar, no_bar), ('Yes', 'No'))

    # Add the x-labels
    if horizontal is True:
        plt.yticks(label_ticks, label_txt)
    else:
        # This set the xlimits to center the xtick with the bin
        # Explanation found here:
        # https://stackoverflow.com/a/27084005/3193951
        plt.xlim([-1, len(df.index)])
        plt.xticks(label_ticks, label_txt, rotation=90)

    # Modifying the whitespaces between the bars and the graph
    plt.margins(0.02, 0.02)

    return fig

plot_y_n(data_to_plot, sort_order=True, horizontal=False, set_label=True)
data_to_plot

def main():

    #  When using this script with ipython and vim
    plt.ion()
    plt.show()
    pd.set_option('display.max_rows', 300)

    # load the dataframe
    df = pd.read_csv('./dataset/cleaned_data.csv')
    location_type_q = './to_plot.json'
    type_questions = get_type_question(location_type_q)

    data_to_plot = count_unique_value(df, type_questions['single_questions']['yes_no'],
                                      normalize=True,dropna=True)
    data_to_plot.sort_values(by='Yes').plot(kind='barh', stacked=True)

    data_to_plot

    for group in type_questions['grouped_questions']:
        pass

    for group in type_questions['single_questions']:
        if group == 'yes_no':  # aggregate in case of grouped_questions
            for q in type_questions['single_questions'][group]:
                process_question(df, q, 'yes_no')

        elif q == 'decision_job':  # ranking
            pass
        elif q.startswith('likert'):  # likert
            pass
        elif q == 'messy_data':  # wordcounts
            pass
        elif q == 'single_item':  # passing
            pass
        else:  # Should be easily plotted in bar chart for all other cases
            pass

if __name__ == "__main__":
    main()
<|endoftext|>