<commit_before>#!/usr/bin/python
import urllib
import os
import zipfile
import sys
import argparse

# todo: change wx solution to use static crt...
# sln file:
# Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "richtext", "wx_richtext.vcxproj", "{7FB0902D-8579-5DCE-B883-DAF66A885005}"
# in vcxproj file: change:
# <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary> to <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
# <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary> to <RuntimeLibrary>MultiThreaded</RuntimeLibrary>

root = os.getcwd()
install_dist = os.path.join(root, 'install-dist')
wx_root = os.path.join(install_dist, 'wx')
proto_root = os.path.join(install_dist, 'proto')

def install_cmd(args):
    global root
    global install_dist
    global wx_root
    global proto_root
    platform='x64'
    if os.environ['PLATFORM'] == 'x86':
        platform='Win32'

    wx_url = "https://github.com/wxWidgets/wxWidgets/releases/download/v3.1.0/wxWidgets-3.1.0.zip"
    wx_zip = os.path.join(install_dist, "wx.zip")
    wx_sln = os.path.join(wx_root, 'build', 'msw', 'wx_vc14.sln')
    wx_msbuild_cmd = 'msbuild /p:Configuration=Release /p:Platform="{platform}" {solution}'.format(platform=platform, solution=wx_sln)

    proto_url = "https://github.com/google/protobuf/archive/v2.6.1.zip"
    proto_zip = os.path.join(install_dist, 'proto.zip')
    proto_sln = os.path.join(proto_root, 'protobuf-2.6.1', 'vsprojects', 'protobuf.sln')
    proto_msbuild_cmd = 'msbuild /p:Configuration=Release /p:Platform="{platform}" {solution}'.format(platform=platform, solution=proto_sln)

    print root
    print wx_sln
    print proto_sln

    os.makedirs(install_dist)
    os.makedirs(wx_root)
    os.makedirs(proto_root)

    urllib.urlretrieve (wx_url, os.path.join(install_dist, wx_zip))
    urllib.urlretrieve (proto_url, os.path.join(install_dist, proto_zip))

    with zipfile.ZipFile(wx_zip, 'r') as z:
        z.extractall(wx_root)

    with zipfile.ZipFile(proto_zip, 'r') as z:
        z.extractall(proto_root)

    print "building wxwidgets"
    print "-----------------------------------"
    sys.stdout.flush()
    os.system(wx_msbuild_cmd)

    print "building protobuf"
    print "-----------------------------------"
    sys.stdout.flush()
    os.system(proto_msbuild_cmd)

def cmake_cmd(args):
    global root
    global install_dist
    global wx_root
    global proto_root
    build = os.path.join(root, 'build')
    os.makedirs(build)
    cmakecmd = ("cmake "
                "-DCMAKE_INSTALL_PREFIX= {install_dist} "
                "-DPROTOBUF_SRC_ROOT_FOLDER={proto_root} "
                "-DwxWidgets_ROOT_DIR={wx_root} "
                "-DRIDE_BUILD_COMMIT=%APPVEYOR_REPO_COMMIT% "
                "-DRIDE_BUILD_NUMBER=%APPVEYOR_BUILD_NUMBER% "
                "-DRIDE_BUILD_BRANCH=%APPVEYOR_REPO_BRANCH% "
                "-DRIDE_BUILD_REPO=%APPVEYOR_REPO_NAME% "
                "{build}").format(
        install_dist=install_dist,
        proto_root=proto_root,
        wx_root=wx_root,
        build=build
    )
    os.system(cmakecmd)

parser = argparse.ArgumentParser(description='Does the windows build')
subparsers = parser.add_subparsers()

install_parser = subparsers.add('install')
install_parser.set_defaults(func=install_cmd)

cmake_parser = subparsers.add('cmake')
cmake_parser.set_defaults(func=cmake_cmd)

args = parser.parse_args()
args.func(args)<commit_msg>fixed cmake cmd<commit_after>#!/usr/bin/python
import urllib
import os
import zipfile
import sys
import argparse

# todo: change wx solution to use static crt...
# sln file:
# Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "richtext", "wx_richtext.vcxproj", "{7FB0902D-8579-5DCE-B883-DAF66A885005}"
# in vcxproj file: change:
# <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary> to <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
# <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary> to <RuntimeLibrary>MultiThreaded</RuntimeLibrary>

root = os.getcwd()
install_dist = os.path.join(root, 'install-dist')
wx_root = os.path.join(install_dist, 'wx')
proto_root = os.path.join(install_dist, 'proto')

def install_cmd(args):
    global root
    global install_dist
    global wx_root
    global proto_root
    platform='x64'
    if os.environ['PLATFORM'] == 'x86':
        platform='Win32'

    wx_url = "https://github.com/wxWidgets/wxWidgets/releases/download/v3.1.0/wxWidgets-3.1.0.zip"
    wx_zip = os.path.join(install_dist, "wx.zip")
    wx_sln = os.path.join(wx_root, 'build', 'msw', 'wx_vc14.sln')
    wx_msbuild_cmd = 'msbuild /p:Configuration=Release /p:Platform="{platform}" {solution}'.format(platform=platform, solution=wx_sln)

    proto_url = "https://github.com/google/protobuf/archive/v2.6.1.zip"
    proto_zip = os.path.join(install_dist, 'proto.zip')
    proto_sln = os.path.join(proto_root, 'protobuf-2.6.1', 'vsprojects', 'protobuf.sln')
    proto_msbuild_cmd = 'msbuild /p:Configuration=Release /p:Platform="{platform}" {solution}'.format(platform=platform, solution=proto_sln)

    print root
    print wx_sln
    print proto_sln

    os.makedirs(install_dist)
    os.makedirs(wx_root)
    os.makedirs(proto_root)

    urllib.urlretrieve (wx_url, os.path.join(install_dist, wx_zip))
    urllib.urlretrieve (proto_url, os.path.join(install_dist, proto_zip))

    with zipfile.ZipFile(wx_zip, 'r') as z:
        z.extractall(wx_root)

    with zipfile.ZipFile(proto_zip, 'r') as z:
        z.extractall(proto_root)

    print "building wxwidgets"
    print "-----------------------------------"
    sys.stdout.flush()
    os.system(wx_msbuild_cmd)

    print "building protobuf"
    print "-----------------------------------"
    sys.stdout.flush()
    os.system(proto_msbuild_cmd)

def cmake_cmd(args):
    global root
    global install_dist
    global wx_root
    global proto_root
    build = os.path.join(root, 'build')
    os.makedirs(build)
    cmakecmd = ("cd {build} && cmake "
                "-DCMAKE_INSTALL_PREFIX= {install_dist} "
                "-DPROTOBUF_SRC_ROOT_FOLDER={proto_root} "
                "-DwxWidgets_ROOT_DIR={wx_root} "
                "-DRIDE_BUILD_COMMIT=%APPVEYOR_REPO_COMMIT% "
                "-DRIDE_BUILD_NUMBER=%APPVEYOR_BUILD_NUMBER% "
                "-DRIDE_BUILD_BRANCH=%APPVEYOR_REPO_BRANCH% "
                "-DRIDE_BUILD_REPO=%APPVEYOR_REPO_NAME% "
                "{root}").format(
        root=root,
        install_dist=install_dist,
        proto_root=proto_root,
        wx_root=wx_root,
        build=build
    )
    os.system(cmakecmd)

parser = argparse.ArgumentParser(description='Does the windows build')
subparsers = parser.add_subparsers()

install_parser = subparsers.add('install')
install_parser.set_defaults(func=install_cmd)

cmake_parser = subparsers.add('cmake')
cmake_parser.set_defaults(func=cmake_cmd)

args = parser.parse_args()
args.func(args)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function
from six import u

import os
import sys
from collections import OrderedDict

import pyprind

try:
    import unicodecsv as csv
except ImportError:
    import csv

class Exporters(object):
    """Maintains a list of available exporters."""
    EXPORT_WARN = 5000
    PAGE_SIZE = 500

    registry = {}

    def register(self, klass):
        if klass.is_available():
            self.registry[klass.name] = klass

    def export(self, exporter, query, *args, **kwargs):
        force = kwargs.pop('force', False)

        if exporter in self.registry:
            nrecords = len(query)
            if nrecords == 0:
                raise Exception('There are no results to export.')
            elif nrecords > self.EXPORT_WARN and not force:
                print('To export {} records, use `force=True`.'
                      .format(nrecords))
                return

            # Increase the efficiency of the export.
            query._page_size = self.PAGE_SIZE
            # Enable progress by default on large exports, but allow toggle.
            show_progress = kwargs.pop('show_progress',
                                       nrecords > self.EXPORT_WARN)
            return self.registry[exporter](query, show_progress=show_progress)\
                .export(*args, **kwargs)
        else:
            raise Exception('Invalid exporter: {}. Available exporters: {}'
                            .format(exporter, ', '.join(self.registry.keys())))

class CSVExporter(object):
    """
    This class includes helper functions to export
    a SolveBio Query object to a CSV file.
    """
    name = 'csv'

    collection = None

    SEP_CHAR = '\r'
    KEY_VAL_CHAR = ': '
    DICT_SEP_CHAR = '\r'
    DICT_OPEN = ''
    DICT_CLOSE = ''

    def __init__(self, query, *args, **kwargs):
        self.query = query
        self.show_progress = kwargs.get('show_progress', False)

    @classmethod
    def is_available(cls):
        return True

    def export(self, filename=None, **kwargs):
        if not filename:
            raise Exception(
                '"filename" is a required parameter '
                'for the CSV exporter.')

        filename = os.path.expanduser(filename)

        from solvebio import Dataset

        result_count = len(self.query)
        if result_count <= 0:
            raise AttributeError('No results found in query!')

        self.rows = []
        self.key_map = OrderedDict()

        for f in Dataset.retrieve(self.query._dataset_id).fields(limit=1000):
            name = f['name']
            splits = [int(s) if s.isdigit() else s
                      for s in name.split('.')]
            self.key_map[name] = splits

        title = 'Exporting query to: {}'.format(filename)

        if self.show_progress:
            progress_bar = pyprind.ProgPercent(
                result_count,
                title=title,
                track_time=False)
        else:
            print(title)

        for ind, record in enumerate(self.query):
            row = self.process_record(record)
            self.rows.append(row)
            if self.show_progress:
                progress_bar.update()

        self.write_csv(filename=filename)
        print('Export complete!')

    def process_cell(self, keys, cell):
        if not keys:
            return [cell]
        elif not cell:
            return []

        # Retrieve the cell values for a nested dict.
        for i, k in enumerate(keys):
            if isinstance(cell, list):
                return [self.process_cell(keys[i:], c)
                        for c in cell]
            elif isinstance(cell, dict):
                return self.process_cell(keys[i + 1:], cell.get(k))
            elif i == len(keys) - 1:
                # Last key, return the cell
                return []

    def process_record(self, record):
        """Process a row of json data against the key map
        """
        row = {}

        for header, keys in self.key_map.items():
            try:
                cells = self.process_cell(keys, record)
            except (KeyError, IndexError, TypeError):
                cells = []

            row[header] = self.SEP_CHAR.join(
                [self.make_string(cell) for cell in cells])

        return row

    def make_string(self, item):
        if isinstance(item, list) or \
                isinstance(item, set) or \
                isinstance(item, tuple):
            return self.SEP_CHAR.join(
                [self.make_string(subitem) for subitem in item])
        elif isinstance(item, dict):
            return self.DICT_OPEN + self.DICT_SEP_CHAR.join(
                [self.KEY_VAL_CHAR.join(
                    [k, self.make_string(val)]
                ) for k, val in item.items()]
            ) + self.DICT_CLOSE
        elif item:
            return u(str(item)).strip()
        else:
            return ''

    def write_csv(self, filename):
        """Write the processed rows to the given filename
        """
        if sys.version_info >= (3, 0, 0):
            f = open(filename, 'w', newline='')
        else:
            f = open(filename, 'wb')

        try:
            writer = csv.DictWriter(f, self.key_map.keys())
            writer.writeheader()
            writer.writerows(self.rows)
        except:
            raise
        finally:
            f.close()

exporters = Exporters()
exporters.register(CSVExporter)
<commit_msg>add XLSX exporter<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import unicode_literals

import os
import sys
from collections import OrderedDict

import pyprind

try:
    import unicodecsv as csv
except ImportError:
    import csv

try:
    import xlsxwriter
except ImportError:
    xlsxwriter = None

class Exporters(object):
    """Maintains a list of available exporters."""
    EXPORT_WARN = 5000
    PAGE_SIZE = 500

    registry = {}

    def register(self, klass):
        self.registry[klass.name.lower()] = klass

    def export(self, exporter, query, *args, **kwargs):
        exporter = exporter.lower()
        force = kwargs.pop('force', False)

        if exporter not in self.registry:
            raise Exception('Invalid exporter: {}. Available exporters: {}'
                            .format(exporter, ', '.join(self.registry.keys())))

        nrecords = len(query)
        if nrecords == 0:
            raise Exception('There are no results to export.')
        elif nrecords > self.EXPORT_WARN and not force:
            print('To export {} records, use `force=True`.'
                  .format(nrecords))
            return

        # Increase the efficiency of the export.
        query._page_size = self.PAGE_SIZE
        # Enable progress by default on large exports, but allow toggle.
        show_progress = kwargs.pop('show_progress',
                                   nrecords > self.EXPORT_WARN)
        return self.registry[exporter](query, show_progress=show_progress)\
            .export(*args, **kwargs)

class CSVExporter(object):
    """
    This class includes helper functions to export
    a SolveBio Query object to a CSV file.
    """
    name = 'csv'

    collection = None

    SEP_CHAR = '\r'
    KEY_VAL_CHAR = ': '
    DICT_SEP_CHAR = '\r'
    DICT_OPEN = ''
    DICT_CLOSE = ''

    def __init__(self, query, *args, **kwargs):
        self.query = query
        self.show_progress = kwargs.get('show_progress', False)

    def export(self, filename=None, **kwargs):
        if not filename:
            raise Exception(
                'The "filename" parameter is required to export.')

        filename = os.path.expanduser(filename)

        from solvebio import Dataset

        result_count = len(self.query)
        if result_count <= 0:
            raise AttributeError('No results found in query!')

        self.rows = []
        self.key_map = OrderedDict()
        self.key_types = {}

        for f in Dataset.retrieve(self.query._dataset_id).fields(limit=1000):
            name = f['name']
            splits = [int(s) if s.isdigit() else s
                      for s in name.split('.')]
            self.key_map[name] = splits
            self.key_types[name] = f['data_type']

        title = 'Exporting query to: {}'.format(filename)

        if self.show_progress:
            progress_bar = pyprind.ProgPercent(
                result_count,
                title=title,
                track_time=False)
        else:
            print(title)

        for ind, record in enumerate(self.query):
            row = self.process_record(record)
            self.rows.append(row)
            if self.show_progress:
                progress_bar.update()

        self.write(filename=filename)
        print('Export complete!')

    def process_cell(self, keys, cell):
        if not keys:
            return [cell]
        elif not cell:
            return []

        # Retrieve the cell values for a nested dict.
        for i, k in enumerate(keys):
            if isinstance(cell, list):
                return [self.process_cell(keys[i:], c)
                        for c in cell]
            elif isinstance(cell, dict):
                return self.process_cell(keys[i + 1:], cell.get(k))
            elif i == len(keys) - 1:
                # Last key, return the cell
                return []

    def process_record(self, record):
        """Process a row of json data against the key map
        """
        row = {}

        for header, keys in self.key_map.items():
            try:
                cells = self.process_cell(keys, record)
            except (KeyError, IndexError, TypeError):
                cells = []

            row[header] = self.SEP_CHAR.join(
                [self.make_string(cell) for cell in cells])

        return row

    def make_string(self, item):
        if isinstance(item, list) or \
                isinstance(item, set) or \
                isinstance(item, tuple):
            return self.SEP_CHAR.join(
                [self.make_string(subitem) for subitem in item])
        elif isinstance(item, dict):
            return self.DICT_OPEN + self.DICT_SEP_CHAR.join(
                [self.KEY_VAL_CHAR.join(
                    [k, self.make_string(val)]
                ) for k, val in item.items()]
            ) + self.DICT_CLOSE
        elif item:
            return str(item).strip()
        else:
            return ''

    def write(self, filename):
        if sys.version_info >= (3, 0, 0):
            f = open(filename, 'w', newline='')
        else:
            f = open(filename, 'wb')

        try:
            writer = csv.DictWriter(f, self.key_map.keys())
            writer.writeheader()
            writer.writerows(self.rows)
        except:
            raise
        finally:
            f.close()

class XLSXExporter(CSVExporter):
    """Uses XlsxWriter to export a valid XLS."""

    name = 'excel'

    SEP_CHAR = '\n'
    KEY_VAL_CHAR = ': '
    DICT_SEP_CHAR = '\n'
    DICT_OPEN = ''
    DICT_CLOSE = ''

    NUMERIC_TYPES = ('integer', 'double', 'long', 'float')

    def __init__(self, query, *args, **kwargs):
        if not xlsxwriter:
            raise Exception(
                'The XLSX exporter requires the xlsxwriter Python module. '
                'Run `pip install XlsxWriter` and reload SolveBio.')

        self.query = query
        self.show_progress = kwargs.get('show_progress', False)

    def write(self, filename):
        workbook = xlsxwriter.Workbook(filename)
        worksheet = workbook.add_worksheet()
        formats = {
            'text': workbook.add_format({'text_wrap': True}),
            'string': workbook.add_format(),
            'header': workbook.add_format({'bold': True}),
            'integer': workbook.add_format(),
            'double': workbook.add_format(),
            'long': workbook.add_format(),
            'float': workbook.add_format(),
        }
        formats['date'] = formats['string']
        formats['boolean'] = formats['string']

        formats['text'].set_align('top')
        formats['nested'] = formats['text']
        formats['object'] = formats['text']

        row = 0
        # Write the header
        for col, k in enumerate(self.key_map.keys()):
            worksheet.write(row, col, k, formats['header'])

        for record in self.rows:
            row += 1
            for col, k in enumerate(sorted(record)):
                fmt = formats.get(self.key_types[k], formats['text'])
                if self.key_types[k] in self.NUMERIC_TYPES:
                    from decimal import Decimal
                    try:
                        val = Decimal(record[k])
                        worksheet.write_number(row, col, val, fmt)
                    except:
                        # Might be multi-line numeric.
                        worksheet.write(row, col, record[k], fmt)
                else:
                    worksheet.write(row, col, record[k].encode('utf-8'), fmt)

        workbook.close()

exporters = Exporters()
exporters.register(CSVExporter)
exporters.register(XLSXExporter)
<|endoftext|>


<commit_before>__version__ = '1.8.7'
<commit_msg>update version 1.8.8<commit_after>__version__ = '1.8.8'
<|endoftext|>


<commit_before>#! /usr/bin/env python

import sys

def import_error(module, package, debian, error):
    sys.stderr.write((
        "Error importing %(module)s: %(error)r\n"
        "To install %(module)s run:\n"
        "  pip install %(package)s\n"
        "or on Debian run:\n"
        "  sudo apt-get install python-%(debian)s\n"
    ) % locals())
    if __name__=='__main__':
        sys.exit(1)

try:
    import jsonschema
except ImportError as e:
    import_error("jsonschema", "jsonschema", "jsonschema", e)
    raise

try:
    import yaml
except ImportError as e:
    import_error("yaml", "PyYAML", "yaml", e)
    raise

import json
import os

def check_response(filepath, request, code, response):
    try:
        example = json.loads(
            response.get('examples', {}).get('application/json', "null")
        )
    except Exception as e:
        raise ValueError("Error parsing JSON example response for %r %r" % (
            request, code
        ), e)
    schema = response.get('schema')
    fileurl = "file://" + os.path.abspath(filepath)
    if example and schema:
        try:
            print ("Checking schema for: %r %r %r" % (filepath, request, code))
            # Setting the 'id' tells jsonschema where the file is so that it
            # can correctly resolve relative $ref references in the schema
            schema['id'] = fileurl
            jsonschema.validate(example, schema)
        except Exception as e:
            raise ValueError("Error validating JSON schema for %r %r" %(
                request, code
            ), e)

def check_swagger_file(filepath):
    with open(filepath) as f:
        swagger = yaml.load(f)

    for path, path_api in swagger['paths'].items():
        for method, request_api in path_api.items():
            request = "%s %s" % (method.upper(), path)
            try:
                responses = request_api['responses']
            except KeyError:
                raise ValueError("No responses for %r" % (request,))
            for code, response in responses.items():
                check_response(filepath, request, code, response)

if __name__=='__main__':
    for path in sys.argv[1:]:
        try:
            check_swagger_file(path)
        except Exception as e:
            raise ValueError("Error checking file %r" % (path,), e)
<commit_msg>Search for yaml swagger files if check_examples.py is run without arguments.<commit_after>#! /usr/bin/env python

import sys
import json
import os

def import_error(module, package, debian, error):
    sys.stderr.write((
        "Error importing %(module)s: %(error)r\n"
        "To install %(module)s run:\n"
        "  pip install %(package)s\n"
        "or on Debian run:\n"
        "  sudo apt-get install python-%(debian)s\n"
    ) % locals())
    if __name__ == '__main__':
        sys.exit(1)

try:
    import jsonschema
except ImportError as e:
    import_error("jsonschema", "jsonschema", "jsonschema", e)
    raise

try:
    import yaml
except ImportError as e:
    import_error("yaml", "PyYAML", "yaml", e)
    raise

def check_response(filepath, request, code, response):
    try:
        example = json.loads(
            response.get('examples', {}).get('application/json', "null")
        )
    except Exception as e:
        raise ValueError("Error parsing JSON example response for %r %r" % (
            request, code
        ), e)
    schema = response.get('schema')
    fileurl = "file://" + os.path.abspath(filepath)
    if example and schema:
        try:
            print ("Checking schema for: %r %r %r" % (filepath, request, code))
            # Setting the 'id' tells jsonschema where the file is so that it
            # can correctly resolve relative $ref references in the schema
            schema['id'] = fileurl
            jsonschema.validate(example, schema)
        except Exception as e:
            raise ValueError("Error validating JSON schema for %r %r" % (
                request, code
            ), e)

def check_swagger_file(filepath):
    with open(filepath) as f:
        swagger = yaml.load(f)

    for path, path_api in swagger.get('paths', {}).items():
        for method, request_api in path_api.items():
            request = "%s %s" % (method.upper(), path)
            try:
                responses = request_api['responses']
            except KeyError:
                raise ValueError("No responses for %r" % (request,))
            for code, response in responses.items():
                check_response(filepath, request, code, response)

if __name__ == '__main__':
    paths = sys.argv[1:]
    if not paths:
        paths = []
        for (root, dirs, files) in os.walk(os.curdir):
            for filename in files:
                if filename.endswith(".yaml"):
                    paths.append(os.path.join(root, filename))
    for path in paths:
        try:
            check_swagger_file(path)
        except Exception as e:
            raise ValueError("Error checking file %r" % (path,), e)
<|endoftext|>


<commit_before>import uuid

from django.conf import settings
from django.contrib.postgres.fields import JSONField
from django.db import models
from django.utils.translation import ugettext_lazy as _

from ..models import UpdatedMixin
from .storages import image_storage

class Image(UpdatedMixin, models.Model):
    author = models.ForeignKey(
        settings.AUTH_USER_MODEL, related_name='images',
        verbose_name=_('Author'), on_delete=models.PROTECT)
    uuid = models.UUIDField(_('UUID'), primary_key=True)
    image = models.ImageField(storage=image_storage)
    mime_type = models.CharField(_('content type'), max_length=99, blank=True)
    meta = JSONField(_('meta-information'), default={}, blank=True)

    class Meta:
        verbose_name = _('image')
        verbose_name_plural = _('images')
        ordering = ('-created_at',)

    def __str__(self):
        return self.image.name

    def save(self, force_insert=False, force_update=False, using=None,
             update_fields=None):
        if not self.uuid:
            uid = uuid.uuid4()
            self.uuid = uid
            self.image.name = image_storage.get_available_name(
                self.image.name, uuid=uid)

        super(Image, self).save(force_insert=force_insert,
                                force_update=force_update,
                                using=using,
                                update_fields=update_fields)
<commit_msg>Add index for image metainformation<commit_after>import uuid

from django.conf import settings
from django.contrib.postgres.fields import JSONField
from django.contrib.postgres.indexes import GinIndex
from django.db import models
from django.utils.translation import ugettext_lazy as _

from ..models import UpdatedMixin
from .storages import image_storage

class Image(UpdatedMixin, models.Model):
    author = models.ForeignKey(
        settings.AUTH_USER_MODEL, related_name='images',
        verbose_name=_('Author'), on_delete=models.PROTECT)
    uuid = models.UUIDField(_('UUID'), primary_key=True)
    image = models.ImageField(storage=image_storage)
    mime_type = models.CharField(_('content type'), max_length=99, blank=True)
    meta = JSONField(_('meta-information'), default={}, blank=True)

    class Meta:
        verbose_name = _('image')
        verbose_name_plural = _('images')
        ordering = ('-created_at',)
        indexes = [GinIndex(fields=['meta'])]

    def __str__(self):
        return self.image.name

    def save(self, force_insert=False, force_update=False, using=None,
             update_fields=None):
        if not self.uuid:
            uid = uuid.uuid4()
            self.uuid = uid
            self.image.name = image_storage.get_available_name(
                self.image.name, uuid=uid)

        super(Image, self).save(force_insert=force_insert,
                                force_update=force_update,
                                using=using,
                                update_fields=update_fields)
<|endoftext|>


<commit_before>"""
sourcemap
~~~~~~~~~

:copyright: (c) 2013 by Matt Robenolt
:license: BSD, see LICENSE for more details.
"""
from .exceptions import SourceMapDecodeError  #NOQA
from .decoder import SourceMapDecoder

__version__ = '0.1.6'

def load(fp, cls=None):
    "Parse a sourcemap from a file-like object"
    return loads(fp.read(), cls)

def loads(source, cls=None):
    "Parse a sourcemap from a string"
    cls = cls or SourceMapDecoder
    return cls().decode(source)

def discover(source):
    "Given a JavaScript file, find the sourceMappingURL line"
    source = source.splitlines()
    # Source maps are only going to exist at either the top or bottom of the document.
    # Technically, there isn't anything indicating *where* it should exist, so we
    # are generous and assume it's somewhere either in the first or last 5 lines.
    # If it's somewhere else in the document, you're probably doing it wrong.
    if len(source) > 10:
        possibilities = source[:5] + source[-5:]
    else:
        possibilities = source

    for line in set(possibilities):
        pragma = line[:21]
        if pragma == '//# sourceMappingURL=' or pragma == '//@ sourceMappingURL=':
            # We want everything AFTER the pragma, which is 21 chars long
            return line[21:].rstrip()
    # XXX: Return None or raise an exception?
    return None
<commit_msg>0.1.7<commit_after>"""
sourcemap
~~~~~~~~~

:copyright: (c) 2013 by Matt Robenolt
:license: BSD, see LICENSE for more details.
"""
from .exceptions import SourceMapDecodeError  #NOQA
from .decoder import SourceMapDecoder

__version__ = '0.1.7'

def load(fp, cls=None):
    "Parse a sourcemap from a file-like object"
    return loads(fp.read(), cls)

def loads(source, cls=None):
    "Parse a sourcemap from a string"
    cls = cls or SourceMapDecoder
    return cls().decode(source)

def discover(source):
    "Given a JavaScript file, find the sourceMappingURL line"
    source = source.splitlines()
    # Source maps are only going to exist at either the top or bottom of the document.
    # Technically, there isn't anything indicating *where* it should exist, so we
    # are generous and assume it's somewhere either in the first or last 5 lines.
    # If it's somewhere else in the document, you're probably doing it wrong.
    if len(source) > 10:
        possibilities = source[:5] + source[-5:]
    else:
        possibilities = source

    for line in set(possibilities):
        pragma = line[:21]
        if pragma == '//# sourceMappingURL=' or pragma == '//@ sourceMappingURL=':
            # We want everything AFTER the pragma, which is 21 chars long
            return line[21:].rstrip()
    # XXX: Return None or raise an exception?
    return None
<|endoftext|>


<commit_before>import avro.schema
import json
import pprint
import copy
from flatten import flatten
import functools
import os
from pathmapper import PathMapper, DockerPathMapper
from job import CommandLineJob
import yaml
import glob
import logging
import hashlib
import random
from process import Process
from process import WorkflowException
import avro_ld.validate as validate
from aslist import aslist
import expression
import re
import urlparse
import tempfile

_logger = logging.getLogger("cwltool")

CONTENT_LIMIT = 64 * 1024

supportedProcessRequirements = ["DockerRequirement",
                                "ExpressionEngineRequirement",
                                "SchemaDefRequirement",
                                "EnvVarRequirement",
                                "CreateFileRequirement",
                                "ScatterFeatureRequirement",
                                "SubworkflowFeatureRequirement"]

def substitute(value, replace):
    if replace[0] == "^":
        return substitute(value[0:value.rindex('.')], replace[1:])
    else:
        return value + replace

class Builder(object):

    def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]):
        bindings = []
        binding = None
        if "inputBinding" in schema and isinstance(schema["inputBinding"], dict):
            binding = copy.copy(schema["inputBinding"])

            if "position" in binding:
                binding["position"] = aslist(lead_pos) + aslist(binding["position"]) + aslist(tail_pos)
            else:
                binding["position"] = aslist(lead_pos) + [0] + aslist(tail_pos)

            if "valueFrom" in binding:
                binding["do_eval"] = binding["valueFrom"]
            binding["valueFrom"] = datum

            if schema["type"] == "File":
                binding["is_file"] = True

        # Handle union types
        if isinstance(schema["type"], list):
            for t in schema["type"]:
                if isinstance(t, basestring) and self.names.has_name(t, ""):
                    avsc = self.names.get_name(t, "")
                else:
                    avsc = avro.schema.make_avsc_object(t, self.names)
                if validate.validate(avsc, datum):
                    schema = copy.deepcopy(schema)
                    schema["type"] = t
                    return self.bind_input(schema, datum, lead_pos=lead_pos, tail_pos=tail_pos)
            raise validate.ValidationException("'%s' is not a valid union %s" % (datum, schema["type"]))
        elif isinstance(schema["type"], dict):
            st = copy.deepcopy(schema["type"])
            if binding and "inputBinding" not in st and "itemSeparator" not in binding and st["type"] in ("array", "map"):
                st["inputBinding"] = {}
            bindings.extend(self.bind_input(st, datum, lead_pos=lead_pos, tail_pos=tail_pos))
        else:
            if schema["type"] in self.schemaDefs:
                schema = self.schemaDefs[schema["type"]]

            if schema["type"] == "record":
                for f in schema["fields"]:
                    if f["name"] in datum:
                        bindings.extend(self.bind_input(f, datum[f["name"]], lead_pos=lead_pos, tail_pos=f["name"]))

            if schema["type"] == "map":
                for n, item in datum.items():
                    b2 = None
                    if binding:
                        b2 = copy.deepcopy(binding)
                        b2["valueFrom"] = [n, item]
                    bindings.extend(self.bind_input({"type": schema["values"], "inputBinding": b2},
                                                    item, lead_pos=n, tail_pos=tail_pos))
                binding = None

            if schema["type"] == "array":
                for n, item in enumerate(datum):
                    b2 = None
                    if binding:
                        b2 = copy.deepcopy(binding)
                        b2["valueFrom"] = item
                    bindings.extend(self.bind_input({"type": schema["items"], "inputBinding": b2},
                                                    item, lead_pos=n, tail_pos=tail_pos))
                binding = None

            if schema["type"] == "File":
                self.files.append(datum)
                if binding:
                    if binding.get("loadContents"):
                        with self.fs_access.open(datum["path"], "rb") as f:
                            datum["contents"] = f.read(CONTENT_LIMIT)

                    if "secondaryFiles" in binding:
                        if "secondaryFiles" not in datum:
                            datum["secondaryFiles"] = []
                        for sf in aslist(binding["secondaryFiles"]):
                            if isinstance(sf, dict):
                                sfpath = self.do_eval(sf, context=datum["path"])
                            else:
                                sfpath = {"path": substitute(datum["path"], sf)}
                            if isinstance(sfpath, list):
                                datum["secondaryFiles"].extend(sfpath)
                            else:
                                datum["secondaryFiles"].append(sfpath)
                            self.files.append(sfpath)

        # Position to front of the sort key
        if binding:
            for bi in bindings:
                bi["position"] = binding["position"] + bi["position"]
            bindings.append(binding)

        return bindings

    def generate_arg(self, binding):
        value = binding["valueFrom"]
        if "do_eval" in binding:
            value = self.do_eval(binding["do_eval"], context=value)

        prefix = binding.get("prefix")
        sep = binding.get("separate", True)

        l = []
        if isinstance(value, list):
            if binding.get("itemSeparator"):
                l = [binding["itemSeparator"].join([str(v) for v in value])]
            elif binding.get("do_eval"):
                return ([prefix] if prefix else []) + value
            elif prefix:
                return [prefix]
            else:
                return []
        elif binding.get("is_file"):
            l = [value["path"]]
        elif isinstance(value, dict):
            return [prefix] if prefix else []
        elif value is True and prefix:
            return [prefix]
        elif value is False or value is None:
            return []
        else:
            l = [value]

        args = []
        for j in l:
            if sep:
                args.extend([prefix, str(j)])
            else:
                args.append(prefix + str(j))

        return [a for a in args if a is not None]

    def do_eval(self, ex, context=None, pull_image=True):
        return expression.do_eval(ex, self.job, self.requirements, self.outdir, self.tmpdir, context=context, pull_image=pull_image)

class Tool(Process):
    def _init_job(self, joborder, input_basedir, **kwargs):
        builder = Builder()
        builder.job = copy.deepcopy(joborder)

        for i in self.tool["inputs"]:
            (_, d) = urlparse.urldefrag(i["id"])
            if d not in builder.job and "default" in i:
                builder.job[d] = i["default"]

        # Validate job order
        try:
            validate.validate_ex(self.names.get_name("input_record_schema", ""), builder.job)
        except validate.ValidationException as e:
            raise WorkflowException("Error validating input record, " + str(e))

        for r in self.requirements:
            if r["class"] not in supportedProcessRequirements:
                raise WorkflowException("Unsupported process requirement %s" % (r["class"]))

        builder.files = []
        builder.bindings = []
        builder.schemaDefs = self.schemaDefs
        builder.names = self.names
        builder.requirements = self.requirements

        dockerReq, _ = self.get_requirement("DockerRequirement")
        if dockerReq and kwargs.get("use_container"):
            builder.outdir = kwargs.get("docker_outdir") or "/tmp/job_output"
            builder.tmpdir = kwargs.get("docker_tmpdir") or "/tmp/job_tmp"
        else:
            builder.outdir = kwargs.get("outdir") or tempfile.mkdtemp()
            builder.tmpdir = kwargs.get("tmpdir") or tempfile.mkdtemp()

        builder.fs_access = kwargs.get("fs_access") or CommandLineTool.DefaultFsAccess(input_basedir)

        builder.bindings.extend(builder.bind_input(self.inputs_record_schema, builder.job))

        return builder

class ExpressionTool(Tool):
    def __init__(self, toolpath_object, **kwargs):
        super(ExpressionTool, self).__init__(toolpath_object, "ExpressionTool", **kwargs)

    class ExpressionJob(object):
        def run(self, **kwargs):
            try:
                self.output_callback(self.builder.do_eval(self.script), "success")
            except Exception:
                self.output_callback({}, "permanentFail")

    def job(self, joborder, input_basedir, output_callback, **kwargs):
        builder = self._init_job(joborder, input_basedir, **kwargs)

        j = ExpressionTool.ExpressionJob()
        j.builder = builder
        j.script = self.tool["expression"]
        j.output_callback = output_callback
        j.requirements = self.requirements
        j.hints = self.hints
        j.outdir = None
        j.tmpdir = None

        yield j

class CommandLineTool(Tool):
    def __init__(self, toolpath_object, **kwargs):
        super(CommandLineTool, self).__init__(toolpath_object, "CommandLineTool", **kwargs)

    def makeJobRunner(self):
        return CommandLineJob()

    def makePathMapper(self, reffiles, input_basedir, **kwargs):
        dockerReq, _ = self.get_requirement("DockerRequirement")
        if dockerReq and kwargs.get("use_container"):
            return DockerPathMapper(reffiles, input_basedir)
        else:
            return PathMapper(reffiles, input_basedir)

    def job(self, joborder, input_basedir, output_callback, **kwargs):
        builder = self._init_job(joborder, input_basedir, **kwargs)

        if self.tool["baseCommand"]:
            for n, b in enumerate(aslist(self.tool["baseCommand"])):
                builder.bindings.append({
                    "position": [-1000000, n],
                    "valueFrom": b
                })

        if self.tool.get("arguments"):
            for i, a in enumerate(self.tool["arguments"]):
                if isinstance(a, dict):
                    a = copy.copy(a)
                    if a.get("position"):
                        a["position"] = [a["position"], i]
                    else:
                        a["position"] = [0, i]
                    a["do_eval"] = a["valueFrom"]
                    a["valueFrom"] = None
                    builder.bindings.append(a)
                else:
                    builder.bindings.append({
                        "position": [0, i],
                        "valueFrom": a
                    })

        builder.bindings.sort(key=lambda a: a["position"])

        reffiles = set((f["path"] for f in builder.files))

        j = self.makeJobRunner()
        j.joborder = builder.job
        j.stdin = None
        j.stdout = None
        j.successCodes = self.tool.get("successCodes")
        j.temporaryFailCodes = self.tool.get("temporaryFailCodes")
        j.permanentFailCodes = self.tool.get("permanentFailCodes")
        j.requirements = self.requirements
        j.hints = self.hints

        builder.pathmapper = None

        if self.tool.get("stdin"):
            j.stdin = builder.do_eval(self.tool["stdin"])
            if isinstance(j.stdin, dict) and "ref" in j.stdin:
                j.stdin = builder.job[j.stdin["ref"][1:]]["path"]
            reffiles.add(j.stdin)

        if self.tool.get("stdout"):
            j.stdout = builder.do_eval(self.tool["stdout"])
            if os.path.isabs(j.stdout) or ".." in j.stdout:
                raise validate.ValidationException("stdout must be a relative path")

        builder.pathmapper = self.makePathMapper(reffiles, input_basedir, **kwargs)

        dockerReq, _ = self.get_requirement("DockerRequirement")
        if dockerReq and kwargs.get("use_container"):
            j.outdir = kwargs.get("outdir") or tempfile.mkdtemp()
            j.tmpdir = kwargs.get("tmpdir") or tempfile.mkdtemp()
        else:
            j.outdir = builder.outdir
            j.tmpdir = builder.tmpdir

        for f in builder.files:
            f["path"] = builder.pathmapper.mapper(f["path"])[1]

        _logger.debug("Bindings is %s", pprint.pformat(builder.bindings))
        _logger.debug("Files is %s", pprint.pformat({p: builder.pathmapper.mapper(p) for p in builder.pathmapper.files()}))

        builder.requirements = j.requirements

        j.generatefiles = {}
        createFiles, _ = self.get_requirement("CreateFileRequirement")
        if createFiles:
            for t in createFiles["fileDef"]:
                j.generatefiles[t["filename"]] = builder.do_eval(t["fileContent"])

        j.environment = {}
        evr, _ = self.get_requirement("EnvVarRequirement")
        if evr:
            for t in evr["envDef"]:
                j.environment[t["envName"]] = builder.do_eval(t["envValue"])

        j.command_line = flatten(map(builder.generate_arg, builder.bindings))

        j.pathmapper = builder.pathmapper
        j.collect_outputs = functools.partial(self.collect_output_ports, self.tool["outputs"], builder)
        j.output_callback = output_callback

        yield j

    class DefaultFsAccess(object):
        def __init__(self, basedir):
            self.basedir = basedir

        def _abs(self, p):
            if os.path.isabs(p):
                return p
            else:
                return os.path.join(self.basedir, p)

        def glob(self, pattern):
            return glob.glob(self._abs(pattern))

        def open(self, fn, mode):
            return open(self._abs(fn), mode)

        def exists(self, fn):
            return os.path.exists(self._abs(fn))

    def collect_output_ports(self, ports, builder, outdir):
        try:
            custom_output = os.path.join(outdir, "cwl.output.json")
            if builder.fs_access.exists(custom_output):
                outputdoc = yaml.load(custom_output)
                validate.validate_ex(self.names.get_name("outputs_record_schema", ""), outputdoc)
                return outputdoc

            ret = {}
            for port in ports:
                doc_url, fragment = urlparse.urldefrag(port['id'])
                ret[fragment] = self.collect_output(port, builder, outdir)
            validate.validate_ex(self.names.get_name("outputs_record_schema", ""), ret)
            return ret if ret is not None else {}
        except validate.ValidationException as e:
            raise WorkflowException("Error validating output record, " + str(e) + "\n in " + json.dumps(ret, indent=4))

    def collect_output(self, schema, builder, outdir):
        r = None
        if "outputBinding" in schema:
            binding = schema["outputBinding"]
            if "glob" in binding:
                r = []
                bg = builder.do_eval(binding["glob"])
                for gb in aslist(bg):
                    r.extend([{"path": g, "class": "File"} for g in builder.fs_access.glob(os.path.join(outdir, gb))])
                for files in r:
                    checksum = hashlib.sha1()
                    with builder.fs_access.open(files["path"], "rb") as f:
                        contents = f.read(CONTENT_LIMIT)
                        if binding.get("loadContents"):
                            files["contents"] = contents
                        filesize = 0
                        while contents != "":
                            checksum.update(contents)
                            filesize += len(contents)
                            contents = f.read(1024*1024)
                    files["checksum"] = "sha1$%s" % checksum.hexdigest()
                    files["size"] = filesize

            if "outputEval" in binding:
                r = builder.do_eval(binding["outputEval"], context=r)
                if schema["type"] == "File" and (not isinstance(r, dict) or "path" not in r):
                    raise WorkflowException("Expression must return a file object.")

            if schema["type"] == "File":
                if not r:
                    raise WorkflowException("No matches for output file with glob: {}.".format(binding["glob"]))
                if len(r) > 1:
                    raise WorkflowException("Multiple matches for output item that is a single file.")
                r = r[0]

            if schema["type"] == "File" and "secondaryFiles" in binding:
                r["secondaryFiles"] = []
                for sf in aslist(binding["secondaryFiles"]):
                    if isinstance(sf, dict):
                        sfpath = builder.do_eval(sf, context=r["path"])
                    else:
                        sfpath = {"path": substitute(r["path"], sf), "class": "File"}
                    if isinstance(sfpath, list):
                        r["secondaryFiles"].extend(sfpath)
                    else:
                        r["secondaryFiles"].append(sfpath)

        if not r and schema["type"] == "record":
            r = {}
            for f in schema["fields"]:
                r[f["name"]] = self.collect_output(f, builder, outdir)

        return r
<commit_msg>DefaultFsAccess renamed to StdFsAccess<commit_after>import avro.schema
import json
import pprint
import copy
from flatten import flatten
import functools
import os
from pathmapper import PathMapper, DockerPathMapper
from job import CommandLineJob
import yaml
import glob
import logging
import hashlib
import random
from process import Process
from process import WorkflowException
import avro_ld.validate as validate
from aslist import aslist
import expression
import re
import urlparse
import tempfile

_logger = logging.getLogger("cwltool")

CONTENT_LIMIT = 64 * 1024

supportedProcessRequirements = ["DockerRequirement",
                                "ExpressionEngineRequirement",
                                "SchemaDefRequirement",
                                "EnvVarRequirement",
                                "CreateFileRequirement",
                                "ScatterFeatureRequirement",
                                "SubworkflowFeatureRequirement"]

def substitute(value, replace):
    if replace[0] == "^":
        return substitute(value[0:value.rindex('.')], replace[1:])
    else:
        return value + replace

class Builder(object):

    def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]):
        bindings = []
        binding = None
        if "inputBinding" in schema and isinstance(schema["inputBinding"], dict):
            binding = copy.copy(schema["inputBinding"])

            if "position" in binding:
                binding["position"] = aslist(lead_pos) + aslist(binding["position"]) + aslist(tail_pos)
            else:
                binding["position"] = aslist(lead_pos) + [0] + aslist(tail_pos)

            if "valueFrom" in binding:
                binding["do_eval"] = binding["valueFrom"]
            binding["valueFrom"] = datum

            if schema["type"] == "File":
                binding["is_file"] = True

        # Handle union types
        if isinstance(schema["type"], list):
            for t in schema["type"]:
                if isinstance(t, basestring) and self.names.has_name(t, ""):
                    avsc = self.names.get_name(t, "")
                else:
                    avsc = avro.schema.make_avsc_object(t, self.names)
                if validate.validate(avsc, datum):
                    schema = copy.deepcopy(schema)
                    schema["type"] = t
                    return self.bind_input(schema, datum, lead_pos=lead_pos, tail_pos=tail_pos)
            raise validate.ValidationException("'%s' is not a valid union %s" % (datum, schema["type"]))
        elif isinstance(schema["type"], dict):
            st = copy.deepcopy(schema["type"])
            if binding and "inputBinding" not in st and "itemSeparator" not in binding and st["type"] in ("array", "map"):
                st["inputBinding"] = {}
            bindings.extend(self.bind_input(st, datum, lead_pos=lead_pos, tail_pos=tail_pos))
        else:
            if schema["type"] in self.schemaDefs:
                schema = self.schemaDefs[schema["type"]]

            if schema["type"] == "record":
                for f in schema["fields"]:
                    if f["name"] in datum:
                        bindings.extend(self.bind_input(f, datum[f["name"]], lead_pos=lead_pos, tail_pos=f["name"]))

            if schema["type"] == "map":
                for n, item in datum.items():
                    b2 = None
                    if binding:
                        b2 = copy.deepcopy(binding)
                        b2["valueFrom"] = [n, item]
                    bindings.extend(self.bind_input({"type": schema["values"], "inputBinding": b2},
                                                    item, lead_pos=n, tail_pos=tail_pos))
                binding = None

            if schema["type"] == "array":
                for n, item in enumerate(datum):
                    b2 = None
                    if binding:
                        b2 = copy.deepcopy(binding)
                        b2["valueFrom"] = item
                    bindings.extend(self.bind_input({"type": schema["items"], "inputBinding": b2},
                                                    item, lead_pos=n, tail_pos=tail_pos))
                binding = None

            if schema["type"] == "File":
                self.files.append(datum)
                if binding:
                    if binding.get("loadContents"):
                        with self.fs_access.open(datum["path"], "rb") as f:
                            datum["contents"] = f.read(CONTENT_LIMIT)

                    if "secondaryFiles" in binding:
                        if "secondaryFiles" not in datum:
                            datum["secondaryFiles"] = []
                        for sf in aslist(binding["secondaryFiles"]):
                            if isinstance(sf, dict):
                                sfpath = self.do_eval(sf, context=datum["path"])
                            else:
                                sfpath = {"path": substitute(datum["path"], sf)}
                            if isinstance(sfpath, list):
                                datum["secondaryFiles"].extend(sfpath)
                            else:
                                datum["secondaryFiles"].append(sfpath)
                            self.files.append(sfpath)

        # Position to front of the sort key
        if binding:
            for bi in bindings:
                bi["position"] = binding["position"] + bi["position"]
            bindings.append(binding)

        return bindings

    def generate_arg(self, binding):
        value = binding["valueFrom"]
        if "do_eval" in binding:
            value = self.do_eval(binding["do_eval"], context=value)

        prefix = binding.get("prefix")
        sep = binding.get("separate", True)

        l = []
        if isinstance(value, list):
            if binding.get("itemSeparator"):
                l = [binding["itemSeparator"].join([str(v) for v in value])]
            elif binding.get("do_eval"):
                return ([prefix] if prefix else []) + value
            elif prefix:
                return [prefix]
            else:
                return []
        elif binding.get("is_file"):
            l = [value["path"]]
        elif isinstance(value, dict):
            return [prefix] if prefix else []
        elif value is True and prefix:
            return [prefix]
        elif value is False or value is None:
            return []
        else:
            l = [value]

        args = []
        for j in l:
            if sep:
                args.extend([prefix, str(j)])
            else:
                args.append(prefix + str(j))

        return [a for a in args if a is not None]

    def do_eval(self, ex, context=None, pull_image=True):
        return expression.do_eval(ex, self.job, self.requirements, self.outdir, self.tmpdir, context=context, pull_image=pull_image)

class Tool(Process):
    def _init_job(self, joborder, input_basedir, **kwargs):
        builder = Builder()
        builder.job = copy.deepcopy(joborder)

        for i in self.tool["inputs"]:
            (_, d) = urlparse.urldefrag(i["id"])
            if d not in builder.job and "default" in i:
                builder.job[d] = i["default"]

        # Validate job order
        try:
            validate.validate_ex(self.names.get_name("input_record_schema", ""), builder.job)
        except validate.ValidationException as e:
            raise WorkflowException("Error validating input record, " + str(e))

        for r in self.requirements:
            if r["class"] not in supportedProcessRequirements:
                raise WorkflowException("Unsupported process requirement %s" % (r["class"]))

        builder.files = []
        builder.bindings = []
        builder.schemaDefs = self.schemaDefs
        builder.names = self.names
        builder.requirements = self.requirements

        dockerReq, _ = self.get_requirement("DockerRequirement")
        if dockerReq and kwargs.get("use_container"):
            builder.outdir = kwargs.get("docker_outdir") or "/tmp/job_output"
            builder.tmpdir = kwargs.get("docker_tmpdir") or "/tmp/job_tmp"
        else:
            builder.outdir = kwargs.get("outdir") or tempfile.mkdtemp()
            builder.tmpdir = kwargs.get("tmpdir") or tempfile.mkdtemp()

        builder.fs_access = kwargs.get("fs_access") or StdFsAccess(input_basedir)

        builder.bindings.extend(builder.bind_input(self.inputs_record_schema, builder.job))

        return builder

class ExpressionTool(Tool):
    def __init__(self, toolpath_object, **kwargs):
        super(ExpressionTool, self).__init__(toolpath_object, "ExpressionTool", **kwargs)

    class ExpressionJob(object):
        def run(self, **kwargs):
            try:
                self.output_callback(self.builder.do_eval(self.script), "success")
            except Exception:
                self.output_callback({}, "permanentFail")

    def job(self, joborder, input_basedir, output_callback, **kwargs):
        builder = self._init_job(joborder, input_basedir, **kwargs)

        j = ExpressionTool.ExpressionJob()
        j.builder = builder
        j.script = self.tool["expression"]
        j.output_callback = output_callback
        j.requirements = self.requirements
        j.hints = self.hints
        j.outdir = None
        j.tmpdir = None

        yield j

class StdFsAccess(object):
    def __init__(self, basedir):
        self.basedir = basedir

    def _abs(self, p):
        if os.path.isabs(p):
            return p
        else:
            return os.path.join(self.basedir, p)

    def glob(self, pattern):
        return glob.glob(self._abs(pattern))

    def open(self, fn, mode):
        return open(self._abs(fn), mode)

    def exists(self, fn):
        return os.path.exists(self._abs(fn))

class CommandLineTool(Tool):
    def __init__(self, toolpath_object, **kwargs):
        super(CommandLineTool, self).__init__(toolpath_object, "CommandLineTool", **kwargs)

    def makeJobRunner(self):
        return CommandLineJob()

    def makePathMapper(self, reffiles, input_basedir, **kwargs):
        dockerReq, _ = self.get_requirement("DockerRequirement")
        if dockerReq and kwargs.get("use_container"):
            return DockerPathMapper(reffiles, input_basedir)
        else:
            return PathMapper(reffiles, input_basedir)

    def job(self, joborder, input_basedir, output_callback, **kwargs):
        builder = self._init_job(joborder, input_basedir, **kwargs)

        if self.tool["baseCommand"]:
            for n, b in enumerate(aslist(self.tool["baseCommand"])):
                builder.bindings.append({
                    "position": [-1000000, n],
                    "valueFrom": b
                })

        if self.tool.get("arguments"):
            for i, a in enumerate(self.tool["arguments"]):
                if isinstance(a, dict):
                    a = copy.copy(a)
                    if a.get("position"):
                        a["position"] = [a["position"], i]
                    else:
                        a["position"] = [0, i]
                    a["do_eval"] = a["valueFrom"]
                    a["valueFrom"] = None
                    builder.bindings.append(a)
                else:
                    builder.bindings.append({
                        "position": [0, i],
                        "valueFrom": a
                    })

        builder.bindings.sort(key=lambda a: a["position"])

        reffiles = set((f["path"] for f in builder.files))

        j = self.makeJobRunner()
        j.joborder = builder.job
        j.stdin = None
        j.stdout = None
        j.successCodes = self.tool.get("successCodes")
        j.temporaryFailCodes = self.tool.get("temporaryFailCodes")
        j.permanentFailCodes = self.tool.get("permanentFailCodes")
        j.requirements = self.requirements
        j.hints = self.hints

        builder.pathmapper = None

        if self.tool.get("stdin"):
            j.stdin = builder.do_eval(self.tool["stdin"])
            if isinstance(j.stdin, dict) and "ref" in j.stdin:
                j.stdin = builder.job[j.stdin["ref"][1:]]["path"]
            reffiles.add(j.stdin)

        if self.tool.get("stdout"):
            j.stdout = builder.do_eval(self.tool["stdout"])
            if os.path.isabs(j.stdout) or ".." in j.stdout:
                raise validate.ValidationException("stdout must be a relative path")

        builder.pathmapper = self.makePathMapper(reffiles, input_basedir, **kwargs)

        dockerReq, _ = self.get_requirement("DockerRequirement")
        if dockerReq and kwargs.get("use_container"):
            j.outdir = kwargs.get("outdir") or tempfile.mkdtemp()
            j.tmpdir = kwargs.get("tmpdir") or tempfile.mkdtemp()
        else:
            j.outdir = builder.outdir
            j.tmpdir = builder.tmpdir

        for f in builder.files:
            f["path"] = builder.pathmapper.mapper(f["path"])[1]

        _logger.debug("Bindings is %s", pprint.pformat(builder.bindings))
        _logger.debug("Files is %s", pprint.pformat({p: builder.pathmapper.mapper(p) for p in builder.pathmapper.files()}))

        builder.requirements = j.requirements

        j.generatefiles = {}
        createFiles, _ = self.get_requirement("CreateFileRequirement")
        if createFiles:
            for t in createFiles["fileDef"]:
                j.generatefiles[t["filename"]] = builder.do_eval(t["fileContent"])

        j.environment = {}
        evr, _ = self.get_requirement("EnvVarRequirement")
        if evr:
            for t in evr["envDef"]:
                j.environment[t["envName"]] = builder.do_eval(t["envValue"])

        j.command_line = flatten(map(builder.generate_arg, builder.bindings))

        j.pathmapper = builder.pathmapper
        j.collect_outputs = functools.partial(self.collect_output_ports, self.tool["outputs"], builder)
        j.output_callback = output_callback

        yield j

    def collect_output_ports(self, ports, builder, outdir):
        try:
            custom_output = os.path.join(outdir, "cwl.output.json")
            if builder.fs_access.exists(custom_output):
                outputdoc = yaml.load(custom_output)
                validate.validate_ex(self.names.get_name("outputs_record_schema", ""), outputdoc)
                return outputdoc

            ret = {}
            for port in ports:
                doc_url, fragment = urlparse.urldefrag(port['id'])
                ret[fragment] = self.collect_output(port, builder, outdir)
            validate.validate_ex(self.names.get_name("outputs_record_schema", ""), ret)
            return ret if ret is not None else {}
        except validate.ValidationException as e:
            raise WorkflowException("Error validating output record, " + str(e) + "\n in " + json.dumps(ret, indent=4))

    def collect_output(self, schema, builder, outdir):
        r = None
        if "outputBinding" in schema:
            binding = schema["outputBinding"]
            if "glob" in binding:
                r = []
                bg = builder.do_eval(binding["glob"])
                for gb in aslist(bg):
                    r.extend([{"path": g, "class": "File"} for g in builder.fs_access.glob(os.path.join(outdir, gb))])
                for files in r:
                    checksum = hashlib.sha1()
                    with builder.fs_access.open(files["path"], "rb") as f:
                        contents = f.read(CONTENT_LIMIT)
                        if binding.get("loadContents"):
                            files["contents"] = contents
                        filesize = 0
                        while contents != "":
                            checksum.update(contents)
                            filesize += len(contents)
                            contents = f.read(1024*1024)
                    files["checksum"] = "sha1$%s" % checksum.hexdigest()
                    files["size"] = filesize

            if "outputEval" in binding:
                r = builder.do_eval(binding["outputEval"], context=r)
                if schema["type"] == "File" and (not isinstance(r, dict) or "path" not in r):
                    raise WorkflowException("Expression must return a file object.")

            if schema["type"] == "File":
                if not r:
                    raise WorkflowException("No matches for output file with glob: {}.".format(binding["glob"]))
                if len(r) > 1:
                    raise WorkflowException("Multiple matches for output item that is a single file.")
                r = r[0]

            if schema["type"] == "File" and "secondaryFiles" in binding:
                r["secondaryFiles"] = []
                for sf in aslist(binding["secondaryFiles"]):
                    if isinstance(sf, dict):
                        sfpath = builder.do_eval(sf, context=r["path"])
                    else:
                        sfpath = {"path": substitute(r["path"], sf), "class": "File"}
                    if isinstance(sfpath, list):
                        r["secondaryFiles"].extend(sfpath)
                    else:
                        r["secondaryFiles"].append(sfpath)

        if not r and schema["type"] == "record":
            r = {}
            for f in schema["fields"]:
                r[f["name"]] = self.collect_output(f, builder, outdir)

        return r
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Tue Mar 04 10:41:28 2014

@author: aitor
"""

import difflib
import json
import mysql.connector

import os, sys

lib_path = os.path.abspath('../')
sys.path.append(lib_path)

base_dir = os.path.dirname(os.path.abspath(__file__))

def load_config():
    from model.dbconnection import dbconfig

    return dbconfig

config = load_config()

def get_complete_names():
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    cursor.execute("SELECT DISTINCT(name) FROM person")
    result = []
    for name in cursor:
        result.append(name[0])
    cursor.close()
    return result

#checks for similar names
def check_similar_names():
    print 'Getting names'
    names = get_complete_names()
    print 'Total names:', len(names)
    # min similarity ratio between strings
    threshold_ratio = 0.90
    repeated = []
    total = float(len(names))

    for i, str_1 in enumerate(names):
        if i%100 == 0:
                print 'Name percentage processed:', str((i/total) * 100)

        for j in range(i+1, len(names)):
            str_2 = names[j]

            if (difflib.SequenceMatcher(None, str_1, str_2).ratio() > threshold_ratio):
                repeated.append((str_1, str_2))

    with open( base_dir + "/cache/repeated_names.json", "wb" ) as outfile:
        json.dump(repeated, outfile)

    return repeated

def get_thesis_names():
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    cursor.execute("SELECT DISTINCT(title) FROM thesis")
    result = []
    for name in cursor:
        result.append(name[0])
    cursor.close()
    return result

def get_repeated_thesis_ids(distinct_titles):
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    repeated_ids = []
    total = float(len(distinct_titles))
    for i, title in enumerate(distinct_titles):
        if i%100 == 0:
            print 'Getting repeated ids:', (i/total)*100
        cursor.execute("SELECT id FROM thesis WHERE title= %s", (title,))
        name_ids = []
        for thesis_id in cursor:
            name_ids.append(thesis_id[0])
        if len(name_ids) > 1:
            print 'Repeated ids', name_ids
            repeated_ids.append(name_ids)

    cursor.close()

    with open( base_dir + "/cache/repeated_thesis_ids.json", "wb" ) as outfile:
        json.dump(repeated_ids, outfile)

    return repeated_ids

def check_repeated_thesis():
    print 'Getting names'
    distinct_names = get_thesis_names()
    print 'Distinct names: ', len(distinct_names)
    repeated_ids = get_repeated_thesis_ids(distinct_names)
    print repeated_ids

def delete_repeated_thesis():
    repeated_ids = json.load(open( base_dir + "/cache/repeated_thesis_ids.json", "rb" ))
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    deleted = 0
    for id_group in repeated_ids:
        id_group.sort()
        to_delete = id_group[0:len(id_group)-1]
        for thesis_id in to_delete:
            print 'Deleting:', thesis_id
            cursor.execute("DELETE FROM thesis WHERE id=" + str(thesis_id))
            deleted +=1

    print 'Deleted tesis:', deleted
    cursor.close()

def get_person_ids():
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    cursor.execute("SELECT DISTINCT(id) FROM person")
    result = []
    for person_id in cursor:
        result.append(person_id[0])
    cursor.close()
    return result

def get_unused_person_ids(person_ids):
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    unused_ids = []
    total = float(len(person_ids))

    for i, person_id in enumerate(person_ids):
        if i%100 == 0:
            print 'Getting unused persons:', (i/total)*100

        author = False
        advisor = False
        panel = False

        cursor.execute("SELECT COUNT(id) FROM thesis WHERE author_id=" + str(person_id))
        for total_thesis in cursor:
            try:
                if total_thesis[0] > 0:
                    author = True
            except:
                pass

        cursor.execute("SELECT COUNT(thesis_id) FROM advisor WHERE person_id="+ str(person_id))
        for total_thesis in cursor:
            try:
                if total_thesis[0] > 0:
                    advisor = True
            except:
                pass

        cursor.execute("SELECT COUNT(thesis_id) FROM panel_member WHERE person_id="+ str(person_id))
        for total_thesis in cursor:
            try:
                if total_thesis[0] > 0:
                    panel = True
            except:
                pass

        if not (author or advisor or panel):
            unused_ids.append(person_id)

    cursor.close()
    with open( base_dir + "/cache/unused_person_ids.json", "wb" ) as outfile:
        json.dump(unused_ids, outfile)

    return unused_ids

def check_unused_person_ids():
    print 'Getting all person ids'
    person_ids = get_person_ids()
    print 'Distinct ids: ', len(person_ids)
    unused_ids = get_unused_person_ids(person_ids)
    print unused_ids

def nuke_unused_persons():
    unused_ids = json.load(open( base_dir + "/cache/unused_person_ids.json", "rb" ))
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    deleted = 0
    for person_id in unused_ids:
        cursor.execute("DELETE FROM advisor WHERE person_id=" + str(person_id))
        cursor.execute("DELETE FROM panel_member WHERE person_id=" + str(person_id))
        cursor.execute("DELETE FROM person WHERE id=" + str(person_id))
        deleted +=1

    print 'Deleted persons:', deleted
    cursor.close()

def get_distinct_names():
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    cursor.execute("SELECT DISTINCT(name) FROM person")
    result = []
    for name in cursor:
        result.append(name[0])
    cursor.close()

    return result

def get_same_name_ids(distinct_names):
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    name_ids = {}
    total = float(len(distinct_names))

    for i, name in enumerate(distinct_names):
        if i%50 == 0:
            print 'Getting name ids:', (i/total)*100
        try:
            cursor.execute("SELECT id FROM person WHERE name=%s", (name,))
            ids = []
            for person_id in cursor:
                ids.append(person_id[0])
            name_ids[name] = ids
        except:
            print 'Problem with name:', name

    with open( base_dir + "/cache/person_name_ids.json", "wb" ) as outfile:
        json.dump(name_ids, outfile)

    cursor.close()
    return name_ids

def check_repeated_name_ids():
    print 'Getting distinct names'
    distinct_names = get_distinct_names()
    print 'Distinct names: ', len(distinct_names)
    name_ids = get_same_name_ids(distinct_names)
    print name_ids

def merge_names():
    print 'Merging all ids of the same names'
    name_ids = json.load(open( base_dir + "/cache/person_name_ids.json", "rb" ))
    print 'Total names:', len(name_ids)
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    total = float(len(name_ids))

    for i, name in enumerate(name_ids):
        if i%50 == 0:
            print 'Merging:', (i/total)*100

        id_group = name_ids[name]
        if len(id_group) > 1:
            base_id = id_group[0]
            dying_ids = id_group[1:]
            for dying_id in dying_ids:
                try:
                    cursor.execute("UPDATE advisor SET person_id = " + str(base_id) + " WHERE person_id = " + str(dying_id))
                except mysql.connector.errors.IntegrityError:
                    print 'Already is an advisor for that thesis'
                cursor.execute("UPDATE thesis SET author_id = " + str(base_id) + " WHERE author_id = " + str(dying_id))
                try:
                    cursor.execute("UPDATE panel_member SET person_id = " + str(base_id) + " WHERE person_id = " + str(dying_id))
                except mysql.connector.errors.IntegrityError:
                    print 'Already is a panel member for that thesis'

                cursor.execute("DELETE FROM person WHERE id=" + str(dying_id))

    cursor.close()
    print 'The great merge has ended, all hail the new clean database'

#all unesco codes must have the following format: ######
def clean_unesco_codes():
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    cnx2 = mysql.connector.connect(**config)
    cursor2 = cnx2.cursor()
    cursor.execute("SELECT id, code FROM descriptor")
    for descriptor in cursor:
        code = descriptor[1]
        desc_id = descriptor[0]
        if len(str(descriptor[1])) == 2:
            code = code * 10000
        elif len(str(descriptor[1])) == 4:
            code = code * 100

        if code != descriptor[1]:
            cursor2.execute("UPDATE descriptor SET code = " + str(code) + " WHERE id = " + str(desc_id))

    cursor.close()
    cursor2.close()

if __name__=='__main__':
    check_repeated_thesis()
    delete_repeated_thesis()
    check_unused_person_ids()
    nuke_unused_persons()
    check_repeated_name_ids()
    check_similar_names()
    merge_names()
    clean_unesco_codes()

    <commit_msg>added method to set genders<commit_after># -*- coding: utf-8 -*-
"""
Created on Tue Mar 04 10:41:28 2014

@author: aitor
"""

import difflib
import json
import mysql.connector

import os, sys

lib_path = os.path.abspath('../')
sys.path.append(lib_path)

base_dir = os.path.dirname(os.path.abspath(__file__))

def load_config():
    from model.dbconnection import dbconfig

    return dbconfig

config = load_config()

def get_complete_names():
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    cursor.execute("SELECT DISTINCT(name) FROM person")
    result = []
    for name in cursor:
        result.append(name[0])
    cursor.close()
    return result

#checks for similar names
def check_similar_names():
    print 'Getting names'
    names = get_complete_names()
    print 'Total names:', len(names)
    # min similarity ratio between strings
    threshold_ratio = 0.90
    repeated = []
    total = float(len(names))

    for i, str_1 in enumerate(names):
        if i%100 == 0:
                print 'Name percentage processed:', str((i/total) * 100)

        for j in range(i+1, len(names)):
            str_2 = names[j]

            if (difflib.SequenceMatcher(None, str_1, str_2).ratio() > threshold_ratio):
                repeated.append((str_1, str_2))

    with open( base_dir + "/cache/repeated_names.json", "wb" ) as outfile:
        json.dump(repeated, outfile)

    return repeated

def get_thesis_names():
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    cursor.execute("SELECT DISTINCT(title) FROM thesis")
    result = []
    for name in cursor:
        result.append(name[0])
    cursor.close()
    return result

def get_repeated_thesis_ids(distinct_titles):
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    repeated_ids = []
    total = float(len(distinct_titles))
    for i, title in enumerate(distinct_titles):
        if i%100 == 0:
            print 'Getting repeated ids:', (i/total)*100
        cursor.execute("SELECT id FROM thesis WHERE title= %s", (title,))
        name_ids = []
        for thesis_id in cursor:
            name_ids.append(thesis_id[0])
        if len(name_ids) > 1:
            print 'Repeated ids', name_ids
            repeated_ids.append(name_ids)

    cursor.close()

    with open( base_dir + "/cache/repeated_thesis_ids.json", "wb" ) as outfile:
        json.dump(repeated_ids, outfile)

    return repeated_ids

def check_repeated_thesis():
    print 'Getting names'
    distinct_names = get_thesis_names()
    print 'Distinct names: ', len(distinct_names)
    repeated_ids = get_repeated_thesis_ids(distinct_names)
    print repeated_ids

def delete_repeated_thesis():
    repeated_ids = json.load(open( base_dir + "/cache/repeated_thesis_ids.json", "rb" ))
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    deleted = 0
    for id_group in repeated_ids:
        id_group.sort()
        to_delete = id_group[0:len(id_group)-1]
        for thesis_id in to_delete:
            print 'Deleting:', thesis_id
            cursor.execute("DELETE FROM thesis WHERE id=" + str(thesis_id))
            deleted +=1

    print 'Deleted tesis:', deleted
    cursor.close()

def get_person_ids():
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    cursor.execute("SELECT DISTINCT(id) FROM person")
    result = []
    for person_id in cursor:
        result.append(person_id[0])
    cursor.close()
    return result

def get_unused_person_ids(person_ids):
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    unused_ids = []
    total = float(len(person_ids))

    for i, person_id in enumerate(person_ids):
        if i%100 == 0:
            print 'Getting unused persons:', (i/total)*100

        author = False
        advisor = False
        panel = False

        cursor.execute("SELECT COUNT(id) FROM thesis WHERE author_id=" + str(person_id))
        for total_thesis in cursor:
            try:
                if total_thesis[0] > 0:
                    author = True
            except:
                pass

        cursor.execute("SELECT COUNT(thesis_id) FROM advisor WHERE person_id="+ str(person_id))
        for total_thesis in cursor:
            try:
                if total_thesis[0] > 0:
                    advisor = True
            except:
                pass

        cursor.execute("SELECT COUNT(thesis_id) FROM panel_member WHERE person_id="+ str(person_id))
        for total_thesis in cursor:
            try:
                if total_thesis[0] > 0:
                    panel = True
            except:
                pass

        if not (author or advisor or panel):
            unused_ids.append(person_id)

    cursor.close()
    with open( base_dir + "/cache/unused_person_ids.json", "wb" ) as outfile:
        json.dump(unused_ids, outfile)

    return unused_ids

def check_unused_person_ids():
    print 'Getting all person ids'
    person_ids = get_person_ids()
    print 'Distinct ids: ', len(person_ids)
    unused_ids = get_unused_person_ids(person_ids)
    print unused_ids

def nuke_unused_persons():
    unused_ids = json.load(open( base_dir + "/cache/unused_person_ids.json", "rb" ))
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    deleted = 0
    for person_id in unused_ids:
        cursor.execute("DELETE FROM advisor WHERE person_id=" + str(person_id))
        cursor.execute("DELETE FROM panel_member WHERE person_id=" + str(person_id))
        cursor.execute("DELETE FROM person WHERE id=" + str(person_id))
        deleted +=1

    print 'Deleted persons:', deleted
    cursor.close()

def get_distinct_names():
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    cursor.execute("SELECT DISTINCT(name) FROM person")
    result = []
    for name in cursor:
        result.append(name[0])
    cursor.close()

    return result

def get_same_name_ids(distinct_names):
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    name_ids = {}
    total = float(len(distinct_names))

    for i, name in enumerate(distinct_names):
        if i%50 == 0:
            print 'Getting name ids:', (i/total)*100
        try:
            cursor.execute("SELECT id FROM person WHERE name=%s", (name,))
            ids = []
            for person_id in cursor:
                ids.append(person_id[0])
            name_ids[name] = ids
        except:
            print 'Problem with name:', name

    with open( base_dir + "/cache/person_name_ids.json", "wb" ) as outfile:
        json.dump(name_ids, outfile)

    cursor.close()
    return name_ids

def check_repeated_name_ids():
    print 'Getting distinct names'
    distinct_names = get_distinct_names()
    print 'Distinct names: ', len(distinct_names)
    name_ids = get_same_name_ids(distinct_names)
    print name_ids

def merge_names():
    print 'Merging all ids of the same names'
    name_ids = json.load(open( base_dir + "/cache/person_name_ids.json", "rb" ))
    print 'Total names:', len(name_ids)
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    total = float(len(name_ids))

    for i, name in enumerate(name_ids):
        if i%50 == 0:
            print 'Merging:', (i/total)*100

        id_group = name_ids[name]
        if len(id_group) > 1:
            base_id = id_group[0]
            dying_ids = id_group[1:]
            for dying_id in dying_ids:
                try:
                    cursor.execute("UPDATE advisor SET person_id = " + str(base_id) + " WHERE person_id = " + str(dying_id))
                except mysql.connector.errors.IntegrityError:
                    print 'Already is an advisor for that thesis'
                cursor.execute("UPDATE thesis SET author_id = " + str(base_id) + " WHERE author_id = " + str(dying_id))
                try:
                    cursor.execute("UPDATE panel_member SET person_id = " + str(base_id) + " WHERE person_id = " + str(dying_id))
                except mysql.connector.errors.IntegrityError:
                    print 'Already is a panel member for that thesis'

                cursor.execute("DELETE FROM person WHERE id=" + str(dying_id))

    cursor.close()
    print 'The great merge has ended, all hail the new clean database'

#all unesco codes must have the following format: ######
def clean_unesco_codes():
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    cnx2 = mysql.connector.connect(**config)
    cursor2 = cnx2.cursor()
    cursor.execute("SELECT id, code FROM descriptor")
    for descriptor in cursor:
        code = descriptor[1]
        desc_id = descriptor[0]
        if len(str(descriptor[1])) == 2:
            code = code * 10000
        elif len(str(descriptor[1])) == 4:
            code = code * 100

        if code != descriptor[1]:
            cursor2.execute("UPDATE descriptor SET code = " + str(code) + " WHERE id = " + str(desc_id))

    cursor.close()
    cursor2.close()

def set_genders():
    from cache import name_genders

    names = get_distinct_names()
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()
    for complete_name in names:
        name = complete_name[0].split(' ')[0]
        gender = name_genders[name]
        cursor.execute('UPDATE person SET gender = %s WHERE name = %s', (gender, name))

    cursor.close()

if __name__=='__main__':
#    check_repeated_thesis()
#    delete_repeated_thesis()
#    check_unused_person_ids()
#    nuke_unused_persons()
#    check_repeated_name_ids()
#    check_similar_names()
#    merge_names()
#    clean_unesco_codes()
    set_genders()
    <|endoftext|>