<commit_before>#!/usr/bin/env python3

# Copyright 2014-2015 Open Source Robotics Foundation, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import re
import sys
import time
from xml.sax.saxutils import escape
from xml.sax.saxutils import quoteattr

from ament_cpplint import cpplint
from ament_cpplint.cpplint import _cpplint_state
from ament_cpplint.cpplint import ParseArguments
from ament_cpplint.cpplint import ProcessFile

# use custom header guard with two underscore between the name parts
def custom_get_header_guard_cpp_variable(filename):
    from ament_cpplint.cpplint import _root
    from ament_cpplint.cpplint import FileInfo
    # Restores original filename in case that cpplint is invoked from Emacs's
    # flymake.
    filename = re.sub(r'_flymake\.h$', '.h', filename)
    filename = re.sub(r'/\.flymake/([^/]*)$', r'/\1', filename)
    # Replace 'c++' with 'cpp'.
    filename = filename.replace('C++', 'cpp').replace('c++', 'cpp')

    fileinfo = FileInfo(filename)
    file_path_from_root = fileinfo.RepositoryName()
    if _root:
        prefix = _root + os.sep
        # use consistent separator on Windows
        if os.sep != '/':
            prefix = prefix.replace(os.sep, '/')
        if file_path_from_root.startswith(prefix):
            file_path_from_root = file_path_from_root[len(prefix):]
        else:
            filename = filename.replace(os.sep, '/')
            if filename.startswith(prefix):
                file_path_from_root = filename[len(prefix):]
    # use double separator
    file_path_from_root = file_path_from_root.replace('/', '//')
    return re.sub(r'[^a-zA-Z0-9]', '_', file_path_from_root).upper() + '_'

cpplint.GetHeaderGuardCPPVariable = custom_get_header_guard_cpp_variable

def main(argv=sys.argv[1:]):
    extensions = ['c', 'cc', 'cpp', 'cxx']
    headers = ['h', 'hh', 'hpp', 'hxx']

    parser = argparse.ArgumentParser(
        description='Check code against the Google style conventions using '
                    'cpplint.',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument(
        '--filters', metavar='FILTER,FILTER,...', type=str,
        help='A comma separated list of category filters to apply')
    parser.add_argument(
        '--linelength', metavar='N', type=int, default=100,
        help='The maximum line length')
    parser.add_argument(
        '--root', type=str,
        help='The --root option for cpplint')
    parser.add_argument(
        'paths',
        nargs='*',
        default=[os.curdir],
        help='The files or directories to check. For directories files ending '
             'in %s will be considered.' %
             ', '.join(["'.%s'" % e for e in extensions + headers]))
    # not using a file handle directly
    # in order to prevent leaving an empty file when something fails early
    parser.add_argument(
        '--xunit-file',
        help='Generate a xunit compliant XML file')
    args = parser.parse_args(argv)

    if args.xunit_file:
        start_time = time.time()

    argv = []
    # collect category based counts
    argv.append('--counting=detailed')
    argv.append('--extensions=%s' % ','.join(extensions))
    argv.append('--headers=%s' % ','.join(headers))
    filters = [
        # we do allow C++11
        '-build/c++11',
        # we consider passing non-const references to be ok
        '-runtime/references',
        # we wrap open curly braces for namespaces, classes and functions
        '-whitespace/braces',
        # we don't indent keywords like public, protected and private with one space
        '-whitespace/indent',
        # we allow closing parenthesis to be on the next line
        '-whitespace/parens',
        # we allow the developer to decide about whitespace after a semicolon
        '-whitespace/semicolon',
    ]
    if args.filters:
        filters += args.filters.split(',')
    argv.append('--filter=%s' % ','.join(filters))

    argv.append('--linelength=%d' % args.linelength)

    groups = get_file_groups(args.paths, extensions + headers)
    if not groups:
        print('No files found', file=sys.stderr)
        return 1

    # hook into error reporting
    import ament_cpplint.cpplint
    DefaultError = ament_cpplint.cpplint.Error  # noqa: N806
    report = []

    # invoke cpplint for each root group of files
    _cpplint_state.ResetErrorCounts()
    for root in sorted(groups.keys()):
        files = groups[root]

        arguments = list(argv)
        if args.root:
            root = os.path.abspath(args.root)
        if root:
            root_arg = '--root=%s' % root
            arguments.append(root_arg)
            print("Using '%s' argument" % root_arg)
        else:
            print("Not using '--root'")
        print('')
        arguments += files
        filenames = ParseArguments(arguments)

        for filename in filenames:
            # hook into error reporting
            errors = []

            def custom_error(filename, linenum, category, confidence, message):
                if ament_cpplint.cpplint._ShouldPrintError(category, confidence, linenum):
                    errors.append({
                        'linenum': linenum,
                        'category': category,
                        'confidence': confidence,
                        'message': message,
                    })
                DefaultError(filename, linenum, category, confidence, message)
            ament_cpplint.cpplint.Error = custom_error

            ProcessFile(filename, _cpplint_state.verbose_level)
            report.append((filename, errors))
            print('')

    # output summary
    for category in sorted(_cpplint_state.errors_by_category.keys()):
        count = _cpplint_state.errors_by_category[category]
        print("Category '%s' errors found: %d" % (category, count),
              file=sys.stderr)
    if _cpplint_state.error_count:
        print('Total errors found: %d' % _cpplint_state.error_count,
              file=sys.stderr)
    else:
        print('No errors found')

    # generate xunit file
    if args.xunit_file:
        folder_name = os.path.basename(os.path.dirname(args.xunit_file))
        file_name = os.path.basename(args.xunit_file)
        suffix = '.xml'
        if file_name.endswith(suffix):
            file_name = file_name[0:-len(suffix)]
            suffix = '.xunit'
            if file_name.endswith(suffix):
                file_name = file_name[0:-len(suffix)]
        testname = '%s.%s' % (folder_name, file_name)

        xml = get_xunit_content(report, testname, time.time() - start_time)
        path = os.path.dirname(os.path.abspath(args.xunit_file))
        if not os.path.exists(path):
            os.makedirs(path)
        with open(args.xunit_file, 'w') as f:
            f.write(xml)

    return 1 if _cpplint_state.error_count else 0

def get_file_groups(paths, extensions):
    # dict mapping root path to files
    groups = {}
    for path in paths:
        if os.path.isdir(path):
            for dirpath, dirnames, filenames in os.walk(path):
                if 'AMENT_IGNORE' in filenames:
                    dirnames[:] = []
                    continue
                # ignore folder starting with . or _
                dirnames[:] = [d for d in dirnames if d[0] not in ['.', '_']]
                dirnames.sort()

                # select files by extension
                for filename in sorted(filenames):
                    _, ext = os.path.splitext(filename)
                    if ext in ['.%s' % e for e in extensions]:
                        append_file_to_group(groups,
                                             os.path.join(dirpath, filename))
        if os.path.isfile(path):
            append_file_to_group(groups, path)
    return groups

def append_file_to_group(groups, path):
    path = os.path.abspath(path)

    root = None

    # try to determine root from path
    base_path = os.path.dirname(path)
    # find longest subpath which ends with one of the following subfolder names
    subfolder_names = ['include', 'src', 'test']
    matches = [
        re.search(
            '^(.+%s%s)%s' %
            (re.escape(os.sep), re.escape(subfolder_name), re.escape(os.sep)), path)
        for subfolder_name in subfolder_names]
    match_groups = [match.group(1) for match in matches if match]
    if match_groups:
        match_groups = [{'group_len': len(x), 'group': x} for x in match_groups]
        sorted_groups = sorted(match_groups, key=lambda k: k['group_len'])
        base_path = sorted_groups[-1]['group']
        root = base_path

    # try to find repository root
    repo_root = None
    p = path
    while p and repo_root is None:
        # abort if root is reached
        if os.path.dirname(p) == p:
            break
        p = os.path.dirname(p)
        for marker in ['.git', '.hg', '.svn']:
            if os.path.exists(os.path.join(p, marker)):
                repo_root = p
                break

    # compute relative --root argument
    if repo_root and repo_root > base_path:
        root = os.path.relpath(base_path, repo_root)

    # add the path to the appropriate group
    if root not in groups:
        groups[root] = []
    groups[root].append(path)

def get_xunit_content(report, testname, elapsed):
    test_count = sum(max(len(r[1]), 1) for r in report)
    error_count = sum(len(r[1]) for r in report)
    data = {
        'testname': testname,
        'test_count': test_count,
        'error_count': error_count,
        'time': '%.3f' % round(elapsed, 3),
    }
    xml = """<?xml version="1.0" encoding="UTF-8"?>
<testsuite
  name="%(testname)s"
  tests="%(test_count)d"
  failures="%(error_count)d"
  time="%(time)s"
>
""" % data

    for (filename, errors) in report:

        if errors:
            # report each cpplint error as a failing testcase
            for error in errors:
                data = {
                    'quoted_name': quoteattr(
                        '%s [%s] (%s:%d)' % (
                            error['category'], error['confidence'],
                            filename, error['linenum'])),
                    'testname': testname,
                    'quoted_message': quoteattr(error['message']),
                }
                xml += """  <testcase
    name=%(quoted_name)s
    classname="%(testname)s"
  >
      <failure message=%(quoted_message)s/>
  </testcase>
""" % data

        else:
            # if there are no cpplint errors report a single successful test
            data = {
                'quoted_location': quoteattr(filename),
                'testname': testname,
            }
            xml += """  <testcase
    name=%(quoted_location)s
    classname="%(testname)s"
    status="No errors"/>
""" % data

    # output list of checked files
    data = {
        'escaped_files': escape(''.join(['\n* %s' % r[0] for r in report])),
    }
    xml += """  <system-out>Checked files:%(escaped_files)s</system-out>
""" % data

    xml += '</testsuite>\n'
    return xml

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>fix sorting of keys in same cases (#127)<commit_after>#!/usr/bin/env python3

# Copyright 2014-2015 Open Source Robotics Foundation, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import re
import sys
import time
from xml.sax.saxutils import escape
from xml.sax.saxutils import quoteattr

from ament_cpplint import cpplint
from ament_cpplint.cpplint import _cpplint_state
from ament_cpplint.cpplint import ParseArguments
from ament_cpplint.cpplint import ProcessFile

# use custom header guard with two underscore between the name parts
def custom_get_header_guard_cpp_variable(filename):
    from ament_cpplint.cpplint import _root
    from ament_cpplint.cpplint import FileInfo
    # Restores original filename in case that cpplint is invoked from Emacs's
    # flymake.
    filename = re.sub(r'_flymake\.h$', '.h', filename)
    filename = re.sub(r'/\.flymake/([^/]*)$', r'/\1', filename)
    # Replace 'c++' with 'cpp'.
    filename = filename.replace('C++', 'cpp').replace('c++', 'cpp')

    fileinfo = FileInfo(filename)
    file_path_from_root = fileinfo.RepositoryName()
    if _root:
        prefix = _root + os.sep
        # use consistent separator on Windows
        if os.sep != '/':
            prefix = prefix.replace(os.sep, '/')
        if file_path_from_root.startswith(prefix):
            file_path_from_root = file_path_from_root[len(prefix):]
        else:
            filename = filename.replace(os.sep, '/')
            if filename.startswith(prefix):
                file_path_from_root = filename[len(prefix):]
    # use double separator
    file_path_from_root = file_path_from_root.replace('/', '//')
    return re.sub(r'[^a-zA-Z0-9]', '_', file_path_from_root).upper() + '_'

cpplint.GetHeaderGuardCPPVariable = custom_get_header_guard_cpp_variable

def main(argv=sys.argv[1:]):
    extensions = ['c', 'cc', 'cpp', 'cxx']
    headers = ['h', 'hh', 'hpp', 'hxx']

    parser = argparse.ArgumentParser(
        description='Check code against the Google style conventions using '
                    'cpplint.',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument(
        '--filters', metavar='FILTER,FILTER,...', type=str,
        help='A comma separated list of category filters to apply')
    parser.add_argument(
        '--linelength', metavar='N', type=int, default=100,
        help='The maximum line length')
    parser.add_argument(
        '--root', type=str,
        help='The --root option for cpplint')
    parser.add_argument(
        'paths',
        nargs='*',
        default=[os.curdir],
        help='The files or directories to check. For directories files ending '
             'in %s will be considered.' %
             ', '.join(["'.%s'" % e for e in extensions + headers]))
    # not using a file handle directly
    # in order to prevent leaving an empty file when something fails early
    parser.add_argument(
        '--xunit-file',
        help='Generate a xunit compliant XML file')
    args = parser.parse_args(argv)

    if args.xunit_file:
        start_time = time.time()

    argv = []
    # collect category based counts
    argv.append('--counting=detailed')
    argv.append('--extensions=%s' % ','.join(extensions))
    argv.append('--headers=%s' % ','.join(headers))
    filters = [
        # we do allow C++11
        '-build/c++11',
        # we consider passing non-const references to be ok
        '-runtime/references',
        # we wrap open curly braces for namespaces, classes and functions
        '-whitespace/braces',
        # we don't indent keywords like public, protected and private with one space
        '-whitespace/indent',
        # we allow closing parenthesis to be on the next line
        '-whitespace/parens',
        # we allow the developer to decide about whitespace after a semicolon
        '-whitespace/semicolon',
    ]
    if args.filters:
        filters += args.filters.split(',')
    argv.append('--filter=%s' % ','.join(filters))

    argv.append('--linelength=%d' % args.linelength)

    groups = get_file_groups(args.paths, extensions + headers)
    if not groups:
        print('No files found', file=sys.stderr)
        return 1

    # hook into error reporting
    import ament_cpplint.cpplint
    DefaultError = ament_cpplint.cpplint.Error  # noqa: N806
    report = []

    # invoke cpplint for each root group of files
    _cpplint_state.ResetErrorCounts()
    for root in sorted(groups.keys()):
        files = groups[root]

        arguments = list(argv)
        if args.root:
            root = os.path.abspath(args.root)
        if root:
            root_arg = '--root=%s' % root
            arguments.append(root_arg)
            print("Using '%s' argument" % root_arg)
        else:
            print("Not using '--root'")
        print('')
        arguments += files
        filenames = ParseArguments(arguments)

        for filename in filenames:
            # hook into error reporting
            errors = []

            def custom_error(filename, linenum, category, confidence, message):
                if ament_cpplint.cpplint._ShouldPrintError(category, confidence, linenum):
                    errors.append({
                        'linenum': linenum,
                        'category': category,
                        'confidence': confidence,
                        'message': message,
                    })
                DefaultError(filename, linenum, category, confidence, message)
            ament_cpplint.cpplint.Error = custom_error

            ProcessFile(filename, _cpplint_state.verbose_level)
            report.append((filename, errors))
            print('')

    # output summary
    for category in sorted(_cpplint_state.errors_by_category.keys()):
        count = _cpplint_state.errors_by_category[category]
        print("Category '%s' errors found: %d" % (category, count),
              file=sys.stderr)
    if _cpplint_state.error_count:
        print('Total errors found: %d' % _cpplint_state.error_count,
              file=sys.stderr)
    else:
        print('No errors found')

    # generate xunit file
    if args.xunit_file:
        folder_name = os.path.basename(os.path.dirname(args.xunit_file))
        file_name = os.path.basename(args.xunit_file)
        suffix = '.xml'
        if file_name.endswith(suffix):
            file_name = file_name[0:-len(suffix)]
            suffix = '.xunit'
            if file_name.endswith(suffix):
                file_name = file_name[0:-len(suffix)]
        testname = '%s.%s' % (folder_name, file_name)

        xml = get_xunit_content(report, testname, time.time() - start_time)
        path = os.path.dirname(os.path.abspath(args.xunit_file))
        if not os.path.exists(path):
            os.makedirs(path)
        with open(args.xunit_file, 'w') as f:
            f.write(xml)

    return 1 if _cpplint_state.error_count else 0

def get_file_groups(paths, extensions):
    # dict mapping root path to files
    groups = {}
    for path in paths:
        if os.path.isdir(path):
            for dirpath, dirnames, filenames in os.walk(path):
                if 'AMENT_IGNORE' in filenames:
                    dirnames[:] = []
                    continue
                # ignore folder starting with . or _
                dirnames[:] = [d for d in dirnames if d[0] not in ['.', '_']]
                dirnames.sort()

                # select files by extension
                for filename in sorted(filenames):
                    _, ext = os.path.splitext(filename)
                    if ext in ['.%s' % e for e in extensions]:
                        append_file_to_group(groups,
                                             os.path.join(dirpath, filename))
        if os.path.isfile(path):
            append_file_to_group(groups, path)
    return groups

def append_file_to_group(groups, path):
    path = os.path.abspath(path)

    root = ''

    # try to determine root from path
    base_path = os.path.dirname(path)
    # find longest subpath which ends with one of the following subfolder names
    subfolder_names = ['include', 'src', 'test']
    matches = [
        re.search(
            '^(.+%s%s)%s' %
            (re.escape(os.sep), re.escape(subfolder_name), re.escape(os.sep)), path)
        for subfolder_name in subfolder_names]
    match_groups = [match.group(1) for match in matches if match]
    if match_groups:
        match_groups = [{'group_len': len(x), 'group': x} for x in match_groups]
        sorted_groups = sorted(match_groups, key=lambda k: k['group_len'])
        base_path = sorted_groups[-1]['group']
        root = base_path

    # try to find repository root
    repo_root = None
    p = path
    while p and repo_root is None:
        # abort if root is reached
        if os.path.dirname(p) == p:
            break
        p = os.path.dirname(p)
        for marker in ['.git', '.hg', '.svn']:
            if os.path.exists(os.path.join(p, marker)):
                repo_root = p
                break

    # compute relative --root argument
    if repo_root and repo_root > base_path:
        root = os.path.relpath(base_path, repo_root)

    # add the path to the appropriate group
    if root not in groups:
        groups[root] = []
    groups[root].append(path)

def get_xunit_content(report, testname, elapsed):
    test_count = sum(max(len(r[1]), 1) for r in report)
    error_count = sum(len(r[1]) for r in report)
    data = {
        'testname': testname,
        'test_count': test_count,
        'error_count': error_count,
        'time': '%.3f' % round(elapsed, 3),
    }
    xml = """<?xml version="1.0" encoding="UTF-8"?>
<testsuite
  name="%(testname)s"
  tests="%(test_count)d"
  failures="%(error_count)d"
  time="%(time)s"
>
""" % data

    for (filename, errors) in report:

        if errors:
            # report each cpplint error as a failing testcase
            for error in errors:
                data = {
                    'quoted_name': quoteattr(
                        '%s [%s] (%s:%d)' % (
                            error['category'], error['confidence'],
                            filename, error['linenum'])),
                    'testname': testname,
                    'quoted_message': quoteattr(error['message']),
                }
                xml += """  <testcase
    name=%(quoted_name)s
    classname="%(testname)s"
  >
      <failure message=%(quoted_message)s/>
  </testcase>
""" % data

        else:
            # if there are no cpplint errors report a single successful test
            data = {
                'quoted_location': quoteattr(filename),
                'testname': testname,
            }
            xml += """  <testcase
    name=%(quoted_location)s
    classname="%(testname)s"
    status="No errors"/>
""" % data

    # output list of checked files
    data = {
        'escaped_files': escape(''.join(['\n* %s' % r[0] for r in report])),
    }
    xml += """  <system-out>Checked files:%(escaped_files)s</system-out>
""" % data

    xml += '</testsuite>\n'
    return xml

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>"""
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
"""

from rest_framework import serializers

from aquifers import models

class AquiferResourceSerializer(serializers.ModelSerializer):
    """Serialize aquifer resourcelist"""
    section_code = serializers.PrimaryKeyRelatedField(
        queryset=models.AquiferResourceSection.objects.all(),
        source='section.code')

    def to_internal_value(self, data):
        """
        Dict of native values <- Dict of primitive datatypes.
        Add instance key to values if `id` present in primitive dict
        :param data:
        """
        obj = super(AquiferResourceSerializer, self).to_internal_value(data)
        instance_id = data.get('id', None)
        if instance_id:
            obj['instance'] = models.AquiferResource.objects.get(id=instance_id)
        return obj

    class Meta:
        model = models.AquiferResource
        fields = (
            'id',
            'name',
            'url',
            'section_code'
        )

class AquiferSerializer(serializers.ModelSerializer):
    """Serialize a aquifer list"""
    demand_description = serializers.SlugRelatedField(
        source='demand', read_only=True, slug_field='description')
    material_description = serializers.SlugRelatedField(
        source='material', read_only=True, slug_field='description')
    productivity_description = serializers.SlugRelatedField(
        source='productivity', read_only=True, slug_field='description')
    subtype_description = serializers.StringRelatedField(
        source='subtype', read_only=True)
    vulnerability_description = serializers.SlugRelatedField(
        source='vulnerability', read_only=True, slug_field='description')
    quality_concern_description = serializers.SlugRelatedField(
        source='quality_concern', read_only=True, slug_field='description')
    known_water_use_description = serializers.SlugRelatedField(
        source='known_water_use', read_only=True, slug_field='description')
    resources = AquiferResourceSerializer(many=True)

    def create(self, validated_data):
        """
        Allow creating resources inline of the aquifer API. ie)

        {
            resources: [{
                url: 'http://...',
                name: 'A resource',
                section_id: 1
            }, {
                ...
            }, ...]
            ...
        }
        """
        resources_data = validated_data.pop('resources')
        aquifer = models.Aquifer.objects.create(**validated_data)
        for resource_item in resources_data:
            models.AquiferResource.objects.create(aquifer=aquifer, **resource_item)
        return aquifer

    def update(self, instance, validated_data):
        """
        Update the resources associated with an aquifer, inline of the aquifer API.
        """
        resources_data = validated_data.pop('resources')
        for k,v in validated_data.items():
            setattr(instance, k, v)
        instance.save()

        # Any items removed from the inline collection are deleted.
        to_delete = models.AquiferResource.objects.filter(
            aquifer=instance
        ).exclude(
            id__in=[r['id'] for r in resources_data if 'id' in r]
        )

        to_delete.delete()

        for resource_item in resources_data:
            if 'instance' in resource_item:
                resource = resource_item['instance']
                resource.section = resource_item['section']['code']
                resource.name = resource_item['name']
                resource.url = resource_item['url']
                resource.save()
            else:
                r = models.AquiferResource(
                    url=resource_item['url'],
                    name=resource_item['name'],
                    aquifer=instance,
                    section_id=resource_item['section']['code'].code)
                r.save()

        return instance

    class Meta:
        model = models.Aquifer
        fields = (
            'aquifer_id',
            'aquifer_name',
            'area',
            'demand_description',
            'demand',
            'known_water_use_description',
            'known_water_use',
            'litho_stratographic_unit',
            'location_description',
            'mapping_year',
            'material_description',
            'material',
            'notes',
            'productivity_description',
            'productivity',
            'quality_concern_description',
            'quality_concern',
            'subtype_description',
            'subtype',
            'vulnerability_description',
            'vulnerability',
            'resources',
        )

class AquiferResourceSectionSerializer(serializers.ModelSerializer):
    """Serialize aquifer section list"""
    class Meta:
        model = models.AquiferResourceSection
        fields = (
            'code',
            'name'
        )

class AquiferMaterialSerializer(serializers.ModelSerializer):
    class Meta:
        model = models.AquiferMaterial
        fields = (
            'code',
            'description'
        )

class QualityConcernSerializer(serializers.ModelSerializer):
    class Meta:
        model = models.QualityConcern
        fields = (
            'code',
            'description'
        )

class AquiferVulnerabilitySerializer(serializers.ModelSerializer):
    class Meta:
        model = models.AquiferVulnerabilityCode
        fields = (
            'code',
            'description'
        )

class AquiferSubtypeSerializer(serializers.ModelSerializer):
    description = serializers.CharField(source='__str__')

    class Meta:
        model = models.AquiferSubtype
        fields = (
            'code',
            'description'
        )

class AquiferProductivitySerializer(serializers.ModelSerializer):
    class Meta:
        model = models.AquiferProductivity
        fields = (
            'code',
            'description'
        )

class AquiferDemandSerializer(serializers.ModelSerializer):
    class Meta:
        model = models.AquiferDemand
        fields = (
            'code',
            'description'
        )

class WaterUseSerializer(serializers.ModelSerializer):
    class Meta:
        model = models.WaterUse
        fields = (
            'code',
            'description'
        )

class AquiferSerializerBasic(serializers.ModelSerializer):
    """Serialize a aquifer list with a simplified format"""

    description = serializers.SerializerMethodField()

    class Meta:
        model = models.Aquifer
        fields = (
            'aquifer_id',
            'description',
        )

    def get_description(self, obj):
        desc = str(obj.aquifer_id)

        # if aquifers have a name (not all do), append it to the aquifer number
        name = obj.aquifer_name
        if name:
            desc += ' - ' + name

        return desc
<commit_msg>fix create flow<commit_after>"""
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
"""

from rest_framework import serializers

from aquifers import models

class AquiferResourceSerializer(serializers.ModelSerializer):
    """Serialize aquifer resourcelist"""
    section_code = serializers.PrimaryKeyRelatedField(
        queryset=models.AquiferResourceSection.objects.all(),
        source='section.code')

    def to_internal_value(self, data):
        """
        Dict of native values <- Dict of primitive datatypes.
        Add instance key to values if `id` present in primitive dict
        :param data:
        """
        obj = super(AquiferResourceSerializer, self).to_internal_value(data)
        instance_id = data.get('id', None)
        if instance_id:
            obj['instance'] = models.AquiferResource.objects.get(id=instance_id)
        return obj

    class Meta:
        model = models.AquiferResource
        fields = (
            'id',
            'name',
            'url',
            'section_code'
        )

class AquiferSerializer(serializers.ModelSerializer):
    """Serialize a aquifer list"""
    demand_description = serializers.SlugRelatedField(
        source='demand', read_only=True, slug_field='description')
    material_description = serializers.SlugRelatedField(
        source='material', read_only=True, slug_field='description')
    productivity_description = serializers.SlugRelatedField(
        source='productivity', read_only=True, slug_field='description')
    subtype_description = serializers.StringRelatedField(
        source='subtype', read_only=True)
    vulnerability_description = serializers.SlugRelatedField(
        source='vulnerability', read_only=True, slug_field='description')
    quality_concern_description = serializers.SlugRelatedField(
        source='quality_concern', read_only=True, slug_field='description')
    known_water_use_description = serializers.SlugRelatedField(
        source='known_water_use', read_only=True, slug_field='description')
    resources = AquiferResourceSerializer(many=True, required=False)

    def create(self, validated_data):
        """
        Allow creating resources inline of the aquifer API. ie)

        {
            resources: [{
                url: 'http://...',
                name: 'A resource',
                section_id: 1
            }, {
                ...
            }, ...]
            ...
        }
        """
        resources_data = validated_data.pop('resources')
        aquifer = models.Aquifer.objects.create(**validated_data)
        for resource_item in resources_data:
            r = models.AquiferResource(
                url=resource_item['url'],
                name=resource_item['name'],
                aquifer=instance,
                section_id=resource_item['section']['code'].code)
            r.save()
        return aquifer

    def update(self, instance, validated_data):
        """
        Update the resources associated with an aquifer, inline of the aquifer API.
        """
        resources_data = validated_data.pop('resources')
        for k,v in validated_data.items():
            setattr(instance, k, v)
        instance.save()

        # Any items removed from the inline collection are deleted.
        to_delete = models.AquiferResource.objects.filter(
            aquifer=instance
        ).exclude(
            id__in=[r['id'] for r in resources_data if 'id' in r]
        )

        to_delete.delete()

        for resource_item in resources_data:
            if 'instance' in resource_item:
                resource = resource_item['instance']
                resource.section = resource_item['section']['code']
                resource.name = resource_item['name']
                resource.url = resource_item['url']
                resource.save()
            else:
                r = models.AquiferResource(
                    url=resource_item['url'],
                    name=resource_item['name'],
                    aquifer=instance,
                    section_id=resource_item['section']['code'].code)
                r.save()

        return instance

    class Meta:
        model = models.Aquifer
        fields = (
            'aquifer_id',
            'aquifer_name',
            'area',
            'demand_description',
            'demand',
            'known_water_use_description',
            'known_water_use',
            'litho_stratographic_unit',
            'location_description',
            'mapping_year',
            'material_description',
            'material',
            'notes',
            'productivity_description',
            'productivity',
            'quality_concern_description',
            'quality_concern',
            'subtype_description',
            'subtype',
            'vulnerability_description',
            'vulnerability',
            'resources',
        )

class AquiferResourceSectionSerializer(serializers.ModelSerializer):
    """Serialize aquifer section list"""
    class Meta:
        model = models.AquiferResourceSection
        fields = (
            'code',
            'name'
        )

class AquiferMaterialSerializer(serializers.ModelSerializer):
    class Meta:
        model = models.AquiferMaterial
        fields = (
            'code',
            'description'
        )

class QualityConcernSerializer(serializers.ModelSerializer):
    class Meta:
        model = models.QualityConcern
        fields = (
            'code',
            'description'
        )

class AquiferVulnerabilitySerializer(serializers.ModelSerializer):
    class Meta:
        model = models.AquiferVulnerabilityCode
        fields = (
            'code',
            'description'
        )

class AquiferSubtypeSerializer(serializers.ModelSerializer):
    description = serializers.CharField(source='__str__')

    class Meta:
        model = models.AquiferSubtype
        fields = (
            'code',
            'description'
        )

class AquiferProductivitySerializer(serializers.ModelSerializer):
    class Meta:
        model = models.AquiferProductivity
        fields = (
            'code',
            'description'
        )

class AquiferDemandSerializer(serializers.ModelSerializer):
    class Meta:
        model = models.AquiferDemand
        fields = (
            'code',
            'description'
        )

class WaterUseSerializer(serializers.ModelSerializer):
    class Meta:
        model = models.WaterUse
        fields = (
            'code',
            'description'
        )

class AquiferSerializerBasic(serializers.ModelSerializer):
    """Serialize a aquifer list with a simplified format"""

    description = serializers.SerializerMethodField()

    class Meta:
        model = models.Aquifer
        fields = (
            'aquifer_id',
            'description',
        )

    def get_description(self, obj):
        desc = str(obj.aquifer_id)

        # if aquifers have a name (not all do), append it to the aquifer number
        name = obj.aquifer_name
        if name:
            desc += ' - ' + name

        return desc
<|endoftext|>


<commit_before>#!/usr/bin/python
import sqlite3, json, os
import logging, sys
from collections import defaultdict

#
# This script moves candidate information from filename.json into the sqlite3 database
#
# !!! UPDATE HERE WHENEVER THE DATABASE TABLE SCHEMA CHANGE !!!
#
# The order matter when we want to insert the value, current schema:
# CREATE TABLE candidates (firstName TEXT, lastName TEXT, prefix TEXT, suffix TEXT, party TEXT, chamber TEXT, state TEXT, district INTEGER, incumbent INTEGER, source TEXT, bioguideId TEXT PRIMARY KEY UNIQUE, fecId TEXT UNIQUE, website TEXT UNIQUE, email TEXT UNIQUE, facebook TEXT UNIQUE, twitter TEXT UNIQUE, youtube TEXT UNIQUE, img_src TEXT, questionnaire_response TEXT, gen_election_candidate INTEGER DEFAULT (0), duplicate INTEGER, candidate_url text);

logging.basicConfig(stream=sys.stderr,level=logging.DEBUG)

def getStateAbbr(s):
  try:
    return {
      'Alabama' : 'AL',
      'Montana' : 'MT',
      'Alaska' : 'AK',
      'Nebraska' : 'NE',
      'Arizona' : 'AZ' ,
      'Nevada' : 'NV',
      'Arkansas' : 'AR' ,
      'New Hampshire' : 'NH',
      'California' : 'CA' ,
      'New Jersey' : 'NJ',
      'Colorado' : 'CO' ,
      'New Mexico' : 'NM',
      'Connecticut' : 'CT',
      'New York' : 'NY',
      'Delaware' : 'DE' ,
      'North Carolina' : 'NC',
      'Florida' : 'FL' ,
      'North Dakota' : 'ND',
      'Georgia' : 'GA' ,
      'Ohio' : 'OH',
      'Hawaii' : 'HI' ,
      'Oklahoma' : 'OK',
      'Idaho' : 'ID' ,
      'Oregon' : 'OR',
      'Illinois' : 'IL' ,
      'Pennsylvania' : 'PA',
      'Indiana' : 'IN' ,
      'Rhode Island' : 'RI',
      'Iowa' : 'IA' ,
      'South Carolina' : 'SC',
      'Kansas' : 'KS' ,
      'South Dakota' : 'SD',
      'Kentucky' : 'KY' ,
      'Tennessee' : 'TN',
      'Louisiana' : 'LA' ,
      'Texas' : 'TX',
      'Maine' : 'ME' ,
      'Utah' : 'UT',
      'Maryland' : 'MD' ,
      'Vermont' : 'VT',
      'Massachusetts' : 'MA' ,
      'Virginia' : 'VA',
      'Michigan' : 'MI' ,
      'Washington' : 'WA',
      'Minnesota' : 'MN' ,
      'West Virginia' : 'WV',
      'Mississippi' : 'MS' ,
      'Wisconsin' : 'WI',
      'Missouri' : 'MO' ,
      'Wyoming' : 'WY',
    }[s]
  except:
    print 'key ' + s + ' not found!'
    return None

dbpath = '../../db/db.sqlite3'
if not (dbpath and os.path.isfile(dbpath)):
  print 'db file not found'
  exit()

try:
  db = sqlite3.connect(dbpath)
  c = db.cursor()
except sqlite3.Error:
  print 'sqlite3 error'
  db.close()

#jsonpath = '/root/CongressionalGuide/app/candidates/import.json'
jsonpath = str(sys.argv[1])
if not (jsonpath and os.path.isfile(jsonpath)):
  print 'candidates json file not found'
  exit()

congressman = json.load(open(jsonpath))

# check first/last name pair
# if exists, update_query
# else insert_query

update_query = 'UPDATE candidates SET candidate_url = ?, img_src = ?, facebook = ?, twitter = ?, website = ?, youtube = ?, source = ?, gen_election_candidate = ?, incumbent = ?, district = ? where firstName like ? and lastName like ? and state = ?'
#update_query = 'UPDATE candidates SET candidate_url = ?, img_src = ?, facebook = ?, twitter = ?, website = ?, youtube = ?, source = ?, gen_election_candidate = ?, incumbent = ? where firstName like ? and lastName like ? and state = ? and district = ?'

# !!! UPDATE HERE WHENEVER THE DATABASE TABLE SCHEMA CHANGE !!!
insert_query = 'INSERT INTO candidates VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)'

for human in congressman:
  firstName=(None,)
  lastName=(None,)
  prefix=(None,)
  suffix=(None,)
  party=(None,)
  chamber=(None,)
  state=(None,)
  district=(0,)
  incumbent=(None,)
  bioguideId=(None,)
  fecId=(None,)
  source=('ballotpedia',)
  website=(None,)
  email=(None,)
  facebook=(None,)
  twitter=(None,)
  youtube=(None,)
  img_src=(None,)
  questionnaire_response=(None,)
  #TODO: NH primary election 9/13, their candidate will have null value here
  gen_election_candidate=(None,)
  candidate_url=(None,)
  duplicate=(None,)

  mesg=''
  for k,v in human.iteritems():
    mesg += '(k,v)=(' + k + ' ,' + str(v) + ')\n'
    if k == 'name':
      lp = v.find('(') #handle name like 'Bill Otto (Missouri)'
      v = v[:lp-1]
      firstName = v.split()[0],
      lastName = v.split()[-1],
    elif k == 'party':
      party = v[0],
    elif k == 'dist':
      dl = [int(d) for d in v if d.isdigit()]
      if len(dl) != 0:
        district = int(''.join(map(str, dl))),
    elif k == 'camp':
      website = v,
    elif k == 'twtr':
      tv = v[v.find('twitter.com')+len('twitter.com')+1:]
      twitter = tv[:tv.find('/')].replace('@',''),
    elif k == 'fb':
      facebook = v,
    elif k == 'state':
      state = getStateAbbr(v),
    elif k == 'pic':
      img_src = v,
    elif k == 'chamber':
      chamber = v,
    elif k == 'youtube':
      youtube = v,
    elif k == 'incumbent':
      incumbent = v,
    elif k == 'gen_election_candidate':
      gen_election_candidate = v,
    elif k == 'url':
      candidate_url = v,

  logging.debug(mesg)
  match_firstName = '%'+firstName[0]+'%',
  match_lastName = '%'+lastName[0]+'%',

# !!! UPDATE HERE WHENEVER THE DATABASE TABLE SCHEMA CHANGE !!!
  insert_values = (firstName + lastName + prefix + suffix + party + chamber + state + district + incumbent + source + bioguideId + fecId + website + email + facebook + twitter + youtube + img_src + questionnaire_response + gen_election_candidate + duplicate + candidate_url)
  update_values = (candidate_url + img_src + facebook + twitter + website + youtube + source + gen_election_candidate + incumbent + district + match_firstName + match_lastName + state)
  #update_values = (candidate_url + img_src + facebook + twitter + website + youtube + source + gen_election_candidate + incumbent + match_firstName + match_lastName + state + district)

  # Match with existing Sunlight data: lastName, first word of firstName, state and district
  # no district for senate
  c.execute('SELECT count(*) FROM candidates where firstName like ? and lastName like ? and state = ? ;', match_firstName + match_lastName + state )
  #c.execute('SELECT count(*) FROM candidates where firstName like ? and lastName like ? and state = ? and district = ?;', match_firstName + match_lastName + state + district)
  obj = c.fetchone()
  if obj[0]:
    logging.info('update_values: %s', update_values)
    c.execute(update_query, update_values)
  else:
    logging.info('insert_values: %s', insert_values)
    c.execute(insert_query, insert_values)
  logging.info('[OK]\n\n')

db.commit()
db.close()

<commit_msg>Add unique requirement for candidate_url, remove unique requirement for website<commit_after>#!/usr/bin/python
import sqlite3, json, os
import logging, sys
from collections import defaultdict

#
# This script moves candidate information from filename.json into the sqlite3 database
#
# !!! UPDATE HERE WHENEVER THE DATABASE TABLE SCHEMA CHANGE !!!
#
# The order matter when we want to insert the value, current schema:
# CREATE TABLE candidates (firstName TEXT, lastName TEXT, prefix TEXT, suffix TEXT, party TEXT, chamber TEXT, state TEXT, district INTEGER, incumbent INTEGER, source TEXT, bioguideId TEXT PRIMARY KEY UNIQUE, fecId TEXT UNIQUE, website TEXT, email TEXT UNIQUE, facebook TEXT UNIQUE, twitter TEXT UNIQUE, youtube TEXT UNIQUE, img_src TEXT, questionnaire_response TEXT, gen_election_candidate INTEGER DEFAULT (0), duplicate INTEGER, candidate_url TEXT UNIQUE);

logging.basicConfig(stream=sys.stderr,level=logging.DEBUG)

def getStateAbbr(s):
  try:
    return {
      'Alabama' : 'AL',
      'Montana' : 'MT',
      'Alaska' : 'AK',
      'Nebraska' : 'NE',
      'Arizona' : 'AZ' ,
      'Nevada' : 'NV',
      'Arkansas' : 'AR' ,
      'New Hampshire' : 'NH',
      'California' : 'CA' ,
      'New Jersey' : 'NJ',
      'Colorado' : 'CO' ,
      'New Mexico' : 'NM',
      'Connecticut' : 'CT',
      'New York' : 'NY',
      'Delaware' : 'DE' ,
      'North Carolina' : 'NC',
      'Florida' : 'FL' ,
      'North Dakota' : 'ND',
      'Georgia' : 'GA' ,
      'Ohio' : 'OH',
      'Hawaii' : 'HI' ,
      'Oklahoma' : 'OK',
      'Idaho' : 'ID' ,
      'Oregon' : 'OR',
      'Illinois' : 'IL' ,
      'Pennsylvania' : 'PA',
      'Indiana' : 'IN' ,
      'Rhode Island' : 'RI',
      'Iowa' : 'IA' ,
      'South Carolina' : 'SC',
      'Kansas' : 'KS' ,
      'South Dakota' : 'SD',
      'Kentucky' : 'KY' ,
      'Tennessee' : 'TN',
      'Louisiana' : 'LA' ,
      'Texas' : 'TX',
      'Maine' : 'ME' ,
      'Utah' : 'UT',
      'Maryland' : 'MD' ,
      'Vermont' : 'VT',
      'Massachusetts' : 'MA' ,
      'Virginia' : 'VA',
      'Michigan' : 'MI' ,
      'Washington' : 'WA',
      'Minnesota' : 'MN' ,
      'West Virginia' : 'WV',
      'Mississippi' : 'MS' ,
      'Wisconsin' : 'WI',
      'Missouri' : 'MO' ,
      'Wyoming' : 'WY',
    }[s]
  except:
    print 'key ' + s + ' not found!'
    return None

dbpath = '../../db/db.sqlite3'
if not (dbpath and os.path.isfile(dbpath)):
  print 'db file not found'
  exit()

try:
  db = sqlite3.connect(dbpath)
  c = db.cursor()
except sqlite3.Error:
  print 'sqlite3 error'
  db.close()

#jsonpath = '/root/CongressionalGuide/app/candidates/import.json'
jsonpath = str(sys.argv[1])
if not (jsonpath and os.path.isfile(jsonpath)):
  print 'candidates json file not found'
  exit()

congressman = json.load(open(jsonpath))

# check first/last name pair
# if exists, update_query
# else insert_query

update_query = 'UPDATE candidates SET candidate_url = ?, img_src = ?, facebook = ?, twitter = ?, website = ?, youtube = ?, source = ?, gen_election_candidate = ?, incumbent = ?, district = ? where firstName like ? and lastName like ? and state = ?'
#update_query = 'UPDATE candidates SET candidate_url = ?, img_src = ?, facebook = ?, twitter = ?, website = ?, youtube = ?, source = ?, gen_election_candidate = ?, incumbent = ? where firstName like ? and lastName like ? and state = ? and district = ?'

# !!! UPDATE HERE WHENEVER THE DATABASE TABLE SCHEMA CHANGE !!!
insert_query = 'INSERT INTO candidates VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)'

for human in congressman:
  firstName=(None,)
  lastName=(None,)
  prefix=(None,)
  suffix=(None,)
  party=(None,)
  chamber=(None,)
  state=(None,)
  district=(0,)
  incumbent=(None,)
  bioguideId=(None,)
  fecId=(None,)
  source=('ballotpedia',)
  website=(None,)
  email=(None,)
  facebook=(None,)
  twitter=(None,)
  youtube=(None,)
  img_src=(None,)
  questionnaire_response=(None,)
  #TODO: NH primary election 9/13, their candidate will have null value here
  gen_election_candidate=(None,)
  candidate_url=(None,)
  duplicate=(None,)

  mesg=''
  for k,v in human.iteritems():
    mesg += '(k,v)=(' + k + ' ,' + str(v) + ')\n'
    if k == 'name':
      lp = v.find('(') #handle name like 'Bill Otto (Missouri)'
      v = v[:lp-1]
      firstName = v.split()[0],
      lastName = v.split()[-1],
    elif k == 'party':
      party = v[0],
    elif k == 'dist':
      dl = [int(d) for d in v if d.isdigit()]
      if len(dl) != 0:
        district = int(''.join(map(str, dl))),
    elif k == 'camp':
      website = v,
    elif k == 'twtr':
      tv = v[v.find('twitter.com')+len('twitter.com')+1:]
      twitter = tv[:tv.find('/')].replace('@',''),
    elif k == 'fb':
      facebook = v,
    elif k == 'state':
      state = getStateAbbr(v),
    elif k == 'pic':
      img_src = v,
    elif k == 'chamber':
      chamber = v,
    elif k == 'youtube':
      youtube = v,
    elif k == 'incumbent':
      incumbent = v,
    elif k == 'gen_election_candidate':
      gen_election_candidate = v,
    elif k == 'url':
      candidate_url = v,

  logging.debug(mesg)
  match_firstName = '%'+firstName[0]+'%',
  match_lastName = '%'+lastName[0]+'%',

# !!! UPDATE HERE WHENEVER THE DATABASE TABLE SCHEMA CHANGE !!!
  insert_values = (firstName + lastName + prefix + suffix + party + chamber + state + district + incumbent + source + bioguideId + fecId + website + email + facebook + twitter + youtube + img_src + questionnaire_response + gen_election_candidate + duplicate + candidate_url)
  update_values = (candidate_url + img_src + facebook + twitter + website + youtube + source + gen_election_candidate + incumbent + district + match_firstName + match_lastName + state)
  #update_values = (candidate_url + img_src + facebook + twitter + website + youtube + source + gen_election_candidate + incumbent + match_firstName + match_lastName + state + district)

  # Match with existing Sunlight data: lastName, first word of firstName, state and district
  # no district for senate
  c.execute('SELECT count(*) FROM candidates where firstName like ? and lastName like ? and state = ? ;', match_firstName + match_lastName + state )
  #c.execute('SELECT count(*) FROM candidates where firstName like ? and lastName like ? and state = ? and district = ?;', match_firstName + match_lastName + state + district)
  obj = c.fetchone()
  if obj[0]:
    logging.info('update_values: %s', update_values)
    c.execute(update_query, update_values)
  else:
    logging.info('insert_values: %s', insert_values)
    c.execute(insert_query, insert_values)
  logging.info('[OK]\n\n')

db.commit()
db.close()

<|endoftext|>


<commit_before>"""Classes for retrieving Marker sequences

Example:
    $ python sequences.py -s stx1

"""
import os
from rdflib import Graph
from middleware.decorators import submit, prefix, tojson
from middleware.graphers import turtle_utils
from routes.job_utils import fetch_job

__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))

@submit
@prefix
def marker_query(marker_uris):

  query = '''
  SELECT ?m
  WHERE {{
    ?m rdf:type :Marker .
    VALUES ?m {{ {} }}
  }}
  '''.format(' '.join(marker_uris))

  return query

@prefix
def sequence_query(marker_rdf, isolate_rdf):
    # query = '''
    #         SELECT ?contig ?contigid ?region ?start ?len ?seq
    #         WHERE {{
    #             ?g a :spfyId .
    #             VALUES ?g {{ {} }}
    #             ?contig a g:Contig ;
    #                 g:Identifier ?contigid .
    #             ?m a :VirulenceFactor .
    #             ?contig g:DNASequence ?dna .
    #             VALUES ?m {{ {} }} .
    #             ?region a faldo:Region ;
    #                 :hasPart ?m ;
    #                 :isFoundIn ?contig ;
    #                 :isFoundIn ?g ;
    #                 faldo:begin ?b ;
    #                 faldo:end ?e .
    #             ?b faldo:position ?beginPos .
    #             ?e faldo:position ?endPos .
    #             BIND( IF(?beginPos < ?endPos,?beginPos,?endPos) as ?start)
    #             BIND( IF(?beginPos < ?endPos,?endPos-?beginPos+1,?beginPos-?endPos+1) as ?len)
    #             BIND( SUBSTR( ?dna, ?start, ?len ) as ?seq )
    #         }}
    #     '''.format(isolate_rdf, ' '.join(marker_rdf))

    query = '''
            SELECT ?contig ?contigid ?region ?start ?len ?seq
            WHERE {{
                ?g a :spfyId .
                VALUES ?g {{ {} }}
                ?contig a g:Contig ;
                    g:Identifier ?contigid .
                ?m a :VirulenceFactor .
                ?contig g:DNASequence ?dna .
                VALUES ?m {{ {} }} .
                ?region a faldo:Region ;
                    :hasPart ?m ;
                    faldo:begin ?b ;
                    faldo:end ?e .
                ?b faldo:position ?beginPos .
                ?e faldo:position ?endPos .
                ?region :isFoundIn ?contig .
                ?contig :isFoundIn ?bagOfContigs .
                ?bagOfContigs :isFoundIn ?genomeHash .
                ?genomeHash :isFoundIn ?g .
                BIND( IF(?beginPos < ?endPos,?beginPos,?endPos) as ?start)
                BIND( IF(?beginPos < ?endPos,?endPos-?beginPos+1,?beginPos-?endPos+1) as ?len)
                BIND( SUBSTR( ?dna, ?start, ?len ) as ?seq )
            }}
        '''.format(isolate_rdf, ' '.join(marker_rdf))

    return query

@tojson
@submit
@prefix
def phylotyper_query(subtypescheme_rdf, isolate_rdf):

    query = '''

        SELECT DISTINCT ?pt ?typeLabel ?score ?region ?contigid ?beginPos ?endPos
        WHERE {{
            ?pt a subt:PTST ;
                subt:isOfPhylotyper {} ;
                subt:hasIdentifiedClass ?type ;
                subt:score ?score ;
                typon:hasIdentifiedAllele ?a .
            ?type subt:subtypeValue ?typeLabel .
            ?a faldo:location ?region .
            ?region :isFoundIn {} ;
                :isFoundIn ?contig ;
                faldo:begin ?b ;
                faldo:end ?e .
            ?contig a g:Contig ;
                g:Identifier ?contigid .
            ?b faldo:position ?beginPos .
            ?e faldo:position ?endPos
        }}
    '''.format(subtypescheme_rdf, isolate_rdf)

    return query

@tojson
@submit
@prefix
def genename_query(locus_rdf):

    query = '''
        SELECT ?markerURI ?markerLabel
        WHERE {{
            VALUES ?l {{ {} }} .
            ?markerURI :isFoundIn ?l ;
                a :Marker ;
                dc:description ?markerLabel
        }}
    '''.format(locus_rdf)

    return query

class MarkerSequences(object):
    """Retrieve DNA region sequences for one or more Markers

    """

    def __init__(self, markers=[':stx2A',':stx2B'], job_id=None, job_turtle=None, job_ectyper_datastruct_vf=None, redis_conn=None):
        """Constructor

        Args:

        """

        # convert to proper RDF terms
        self.marker_uris = [turtle_utils.normalize_rdfterm(m) for m in markers]
        # Read the phylotyper ontology as a starter graph.
        g = Graph()
        ontology_turtle_file = os.path.join(__location__, 'superphy_subtyping.ttl')
        g.parse(ontology_turtle_file, format="turtle")
        # Retrieve and merge graphs from pre-req. jobs.
        self.graph = g + fetch_job(job_id, redis_conn).result + fetch_job(job_turtle, redis_conn).result + fetch_job(job_ectyper_datastruct_vf, redis_conn).result

    def sequences(self, genome_uri):
        """Retrieve sequences for object alleles

          Args:
            genome_uri(str): Genome URI

          Returns:
            dictionary

        """
        #raise Exception('graph: {0}'.format(self.graph.serialize(format="turtle")))
        genome_rdf = turtle_utils.normalize_rdfterm(genome_uri)
        query = sequence_query(self.marker_uris, genome_rdf)
        # query_result = sequence_query(self.marker_uris, genome_rdf)
        query_result = self.graph.query(query)
        query_result = [row for row in query_result]
        raise Exception('sequences() query_result: {0}'.format(query_result))

        # Unroll result into dictionary with fasta-like keys
        seqdict = { "spfy|{}| {}:{}..{}".format(
            turtle_utils.fulluri_to_basename(r['region']),
            r['contigid'], r['start'], r['start']+r['len']-1): r['seq'] for r in query_result }

        return seqdict

    def fasta(self, genome_uri):
        """Retrieve sequences for object alleles

          Args:
            genome_uri(str): Genome URI

          Returns:
             fasta format string or None

        """

        seqdict = self.sequences(genome_uri)

        if not seqdict:
            return None

        fasta_string = ''
        for (h,s) in seqdict.iteritems():
            fasta_string += ">{}\n{}\n".format(h,s)

        return fasta_string

if __name__=='__main__':
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-m",
        nargs="*",
        help="Marker URI",
        required=True
    )
    parser.add_argument(
        "-g",
        help="Genome URI",
        required=True
    )
    args = parser.parse_args()

    ms = MarkerSequences(args.m)
    print ms.sequences(args.g)
<commit_msg>ADD: convert rdflibs query() result to our format<commit_after>"""Classes for retrieving Marker sequences

Example:
    $ python sequences.py -s stx1

"""
import os
from rdflib import Graph
from middleware.decorators import submit, prefix, tojson
from middleware.graphers import turtle_utils
from routes.job_utils import fetch_job

__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))

@submit
@prefix
def marker_query(marker_uris):

  query = '''
  SELECT ?m
  WHERE {{
    ?m rdf:type :Marker .
    VALUES ?m {{ {} }}
  }}
  '''.format(' '.join(marker_uris))

  return query

@prefix
def sequence_query(marker_rdf, isolate_rdf):
    # query = '''
    #         SELECT ?contig ?contigid ?region ?start ?len ?seq
    #         WHERE {{
    #             ?g a :spfyId .
    #             VALUES ?g {{ {} }}
    #             ?contig a g:Contig ;
    #                 g:Identifier ?contigid .
    #             ?m a :VirulenceFactor .
    #             ?contig g:DNASequence ?dna .
    #             VALUES ?m {{ {} }} .
    #             ?region a faldo:Region ;
    #                 :hasPart ?m ;
    #                 :isFoundIn ?contig ;
    #                 :isFoundIn ?g ;
    #                 faldo:begin ?b ;
    #                 faldo:end ?e .
    #             ?b faldo:position ?beginPos .
    #             ?e faldo:position ?endPos .
    #             BIND( IF(?beginPos < ?endPos,?beginPos,?endPos) as ?start)
    #             BIND( IF(?beginPos < ?endPos,?endPos-?beginPos+1,?beginPos-?endPos+1) as ?len)
    #             BIND( SUBSTR( ?dna, ?start, ?len ) as ?seq )
    #         }}
    #     '''.format(isolate_rdf, ' '.join(marker_rdf))

    query = '''
            SELECT ?contig ?contigid ?region ?start ?len ?seq
            WHERE {{
                ?g a :spfyId .
                VALUES ?g {{ {} }}
                ?contig a g:Contig ;
                    g:Identifier ?contigid .
                ?m a :VirulenceFactor .
                ?contig g:DNASequence ?dna .
                VALUES ?m {{ {} }} .
                ?region a faldo:Region ;
                    :hasPart ?m ;
                    faldo:begin ?b ;
                    faldo:end ?e .
                ?b faldo:position ?beginPos .
                ?e faldo:position ?endPos .
                ?region :isFoundIn ?contig .
                ?contig :isFoundIn ?bagOfContigs .
                ?bagOfContigs :isFoundIn ?genomeHash .
                ?genomeHash :isFoundIn ?g .
                BIND( IF(?beginPos < ?endPos,?beginPos,?endPos) as ?start)
                BIND( IF(?beginPos < ?endPos,?endPos-?beginPos+1,?beginPos-?endPos+1) as ?len)
                BIND( SUBSTR( ?dna, ?start, ?len ) as ?seq )
            }}
        '''.format(isolate_rdf, ' '.join(marker_rdf))

    return query

@tojson
@submit
@prefix
def phylotyper_query(subtypescheme_rdf, isolate_rdf):

    query = '''

        SELECT DISTINCT ?pt ?typeLabel ?score ?region ?contigid ?beginPos ?endPos
        WHERE {{
            ?pt a subt:PTST ;
                subt:isOfPhylotyper {} ;
                subt:hasIdentifiedClass ?type ;
                subt:score ?score ;
                typon:hasIdentifiedAllele ?a .
            ?type subt:subtypeValue ?typeLabel .
            ?a faldo:location ?region .
            ?region :isFoundIn {} ;
                :isFoundIn ?contig ;
                faldo:begin ?b ;
                faldo:end ?e .
            ?contig a g:Contig ;
                g:Identifier ?contigid .
            ?b faldo:position ?beginPos .
            ?e faldo:position ?endPos
        }}
    '''.format(subtypescheme_rdf, isolate_rdf)

    return query

@tojson
@submit
@prefix
def genename_query(locus_rdf):

    query = '''
        SELECT ?markerURI ?markerLabel
        WHERE {{
            VALUES ?l {{ {} }} .
            ?markerURI :isFoundIn ?l ;
                a :Marker ;
                dc:description ?markerLabel
        }}
    '''.format(locus_rdf)

    return query

class MarkerSequences(object):
    """Retrieve DNA region sequences for one or more Markers

    """

    def __init__(self, markers=[':stx2A',':stx2B'], job_id=None, job_turtle=None, job_ectyper_datastruct_vf=None, redis_conn=None):
        """Constructor

        Args:

        """

        # convert to proper RDF terms
        self.marker_uris = [turtle_utils.normalize_rdfterm(m) for m in markers]
        # Read the phylotyper ontology as a starter graph.
        g = Graph()
        ontology_turtle_file = os.path.join(__location__, 'superphy_subtyping.ttl')
        g.parse(ontology_turtle_file, format="turtle")
        # Retrieve and merge graphs from pre-req. jobs.
        self.graph = g + fetch_job(job_id, redis_conn).result + fetch_job(job_turtle, redis_conn).result + fetch_job(job_ectyper_datastruct_vf, redis_conn).result

    def _unroll(self, query_result):
        '''
        Custom method to convert query results into the same dictionary format as decorators.to_json().
        :param query_result:
        :return:
        '''
        l = []
        for tup in query_result:

    def sequences(self, genome_uri):
        """Retrieve sequences for object alleles

          Args:
            genome_uri(str): Genome URI

          Returns:
            dictionary

        """
        #raise Exception('graph: {0}'.format(self.graph.serialize(format="turtle")))
        genome_rdf = turtle_utils.normalize_rdfterm(genome_uri)
        query = sequence_query(self.marker_uris, genome_rdf)
        # query_result = sequence_query(self.marker_uris, genome_rdf)
        query_result = self.graph.query(query)
        query_result = [row for row in query_result]
        # ?contig ?contigid ?region ?start ?len ?seq
        l = [
            {
                'contig': tup[0].toPython(),
                'contigid': tup[1].toPython(),
                'region': tup[2].toPython(),
                'start': tup[3].toPython(),
                'len': tup[4].toPython(),
                'seq': tup[5].toPython()
            }
            for tup in query_result
        ]
        raise Exception('sequences() query_result: {0}'.format(l))

        # Unroll result into dictionary with fasta-like keys
        seqdict = { "spfy|{}| {}:{}..{}".format(
            turtle_utils.fulluri_to_basename(r['region']),
            r['contigid'], r['start'], r['start']+r['len']-1): r['seq'] for r in query_result }

        return seqdict

    def fasta(self, genome_uri):
        """Retrieve sequences for object alleles

          Args:
            genome_uri(str): Genome URI

          Returns:
             fasta format string or None

        """

        seqdict = self.sequences(genome_uri)

        if not seqdict:
            return None

        fasta_string = ''
        for (h,s) in seqdict.iteritems():
            fasta_string += ">{}\n{}\n".format(h,s)

        return fasta_string

if __name__=='__main__':
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-m",
        nargs="*",
        help="Marker URI",
        required=True
    )
    parser.add_argument(
        "-g",
        help="Genome URI",
        required=True
    )
    args = parser.parse_args()

    ms = MarkerSequences(args.m)
    print ms.sequences(args.g)
<|endoftext|>


<commit_before>#!/usr/bin/env python2.5
#
# Copyright 2011 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module for the admin pages.
"""

__authors__ = [
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  ]

import logging

from google.appengine.api import users
from google.appengine.ext import db

from django import forms as djangoforms
from django.conf.urls.defaults import url
from django.utils import simplejson
from django.utils.translation import ugettext

from soc.logic import accounts
from soc.logic import cleaning
from soc.logic import dicts
from soc.logic.exceptions import AccessViolation
from soc.logic.exceptions import BadRequest
from soc.views import forms
from soc.views import readonly_template
from soc.views.template import Template

from soc.models.user import User

from soc.modules.gsoc.models.profile import GSoCProfile
from soc.modules.gsoc.models.organization import GSoCOrganization
from soc.modules.gsoc.views.base import RequestHandler
from soc.modules.gsoc.views.base_templates import LoggedInMsg
from soc.modules.gsoc.views.helper import lists
from soc.modules.gsoc.views.helper import url_patterns

class LookupForm(forms.ModelForm):
  """Django form for the invite page.
  """

  class Meta:
    model = None

  def __init__(self, request_data, *args):
    super(LookupForm, self).__init__(*args)
    self.request_data = request_data

  email = djangoforms.CharField(label='Email')

  def clean_email(self):
    email_cleaner = cleaning.clean_email('email')

    try:
      email_address = email_cleaner(self)
    except djangoforms.ValidationError, e:
      if e.code != 'invalid':
        raise
      msg = ugettext(u'Enter a valid email address.')
      raise djangoforms.ValidationError(msg, code='invalid')

    account = users.User(email_address)
    user_account = accounts.normalizeAccount(account)
    user = User.all().filter('account', user_account).get()

    if not user:
      raise djangoforms.ValidationError(
          "There is no user with that email address")

    self.cleaned_data['user'] = user

    q = GSoCProfile.all()
    q.filter('scope', self.request_data.program)
    q.ancestor(user)
    self.cleaned_data['profile'] = q.get()

class DashboardPage(RequestHandler):
  """Dashboard for admins.
  """

  def djangoURLPatterns(self):
    return [
        url(r'^gsoc/admin/%s$' % url_patterns.PROGRAM,
         self, name='gsoc_admin_dashboard'),
    ]

  def checkAccess(self):
    self.check.isHost()

  def templatePath(self):
    return 'v2/modules/gsoc/admin/dashboard.html'

  def context(self):
    r = self.data.redirect
    r.program()

    return {
        'page_name': 'Admin dashboard',
        'lookup_link': r.urlOf('lookup_gsoc_profile'),
        'slots_link': r.urlOf('gsoc_slots'),
        'program_link': r.urlOf('edit_gsoc_program'),
        'timeline_link': r.urlOf('edit_gsoc_timeline'),
    }

class LookupLinkIdPage(RequestHandler):
  """View for the participant profile.
  """

  def djangoURLPatterns(self):
    return [
        url(r'^gsoc/admin/lookup/%s$' % url_patterns.PROGRAM,
         self, name='lookup_gsoc_profile'),
    ]

  def checkAccess(self):
    self.check.isHost()

  def templatePath(self):
    return 'v2/modules/gsoc/admin/lookup.html'

  def post(self):
    self.get()

  def context(self):
    form = LookupForm(self.data, self.data.POST or None)
    error = bool(form.errors)

    forms = [form]
    profile = None

    if not form.errors and self.data.request.method == 'POST':
      profile = form.cleaned_data.get('profile')

    if profile:
      self.redirect.profile(profile.link_id)
      self.redirect.to('gsoc_profile_admin')

    return {
      'forms': forms,
      'error': error,
      'posted': error,
      'page_name': 'Lookup profile',
    }

class SlotsList(Template):
  """Template for list of accepted organizations.
  """

  def __init__(self, request, data):
    self.request = request
    self.data = data

    list_config = lists.ListConfiguration()
    list_config.addColumn('name', 'Name',
        (lambda e, *args: e.short_name.strip()), width=75)
    list_config.addSimpleColumn('link_id', 'Link ID', hidden=True)
    options = [('', 'All'), ('true', 'New'), ('false', 'Veteran')]
    list_config.addSimpleColumn('new_org', 'New', width=25, options=options)
    list_config.addSimpleColumn('slots_desired', 'min', width=25)
    list_config.addSimpleColumn('max_slots_desired', 'max', width=25)
    list_config.addSimpleColumn('slots', 'Slots', width=50)
    list_config.setColumnEditable('slots', True)
    list_config.setColumnSummary('slots', 'sum', "Total: {0}")
    list_config.addSimpleColumn('note', 'Note')
    list_config.setColumnEditable('note', True) #, edittype='textarea')
    list_config.setDefaultPagination(False)
    list_config.setDefaultSort('name')
    list_config.addPostEditButton('save', "Save", "", [])

    self._list_config = list_config

  def context(self):
    description = 'List of organizations accepted into %s' % (
            self.data.program.name)

    list = lists.ListConfigurationResponse(
        self.data, self._list_config, 0, description)

    return {
        'lists': [list],
    }

  def post(self):
    idx = lists.getListIndex(self.request)
    if idx != 0:
      return False

    data = self.data.POST.get('data')

    if not data:
      raise BadRequest("Missing data")

    parsed = simplejson.loads(data)

    for key_name, properties in parsed.iteritems():
      note = properties.get('note')
      slots = properties.get('slots')

      if 'note' not in properties and 'slots' not in properties:
        logging.warning("Neither note or slots present in '%s'" % properties)
        continue

      if 'slots' in properties:
        if not slots.isdigit():
          logging.warning("Non-int value for slots: '%s'" % slots)
          properties.pop('slots')
        else:
          slots = int(slots)

      def update_org_txn():
        org = GSoCOrganization.get_by_key_name(key_name)
        if not org:
          logging.warning("Invalid org_key '%s'" % key_name)
          return
        if 'note' in properties:
          org.note = note
        if 'slots' in properties:
          org.slots = slots
        org.put()

      db.run_in_transaction(update_org_txn)

    return True

  def getListData(self):
    idx = lists.getListIndex(self.request)
    if idx != 0:
      return None

    q = GSoCOrganization.all().filter('scope', self.data.program)

    starter = lists.keyStarter

    response_builder = lists.RawQueryContentResponseBuilder(
        self.request, self._list_config, q, starter)

    return response_builder.build()

  def templatePath(self):
    return "v2/modules/gsoc/admin/_slots_list.html"

class SlotsPage(RequestHandler):
  """View for the participant profile.
  """

  def djangoURLPatterns(self):
    return [
        url(r'^gsoc/admin/slots/%s$' % url_patterns.PROGRAM,
         self, name='gsoc_slots'),
    ]

  def checkAccess(self):
    self.check.isHost()

  def templatePath(self):
    return 'v2/modules/gsoc/admin/slots.html'

  def jsonContext(self):
    list_content = SlotsList(self.request, self.data).getListData()

    if not list_content:
      raise AccessViolation(
          'You do not have access to this data')

    return list_content.content()

  def post(self):
    slots_list = SlotsList(self.request, self.data)

    if not slots_list.post():
      raise AccessViolation(
          'You cannot change this data')

  def context(self):
    return {
      'page_name': 'Slots page',
      'slots_list': SlotsList(self.request, self.data),
    }
<commit_msg>Disable auto-refresh in slots page<commit_after>#!/usr/bin/env python2.5
#
# Copyright 2011 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module for the admin pages.
"""

__authors__ = [
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  ]

import logging

from google.appengine.api import users
from google.appengine.ext import db

from django import forms as djangoforms
from django.conf.urls.defaults import url
from django.utils import simplejson
from django.utils.translation import ugettext

from soc.logic import accounts
from soc.logic import cleaning
from soc.logic import dicts
from soc.logic.exceptions import AccessViolation
from soc.logic.exceptions import BadRequest
from soc.views import forms
from soc.views import readonly_template
from soc.views.template import Template

from soc.models.user import User

from soc.modules.gsoc.models.profile import GSoCProfile
from soc.modules.gsoc.models.organization import GSoCOrganization
from soc.modules.gsoc.views.base import RequestHandler
from soc.modules.gsoc.views.base_templates import LoggedInMsg
from soc.modules.gsoc.views.helper import lists
from soc.modules.gsoc.views.helper import url_patterns

class LookupForm(forms.ModelForm):
  """Django form for the invite page.
  """

  class Meta:
    model = None

  def __init__(self, request_data, *args):
    super(LookupForm, self).__init__(*args)
    self.request_data = request_data

  email = djangoforms.CharField(label='Email')

  def clean_email(self):
    email_cleaner = cleaning.clean_email('email')

    try:
      email_address = email_cleaner(self)
    except djangoforms.ValidationError, e:
      if e.code != 'invalid':
        raise
      msg = ugettext(u'Enter a valid email address.')
      raise djangoforms.ValidationError(msg, code='invalid')

    account = users.User(email_address)
    user_account = accounts.normalizeAccount(account)
    user = User.all().filter('account', user_account).get()

    if not user:
      raise djangoforms.ValidationError(
          "There is no user with that email address")

    self.cleaned_data['user'] = user

    q = GSoCProfile.all()
    q.filter('scope', self.request_data.program)
    q.ancestor(user)
    self.cleaned_data['profile'] = q.get()

class DashboardPage(RequestHandler):
  """Dashboard for admins.
  """

  def djangoURLPatterns(self):
    return [
        url(r'^gsoc/admin/%s$' % url_patterns.PROGRAM,
         self, name='gsoc_admin_dashboard'),
    ]

  def checkAccess(self):
    self.check.isHost()

  def templatePath(self):
    return 'v2/modules/gsoc/admin/dashboard.html'

  def context(self):
    r = self.data.redirect
    r.program()

    return {
        'page_name': 'Admin dashboard',
        'lookup_link': r.urlOf('lookup_gsoc_profile'),
        'slots_link': r.urlOf('gsoc_slots'),
        'program_link': r.urlOf('edit_gsoc_program'),
        'timeline_link': r.urlOf('edit_gsoc_timeline'),
    }

class LookupLinkIdPage(RequestHandler):
  """View for the participant profile.
  """

  def djangoURLPatterns(self):
    return [
        url(r'^gsoc/admin/lookup/%s$' % url_patterns.PROGRAM,
         self, name='lookup_gsoc_profile'),
    ]

  def checkAccess(self):
    self.check.isHost()

  def templatePath(self):
    return 'v2/modules/gsoc/admin/lookup.html'

  def post(self):
    self.get()

  def context(self):
    form = LookupForm(self.data, self.data.POST or None)
    error = bool(form.errors)

    forms = [form]
    profile = None

    if not form.errors and self.data.request.method == 'POST':
      profile = form.cleaned_data.get('profile')

    if profile:
      self.redirect.profile(profile.link_id)
      self.redirect.to('gsoc_profile_admin')

    return {
      'forms': forms,
      'error': error,
      'posted': error,
      'page_name': 'Lookup profile',
    }

class SlotsList(Template):
  """Template for list of accepted organizations.
  """

  def __init__(self, request, data):
    self.request = request
    self.data = data

    list_config = lists.ListConfiguration()
    list_config.addColumn('name', 'Name',
        (lambda e, *args: e.short_name.strip()), width=75)
    list_config.addSimpleColumn('link_id', 'Link ID', hidden=True)
    options = [('', 'All'), ('true', 'New'), ('false', 'Veteran')]
    list_config.addSimpleColumn('new_org', 'New', width=25, options=options)
    list_config.addSimpleColumn('slots_desired', 'min', width=25)
    list_config.addSimpleColumn('max_slots_desired', 'max', width=25)
    list_config.addSimpleColumn('slots', 'Slots', width=50)
    list_config.setColumnEditable('slots', True)
    list_config.setColumnSummary('slots', 'sum', "Total: {0}")
    list_config.addSimpleColumn('note', 'Note')
    list_config.setColumnEditable('note', True) #, edittype='textarea')
    list_config.setDefaultPagination(False)
    list_config.setDefaultSort('name')
    list_config.addPostEditButton('save', "Save", "", [], refresh="none")

    self._list_config = list_config

  def context(self):
    description = 'List of organizations accepted into %s' % (
            self.data.program.name)

    list = lists.ListConfigurationResponse(
        self.data, self._list_config, 0, description)

    return {
        'lists': [list],
    }

  def post(self):
    idx = lists.getListIndex(self.request)
    if idx != 0:
      return False

    data = self.data.POST.get('data')

    if not data:
      raise BadRequest("Missing data")

    parsed = simplejson.loads(data)

    for key_name, properties in parsed.iteritems():
      note = properties.get('note')
      slots = properties.get('slots')

      if 'note' not in properties and 'slots' not in properties:
        logging.warning("Neither note or slots present in '%s'" % properties)
        continue

      if 'slots' in properties:
        if not slots.isdigit():
          logging.warning("Non-int value for slots: '%s'" % slots)
          properties.pop('slots')
        else:
          slots = int(slots)

      def update_org_txn():
        org = GSoCOrganization.get_by_key_name(key_name)
        if not org:
          logging.warning("Invalid org_key '%s'" % key_name)
          return
        if 'note' in properties:
          org.note = note
        if 'slots' in properties:
          org.slots = slots
        org.put()

      db.run_in_transaction(update_org_txn)

    return True

  def getListData(self):
    idx = lists.getListIndex(self.request)
    if idx != 0:
      return None

    q = GSoCOrganization.all().filter('scope', self.data.program)

    starter = lists.keyStarter

    response_builder = lists.RawQueryContentResponseBuilder(
        self.request, self._list_config, q, starter)

    return response_builder.build()

  def templatePath(self):
    return "v2/modules/gsoc/admin/_slots_list.html"

class SlotsPage(RequestHandler):
  """View for the participant profile.
  """

  def djangoURLPatterns(self):
    return [
        url(r'^gsoc/admin/slots/%s$' % url_patterns.PROGRAM,
         self, name='gsoc_slots'),
    ]

  def checkAccess(self):
    self.check.isHost()

  def templatePath(self):
    return 'v2/modules/gsoc/admin/slots.html'

  def jsonContext(self):
    list_content = SlotsList(self.request, self.data).getListData()

    if not list_content:
      raise AccessViolation(
          'You do not have access to this data')

    return list_content.content()

  def post(self):
    slots_list = SlotsList(self.request, self.data)

    if not slots_list.post():
      raise AccessViolation(
          'You cannot change this data')

  def context(self):
    return {
      'page_name': 'Slots page',
      'slots_list': SlotsList(self.request, self.data),
    }
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Oct 31 23:25:33 2017

@author: asadm2
"""

from __future__ import division, print_function, absolute_import
from glob import glob
from astropy.io import fits
from astropy.wcs import WCS
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import os
import subprocess

eco_repo_path = '/fs1/masad/Research/Repositories/ECO_Globular_Clusters/'

goodObj = '../../../data/interim/goodObj.txt'
good_Obj = pd.read_csv(goodObj,header=None)
good_Obj.columns = ['ECOID']

ECOnew  = '../../../data/interim/ECO_formatted.txt'
ECO_new  = pd.read_csv(ECOnew, delimiter='\t')

ECOphotcat = '../../../data/interim/ECOphot_final.txt'
ECO_phot_cat  = pd.read_csv(ECOphotcat, delim_whitespace=True,header=None,\
                            skiprows=1,names=['ECOID','RA','DEC','VLG','umag',\
                                              'gmag','rmag','imag','zmag',\
                                              'Jmag','Hmag','Kmag'])

#good_Obj_new = good_Obj.ECOID[:4].append(good_Obj.ECOID[5:11].append(good_Obj\
#                             .ECOID[104:105]))

good_Obj_new = ['ECO00026'] #REMOVE THIS

sdssr_calc = []
sdssr_cat = []
for obj in good_Obj_new:
    print(obj)
    dir_path = os.getcwd()
    if os.path.basename(dir_path) == 'main':
        os.chdir('../../../data/interim/'+obj)
    elif os.path.basename(dir_path) != obj:
        os.chdir('../interim/'+obj)

    comb_coadd = glob(obj+'_comb_coadd.fits')[0]
    f814_coadd = glob(obj+'_acs_wfc_f814w_coadd.fits')[0]

    print('Starting source extractor')
    subprocess.call(['sex',comb_coadd+","+f814_coadd,'-ANALYSIS_THRESH','1.5',\
    '-BACK_SIZE','128','-DEBLEND_MINCONT','0.0025', '-DETECT_THRESH','1.5',\
    '-DETECT_MINAREA','9','-SEEING_FWHM','0.1',\
    '-CATALOG_NAME',obj+'_acs_wfc_f814w.cat']) #CHANGE THIS
    print('Finished running source extractor')

    hdu_f814w_coadd = fits.open(f814_coadd)
    prihdr = hdu_f814w_coadd[0].header
    photflam814 = prihdr['PHOTFLAM']
    photzpt814 = prihdr['PHOTZPT']
    print('Calculating zeropoint')
    zpt814 = -2.5*np.log10(photflam814) + photzpt814

    ra = np.unique(ECO_new.RA.loc[ECO_new.ECOID==obj])[0]
    dec = np.unique(ECO_new.DEC.loc[ECO_new.ECOID==obj])[0]

    print('Converting to pixel coordinates')
    wcs = WCS(prihdr)
    print(wcs)
    xx,yy = wcs.wcs_world2pix(ra,dec,1)
    print(xx,yy)

#remember to copy already uncommented default.param file and remove other one
#remember to copy all default files to last obj in contents_new

#glob comb_coadd and store as image1
#glob f814 coadd and store as image2
#use subprocess to pass parameters
#open filter image and get necessary keywords for finding zeropoint
#get RA and DEC of galaxy from ECO catalog
#open filter image and get WCS info to find world2pix transformation for RA and
#DEC of galaxy
#open catalog that sextractor generated and get petro mag in f814 at
#calculated x and y pixel values
#Apply zeropoint to magnitude
#Use conversion to change to r band
#Get r band mag from photometry catalog
#Plot both points for each galaxy

#need to use WCS

    print('Reading test.cat')
    #CHANGE NAME
    f814w_cat = pd.read_csv(obj+'acs_wfc_f814w_scaletest.cat',header=None,\
                            delim_whitespace=True,skiprows=7,\
                            names=['petro_mag','petro_magerr','petro_radius',\
                                  'x_image','y_image','a_image','class_star'])

    print('Getting petro mag from test.cat')
#    print(f814w_cat.x_image==xx)
#    print(f814w_cat.y_image==yy)
    f814mag = f814w_cat.petro_mag.loc[(np.abs(f814w_cat.x_image==xx))&\
                                      (np.abs(f814w_cat.y_image==yy))]\
                                      .values[0]

    print('Calculating rmag')
    f814mag += zpt814
    sdss_r = f814mag - 1

    print('Retrieving rmag from catalog')
    sdss_r_cat = ECO_phot_cat.rmag.loc[ECO_phot_cat.ECOID==obj].values[0]

    sdssr_calc.append(sdss_r)
    sdssr_cat.append(sdss_r_cat)
    hdu_f814w_coadd.close()

print('Plotting')
x = np.linspace(0,len(good_Obj_new.values)+1,len(good_Obj_new.values))
my_xticks = good_Obj_new.values
fig1 = plt.figure(figsize=(10,8))
plt.xticks(x, my_xticks,rotation=90)
plt.scatter(x,sdssr_calc, c='r',label='calculated rmag')
plt.scatter(x,sdssr_cat, c='g',label='catalog rmag')
plt.xlabel('ECOID')
plt.ylabel('rmag')
plt.gca().invert_yaxis()
plt.title('Comparison of calculated rmag and rmag from ECO photometric table')
plt.savefig('calcrmag_catrmag.png')

    <commit_msg>Testing ECO00026<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Oct 31 23:25:33 2017

@author: asadm2
"""

from __future__ import division, print_function, absolute_import
from glob import glob
from astropy.io import fits
from astropy.wcs import WCS
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import os
import subprocess

eco_repo_path = '/fs1/masad/Research/Repositories/ECO_Globular_Clusters/'

goodObj = '../../../data/interim/goodObj.txt'
good_Obj = pd.read_csv(goodObj,header=None)
good_Obj.columns = ['ECOID']

ECOnew  = '../../../data/interim/ECO_formatted.txt'
ECO_new  = pd.read_csv(ECOnew, delimiter='\t')

ECOphotcat = '../../../data/interim/ECOphot_final.txt'
ECO_phot_cat  = pd.read_csv(ECOphotcat, delim_whitespace=True,header=None,\
                            skiprows=1,names=['ECOID','RA','DEC','VLG','umag',\
                                              'gmag','rmag','imag','zmag',\
                                              'Jmag','Hmag','Kmag'])

#good_Obj_new = good_Obj.ECOID[:4].append(good_Obj.ECOID[5:11].append(good_Obj\
#                             .ECOID[104:105]))

good_Obj_new = ['ECO00026'] #REMOVE THIS

sdssr_calc = []
sdssr_cat = []
for obj in good_Obj_new:
    print('{0}/{1}'.format(obj,len(good_Obj_new)))
    dir_path = os.getcwd()
    if os.path.basename(dir_path) == 'main':
        os.chdir('../../../data/interim/'+obj)
    elif os.path.basename(dir_path) != obj:
        os.chdir('../interim/'+obj)

    comb_coadd = glob(obj+'_comb_coadd.fits')[0]
    f814_coadd = glob(obj+'_acs_wfc_f814w_coadd.fits')[0]

    print('Starting source extractor')
    subprocess.call(['sex',comb_coadd+","+f814_coadd,'-ANALYSIS_THRESH','1.5',\
    '-BACK_SIZE','128','-DEBLEND_MINCONT','0.0025', '-DETECT_THRESH','1.5',\
    '-DETECT_MINAREA','9','-SEEING_FWHM','0.1',\
    '-CATALOG_NAME',obj+'_acs_wfc_f814w.cat']) #CHANGE THIS
    print('Finished running source extractor')

    hdu_f814w_coadd = fits.open(f814_coadd)
    prihdr = hdu_f814w_coadd[0].header
    photflam814 = prihdr['PHOTFLAM']
    photzpt814 = prihdr['PHOTZPT']
    print('Calculating zeropoint')
    zpt814 = -2.5*np.log10(photflam814) + photzpt814

    ra = np.unique(ECO_new.RA.loc[ECO_new.ECOID==obj])[0]
    dec = np.unique(ECO_new.DEC.loc[ECO_new.ECOID==obj])[0]

    print('Converting to pixel coordinates')
    wcs = WCS(prihdr)
    print(wcs)
    xx,yy = wcs.wcs_world2pix(ra,dec,1)
    print(xx,yy)

#remember to copy already uncommented default.param file and remove other one
#remember to copy all default files to last obj in contents_new

#glob comb_coadd and store as image1
#glob f814 coadd and store as image2
#use subprocess to pass parameters
#open filter image and get necessary keywords for finding zeropoint
#get RA and DEC of galaxy from ECO catalog
#open filter image and get WCS info to find world2pix transformation for RA and
#DEC of galaxy
#open catalog that sextractor generated and get petro mag in f814 at
#calculated x and y pixel values
#Apply zeropoint to magnitude
#Use conversion to change to r band
#Get r band mag from photometry catalog
#Plot both points for each galaxy

#need to use WCS

    print('Reading test.cat')
    #CHANGE NAME
    f814w_cat = pd.read_csv(obj+'acs_wfc_f814w_scaletest.cat',header=None,\
                            delim_whitespace=True,skiprows=7,\
                            names=['petro_mag','petro_magerr','petro_radius',\
                                  'x_image','y_image','a_image','class_star'])

    print('Getting petro mag from test.cat')
#    print(f814w_cat.x_image==xx)
#    print(f814w_cat.y_image==yy)
    f814mag = f814w_cat.petro_mag.loc[(np.abs(f814w_cat.x_image==xx))&\
                                      (np.abs(f814w_cat.y_image==yy))]\
                                      .values[0]

    print('Calculating rmag')
    f814mag += zpt814
    sdss_r = f814mag - 1

    print('Retrieving rmag from catalog')
    sdss_r_cat = ECO_phot_cat.rmag.loc[ECO_phot_cat.ECOID==obj].values[0]

    sdssr_calc.append(sdss_r)
    sdssr_cat.append(sdss_r_cat)
    hdu_f814w_coadd.close()

print('Plotting')
x = np.linspace(0,len(good_Obj_new.values)+1,len(good_Obj_new.values))
my_xticks = good_Obj_new.values
fig1 = plt.figure(figsize=(10,8))
plt.xticks(x, my_xticks,rotation=90)
plt.scatter(x,sdssr_calc, c='r',label='calculated rmag')
plt.scatter(x,sdssr_cat, c='g',label='catalog rmag')
plt.xlabel('ECOID')
plt.ylabel('rmag')
plt.gca().invert_yaxis()
plt.title('Comparison of calculated rmag and rmag from ECO photometric table')
plt.savefig('calcrmag_catrmag.png')

    <|endoftext|>