<commit_before>from django.http import HttpResponse
from django.utils import simplejson
import models

from scipy_central.screenshot.forms import ScreenshotForm

def add_screenshot(request):
    """
    Received POST request to add a new screenshot.
    """
    img_form = ScreenshotForm(request.POST, request.FILES)
    if img_form.is_valid():

        # Somehow (not sure how yet) the file name is slugified, so that there
        # will never be characters such as ";", "#", "&" etc in the names
        # We benefit from that by allowing the magnifier setting to be
        # specified after a ";" character.
        img = models.Screenshot(img_file_raw=img_form.\
                                              cleaned_data['spc_image_upload'])
        img.save()
        msg = ('
'
               'Upload successful. Insert the image in your description as'
               '    <tt>:image:`%s`</tt>
'
               'Want a smaller image? e.g. scaled down to 40%%: '
               '   <tt>:image:`%s;40`</tt>

') %\
                            (img.img_file_raw.name.partition('/')[2],
                             img.img_file_raw.name.partition('/')[2])
        return HttpResponse(msg)
    else:
        msg = ('
%s
') % img_form.errors.get('spc_image_upload')[0]
        return HttpResponse(msg)
<commit_msg>MAINT: improve image upload message<commit_after>from django.http import HttpResponse
from django.utils import simplejson
import models

from scipy_central.screenshot.forms import ScreenshotForm

def add_screenshot(request):
    """
    Received POST request to add a new screenshot.
    """
    img_form = ScreenshotForm(request.POST, request.FILES)
    if img_form.is_valid():

        # Somehow (not sure how yet) the file name is slugified, so that there
        # will never be characters such as ";", "#", "&" etc in the names
        # We benefit from that by allowing the magnifier setting to be
        # specified after a ";" character.
        img = models.Screenshot(img_file_raw=img_form.\
                                              cleaned_data['spc_image_upload'])
        img.save()
        msg = ('Upload successful. Insert the image in your description as'
               '    <tt>:image:`%s`</tt>
'
               '
'
               'Want a smaller image? e.g. scaled down to 40%%: '
               ' <tt>:image:`%s;40`</tt>
') %\
                            (img.img_file_raw.name.partition('/')[2],
                             img.img_file_raw.name.partition('/')[2])
        return HttpResponse(msg)
    else:
        msg = ('
%s
') % img_form.errors.get('spc_image_upload')[0]
        return HttpResponse(msg)
<|endoftext|>


<commit_before>from django.contrib import admin

# Register your models here.
<commit_msg>registered game and round as models on admin site.<commit_after>from django.contrib import admin

from .models import Game, Round

admin.site.register(Game)
admin.site.register(Round)
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright: (c) 2012, Jan-Piet Mens <jpmens () gmail.com>
# Copyright: (c) 2015, Ales Nosek <anosek.nosek () gmail.com>
# Copyright: (c) 2017, Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)

from __future__ import absolute_import, division, print_function
__metaclass__ = type
#

ANSIBLE_METADATA = {'metadata_version': '1.1',
                    'status': ['preview'],
                    'supported_by': 'community'}

DOCUMENTATION = '''
---
module: ini_file
short_description: Tweak settings in INI files
extends_documentation_fragment: files
description:
     - Manage (add, remove, change) individual settings in an INI-style file without having
       to manage the file as a whole with, say, M(template) or M(assemble). Adds missing
       sections if they don't exist.
     - Before version 2.0, comments are discarded when the source file is read, and therefore will not show up in the destination file.
     - Since version 2.3, this module adds missing ending newlines to files to keep in line with the POSIX standard, even when
       no other modifications need to be applied.
version_added: "0.9"
options:
  path:
    description:
      - Path to the INI-style file; this file is created if required.
      - Before 2.3 this option was only usable as I(dest).
    aliases: [ dest ]
    required: true
  section:
    description:
      - Section name in INI file. This is added if C(state=present) automatically when
        a single value is being set.
      - If left empty or set to `null`, the I(option) will be placed before the first I(section).
        Using `null` is also required if the config format does not support sections.
    required: true
  option:
    description:
      - If set (required for changing a I(value)), this is the name of the option.
      - May be omitted if adding/removing a whole I(section).
  value:
    description:
     - The string value to be associated with an I(option). May be omitted when removing an I(option).
  backup:
    description:
      - Create a backup file including the timestamp information so you can get
        the original file back if you somehow clobbered it incorrectly.
    type: bool
    default: 'no'
  others:
     description:
       - All arguments accepted by the M(file) module also work here
  state:
     description:
       - If set to C(absent) the option or section will be removed if present instead of created.
     choices: [ absent, present ]
     default: present
  no_extra_spaces:
     description:
       - Do not insert spaces before and after '=' symbol
     type: bool
     default: 'no'
     version_added: "2.1"
  create:
     description:
       - If set to 'no', the module will fail if the file does not already exist.
         By default it will create the file if it is missing.
     type: bool
     default: 'yes'
     version_added: "2.2"
  allow_no_value:
     description:
       - allow option without value and without '=' symbol
     type: bool
     required: false
     default: false
     version_added: "2.6"
notes:
   - While it is possible to add an I(option) without specifying a I(value), this makes
     no sense.
   - As of Ansible 2.3, the I(dest) option has been changed to I(path) as default, but
     I(dest) still works as well.
author:
    - Jan-Piet Mens (@jpmens)
    - Ales Nosek (@noseka1)
'''

EXAMPLES = '''
# Before 2.3, option 'dest' was used instead of 'path'
- name: Ensure "fav=lemonade is in section "[drinks]" in specified file
  ini_file:
    path: /etc/conf
    section: drinks
    option: fav
    value: lemonade
    mode: 0600
    backup: yes

- ini_file:
    path: /etc/anotherconf
    section: drinks
    option: temperature
    value: cold
    backup: yes
'''

import os
import re
import tempfile
import traceback

from ansible.module_utils.basic import AnsibleModule

def match_opt(option, line):
    option = re.escape(option)
    return re.match('( |\t)*%s( |\t)*(=|$)' % option, line) \
        or re.match('#( |\t)*%s( |\t)*(=|$)' % option, line) \
        or re.match(';( |\t)*%s( |\t)*(=|$)' % option, line)

def match_active_opt(option, line):
    option = re.escape(option)
    return re.match('( |\t)*%s( |\t)*(=|$)' % option, line)

def do_ini(module, filename, section=None, option=None, value=None,
           state='present', backup=False, no_extra_spaces=False, create=True,
           allow_no_value=False):

    diff = dict(
        before='',
        after='',
        before_header='%s (content)' % filename,
        after_header='%s (content)' % filename,
    )

    if not os.path.exists(filename):
        if not create:
            module.fail_json(rc=257, msg='Destination %s does not exist !' % filename)
        destpath = os.path.dirname(filename)
        if not os.path.exists(destpath) and not module.check_mode:
            os.makedirs(destpath)
        ini_lines = []
    else:
        ini_file = open(filename, 'r')
        try:
            ini_lines = ini_file.readlines()
        finally:
            ini_file.close()

    if module._diff:
        diff['before'] = ''.join(ini_lines)

    changed = False

    # ini file could be empty
    if not ini_lines:
        ini_lines.append('\n')

    # last line of file may not contain a trailing newline
    if ini_lines[-1] == "" or ini_lines[-1][-1] != '\n':
        ini_lines[-1] += '\n'
        changed = True

    # append a fake section line to simplify the logic
    ini_lines.append('[')

    within_section = not section
    section_start = 0
    msg = 'OK'
    if no_extra_spaces:
        assignment_format = '%s=%s\n'
    else:
        assignment_format = '%s = %s\n'

    for index, line in enumerate(ini_lines):
        if line.startswith('[%s]' % section):
            within_section = True
            section_start = index
        elif line.startswith('['):
            if within_section:
                if state == 'present':
                    # insert missing option line at the end of the section
                    for i in range(index, 0, -1):
                        # search backwards for previous non-blank or non-comment line
                        if not re.match(r'^[ \t]*([#;].*)?$', ini_lines[i - 1]):
                            if not value and allow_no_value:
                                ini_lines.insert(i, '%s\n' % option)
                            else:
                                ini_lines.insert(i, assignment_format % (option, value))
                            msg = 'option added'
                            changed = True
                            break
                elif state == 'absent' and not option:
                    # remove the entire section
                    del ini_lines[section_start:index]
                    msg = 'section removed'
                    changed = True
                break
        else:
            if within_section and option:
                if state == 'present':
                    # change the existing option line
                    if match_opt(option, line):
                        if not value and allow_no_value:
                            newline = '%s\n' % option
                        else:
                            newline = assignment_format % (option, value)
                        option_changed = ini_lines[index] != newline
                        changed = changed or option_changed
                        if option_changed:
                            msg = 'option changed'
                        ini_lines[index] = newline
                        if option_changed:
                            # remove all possible option occurrences from the rest of the section
                            index = index + 1
                            while index < len(ini_lines):
                                line = ini_lines[index]
                                if line.startswith('['):
                                    break
                                if match_active_opt(option, line):
                                    del ini_lines[index]
                                else:
                                    index = index + 1
                        break
                elif state == 'absent':
                    # delete the existing line
                    if match_active_opt(option, line):
                        del ini_lines[index]
                        changed = True
                        msg = 'option changed'
                        break

    # remove the fake section line
    del ini_lines[-1:]

    if not within_section and option and state == 'present':
        ini_lines.append('[%s]\n' % section)
        if not value and allow_no_value:
            ini_lines.append('%s\n' % option)
        else:
            ini_lines.append(assignment_format % (option, value))
        changed = True
        msg = 'section and option added'

    if module._diff:
        diff['after'] = ''.join(ini_lines)

    backup_file = None
    if changed and not module.check_mode:
        if backup:
            backup_file = module.backup_local(filename)

        try:
            tmpfd, tmpfile = tempfile.mkstemp(dir=module.tmpdir)
            f = os.fdopen(tmpfd, 'w')
            f.writelines(ini_lines)
            f.close()
        except IOError:
            module.fail_json(msg="Unable to create temporary file %s", traceback=traceback.format_exc())

        try:
            module.atomic_move(tmpfile, filename)
        except IOError:
            module.ansible.fail_json(msg='Unable to move temporary \
                                   file %s to %s, IOError' % (tmpfile, filename), traceback=traceback.format_exc())

    return (changed, backup_file, diff, msg)

def main():

    module = AnsibleModule(
        argument_spec=dict(
            path=dict(type='path', required=True, aliases=['dest']),
            section=dict(type='str', required=True),
            option=dict(type='str'),
            value=dict(type='str'),
            backup=dict(type='bool', default=False),
            state=dict(type='str', default='present', choices=['absent', 'present']),
            no_extra_spaces=dict(type='bool', default=False),
            allow_no_value=dict(type='bool', default=False, required=False),
            create=dict(type='bool', default=True)
        ),
        add_file_common_args=True,
        supports_check_mode=True,
    )

    path = module.params['path']
    section = module.params['section']
    option = module.params['option']
    value = module.params['value']
    state = module.params['state']
    backup = module.params['backup']
    no_extra_spaces = module.params['no_extra_spaces']
    allow_no_value = module.params['allow_no_value']
    create = module.params['create']

    (changed, backup_file, diff, msg) = do_ini(module, path, section, option, value, state, backup, no_extra_spaces, create, allow_no_value)

    if not module.check_mode and os.path.exists(path):
        file_args = module.load_file_common_arguments(module.params)
        changed = module.set_fs_attributes_if_different(file_args, changed)

    results = dict(
        changed=changed,
        diff=diff,
        msg=msg,
        path=path,
    )
    if backup_file is not None:
        results['backup_file'] = backup_file

    # Mission complete
    module.exit_json(**results)

if __name__ == '__main__':
    main()
<commit_msg>doc: add name to tasks in ini_file module examples (#38888)<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright: (c) 2012, Jan-Piet Mens <jpmens () gmail.com>
# Copyright: (c) 2015, Ales Nosek <anosek.nosek () gmail.com>
# Copyright: (c) 2017, Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)

from __future__ import absolute_import, division, print_function
__metaclass__ = type
#

ANSIBLE_METADATA = {'metadata_version': '1.1',
                    'status': ['preview'],
                    'supported_by': 'community'}

DOCUMENTATION = '''
---
module: ini_file
short_description: Tweak settings in INI files
extends_documentation_fragment: files
description:
     - Manage (add, remove, change) individual settings in an INI-style file without having
       to manage the file as a whole with, say, M(template) or M(assemble). Adds missing
       sections if they don't exist.
     - Before version 2.0, comments are discarded when the source file is read, and therefore will not show up in the destination file.
     - Since version 2.3, this module adds missing ending newlines to files to keep in line with the POSIX standard, even when
       no other modifications need to be applied.
version_added: "0.9"
options:
  path:
    description:
      - Path to the INI-style file; this file is created if required.
      - Before 2.3 this option was only usable as I(dest).
    aliases: [ dest ]
    required: true
  section:
    description:
      - Section name in INI file. This is added if C(state=present) automatically when
        a single value is being set.
      - If left empty or set to `null`, the I(option) will be placed before the first I(section).
        Using `null` is also required if the config format does not support sections.
    required: true
  option:
    description:
      - If set (required for changing a I(value)), this is the name of the option.
      - May be omitted if adding/removing a whole I(section).
  value:
    description:
     - The string value to be associated with an I(option). May be omitted when removing an I(option).
  backup:
    description:
      - Create a backup file including the timestamp information so you can get
        the original file back if you somehow clobbered it incorrectly.
    type: bool
    default: 'no'
  others:
     description:
       - All arguments accepted by the M(file) module also work here
  state:
     description:
       - If set to C(absent) the option or section will be removed if present instead of created.
     choices: [ absent, present ]
     default: present
  no_extra_spaces:
     description:
       - Do not insert spaces before and after '=' symbol
     type: bool
     default: 'no'
     version_added: "2.1"
  create:
     description:
       - If set to 'no', the module will fail if the file does not already exist.
         By default it will create the file if it is missing.
     type: bool
     default: 'yes'
     version_added: "2.2"
  allow_no_value:
     description:
       - allow option without value and without '=' symbol
     type: bool
     required: false
     default: false
     version_added: "2.6"
notes:
   - While it is possible to add an I(option) without specifying a I(value), this makes
     no sense.
   - As of Ansible 2.3, the I(dest) option has been changed to I(path) as default, but
     I(dest) still works as well.
author:
    - Jan-Piet Mens (@jpmens)
    - Ales Nosek (@noseka1)
'''

EXAMPLES = '''
# Before 2.3, option 'dest' was used instead of 'path'
- name: Ensure "fav=lemonade is in section "[drinks]" in specified file
  ini_file:
    path: /etc/conf
    section: drinks
    option: fav
    value: lemonade
    mode: 0600
    backup: yes

- name: Ensure "temperature=cold is in section "[drinks]" in specified file
  ini_file:
    path: /etc/anotherconf
    section: drinks
    option: temperature
    value: cold
    backup: yes
'''

import os
import re
import tempfile
import traceback

from ansible.module_utils.basic import AnsibleModule

def match_opt(option, line):
    option = re.escape(option)
    return re.match('( |\t)*%s( |\t)*(=|$)' % option, line) \
        or re.match('#( |\t)*%s( |\t)*(=|$)' % option, line) \
        or re.match(';( |\t)*%s( |\t)*(=|$)' % option, line)

def match_active_opt(option, line):
    option = re.escape(option)
    return re.match('( |\t)*%s( |\t)*(=|$)' % option, line)

def do_ini(module, filename, section=None, option=None, value=None,
           state='present', backup=False, no_extra_spaces=False, create=True,
           allow_no_value=False):

    diff = dict(
        before='',
        after='',
        before_header='%s (content)' % filename,
        after_header='%s (content)' % filename,
    )

    if not os.path.exists(filename):
        if not create:
            module.fail_json(rc=257, msg='Destination %s does not exist !' % filename)
        destpath = os.path.dirname(filename)
        if not os.path.exists(destpath) and not module.check_mode:
            os.makedirs(destpath)
        ini_lines = []
    else:
        ini_file = open(filename, 'r')
        try:
            ini_lines = ini_file.readlines()
        finally:
            ini_file.close()

    if module._diff:
        diff['before'] = ''.join(ini_lines)

    changed = False

    # ini file could be empty
    if not ini_lines:
        ini_lines.append('\n')

    # last line of file may not contain a trailing newline
    if ini_lines[-1] == "" or ini_lines[-1][-1] != '\n':
        ini_lines[-1] += '\n'
        changed = True

    # append a fake section line to simplify the logic
    ini_lines.append('[')

    within_section = not section
    section_start = 0
    msg = 'OK'
    if no_extra_spaces:
        assignment_format = '%s=%s\n'
    else:
        assignment_format = '%s = %s\n'

    for index, line in enumerate(ini_lines):
        if line.startswith('[%s]' % section):
            within_section = True
            section_start = index
        elif line.startswith('['):
            if within_section:
                if state == 'present':
                    # insert missing option line at the end of the section
                    for i in range(index, 0, -1):
                        # search backwards for previous non-blank or non-comment line
                        if not re.match(r'^[ \t]*([#;].*)?$', ini_lines[i - 1]):
                            if not value and allow_no_value:
                                ini_lines.insert(i, '%s\n' % option)
                            else:
                                ini_lines.insert(i, assignment_format % (option, value))
                            msg = 'option added'
                            changed = True
                            break
                elif state == 'absent' and not option:
                    # remove the entire section
                    del ini_lines[section_start:index]
                    msg = 'section removed'
                    changed = True
                break
        else:
            if within_section and option:
                if state == 'present':
                    # change the existing option line
                    if match_opt(option, line):
                        if not value and allow_no_value:
                            newline = '%s\n' % option
                        else:
                            newline = assignment_format % (option, value)
                        option_changed = ini_lines[index] != newline
                        changed = changed or option_changed
                        if option_changed:
                            msg = 'option changed'
                        ini_lines[index] = newline
                        if option_changed:
                            # remove all possible option occurrences from the rest of the section
                            index = index + 1
                            while index < len(ini_lines):
                                line = ini_lines[index]
                                if line.startswith('['):
                                    break
                                if match_active_opt(option, line):
                                    del ini_lines[index]
                                else:
                                    index = index + 1
                        break
                elif state == 'absent':
                    # delete the existing line
                    if match_active_opt(option, line):
                        del ini_lines[index]
                        changed = True
                        msg = 'option changed'
                        break

    # remove the fake section line
    del ini_lines[-1:]

    if not within_section and option and state == 'present':
        ini_lines.append('[%s]\n' % section)
        if not value and allow_no_value:
            ini_lines.append('%s\n' % option)
        else:
            ini_lines.append(assignment_format % (option, value))
        changed = True
        msg = 'section and option added'

    if module._diff:
        diff['after'] = ''.join(ini_lines)

    backup_file = None
    if changed and not module.check_mode:
        if backup:
            backup_file = module.backup_local(filename)

        try:
            tmpfd, tmpfile = tempfile.mkstemp(dir=module.tmpdir)
            f = os.fdopen(tmpfd, 'w')
            f.writelines(ini_lines)
            f.close()
        except IOError:
            module.fail_json(msg="Unable to create temporary file %s", traceback=traceback.format_exc())

        try:
            module.atomic_move(tmpfile, filename)
        except IOError:
            module.ansible.fail_json(msg='Unable to move temporary \
                                   file %s to %s, IOError' % (tmpfile, filename), traceback=traceback.format_exc())

    return (changed, backup_file, diff, msg)

def main():

    module = AnsibleModule(
        argument_spec=dict(
            path=dict(type='path', required=True, aliases=['dest']),
            section=dict(type='str', required=True),
            option=dict(type='str'),
            value=dict(type='str'),
            backup=dict(type='bool', default=False),
            state=dict(type='str', default='present', choices=['absent', 'present']),
            no_extra_spaces=dict(type='bool', default=False),
            allow_no_value=dict(type='bool', default=False, required=False),
            create=dict(type='bool', default=True)
        ),
        add_file_common_args=True,
        supports_check_mode=True,
    )

    path = module.params['path']
    section = module.params['section']
    option = module.params['option']
    value = module.params['value']
    state = module.params['state']
    backup = module.params['backup']
    no_extra_spaces = module.params['no_extra_spaces']
    allow_no_value = module.params['allow_no_value']
    create = module.params['create']

    (changed, backup_file, diff, msg) = do_ini(module, path, section, option, value, state, backup, no_extra_spaces, create, allow_no_value)

    if not module.check_mode and os.path.exists(path):
        file_args = module.load_file_common_arguments(module.params)
        changed = module.set_fs_attributes_if_different(file_args, changed)

    results = dict(
        changed=changed,
        diff=diff,
        msg=msg,
        path=path,
    )
    if backup_file is not None:
        results['backup_file'] = backup_file

    # Mission complete
    module.exit_json(**results)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Authors: Luis Rei <luis.rei@ijs.si>
# License: MIT License

"""
Generate a POS/NEG/NEUTRAL sentiment dataset from LD-JSON tweets (e.g. sample)
and collected news tweets also in LD-JSON format.

0 - optionally process newsfeed tweets into LD-JSON lines
1 - Create tweets.nf.lang.txt: LD tweet text for a given lang
2 - Create tweets.lang.txt: Filter through langid.py and ldig
3 - Create tweets.pp.lang.tx: Preprocessing - Tokenization and Lower-Casing
4 - Create pos.lang.txt / neg.lang.txt / back.lang.txt:
    filter emoticons with a probability of keeping them in
5 - Create neutal.lang.txt from news_tweets
"""

import sys
import os
from newsfeed_tweets import convert_tweets

def main():
    cmd_name = sys.argv[0]

    # usage
    if len(sys.argv) not in [5, 6]:
        usage = 'Usage:\n\t%{cmd} ' \
                'lang_code prob_smiley min_langid_prob ' \
                'tweets_path tweet_news_path [news_feed_path]'
        usage.format(cmd=cmd_name)
        print usage
        sys.exit(0)

    lang_code = sys.argv[1]
    prob_smiley = float(sys.argv[2])
    min_langid_prob = float(sys.argv[3])
    tweets_path = sys.argv[4]
    news_tweets_path = sys.argv[5]
    if len(sys.argv) == 6:
        newsfeed_path = sys.argv[6]
    else:
        newsfeed_path = None

    #@todo: create tmpdir
    tmpdir = './tmp'

    # Read newsfeed pickled tweets
    if newsfeed_path is not None:
        # @todo remove if tweets_path exists
        convert_tweets(newsfeed_path, tweets_path)

    # Filter Based on Language

    # Preprocess Text

    #@todo remove tmpdir
<commit_msg>added hashbang and call to main<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Authors: Luis Rei <luis.rei@ijs.si>
# License: MIT License

"""
Generate a POS/NEG/NEUTRAL sentiment dataset from LD-JSON tweets (e.g. sample)
and collected news tweets also in LD-JSON format.

0 - optionally process newsfeed tweets into LD-JSON lines
1 - Create tweets.nf.lang.txt: LD tweet text for a given lang
2 - Create tweets.lang.txt: Filter through langid.py and ldig
3 - Create tweets.pp.lang.tx: Preprocessing - Tokenization and Lower-Casing
4 - Create pos.lang.txt / neg.lang.txt / back.lang.txt:
    filter emoticons with a probability of keeping them in
5 - Create neutal.lang.txt from news_tweets
"""

import sys
import os
from newsfeed_tweets import convert_tweets

def main():
    cmd_name = sys.argv[0]

    # usage
    if len(sys.argv) not in [5, 6]:
        usage = 'Usage:\n\t%{cmd} ' \
                'lang_code prob_smiley min_langid_prob ' \
                'tweets_path tweet_news_path [news_feed_path]'
        usage.format(cmd=cmd_name)
        print usage
        sys.exit(0)

    lang_code = sys.argv[1]
    prob_smiley = float(sys.argv[2])
    min_langid_prob = float(sys.argv[3])
    tweets_path = sys.argv[4]
    news_tweets_path = sys.argv[5]
    if len(sys.argv) == 6:
        newsfeed_path = sys.argv[6]
    else:
        newsfeed_path = None

    #@todo: create tmpdir
    tmpdir = './tmp'

    # Read newsfeed pickled tweets
    if newsfeed_path is not None:
        # @todo remove if tweets_path exists
        convert_tweets(newsfeed_path, tweets_path)

    # Filter Based on Language

    # Preprocess Text

    #@todo remove tmpdir

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
tabimport is a Python utility to ease imports of tabular data from CSV, ODF,
XLS or XLSX files.

Usage:
>>> smart_file = FileFactory(file_path)
>>> for data_line in smart_file:
>>>     do_something_with(data_line['header'])

"""
import csv
import logging
import tempfile
from datetime import datetime

try:
    # XLS/XLSX format
    # http://www.lexicon.net/sjmachin/xlrd.html
    import xlrd
    has_xlrd = True
except ImportError:
    has_xlrd = False

try:
    # ODF format
    import ooolib
    has_ooolib = True
except ImportError:
    has_ooolib = False

from django.utils.datastructures import SortedDict
from django.utils.translation import ugettext as _

class UnsupportedFileFormat(Exception):
    pass

class HeaderError(Exception):
    pass

class FileFactory(object):
    """ Returns a file object depending on the file format """
    def __new__(cls, datafile, **imp_params):
        format = cls._sniff_format(datafile)
        if format == 'ods':
            return ODSImportedFile(datafile, **imp_params)
        elif format in ('xls', 'xlsx'):
            return XLSImportedFile(datafile, **imp_params)
        elif format == 'csv':
            return CSVImportedFile(datafile, **imp_params)
        else:
            raise UnsupportedFileFormat(_("Unknown file extension '%s'") % format)

    @classmethod
    def _sniff_format(cls, dfile):
        """ dfile may be a file path or a django (Uploaded)File object """
        if isinstance(dfile, basestring):
            format = dfile.rsplit('.', 1)[-1]
        else:
            if "opendocument.spreadsheet" in dfile.content_type or dfile.name.endswith(".ods"):
                format = "ods"
            elif dfile.name.endswith(".xlsx"):
                format = "xlsx"
            elif "excel" in dfile.content_type or dfile.name.endswith(".xls"):
                format = "xls"
            elif "text/csv" in dfile.content_type or dfile.name.endswith(".csv"):
                format = "csv"
            else:
                raise UnsupportedFileFormat(_("This file is not in an accepted format (ods, xls, csv)"))
        return format

class ImportedFile(object):
    """ Abstract class to get file object in different formats """
    # Set to True if the external lib does not support file-like objects
    force_file_to_disk = False

    def __init__(self, datafile, sheet_index=0, skip_lines=None):
        """ datafile can be either a path (string) or a file object
            sheet_index is the spreadsheet index, if applicable
            skip_lines is a list of row indexes to skip """
        # Some internal variables are initialized in activate_sheet so as they
        # are resetted whenever a new sheet is activated
        self._headers = {} # dict of lists
        self._ignored_headers_idx = {} # dict of lists
        self.data_sheet_indexes = [sheet_index]
        self.skip_lines = skip_lines
        self.file_content = None
        if isinstance(datafile, basestring):
            self.file_path = datafile
        else:
            try:
                self.file_path = datafile.temporary_file_path()
            except AttributeError:
                try:
                    self.file_path = datafile.file.name
                except AttributeError:
                    if self.force_file_to_disk:
                        # set to self, so it is not removed before the instance is deleted
                        self.temp_f = tempfile.NamedTemporaryFile()
                        for data in datafile.chunks():
                            self.temp_f.write(data)
                        assert self.temp_f.tell() == datafile.size
                        self.file_path = self.temp_f.name
                    else:
                        self.file_path = None
                        self.file_content = datafile.file.read()

    def __iter__(self):
        return self

    def next(self):
        raise NotImplementedError("Abstract class")

    def get_headers(self):
        raise NotImplementedError("Abstract class")

    def activate_sheet(self, idx):
        self.current_index = idx
        self._row_index = 1 # skip first line

    def current_sheet_name(self):
        raise NotImplementedError("Abstract class")

    def check_header_validity(self, possible_headers, mandatory_headers, case_sensitive=False): # TODO: ignore_pattern
        """ This method has the side effect of swallowing the first line (headers) of the file """
        errors = []
        warnings = [] # List of tuples: (sheet name, header name, motive)
        def idx_to_header(idx):
            return (idx>25 and chr(64 + idx/26) or '') + chr(65 + idx%26)
        for sheet_idx in self.data_sheet_indexes:
            self.activate_sheet(sheet_idx)
            good_headers = set()
            if not case_sensitive:
                possible_headers = [h.lower() for h in possible_headers]
            for i, h in enumerate(self.get_headers()):
                h_norm = case_sensitive and h.strip() or h.strip().lower()
                if h_norm == "":
                    warnings.append(
                        (self.current_sheet_name(),
                         "(%s header empty)" % idx_to_header(i),
                         _("Empty header")))
                    self._ignored_headers_idx[self.current_index].append(i)
                elif h_norm not in possible_headers:
                    warnings.append((self.current_sheet_name(), h, _("Unknown header")))
                    self._ignored_headers_idx[self.current_index].append(i)
                elif h in good_headers:
                    errors.append(_("The column '%s' is twice in your file's headers (sheet '%s')") % (h, self.current_sheet_name()))
                else:
                    good_headers.add(h)
            for h in mandatory_headers:
                if h not in good_headers:
                    errors.append(_("The header '%s' is mandatory and is missing in sheet '%s' of your file") % (h, self.current_sheet_name()))
        self.activate_sheet(0)
        if errors:
            raise HeaderError(u"\n".join(errors))
        return warnings

class CSVImportedFile(ImportedFile):
    encoding = 'latin-1'
    def __init__(self, datafile, sheet_index=0, **kwds):
        super(CSVImportedFile, self).__init__(datafile, sheet_index)
        if isinstance(datafile, basestring):
            # if datafile is a path, try to open the file
            datafile = open(datafile, 'r')
        try:
            dialect = csv.Sniffer().sniff(datafile.read(2048))
            # Python 2.4 csv module weakness ?
            if not dialect.delimiter or dialect.delimiter == '\r':
                dialect.delimiter = ";"
        except Exception as e:
            dialect = csv.excel
            dialect.delimiter = ";"
        self.delimiter = dialect.delimiter
        datafile.seek(0)
        datafile = UTF8Recoder(datafile, self.encoding)
        self.reader = csv.DictReader(datafile, dialect=dialect, **kwds)
        # It may happen that fieldnames are not filled before first line has been read
        self._first_line_read = False
        self._first_line = None

    def get_headers(self):
        if not self.current_index in self._headers:
            self._ignored_headers_idx[self.current_index] = []
            if not self._first_line_read:
                self._first_line = self.reader.next()
                self._first_line_read = True
            self._headers[self.current_index] = self.reader.fieldnames
            if not isinstance(self._headers[self.current_index], list):
                self._headers[self.current_index] = self._headers[self.current_index].split(self.delimiter)
        return self._headers[self.current_index]

    def next(self):
        """ Returns a SortedDict : {'DESCRIPTOR': value, ...} """
        if self._first_line:
            row = self._first_line
            self._first_line = None
        else:
            row = self.reader.next()
        for key, val in row.items():
            if val is None: val = ""
            row[key] = unicode(val, "utf-8")
        return row

    def current_sheet_name(self):
        return u"1"

class XLSImportedFile(ImportedFile):
    """ XLS reader based on xlrd (multiple sheets not yet implemented)"""
    def __init__(self, datafile, sheet_index=0, skip_lines=None):
        if not has_xlrd:
            raise NotImplementedError("The xlrd library is not available")
        super(XLSImportedFile, self).__init__(datafile, sheet_index, skip_lines)
        try:
            self.book = xlrd.open_workbook(filename=self.file_path, file_contents=self.file_content)
        except xlrd.XLRDError as e:
            logging.warn("XLS import error: %s" % str(e))
            raise UnsupportedFileFormat(_(u"Unable to read the file. Are you sure it is an XLS file?"))
        self.data_sheet_indexes = [i for i, ws in enumerate(self.book.sheets()) if (ws.nrows > 0 and ws.ncols > 0)]
        self.activate_sheet(self.data_sheet_indexes[0])

    def get_headers(self):
        if not self.current_index in self._headers:
            self._headers[self.current_index] = []
            self._ignored_headers_idx[self.current_index] = []
            row = self.current_sheet.row(0)
            for i, cell in enumerate(row):
                self._headers[self.current_index].append(unicode(cell.value).strip())
        return self._headers[self.current_index]

    def next(self):
        """ Returns a SortedDict : {'DESCRIPTOR': value, ...} """
        while self.skip_lines and self._row_index in self.skip_lines:
            self._row_index += 1
        if self._row_index >= self._nrows:
            # Increment current_index and skip to next sheet, if any
            new_index = self.data_sheet_indexes.index(self.current_index)+1
            if new_index < len(self.book.sheets()):
                self.activate_sheet(new_index)
                return self.next()
            raise StopIteration
        row_dict = SortedDict()
        row = self.current_sheet.row(self._row_index)
        headers = self.get_headers()
        for i, cell in enumerate(row):
            if i in self._ignored_headers_idx[self.current_index]:
                continue
            if cell.ctype == xlrd.XL_CELL_DATE:
                value = datetime(*xlrd.xldate_as_tuple(cell.value, self.book.datemode))
            else:
                value = cell.value
            row_dict[headers[i]] = value
        self._row_index += 1
        return row_dict

    def activate_sheet(self, idx):
        super(XLSImportedFile, self).activate_sheet(idx)
        self.current_sheet = self.book.sheet_by_index(idx)
        self._nrows = self.current_sheet.nrows
        self._ncols = self.current_sheet.ncols

    def current_sheet_name(self):
        return self.book.sheet_by_index(self.current_index).name

class ODSImportedFile(ImportedFile):
    """ OO Calc reader based on ooolib (multiple sheets not yet implemented)"""
    # FIXME: missing multi-sheet import
    def __init__(self, datafile, sheet_index=0, skip_lines=None):
        if not has_ooolib:
            raise NotImplementedError("The ooolib library is not available")
        super(ODSImportedFile, self).__init__(datafile, sheet_index, skip_lines)
        book = ooolib.Calc(opendoc=self.file_path)
        book.set_sheet_index(sheet_index)
        self.current_sheet = book
        (self._ncols, self._nrows) = self.current_sheet.get_sheet_dimensions()

    def get_headers(self):
        if not self.current_index in self._headers:
            self._headers[self.current_index] = []
            self._ignored_headers_idx[self.current_index] = []
            for i in range(self._ncols):
                cell_value = self.current_sheet.get_cell_value(i+1, 1)
                if cell_value:
                    self._headers[self.current_index].append(self.current_sheet.get_cell_value(i+1, 1)[1])
                else:
                    logging.warn("Empty header in %s" % self.file_path)
                    self._headers[self.current_index].append("--empty--")
        return self._headers[self.current_index]

    def next(self):
        """ Returns a SortedDict : {'DESCRIPTOR': value, ...} """
        while self.skip_lines and self._row_index in self.skip_lines:
            self._row_index += 1
        if self._row_index >= self._nrows:
            raise StopIteration
        row_dict = SortedDict()
        for i in range(self._ncols):
            if i in self._ignored_headers_idx[self.current_index]:
                continue
            cell_value = self.current_sheet.get_cell_value(i+1, self._row_index+1)
            if cell_value and cell_value[0] == 'formula' and cell_value[1]:
                raise ValueError(_("The ODS file contains formula. Please convert them to raw values before importing the file."))
            row_dict[self.get_headers()[i]] = cell_value and cell_value[1] or u""
        self._row_index += 1
        return row_dict

    def current_sheet_name(self):
        return "??" # FIXME: self.current_sheet.title

<commit_msg>Abstract string type (Python 3 compatibility)<commit_after># -*- coding: utf-8 -*-
"""
tabimport is a Python utility to ease imports of tabular data from CSV, ODF,
XLS or XLSX files.

Usage:
>>> smart_file = FileFactory(file_path)
>>> for data_line in smart_file:
>>>     do_something_with(data_line['header'])

"""
import csv
import logging
import sys
import tempfile
from datetime import datetime

PY3 = sys.version_info >= (3, 0)
string_type = str if PY3 else basestring

try:
    # XLS/XLSX format
    # http://www.lexicon.net/sjmachin/xlrd.html
    import xlrd
    has_xlrd = True
except ImportError:
    has_xlrd = False

try:
    # ODF format
    import ooolib
    has_ooolib = True
except ImportError:
    has_ooolib = False

from django.utils.datastructures import SortedDict
from django.utils.translation import ugettext as _

class UnsupportedFileFormat(Exception):
    pass

class HeaderError(Exception):
    pass

class FileFactory(object):
    """ Returns a file object depending on the file format """
    def __new__(cls, datafile, **imp_params):
        format = cls._sniff_format(datafile)
        if format == 'ods':
            return ODSImportedFile(datafile, **imp_params)
        elif format in ('xls', 'xlsx'):
            return XLSImportedFile(datafile, **imp_params)
        elif format == 'csv':
            return CSVImportedFile(datafile, **imp_params)
        else:
            raise UnsupportedFileFormat(_("Unknown file extension '%s'") % format)

    @classmethod
    def _sniff_format(cls, dfile):
        """ dfile may be a file path or a django (Uploaded)File object """
        if isinstance(dfile, string_type):
            format = dfile.rsplit('.', 1)[-1]
        else:
            if "opendocument.spreadsheet" in dfile.content_type or dfile.name.endswith(".ods"):
                format = "ods"
            elif dfile.name.endswith(".xlsx"):
                format = "xlsx"
            elif "excel" in dfile.content_type or dfile.name.endswith(".xls"):
                format = "xls"
            elif "text/csv" in dfile.content_type or dfile.name.endswith(".csv"):
                format = "csv"
            else:
                raise UnsupportedFileFormat(_("This file is not in an accepted format (ods, xls, csv)"))
        return format

class ImportedFile(object):
    """ Abstract class to get file object in different formats """
    # Set to True if the external lib does not support file-like objects
    force_file_to_disk = False

    def __init__(self, datafile, sheet_index=0, skip_lines=None):
        """ datafile can be either a path (string) or a file object
            sheet_index is the spreadsheet index, if applicable
            skip_lines is a list of row indexes to skip """
        # Some internal variables are initialized in activate_sheet so as they
        # are resetted whenever a new sheet is activated
        self._headers = {} # dict of lists
        self._ignored_headers_idx = {} # dict of lists
        self.data_sheet_indexes = [sheet_index]
        self.skip_lines = skip_lines
        self.file_content = None
        if isinstance(datafile, string_type):
            self.file_path = datafile
        else:
            try:
                self.file_path = datafile.temporary_file_path()
            except AttributeError:
                try:
                    self.file_path = datafile.file.name
                except AttributeError:
                    if self.force_file_to_disk:
                        # set to self, so it is not removed before the instance is deleted
                        self.temp_f = tempfile.NamedTemporaryFile()
                        for data in datafile.chunks():
                            self.temp_f.write(data)
                        assert self.temp_f.tell() == datafile.size
                        self.file_path = self.temp_f.name
                    else:
                        self.file_path = None
                        self.file_content = datafile.file.read()

    def __iter__(self):
        return self

    def next(self):
        raise NotImplementedError("Abstract class")

    def get_headers(self):
        raise NotImplementedError("Abstract class")

    def activate_sheet(self, idx):
        self.current_index = idx
        self._row_index = 1 # skip first line

    def current_sheet_name(self):
        raise NotImplementedError("Abstract class")

    def check_header_validity(self, possible_headers, mandatory_headers, case_sensitive=False): # TODO: ignore_pattern
        """ This method has the side effect of swallowing the first line (headers) of the file """
        errors = []
        warnings = [] # List of tuples: (sheet name, header name, motive)
        def idx_to_header(idx):
            return (idx>25 and chr(64 + idx/26) or '') + chr(65 + idx%26)
        for sheet_idx in self.data_sheet_indexes:
            self.activate_sheet(sheet_idx)
            good_headers = set()
            if not case_sensitive:
                possible_headers = [h.lower() for h in possible_headers]
            for i, h in enumerate(self.get_headers()):
                h_norm = case_sensitive and h.strip() or h.strip().lower()
                if h_norm == "":
                    warnings.append(
                        (self.current_sheet_name(),
                         "(%s header empty)" % idx_to_header(i),
                         _("Empty header")))
                    self._ignored_headers_idx[self.current_index].append(i)
                elif h_norm not in possible_headers:
                    warnings.append((self.current_sheet_name(), h, _("Unknown header")))
                    self._ignored_headers_idx[self.current_index].append(i)
                elif h in good_headers:
                    errors.append(_("The column '%s' is twice in your file's headers (sheet '%s')") % (h, self.current_sheet_name()))
                else:
                    good_headers.add(h)
            for h in mandatory_headers:
                if h not in good_headers:
                    errors.append(_("The header '%s' is mandatory and is missing in sheet '%s' of your file") % (h, self.current_sheet_name()))
        self.activate_sheet(0)
        if errors:
            raise HeaderError(u"\n".join(errors))
        return warnings

class CSVImportedFile(ImportedFile):
    encoding = 'latin-1'
    def __init__(self, datafile, sheet_index=0, **kwds):
        super(CSVImportedFile, self).__init__(datafile, sheet_index)
        if isinstance(datafile, string_type):
            # if datafile is a path, try to open the file
            datafile = open(datafile, 'r')
        try:
            dialect = csv.Sniffer().sniff(datafile.read(2048))
            # Python 2.4 csv module weakness ?
            if not dialect.delimiter or dialect.delimiter == '\r':
                dialect.delimiter = ";"
        except Exception as e:
            dialect = csv.excel
            dialect.delimiter = ";"
        self.delimiter = dialect.delimiter
        datafile.seek(0)
        datafile = UTF8Recoder(datafile, self.encoding)
        self.reader = csv.DictReader(datafile, dialect=dialect, **kwds)
        # It may happen that fieldnames are not filled before first line has been read
        self._first_line_read = False
        self._first_line = None

    def get_headers(self):
        if not self.current_index in self._headers:
            self._ignored_headers_idx[self.current_index] = []
            if not self._first_line_read:
                self._first_line = self.reader.next()
                self._first_line_read = True
            self._headers[self.current_index] = self.reader.fieldnames
            if not isinstance(self._headers[self.current_index], list):
                self._headers[self.current_index] = self._headers[self.current_index].split(self.delimiter)
        return self._headers[self.current_index]

    def next(self):
        """ Returns a SortedDict : {'DESCRIPTOR': value, ...} """
        if self._first_line:
            row = self._first_line
            self._first_line = None
        else:
            row = self.reader.next()
        for key, val in row.items():
            if val is None: val = ""
            row[key] = unicode(val, "utf-8")
        return row

    def current_sheet_name(self):
        return u"1"

class XLSImportedFile(ImportedFile):
    """ XLS reader based on xlrd (multiple sheets not yet implemented)"""
    def __init__(self, datafile, sheet_index=0, skip_lines=None):
        if not has_xlrd:
            raise NotImplementedError("The xlrd library is not available")
        super(XLSImportedFile, self).__init__(datafile, sheet_index, skip_lines)
        try:
            self.book = xlrd.open_workbook(filename=self.file_path, file_contents=self.file_content)
        except xlrd.XLRDError as e:
            logging.warn("XLS import error: %s" % str(e))
            raise UnsupportedFileFormat(_(u"Unable to read the file. Are you sure it is an XLS file?"))
        self.data_sheet_indexes = [i for i, ws in enumerate(self.book.sheets()) if (ws.nrows > 0 and ws.ncols > 0)]
        self.activate_sheet(self.data_sheet_indexes[0])

    def get_headers(self):
        if not self.current_index in self._headers:
            self._headers[self.current_index] = []
            self._ignored_headers_idx[self.current_index] = []
            row = self.current_sheet.row(0)
            for i, cell in enumerate(row):
                self._headers[self.current_index].append(unicode(cell.value).strip())
        return self._headers[self.current_index]

    def next(self):
        """ Returns a SortedDict : {'DESCRIPTOR': value, ...} """
        while self.skip_lines and self._row_index in self.skip_lines:
            self._row_index += 1
        if self._row_index >= self._nrows:
            # Increment current_index and skip to next sheet, if any
            new_index = self.data_sheet_indexes.index(self.current_index)+1
            if new_index < len(self.book.sheets()):
                self.activate_sheet(new_index)
                return self.next()
            raise StopIteration
        row_dict = SortedDict()
        row = self.current_sheet.row(self._row_index)
        headers = self.get_headers()
        for i, cell in enumerate(row):
            if i in self._ignored_headers_idx[self.current_index]:
                continue
            if cell.ctype == xlrd.XL_CELL_DATE:
                value = datetime(*xlrd.xldate_as_tuple(cell.value, self.book.datemode))
            else:
                value = cell.value
            row_dict[headers[i]] = value
        self._row_index += 1
        return row_dict

    def activate_sheet(self, idx):
        super(XLSImportedFile, self).activate_sheet(idx)
        self.current_sheet = self.book.sheet_by_index(idx)
        self._nrows = self.current_sheet.nrows
        self._ncols = self.current_sheet.ncols

    def current_sheet_name(self):
        return self.book.sheet_by_index(self.current_index).name

class ODSImportedFile(ImportedFile):
    """ OO Calc reader based on ooolib (multiple sheets not yet implemented)"""
    # FIXME: missing multi-sheet import
    def __init__(self, datafile, sheet_index=0, skip_lines=None):
        if not has_ooolib:
            raise NotImplementedError("The ooolib library is not available")
        super(ODSImportedFile, self).__init__(datafile, sheet_index, skip_lines)
        book = ooolib.Calc(opendoc=self.file_path)
        book.set_sheet_index(sheet_index)
        self.current_sheet = book
        (self._ncols, self._nrows) = self.current_sheet.get_sheet_dimensions()

    def get_headers(self):
        if not self.current_index in self._headers:
            self._headers[self.current_index] = []
            self._ignored_headers_idx[self.current_index] = []
            for i in range(self._ncols):
                cell_value = self.current_sheet.get_cell_value(i+1, 1)
                if cell_value:
                    self._headers[self.current_index].append(self.current_sheet.get_cell_value(i+1, 1)[1])
                else:
                    logging.warn("Empty header in %s" % self.file_path)
                    self._headers[self.current_index].append("--empty--")
        return self._headers[self.current_index]

    def next(self):
        """ Returns a SortedDict : {'DESCRIPTOR': value, ...} """
        while self.skip_lines and self._row_index in self.skip_lines:
            self._row_index += 1
        if self._row_index >= self._nrows:
            raise StopIteration
        row_dict = SortedDict()
        for i in range(self._ncols):
            if i in self._ignored_headers_idx[self.current_index]:
                continue
            cell_value = self.current_sheet.get_cell_value(i+1, self._row_index+1)
            if cell_value and cell_value[0] == 'formula' and cell_value[1]:
                raise ValueError(_("The ODS file contains formula. Please convert them to raw values before importing the file."))
            row_dict[self.get_headers()[i]] = cell_value and cell_value[1] or u""
        self._row_index += 1
        return row_dict

    def current_sheet_name(self):
        return "??" # FIXME: self.current_sheet.title

<|endoftext|>


<commit_before># (c) 2015, Ansible Inc,
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.action import ActionBase

class ActionModule(ActionBase):

    TRANSFERS_FILES = True

    def run(self, tmp=None, task_vars=dict()):
        ''' handler for package operations '''

        name  = self._task.args.get('name', None)
        state = self._task.args.get('state', None)
        module = self._task.args.get('use', None)

        if module is None:
            try:
                module = self._templar.template('{{ansible_pkg_mgr}}')
            except:
                pass # could not get it from template!

        if module is None:
            #TODO: autodetect the package manager, by invoking that specific fact snippet remotely
            pass

        if module is not None:
            # run the 'package' module
            new_module_args = self._task.args.copy()
            if 'use' in new_module_args:
                del new_module_args['use']

            return self._execute_module(module_name=module, module_args=new_module_args, task_vars=task_vars)

        else:

            return {'failed': True, 'msg': 'Could not detect which package manager to use. Try gathering facts or setting the "use" option.'}
<commit_msg>changed to default 'auto' as it better describes the use= option<commit_after># (c) 2015, Ansible Inc,
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.action import ActionBase

class ActionModule(ActionBase):

    TRANSFERS_FILES = True

    def run(self, tmp=None, task_vars=dict()):
        ''' handler for package operations '''

        name  = self._task.args.get('name', None)
        state = self._task.args.get('state', None)
        module = self._task.args.get('use', 'auto')

        if module == 'auto':
            try:
                module = self._templar.template('{{ansible_pkg_mgr}}')
            except:
                pass # could not get it from template!

        if module == 'auto':
            #FIXME: autodetect the package manager run facts module remotely to get ansible_pkg_mgr
            #module = self._execute_module(module_name=setup, module_args={filter: 'ansible_pkg_mgr'}, task_vars=task_vars)
            pass

        if module != 'auto':
            # run the 'package' module
            new_module_args = self._task.args.copy()
            if 'use' in new_module_args:
                del new_module_args['use']

            return self._execute_module(module_name=module, module_args=new_module_args, task_vars=task_vars)

        else:

            return {'failed': True, 'msg': 'Could not detect which package manager to use. Try gathering facts or setting the "use" option.'}
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8

import json
import math
import os
import time
from collections import OrderedDict

import tensorflow as tf

from utils import SEP_TOKEN, PAD_TOKEN, VOCAB_SIZE, MODEL_DIR
from data_utils import gen_batch_train_data
from seq2seq_model import Seq2SeqModel
from word2vec import get_word_embedding

# Data loading parameters
tf.app.flags.DEFINE_string('source_vocabulary', 'data/europarl-v7.1.4M.de.json', 'Path to source vocabulary')
tf.app.flags.DEFINE_string('target_vocabulary', 'data/europarl-v7.1.4M.fr.json', 'Path to target vocabulary')
tf.app.flags.DEFINE_string('source_train_data', 'data/europarl-v7.1.4M.de', 'Path to source training data')
tf.app.flags.DEFINE_string('target_train_data', 'data/europarl-v7.1.4M.fr', 'Path to target training data')
tf.app.flags.DEFINE_string('source_valid_data', 'data/newstest2012.bpe.de', 'Path to source validation data')
tf.app.flags.DEFINE_string('target_valid_data', 'data/newstest2012.bpe.fr', 'Path to target validation data')

# Network parameters
tf.app.flags.DEFINE_string('cell_type', 'lstm', 'RNN cell for encoder and decoder, default: lstm')
tf.app.flags.DEFINE_string('attention_type', 'bahdanau', 'Attention mechanism: (bahdanau, luong), default: bahdanau')
tf.app.flags.DEFINE_integer('hidden_units', 128, 'Number of hidden units in each layer')
tf.app.flags.DEFINE_integer('depth', 4, 'Number of layers in each encoder and decoder')
tf.app.flags.DEFINE_integer('embedding_size', 128, 'Embedding dimensions of encoder and decoder inputs')
tf.app.flags.DEFINE_integer('num_encoder_symbols', 30000, 'Source vocabulary size')
tf.app.flags.DEFINE_integer('num_decoder_symbols', 30000, 'Target vocabulary size')
# NOTE(sdsuo): We used the same vocab for source and target
tf.app.flags.DEFINE_integer('vocab_size', VOCAB_SIZE, 'General vocabulary size')

tf.app.flags.DEFINE_boolean('use_residual', True, 'Use residual connection between layers')
tf.app.flags.DEFINE_boolean('attn_input_feeding', False, 'Use input feeding method in attentional decoder')
tf.app.flags.DEFINE_boolean('use_dropout', True, 'Use dropout in each rnn cell')
tf.app.flags.DEFINE_float('dropout_rate', 0.3, 'Dropout probability for input/output/state units (0.0: no dropout)')

# Training parameters
tf.app.flags.DEFINE_float('learning_rate', 0.0002, 'Learning rate')
tf.app.flags.DEFINE_float('max_gradient_norm', 1.0, 'Clip gradients to this norm')
tf.app.flags.DEFINE_integer('batch_size', 64, 'Batch size')
tf.app.flags.DEFINE_integer('max_epochs', 100000, 'Maximum # of training epochs')
tf.app.flags.DEFINE_integer('max_load_batches', 20, 'Maximum # of batches to load at one time')
tf.app.flags.DEFINE_integer('max_seq_length', 50, 'Maximum sequence length')
tf.app.flags.DEFINE_integer('display_freq', 100, 'Display training status every this iteration')
tf.app.flags.DEFINE_integer('save_freq', 100, 'Save model checkpoint every this iteration')
tf.app.flags.DEFINE_integer('valid_freq', 1150000, 'Evaluate model every this iteration: valid_data needed')
tf.app.flags.DEFINE_string('optimizer', 'adam', 'Optimizer for training: (adadelta, adam, rmsprop)')
tf.app.flags.DEFINE_string('model_dir', MODEL_DIR, 'Path to save model checkpoints')
tf.app.flags.DEFINE_string('summary_dir', 'model/summary', 'Path to save model summary')
tf.app.flags.DEFINE_string('model_name', 'translate.ckpt', 'File name used for model checkpoints')
tf.app.flags.DEFINE_boolean('shuffle_each_epoch', True, 'Shuffle training dataset for each epoch')
tf.app.flags.DEFINE_boolean('sort_by_length', True, 'Sort pre-fetched minibatches by their target sequence lengths')
tf.app.flags.DEFINE_boolean('use_fp16', False, 'Use half precision float16 instead of float32 as dtype')
tf.app.flags.DEFINE_boolean('bidirectional', True, 'Use bidirectional encoder')
tf.app.flags.DEFINE_string('train_mode', 'ground_truth', 'Decode helper to use for training')
tf.app.flags.DEFINE_string('sampling_probability', 0.1, 'Probability of sampling from decoder output instead of using ground truth')

# TODO(sdsuo): Make start token and end token more robust
tf.app.flags.DEFINE_integer('start_token', SEP_TOKEN, 'Start token')
tf.app.flags.DEFINE_integer('end_token', PAD_TOKEN, 'End token')

# Runtime parameters
tf.app.flags.DEFINE_boolean('allow_soft_placement', True, 'Allow device soft placement')
tf.app.flags.DEFINE_boolean('log_device_placement', False, 'Log placement of ops on devices')

FLAGS = tf.app.flags.FLAGS

def load_or_create_model(sess, model, saver, FLAGS):
    ckpt = tf.train.get_checkpoint_state(FLAGS.model_dir)
    if ckpt and tf.train.checkpoint_exists(ckpt.model_checkpoint_path):
        print 'Reloading model parameters...'
        model.restore(sess, saver, ckpt.model_checkpoint_path)
    else:
        if not os.path.exists(FLAGS.model_dir):
            os.makedirs(FLAGS.model_dir)
        print 'Created new model parameters...'
        sess.run(tf.global_variables_initializer())

def train():
    config_proto = tf.ConfigProto(
        allow_soft_placement=FLAGS.allow_soft_placement,
        log_device_placement=FLAGS.log_device_placement,
        gpu_options=tf.GPUOptions(allow_growth=True)
    )

    with tf.Session(config=config_proto) as sess:
        # Build the model
        config = OrderedDict(sorted(FLAGS.__flags.items()))
        model = Seq2SeqModel(config, 'train')

        # Create a log writer object
        log_writer = tf.summary.FileWriter(FLAGS.model_dir, graph=sess.graph)

        # Create a saver
        # Using var_list = None returns the list of all saveable variables
        saver = tf.train.Saver(var_list=None)

        # Initiaize global variables or reload existing checkpoint
        load_or_create_model(sess, model, saver, FLAGS)

        # Load word2vec embedding
        embedding = get_word_embedding(FLAGS.hidden_units)
        model.init_vars(sess, embedding=embedding)

        step_time, loss = 0.0, 0.0
        sents_seen = 0

        start_time = time.time()

        print 'Training...'
        for epoch_idx in xrange(FLAGS.max_epochs):
            if model.global_epoch_step.eval() >= FLAGS.max_epochs:
                print 'Training is already complete.', \
                      'Current epoch: {}, Max epoch: {}'.format(model.global_epoch_step.eval(), FLAGS.max_epochs)
                break

            # Prepare batch training data
            # TODO(sdsuo): Make corresponding changes in data_utils
            for source, source_len, target, target_len in gen_batch_train_data(FLAGS.batch_size, prev=True, rev=True, align=True):
                step_loss, summary = model.train(
                    sess,
                    encoder_inputs=source,
                    encoder_inputs_length=source_len,
                    decoder_inputs=target,
                    decoder_inputs_length=target_len
                )

                loss += float(step_loss) / FLAGS.display_freq
                sents_seen += float(source.shape[0]) # batch_size

                # Display information
                if model.global_step.eval() % FLAGS.display_freq == 0:

                    avg_perplexity = math.exp(float(loss)) if loss < 300 else float("inf")

                    time_elapsed = time.time() - start_time
                    step_time = time_elapsed / FLAGS.display_freq

                    sents_per_sec = sents_seen / time_elapsed

                    print 'Epoch ', model.global_epoch_step.eval(), 'Step ', model.global_step.eval(), \
                          'Perplexity {0:.2f}'.format(avg_perplexity), 'Step-time ', step_time, \
                          '{0:.2f} sents/s'.format(sents_per_sec)

                    loss = 0
                    sents_seen = 0
                    start_time = time.time()

                    # Record training summary for the current batch
                    log_writer.add_summary(summary, model.global_step.eval())

                # Save the model checkpoint
                if model.global_step.eval() % FLAGS.save_freq == 0:
                    print 'Saving the model..'
                    checkpoint_path = os.path.join(FLAGS.model_dir, FLAGS.model_name)
                    model.save(sess, saver, checkpoint_path, global_step=model.global_step)
                    json.dump(model.config,
                              open('%s-%d.json' % (checkpoint_path, model.global_step.eval()), 'wb'),
                              indent=2)

            # Increase the epoch index of the model
            model.increment_global_epoch_step_op.eval()
            print 'Epoch {0:} DONE'.format(model.global_epoch_step.eval())

        print 'Saving the last model'
        checkpoint_path = os.path.join(FLAGS.model_dir, FLAGS.model_name)
        model.save(sess, saver, checkpoint_path, global_step=model.global_step)
        json.dump(model.config,
                  open('%s-%d.json' % (checkpoint_path, model.global_step.eval()), 'wb'),
                  indent=2)

    print 'Training terminated'

def main(_):
    train()

if __name__ == '__main__':
    tf.app.run()
<commit_msg>add default use word2vec with alignment<commit_after>#!/usr/bin/env python
# coding: utf-8

import json
import math
import os
import time
from collections import OrderedDict

import tensorflow as tf

from utils import SEP_TOKEN, PAD_TOKEN, VOCAB_SIZE, MODEL_DIR
from data_utils import gen_batch_train_data
from seq2seq_model import Seq2SeqModel
from word2vec import get_word_embedding

# Data loading parameters
tf.app.flags.DEFINE_string('source_vocabulary', 'data/europarl-v7.1.4M.de.json', 'Path to source vocabulary')
tf.app.flags.DEFINE_string('target_vocabulary', 'data/europarl-v7.1.4M.fr.json', 'Path to target vocabulary')
tf.app.flags.DEFINE_string('source_train_data', 'data/europarl-v7.1.4M.de', 'Path to source training data')
tf.app.flags.DEFINE_string('target_train_data', 'data/europarl-v7.1.4M.fr', 'Path to target training data')
tf.app.flags.DEFINE_string('source_valid_data', 'data/newstest2012.bpe.de', 'Path to source validation data')
tf.app.flags.DEFINE_string('target_valid_data', 'data/newstest2012.bpe.fr', 'Path to target validation data')

# Network parameters
tf.app.flags.DEFINE_string('cell_type', 'lstm', 'RNN cell for encoder and decoder, default: lstm')
tf.app.flags.DEFINE_string('attention_type', 'bahdanau', 'Attention mechanism: (bahdanau, luong), default: bahdanau')
tf.app.flags.DEFINE_integer('hidden_units', 128, 'Number of hidden units in each layer')
tf.app.flags.DEFINE_integer('depth', 4, 'Number of layers in each encoder and decoder')
tf.app.flags.DEFINE_integer('embedding_size', 128, 'Embedding dimensions of encoder and decoder inputs')
tf.app.flags.DEFINE_integer('num_encoder_symbols', 30000, 'Source vocabulary size')
tf.app.flags.DEFINE_integer('num_decoder_symbols', 30000, 'Target vocabulary size')
# NOTE(sdsuo): We used the same vocab for source and target
tf.app.flags.DEFINE_integer('vocab_size', VOCAB_SIZE, 'General vocabulary size')

tf.app.flags.DEFINE_boolean('use_residual', True, 'Use residual connection between layers')
tf.app.flags.DEFINE_boolean('attn_input_feeding', False, 'Use input feeding method in attentional decoder')
tf.app.flags.DEFINE_boolean('use_dropout', True, 'Use dropout in each rnn cell')
tf.app.flags.DEFINE_float('dropout_rate', 0.3, 'Dropout probability for input/output/state units (0.0: no dropout)')

# Training parameters
tf.app.flags.DEFINE_float('learning_rate', 0.0002, 'Learning rate')
tf.app.flags.DEFINE_float('max_gradient_norm', 1.0, 'Clip gradients to this norm')
tf.app.flags.DEFINE_integer('batch_size', 64, 'Batch size')
tf.app.flags.DEFINE_integer('max_epochs', 100000, 'Maximum # of training epochs')
tf.app.flags.DEFINE_integer('max_load_batches', 20, 'Maximum # of batches to load at one time')
tf.app.flags.DEFINE_integer('max_seq_length', 50, 'Maximum sequence length')
tf.app.flags.DEFINE_integer('display_freq', 100, 'Display training status every this iteration')
tf.app.flags.DEFINE_integer('save_freq', 100, 'Save model checkpoint every this iteration')
tf.app.flags.DEFINE_integer('valid_freq', 1150000, 'Evaluate model every this iteration: valid_data needed')
tf.app.flags.DEFINE_string('optimizer', 'adam', 'Optimizer for training: (adadelta, adam, rmsprop)')
tf.app.flags.DEFINE_string('model_dir', MODEL_DIR, 'Path to save model checkpoints')
tf.app.flags.DEFINE_string('summary_dir', 'model/summary', 'Path to save model summary')
tf.app.flags.DEFINE_string('model_name', 'translate.ckpt', 'File name used for model checkpoints')
tf.app.flags.DEFINE_boolean('shuffle_each_epoch', True, 'Shuffle training dataset for each epoch')
tf.app.flags.DEFINE_boolean('sort_by_length', True, 'Sort pre-fetched minibatches by their target sequence lengths')
tf.app.flags.DEFINE_boolean('use_fp16', False, 'Use half precision float16 instead of float32 as dtype')
tf.app.flags.DEFINE_boolean('bidirectional', True, 'Use bidirectional encoder')
tf.app.flags.DEFINE_string('train_mode', 'ground_truth', 'Decode helper to use for training')
tf.app.flags.DEFINE_string('sampling_probability', 0.1, 'Probability of sampling from decoder output instead of using ground truth')

# TODO(sdsuo): Make start token and end token more robust
tf.app.flags.DEFINE_integer('start_token', SEP_TOKEN, 'Start token')
tf.app.flags.DEFINE_integer('end_token', PAD_TOKEN, 'End token')

# Runtime parameters
tf.app.flags.DEFINE_boolean('allow_soft_placement', True, 'Allow device soft placement')
tf.app.flags.DEFINE_boolean('log_device_placement', False, 'Log placement of ops on devices')

FLAGS = tf.app.flags.FLAGS

def load_or_create_model(sess, model, saver, FLAGS):
    ckpt = tf.train.get_checkpoint_state(FLAGS.model_dir)
    if ckpt and tf.train.checkpoint_exists(ckpt.model_checkpoint_path):
        print 'Reloading model parameters...'
        model.restore(sess, saver, ckpt.model_checkpoint_path)
    else:
        if not os.path.exists(FLAGS.model_dir):
            os.makedirs(FLAGS.model_dir)
        print 'Created new model parameters...'
        sess.run(tf.global_variables_initializer())

def train():
    config_proto = tf.ConfigProto(
        allow_soft_placement=FLAGS.allow_soft_placement,
        log_device_placement=FLAGS.log_device_placement,
        gpu_options=tf.GPUOptions(allow_growth=True)
    )

    with tf.Session(config=config_proto) as sess:
        # Build the model
        config = OrderedDict(sorted(FLAGS.__flags.items()))
        model = Seq2SeqModel(config, 'train')

        # Create a log writer object
        log_writer = tf.summary.FileWriter(FLAGS.model_dir, graph=sess.graph)

        # Create a saver
        # Using var_list = None returns the list of all saveable variables
        saver = tf.train.Saver(var_list=None)

        # Initiaize global variables or reload existing checkpoint
        load_or_create_model(sess, model, saver, FLAGS)

        # Load word2vec embedding
        embedding = get_word_embedding(FLAGS.hidden_units, alignment=True)
        model.init_vars(sess, embedding=embedding)

        step_time, loss = 0.0, 0.0
        sents_seen = 0

        start_time = time.time()

        print 'Training...'
        for epoch_idx in xrange(FLAGS.max_epochs):
            if model.global_epoch_step.eval() >= FLAGS.max_epochs:
                print 'Training is already complete.', \
                      'Current epoch: {}, Max epoch: {}'.format(model.global_epoch_step.eval(), FLAGS.max_epochs)
                break

            # Prepare batch training data
            # TODO(sdsuo): Make corresponding changes in data_utils
            for source, source_len, target, target_len in gen_batch_train_data(FLAGS.batch_size, prev=True, rev=True, align=True):
                step_loss, summary = model.train(
                    sess,
                    encoder_inputs=source,
                    encoder_inputs_length=source_len,
                    decoder_inputs=target,
                    decoder_inputs_length=target_len
                )

                loss += float(step_loss) / FLAGS.display_freq
                sents_seen += float(source.shape[0]) # batch_size

                # Display information
                if model.global_step.eval() % FLAGS.display_freq == 0:

                    avg_perplexity = math.exp(float(loss)) if loss < 300 else float("inf")

                    time_elapsed = time.time() - start_time
                    step_time = time_elapsed / FLAGS.display_freq

                    sents_per_sec = sents_seen / time_elapsed

                    print 'Epoch ', model.global_epoch_step.eval(), 'Step ', model.global_step.eval(), \
                          'Perplexity {0:.2f}'.format(avg_perplexity), 'Step-time ', step_time, \
                          '{0:.2f} sents/s'.format(sents_per_sec)

                    loss = 0
                    sents_seen = 0
                    start_time = time.time()

                    # Record training summary for the current batch
                    log_writer.add_summary(summary, model.global_step.eval())

                # Save the model checkpoint
                if model.global_step.eval() % FLAGS.save_freq == 0:
                    print 'Saving the model..'
                    checkpoint_path = os.path.join(FLAGS.model_dir, FLAGS.model_name)
                    model.save(sess, saver, checkpoint_path, global_step=model.global_step)
                    json.dump(model.config,
                              open('%s-%d.json' % (checkpoint_path, model.global_step.eval()), 'wb'),
                              indent=2)

            # Increase the epoch index of the model
            model.increment_global_epoch_step_op.eval()
            print 'Epoch {0:} DONE'.format(model.global_epoch_step.eval())

        print 'Saving the last model'
        checkpoint_path = os.path.join(FLAGS.model_dir, FLAGS.model_name)
        model.save(sess, saver, checkpoint_path, global_step=model.global_step)
        json.dump(model.config,
                  open('%s-%d.json' % (checkpoint_path, model.global_step.eval()), 'wb'),
                  indent=2)

    print 'Training terminated'

def main(_):
    train()

if __name__ == '__main__':
    tf.app.run()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import division
from __future__ import print_function
from __future__ import absolute_import
from __future__ import unicode_literals

import io
import six
import json
from copy import deepcopy
from .profile import Profile
from .field import Field
from . import exceptions
from . import helpers
from . import config
from . import types

# Module API

class Schema(object):

    # Public

    def __init__(self, descriptor={}, strict=False):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """

        # Process descriptor
        descriptor = helpers.retrieve_descriptor(descriptor)

        # Set attributes
        self.__strict = strict
        self.__current_descriptor = deepcopy(descriptor)
        self.__next_descriptor = deepcopy(descriptor)
        self.__profile = Profile('table-schema')
        self.__errors = []
        self.__fields = []

        # Build instance
        self.__build()

    @property
    def valid(self):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        return not bool(self.__errors)

    @property
    def errors(self):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        return self.__errors

    @property
    def descriptor(self):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        # Never use this.descriptor inside this class (!!!)
        return self.__next_descriptor

    @property
    def primary_key(self):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        primary_key = self.__current_descriptor.get('primaryKey', [])
        if not isinstance(primary_key, list):
            primary_key = [primary_key]
        return primary_key

    @property
    def foreign_keys(self):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        foreign_keys = self.__current_descriptor.get('foreignKeys', [])
        for key in foreign_keys:
            key.setdefault('fields', [])
            key.setdefault('reference', {})
            key['reference'].setdefault('resource', '')
            key['reference'].setdefault('fields', [])
            if not isinstance(key['fields'], list):
                key['fields'] = [key['fields']]
            if not isinstance(key['reference']['fields'], list):
                key['reference']['fields'] = [key['reference']['fields']]
        return foreign_keys

    @property
    def fields(self):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        return self.__fields

    @property
    def field_names(self):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        return [field.name for field in self.fields]

    def get_field(self, name):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        for field in self.fields:
            if field.name == name:
                return field
        return None

    def add_field(self, descriptor):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        self.__next_descriptor.setdefault('fields', [])
        self.__next_descriptor['fields'].append(descriptor)
        self.commit()
        return self.__fields[-1]

    def remove_field(self, name):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        field = self.get_field(name)
        if field:
            predicat = lambda field: field.get('name') != name
            self.__next_descriptor['fields'] = filter(
                predicat, self.__next_descriptor['fields'])
            self.commit()
        return field

    def cast_row(self, row, fail_fast=False):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """

        # Prepare
        errors = []

        # Check row length
        if len(row) != len(self.fields):
            message = 'Row length (%s) doesn\'t match fields count (%s)'
            message = message % (len(row), len(self.fields))
            exception = exceptions.CastError(message)
            if fail_fast:
                raise exception
            errors.append(exception)

        # Cast
        result = []
        if not errors:
            for field, value in zip(self.fields, row):
                try:
                    result.append(field.cast_value(value))
                except exceptions.CastError as exception:
                    if fail_fast:
                        raise
                    errors.append(exception)

        # Raise
        if errors:
            message = 'There are %s cast errors (see exception.errors)' % len(errors)
            raise exceptions.CastError(message, errors=errors)

        return result

    def infer(self, rows, headers=1):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """

        # Get headers
        if not isinstance(headers, list):
            headers_row = headers
            while True:
                headers_row -= 1
                headers = rows.pop(0)
                if not headers_row:
                    break

        # Get descriptor
        guesser = _TypeGuesser()
        resolver = _TypeResolver()
        descriptor = {'fields': []}
        type_matches = {}
        for header in headers:
            descriptor['fields'].append({'name': header})
        for index, row in enumerate(rows):
            # Normalize rows with invalid dimensions for sanity
            row_length = len(row)
            headers_length = len(headers)
            if row_length > headers_length:
                row = row[:len(headers)]
            if row_length < headers_length:
                diff = headers_length - row_length
                fill = [''] * diff
                row = row + fill
            # build a column-wise lookup of type matches
            for index, value in enumerate(row):
                rv = guesser.cast(value)
                if type_matches.get(index):
                    type_matches[index].append(rv)
                else:
                    type_matches[index] = [rv]
        # choose a type/format for each column based on the matches
        for index, results in type_matches.items():
            rv = resolver.get(results)
            descriptor['fields'][index].update(**rv)

        # Commit descriptor
        self.__next_descriptor = descriptor
        self.commit()

        return descriptor

    def commit(self, strict=None):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        if strict is not None:
            self.__strict = strict
        elif self.__current_descriptor == self.__next_descriptor:
            return False
        self.__current_descriptor = deepcopy(self.__next_descriptor)
        self.__build()
        return True

    def save(self, target):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        mode = 'w'
        encoding = 'utf-8'
        if six.PY2:
            mode = 'wb'
            encoding = None
        helpers.ensure_dir(target)
        with io.open(target, mode=mode, encoding=encoding) as file:
            json.dump(self.__current_descriptor, file, indent=4)

    # Internal

    def __build(self):

        # Process descriptor
        self.__current_descriptor = helpers.expand_schema_descriptor(
            self.__current_descriptor)
        self.__next_descriptor = deepcopy(self.__current_descriptor)

        # Validate descriptor
        try:
            self.__profile.validate(self.__current_descriptor)
            self.__errors = []
        except exceptions.ValidationError as exception:
            self.__errors = exception.errors
            if self.__strict:
                message = 'There are %s validation errors (see exception.errors)'
                raise exceptions.ValidationError(
                    message % exception.errors, errors=exception.errors)

        # Populate fields
        self.__fields = []
        for field in self.__current_descriptor.get('fields', []):
            missing_values = self.__current_descriptor['missingValues']
            try:
                field = Field(field, missing_values=missing_values)
            except Exception:
                field = False
            self.__fields.append(field)

    # Deprecated

    headers = field_names
    has_field = get_field

# Internal

_INFER_TYPE_ORDER = [
    'duration',
    'geojson',
    'geopoint',
    'object',
    'array',
    'datetime',
    'time',
    'date',
    'integer',
    'number',
    'boolean',
    'string',
    'any',
]

class _TypeGuesser(object):
    """Guess the type for a value returning a tuple of ('type', 'format')
    """

    # Public

    def cast(self, value):
        for name in _INFER_TYPE_ORDER:
            cast = getattr(types, 'cast_%s' % name)
            result = cast('default', value)
            if result != config.ERROR:
                return (name, 'default')

class _TypeResolver(object):
    """Get the best matching type/format from a list of possible ones.
    """

    # Public

    @staticmethod
    def _sort_key(item):
        return (item[1], _INFER_TYPE_ORDER.index(item[0][0]))

    def get(self, results):
        variants = set(results)
        # only one candidate... that's easy.
        if len(variants) == 1:
            rv = {'type': results[0][0], 'format': results[0][1]}
        else:
            counts = {}
            for result in results:
                if counts.get(result):
                    counts[result] += 1
                else:
                    counts[result] = 1
            # tuple representation of `counts` dict sorted by values
            sorted_counts = sorted(counts.items(), key=self._sort_key, reverse=True)
            rv = {'type': sorted_counts[0][0][0], 'format': sorted_counts[0][0][1]}
        return rv
<commit_msg>Simplified exception raising in Schema<commit_after># -*- coding: utf-8 -*-
from __future__ import division
from __future__ import print_function
from __future__ import absolute_import
from __future__ import unicode_literals

import io
import six
import json
from copy import deepcopy
from .profile import Profile
from .field import Field
from . import exceptions
from . import helpers
from . import config
from . import types

# Module API

class Schema(object):

    # Public

    def __init__(self, descriptor={}, strict=False):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """

        # Process descriptor
        descriptor = helpers.retrieve_descriptor(descriptor)

        # Set attributes
        self.__strict = strict
        self.__current_descriptor = deepcopy(descriptor)
        self.__next_descriptor = deepcopy(descriptor)
        self.__profile = Profile('table-schema')
        self.__errors = []
        self.__fields = []

        # Build instance
        self.__build()

    @property
    def valid(self):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        return not bool(self.__errors)

    @property
    def errors(self):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        return self.__errors

    @property
    def descriptor(self):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        # Never use this.descriptor inside this class (!!!)
        return self.__next_descriptor

    @property
    def primary_key(self):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        primary_key = self.__current_descriptor.get('primaryKey', [])
        if not isinstance(primary_key, list):
            primary_key = [primary_key]
        return primary_key

    @property
    def foreign_keys(self):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        foreign_keys = self.__current_descriptor.get('foreignKeys', [])
        for key in foreign_keys:
            key.setdefault('fields', [])
            key.setdefault('reference', {})
            key['reference'].setdefault('resource', '')
            key['reference'].setdefault('fields', [])
            if not isinstance(key['fields'], list):
                key['fields'] = [key['fields']]
            if not isinstance(key['reference']['fields'], list):
                key['reference']['fields'] = [key['reference']['fields']]
        return foreign_keys

    @property
    def fields(self):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        return self.__fields

    @property
    def field_names(self):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        return [field.name for field in self.fields]

    def get_field(self, name):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        for field in self.fields:
            if field.name == name:
                return field
        return None

    def add_field(self, descriptor):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        self.__next_descriptor.setdefault('fields', [])
        self.__next_descriptor['fields'].append(descriptor)
        self.commit()
        return self.__fields[-1]

    def remove_field(self, name):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        field = self.get_field(name)
        if field:
            predicat = lambda field: field.get('name') != name
            self.__next_descriptor['fields'] = filter(
                predicat, self.__next_descriptor['fields'])
            self.commit()
        return field

    def cast_row(self, row, fail_fast=False):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """

        # Prepare
        errors = []

        # Check row length
        if len(row) != len(self.fields):
            message = 'Row length (%s) doesn\'t match fields count (%s)'
            message = message % (len(row), len(self.fields))
            exception = exceptions.CastError(message)
            if fail_fast:
                raise exception
            errors.append(exception)

        # Cast
        result = []
        if not errors:
            for field, value in zip(self.fields, row):
                try:
                    result.append(field.cast_value(value))
                except exceptions.CastError as exception:
                    if fail_fast:
                        raise
                    errors.append(exception)

        # Raise
        if errors:
            message = 'There are %s cast errors (see exception.errors)' % len(errors)
            raise exceptions.CastError(message, errors=errors)

        return result

    def infer(self, rows, headers=1):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """

        # Get headers
        if not isinstance(headers, list):
            headers_row = headers
            while True:
                headers_row -= 1
                headers = rows.pop(0)
                if not headers_row:
                    break

        # Get descriptor
        guesser = _TypeGuesser()
        resolver = _TypeResolver()
        descriptor = {'fields': []}
        type_matches = {}
        for header in headers:
            descriptor['fields'].append({'name': header})
        for index, row in enumerate(rows):
            # Normalize rows with invalid dimensions for sanity
            row_length = len(row)
            headers_length = len(headers)
            if row_length > headers_length:
                row = row[:len(headers)]
            if row_length < headers_length:
                diff = headers_length - row_length
                fill = [''] * diff
                row = row + fill
            # build a column-wise lookup of type matches
            for index, value in enumerate(row):
                rv = guesser.cast(value)
                if type_matches.get(index):
                    type_matches[index].append(rv)
                else:
                    type_matches[index] = [rv]
        # choose a type/format for each column based on the matches
        for index, results in type_matches.items():
            rv = resolver.get(results)
            descriptor['fields'][index].update(**rv)

        # Commit descriptor
        self.__next_descriptor = descriptor
        self.commit()

        return descriptor

    def commit(self, strict=None):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        if strict is not None:
            self.__strict = strict
        elif self.__current_descriptor == self.__next_descriptor:
            return False
        self.__current_descriptor = deepcopy(self.__next_descriptor)
        self.__build()
        return True

    def save(self, target):
        """https://github.com/frictionlessdata/tableschema-py#schema
        """
        mode = 'w'
        encoding = 'utf-8'
        if six.PY2:
            mode = 'wb'
            encoding = None
        helpers.ensure_dir(target)
        with io.open(target, mode=mode, encoding=encoding) as file:
            json.dump(self.__current_descriptor, file, indent=4)

    # Internal

    def __build(self):

        # Process descriptor
        self.__current_descriptor = helpers.expand_schema_descriptor(
            self.__current_descriptor)
        self.__next_descriptor = deepcopy(self.__current_descriptor)

        # Validate descriptor
        try:
            self.__profile.validate(self.__current_descriptor)
            self.__errors = []
        except exceptions.ValidationError as exception:
            self.__errors = exception.errors
            if self.__strict:
                raise exception

        # Populate fields
        self.__fields = []
        for field in self.__current_descriptor.get('fields', []):
            missing_values = self.__current_descriptor['missingValues']
            try:
                field = Field(field, missing_values=missing_values)
            except Exception:
                field = False
            self.__fields.append(field)

    # Deprecated

    headers = field_names
    has_field = get_field

# Internal

_INFER_TYPE_ORDER = [
    'duration',
    'geojson',
    'geopoint',
    'object',
    'array',
    'datetime',
    'time',
    'date',
    'integer',
    'number',
    'boolean',
    'string',
    'any',
]

class _TypeGuesser(object):
    """Guess the type for a value returning a tuple of ('type', 'format')
    """

    # Public

    def cast(self, value):
        for name in _INFER_TYPE_ORDER:
            cast = getattr(types, 'cast_%s' % name)
            result = cast('default', value)
            if result != config.ERROR:
                return (name, 'default')

class _TypeResolver(object):
    """Get the best matching type/format from a list of possible ones.
    """

    # Public

    @staticmethod
    def _sort_key(item):
        return (item[1], _INFER_TYPE_ORDER.index(item[0][0]))

    def get(self, results):
        variants = set(results)
        # only one candidate... that's easy.
        if len(variants) == 1:
            rv = {'type': results[0][0], 'format': results[0][1]}
        else:
            counts = {}
            for result in results:
                if counts.get(result):
                    counts[result] += 1
                else:
                    counts[result] = 1
            # tuple representation of `counts` dict sorted by values
            sorted_counts = sorted(counts.items(), key=self._sort_key, reverse=True)
            rv = {'type': sorted_counts[0][0][0], 'format': sorted_counts[0][0][1]}
        return rv
<|endoftext|>