<commit_before>import textwrap
from pathlib import Path
import click
from plaintable import Table
import pynote
from pynote import config
from pynote.formatting import echo, echo_info, echo_error, highlight_
from pynote.container import Note, load_notes, get_note, filter_tags

class Context:

    def __init__(self, data=None, no_pager=False):
        self.data = data
        self.no_pager = no_pager

pass_ctx = click.make_pass_decorator(Context, ensure=True)

class AliasedGroup(click.Group):

    def get_command(self, ctx, cmd_name):
        rv = click.Group.get_command(self, ctx, cmd_name)
        if rv is not None:
            return rv
        matches = [x for x in self.list_commands(ctx)
                   if x.startswith(cmd_name)]
        if not matches:
            return None
        elif len(matches) == 1:
            return click.Group.get_command(self, ctx, matches[0])
        ctx.fail('Too many matches: %s' % ', '.join(sorted(matches)))

@click.command(cls=AliasedGroup)
@click.version_option(version=pynote.__version__, prog_name='pynote')
@click.option('--no-pager', is_flag=True, help='Supress paging long output.')
@click.option('--no-header', is_flag=True, help='Supress header.')
@pass_ctx
def cli(ctx, no_pager, no_header):
    ctx.data = load_notes()
    ctx.trash = load_notes(config.trash_path)
    ctx.no_pager = no_pager
    ctx.no_header = no_header

@cli.command(name='list')
@click.option('--trash', is_flag=True, help='Show trash.')
@click.option('-t', '--tags', default=None, help='Filter appropriate tags.')
@click.option('-e', '--extended', is_flag=True, help='Add a tags column.')
@pass_ctx
def list_(ctx, trash, tags, extended):
    """Print out a table with all notes."""
    notes = []
    # Choose between data and trash depending on --trash.
    data = enumerate(ctx.data) if not trash else enumerate(ctx.trash)
    data = filter_tags(ctx.data, tags.split()) if tags else data
    for i, note in data:
        if config.reldates:
            if extended:
                header = ['ID', 'Title', 'Age', 'Tags']
                notes.append(
                    [i, note.title, note.format_age(), ', '.join(note.tags)]
                )
            else:
                header = ['ID', 'Title', 'Age']
                notes.append([i, note.title, note.format_age()])
        else:
            if extended:
                header = ['ID', 'Title', 'Updated', 'Tags']
                notes.append(
                    [i, note.title, note.format_age(), ', '.join(note.tags)]
                )
            else:
                header = ['ID', 'Title', 'Updated']
                notes.append([i, note.title, note.format_updated()])
    # Error handling
    if notes:
        echo(str(Table(notes, headline=header)))
    elif not notes and trash:
        echo_error('No notes in trash!')
    else:
        echo_error('No notes exists! Create new ones with "note new TITLE"!')

@cli.command()
@click.argument('key', type=int)
@click.option('-l', '--lang', default=None)
@click.option('-w', '--wrap-text', is_flag=True,
              help="Wrap output text at 70 chars.")
@pass_ctx
def show(ctx, key, lang, wrap_text):
    """Show a specific note."""
    note = get_note(ctx.data, key)
    if lang and config.colors:
        content = highlight_(note.content.decode(), lang)
    elif lang and config.colors is False:
        echo_error('Color support is not enabled!')
        exit(1)
    else:
        content = note.content.decode()

    if ctx.no_header:
        output = content
    else:
        output = note.format_header() + '\n\n' + content

    if wrap_text:
        output = textwrap.fill(output, replace_whitespace=False)

    echo(output, ctx.no_pager)

@cli.command()
@pass_ctx
def all(ctx):
    """Print out all notes in the data directory."""
    output = ''
    for i, note in enumerate(ctx.data):
        counter = '-- note {} --'.format(i)
        counter = click.style(counter, bold=True) if config.colors else counter

        output += '\n\n'
        output += counter
        output += '\n\n'

        if note.is_encrypted:
            output += 'Encrypted note!'
            output += '\n'
        else:
            if ctx.no_header:
                output += note.content.decode()
            else:
                output += note.format_header()
                output += '\n\n'
                output += note.content.decode()

    echo(output, ctx.no_pager)

@cli.command()
@click.argument('key', type=int)
@click.option('--title', is_flag=True, help='Edit the title instead.')
@click.option('--tags', is_flag=True, help='Edit assigned tags instead.')
@pass_ctx
def edit(ctx, key, title, tags):
    """Edit a specific note."""
    note = get_note(ctx.data, key)
    if title:
        new_title = click.edit(note.title, editor=config.editor)
        if new_title:
            # Delete old tags from tagsfile and save them in a variable.
            tags = note.tags
            del note.tags
            new_title = new_title.strip()
            new_path = note.path.parent / Path(new_title)
            note.path.rename(new_path)
            note.update_path(new_path)
            # Add saved tags to the renamed note.
            # This ensures correct dictionary keys.
            note.tags = tags
            # REVIEW: Can we remove this?
            note.path.touch()
        else:
            echo_info('No changes detected')
    elif tags:
        tag_str = '# Put each tag in one line! This line will be ignored.\n'
        tag_str += '\n'.join(note.tags)
        new_tags = click.edit(tag_str, editor=config.editor)
        if new_tags:
            new_tags = new_tags.strip().splitlines()
            try:
                note.tags = new_tags[1:]  # strip comment
            except AttributeError:
                echo_error('Tags with spaces are not allowed!')
                exit(1)
        else:
            echo_info('No changes detected')
    else:
        new_content = click.edit(note.content.decode(), editor=config.editor,
                                 extension=config.extension)
        if new_content:
            note.content = new_content.encode()
        else:
            echo_info('No changes detected')

@cli.command()
@click.argument('title', type=str)
@click.option('-t', '--tags', default=None, help='Assign tags to new note.')
def new(title, tags):
    """Create a new note."""
    if '/' in title:
        echo_error('Slashes in the title are not allowed!')
        exit(1)
    try:
        note = Note.create(title)
    except FileExistsError:
        echo_error('This note already exists!')
        exit(1)
    content = click.edit(note.content.decode(), editor=config.editor)
    note.content = content.encode() if content else b''
    echo_info("New note '{}' created!".format(note.title))
    if tags:
        note.tags = tags.split()
        echo_info("Assigned tags: {}".format(', '.join(tags.split())))

@cli.command()
@click.argument('key', type=int)
@pass_ctx
def delete(ctx, key):
    """Move a note to trash."""
    note = get_note(ctx.data, key)
    if not config.trash_path.exists():
        config.trash_path.mkdir()
    new_path = config.trash_path / note.title
    note.path.rename(new_path)
    note.update_path(new_path)
    note.path.touch()
    echo_info("Note '{}' moved to trash!".format(note.title))

@cli.command()
@click.argument('key', type=int)
@pass_ctx
def restore(ctx, key):
    """Restore a note from trash."""
    note = get_note(ctx.trash, key)
    new_path = config.data_path / note.title
    note.path.rename(new_path)
    note.update_path(new_path)
    note.path.touch()
    echo_info("Note '{}' restored!".format(note.title))

@cli.command()
def conf():
    """Show pynote's configuration (for debugging)."""
    varlist = [
        ['global_config', config.global_config],
        ['local_config', config.local_config],
        ['data_path', config.data_path],
        ['trash_path', config.trash_path],
        ['editor', config.editor],
        ['colors', config.colors],
        ['dateformat', config.dateformat],
        ['reldates', config.reldates],
        ['locale', config.locale],
        ['extension', config.extension],
        ['ignore_extensions', config.ignore_extensions],
        ['pygments_theme', config.pygments_theme],
    ]
    echo(str(Table(varlist)))
<commit_msg>Fixed error message, fixes #25<commit_after>import textwrap
from pathlib import Path
import click
from plaintable import Table
import pynote
from pynote import config
from pynote.formatting import echo, echo_info, echo_error, highlight_
from pynote.container import Note, load_notes, get_note, filter_tags

class Context:

    def __init__(self, data=None, no_pager=False):
        self.data = data
        self.no_pager = no_pager

pass_ctx = click.make_pass_decorator(Context, ensure=True)

class AliasedGroup(click.Group):

    def get_command(self, ctx, cmd_name):
        rv = click.Group.get_command(self, ctx, cmd_name)
        if rv is not None:
            return rv
        matches = [x for x in self.list_commands(ctx)
                   if x.startswith(cmd_name)]
        if not matches:
            return None
        elif len(matches) == 1:
            return click.Group.get_command(self, ctx, matches[0])
        ctx.fail('Too many matches: %s' % ', '.join(sorted(matches)))

@click.command(cls=AliasedGroup)
@click.version_option(version=pynote.__version__, prog_name='pynote')
@click.option('--no-pager', is_flag=True, help='Supress paging long output.')
@click.option('--no-header', is_flag=True, help='Supress header.')
@pass_ctx
def cli(ctx, no_pager, no_header):
    ctx.data = load_notes()
    ctx.trash = load_notes(config.trash_path)
    ctx.no_pager = no_pager
    ctx.no_header = no_header

@cli.command(name='list')
@click.option('--trash', is_flag=True, help='Show trash.')
@click.option('-t', '--tags', default=None, help='Filter appropriate tags.')
@click.option('-e', '--extended', is_flag=True, help='Add a tags column.')
@pass_ctx
def list_(ctx, trash, tags, extended):
    """Print out a table with all notes."""
    notes = []
    # Choose between data and trash depending on --trash.
    data = enumerate(ctx.data) if not trash else enumerate(ctx.trash)
    data = filter_tags(ctx.data, tags.split()) if tags else data
    for i, note in data:
        if config.reldates:
            if extended:
                header = ['ID', 'Title', 'Age', 'Tags']
                notes.append(
                    [i, note.title, note.format_age(), ', '.join(note.tags)]
                )
            else:
                header = ['ID', 'Title', 'Age']
                notes.append([i, note.title, note.format_age()])
        else:
            if extended:
                header = ['ID', 'Title', 'Updated', 'Tags']
                notes.append(
                    [i, note.title, note.format_age(), ', '.join(note.tags)]
                )
            else:
                header = ['ID', 'Title', 'Updated']
                notes.append([i, note.title, note.format_updated()])
    # Error handling
    if notes:
        echo(str(Table(notes, headline=header)))
    elif not notes and trash:
        echo_error('No notes in trash!')
    elif not notes and tags:
        echo_error('No note matches the given tags!')
    else:
        echo_error('No notes exists! Create new ones with "note new TITLE"!')

@cli.command()
@click.argument('key', type=int)
@click.option('-l', '--lang', default=None)
@click.option('-w', '--wrap-text', is_flag=True,
              help="Wrap output text at 70 chars.")
@pass_ctx
def show(ctx, key, lang, wrap_text):
    """Show a specific note."""
    note = get_note(ctx.data, key)
    if lang and config.colors:
        content = highlight_(note.content.decode(), lang)
    elif lang and config.colors is False:
        echo_error('Color support is not enabled!')
        exit(1)
    else:
        content = note.content.decode()

    if ctx.no_header:
        output = content
    else:
        output = note.format_header() + '\n\n' + content

    if wrap_text:
        output = textwrap.fill(output, replace_whitespace=False)

    echo(output, ctx.no_pager)

@cli.command()
@pass_ctx
def all(ctx):
    """Print out all notes in the data directory."""
    output = ''
    for i, note in enumerate(ctx.data):
        counter = '-- note {} --'.format(i)
        counter = click.style(counter, bold=True) if config.colors else counter

        output += '\n\n'
        output += counter
        output += '\n\n'

        if note.is_encrypted:
            output += 'Encrypted note!'
            output += '\n'
        else:
            if ctx.no_header:
                output += note.content.decode()
            else:
                output += note.format_header()
                output += '\n\n'
                output += note.content.decode()

    echo(output, ctx.no_pager)

@cli.command()
@click.argument('key', type=int)
@click.option('--title', is_flag=True, help='Edit the title instead.')
@click.option('--tags', is_flag=True, help='Edit assigned tags instead.')
@pass_ctx
def edit(ctx, key, title, tags):
    """Edit a specific note."""
    note = get_note(ctx.data, key)
    if title:
        new_title = click.edit(note.title, editor=config.editor)
        if new_title:
            # Delete old tags from tagsfile and save them in a variable.
            tags = note.tags
            del note.tags
            new_title = new_title.strip()
            new_path = note.path.parent / Path(new_title)
            note.path.rename(new_path)
            note.update_path(new_path)
            # Add saved tags to the renamed note.
            # This ensures correct dictionary keys.
            note.tags = tags
            # REVIEW: Can we remove this?
            note.path.touch()
        else:
            echo_info('No changes detected')
    elif tags:
        tag_str = '# Put each tag in one line! This line will be ignored.\n'
        tag_str += '\n'.join(note.tags)
        new_tags = click.edit(tag_str, editor=config.editor)
        if new_tags:
            new_tags = new_tags.strip().splitlines()
            try:
                note.tags = new_tags[1:]  # strip comment
            except AttributeError:
                echo_error('Tags with spaces are not allowed!')
                exit(1)
        else:
            echo_info('No changes detected')
    else:
        new_content = click.edit(note.content.decode(), editor=config.editor,
                                 extension=config.extension)
        if new_content:
            note.content = new_content.encode()
        else:
            echo_info('No changes detected')

@cli.command()
@click.argument('title', type=str)
@click.option('-t', '--tags', default=None, help='Assign tags to new note.')
def new(title, tags):
    """Create a new note."""
    if '/' in title:
        echo_error('Slashes in the title are not allowed!')
        exit(1)
    try:
        note = Note.create(title)
    except FileExistsError:
        echo_error('This note already exists!')
        exit(1)
    content = click.edit(note.content.decode(), editor=config.editor)
    note.content = content.encode() if content else b''
    echo_info("New note '{}' created!".format(note.title))
    if tags:
        note.tags = tags.split()
        echo_info("Assigned tags: {}".format(', '.join(tags.split())))

@cli.command()
@click.argument('key', type=int)
@pass_ctx
def delete(ctx, key):
    """Move a note to trash."""
    note = get_note(ctx.data, key)
    if not config.trash_path.exists():
        config.trash_path.mkdir()
    new_path = config.trash_path / note.title
    note.path.rename(new_path)
    note.update_path(new_path)
    note.path.touch()
    echo_info("Note '{}' moved to trash!".format(note.title))

@cli.command()
@click.argument('key', type=int)
@pass_ctx
def restore(ctx, key):
    """Restore a note from trash."""
    note = get_note(ctx.trash, key)
    new_path = config.data_path / note.title
    note.path.rename(new_path)
    note.update_path(new_path)
    note.path.touch()
    echo_info("Note '{}' restored!".format(note.title))

@cli.command()
def conf():
    """Show pynote's configuration (for debugging)."""
    varlist = [
        ['global_config', config.global_config],
        ['local_config', config.local_config],
        ['data_path', config.data_path],
        ['trash_path', config.trash_path],
        ['editor', config.editor],
        ['colors', config.colors],
        ['dateformat', config.dateformat],
        ['reldates', config.reldates],
        ['locale', config.locale],
        ['extension', config.extension],
        ['ignore_extensions', config.ignore_extensions],
        ['pygments_theme', config.pygments_theme],
    ]
    echo(str(Table(varlist)))
<|endoftext|>


<commit_before>import sys
import os
sys.path.append(os.path.join(os.path.dirname(__file__), '.'))

# use block explorer to check balance, block height to check fork

import requests
from config import *
from mnb_misc import *
import simplejson as json

def make_request(url):
    USERAGET = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/602.2.14 (KHTML, like Gecko) Version/10.0.1 Safari/602.2.14'
    headers = {'user-agent': USERAGET}

    try:
        response = requests.get(url, headers=headers, timeout=(5, 5))
        if response.status_code == requests.codes.ok and float(
                response.text) >= 0:
            return response.text

        else:
            return None

    except requests.exceptions.RequestException:
        err_msg = 'requests.exceptions.RequestException'
        print_err_exit(
            get_caller_name(),
            get_function_name(),
            err_msg)

    except Exception as e:
        err_msg = str(e.args)
        print_err_exit(
            get_caller_name(),
            get_function_name(),
            err_msg)

def make_request_version_txt(url):
    USERAGET = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/602.2.14 (KHTML, like Gecko) Version/10.0.1 Safari/602.2.14'
    headers = {'user-agent': USERAGET}

    try:
        response = requests.get(url, headers=headers, timeout=(4, 3))
        if response.status_code == requests.codes.ok and len(
                response.text) > 2:
            return response.json()

        else:
            return None

    except requests.exceptions.RequestException:
        if get_caller_name() == 'get_version_txt':
            err_msg = 'requests.exceptions.RequestException : error on checking version, use -k to skip version check'
        else:
            err_msg = 'requests.exceptions.RequestException'

        print_err_exit(
            get_caller_name(),
            get_function_name(),
            err_msg)

    except Exception as e:
        err_msg = str(e.args)
        print_err_exit(
            get_caller_name(),
            get_function_name(),
            err_msg)

def make_insight_request(url):
    USERAGET = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/602.2.14 (KHTML, like Gecko) Version/10.0.1 Safari/602.2.14'
    headers = {'user-agent': USERAGET}

    try:
        response = requests.get(url, headers=headers, timeout=(2,5))
        try:
            if response.status_code == requests.codes.ok and len(response.text) > 2:
                if isinstance(response.json(), list):
                    return response.json()[0]

                else:
                    return response.json()

        except Exception as e:
            print(e.args[0])
            return None

    except requests.exceptions.RequestException:
        return None

def getinfo_insight(url):
    getinfourl = url + '/status?q=getinfo'
    rawjson = make_insight_request(getinfourl)
    if rawjson:
        blockcnt = rawjson['info'].get('blocks', 0)

        return blockcnt

    else:
        return 0

def get_insight_blockcount():
    import random

    exp = [
        "https://insight.dashevo.org/insight-api-dash",
        "http://insight.dash.org/api",
        "https://insight.dash.siampm.com/api"
    ]

    IURL = exp[random.randrange(0,len(exp))]

    response = getinfo_insight(IURL)

    return response

def get_explorer_blockcount():
    if MAINNET:
        url = 'https://explorer.dash.org/chain/Dash/q/getblockcount'
    else:
        url = 'https://test.explorer.dash.org/chain/tDash/q/getblockcount'

    response = make_request(url)

    return response

def get_version_txt():
    url = 'https://raw.githubusercontent.com/chaeplin/dashmnb/master/dashlib/version.txt'
    response = make_request_version_txt(url)
    return response

def get_mnstatus_dashninja(vins):
    try:
        url = 'https://www.dashninja.pl/api/masternodes?testnet=0&portcheck=0&balance=0&exstatus=0&vins=' + json.dumps(vins)
        response = make_request_version_txt(url)
        return response
    except:
        return None

# end
<commit_msg>remove insight.dash.siampm.com (#56)<commit_after>import sys
import os
sys.path.append(os.path.join(os.path.dirname(__file__), '.'))

# use block explorer to check balance, block height to check fork

import requests
from config import *
from mnb_misc import *
import simplejson as json

def make_request(url):
    USERAGET = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/602.2.14 (KHTML, like Gecko) Version/10.0.1 Safari/602.2.14'
    headers = {'user-agent': USERAGET}

    try:
        response = requests.get(url, headers=headers, timeout=(5, 5))
        if response.status_code == requests.codes.ok and float(
                response.text) >= 0:
            return response.text

        else:
            return None

    except requests.exceptions.RequestException:
        err_msg = 'requests.exceptions.RequestException'
        print_err_exit(
            get_caller_name(),
            get_function_name(),
            err_msg)

    except Exception as e:
        err_msg = str(e.args)
        print_err_exit(
            get_caller_name(),
            get_function_name(),
            err_msg)

def make_request_version_txt(url):
    USERAGET = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/602.2.14 (KHTML, like Gecko) Version/10.0.1 Safari/602.2.14'
    headers = {'user-agent': USERAGET}

    try:
        response = requests.get(url, headers=headers, timeout=(4, 3))
        if response.status_code == requests.codes.ok and len(
                response.text) > 2:
            return response.json()

        else:
            return None

    except requests.exceptions.RequestException:
        if get_caller_name() == 'get_version_txt':
            err_msg = 'requests.exceptions.RequestException : error on checking version, use -k to skip version check'
        else:
            err_msg = 'requests.exceptions.RequestException'

        print_err_exit(
            get_caller_name(),
            get_function_name(),
            err_msg)

    except Exception as e:
        err_msg = str(e.args)
        print_err_exit(
            get_caller_name(),
            get_function_name(),
            err_msg)

def make_insight_request(url):
    USERAGET = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/602.2.14 (KHTML, like Gecko) Version/10.0.1 Safari/602.2.14'
    headers = {'user-agent': USERAGET}

    try:
        response = requests.get(url, headers=headers, timeout=(2,5))
        try:
            if response.status_code == requests.codes.ok and len(response.text) > 2:
                if isinstance(response.json(), list):
                    return response.json()[0]

                else:
                    return response.json()

        except Exception as e:
            print(e.args[0])
            return None

    except requests.exceptions.RequestException:
        return None

def getinfo_insight(url):
    getinfourl = url + '/status?q=getinfo'
    rawjson = make_insight_request(getinfourl)
    if rawjson:
        blockcnt = rawjson['info'].get('blocks', 0)

        return blockcnt

    else:
        return 0

def get_insight_blockcount():
    import random

    exp = [
        "https://insight.dashevo.org/insight-api-dash",
        "http://insight.dash.org/api"
    ]

    IURL = exp[random.randrange(0,len(exp))]

    response = getinfo_insight(IURL)

    return response

def get_explorer_blockcount():
    if MAINNET:
        url = 'https://explorer.dash.org/chain/Dash/q/getblockcount'
    else:
        url = 'https://test.explorer.dash.org/chain/tDash/q/getblockcount'

    response = make_request(url)

    return response

def get_version_txt():
    url = 'https://raw.githubusercontent.com/chaeplin/dashmnb/master/dashlib/version.txt'
    response = make_request_version_txt(url)
    return response

def get_mnstatus_dashninja(vins):
    try:
        url = 'https://www.dashninja.pl/api/masternodes?testnet=0&portcheck=0&balance=0&exstatus=0&vins=' + json.dumps(vins)
        response = make_request_version_txt(url)
        return response
    except:
        return None

# end
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from datetime import timedelta
from openerp import models, fields, api, exceptions, _

class Session(models.Model):
    _name = 'openacademy.session'

    _sql_constraints = [
        ('check_end_gte_start',
         'CHECK(end_date >= start_date)',
         "The end date must be greater than the start date"),

        ('check_duration_gte_zero',
         'CHECK(duration >= 0)',
         "The duration must be positive"),
    ]

    name = fields.Char(string='Title', required=True)
    start_date = fields.Date(default=fields.Date.today)
    duration = fields.Float(
        digits=(6, 2),
        string="Duration in days",
        help="Duration in days"
    )
    end_date = fields.Date(
        store=True,
        compute="_get_end_date",
        inverse="_set_end_date"
    )
    hours = fields.Float(
        string="Duration in hours",
        compute="_get_hours",
        inverse="_set_hours"
    )
    seats = fields.Integer(string="Number of seats")
    taken_seats = fields.Float(compute="_taken_seats", default=0.0)
    active = fields.Boolean(default=True)
    color = fields.Integer()
    instructor_id = fields.Many2one(
        'res.partner',
        string='Instructor',
        domain=[
            '|',
            ('instructor', '=', True),
            ('category_id', 'ilike', 'Teacher'),
        ]
    )
    course_id = fields.Many2one(
        'openacademy.course',
        ondelete='cascade',
        string='Course',
        required=True
    )
    attendee_ids = fields.Many2many('res.partner', string='Attendees')
    attendees_count = fields.Integer(
        store=True,
        compute='_get_attendees_count'
    )
    state = fields.Selection(
        [
            ('draft', "Draft"),
            ('confirmed', "Confirmed"),
            ('done', "Done"),
        ]
    )

    @api.one
    @api.depends('start_date', 'duration')
    def _get_end_date(self):
        if not (self.start_date and self.duration):
            self.end_date = self.start_date
        else:
            # Add duration to start_date, but: Mon + 5 days = Sat, so
            # subtract one second to get on Fri instead
            start = fields.Datetime.from_string(self.start_date)
            duration = timedelta(days=self.duration, seconds=-1)
            self.end_date = start + duration

    def _set_end_date(self):
        if self.start_date and self.end_date:
            # Compute the difference between dates, but: Fri - Mon = 4 days,
            # so add one day to get 5 days instead
            start_date = fields.Datetime.from_string(self.start_date)
            end_date = fields.Datetime.from_string(self.end_date)
            self.duration = (end_date - start_date).days + 1

    @api.one
    @api.depends('duration')
    def _get_hours(self):
        self.hours = self.duration * 24

    def _set_hours(self):
        self.duration = self.hours / 24

    @api.one
    @api.depends('seats', 'attendee_ids')
    def _taken_seats(self):
        if self.seats:
            self.taken_seats = 100 * len(self.attendee_ids) / self.seats
        else:
            self.taken_seats = 0

    @api.one
    @api.depends('attendee_ids')
    def _get_attendees_count(self):
        self.attendees_count = len(self.attendee_ids)

    @api.onchange('seats', 'attendee_ids')
    def _verify_valid_seats(self):
        if self.seats < 0:
            return {
                'warning': {
                    'title': _("Incorrect 'seats' value"),
                    'message': _("The number of available seats may not "
                               "be negative"),
                }
            }
        if len(self.attendee_ids) > self.seats:
            return {
                'warning': {
                    'title': _("Too many attendees"),
                    'message': _("Increase seats or remove excess attendees"),
                }
            }

    @api.constrains('instructor_id', 'attendee_ids')
    def _check_instructor_not_in_attendees(self):
        if self.instructor_id and self.instructor_id in self.attendee_ids:
            raise exceptions.ValidationError(_("A session's instructor can't be "
                                             "an attendee"))

    @api.multi
    def action_draft(self):
        self.ensure_one()
        self.state = 'draft'

    @api.multi
    def action_confirm(self):
        self.ensure_one()
        self.state = 'confirmed'

    @api.multi
    def action_done(self):
        self.ensure_one()
        self.state = 'done'
<commit_msg>[REF] openacademy: fix flake8 errors in model.session<commit_after># -*- coding: utf-8 -*-

from datetime import timedelta
from openerp import models, fields, api, exceptions, _

class Session(models.Model):
    _name = 'openacademy.session'

    _sql_constraints = [
        ('check_end_gte_start',
         'CHECK(end_date >= start_date)',
         "The end date must be greater than the start date"),

        ('check_duration_gte_zero',
         'CHECK(duration >= 0)',
         "The duration must be positive"),
    ]

    name = fields.Char(string='Title', required=True)
    start_date = fields.Date(default=fields.Date.today)
    duration = fields.Float(
        digits=(6, 2),
        string="Duration in days",
        help="Duration in days"
    )
    end_date = fields.Date(
        store=True,
        compute="_get_end_date",
        inverse="_set_end_date"
    )
    hours = fields.Float(
        string="Duration in hours",
        compute="_get_hours",
        inverse="_set_hours"
    )
    seats = fields.Integer(string="Number of seats")
    taken_seats = fields.Float(compute="_taken_seats", default=0.0)
    active = fields.Boolean(default=True)
    color = fields.Integer()
    instructor_id = fields.Many2one(
        'res.partner',
        string='Instructor',
        domain=[
            '|',
            ('instructor', '=', True),
            ('category_id', 'ilike', 'Teacher'),
        ]
    )
    course_id = fields.Many2one(
        'openacademy.course',
        ondelete='cascade',
        string='Course',
        required=True
    )
    attendee_ids = fields.Many2many('res.partner', string='Attendees')
    attendees_count = fields.Integer(
        store=True,
        compute='_get_attendees_count'
    )
    state = fields.Selection(
        [
            ('draft', "Draft"),
            ('confirmed', "Confirmed"),
            ('done', "Done"),
        ]
    )

    @api.one
    @api.depends('start_date', 'duration')
    def _get_end_date(self):
        if not (self.start_date and self.duration):
            self.end_date = self.start_date
        else:
            # Add duration to start_date, but: Mon + 5 days = Sat, so
            # subtract one second to get on Fri instead
            start = fields.Datetime.from_string(self.start_date)
            duration = timedelta(days=self.duration, seconds=-1)
            self.end_date = start + duration

    def _set_end_date(self):
        if self.start_date and self.end_date:
            # Compute the difference between dates, but: Fri - Mon = 4 days,
            # so add one day to get 5 days instead
            start_date = fields.Datetime.from_string(self.start_date)
            end_date = fields.Datetime.from_string(self.end_date)
            self.duration = (end_date - start_date).days + 1

    @api.one
    @api.depends('duration')
    def _get_hours(self):
        self.hours = self.duration * 24

    def _set_hours(self):
        self.duration = self.hours / 24

    @api.one
    @api.depends('seats', 'attendee_ids')
    def _taken_seats(self):
        if self.seats:
            self.taken_seats = 100 * len(self.attendee_ids) / self.seats
        else:
            self.taken_seats = 0

    @api.one
    @api.depends('attendee_ids')
    def _get_attendees_count(self):
        self.attendees_count = len(self.attendee_ids)

    @api.onchange('seats', 'attendee_ids')
    def _verify_valid_seats(self):
        if self.seats < 0:
            return {
                'warning': {
                    'title': _("Incorrect 'seats' value"),
                    'message': _(
                        "The number of available seats may not be negative"
                    ),
                }
            }
        if len(self.attendee_ids) > self.seats:
            return {
                'warning': {
                    'title': _("Too many attendees"),
                    'message': _("Increase seats or remove excess attendees"),
                }
            }

    @api.constrains('instructor_id', 'attendee_ids')
    def _check_instructor_not_in_attendees(self):
        if self.instructor_id and self.instructor_id in self.attendee_ids:
            raise exceptions.ValidationError(
                _("A session's instructor can't be an attendee")
            )

    @api.multi
    def action_draft(self):
        self.ensure_one()
        self.state = 'draft'

    @api.multi
    def action_confirm(self):
        self.ensure_one()
        self.state = 'confirmed'

    @api.multi
    def action_done(self):
        self.ensure_one()
        self.state = 'done'
<|endoftext|>


<commit_before>from future.standard_library import install_aliases
install_aliases()

from urllib.parse import urlparse
from airflow.hooks import BaseHook
from airflow.utils import AirflowException

import gcloud.storage as gcs

def parse_gcs_url(gsurl):
    """
    Given a Google Cloud Storage URL (gs://<bucket>/<blob>), returns a
    tuple containing the corresponding bucket and blob.
    """
    parsed_url = urlparse(gsurl)
    if not parsed_url.netloc:
        raise AirflowException('Please provide a bucket name')
    else:
        bucket = parsed_url.netloc
        if parsed_url.path.startswith('/'):
            blob = parsed_url.path[1:]
        else:
            blob = parsed_url.path
        return (bucket, blob)

class GCSHook(BaseHook):
    """
    A hook for working wth Google Cloud Storage via the gcloud library.

    GCS Connections can contain two optional fields in "extras":
    {
        "project": "<google cloud project id>",
        "keyfile_path": "<path to service account JSON keyfile>"
    }

    If the project field is missing, the project will be inferred from the host
    environment (if possible). To set a default project, use:
        gcloud config set project <project-id>

    If the keyfile_path is missing, the host authorization credentials will be
    used (if possible). To log in, use:
        gcloud auth
    """
    def __init__(self, gcs_conn_id=None):
        self.gcs_conn_id = gcs_conn_id
        self.gcs_conn = self.get_conn()

    def get_conn(self):
        project, keyfile_path = None, None
        if self.gcs_conn_id:
            conn = self.get_connection(self.gcs_conn_id)
            extras = conn.extra_dejson
            project = extras.get('project', None)
            keyfile_path = extras.get('keyfile_path', None)

        if not project:
            project = gcloud._helpers._determine_default_project()
            # workaround for
            # https://github.com/GoogleCloudPlatform/gcloud-python/issues/1470
            if isinstance(project, bytes):
                project = project.decode()

        if keyfile_path:
            client = gcs.Client.from_service_account_json(
                keyfile_path, project=project)
        else:
            client = gcs.Client(project=project)

        return client

    def bucket_exists(self, bucket):
        return self.get_conn().bucket(bucket).exists()

    def get_bucket(self, bucket):
        return self.get_conn().get_bucket(bucket)

    def list_blobs(
            self,
            bucket,
            max_results=None,
            page_token=None,
            prefix=None,
            delimiter=None):
        return self.get_conn().bucket(bucket).list_blobs(
            max_results=max_results,
            page_token=page_token,
            prefix=prefix,
            delimiter=delimiter)

    def get_blob(self, blob, bucket=None):
        """
        Returns None if the blob does not exist
        """
        if not bucket:
            bucket, blob = parse_gcs_url(blob)
        return self.get_conn().bucket(bucket).get_blob(blob)

    def blob_exists(self, blob, bucket=None):
        if not bucket:
            bucket, blob = parse_gcs_url(blob)
        return self.get_conn().bucket(bucket).blob(blob).exists()

    def upload_from_file(
            self,
            file_obj,
            blob,
            bucket=None,
            replace=False):
        if not bucket:
            bucket, blob = parse_gcs_url(blob)
        gcs_blob = self.get_conn().bucket(bucket).blob(blob)
        if gcs_blob.exists() and not replace:
            raise ValueError(
                'The blob {bucket}/{blob} already exists.'.format(**locals()))
        gcs_blob.upload_from_file(file_obj)

    def upload_from_filename(
            self,
            filename,
            blob,
            bucket=None,
            replace=False):
        if not bucket:
            bucket, blob = parse_gcs_url(blob)
        gcs_blob = self.get_conn().bucket(bucket).blob(blob)
        if gcs_blob.exists() and not replace:
            raise ValueError(
                'The blob {bucket}/{blob} already exists.'.format(**locals()))
        gcs_blob.upload_from_filename(filename)

    def upload_from_string(
            self,
            string,
            blob,
            bucket=None,
            replace=False):
        if not bucket:
            bucket, blob = parse_gcs_url(blob)
        gcs_blob = self.get_conn().bucket(bucket).blob(blob)
        if gcs_blob.exists() and not replace:
            raise ValueError(
                'The blob {bucket}/{blob} already exists.'.format(**locals()))
        gcs_blob.upload_from_string(string)

    def download_as_string(
            self,
            blob,
            bucket=None):
        if not bucket:
            bucket, blob = parse_gcs_url(blob)
        gcs_blob = self.get_conn().bucket(bucket).get_blob(blob)
        if not gcs_blob:
            raise ValueError(
                'Blob does not exist: {bucket}/{blob}'.format(**locals()))
        return gcs_blob.download_as_string()

    def download_to_file(
            self,
            file_obj,
            blob,
            bucket=None):
        if not bucket:
            bucket, blob = parse_gcs_url(blob)
        gcs_blob = self.get_conn().bucket(bucket).get_blob(blob)
        if not gcs_blob:
            raise ValueError(
                'Blob does not exist: {bucket}/{blob}'.format(**locals()))
        return gcs_blob.download_to_file(file_obj)

    def download_to_filename(
            self,
            filename,
            blob,
            bucket=None):
        if not bucket:
            bucket, blob = parse_gcs_url(blob)
        gcs_blob = self.get_conn().bucket(bucket).get_blob(blob)
        if not gcs_blob:
            raise ValueError(
                'Blob does not exist: {bucket}/{blob}'.format(**locals()))
        return gcs_blob.download_to_filename(filename)
<commit_msg>support P12 credentials; compatibility with existing GCS connections<commit_after>from future.standard_library import install_aliases
install_aliases()

from urllib.parse import urlparse
from airflow.hooks import BaseHook
from airflow.utils import AirflowException

import gcloud.storage as gcs

def parse_gcs_url(gsurl):
    """
    Given a Google Cloud Storage URL (gs://<bucket>/<blob>), returns a
    tuple containing the corresponding bucket and blob.
    """
    parsed_url = urlparse(gsurl)
    if not parsed_url.netloc:
        raise AirflowException('Please provide a bucket name')
    else:
        bucket = parsed_url.netloc
        if parsed_url.path.startswith('/'):
            blob = parsed_url.path[1:]
        else:
            blob = parsed_url.path
        return (bucket, blob)

class GCSHook(BaseHook):
    """
    A hook for working wth Google Cloud Storage via the gcloud library.

    GCS Connections can contain three optional fields in "extras":
    {
        "project": "<google cloud project id>",
        "key_path": "<path to service account keyfile, either JSON or P12>"
        "service_account": "<google service account email, required for P12>"
    }

    If the project field is missing, the project will be inferred from the host
    environment (if possible). To set a default project, use:
        gcloud config set project <project-id>

    If the key_path is missing, the host authorization credentials will be
    used (if possible). To log in, use:
        gcloud auth

    The service_account is only required if the key_path points to a P12 file.
    """
    def __init__(self, gcs_conn_id=None):
        self.gcs_conn_id = gcs_conn_id
        self.gcs_conn = self.get_conn()

    def get_conn(self):
        project, key_path = None, None
        if self.gcs_conn_id:
            conn = self.get_connection(self.gcs_conn_id)
            extras = conn.extra_dejson
            project = extras.get('project', None)
            key_path = extras.get('key_path', None)
            service_account = extras.get('service_account', None)

        if not project:
            project = gcloud._helpers._determine_default_project()
            # workaround for
            # https://github.com/GoogleCloudPlatform/gcloud-python/issues/1470
            if isinstance(project, bytes):
                project = project.decode()

        if key_path:
            if key_path.endswith('.json') or key_path.endswith('.JSON'):
                client = gcs.Client.from_service_account_json(
                    json_credentials_path=key_path,
                    project=project)
            elif key_path.endswith('.p12') or key_path.endswith('.P12'):
                client = gcs.Client.from_service_account_p12(
                    client_email=service_account,
                    private_key_path=key_path,
                    project=project)
            else:
                raise ValueError('Unrecognized keyfile: {}'.format(key_path))
        else:
            client = gcs.Client(project=project)

        return client

    def bucket_exists(self, bucket):
        return self.get_conn().bucket(bucket).exists()

    def get_bucket(self, bucket):
        return self.get_conn().get_bucket(bucket)

    def list_blobs(
            self,
            bucket,
            max_results=None,
            page_token=None,
            prefix=None,
            delimiter=None):
        return self.get_conn().bucket(bucket).list_blobs(
            max_results=max_results,
            page_token=page_token,
            prefix=prefix,
            delimiter=delimiter)

    def get_blob(self, blob, bucket=None):
        """
        Returns None if the blob does not exist
        """
        if not bucket:
            bucket, blob = parse_gcs_url(blob)
        return self.get_conn().bucket(bucket).get_blob(blob)

    def blob_exists(self, blob, bucket=None):
        if not bucket:
            bucket, blob = parse_gcs_url(blob)
        return self.get_conn().bucket(bucket).blob(blob).exists()

    def upload_from_file(
            self,
            file_obj,
            blob,
            bucket=None,
            replace=False):
        if not bucket:
            bucket, blob = parse_gcs_url(blob)
        gcs_blob = self.get_conn().bucket(bucket).blob(blob)
        if gcs_blob.exists() and not replace:
            raise ValueError(
                'The blob {bucket}/{blob} already exists.'.format(**locals()))
        gcs_blob.upload_from_file(file_obj)

    def upload_from_filename(
            self,
            filename,
            blob,
            bucket=None,
            replace=False):
        if not bucket:
            bucket, blob = parse_gcs_url(blob)
        gcs_blob = self.get_conn().bucket(bucket).blob(blob)
        if gcs_blob.exists() and not replace:
            raise ValueError(
                'The blob {bucket}/{blob} already exists.'.format(**locals()))
        gcs_blob.upload_from_filename(filename)

    def upload_from_string(
            self,
            string,
            blob,
            bucket=None,
            replace=False):
        if not bucket:
            bucket, blob = parse_gcs_url(blob)
        gcs_blob = self.get_conn().bucket(bucket).blob(blob)
        if gcs_blob.exists() and not replace:
            raise ValueError(
                'The blob {bucket}/{blob} already exists.'.format(**locals()))
        gcs_blob.upload_from_string(string)

    def download_as_string(
            self,
            blob,
            bucket=None):
        if not bucket:
            bucket, blob = parse_gcs_url(blob)
        gcs_blob = self.get_conn().bucket(bucket).get_blob(blob)
        if not gcs_blob:
            raise ValueError(
                'Blob does not exist: {bucket}/{blob}'.format(**locals()))
        return gcs_blob.download_as_string()

    def download_to_file(
            self,
            file_obj,
            blob,
            bucket=None):
        if not bucket:
            bucket, blob = parse_gcs_url(blob)
        gcs_blob = self.get_conn().bucket(bucket).get_blob(blob)
        if not gcs_blob:
            raise ValueError(
                'Blob does not exist: {bucket}/{blob}'.format(**locals()))
        return gcs_blob.download_to_file(file_obj)

    def download_to_filename(
            self,
            filename,
            blob,
            bucket=None):
        if not bucket:
            bucket, blob = parse_gcs_url(blob)
        gcs_blob = self.get_conn().bucket(bucket).get_blob(blob)
        if not gcs_blob:
            raise ValueError(
                'Blob does not exist: {bucket}/{blob}'.format(**locals()))
        return gcs_blob.download_to_filename(filename)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Docstring."""

import os
import pickle

class PickleCache(object):

    def __init__(self, file_path='datastore.pkl', autosync=False):
        """Docstring."""
        self.__file_path = file_path
        self.__data = {}
        self.autosync = autosync
        self.__file_object = None

    def __setitem__(self, key, value):
        """Docstring."""
        self.__data[key] = value
        if self.autosync is not False:
            self.flush()

    def __len__(self):
        """Docstring."""
        return len(self.__data)

    def __getitem__(self,key):
        try:
            return self.__data[key]
        except TypeError or KeyError:
            raise KeyError

<commit_msg>encountering keyerrors<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Docstring."""

import os
import pickle

class PickleCache(object):

    def __init__(self, file_path='datastore.pkl', autosync=False):
        """Docstring."""
        self.__file_path = file_path
        self.__data = {}
        self.autosync = autosync
        self.__file_object = None

    def __setitem__(self, key, value):
        """Docstring."""
        self.__data[key] = value
        if self.autosync is not False:
            self.flush()

    def __len__(self):
        """Docstring."""
        return len(self.__data)

    def __getitem__(self, key):
        try:
            if self.__data[key]:
                return self.__data[key]
        except KeyError, TypeError:
            raise KeyError

    def __delitem__(self, key):
        """Docstring."""
        if key in self.__data:
            del self.__data[key]
            if self.autosync is not False:
                self.flush()

    def load(self):
        """Docstring."""
        self.__file_path = load
        if os.path.exists(load) is True and os.path.getsize(load) > 0:
            fhandler = open(self.__file_path, 'r')
            self.__data = pickle.load(fhandler)
            fhandler.close()

    def flush(self):
        fhandler = open(self.__file_path, 'w')
        pickle.dump(self.__data, fhandler)
        fhandler.close()

<|endoftext|>


<commit_before>#    Copyright 2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import re
import logging
from itertools import chain

from devlib.trace.ftrace import TRACE_MARKER_START, TRACE_MARKER_STOP

from wa.utils.misc import isiterable
from wa.utils.types import numeric

logger = logging.getLogger('trace-cmd')

class TraceCmdEvent(object):
    """
    A single trace-cmd event. This will appear in the trace cmd report in the format ::

          <idle>-0     [000]  3284.126993: sched_rq_runnable_load: cpu=0 load=54
             |           |         |              |                |___________|
             |           |         |              |                      |
          thread        cpu    timestamp        name                    body

    """

    __slots__ = ['thread', 'reporting_cpu_id', 'timestamp', 'name', 'text', 'fields']

    def __init__(self, thread, cpu_id, ts, name, body, parser=None):
        """
        parameters:

        :thread: thread which generated the event
        :cpu: cpu on which the event has occurred
        :ts: timestamp of the event
        :name: the name of the event
        :bodytext: a string with the rest of the event text
        :parser: optionally, a function that will parse body text to populate
                 this event's attributes

        The parser can be any callable that can be invoked with

            parser(event, text)

        Where ``event`` is this TraceCmdEvent instance, and ``text`` is the body text to be
        parsed. The parser should updated the passed event instance and not return anything
        (the return value will be ignored). Any exceptions raised by the parser will be silently
        ignored (note that this means that the event's attributes may be partially initialized).

        """
        self.thread = thread
        self.reporting_cpu_id = int(cpu_id)
        self.timestamp = numeric(ts)
        self.name = name
        self.text = body
        self.fields = {}

        if parser:
            try:
                parser(self, self.text)
            except Exception:  # pylint: disable=broad-except
                # unknown format assume user does not care or know how to
                # parse self.text
                pass

    def __getattr__(self, name):
        try:
            return self.fields[name]
        except KeyError:
            raise AttributeError(name)

    def __str__(self):
        return 'TE({} @ {})'.format(self.name, self.timestamp)

    __repr__ = __str__

class DroppedEventsEvent(object):

    __slots__ = ['thread', 'reporting_cpu_id', 'timestamp', 'name', 'text', 'fields']

    def __init__(self, cpu_id):
        self.thread = None
        self.reporting_cpu_id = None
        self.timestamp = None
        self.name = 'DROPPED EVENTS DETECTED'
        self.text = None
        self.fields = {'cpu_id': int(cpu_id)}

    def __getattr__(self, name):
        try:
            return self.fields[name]
        except KeyError:
            raise AttributeError(name)

    def __str__(self):
        return 'DROPPED_EVENTS_ON_CPU{}'.format(self.cpu_id)

    __repr__ = __str__

def try_convert_to_numeric(v):
    try:
        if isiterable(v):
            return map(numeric, v)
        else:
            return numeric(v)
    except ValueError:
        return v

def default_body_parser(event, text):
    """
    Default parser to attempt to use to parser body text for the event (i.e. after
    the "header" common to all events has been parsed). This assumes that the body is
    a whitespace-separated list of key=value pairs. The parser will attempt to convert
    the value into a numeric type, and failing that, keep it as string.

    """
    parts = [e.rsplit(' ', 1) for e in text.strip().split('=')]
    parts = [p.strip() for p in chain.from_iterable(parts)]
    if not len(parts) % 2:
        i = iter(parts)
        for k, v in zip(i, i):
            try:
                v = int(v)
            except ValueError:
                pass
            event.fields[k] = v

def regex_body_parser(regex, flags=0):
    """
    Creates an event body parser form the specified regular expression (could be an
    ``re.RegexObject``, or a string). The regular expression should contain some named
    groups, as those will be extracted as the event attributes (unnamed groups and the
    reset of the match will be ignored).

    If the specified regex is a string, it will be compiled, in which case ``flags`` may
    be provided for the resulting regex object (see ``re`` standard module documentation).
    If regex is a pre-compiled object, flags will be ignored.

    """
    if isinstance(regex, basestring):
        regex = re.compile(regex, flags)

    def regex_parser_func(event, text):
        match = regex.search(text)
        if match:
            for k, v in match.groupdict().iteritems():
                try:
                    event.fields[k] = int(v)
                except ValueError:
                    event.fields[k] = v

    return regex_parser_func

def sched_switch_parser(event, text):
    """
    Sched switch output may be presented in a couple of different formats. One is handled
    by a regex. The other format can *almost* be handled by the default parser, if it
    weren't for the ``==>`` that appears in the middle.
    """
    if text.count('=') == 2:  # old format
        regex = re.compile(
            r'(?P<prev_comm>\S.*):(?P<prev_pid>\d+) \[(?P<prev_prio>\d+)\] (?P<status>\S+)'
            r' ==> '
            r'(?P<next_comm>\S.*):(?P<next_pid>\d+) \[(?P<next_prio>\d+)\]'
        )
        parser_func = regex_body_parser(regex)
        return parser_func(event, text)
    else:  # there are more than two "=" -- new format
        return default_body_parser(event, text.replace('==>', ''))

# Maps event onto the corresponding parser for its body text. A parser may be
# a callable with signature
#
#   parser(event, bodytext)
#
# a re.RegexObject, or a string (in which case it will be compiled into a
# regex). In case of a string/regex, its named groups will be used to populate
# the event's attributes.
EVENT_PARSER_MAP = {
    'sched_switch': sched_switch_parser,
}

TRACE_EVENT_REGEX = re.compile(r'^\s+(?P<thread>\S+.*?\S+)\s+\[(?P<cpu_id>\d+)\]\s+(?P<ts>[\d.]+):\s+'
                               r'(?P<name>[^:]+):\s+(?P<body>.*?)\s*$')

HEADER_REGEX = re.compile(r'^\s*(?:version|cpus)\s*=\s*([\d.]+)\s*$')

DROPPED_EVENTS_REGEX = re.compile(r'CPU:(?P<cpu_id>\d+) \[\d*\s*EVENTS DROPPED\]')

EMPTY_CPU_REGEX = re.compile(r'CPU \d+ is empty')

class TraceCmdParser(object):
    """
    A parser for textual representation of ftrace as reported by trace-cmd

    """

    def __init__(self, filter_markers=True):
        """
        Initialize a new trace parser.

        :param filter_markers: Specifies whether the trace before the start
                               marker and after the stop marker should be
                               filtered out (so only events between the two
                               markers will be reported). This maybe overriden
                               based on `check_for_markers` parameter of
                               `parse()`

        """
        self.filter_markers = filter_markers

    def parse(self, filepath, events=None, check_for_markers=True):  # pylint: disable=too-many-branches,too-many-locals
        """
        This is a generator for the trace event stream.

        :param filepath: The path to the file containg text trace as reported
                         by trace-cmd
        :param events: A list of event names to be reported; if not specified,
                       all events will be reported.
        :param check_for_markers: Check if the start/stop markers are present
                                  in the trace and ensure that `filter_markers`
                                  is `False` if they aren't

        """
        inside_maked_region = False
        filters = [re.compile('^{}$'.format(e)) for e in (events or [])]
        filter_markers = self.filter_markers
        if filter_markers and check_for_markers:
            with open(filepath) as fh:
                for line in fh:
                    if TRACE_MARKER_START in line:
                        break
                else:
                    # maker not found force filtering by marker to False
                    filter_markers = False

        with open(filepath) as fh:
            for line in fh:
                # if processing trace markers, skip marker lines as well as all
                # lines outside marked region
                if filter_markers:
                    if not inside_maked_region:
                        if TRACE_MARKER_START in line:
                            inside_maked_region = True
                        continue
                    elif TRACE_MARKER_STOP in line:
                        inside_maked_region = False
                        continue

                match = DROPPED_EVENTS_REGEX.search(line)
                if match:
                    yield DroppedEventsEvent(match.group('cpu_id'))
                    continue

                matched = False
                for rx in [HEADER_REGEX, EMPTY_CPU_REGEX]:
                    match = rx.search(line)
                    if match:
                        logger.debug(line.strip())
                        matched = True
                        break
                if matched:
                    continue

                match = TRACE_EVENT_REGEX.search(line)
                if not match:
                    logger.warning('Invalid trace event: "{}"'.format(line))
                    continue

                event_name = match.group('name')

                if filters:
                    found = False
                    for f in filters:
                        if f.search(event_name):
                            found = True
                            break
                    if not found:
                        continue

                body_parser = EVENT_PARSER_MAP.get(event_name, default_body_parser)
                if isinstance(body_parser, basestring) or isinstance(body_parser, re._pattern_type):  # pylint: disable=protected-access
                    body_parser = regex_body_parser(body_parser)
                yield TraceCmdEvent(parser=body_parser, **match.groupdict())

<commit_msg>utils/trace_cmd: add parsers for more sched events<commit_after>#    Copyright 2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import re
import logging
from itertools import chain

from devlib.trace.ftrace import TRACE_MARKER_START, TRACE_MARKER_STOP

from wa.utils.misc import isiterable
from wa.utils.types import numeric

logger = logging.getLogger('trace-cmd')

class TraceCmdEvent(object):
    """
    A single trace-cmd event. This will appear in the trace cmd report in the format ::

          <idle>-0     [000]  3284.126993: sched_rq_runnable_load: cpu=0 load=54
             |           |         |              |                |___________|
             |           |         |              |                      |
          thread        cpu    timestamp        name                    body

    """

    __slots__ = ['thread', 'reporting_cpu_id', 'timestamp', 'name', 'text', 'fields']

    def __init__(self, thread, cpu_id, ts, name, body, parser=None):
        """
        parameters:

        :thread: thread which generated the event
        :cpu: cpu on which the event has occurred
        :ts: timestamp of the event
        :name: the name of the event
        :bodytext: a string with the rest of the event text
        :parser: optionally, a function that will parse body text to populate
                 this event's attributes

        The parser can be any callable that can be invoked with

            parser(event, text)

        Where ``event`` is this TraceCmdEvent instance, and ``text`` is the body text to be
        parsed. The parser should updated the passed event instance and not return anything
        (the return value will be ignored). Any exceptions raised by the parser will be silently
        ignored (note that this means that the event's attributes may be partially initialized).

        """
        self.thread = thread
        self.reporting_cpu_id = int(cpu_id)
        self.timestamp = numeric(ts)
        self.name = name
        self.text = body
        self.fields = {}

        if parser:
            try:
                parser(self, self.text)
            except Exception:  # pylint: disable=broad-except
                # unknown format assume user does not care or know how to
                # parse self.text
                pass

    def __getattr__(self, name):
        try:
            return self.fields[name]
        except KeyError:
            raise AttributeError(name)

    def __str__(self):
        return 'TE({} @ {})'.format(self.name, self.timestamp)

    __repr__ = __str__

class DroppedEventsEvent(object):

    __slots__ = ['thread', 'reporting_cpu_id', 'timestamp', 'name', 'text', 'fields']

    def __init__(self, cpu_id):
        self.thread = None
        self.reporting_cpu_id = None
        self.timestamp = None
        self.name = 'DROPPED EVENTS DETECTED'
        self.text = None
        self.fields = {'cpu_id': int(cpu_id)}

    def __getattr__(self, name):
        try:
            return self.fields[name]
        except KeyError:
            raise AttributeError(name)

    def __str__(self):
        return 'DROPPED_EVENTS_ON_CPU{}'.format(self.cpu_id)

    __repr__ = __str__

def try_convert_to_numeric(v):
    try:
        if isiterable(v):
            return map(numeric, v)
        else:
            return numeric(v)
    except ValueError:
        return v

def default_body_parser(event, text):
    """
    Default parser to attempt to use to parser body text for the event (i.e. after
    the "header" common to all events has been parsed). This assumes that the body is
    a whitespace-separated list of key=value pairs. The parser will attempt to convert
    the value into a numeric type, and failing that, keep it as string.

    """
    parts = [e.rsplit(' ', 1) for e in text.strip().split('=')]
    parts = [p.strip() for p in chain.from_iterable(parts)]
    if not len(parts) % 2:
        i = iter(parts)
        for k, v in zip(i, i):
            try:
                v = int(v)
            except ValueError:
                pass
            event.fields[k] = v

def regex_body_parser(regex, flags=0):
    """
    Creates an event body parser form the specified regular expression (could be an
    ``re.RegexObject``, or a string). The regular expression should contain some named
    groups, as those will be extracted as the event attributes (unnamed groups and the
    reset of the match will be ignored).

    If the specified regex is a string, it will be compiled, in which case ``flags`` may
    be provided for the resulting regex object (see ``re`` standard module documentation).
    If regex is a pre-compiled object, flags will be ignored.

    """
    if isinstance(regex, basestring):
        regex = re.compile(regex, flags)

    def regex_parser_func(event, text):
        match = regex.search(text)
        if match:
            for k, v in match.groupdict().iteritems():
                try:
                    event.fields[k] = int(v)
                except ValueError:
                    event.fields[k] = v

    return regex_parser_func

def sched_switch_parser(event, text):
    """
    Sched switch output may be presented in a couple of different formats. One is handled
    by a regex. The other format can *almost* be handled by the default parser, if it
    weren't for the ``==>`` that appears in the middle.
    """
    if text.count('=') == 2:  # old format
        regex = re.compile(
            r'(?P<prev_comm>\S.*):(?P<prev_pid>\d+) \[(?P<prev_prio>\d+)\] (?P<status>\S+)'
            r' ==> '
            r'(?P<next_comm>\S.*):(?P<next_pid>\d+) \[(?P<next_prio>\d+)\]'
        )
        parser_func = regex_body_parser(regex)
        return parser_func(event, text)
    else:  # there are more than two "=" -- new format
        return default_body_parser(event, text.replace('==>', ''))

def sched_stat_parser(event, text):
    """
    sched_stat_* events unclude the units, "[ns]", in an otherwise
    regular key=value sequence; so the units  need to be stripped out first.
    """
    return default_body_parser(event, text.replace(' [ns]', ''))

def sched_wakeup_parser(event, text):
    regex = re.compile(r'(?P<comm>\S+):(?P<pid>\d+) \[(?P<prio>\d+)\] success=(?P<success>\d) CPU:(?P<cpu>\d+)')
    parse_func = regex_body_parser(regex)
    return parse_func(event, text)

# Maps event onto the corresponding parser for its body text. A parser may be
# a callable with signature
#
#   parser(event, bodytext)
#
# a re.RegexObject, or a string (in which case it will be compiled into a
# regex). In case of a string/regex, its named groups will be used to populate
# the event's attributes.
EVENT_PARSER_MAP = {
    'sched_stat_blocked': sched_stat_parser,
    'sched_stat_iowait': sched_stat_parser,
    'sched_stat_runtime': sched_stat_parser,
    'sched_stat_sleep': sched_stat_parser,
    'sched_stat_wait': sched_stat_parser,
    'sched_switch': sched_switch_parser,
    'sched_wakeup': sched_wakeup_parser,
    'sched_wakeup_new': sched_wakeup_parser,
}

TRACE_EVENT_REGEX = re.compile(r'^\s+(?P<thread>\S+.*?\S+)\s+\[(?P<cpu_id>\d+)\]\s+(?P<ts>[\d.]+):\s+'
                               r'(?P<name>[^:]+):\s+(?P<body>.*?)\s*$')

HEADER_REGEX = re.compile(r'^\s*(?:version|cpus)\s*=\s*([\d.]+)\s*$')

DROPPED_EVENTS_REGEX = re.compile(r'CPU:(?P<cpu_id>\d+) \[\d*\s*EVENTS DROPPED\]')

EMPTY_CPU_REGEX = re.compile(r'CPU \d+ is empty')

class TraceCmdParser(object):
    """
    A parser for textual representation of ftrace as reported by trace-cmd

    """

    def __init__(self, filter_markers=True):
        """
        Initialize a new trace parser.

        :param filter_markers: Specifies whether the trace before the start
                               marker and after the stop marker should be
                               filtered out (so only events between the two
                               markers will be reported). This maybe overriden
                               based on `check_for_markers` parameter of
                               `parse()`

        """
        self.filter_markers = filter_markers

    def parse(self, filepath, events=None, check_for_markers=True):  # pylint: disable=too-many-branches,too-many-locals
        """
        This is a generator for the trace event stream.

        :param filepath: The path to the file containg text trace as reported
                         by trace-cmd
        :param events: A list of event names to be reported; if not specified,
                       all events will be reported.
        :param check_for_markers: Check if the start/stop markers are present
                                  in the trace and ensure that `filter_markers`
                                  is `False` if they aren't

        """
        inside_maked_region = False
        filters = [re.compile('^{}$'.format(e)) for e in (events or [])]
        filter_markers = self.filter_markers
        if filter_markers and check_for_markers:
            with open(filepath) as fh:
                for line in fh:
                    if TRACE_MARKER_START in line:
                        break
                else:
                    # maker not found force filtering by marker to False
                    filter_markers = False

        with open(filepath) as fh:
            for line in fh:
                # if processing trace markers, skip marker lines as well as all
                # lines outside marked region
                if filter_markers:
                    if not inside_maked_region:
                        if TRACE_MARKER_START in line:
                            inside_maked_region = True
                        continue
                    elif TRACE_MARKER_STOP in line:
                        inside_maked_region = False
                        continue

                match = DROPPED_EVENTS_REGEX.search(line)
                if match:
                    yield DroppedEventsEvent(match.group('cpu_id'))
                    continue

                matched = False
                for rx in [HEADER_REGEX, EMPTY_CPU_REGEX]:
                    match = rx.search(line)
                    if match:
                        logger.debug(line.strip())
                        matched = True
                        break
                if matched:
                    continue

                match = TRACE_EVENT_REGEX.search(line)
                if not match:
                    logger.warning('Invalid trace event: "{}"'.format(line))
                    continue

                event_name = match.group('name')

                if filters:
                    found = False
                    for f in filters:
                        if f.search(event_name):
                            found = True
                            break
                    if not found:
                        continue

                body_parser = EVENT_PARSER_MAP.get(event_name, default_body_parser)
                if isinstance(body_parser, basestring) or isinstance(body_parser, re._pattern_type):  # pylint: disable=protected-access
                    body_parser = regex_body_parser(body_parser)
                yield TraceCmdEvent(parser=body_parser, **match.groupdict())

<|endoftext|>


<commit_before>from os.path import dirname, join
from StringIO import StringIO
from urlparse import urlunparse

from openspending.model import Dataset, Source
from openspending.model import meta as db
from openspending.lib import json

from openspending.importer import CSVImporter

from ... import DatabaseTestCase, helpers as h

def csvimport_fixture_path(name, path):
    return h.fixture_path('csv_import/%s/%s' % (name, path))

def csvimport_fixture_file(name, path):
    try:
        fp = open(csvimport_fixture_path(name, path))
    except IOError:
        if name == 'default':
            fp = None
        else:
            fp = csvimport_fixture_file('default', path)

    return fp

def csvimport_fixture(name):
    model_fp = csvimport_fixture_file(name, 'model.json')
    mapping_fp = csvimport_fixture_file(name, 'mapping.json')
    model = json.load(model_fp)
    if mapping_fp:
        model['mapping'] = json.load(mapping_fp)
    dataset = Dataset(model)
    dataset.generate()
    db.session.add(dataset)
    data_path = csvimport_fixture_path(name, 'data.csv')
    user = h.make_account()
    source = Source(dataset, user, data_path)
    db.session.add(source)
    db.session.commit()
    return source

class TestCSVImporter(DatabaseTestCase):

    def test_successful_import(self):
        source = csvimport_fixture('successful_import')
        importer = CSVImporter(source)
        importer.run()
        dataset = db.session.query(Dataset).first()
        h.assert_true(dataset is not None, "Dataset should not be None")
        h.assert_equal(dataset.name, "test-csv")
        entries = dataset.entries()
        h.assert_equal(len(list(entries)), 4)

        # TODO: provenance
        entry = list(dataset.entries(limit=1, offset=1)).pop()
        h.assert_true(entry is not None,
                      "Entry with name could not be found")
        h.assert_equal(entry['amount'], 66097.77)

    def test_no_dimensions_for_measures(self):
        source = csvimport_fixture('simple')
        importer = CSVImporter(source)
        importer.run()
        dataset = db.session.query(Dataset).first()

        dimensions = [str(d.name) for d in dataset.dimensions]
        h.assert_equal(sorted(dimensions), ['entry_id', 'from', 'time', 'to'])

    def test_successful_import_with_simple_testdata(self):
        source = csvimport_fixture('simple')
        importer = CSVImporter(source)
        importer.run()
        h.assert_equal(importer.errors, 0)

        dataset = db.session.query(Dataset).first()
        h.assert_true(dataset is not None, "Dataset should not be None")

        entries = list(dataset.entries())
        h.assert_equal(len(entries), 5)

        entry = entries[0]
        h.assert_equal(entry['from']['label'], 'Test From')
        h.assert_equal(entry['to']['label'], 'Test To')
        h.assert_equal(entry['time']['name'], '2010-01-01')
        h.assert_equal(entry['amount'], 100.00)

    def test_import_errors(self):
        source = csvimport_fixture('import_errors')

        importer = CSVImporter(source)
        importer.run(dry_run=True)
        h.assert_true(importer.errors > 1, "Should have errors")
        records = list(importer._run.records)
        h.assert_equal(records[0].row, 1,
                       "Should detect missing date colum in line 1")

    def test_empty_csv(self):
        source = csvimport_fixture('default')
        source.url = 'file:///dev/null'
        importer = CSVImporter(source)
        importer.run(dry_run=True)

        h.assert_equal(importer.errors, 2)
        records = list(importer._run.records)
        h.assert_equal(records[0].row, 0)
        h.assert_equal(records[1].row, 0)
        h.assert_true("Didn't read any lines of data" in str(records[1].message))

    def test_malformed_csv(self):
        source = csvimport_fixture('malformed')
        importer = CSVImporter(source)
        importer.run(dry_run=True)
        h.assert_equal(importer.errors, 1)

    def test_erroneous_values(self):
        source = csvimport_fixture('erroneous_values')
        importer = CSVImporter(source)
        importer.run(dry_run=True)
        h.assert_equal(importer.errors, 2)
        records = list(importer._run.records)
        h.assert_true("time" in records[1].attribute,
                      "Should find badly formatted date")
        h.assert_equal(records[1].row, 5)

    def test_error_with_empty_additional_date(self):
        source = csvimport_fixture('empty_additional_date')
        importer = CSVImporter(source)
        importer.run()
        # We are currently not able to import date cells without a value. See:
        # http://trac.openspending.org/ticket/170
        h.assert_equal(importer.errors, 1)

    def test_quoting(self):
        source = csvimport_fixture('quoting')
        importer = CSVImporter(source)
        importer.run()
        h.assert_equal(importer.errors, 0)

class TestCSVImportDatasets(DatabaseTestCase):

    datasets_to_test = ('lbhf', 'mexico', 'sample', 'uganda')

    def count_lines_in_stream(self, f):
        try:
            return len(f.read().splitlines())
        finally:
            f.seek(0)

    def _test_import(self, name):
        source = csvimport_fixture(name)
        data = open(source.url)
        lines = self.count_lines_in_stream(data) - 1 # -1 for header row

        importer = CSVImporter(source)
        importer.run()

        h.assert_equal(importer.errors, 0)

        # check correct number of entries
        dataset = db.session.query(Dataset).first()
        entries = list(dataset.entries())
        h.assert_equal(len(entries), lines)

    def test_all_imports(self):
        for dir in self.datasets_to_test:
            yield self._test_import, dir

<commit_msg>Updated test for erroneous values. The test now also grabs the unique key constraint errors (it's a dry run).<commit_after>from os.path import dirname, join
from StringIO import StringIO
from urlparse import urlunparse

from openspending.model import Dataset, Source
from openspending.model import meta as db
from openspending.lib import json

from openspending.importer import CSVImporter

from ... import DatabaseTestCase, helpers as h

def csvimport_fixture_path(name, path):
    return h.fixture_path('csv_import/%s/%s' % (name, path))

def csvimport_fixture_file(name, path):
    try:
        fp = open(csvimport_fixture_path(name, path))
    except IOError:
        if name == 'default':
            fp = None
        else:
            fp = csvimport_fixture_file('default', path)

    return fp

def csvimport_fixture(name):
    model_fp = csvimport_fixture_file(name, 'model.json')
    mapping_fp = csvimport_fixture_file(name, 'mapping.json')
    model = json.load(model_fp)
    if mapping_fp:
        model['mapping'] = json.load(mapping_fp)
    dataset = Dataset(model)
    dataset.generate()
    db.session.add(dataset)
    data_path = csvimport_fixture_path(name, 'data.csv')
    user = h.make_account()
    source = Source(dataset, user, data_path)
    db.session.add(source)
    db.session.commit()
    return source

class TestCSVImporter(DatabaseTestCase):

    def test_successful_import(self):
        source = csvimport_fixture('successful_import')
        importer = CSVImporter(source)
        importer.run()
        dataset = db.session.query(Dataset).first()
        h.assert_true(dataset is not None, "Dataset should not be None")
        h.assert_equal(dataset.name, "test-csv")
        entries = dataset.entries()
        h.assert_equal(len(list(entries)), 4)

        # TODO: provenance
        entry = list(dataset.entries(limit=1, offset=1)).pop()
        h.assert_true(entry is not None,
                      "Entry with name could not be found")
        h.assert_equal(entry['amount'], 66097.77)

    def test_no_dimensions_for_measures(self):
        source = csvimport_fixture('simple')
        importer = CSVImporter(source)
        importer.run()
        dataset = db.session.query(Dataset).first()

        dimensions = [str(d.name) for d in dataset.dimensions]
        h.assert_equal(sorted(dimensions), ['entry_id', 'from', 'time', 'to'])

    def test_successful_import_with_simple_testdata(self):
        source = csvimport_fixture('simple')
        importer = CSVImporter(source)
        importer.run()
        h.assert_equal(importer.errors, 0)

        dataset = db.session.query(Dataset).first()
        h.assert_true(dataset is not None, "Dataset should not be None")

        entries = list(dataset.entries())
        h.assert_equal(len(entries), 5)

        entry = entries[0]
        h.assert_equal(entry['from']['label'], 'Test From')
        h.assert_equal(entry['to']['label'], 'Test To')
        h.assert_equal(entry['time']['name'], '2010-01-01')
        h.assert_equal(entry['amount'], 100.00)

    def test_import_errors(self):
        source = csvimport_fixture('import_errors')

        importer = CSVImporter(source)
        importer.run(dry_run=True)
        h.assert_true(importer.errors > 1, "Should have errors")
        records = list(importer._run.records)
        h.assert_equal(records[0].row, 1,
                       "Should detect missing date colum in line 1")

    def test_empty_csv(self):
        source = csvimport_fixture('default')
        source.url = 'file:///dev/null'
        importer = CSVImporter(source)
        importer.run(dry_run=True)

        h.assert_equal(importer.errors, 2)
        records = list(importer._run.records)
        h.assert_equal(records[0].row, 0)
        h.assert_equal(records[1].row, 0)
        h.assert_true("Didn't read any lines of data" in str(records[1].message))

    def test_malformed_csv(self):
        source = csvimport_fixture('malformed')
        importer = CSVImporter(source)
        importer.run(dry_run=True)
        h.assert_equal(importer.errors, 1)

    def test_erroneous_values(self):
        source = csvimport_fixture('erroneous_values')
        importer = CSVImporter(source)
        importer.run(dry_run=True)

        # Expected failures:
        # * unique key constraint not met (2x)
        # * amount cannot be parsed
        # * time cannot be parse
        h.assert_equal(importer.errors, 4)
        records = list(importer._run.records)
        # The fourth record should be about badly formed date
        h.assert_true("time" in records[3].attribute,
                      "Should find badly formatted date")
        # The row number of the badly formed date should be 5
        h.assert_equal(records[3].row, 5)

    def test_error_with_empty_additional_date(self):
        source = csvimport_fixture('empty_additional_date')
        importer = CSVImporter(source)
        importer.run()
        # We are currently not able to import date cells without a value. See:
        # http://trac.openspending.org/ticket/170
        h.assert_equal(importer.errors, 1)

    def test_quoting(self):
        source = csvimport_fixture('quoting')
        importer = CSVImporter(source)
        importer.run()
        h.assert_equal(importer.errors, 0)

class TestCSVImportDatasets(DatabaseTestCase):

    datasets_to_test = ('lbhf', 'mexico', 'sample', 'uganda')

    def count_lines_in_stream(self, f):
        try:
            return len(f.read().splitlines())
        finally:
            f.seek(0)

    def _test_import(self, name):
        source = csvimport_fixture(name)
        data = open(source.url)
        lines = self.count_lines_in_stream(data) - 1 # -1 for header row

        importer = CSVImporter(source)
        importer.run()

        h.assert_equal(importer.errors, 0)

        # check correct number of entries
        dataset = db.session.query(Dataset).first()
        entries = list(dataset.entries())
        h.assert_equal(len(entries), lines)

    def test_all_imports(self):
        for dir in self.datasets_to_test:
            yield self._test_import, dir

<|endoftext|>


<commit_before># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt

from __future__ import unicode_literals
import frappe
from frappe.utils import cstr, has_gravatar, cint
from frappe import _
from frappe.model.document import Document
from frappe.core.doctype.dynamic_link.dynamic_link import deduplicate_dynamic_links
from six import iteritems
from past.builtins import cmp
from frappe.model.naming import append_number_if_name_exists
from frappe.contacts.address_and_contact import set_link_title

import functools

class Contact(Document):
	def autoname(self):
		# concat first and last name
		self.name = " ".join(filter(None,
			[cstr(self.get(f)).strip() for f in ["first_name", "last_name"]]))

		if frappe.db.exists("Contact", self.name):
			self.name = append_number_if_name_exists('Contact', self.name)

		# concat party name if reqd
		for link in self.links:
			self.name = self.name + '-' + link.link_name.strip()
			break

	def validate(self):
		self.set_primary_email()
		self.set_primary("phone")
		self.set_primary("mobile_no")

		self.set_user()

		set_link_title(self)

		if self.email_id and not self.image:
			self.image = has_gravatar(self.email_id)

		if self.get("sync_with_google_contacts") and not self.get("google_contacts"):
			frappe.throw(_("Select Google Contacts to which contact should be synced."))

		deduplicate_dynamic_links(self)

	def set_user(self):
		if not self.user and self.email_id:
			self.user = frappe.db.get_value("User", {"email": self.email_id})

	def get_link_for(self, link_doctype):
		'''Return the link name, if exists for the given link DocType'''
		for link in self.links:
			if link.link_doctype==link_doctype:
				return link.link_name

		return None

	def has_link(self, doctype, name):
		for link in self.links:
			if link.link_doctype==doctype and link.link_name== name:
				return True

	def has_common_link(self, doc):
		reference_links = [(link.link_doctype, link.link_name) for link in doc.links]
		for link in self.links:
			if (link.link_doctype, link.link_name) in reference_links:
				return True

	def add_email(self, email_id, is_primary=0, autosave=False):
		if not frappe.db.exists("Contact Email", {"email_id": email_id, "parent": self.name}):
			self.append("email_ids", {
				"email_id": email_id,
				"is_primary": is_primary
			})

			if autosave:
				self.save(ignore_permissions=True)

	def add_phone(self, phone, is_primary_phone=0, is_primary_mobile_no=0, autosave=False):
		if not frappe.db.exists("Contact Phone", {"phone": phone, "parent": self.name}):
			self.append("phone_nos", {
				"phone": phone,
				"is_primary_phone": is_primary_phone,
				"is_primary_mobile_no": is_primary_mobile_no
			})

			if autosave:
				self.save(ignore_permissions=True)

	def set_primary_email(self):
		if not self.email_ids:
			self.email_id = ""
			return

		if len([email.email_id for email in self.email_ids if email.is_primary]) > 1:
			frappe.throw(_("Only one {0} can be set as primary.").format(frappe.bold("Email ID")))

		for d in self.email_ids:
			if d.is_primary == 1:
				self.email_id = d.email_id.strip()
				break

	def set_primary(self, fieldname):
		# Used to set primary mobile and phone no.
		if len(self.phone_nos) == 0:
			setattr(self, fieldname, "")
			return

		field_name = "is_primary_" + fieldname

		is_primary = [phone.phone for phone in self.phone_nos if phone.get(field_name)]

		if len(is_primary) > 1:
			frappe.throw(_("Only one {0} can be set as primary.").format(frappe.bold(frappe.unscrub(fieldname))))

		for d in self.phone_nos:
			if d.get(field_name) == 1:
				setattr(self, fieldname, d.phone)
				break

def get_default_contact(doctype, name):
	'''Returns default contact for the given doctype, name'''
	out = frappe.db.sql('''select parent,
			IFNULL((select is_primary_contact from tabContact c where c.name = dl.parent), 0)
				as is_primary_contact
		from
			`tabDynamic Link` dl
		where
			dl.link_doctype=%s and
			dl.link_name=%s and
			dl.parenttype = "Contact"''', (doctype, name))

	if out:
		return sorted(out, key = functools.cmp_to_key(lambda x,y: cmp(cint(y[1]), cint(x[1]))))[0][0]
	else:
		return None

@frappe.whitelist()
def invite_user(contact):
	contact = frappe.get_doc("Contact", contact)

	if not contact.email_id:
		frappe.throw(_("Please set Email Address"))

	if contact.has_permission("write"):
		user = frappe.get_doc({
			"doctype": "User",
			"first_name": contact.first_name,
			"last_name": contact.last_name,
			"email": contact.email_id,
			"user_type": "Website User",
			"send_welcome_email": 1
		}).insert(ignore_permissions = True)

		return user.name

@frappe.whitelist()
def get_contact_details(contact):
	contact = frappe.get_doc("Contact", contact)
	out = {
		"contact_person": contact.get("name"),
		"contact_display": " ".join(filter(None,
			[contact.get("salutation"), contact.get("first_name"), contact.get("last_name")])),
		"contact_email": contact.get("email_id"),
		"contact_mobile": contact.get("mobile_no"),
		"contact_phone": contact.get("phone"),
		"contact_designation": contact.get("designation"),
		"contact_department": contact.get("department")
	}
	return out

def update_contact(doc, method):
	'''Update contact when user is updated, if contact is found. Called via hooks'''
	contact_name = frappe.db.get_value("Contact", {"email_id": doc.name})
	if contact_name:
		contact = frappe.get_doc("Contact", contact_name)
		for key in ("first_name", "last_name", "phone"):
			if doc.get(key):
				contact.set(key, doc.get(key))
		contact.flags.ignore_mandatory = True
		contact.save(ignore_permissions=True)

@frappe.whitelist()
@frappe.validate_and_sanitize_search_inputs
def contact_query(doctype, txt, searchfield, start, page_len, filters):
	from frappe.desk.reportview import get_match_cond

	if not frappe.get_meta("Contact").get_field(searchfield)\
		or searchfield not in frappe.db.DEFAULT_COLUMNS:
		return []

	link_doctype = filters.pop('link_doctype')
	link_name = filters.pop('link_name')

	return frappe.db.sql("""select
			`tabContact`.name, `tabContact`.first_name, `tabContact`.last_name
		from
			`tabContact`, `tabDynamic Link`
		where
			`tabDynamic Link`.parent = `tabContact`.name and
			`tabDynamic Link`.parenttype = 'Contact' and
			`tabDynamic Link`.link_doctype = %(link_doctype)s and
			`tabDynamic Link`.link_name = %(link_name)s and
			`tabContact`.`{key}` like %(txt)s
			{mcond}
		order by
			if(locate(%(_txt)s, `tabContact`.name), locate(%(_txt)s, `tabContact`.name), 99999),
			`tabContact`.idx desc, `tabContact`.name
		limit %(start)s, %(page_len)s """.format(mcond=get_match_cond(doctype), key=searchfield), {
			'txt': '%' + txt + '%',
			'_txt': txt.replace("%", ""),
			'start': start,
			'page_len': page_len,
			'link_name': link_name,
			'link_doctype': link_doctype
		})

@frappe.whitelist()
def address_query(links):
	import json

	links = [{"link_doctype": d.get("link_doctype"), "link_name": d.get("link_name")} for d in json.loads(links)]
	result = []

	for link in links:
		if not frappe.has_permission(doctype=link.get("link_doctype"), ptype="read", doc=link.get("link_name")):
			continue

		res = frappe.db.sql("""
			SELECT `tabAddress`.name
			FROM `tabAddress`, `tabDynamic Link`
			WHERE `tabDynamic Link`.parenttype='Address'
				AND `tabDynamic Link`.parent=`tabAddress`.name
				AND `tabDynamic Link`.link_doctype = %(link_doctype)s
				AND `tabDynamic Link`.link_name = %(link_name)s
		""", {
			"link_doctype": link.get("link_doctype"),
			"link_name": link.get("link_name"),
		}, as_dict=True)

		result.extend([l.name for l in res])

	return result

def get_contact_with_phone_number(number):
	if not number: return

	contacts = frappe.get_all('Contact Phone', filters=[
		['phone', 'like', '%{0}'.format(number)]
	], fields=["parent"], limit=1)

	return contacts[0].parent if contacts else None

def get_contact_name(email_id):
	contact = frappe.get_list("Contact Email", filters={"email_id": email_id}, fields=["parent"], limit=1)
	return contact[0].parent if contact else None
<commit_msg>fix: contact not showing in form (bp #11363) (#11379)<commit_after># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt

from __future__ import unicode_literals
import frappe
from frappe.utils import cstr, has_gravatar, cint
from frappe import _
from frappe.model.document import Document
from frappe.core.doctype.dynamic_link.dynamic_link import deduplicate_dynamic_links
from six import iteritems
from past.builtins import cmp
from frappe.model.naming import append_number_if_name_exists
from frappe.contacts.address_and_contact import set_link_title

import functools

class Contact(Document):
	def autoname(self):
		# concat first and last name
		self.name = " ".join(filter(None,
			[cstr(self.get(f)).strip() for f in ["first_name", "last_name"]]))

		if frappe.db.exists("Contact", self.name):
			self.name = append_number_if_name_exists('Contact', self.name)

		# concat party name if reqd
		for link in self.links:
			self.name = self.name + '-' + link.link_name.strip()
			break

	def validate(self):
		self.set_primary_email()
		self.set_primary("phone")
		self.set_primary("mobile_no")

		self.set_user()

		set_link_title(self)

		if self.email_id and not self.image:
			self.image = has_gravatar(self.email_id)

		if self.get("sync_with_google_contacts") and not self.get("google_contacts"):
			frappe.throw(_("Select Google Contacts to which contact should be synced."))

		deduplicate_dynamic_links(self)

	def set_user(self):
		if not self.user and self.email_id:
			self.user = frappe.db.get_value("User", {"email": self.email_id})

	def get_link_for(self, link_doctype):
		'''Return the link name, if exists for the given link DocType'''
		for link in self.links:
			if link.link_doctype==link_doctype:
				return link.link_name

		return None

	def has_link(self, doctype, name):
		for link in self.links:
			if link.link_doctype==doctype and link.link_name== name:
				return True

	def has_common_link(self, doc):
		reference_links = [(link.link_doctype, link.link_name) for link in doc.links]
		for link in self.links:
			if (link.link_doctype, link.link_name) in reference_links:
				return True

	def add_email(self, email_id, is_primary=0, autosave=False):
		if not frappe.db.exists("Contact Email", {"email_id": email_id, "parent": self.name}):
			self.append("email_ids", {
				"email_id": email_id,
				"is_primary": is_primary
			})

			if autosave:
				self.save(ignore_permissions=True)

	def add_phone(self, phone, is_primary_phone=0, is_primary_mobile_no=0, autosave=False):
		if not frappe.db.exists("Contact Phone", {"phone": phone, "parent": self.name}):
			self.append("phone_nos", {
				"phone": phone,
				"is_primary_phone": is_primary_phone,
				"is_primary_mobile_no": is_primary_mobile_no
			})

			if autosave:
				self.save(ignore_permissions=True)

	def set_primary_email(self):
		if not self.email_ids:
			self.email_id = ""
			return

		if len([email.email_id for email in self.email_ids if email.is_primary]) > 1:
			frappe.throw(_("Only one {0} can be set as primary.").format(frappe.bold("Email ID")))

		for d in self.email_ids:
			if d.is_primary == 1:
				self.email_id = d.email_id.strip()
				break

	def set_primary(self, fieldname):
		# Used to set primary mobile and phone no.
		if len(self.phone_nos) == 0:
			setattr(self, fieldname, "")
			return

		field_name = "is_primary_" + fieldname

		is_primary = [phone.phone for phone in self.phone_nos if phone.get(field_name)]

		if len(is_primary) > 1:
			frappe.throw(_("Only one {0} can be set as primary.").format(frappe.bold(frappe.unscrub(fieldname))))

		for d in self.phone_nos:
			if d.get(field_name) == 1:
				setattr(self, fieldname, d.phone)
				break

def get_default_contact(doctype, name):
	'''Returns default contact for the given doctype, name'''
	out = frappe.db.sql('''select parent,
			IFNULL((select is_primary_contact from tabContact c where c.name = dl.parent), 0)
				as is_primary_contact
		from
			`tabDynamic Link` dl
		where
			dl.link_doctype=%s and
			dl.link_name=%s and
			dl.parenttype = "Contact"''', (doctype, name))

	if out:
		return sorted(out, key = functools.cmp_to_key(lambda x,y: cmp(cint(y[1]), cint(x[1]))))[0][0]
	else:
		return None

@frappe.whitelist()
def invite_user(contact):
	contact = frappe.get_doc("Contact", contact)

	if not contact.email_id:
		frappe.throw(_("Please set Email Address"))

	if contact.has_permission("write"):
		user = frappe.get_doc({
			"doctype": "User",
			"first_name": contact.first_name,
			"last_name": contact.last_name,
			"email": contact.email_id,
			"user_type": "Website User",
			"send_welcome_email": 1
		}).insert(ignore_permissions = True)

		return user.name

@frappe.whitelist()
def get_contact_details(contact):
	contact = frappe.get_doc("Contact", contact)
	out = {
		"contact_person": contact.get("name"),
		"contact_display": " ".join(filter(None,
			[contact.get("salutation"), contact.get("first_name"), contact.get("last_name")])),
		"contact_email": contact.get("email_id"),
		"contact_mobile": contact.get("mobile_no"),
		"contact_phone": contact.get("phone"),
		"contact_designation": contact.get("designation"),
		"contact_department": contact.get("department")
	}
	return out

def update_contact(doc, method):
	'''Update contact when user is updated, if contact is found. Called via hooks'''
	contact_name = frappe.db.get_value("Contact", {"email_id": doc.name})
	if contact_name:
		contact = frappe.get_doc("Contact", contact_name)
		for key in ("first_name", "last_name", "phone"):
			if doc.get(key):
				contact.set(key, doc.get(key))
		contact.flags.ignore_mandatory = True
		contact.save(ignore_permissions=True)

@frappe.whitelist()
@frappe.validate_and_sanitize_search_inputs
def contact_query(doctype, txt, searchfield, start, page_len, filters):
	from frappe.desk.reportview import get_match_cond

	if not frappe.get_meta("Contact").get_field(searchfield)\
		and searchfield not in frappe.db.DEFAULT_COLUMNS:
		return []

	link_doctype = filters.pop('link_doctype')
	link_name = filters.pop('link_name')

	return frappe.db.sql("""select
			`tabContact`.name, `tabContact`.first_name, `tabContact`.last_name
		from
			`tabContact`, `tabDynamic Link`
		where
			`tabDynamic Link`.parent = `tabContact`.name and
			`tabDynamic Link`.parenttype = 'Contact' and
			`tabDynamic Link`.link_doctype = %(link_doctype)s and
			`tabDynamic Link`.link_name = %(link_name)s and
			`tabContact`.`{key}` like %(txt)s
			{mcond}
		order by
			if(locate(%(_txt)s, `tabContact`.name), locate(%(_txt)s, `tabContact`.name), 99999),
			`tabContact`.idx desc, `tabContact`.name
		limit %(start)s, %(page_len)s """.format(mcond=get_match_cond(doctype), key=searchfield), {
			'txt': '%' + txt + '%',
			'_txt': txt.replace("%", ""),
			'start': start,
			'page_len': page_len,
			'link_name': link_name,
			'link_doctype': link_doctype
		})

@frappe.whitelist()
def address_query(links):
	import json

	links = [{"link_doctype": d.get("link_doctype"), "link_name": d.get("link_name")} for d in json.loads(links)]
	result = []

	for link in links:
		if not frappe.has_permission(doctype=link.get("link_doctype"), ptype="read", doc=link.get("link_name")):
			continue

		res = frappe.db.sql("""
			SELECT `tabAddress`.name
			FROM `tabAddress`, `tabDynamic Link`
			WHERE `tabDynamic Link`.parenttype='Address'
				AND `tabDynamic Link`.parent=`tabAddress`.name
				AND `tabDynamic Link`.link_doctype = %(link_doctype)s
				AND `tabDynamic Link`.link_name = %(link_name)s
		""", {
			"link_doctype": link.get("link_doctype"),
			"link_name": link.get("link_name"),
		}, as_dict=True)

		result.extend([l.name for l in res])

	return result

def get_contact_with_phone_number(number):
	if not number: return

	contacts = frappe.get_all('Contact Phone', filters=[
		['phone', 'like', '%{0}'.format(number)]
	], fields=["parent"], limit=1)

	return contacts[0].parent if contacts else None

def get_contact_name(email_id):
	contact = frappe.get_list("Contact Email", filters={"email_id": email_id}, fields=["parent"], limit=1)
	return contact[0].parent if contact else None
<|endoftext|>