<commit_before>from __future__ import print_function

import argparse
from typing import IO
from typing import List
from typing import Optional
from typing import Sequence

PASS = 0
FAIL = 1

class Requirement(object):

    def __init__(self):  # type: () -> None
        super(Requirement, self).__init__()
        self.value = None  # type: Optional[bytes]
        self.comments = []   # type: List[bytes]

    @property
    def name(self):  # type: () -> bytes
        assert self.value is not None, self.value
        if self.value.startswith(b'-e '):
            return self.value.lower().partition(b'=')[-1]

        return self.value.lower().partition(b'==')[0]

    def __lt__(self, requirement):  # type: (Requirement) -> int
        # \n means top of file comment, so always return True,
        # otherwise just do a string comparison with value.
        assert self.value is not None, self.value
        if self.value == b'\n':
            return True
        elif requirement.value == b'\n':
            return False
        else:
            return self.name < requirement.name

def fix_requirements(f):  # type: (IO[bytes]) -> int
    requirements = []  # type: List[Requirement]
    before = list(f)  # type: List[bytes]
    after = []  # type: List[bytes]

    before_string = b''.join(before)

    # adds new line in case one is missing
    # AND a change to the requirements file is needed regardless:
    if before and not before[-1].endswith(b'\n'):
        before[-1] += b'\n'

    # If the file is empty (i.e. only whitespace/newlines) exit early
    if before_string.strip() == b'':
        return PASS

    for line in before:
        # If the most recent requirement object has a value, then it's
        # time to start building the next requirement object.

        if not len(requirements) or requirements[-1].value is not None:
            requirements.append(Requirement())

        requirement = requirements[-1]

        # If we see a newline before any requirements, then this is a
        # top of file comment.
        if len(requirements) == 1 and line.strip() == b'':
            if (
                    len(requirement.comments) and
                    requirement.comments[0].startswith(b'#')
            ):
                requirement.value = b'\n'
            else:
                requirement.comments.append(line)
        elif line.startswith(b'#') or line.strip() == b'':
            requirement.comments.append(line)
        else:
            requirement.value = line

    # if a file ends in a comment, preserve it at the end
    if requirements[-1].value is None:
        rest = requirements.pop().comments
    else:
        rest = []

    # find and remove pkg-resources==0.0.0
    # which is automatically added by broken pip package under Debian
    requirements = [
        req for req in requirements
        if req.value != b'pkg-resources==0.0.0\n'
    ]

    for requirement in sorted(requirements):
        after.extend(requirement.comments)
        assert requirement.value, requirement.value
        after.append(requirement.value)
    after.extend(rest)

    after_string = b''.join(after)

    if before_string == after_string:
        return PASS
    else:
        f.seek(0)
        f.write(after_string)
        f.truncate()
        return FAIL

def main(argv=None):  # type: (Optional[Sequence[str]]) -> int
    parser = argparse.ArgumentParser()
    parser.add_argument('filenames', nargs='*', help='Filenames to fix')
    args = parser.parse_args(argv)

    retv = PASS

    for arg in args.filenames:
        with open(arg, 'rb+') as file_obj:
            ret_for_file = fix_requirements(file_obj)

            if ret_for_file:
                print('Sorting {}'.format(arg))

            retv |= ret_for_file

    return retv

if __name__ == '__main__':
    exit(main())
<commit_msg>remove redundent type annotation<commit_after>from __future__ import print_function

import argparse
from typing import IO
from typing import List
from typing import Optional
from typing import Sequence

PASS = 0
FAIL = 1

class Requirement(object):

    def __init__(self):  # type: () -> None
        super(Requirement, self).__init__()
        self.value = None  # type: Optional[bytes]
        self.comments = []   # type: List[bytes]

    @property
    def name(self):  # type: () -> bytes
        assert self.value is not None, self.value
        if self.value.startswith(b'-e '):
            return self.value.lower().partition(b'=')[-1]

        return self.value.lower().partition(b'==')[0]

    def __lt__(self, requirement):  # type: (Requirement) -> int
        # \n means top of file comment, so always return True,
        # otherwise just do a string comparison with value.
        assert self.value is not None, self.value
        if self.value == b'\n':
            return True
        elif requirement.value == b'\n':
            return False
        else:
            return self.name < requirement.name

def fix_requirements(f):  # type: (IO[bytes]) -> int
    requirements = []  # type: List[Requirement]
    before = list(f)
    after = []  # type: List[bytes]

    before_string = b''.join(before)

    # adds new line in case one is missing
    # AND a change to the requirements file is needed regardless:
    if before and not before[-1].endswith(b'\n'):
        before[-1] += b'\n'

    # If the file is empty (i.e. only whitespace/newlines) exit early
    if before_string.strip() == b'':
        return PASS

    for line in before:
        # If the most recent requirement object has a value, then it's
        # time to start building the next requirement object.

        if not len(requirements) or requirements[-1].value is not None:
            requirements.append(Requirement())

        requirement = requirements[-1]

        # If we see a newline before any requirements, then this is a
        # top of file comment.
        if len(requirements) == 1 and line.strip() == b'':
            if (
                    len(requirement.comments) and
                    requirement.comments[0].startswith(b'#')
            ):
                requirement.value = b'\n'
            else:
                requirement.comments.append(line)
        elif line.startswith(b'#') or line.strip() == b'':
            requirement.comments.append(line)
        else:
            requirement.value = line

    # if a file ends in a comment, preserve it at the end
    if requirements[-1].value is None:
        rest = requirements.pop().comments
    else:
        rest = []

    # find and remove pkg-resources==0.0.0
    # which is automatically added by broken pip package under Debian
    requirements = [
        req for req in requirements
        if req.value != b'pkg-resources==0.0.0\n'
    ]

    for requirement in sorted(requirements):
        after.extend(requirement.comments)
        assert requirement.value, requirement.value
        after.append(requirement.value)
    after.extend(rest)

    after_string = b''.join(after)

    if before_string == after_string:
        return PASS
    else:
        f.seek(0)
        f.write(after_string)
        f.truncate()
        return FAIL

def main(argv=None):  # type: (Optional[Sequence[str]]) -> int
    parser = argparse.ArgumentParser()
    parser.add_argument('filenames', nargs='*', help='Filenames to fix')
    args = parser.parse_args(argv)

    retv = PASS

    for arg in args.filenames:
        with open(arg, 'rb+') as file_obj:
            ret_for_file = fix_requirements(file_obj)

            if ret_for_file:
                print('Sorting {}'.format(arg))

            retv |= ret_for_file

    return retv

if __name__ == '__main__':
    exit(main())
<|endoftext|>


<commit_before>import sqlite3
import re
conn = sqlite3.connect('sitedb.sqlite')
cur = conn.cursor()

cur.execute('''UPDATE Data set 'site_id' = NULL''')
cur.execute('''SELECT * FROM Sites''')
results = [cur.fetchall()]
for row in results[0]:
	site = row[1]
	id = row[0]

	cur.execute('''
				UPDATE Data SET site_id = (?) WHERE site = (?) ''', (id, site))
# made executive decision to set Ventura Marina Harbor equal to Ventura Harbor BB
cur.execute('''UPDATE Data SET site_id = 2103 WHERE site = "Ventura Marina Harbor"
			''')
# executive decision to set Ormond Beach Pier equal to Hueneme Pier
cur.execute('''UPDATE Data SET site_id = 1001 WHERE site = "Ormond Beach Pier"
			''')

conn.commit()
cur.close()<commit_msg>Delete add_site_id_to_data_db.py<commit_after><|endoftext|>


<commit_before>import os
import genapi

h_template = r"""
#ifdef _UMATHMODULE

#ifdef NPY_ENABLE_SEPARATE_COMPILATION
extern NPY_NO_EXPORT PyTypeObject PyUFunc_Type;
#else
NPY_NO_EXPORT PyTypeObject PyUFunc_Type;
#endif

%s

#else

#if defined(PY_UFUNC_UNIQUE_SYMBOL)
#define PyUFunc_API PY_UFUNC_UNIQUE_SYMBOL
#endif

#if defined(NO_IMPORT) || defined(NO_IMPORT_UFUNC)
extern void **PyUFunc_API;
#else
#if defined(PY_UFUNC_UNIQUE_SYMBOL)
void **PyUFunc_API;
#else
static void **PyUFunc_API=NULL;
#endif
#endif

#define PyUFunc_Type (*(PyTypeObject *)PyUFunc_API[0])

%s

static int
_import_umath(void)
{
  PyObject *numpy = PyImport_ImportModule("numpy.core.umath");
  PyObject *c_api = NULL;

  if (numpy == NULL) return -1;
  c_api = PyObject_GetAttrString(numpy, "_UFUNC_API");
  if (c_api == NULL) {Py_DECREF(numpy); return -1;}
  if (PyCObject_Check(c_api)) {
      PyUFunc_API = (void **)PyCObject_AsVoidPtr(c_api);
  }
  Py_DECREF(c_api);
  Py_DECREF(numpy);
  if (PyUFunc_API == NULL) return -1;
  return 0;
}

#define import_umath() { UFUNC_NOFPE if (_import_umath() < 0) {PyErr_Print(); PyErr_SetString(PyExc_ImportError, "numpy.core.umath failed to import"); return; }}

#define import_umath1(ret) { UFUNC_NOFPE if (_import_umath() < 0) {PyErr_Print(); PyErr_SetString(PyExc_ImportError, "numpy.core.umath failed to import"); return ret; }}

#define import_umath2(msg, ret) { UFUNC_NOFPE if (_import_umath() < 0) {PyErr_Print(); PyErr_SetString(PyExc_ImportError, msg); return ret; }}

#define import_ufunc() { UFUNC_NOFPE if (_import_umath() < 0) {PyErr_Print(); PyErr_SetString(PyExc_ImportError, "numpy.core.umath failed to import"); }}

#endif
"""

c_template = r"""
/* These pointers will be stored in the C-object for use in other
    extension modules
*/

void *PyUFunc_API[] = {
        (void *) &PyUFunc_Type,
%s
};
"""

def generate_api(output_dir, force=False):
    basename = 'ufunc_api'

    h_file = os.path.join(output_dir, '__%s.h' % basename)
    c_file = os.path.join(output_dir, '__%s.c' % basename)
    d_file = os.path.join(output_dir, '%s.txt' % basename)
    targets = (h_file, c_file, d_file)

    sources = ['ufunc_api_order.txt']

    if (not force and not genapi.should_rebuild(targets, sources + [__file__])):
        return targets
    else:
        do_generate_api(targets, sources)

    return targets

def do_generate_api(targets, sources):
    header_file = targets[0]
    c_file = targets[1]
    doc_file = targets[2]

    ufunc_api_list = genapi.get_api_functions('UFUNC_API', sources[0])

    # API fixes for __arrayobject_api.h

    fixed = 1
    nummulti = len(ufunc_api_list)
    numtotal = fixed + nummulti

    module_list = []
    extension_list = []
    init_list = []

    # set up object API
    genapi.add_api_list(fixed, 'PyUFunc_API', ufunc_api_list,
                        module_list, extension_list, init_list)

    # Write to header
    fid = open(header_file, 'w')
    s = h_template % ('\n'.join(module_list), '\n'.join(extension_list))
    fid.write(s)
    fid.close()

    # Write to c-code
    fid = open(c_file, 'w')
    s = c_template % '\n'.join(init_list)
    fid.write(s)
    fid.close()

    # Write to documentation
    fid = open(doc_file, 'w')
    fid.write('''
=================
Numpy Ufunc C-API
=================
''')
    for func in ufunc_api_list:
        fid.write(func.to_ReST())
        fid.write('\n\n')
    fid.close()

    return targets
<commit_msg>ENH: use api dict for ufunc as well.<commit_after>import os
import genapi
import genapi2

import numpy_api

from genapi2 import \
        TypeApi, GlobalVarApi, FunctionApi, BoolValuesApi

h_template = r"""
#ifdef _UMATHMODULE

#ifdef NPY_ENABLE_SEPARATE_COMPILATION
extern NPY_NO_EXPORT PyTypeObject PyUFunc_Type;
#else
NPY_NO_EXPORT PyTypeObject PyUFunc_Type;
#endif

%s

#else

#if defined(PY_UFUNC_UNIQUE_SYMBOL)
#define PyUFunc_API PY_UFUNC_UNIQUE_SYMBOL
#endif

#if defined(NO_IMPORT) || defined(NO_IMPORT_UFUNC)
extern void **PyUFunc_API;
#else
#if defined(PY_UFUNC_UNIQUE_SYMBOL)
void **PyUFunc_API;
#else
static void **PyUFunc_API=NULL;
#endif
#endif

%s

static int
_import_umath(void)
{
  PyObject *numpy = PyImport_ImportModule("numpy.core.umath");
  PyObject *c_api = NULL;

  if (numpy == NULL) return -1;
  c_api = PyObject_GetAttrString(numpy, "_UFUNC_API");
  if (c_api == NULL) {Py_DECREF(numpy); return -1;}
  if (PyCObject_Check(c_api)) {
      PyUFunc_API = (void **)PyCObject_AsVoidPtr(c_api);
  }
  Py_DECREF(c_api);
  Py_DECREF(numpy);
  if (PyUFunc_API == NULL) return -1;
  return 0;
}

#define import_umath() { UFUNC_NOFPE if (_import_umath() < 0) {PyErr_Print(); PyErr_SetString(PyExc_ImportError, "numpy.core.umath failed to import"); return; }}

#define import_umath1(ret) { UFUNC_NOFPE if (_import_umath() < 0) {PyErr_Print(); PyErr_SetString(PyExc_ImportError, "numpy.core.umath failed to import"); return ret; }}

#define import_umath2(msg, ret) { UFUNC_NOFPE if (_import_umath() < 0) {PyErr_Print(); PyErr_SetString(PyExc_ImportError, msg); return ret; }}

#define import_ufunc() { UFUNC_NOFPE if (_import_umath() < 0) {PyErr_Print(); PyErr_SetString(PyExc_ImportError, "numpy.core.umath failed to import"); }}

#endif
"""

c_template = r"""
/* These pointers will be stored in the C-object for use in other
    extension modules
*/

void *PyUFunc_API[] = {
%s
};
"""

def generate_api(output_dir, force=False):
    basename = 'ufunc_api'

    h_file = os.path.join(output_dir, '__%s.h' % basename)
    c_file = os.path.join(output_dir, '__%s.c' % basename)
    d_file = os.path.join(output_dir, '%s.txt' % basename)
    targets = (h_file, c_file, d_file)

    sources = ['ufunc_api_order.txt']

    if (not force and not genapi.should_rebuild(targets, sources + [__file__])):
        return targets
    else:
        do_generate_api(targets, sources)

    return targets

def do_generate_api(targets, sources):
    header_file = targets[0]
    c_file = targets[1]
    doc_file = targets[2]

    ufunc_api_index = genapi2.merge_api_dicts((
            numpy_api.ufunc_funcs_api,
            numpy_api.ufunc_types_api))
    genapi2.check_api_dict(ufunc_api_index)

    ufunc_api_list = genapi2.get_api_functions('UFUNC_API', numpy_api.ufunc_funcs_api)

    # Create dict name -> *Api instance
    ufunc_api_dict = {}
    api_name = 'PyUFunc_API'
    for f in ufunc_api_list:
        name = f.name
        index = ufunc_api_index[name]
        ufunc_api_dict[name] = FunctionApi(f.name, index, f.return_type,
                                           f.args, api_name)

    for name, index in numpy_api.ufunc_types_api.items():
        ufunc_api_dict[name] = TypeApi(name, index, 'PyTypeObject', api_name)

    # set up object API
    module_list = []
    extension_list = []
    init_list = []

    for name, index in genapi2.order_dict(ufunc_api_index):
        api_item = ufunc_api_dict[name]
        extension_list.append(api_item.define_from_array_api_string())
        init_list.append(api_item.array_api_define())
        module_list.append(api_item.internal_define())

    # Write to header
    fid = open(header_file, 'w')
    s = h_template % ('\n'.join(module_list), '\n'.join(extension_list))
    fid.write(s)
    fid.close()

    # Write to c-code
    fid = open(c_file, 'w')
    s = c_template % ',\n'.join(init_list)
    fid.write(s)
    fid.close()

    # Write to documentation
    fid = open(doc_file, 'w')
    fid.write('''
=================
Numpy Ufunc C-API
=================
''')
    for func in ufunc_api_list:
        fid.write(func.to_ReST())
        fid.write('\n\n')
    fid.close()

    return targets
<|endoftext|>


<commit_before>#! /usr/bin/env python3

# Copyright 2019 John Hanley.
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# The software is provided "AS IS", without warranty of any kind, express or
# implied, including but not limited to the warranties of merchantability,
# fitness for a particular purpose and noninfringement. In no event shall
# the authors or copyright holders be liable for any claim, damages or
# other liability, whether in an action of contract, tort or otherwise,
# arising from, out of or in connection with the software or the use or
# other dealings in the software.

"""Systematically finds aggregate stats for a table's columns."""

import click
import sqlalchemy as sa
import sqlalchemy.sql.sqltypes as sqltypes
import sqlalchemy.dialects.postgresql.base as pg_base
import uszipcode

def get_zipcode_session():
    return uszipcode.SearchEngine().ses

def get_zipcode_cs():
    """Returns a JDBC connect string for the zipcode database."""
    # typical value: sqlite:////Users/foo/.uszipcode/simple_db.sqlite
    return get_zipcode_session().connection().engine.url

class ColumnExplorer:

    def __init__(self, cs_or_engine):
        self.engine = sa.create_engine(cs_or_engine)

    def _get_table(self, table_name):
        kwargs = dict(autoload=True)
        table_short_name = table_name
        if '.' in table_name:
            schema, table_short_name = table_name.split('.')
            kwargs['schema'] = schema
        meta = sa.MetaData(bind=self.engine)
        return sa.Table(table_short_name, meta, **kwargs)

    def show_informative_columns(self, table_name):
        """A column is uninformative if it has a constant value, e.g. always NULL."""
        tbl = self._get_table(table_name)
        cnt, = self.engine.execute(f'select count(*) from {table_name}').fetchone()
        if cnt <= 1:
            return  # Nothing to see here, by definition there are no informative columns.
        for column, _ in self._get_cols(tbl):
            select = f'select count(distinct {column}) from {table_name}'
            cnt, = self.engine.execute(select).fetchone()
            # print(f'{cnt:8d}  {column}')
            if cnt > 1:
                print(column + ',')

    def report(self, table_name, round_digits=3):

        tbl = self._get_table(table_name)
        stat = None
        non_numeric = set([
            sqltypes.TEXT,
            pg_base.TIMESTAMP,
        ])

        cnt, = self.engine.execute(f'select count(*) from {table_name}').fetchone()
        print(f'# {table_name}\n{cnt} rows, {len(tbl.c)} columns\n')

        for column, typ in self._get_cols(tbl):
            print('\n## ' + column)
            for agg in ['min(', 'avg(', 'max(', 'mode',
                        'mode count', 'nulls', 'count(distinct ']:
                params = {}
                select = f'select {agg}{column}) from {table_name}'
                if agg == 'avg(' and typ in non_numeric:
                    continue
                if agg == 'mode':
                    select = (f'select {column}  from {table_name}'
                              f' group by {column}  order by count(*) desc  limit 1')
                if agg == 'mode count':
                    params = dict(val=stat)
                    select = f'select count(*) from {table_name} where {column} = :val'
                if ((agg == 'nulls')
                        or (agg == 'mode count' and stat is None)):
                    select = f'select count(*) from {table_name} where {column} is null'

                stat, = self.engine.execute(sa.text(select), params).fetchone()

                if agg == 'avg(' and stat is not None:
                    stat = round(stat, round_digits)
                if agg == 'nulls':
                    pct = round(100 * stat / cnt, round_digits)
                    stat = f'{stat} ({pct} %)'
                print('-', agg.replace('(', ' '), stat)

        print(f'\n{cnt} rows in {table_name}')

    def _get_cols(self, table):
        for col in table.columns:
            if type(col.type) != sqltypes.BOOLEAN:  # Can't take max(B) of boolean B.
                yield str(col).split('.')[-1], type(col.type)

@click.command()
@click.option('--uri-getter', default='get_zipcode_cs')
@click.option('--table', default='simple_zipcode')
def main(uri_getter, table):
    callable = globals()[uri_getter]
    ce = ColumnExplorer(callable())
    # ce.show_informative_columns(table)
    ce.report(table)

if __name__ == '__main__':
    main()
<commit_msg>Add varchar etc. On empty tables skip the mode.<commit_after>#! /usr/bin/env python3

# Copyright 2019 John Hanley.
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# The software is provided "AS IS", without warranty of any kind, express or
# implied, including but not limited to the warranties of merchantability,
# fitness for a particular purpose and noninfringement. In no event shall
# the authors or copyright holders be liable for any claim, damages or
# other liability, whether in an action of contract, tort or otherwise,
# arising from, out of or in connection with the software or the use or
# other dealings in the software.

"""Systematically finds aggregate stats for a table's columns."""

import click
import sqlalchemy as sa
import sqlalchemy.sql.sqltypes as sqltypes
import sqlalchemy.dialects.postgresql.base as pg_base
import uszipcode

def get_zipcode_session():
    return uszipcode.SearchEngine().ses

def get_zipcode_cs():
    """Returns a JDBC connect string for the zipcode database."""
    # typical value: sqlite:////Users/foo/.uszipcode/simple_db.sqlite
    return get_zipcode_session().connection().engine.url

class ColumnExplorer:

    def __init__(self, cs_or_engine):
        self.engine = sa.create_engine(cs_or_engine)

    def _get_table(self, table_name):
        kwargs = dict(autoload=True)
        table_short_name = table_name
        if '.' in table_name:
            schema, table_short_name = table_name.split('.')
            kwargs['schema'] = schema
        meta = sa.MetaData(bind=self.engine)
        return sa.Table(table_short_name, meta, **kwargs)

    def show_informative_columns(self, table_name):
        """A column is uninformative if it has a constant value, e.g. always NULL."""
        tbl = self._get_table(table_name)
        cnt, = self.engine.execute(f'select count(*) from {table_name}').fetchone()
        if cnt <= 1:
            return  # Nothing to see here, by definition there are no informative columns.
        for column, _ in self._get_cols(tbl):
            select = f'select count(distinct {column}) from {table_name}'
            cnt, = self.engine.execute(select).fetchone()
            # print(f'{cnt:8d}  {column}')
            if cnt > 1:
                print(column + ',')

    def report(self, table_name, round_digits=3):

        tbl = self._get_table(table_name)
        stat = None
        non_numeric = set([
            sqltypes.BLOB,
            sqltypes.BOOLEAN,
            sqltypes.CHAR,
            sqltypes.DATE,
            sqltypes.TEXT,
            sqltypes.VARCHAR,
            pg_base.CIDR,
            pg_base.ENUM,
            pg_base.INET,
            pg_base.INTERVAL,
            pg_base.TIME,
            pg_base.TIMESTAMP,
        ])

        cnt, = self.engine.execute(f'select count(*) from {table_name}').fetchone()
        print(f'# {table_name}\n{cnt} rows, {len(tbl.c)} columns\n')

        for column, typ in self._get_cols(tbl):
            print('\n## ' + column)
            for agg in ['min(', 'avg(', 'max(', 'mode',
                        'mode count', 'nulls', 'count(distinct ']:
                params = {}
                select = f'select {agg}{column}) from {table_name}'
                if agg == 'avg(' and typ in non_numeric:
                    continue
                if agg == 'mode':
                    select = (f'select {column}  from {table_name}'
                              f' group by {column}  order by count(*) desc  limit 1')
                    if cnt == 0:
                        continue
                if agg == 'mode count':
                    params = dict(val=stat)
                    select = f'select count(*) from {table_name} where {column} = :val'
                if ((agg == 'nulls')
                        or (agg == 'mode count' and stat is None)):
                    select = f'select count(*) from {table_name} where {column} is null'

                stat, = self.engine.execute(sa.text(select), params).fetchone()

                if agg == 'avg(' and stat is not None:
                    stat = round(stat, round_digits)
                if agg == 'nulls':
                    pct = round(100 * stat / cnt, round_digits)
                    stat = f'{stat} ({pct} %)'
                print('-', agg.replace('(', ' '), stat)

        print(f'\n{cnt} rows in {table_name}')

    def _get_cols(self, table):
        for col in table.columns:
            if type(col.type) != sqltypes.BOOLEAN:  # Can't take max(B) of boolean B.
                yield str(col).split('.')[-1], type(col.type)

@click.command()
@click.option('--uri-getter', default='get_zipcode_cs')
@click.option('--table', default='simple_zipcode')
def main(uri_getter, table):
    callable = globals()[uri_getter]
    ce = ColumnExplorer(callable())
    # ce.show_informative_columns(table)
    ce.report(table)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""Main executable for ShivyC compiler.

For usage, run "./shivyc.py --help".

"""

import argparse
import subprocess

import token_kinds
from errors import CompilerError
from lexer import Lexer
from parser import Parser
from il_gen import ILCode
from il_gen import SymbolTable
from asm_gen import ASMCode
from asm_gen import ASMGen

def main():
    """Load the input files, compile them, and save output.

    The main function handles interfacing with the user, like reading the
    command line arguments, printing errors, and generating output files.

    """
    arguments = get_arguments()

    try:
        with open(arguments.file_name) as c_file:
            code_lines = [
                (line_text.strip(), arguments.file_name, line_num + 1)
                for line_num, line_text in enumerate(c_file)
            ]
    except IOError:
        raise CompilerError("could not read file: '{}'"
                            .format(arguments.file_name))

    asm_source = compile_to_asm(code_lines)
    asm_filename = "out.s"
    write_asm(asm_source, asm_filename)

    assemble_and_link("out", asm_filename, "out.o")

def write_asm(asm_source, asm_filename):
    """Save the given assembly source to disk at asm_filename.

    asm_source (str) - Full assembly source code.
    asm_filename (str) - Filename to which to save the generated assembly.

    """
    try:
        with open(asm_filename, "w") as s_file:
            s_file.write(asm_source)
    except IOError:
        raise CompilerError("could not write output file '{}'".format(
            asm_filename))

def compile_to_asm(code_lines):
    """Compile the given code lines to asm.

    code_lines (List(tuple)) - Lines of code. First element is the line of code
    itself, second element is the file name, third element is the line number.

    """
    lexer = Lexer(token_kinds.symbol_kinds, token_kinds.keyword_kinds)
    token_list = lexer.tokenize(code_lines)

    ast_root = Parser(token_list).parse()

    il_code = ILCode()
    symbol_table = SymbolTable()
    ast_root.make_code(il_code, symbol_table)

    asm_code = ASMCode()
    ASMGen(il_code, asm_code).make_asm()

    return asm_code.full_code()

def get_arguments():
    """Get the command-line arguments.

    This function sets up the argument parser and returns an object storing the
    argument values (as returned by argparse.parse_args()).

    """
    parser = argparse.ArgumentParser(description="Compile C files.")

    # The file name of the C file to compile. The file name gets saved to the
    # file_name attribute of the returned object, but this parameter appears as
    # "filename" (no underscore) on the command line.
    parser.add_argument("file_name", metavar="filename")
    return parser.parse_args()

def assemble_and_link(binary_name, asm_name, obj_name):
    """Assmble and link the assembly file into an object file and binary.

    If the assembly/linking fails, raise an exception. TODO: Deal with linker
    failures more smoothly.

    binary_name (str) - Name of the binary file to output
    asm_name (str) - Name of the assembly file to read in
    obj_name (str) - Name of the obj file to output

    """
    subprocess.check_call(["nasm", "-f", "elf64", "-o", obj_name, asm_name])
    subprocess.check_call([
        "ld", "-dynamic-linker", "/lib64/ld-linux-x86-64.so.2",
        "/usr/lib/x86_64-linux-gnu/crt1.o", "/usr/lib/x86_64-linux-gnu/crti.o",
        "-lc", obj_name, "/usr/lib/x86_64-linux-gnu/crtn.o", "-o", binary_name
    ])

if __name__ == "__main__":
    try:
        main()
    except CompilerError as e:
        print(e)
<commit_msg>Reorder functions in shivyc.py<commit_after>#!/usr/bin/env python3
"""Main executable for ShivyC compiler.

For usage, run "./shivyc.py --help".

"""

import argparse
import subprocess

import token_kinds
from errors import CompilerError
from lexer import Lexer
from parser import Parser
from il_gen import ILCode
from il_gen import SymbolTable
from asm_gen import ASMCode
from asm_gen import ASMGen

def main():
    """Load the input files, compile them, and save output.

    The main function handles interfacing with the user, like reading the
    command line arguments, printing errors, and generating output files.

    """
    arguments = get_arguments()

    try:
        with open(arguments.file_name) as c_file:
            code_lines = [
                (line_text.strip(), arguments.file_name, line_num + 1)
                for line_num, line_text in enumerate(c_file)
            ]
    except IOError:
        raise CompilerError("could not read file: '{}'"
                            .format(arguments.file_name))

    asm_source = compile_to_asm(code_lines)
    asm_filename = "out.s"
    write_asm(asm_source, asm_filename)

    assemble_and_link("out", asm_filename, "out.o")

def get_arguments():
    """Get the command-line arguments.

    This function sets up the argument parser and returns an object storing the
    argument values (as returned by argparse.parse_args()).

    """
    parser = argparse.ArgumentParser(description="Compile C files.")

    # The file name of the C file to compile. The file name gets saved to the
    # file_name attribute of the returned object, but this parameter appears as
    # "filename" (no underscore) on the command line.
    parser.add_argument("file_name", metavar="filename")
    return parser.parse_args()

def compile_to_asm(code_lines):
    """Compile the given code lines to asm.

    code_lines (List(tuple)) - Lines of code. First element is the line of code
    itself, second element is the file name, third element is the line number.

    """
    lexer = Lexer(token_kinds.symbol_kinds, token_kinds.keyword_kinds)
    token_list = lexer.tokenize(code_lines)

    ast_root = Parser(token_list).parse()

    il_code = ILCode()
    symbol_table = SymbolTable()
    ast_root.make_code(il_code, symbol_table)

    asm_code = ASMCode()
    ASMGen(il_code, asm_code).make_asm()

    return asm_code.full_code()

def write_asm(asm_source, asm_filename):
    """Save the given assembly source to disk at asm_filename.

    asm_source (str) - Full assembly source code.
    asm_filename (str) - Filename to which to save the generated assembly.

    """
    try:
        with open(asm_filename, "w") as s_file:
            s_file.write(asm_source)
    except IOError:
        raise CompilerError("could not write output file '{}'".format(
            asm_filename))

def assemble_and_link(binary_name, asm_name, obj_name):
    """Assmble and link the assembly file into an object file and binary.

    If the assembly/linking fails, raise an exception. TODO: Deal with linker
    failures more smoothly.

    binary_name (str) - Name of the binary file to output
    asm_name (str) - Name of the assembly file to read in
    obj_name (str) - Name of the obj file to output

    """
    subprocess.check_call(["nasm", "-f", "elf64", "-o", obj_name, asm_name])
    subprocess.check_call([
        "ld", "-dynamic-linker", "/lib64/ld-linux-x86-64.so.2",
        "/usr/lib/x86_64-linux-gnu/crt1.o", "/usr/lib/x86_64-linux-gnu/crti.o",
        "-lc", obj_name, "/usr/lib/x86_64-linux-gnu/crtn.o", "-o", binary_name
    ])

if __name__ == "__main__":
    try:
        main()
    except CompilerError as e:
        print(e)
<|endoftext|>


<commit_before>#!/usr/bin/python

import sys
import os
import stat
import re

def checkglobals(libname, exceptsymbols, verbose):
    badglobals = 0
    nmcmd = "nm -g " + libname
    f = os.popen(nmcmd)
    b = f.readline()
    while b != "":
        match = re.match("^([0-9a-f]+)\s(.?)\s(.*)$", b)
        if match == None:
            match = re.match("^\s+(.*)$", b)
            if match == None:
                print "unknown", b
                badglobals = 1
        else:
            type = match.group(2)
            symbol = match.group(3)
            if verbose: print type, symbol
            match = re.match("^toku_|^__toku|^db_env_set", symbol)
            if match == None and not exceptsymbols.has_key(symbol):
                print "non toku symbol=", symbol
                badglobals = 1
        b = f.readline()
    f.close()
    return badglobals

def main():
    verbose = 0
    libname = "libdb.so"
    for arg in sys.argv[1:]:
        if arg == "-v":
            verbose += 1
        elif arg[0:3] == "lib":
            libname = arg

    try: st = os.stat(libname)
    except: return 1
    mode = st[stat.ST_MODE]
    if not (mode & stat.S_IREAD): return 1

    exceptsymbols = {}
    for n in [ "_init", "_fini", "_end", "_edata", "__bss_start" ]:
        exceptsymbols[n] = 1
    for n in [ "db_env_create", "db_create", "db_strerror", "db_version", "log_compare",
               "db_env_set_func_fsync",
               "db_env_set_func_free",
               "db_env_set_func_malloc",
               "db_env_set_func_realloc",
               "db_env_set_func_pwrite",
               "db_env_set_func_write",
               "db_env_set_func_loader_fwrite",
               "db_env_set_checkpoint_callback",
               "db_env_set_checkpoint_callback2",
               "db_env_set_recover_callback",
               "db_env_set_recover_callback2",
               "setup_dlmalloc", "dlfree", "dlmalloc", "dlrealloc"
               ]:
        exceptsymbols[n] = 1
    return checkglobals(libname, exceptsymbols, verbose)

sys.exit(main())
<commit_msg>[t:3028] Add db_env_enable_engine_status to tokuglobals.py.<commit_after>#!/usr/bin/python

import sys
import os
import stat
import re

def checkglobals(libname, exceptsymbols, verbose):
    badglobals = 0
    nmcmd = "nm -g " + libname
    f = os.popen(nmcmd)
    b = f.readline()
    while b != "":
        match = re.match("^([0-9a-f]+)\s(.?)\s(.*)$", b)
        if match == None:
            match = re.match("^\s+(.*)$", b)
            if match == None:
                print "unknown", b
                badglobals = 1
        else:
            type = match.group(2)
            symbol = match.group(3)
            if verbose: print type, symbol
            match = re.match("^toku_|^__toku|^db_env_set", symbol)
            if match == None and not exceptsymbols.has_key(symbol):
                print "non toku symbol=", symbol
                badglobals = 1
        b = f.readline()
    f.close()
    return badglobals

def main():
    verbose = 0
    libname = "libdb.so"
    for arg in sys.argv[1:]:
        if arg == "-v":
            verbose += 1
        elif arg[0:3] == "lib":
            libname = arg

    try: st = os.stat(libname)
    except: return 1
    mode = st[stat.ST_MODE]
    if not (mode & stat.S_IREAD): return 1

    exceptsymbols = {}
    for n in [ "_init", "_fini", "_end", "_edata", "__bss_start" ]:
        exceptsymbols[n] = 1
    for n in [ "db_env_create", "db_create", "db_strerror", "db_version", "log_compare",
               "db_env_set_func_fsync",
               "db_env_set_func_free",
               "db_env_set_func_malloc",
               "db_env_set_func_realloc",
               "db_env_set_func_pwrite",
               "db_env_set_func_write",
               "db_env_set_func_loader_fwrite",
               "db_env_set_checkpoint_callback",
               "db_env_set_checkpoint_callback2",
               "db_env_set_recover_callback",
               "db_env_set_recover_callback2",
               "db_env_enable_engine_status",
               "setup_dlmalloc", "dlfree", "dlmalloc", "dlrealloc"
               ]:
        exceptsymbols[n] = 1
    return checkglobals(libname, exceptsymbols, verbose)

sys.exit(main())
<|endoftext|>


<commit_before>from __future__ import print_function
from sklearn.decomposition import PCA
import pandas as pd
import pickle as pk
import re
import seaborn as sns
import numpy as np
import matplotlib.pyplot as plt
plt.style.use('seaborn-dark')

hists = pk.load(open('../pickle_files/fgHists.pd.pk', 'rb'))
fullHists = []
for i in range(hists.shape[0]):
    # flatten histogram
    fullHists.append(np.hstack((hists.iloc[i].fgHist['blue'][:,0], hists.iloc[i].fgHist['green'][:,0], hists.iloc[i].fgHist['red'][:,0])))

iHists = pd.DataFrame(columns = ['bin{}'.format(i) for i in range(len(fullHists[0]))], data=fullHists)
iHists['breed'] = hists.breed
iHists['file'] = hists.file

removeIdx = []
# some values are NaN for some reason, need to remove them
for i in range(iHists.shape[0]):
    if sum(iHists.iloc[i].isnull()) > 0:
        removeIdx.append(i)

newIhists = iHists.drop(removeIdx)
tempH = newIhists.drop('breed', 1)
tempH = tempH.drop('file', 1)

pcaFG = PCA(n_components=20)
pcaFG.fit(tempH)
varianceFG = pcaFG.explained_variance_ratio_
print(varianceFG)
print('top 20 fg components:', varianceFG[:20].sum())

pcaHists = pd.DataFrame(pcaFG.transform(tempH), columns=['dim{}'.format(i) for i in range(20)])
pcaHists['breed'] = newIhists.breed
pcaHists['file'] = newIhists.file

pk.dump(pcaFG, open('../pickle_files/pcaFG20.pd.pk', 'wb'))

pk.dump(pcaHists, open('../pickle_files/pcaHists.pd.pk', 'wb'))<commit_msg>chore: housekeeping<commit_after># does PCA on color histograms
from __future__ import print_function
from sklearn.decomposition import PCA
import pandas as pd
import pickle as pk
import re
import seaborn as sns
import numpy as np
import matplotlib.pyplot as plt
plt.style.use('seaborn-dark')

hists = pk.load(open('../pickle_files/fgHists.pd.pk', 'rb'))
fullHists = []
for i in range(hists.shape[0]):
    # flatten histogram
    fullHists.append(np.hstack((hists.iloc[i].fgHist['blue'][:,0], hists.iloc[i].fgHist['green'][:,0], hists.iloc[i].fgHist['red'][:,0])))

iHists = pd.DataFrame(columns = ['bin{}'.format(i) for i in range(len(fullHists[0]))], data=fullHists)
iHists['breed'] = hists.breed
iHists['file'] = hists.file

removeIdx = []
# some values are NaN for some reason, need to remove them
for i in range(iHists.shape[0]):
    if sum(iHists.iloc[i].isnull()) > 0:
        removeIdx.append(i)

newIhists = iHists.drop(removeIdx)
tempH = newIhists.drop('breed', 1)
tempH = tempH.drop('file', 1)

pcaFG = PCA(n_components=20)
pcaFG.fit(tempH)
varianceFG = pcaFG.explained_variance_ratio_
print(varianceFG)
print('top 20 fg components:', varianceFG[:20].sum())

pcaHists = pd.DataFrame(pcaFG.transform(tempH), columns=['dim{}'.format(i) for i in range(20)])
pcaHists['breed'] = newIhists.breed
pcaHists['file'] = newIhists.file

pk.dump(pcaFG, open('../pickle_files/pcaFG20.pd.pk', 'wb'))

pk.dump(pcaHists, open('../pickle_files/pcaHists.pd.pk', 'wb'))<|endoftext|>


<commit_before># coding=utf-8
# auto defenestration is key

"""
Cyther: The Cross Platform Cython/Python Compiler

We all know the beauties of Cython:
    1) Writing C extensions is as easy as Python
    2) Almost any valid Python is valid Cython, as Cython is a superset of Python
    3) It has the readability of Python, but the speed of C
    4) Minimal effort has to be taken in order to speed up some programs by three to four orders of magnitude

However, compiling is not always easy. There are a few places that disutils' 'setup.py' can get tripped up.
    1) vcvarsall.bat not found error
    2) gcc: undefined reference to...
    3) Other errors basically referring to 'compiler not found'

Cython may be almost as easy to write as Python, but sometimes nowhere near the level of easiness that it
takes to run Python. This is where Cyther comes into play. Cyther is an attempt at a cross platform compiler
that wields both the standard cython compiler and gcc to make sure that these errors don't happen.

Cyther is extremely easy to use. One can call cyther.py from the command line, or import `core` on the
module level, then call that. Here are a few examples of Cyther's features:

        from cyther import core
        core('cython_file.pyx')

    same can be done with:

        C:/Python35> cyther cython_file.pyx

    Here are some neat little option examples:

        from cyther import core
        core('python_file.py -t -l')
        # -t means that cyther will not compile it if the source file is not older than the compiled file
        # -l means that cyther will build locally, if not given, it builds in __cythercache__

    Perhaps the most useful feature of Cyther:

        from cyther import core
        core('cython_file.pyx -w')

    This `-w` command means that cyther will keep looking at that file indefinitely and whenever it sees a change
    in the source code, it will automatically compile it without the user having to do anything. Here is the
    output of the -w option in the command line and stdout:

        Compiling the file 'D:\python\notes.py'

        cython -a -p -o D:\python\notes.c D:\python\notes.py -l

        gcc -shared -w -O3 -I D:\Python35\include -L D:\Python35\libs
            -o D:\python\notes.pyd D:\python\notes.c -l python35

        ...<count:1>...

        Compiling the file 'D:\python\test.pyx'

        ...<count:2>...

        Compiling the file 'D:\python\test.pyx'

        ...<count:3>...

        Compiling the file 'D:\python\test2.pyx'

        ...<count:4>...

    Keep in mind that anything that you pass to core, you can also pass to cyther from the command line. Now,
    try to meditate on this command:

        C:/Python35> cyther cython_file.pyx python_file.py test.pyx -w -l -o something -cython _l

    This command will compile these three files, then proceed to watch them continuously, and if they change,
    they will be recompiled. Also, their .c and .a files will be built in the same directory. Even further,
    we pass the option _l (-l) to cython, which will create listing files for the three files specified.
    Notice that we put a -o option, when in reality this makes no sense. Cyther knows this and will erase this
    option before it goes to compile, so the files will not be compiled under the same name. To get an idea
    of what Cyther is currently capable of, type `core('-h')` or `cyther -h` from the command line.

Cyther isn't quite perfect yet, so all the incompatabilities and assumptions that Cyther makes are listed
below. We strongly recommend that you look them over before even touching the download button. In the
near future we hope to make Cyther as polished as possible, and bring the list of assumptions listed below
to a minimum. There are even plans in the works to be able to automatically recompile shared object libraries
that are entirely missing on one's system; critical to Cython compilation.

Assumptions cyther makes about your system:
    1) Your environment path variable is able to be found by `shutil.which`
    2) gcc can work with the option -l pythonXY (libpythonXY.a exists in your libs directory)
    3) Almost any gcc compiled C program will work on Windows

Hey you! Yes you. If you notice any bugs or peculiarities, please report them to our bug tracker, it will
help us out a lot:
    <LINK FOR BUG TRACKING>

If you have any questions or concerns, or even any suggestions, don't hesitate to contact me at:
    npandolfi@wpi.edu

Happy compiling!!

Tags: Cyther, antibug, vcvarsall.bat, MinGW32, Python, Cython, Python 3.x, compiler, command-line, script
user-friendly, setup.py, gcc
"""

import os, sys, subprocess
import argparse, platform, errno
import time

from shutil import which

__author__ = 'Nicholas C. Pandolfi'

__version__ = '0.3.5'

__license__ = '''
Copyright (c) 2016 Nicholas C. Pandolfi ALL RIGHTS RESERVED (MIT)

Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without
restriction, including without limitation the rights to use,
copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall
be included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
'''

__all__ = ['CytherError', '_make_directory', 'core', 'where']

CYTHER_FAIL = 0
CYTHER_SUCCESS = 1
ERROR_PASSOFF = -3
INTERVAL = .25

CYTHONIZABLE_FILE_EXTS = ('.pyx', '.py')

PLEASE_ADD = ", please add it to the system's path"
NOT_NEEDED_MESSAGE = "Module '{}' does not have to be included, or has no .get_include() method"

LIB_A_MISSING_MESSAGE = '''
Currently, cyther does not support the automatic construction of 'libpython{0}.a',
so you will have to do it yourself. There are several tools that can help you with
this, including dlltool, gendef, pexport, reimp. All of which are a part of mingw32.

The general idea is to first look into the python{0}.dll and figure out all the
objects that belong to that file. Then, take the names of these objects, and put
them in a .def file. Take this file, and pass it to dll tool to tell it what to
extract from the python{0}.dll and inject into the libpython{0}.a static library.
'''

ASSUMPTIONS = """
Assumptions cyther makes about your system:
1) Your environment path variable is able to be found by `shutil.which`
2) gcc can work with the option -l pythonXY (libpythonXY.a exists in your libs directory)
3) Almost any gcc compiled C program will work on Windows
"""

class CytherError(Exception):
    """A helpful custom error to be called when a general python error just doesn't make sense in context"""
    def __init__(self, *args, **kwargs):
        Exception.__init__(self, *args, **kwargs)

def where(cmd, mode=os.X_OK, path=None):
    raw_result = which(cmd, mode, path)
    if raw_result:
        return os.path.abspath(raw_result)
    else:
        return ''

def deal_with_lib_a(direc, message):
    """What to do if the libpythonXY.a is missing. Currently, it raises an error, and prints a helpful message"""
    raise CytherError("The file '{}' does not exist.\n\n{}".format(direc, message))

INFO = str()

INFO += "\nSystem:"

OPERATING_SYSTEM = platform.platform().split('-')[0]
INFO += "\n\tOperating System: {}".format(OPERATING_SYSTEM)
IS_WINDOWS = OPERATING_SYSTEM == 'Windows'
INFO += "\n\t\tOS is Windows: {}".format(IS_WINDOWS)
DLL_EXTENSION = '.dll' if IS_WINDOWS else '.so'
INFO += "\n\tLinked Library Extension: {}".format(DLL_EXTENSION)
DEFAULT_OUTPUT_EXTENSION = '.pyd' if IS_WINDOWS else '.so'
INFO += "\n\tDefault Output Extension: {}".format(DEFAULT_OUTPUT_EXTENSION)

INFO += "\nPython:"

python_found = where('python')
if not python_found:
    raise CytherError("Python is not able to be called, please add it to the system's path")

VER = str(sys.version_info.major) + str(sys.version_info.minor)
INFO += "\n\tVersion: {}".format('.'.join(list(VER)))

LIB_A_MISSING_MESSAGE = LIB_A_MISSING_MESSAGE.format(VER)

PYTHON_DIRECTORY = sys.exec_prefix
PYTHON_DIRECTORY_EXISTS = os.path.exists(PYTHON_DIRECTORY)
if not PYTHON_DIRECTORY_EXISTS:
    raise CytherError("The Python main installation directory doesn't exist")
INFO += "\n\tInstallation Directory: {}".format(PYTHON_DIRECTORY)
INFO += "\n\t\tExists: {}".format(PYTHON_DIRECTORY_EXISTS)

LIBS_DIRECTORY = os.path.normpath(os.path.join(PYTHON_DIRECTORY, 'libs'))
LIBS_DIRECTORY_EXISTS = os.path.exists(LIBS_DIRECTORY)
if not LIBS_DIRECTORY_EXISTS:
    raise CytherError("The Python 'libs' directory doesn't exist")
INFO += "\n\tDirectory 'libs': {}".format(LIBS_DIRECTORY)
INFO += "\n\t\tExists: {}".format(LIBS_DIRECTORY_EXISTS)

INCLUDE_DIRECTORY = os.path.normpath(os.path.join(PYTHON_DIRECTORY, 'include'))
INCLUDE_DIRECTORY_EXISTS = os.path.exists(INCLUDE_DIRECTORY)
if not INCLUDE_DIRECTORY_EXISTS:
    raise CytherError("The Python 'include' directory doesn't exist")
INFO += "\n\tDirectory 'include': {}".format(INCLUDE_DIRECTORY)
INFO += "\n\t\tExists: {}".format(INCLUDE_DIRECTORY_EXISTS)

LIB_A_NAME = 'python' + VER
LIB_A = 'lib' + LIB_A_NAME + '.a'
LIB_A_DIRECTORY = os.path.normpath(os.path.join(LIBS_DIRECTORY, LIB_A))
LIB_A_DIRECTORY_EXISTS = os.path.exists(LIB_A_DIRECTORY)
if not LIB_A_DIRECTORY_EXISTS:
    deal_with_lib_a(LIB_A_DIRECTORY, LIB_A_MISSING_MESSAGE)
INFO += "\n\tLibrary '.a' Directory: {}".format(LIB_A_DIRECTORY)
INFO += "\n\t\tExists: {}".format(LIB_A_DIRECTORY_EXISTS)

DLL_NAME = LIB_A_NAME + DLL_EXTENSION
DLL_LOCATION = where(DLL_NAME)
DLL_EXISTS = os.path.exists(DLL_LOCATION)

DLL_DIRECTORY = os.path.normpath(DLL_LOCATION)
INFO += "\n\tPython '{}' Directory: {}".format(DLL_EXTENSION, DLL_LOCATION)
INFO += "\n\t\tExists: {}".format(DLL_EXISTS)

cython_found = where('cython')
if not cython_found:
    try:
        import cython
        raise CytherError("Cython exists and is able to be imported by Python, but is unable to be called" + PLEASE_ADD)
    except ImportError:
        raise CytherError("Cython is unable to be imported, and is probably not installed")

gcc_found = where('gcc')
if not gcc_found:
    raise CytherError("gcc is not able to be called" + PLEASE_ADD)

__cytherinfo__ = INFO

del INFO

help_filenames = 'The Cython source files'
help_preset = 'The preset options for using cython and gcc (ninja, verbose, beast)'
help_timestamp = 'If this flag is provided, cyther will not compile files that have a modified' \
                 'time before that of your compiled .pyd or .so files'
help_output_name = 'Change the name of the output file, default is basename plus .pyd'
help_include = 'The names of the python modules that have an include library that needs to be passed to gcc'
help_assumptions = 'Print the list of assumptions cyther makes about your system before running'
help_local = 'When not flagged, builds in __cythercache__, when flagged, it builds locally in the same directory'
help_watch = "When given, cyther will watch the directory with the 't' option implied and compile," \
             "when necessary, the files given"
help_cython = "Arguments to pass to Cython (use '_' or '__' instead of '-' or '--'"
help_gcc = "Arguments to pass to gcc (use '_' or '__' instead of '-' or '--'"

description_text = 'Auto compile and build .pyx files in place.\n{}\n{}'

parser = argparse.ArgumentParser(description=description_text.format(ASSUMPTIONS, __cytherinfo__),
                                 formatter_class=argparse.RawDescriptionHelpFormatter,
                                 usage='cyther [options] input_file')
parser.add_argument('filenames', action='store', nargs='+', help=help_filenames)
parser.add_argument('-p', '--preset', action='store', default='', help=help_preset)
parser.add_argument('-t', '--timestamp', action='store_true', default=False, help=help_timestamp)
parser.add_argument('-o', '--output_name', action='store', help=help_output_name)
parser.add_argument('-i', '--include', action='store', default='', help=help_include)
parser.add_argument('-l', '--local', action='store_true', dest='local', default=False, help=help_local)
parser.add_argument('-w', '--watch', action='store_true', dest='watch', default=False, help=help_watch)
parser.add_argument('-cython', action='store', nargs='+', dest='cython_args', default=[], help=help_cython)
parser.add_argument('-gcc', action='store', nargs='+', dest='gcc_args', default=[], help=help_gcc)

def _make_directory(directory):
    """A attempt at a raceless make directory function. Should be as safe as possible"""
    if not os.path.exists(directory):
        try:
            os.makedirs(directory)
        except OSError as error:
            if error.errno != errno.EEXIST:
                raise

def _get_full_path(filename):
    """This will get the full path of a path of a filename"""
    if os.path.exists(filename) and (filename not in os.listdir(os.getcwd())):
        ret = filename
    elif os.path.exists(os.path.join(os.getcwd(), filename)):
        ret = os.path.join(os.getcwd(), filename)
    else:
        raise CytherError("The file '{}' does not exist".format(filename))
    return ret

def _process_files(args):
    """Generates and error checks each file's information before the compilation actually starts"""
    to_process = []
    for filename in args['filenames']:
        file = dict()
        file['file_path'] = _get_full_path(filename)
        file['file_base_name'] = os.path.splitext(os.path.basename(file['file_path']))[0]
        file['no_extension'], file['extension'] = os.path.splitext(file['file_path'])
        if file['extension'] not in CYTHONIZABLE_FILE_EXTS:
            raise CytherError("The file '{}' is not a designated cython file".format(file['file_path']))
        base_path = os.path.dirname(file['file_path'])
        local_build = args['local']
        if not local_build:
            cache_name = os.path.join(base_path, '__cythercache__')
            _make_directory(cache_name)
            file['c_name'] = os.path.join(cache_name, file['file_base_name']) + '.c'
        else:
            file['c_name'] = file['no_extension'] + '.c'

        output_name = args['output_name']
        if args['watch']:
            file['output_name'] = file['no_extension'] + DEFAULT_OUTPUT_EXTENSION
        elif output_name:
            if os.path.exists(output_name) and os.path.isfile(output_name):
                file['output_name'] = output_name
            else:
                dirname = os.path.dirname(output_name)
                if not dirname:
                    dirname = os.getcwd()
                print('dirname: {}'.format(dirname))
                if os.path.exists(dirname):
                    file['output_name'] = output_name
                else:
                    raise CytherError('The directory specified to write the output file in does not exist')
        else:
            file['output_name'] = file['no_extension'] + DEFAULT_OUTPUT_EXTENSION

        file['stamp_if_error'] = 0
        to_process.append(file)
    return to_process

def _should_compile(file):
    """Figures out if Cyther should compile the given file. For use with the -t option"""
    if os.path.exists(file['output_name']):
        source_time = os.path.getmtime(file['file_path'])
        output_time = os.path.getmtime(file['output_name'])
        if source_time > output_time:
            return True
    else:
        return True
    return False

def _commands_from_preset(preset, file):
    """Given a high level preset, it will construct the basic args to pass over. 'ninja', 'beast', and 'minimal'"""
    if not preset:
        preset = 'ninja'

    if preset == 'ninja':
        cython_command = ['cython', '-a', '-p', '-o', file['c_name'], file['file_path']]
        gcc_command = ['gcc', '-shared', '-w', '-O3', '-I', INCLUDE_DIRECTORY, '-L', LIBS_DIRECTORY, '-o',
                       file['output_name'], file['c_name'],
                       '-l', LIB_A_NAME]
    elif preset == 'beast':
        cython_command = ['cython', '-a', '-l', '-p', '-o', file['c_name'], file['file_path']]
        gcc_command = ['gcc', '-shared', '-Wall', '-O3', '-I', INCLUDE_DIRECTORY, '-L', LIBS_DIRECTORY, '-o',
                       file['output_name'],
                       file['c_name'], '-l', LIB_A_NAME]
    elif preset == 'minimal':
        cython_command = ['cython', '-o', file['c_name'], file['file_path']]
        gcc_command = ['gcc', '-shared', '-I', INCLUDE_DIRECTORY, '-L', LIBS_DIRECTORY, '-o', file['output_name'],
                       file['c_name'], '-l', LIB_A_NAME]
    else:
        raise CytherError("The preset '{}' is not supported".format(preset))

    return cython_command, gcc_command

def _print_commands(commands):
    """Simply prints all of the commands on screen for the user to see. Nice for debugging."""
    print(' '.join(commands).strip())

def _underscores_to_dashes(commands):
    """Converts Cyther's pass off notation to the regular dash notation (_a becomes -a)"""
    processed = []
    for command in commands:
        if command[0] == '_':
            if command[1] == '_':
                command = '--' + command[2:]
            else:
                command = '-' + command[1:]
        processed.append(command)
    return processed

def _filter_passoff_commands(pass_off_commands, commands):
    """Puts items of pass_off_commands in commands if commands does not already have them"""
    if pass_off_commands:
        for item in pass_off_commands:
            if item[0] == '-':
                if item not in commands:
                    commands.append(item)

def _extract_inc_dirs(obj):
    """Used by _include_directories. Does some nice error checking and splitting and stripping. Cleans up."""
    results = []
    modules = obj.strip().split('-')
    for m in modules:
        try:
            exec('import {}'.format(m))
        except ImportError:
            raise CytherError("The module '{}' does not exist".format(m))
        try:
            results.append(eval(m).get_include())
        except AttributeError:
            print(NOT_NEEDED_MESSAGE.format(m))
    return results

def _include_directories(modules):
    """Given a string of module names, it will return the 'include' directories essential to their compilation"""
    dirs = []
    if modules:
        for path in _extract_inc_dirs(modules):
            dirs.append('-I {}'.format(path))
    return dirs

def _cytherize(args, commands, actions):
    """The function where the actual compilation takes place"""
    for command, action in zip(commands, actions):
        ret = subprocess.call(command)
        if ret:
            if ret != 1:
                string = str(ret)
            else:
                string = ''
            if args['watch']:
                print(string)
                return ERROR_PASSOFF
            else:
                raise CytherError(string)
    return CYTHER_SUCCESS

def _integrate(args, file, print_command):
    """Used by core to integrate all the pieces of information, and to make sure everything is good to go"""
    cython_commands, gcc_commands = _commands_from_preset(args['preset'], file)

    cython_pass_off = _underscores_to_dashes(args['cython_args'])
    gcc_pass_off = _underscores_to_dashes(args['gcc_args'])

    _filter_passoff_commands(cython_pass_off, cython_commands)
    _filter_passoff_commands(gcc_pass_off, gcc_commands)

    dirs = _include_directories(args['include'])
    gcc_commands.extend(dirs)

    if print_command:
        _print_commands(cython_commands)
        _print_commands(gcc_commands)

    response = _cytherize(args, (cython_commands, gcc_commands), ('cython', 'gcc'))
    return response

def _string_to_dictionary(string):
    """changes a command line string to a dictionary"""
    unprocessed = string.strip().split(' ')
    if unprocessed[0] == 'cyther':
        del unprocessed[0]
    namespace = parser.parse_args(unprocessed).__dict__
    return namespace

def core(args):
    """The heart of Cyther, this function controlls the main loop"""
    if isinstance(args, str):
        args = _string_to_dictionary(args)
    elif isinstance(args, argparse.Namespace):
        args = args.__dict__
    else:
        raise CytherError("Args must be a instance of str or argparse.Namespace, not '{}'".format(str(type(args))))

    numfiles = len(args['filenames'])
    interval = INTERVAL / numfiles
    files = _process_files(args)
    if not args['timestamp'] and args['watch']:
        args['timestamp'] = True
    counter = 1
    should_i_print = True
    while True:
        for file in files:
            if args['timestamp']:
                modified_time = os.path.getmtime(file['file_path'])
                no_error = modified_time > file['stamp_if_error']
                if _should_compile(file) and no_error:
                    if args['watch']:
                        if len(args['filenames']) > 1:
                            print("Compiling the file '{}'".format(file['file_path']))
                        else:
                            print('Compiling the file')
                    print('')
                    ret = _integrate(args, file, should_i_print)
                    should_i_print = False
                    if ret == ERROR_PASSOFF:
                        file['stamp_if_error'] = time.time()
                    if args['watch']:
                        print("\n...<count:{}>...\n".format(counter))
                        counter += 1
                else:
                    if not args['watch']:
                        if len(args['filenames']) > 1:
                            print("Skipping the file '{}'".format(file['file_path']))
                        else:
                            print('Skipping compilation')
                    else:
                        pass
                    continue
            else:
                _integrate(args, file, should_i_print)
        if not args['watch']:
            break
        else:
            time.sleep(interval)

def test():
    core('test.pyx -w')

if __name__ == '__main__':
    command_line_args = parser.parse_args()
    core(command_line_args)
<commit_msg>Delete cyther.py<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright 2015 Vauxoo
#    Author: Moisés Lopez, Osval Reyes
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from openerp import _, api, exceptions, models

class StockMove(models.Model):

    """
    Stock Move
    """
    _inherit = 'stock.move'

    @api.model
    def check_after_action_done(self, operation_or_move):
        super(StockMove, self).\
            check_after_action_done(operation_or_move)
        return self.check_unicity_qty_available(operation_or_move)

    @api.model
    def check_before_action_done(self, operation_or_move):
        res = super(StockMove, self).\
            check_before_action_done(operation_or_move)

        elements = [
            (x.product_id, x.lot_id)
            for x in operation_or_move.picking_id.pack_operation_ids
            if x.product_id.lot_unique_ok
            ]
        repeat_elements = [x for x in elements if elements.count(x) > 1]
        if repeat_elements:
            raise exceptions.ValidationError(_(
                'Same lot with product unique in multiple operations'
                ))
        return res

    @api.multi
    def check_unicity_move_qty(self):
        """
        Check move quantity to verify that has qty = 1
        if 'lot unique' is ok on product
        """
        for move in self:
            if move.product_id.lot_unique_ok:
                for move_operation in \
                        move.linked_move_operation_ids:
                    if abs(move_operation.qty) > 1:
                        raise exceptions.ValidationError(_(
                            "Product '%s' has active"
                            " 'unique lot' "
                            "but has qty > 1"
                        ) % (move.product_id.name))

    @api.model
    def check_unicity_qty_available(self, operation_or_move):
        """
        Check quantity on lot to verify that has qty = 1
        if 'lot unique' is ok on product
        """
        lot_id = operation_or_move.lot_id
        if operation_or_move.product_id.lot_unique_ok and lot_id:
            qty = sum([x.qty + x.propagated_from_id.qty for x in operation_or_move.lot_id.quant_ids])
            if qty != 1 or operation_or_move.product_qty != 1:
                raise exceptions.ValidationError(_(
                    "Product '%s' has active "
                    "'unique lot'\n"
                    "but with this move "
                    "you will have a quantity of "
                    "'%s' in lot '%s'"
                ) % (operation_or_move.product_id.name, qty, lot_id.name))
        return True

    @api.model
    def check_tracking(self, move, lot_id):
        res = super(StockMove, self).check_tracking(move, lot_id)
        self.check_unicity_move_qty()
        return res
<commit_msg>flake8<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright 2015 Vauxoo
#    Author: Moisés Lopez, Osval Reyes
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from openerp import _, api, exceptions, models

class StockMove(models.Model):

    """
    Stock Move
    """
    _inherit = 'stock.move'

    @api.model
    def check_after_action_done(self, operation_or_move):
        super(StockMove, self).\
            check_after_action_done(operation_or_move)
        return self.check_unicity_qty_available(operation_or_move)

    @api.model
    def check_before_action_done(self, operation_or_move):
        res = super(StockMove, self).\
            check_before_action_done(operation_or_move)

        elements = [
            (x.product_id, x.lot_id)
            for x in operation_or_move.picking_id.pack_operation_ids
            if x.product_id.lot_unique_ok
            ]
        repeat_elements = [x for x in elements if elements.count(x) > 1]
        if repeat_elements:
            raise exceptions.ValidationError(_(
                'Same lot with product unique in multiple operations'
                ))
        return res

    @api.multi
    def check_unicity_move_qty(self):
        """
        Check move quantity to verify that has qty = 1
        if 'lot unique' is ok on product
        """
        for move in self:
            if move.product_id.lot_unique_ok:
                for move_operation in \
                        move.linked_move_operation_ids:
                    if abs(move_operation.qty) > 1:
                        raise exceptions.ValidationError(_(
                            "Product '%s' has active"
                            " 'unique lot' "
                            "but has qty > 1"
                        ) % (move.product_id.name))

    @api.model
    def check_unicity_qty_available(self, operation_or_move):
        """
        Check quantity on lot to verify that has qty = 1
        if 'lot unique' is ok on product
        """
        lot_id = operation_or_move.lot_id
        if operation_or_move.product_id.lot_unique_ok and lot_id:
            qty = sum([x.qty + x.propagated_from_id.qty
                       for x in operation_or_move.lot_id.quant_ids])
            if qty != 1 or operation_or_move.product_qty != 1:
                raise exceptions.ValidationError(_(
                    "Product '%s' has active "
                    "'unique lot'\n"
                    "but with this move "
                    "you will have a quantity of "
                    "'%s' in lot '%s'"
                ) % (operation_or_move.product_id.name, qty, lot_id.name))
        return True

    @api.model
    def check_tracking(self, move, lot_id):
        res = super(StockMove, self).check_tracking(move, lot_id)
        self.check_unicity_move_qty()
        return res
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import multiprocessing, subprocess, sys, threading, traceback
import time, os.path, socket
from select import select
from settings import Settings

_states = {
	1<<0: 'STOPPED',
	1<<1: 'STOPPING',
	1<<2: 'STARTING',
	1<<3: 'RUNNING',
	1<<4: 'CRASHED'
}

# the reverse of k,v is deliberate
for v,k in _states.items():
	sys.modules[__name__].__dict__[k] = v

def statename(state):
	return _states[state]

class StateError(Exception):
	pass

class _Log:
	def __init__(self, size=5):
		self.size = size
		self.lines = []

	def push(self, line):
		self.lines.append(line)
		if len(self.lines) > self.size:
			self.lines.pop(0)

	def __iter__(self):
		return self.lines.__iter__()

class _Daemon(threading.Thread):
	def __init__(self):
		threading.Thread.__init__(self)
		self._pid = None
		self._ipc = None
		self._instance = None
		self.log = _Log(size=50)

		self._state = STOPPED
		self._state_lock = threading.Lock()

		self._queue = []
		self._sem = multiprocessing.Semaphore(0) # not using threading.Semaphore since it doesn't support timeouts

		self._running = False

	def __del__(self):
		print '__del__'
		self._instance = None

	def stop(self):
		self._running = False

	def do_start(self, pid, ipc):
		if not self._state in [STOPPED, CRASHED]:
			raise StateError, 'Cannot start daemon while in state ' + statename(self._state)

		self._state = STARTING
		settings = Settings('settings.xml', 'settings.json')

		cmd = settings['Files.BinaryPath']
		args = [
			'--uds-log', 'slideshow.sock'
		]
		env = dict(
			TERM='xterm'
		)
		print cmd
		instance = subprocess.Popen(
			[cmd] + args,
			stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
			cwd=settings['Path.BasePath'], env=None
		)

		self._logobj = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)

		n = 0
		while True:
			n += 1

			if n == 20:
				raise RuntimeError, "Failed to connect log"

			try:
				self._logobj.connect(os.path.join(settings['Path.BasePath'], 'slideshow.sock'))
				break
			except:
				time.sleep(0.1)

		self._instance = instance

		self._state = RUNNING

	def state(self):
		self._state_lock.acquire()
		tmp = self._state
		self._state_lock.release()
		return tmp

	def __str__(self):
		return '<daemon id="{id}" />'.format(id=id(self))

	def run(self):
		self._running = True
		while self._running:
			if self._instance:
				self._instance.poll()

				(rd, _, _) = select([self._logobj], [], [], 0)
				if len(rd) > 0:
					for line in self._logobj.recv(4096).split("\n"):
						self.log.push(line)

				if self._instance.returncode != None:
					if self._instance.returncode == 0:
						self._state = STOPPED
					else:
						self._state = CRASHED

					self._instance = None

			# check if any commands has been issued
			if not self._sem.acquire(timeout=1.0):
				continue

			func, args, kwargs, sem, ret = self._queue.pop()
			try:
				ret.set(func(self, *args, **kwargs))
			except Exception as e:
				traceback.print_exc()
				e.exc_info = sys.exc_info()
				ret.set(e)
			sem.release()

	def push(self, func, args, kwargs, sem, ret):
		self._queue.append((func, args, kwargs, sem, ret))
		self._sem.release()

_daemon = _Daemon()

def subscribe(engine):
	engine.subscribe('start', _daemon.start)
	engine.subscribe('stop', _daemon.stop)

def _call(func, *args, **kwargs):
	class V:
		def __init__(self, value=None):
			self.value = value

		def set(self, value):
			self.value = value

		def get(self):
			return self.value

	sem = threading.Semaphore(0)
	ret = V()
	_daemon.push(func, args, kwargs, sem, ret)
	sem.acquire()

	if isinstance(ret.get(), Exception):
		exc_info = ret.get().exc_info
		raise exc_info[0], exc_info[1], exc_info[2]

	print 'return', ret.get()
	return ret.get()

def start(pid, ipc):
	return _call(_Daemon.do_start, pid, ipc)

def state():
	return _daemon.state()

def log():
	return _daemon.log<commit_msg>setting env<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import multiprocessing, subprocess, sys, threading, traceback
import time, os.path, socket
from select import select
from settings import Settings

_states = {
	1<<0: 'STOPPED',
	1<<1: 'STOPPING',
	1<<2: 'STARTING',
	1<<3: 'RUNNING',
	1<<4: 'CRASHED'
}

# the reverse of k,v is deliberate
for v,k in _states.items():
	sys.modules[__name__].__dict__[k] = v

def statename(state):
	return _states[state]

class StateError(Exception):
	pass

class _Log:
	def __init__(self, size=5):
		self.size = size
		self.lines = []

	def push(self, line):
		self.lines.append(line)
		if len(self.lines) > self.size:
			self.lines.pop(0)

	def __iter__(self):
		return self.lines.__iter__()

class _Daemon(threading.Thread):
	def __init__(self):
		threading.Thread.__init__(self)
		self._pid = None
		self._ipc = None
		self._instance = None
		self.log = _Log(size=50)

		self._state = STOPPED
		self._state_lock = threading.Lock()

		self._queue = []
		self._sem = multiprocessing.Semaphore(0) # not using threading.Semaphore since it doesn't support timeouts

		self._running = False

	def __del__(self):
		print '__del__'
		self._instance = None

	def stop(self):
		self._running = False

	def do_start(self, pid, ipc):
		if not self._state in [STOPPED, CRASHED]:
			raise StateError, 'Cannot start daemon while in state ' + statename(self._state)

		self._state = STARTING
		settings = Settings('settings.xml', 'settings.json')

		cmd = settings['Files.BinaryPath']
		args = [
			'--uds-log', 'slideshow.sock'
		]
		env = dict(
			#TERM='xterm'
			DISPLAY=settings['Apparence.Display']
		)
		env.update(settings['Env'])
		print cmd, args, env
		instance = subprocess.Popen(
			[cmd] + args,
			stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
			cwd=settings['Path.BasePath'], env=env
		)

		self._logobj = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)

		n = 0
		while True:
			n += 1

			if n == 20:
				raise RuntimeError, "Failed to connect log"

			try:
				self._logobj.connect(os.path.join(settings['Path.BasePath'], 'slideshow.sock'))
				break
			except:
				time.sleep(0.1)

		self._instance = instance

		self._state = RUNNING

	def state(self):
		self._state_lock.acquire()
		tmp = self._state
		self._state_lock.release()
		return tmp

	def __str__(self):
		return '<daemon id="{id}" />'.format(id=id(self))

	def run(self):
		self._running = True
		while self._running:
			if self._instance:
				self._instance.poll()

				(rd, _, _) = select([self._logobj], [], [], 0)
				if len(rd) > 0:
					for line in self._logobj.recv(4096).split("\n"):
						self.log.push(line)

				if self._instance.returncode != None:
					if self._instance.returncode == 0:
						self._state = STOPPED
					else:
						self._state = CRASHED

					self._instance = None

			# check if any commands has been issued
			if not self._sem.acquire(timeout=1.0):
				continue

			func, args, kwargs, sem, ret = self._queue.pop()
			try:
				ret.set(func(self, *args, **kwargs))
			except Exception as e:
				traceback.print_exc()
				e.exc_info = sys.exc_info()
				ret.set(e)
			sem.release()

	def push(self, func, args, kwargs, sem, ret):
		self._queue.append((func, args, kwargs, sem, ret))
		self._sem.release()

_daemon = _Daemon()

def subscribe(engine):
	engine.subscribe('start', _daemon.start)
	engine.subscribe('stop', _daemon.stop)

def _call(func, *args, **kwargs):
	class V:
		def __init__(self, value=None):
			self.value = value

		def set(self, value):
			self.value = value

		def get(self):
			return self.value

	sem = threading.Semaphore(0)
	ret = V()
	_daemon.push(func, args, kwargs, sem, ret)
	sem.acquire()

	if isinstance(ret.get(), Exception):
		exc_info = ret.get().exc_info
		raise exc_info[0], exc_info[1], exc_info[2]

	print 'return', ret.get()
	return ret.get()

def start(pid, ipc):
	return _call(_Daemon.do_start, pid, ipc)

def state():
	return _daemon.state()

def log():
	return _daemon.log<|endoftext|>