<commit_before>from __future__ import unicode_literals

import unittest
from six import StringIO

import pyaavso
from pyaavso.formats.visual import VisualFormatWriter, VisualFormatReader, \
    FormatException

class VisualFormatWriterTestCase(unittest.TestCase):
    """
    Tests for VisualFormatWriter class.
    """
    def setUp(self):
        self.fp = StringIO()

    def tearDown(self):
        self.fp.close()

    def test_header_type(self):
        """
        Check that TYPE parameter is always Visual.
        """
        writer = VisualFormatWriter(self.fp, 'XYZ')
        contents = self.fp.getvalue()
        self.assertIn("#TYPE=Visual", contents)

    def test_header_obscode(self):
        """
        Check that observer code is written into file.
        """
        writer = VisualFormatWriter(self.fp, 'XYZ')
        contents = self.fp.getvalue()
        self.assertIn("#OBSCODE=XYZ", contents)

    def test_header_software(self):
        """
        Check that the SOFTWARE parameter is correct.
        """
        writer = VisualFormatWriter(self.fp, 'XYZ')
        contents = self.fp.getvalue()
        self.assertIn("#SOFTWARE=pyaavso %s" % pyaavso.get_version(), contents)

    def test_header_date(self):
        """
        Check that the DATE parameter represents date format used.
        """
        writer = VisualFormatWriter(self.fp, 'XYZ', date_format='jd')
        contents = self.fp.getvalue()
        self.assertIn("#DATE=JD", contents)

    def test_header_obstype(self):
        """
        Check that OBSTYPE parameter is correctly set.
        """
        writer = VisualFormatWriter(self.fp, 'XYZ', obstype='Visual')
        contents = self.fp.getvalue()
        self.assertIn("#OBSTYPE=Visual", contents)

    def test_header_obstype_ptg(self):
        """
        Check that OBSTYPE can be set to PTG (Photographic).
        """
        writer = VisualFormatWriter(self.fp, 'XYZ', obstype='PTG')
        contents = self.fp.getvalue()
        self.assertIn("#OBSTYPE=PTG", contents)

    def test_write_raw_data(self):
        """
        Check that writerow() can accept raw data record.
        """
        data = [
            'SS CYG', '2450702.1234', '<11.1', 'na', '110', '113', '070613',
            'This is a test',
        ]
        writer = VisualFormatWriter(self.fp, 'XYZ')
        writer.writerow(data)
        lines = self.fp.getvalue().splitlines()
        self.assertEqual(
            lines[5],
            "SS CYG,2450702.1234,<11.1,na,110,113,070613,This is a test"
        )

    def test_write_dict(self):
        """
        Check that dictionary of observation data can be written to file.
        """
        data = {
            'name': 'SS CYG',
            'date': '2450702.1234',
            'magnitude': '<11.1',
            'comment_code': 'na',
            'comp1': '110',
            'comp2': '113',
            'chart': '070613',
            'notes': 'This is a test',
        }
        writer = VisualFormatWriter(self.fp, 'XYZ')
        writer.writerow(data)
        lines = self.fp.getvalue().splitlines()
        self.assertEqual(
            lines[5],
            "SS CYG,2450702.1234,<11.1,na,110,113,070613,This is a test"
        )

class VisualFormatReaderTestCase(unittest.TestCase):
    """
    Tests for VisualFormatReader class.
    """
    def setUp(self):
        self.lines = [
            "#TYPE=VISUAL",
            "#OBSCODE=XYZ",
            "#SOFTWARE=Notepad",
            "#DELIM=,",
            "#DATE=JD",
            "#OBSTYPE=Visual",
            "SS CYG,2450702.1234,<11.1,na,110,113,070613,This is a test",
        ]
        self.fp = StringIO("\n".join(self.lines))

    def tearDown(self):
        self.fp.close()

    def _drop_header(self, header_name):
        self.fp = StringIO("\n".join(
            line for line in self.lines if header_name not in line
        ))

    def test_missing_type(self):
        """
        Check that a missing TYPE header raises an exception.
        """
        self._drop_header('TYPE')
        with self.assertRaises(FormatException):
            reader = VisualFormatReader(self.fp)

    def test_observer_code(self):
        """
        Check that observer code matches header information.
        """
        reader = VisualFormatReader(self.fp)
        self.assertEqual(reader.observer_code, 'XYZ')

    def test_missing_observer_code(self):
        """
        Check that a missing OBSCODE header raises an exception.
        """
        self._drop_header('OBSCODE')
        with self.assertRaises(FormatException):
            reader = VisualFormatReader(self.fp)

    def test_date_format(self):
        """
        Check that the date format is read correctly.
        """
        reader = VisualFormatReader(self.fp)
        self.assertEqual(reader.date_format, 'JD')

    def test_missing_date_format(self):
        """
        Check that a missing DATE header raises an exception.
        """
        self._drop_header('DATE')
        with self.assertRaises(FormatException):
            reader = VisualFormatReader(self.fp)

    def test_software(self):
        """
        Check that SOFTWARE header is read correctly.
        """
        reader = VisualFormatReader(self.fp)
        self.assertEqual(reader.software, 'Notepad')

    def test_missing_software(self):
        """
        Check that SOFTWARE header is not required.
        """
        self._drop_header('SOFTWARE')
        reader = VisualFormatReader(self.fp)
        self.assertEqual(reader.software, '')

    def test_delimiter(self):
        """
        Check that the delimiter character is read correctly.
        """
        reader = VisualFormatReader(self.fp)
        self.assertEqual(reader.delimiter, str(','))

    def test_missing_delimiter(self):
        """
        Check that in case of missing delimiter, a comma is assumed.
        """
        self._drop_header('DELIM')
        reader = VisualFormatReader(self.fp)
        self.assertEqual(reader.delimiter, str(','))

    def test_obstype(self):
        """
        Check that observation type matches data in header.
        """
        reader = VisualFormatReader(self.fp)
        self.assertEqual(reader.obstype, str('Visual'))

    def test_missing_obstype(self):
        """
        Check that Visual is assumed obstype when OBSTYPE header is missing.
        """
        self._drop_header('OBSTYPE')
        reader = VisualFormatReader(self.fp)
        self.assertEqual(reader.obstype, 'Visual')

    def test_row_count(self):
        """
        Check that number of records in initial input file is exactly 1.
        """
        reader = VisualFormatReader(self.fp)
        rows = [row for row in reader]
        self.assertEqual(len(rows), 1)

    def test_row_count_duplicated(self):
        """
        Check that there are exactly 2 records after duplicating the last line.
        """
        # duplicate the last line
        self.lines.append(self.lines[-1])
        self.fp = StringIO("\n".join(self.lines))
        reader = VisualFormatReader(self.fp)
        rows = [row for row in reader]
        self.assertEqual(len(rows), 2)
<commit_msg>Added test for row dictionary.<commit_after>from __future__ import unicode_literals

import unittest
from six import StringIO

import pyaavso
from pyaavso.formats.visual import VisualFormatWriter, VisualFormatReader, \
    FormatException

class VisualFormatWriterTestCase(unittest.TestCase):
    """
    Tests for VisualFormatWriter class.
    """
    def setUp(self):
        self.fp = StringIO()

    def tearDown(self):
        self.fp.close()

    def test_header_type(self):
        """
        Check that TYPE parameter is always Visual.
        """
        writer = VisualFormatWriter(self.fp, 'XYZ')
        contents = self.fp.getvalue()
        self.assertIn("#TYPE=Visual", contents)

    def test_header_obscode(self):
        """
        Check that observer code is written into file.
        """
        writer = VisualFormatWriter(self.fp, 'XYZ')
        contents = self.fp.getvalue()
        self.assertIn("#OBSCODE=XYZ", contents)

    def test_header_software(self):
        """
        Check that the SOFTWARE parameter is correct.
        """
        writer = VisualFormatWriter(self.fp, 'XYZ')
        contents = self.fp.getvalue()
        self.assertIn("#SOFTWARE=pyaavso %s" % pyaavso.get_version(), contents)

    def test_header_date(self):
        """
        Check that the DATE parameter represents date format used.
        """
        writer = VisualFormatWriter(self.fp, 'XYZ', date_format='jd')
        contents = self.fp.getvalue()
        self.assertIn("#DATE=JD", contents)

    def test_header_obstype(self):
        """
        Check that OBSTYPE parameter is correctly set.
        """
        writer = VisualFormatWriter(self.fp, 'XYZ', obstype='Visual')
        contents = self.fp.getvalue()
        self.assertIn("#OBSTYPE=Visual", contents)

    def test_header_obstype_ptg(self):
        """
        Check that OBSTYPE can be set to PTG (Photographic).
        """
        writer = VisualFormatWriter(self.fp, 'XYZ', obstype='PTG')
        contents = self.fp.getvalue()
        self.assertIn("#OBSTYPE=PTG", contents)

    def test_write_raw_data(self):
        """
        Check that writerow() can accept raw data record.
        """
        data = [
            'SS CYG', '2450702.1234', '<11.1', 'na', '110', '113', '070613',
            'This is a test',
        ]
        writer = VisualFormatWriter(self.fp, 'XYZ')
        writer.writerow(data)
        lines = self.fp.getvalue().splitlines()
        self.assertEqual(
            lines[5],
            "SS CYG,2450702.1234,<11.1,na,110,113,070613,This is a test"
        )

    def test_write_dict(self):
        """
        Check that dictionary of observation data can be written to file.
        """
        data = {
            'name': 'SS CYG',
            'date': '2450702.1234',
            'magnitude': '<11.1',
            'comment_code': 'na',
            'comp1': '110',
            'comp2': '113',
            'chart': '070613',
            'notes': 'This is a test',
        }
        writer = VisualFormatWriter(self.fp, 'XYZ')
        writer.writerow(data)
        lines = self.fp.getvalue().splitlines()
        self.assertEqual(
            lines[5],
            "SS CYG,2450702.1234,<11.1,na,110,113,070613,This is a test"
        )

class VisualFormatReaderTestCase(unittest.TestCase):
    """
    Tests for VisualFormatReader class.
    """
    def setUp(self):
        self.lines = [
            "#TYPE=VISUAL",
            "#OBSCODE=XYZ",
            "#SOFTWARE=Notepad",
            "#DELIM=,",
            "#DATE=JD",
            "#OBSTYPE=Visual",
            "SS CYG,2450702.1234,<11.1,na,110,113,070613,This is a test",
        ]
        self.fp = StringIO("\n".join(self.lines))

    def tearDown(self):
        self.fp.close()

    def _drop_header(self, header_name):
        self.fp = StringIO("\n".join(
            line for line in self.lines if header_name not in line
        ))

    def test_missing_type(self):
        """
        Check that a missing TYPE header raises an exception.
        """
        self._drop_header('TYPE')
        with self.assertRaises(FormatException):
            reader = VisualFormatReader(self.fp)

    def test_observer_code(self):
        """
        Check that observer code matches header information.
        """
        reader = VisualFormatReader(self.fp)
        self.assertEqual(reader.observer_code, 'XYZ')

    def test_missing_observer_code(self):
        """
        Check that a missing OBSCODE header raises an exception.
        """
        self._drop_header('OBSCODE')
        with self.assertRaises(FormatException):
            reader = VisualFormatReader(self.fp)

    def test_date_format(self):
        """
        Check that the date format is read correctly.
        """
        reader = VisualFormatReader(self.fp)
        self.assertEqual(reader.date_format, 'JD')

    def test_missing_date_format(self):
        """
        Check that a missing DATE header raises an exception.
        """
        self._drop_header('DATE')
        with self.assertRaises(FormatException):
            reader = VisualFormatReader(self.fp)

    def test_software(self):
        """
        Check that SOFTWARE header is read correctly.
        """
        reader = VisualFormatReader(self.fp)
        self.assertEqual(reader.software, 'Notepad')

    def test_missing_software(self):
        """
        Check that SOFTWARE header is not required.
        """
        self._drop_header('SOFTWARE')
        reader = VisualFormatReader(self.fp)
        self.assertEqual(reader.software, '')

    def test_delimiter(self):
        """
        Check that the delimiter character is read correctly.
        """
        reader = VisualFormatReader(self.fp)
        self.assertEqual(reader.delimiter, str(','))

    def test_missing_delimiter(self):
        """
        Check that in case of missing delimiter, a comma is assumed.
        """
        self._drop_header('DELIM')
        reader = VisualFormatReader(self.fp)
        self.assertEqual(reader.delimiter, str(','))

    def test_obstype(self):
        """
        Check that observation type matches data in header.
        """
        reader = VisualFormatReader(self.fp)
        self.assertEqual(reader.obstype, str('Visual'))

    def test_missing_obstype(self):
        """
        Check that Visual is assumed obstype when OBSTYPE header is missing.
        """
        self._drop_header('OBSTYPE')
        reader = VisualFormatReader(self.fp)
        self.assertEqual(reader.obstype, 'Visual')

    def test_row_count(self):
        """
        Check that number of records in initial input file is exactly 1.
        """
        reader = VisualFormatReader(self.fp)
        rows = [row for row in reader]
        self.assertEqual(len(rows), 1)

    def test_row_count_duplicated(self):
        """
        Check that there are exactly 2 records after duplicating the last line.
        """
        # duplicate the last line
        self.lines.append(self.lines[-1])
        self.fp = StringIO("\n".join(self.lines))
        reader = VisualFormatReader(self.fp)
        rows = [row for row in reader]
        self.assertEqual(len(rows), 2)

    def test_row_dictionary(self):
        """
        Check that an observation is returned as a dictionary of values.
        """
        reader = VisualFormatReader(self.fp)
        rows = [row for row in reader]
        observation = rows[0]
        self.assertEqual(observation['name'], 'SS CYG')
        self.assertEqual(observation['comp1'], '110')
        self.assertEqual(observation['comp2'], '113')
<|endoftext|>


<commit_before>import core
import data

from .data import get
#from .core import year_frac, PerformanceStats, GroupStats, merge
from .core import *

core.extend_pandas()

__version__ = (0, 1, 7)
<commit_msg>version bump to 0.1.8<commit_after>import core
import data

from .data import get
#from .core import year_frac, PerformanceStats, GroupStats, merge
from .core import *

core.extend_pandas()

__version__ = (0, 1, 8)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import os
import sys

section_delimiter="-"
word_delimiter=" "
new_word_delimiter="_"
#extension = None
replacement=[
             ["'",""],
             [" ","_"],
             [".",""],
             ["","ss"],
             ["[","("],
             ["]",")"]
             ]

def replace_chars(string,replacement_list):
    if string and replacement_list:
        for item in replacement_list:
            string = string.replace(item[0],item[1])

    return string

def rename_files(directory):
    if os.path.exists(directory) and os.path.isdir(directory):
        os.chdir(directory)
        cwd = os.getcwd()
        for file in os.listdir(cwd):
            if os.path.isfile(file):
                extension = os.path.splitext(file)[-1]
                file_name = os.path.splitext(file)[0]
                splitted_file_name = file_name.split("-")
                if len(splitted_file_name[0]) == 1:
                    splitted_file_name[0] = "0"+splitted_file_name[0]

                file_name = "-".join(item.strip() for item in splitted_file_name)

                file_name = replace_chars(file_name,replacement)
                file_name = file_name+extension
                file_name = file_name.lower()
                if file_name != file:
                    if not os.path.exists(file_name):
                        os.renames(file,file_name)
                    else:
                        print file_name+" already exists"

        return 0
    else:
        return 1

def main():
    if len(sys.argv) > 1:
        directory = os.path.abspath(sys.argv[1])
        rename_files(directory)

        sys.exit(0)
    else:
        print "No directory specified."
        sys.exit(1)

if __name__ == "__main__":
     main()

 <commit_msg>extended char replacement dictionary<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import os
import sys

section_delimiter="-"
word_delimiter=" "
new_word_delimiter="_"
#extension = None
replacement=[
             ["'","_"],
             [" ","_"],
             [".",""],
             ["","ss"],
             ["[","("],
	     [",",""],
             ["]",")"],
	     ["!",""],
	     ["?",""]
             ]

def replace_chars(string,replacement_list):
    if string and replacement_list:
        for item in replacement_list:
            string = string.replace(item[0],item[1])

    return string

def rename_files(directory):
    if os.path.exists(directory) and os.path.isdir(directory):
        os.chdir(directory)
        cwd = os.getcwd()
        for file in os.listdir(cwd):
            if os.path.isfile(file):
                extension = os.path.splitext(file)[-1]
                file_name = os.path.splitext(file)[0]
                splitted_file_name = file_name.split("-")
                if len(splitted_file_name[0]) == 1:
                    splitted_file_name[0] = "0"+splitted_file_name[0]

                file_name = "-".join(item.strip() for item in splitted_file_name)

                file_name = replace_chars(file_name,replacement)
                file_name = file_name+extension
                file_name = file_name.lower()
                if file_name != file:
                    if not os.path.exists(file_name):
                        os.renames(file,file_name)
                    else:
                        print file_name+" already exists"

        return 0
    else:
        return 1

def main():
    if len(sys.argv) > 1:
        directory = os.path.abspath(sys.argv[1])
        rename_files(directory)

        sys.exit(0)
    else:
        print "No directory specified."
        sys.exit(1)

if __name__ == "__main__":
     main()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import datetime

from pyramid.decorator import reify
from sqlalchemy import Column
from sqlalchemy import DateTime
from sqlalchemy import ForeignKey
from sqlalchemy import func
from sqlalchemy import Integer
from sqlalchemy import Unicode
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.orm import aliased

from ines.api.database.sql import Options
from ines.api.database.sql import sql_declarative_base
from ines.exceptions import Error
from ines.utils import make_uuid_hash

class CoreTypesMissing(dict):
    def __missing__(self, key):
        self[key] = {
            'table': None,
            'parent': None,
            'childs': set(),
            'branches': set()}
        return self[key]

CORE_TYPES = CoreTypesMissing()
CORE_KEYS = set()
NOW_DATE = datetime.datetime.now
DeclarativeBase = sql_declarative_base('core')

class CoreAliased(dict):
    def __missing__(self, key):
        self[key] = aliased(Core)
        return self[key]

class Core(DeclarativeBase):
    __tablename__ = 'core'

    id = Column(Integer, primary_key=True, nullable=False)
    key = Column(Unicode(70), unique=True, index=True, nullable=False)
    # If null, means this object is a relation type
    type = Column(Unicode(50), index=True)
    parent_id = Column(Integer, ForeignKey('core.id'))

    start_date = Column(DateTime)
    end_date = Column(DateTime)
    updated_date = Column(
        DateTime,
        default=func.now(), onupdate=func.now(),
        nullable=False)
    created_date = Column(DateTime, default=func.now(), nullable=False)

    def make_key(self):
        return make_uuid_hash()

CORE_TYPES['core']['table'] = Core

class CoreColumnParent(object):
    def __init__(self, table, attribute):
        self._table = table
        self.attribute = attribute
        self.with_label = None

    def clone(self):
        return CoreColumnParent(self._table, self.attribute)

    @reify
    def table(self):
        return self._table.__table__

    @reify
    def name(self):
        return '%s.%s' % (self._table.__tablename__, self.attribute)

    def __repr__(self):
        return self.name

    def get_core_column(self):
        return getattr(Core, self.attribute)

    def get_alias_column(self, alias):
        column = getattr(alias, self.attribute)
        if self.with_label:
            return column.label(self.with_label)
        else:
            return column

    def label(self, name):
        self.with_label = name
        return self

def replace_core_attribute(wrapped):
    def decorator(self):
        name = wrapped.__name__
        value = CoreColumnParent(self, name)
        setattr(self, name, value)
        CORE_KEYS.add(name)
        return value
    return decorator

class CoreType(object):
    core_name = None

    @declared_attr
    def __tablename__(self):
        if self.core_name in CORE_TYPES:
            message = u'Core "%s" already defined' % self.core_name
            raise Error('core', message)
        else:
            CORE_TYPES[self.core_name]['table'] = self

            core_relation = getattr(self, 'core_relation', None)
            if core_relation:
                relation, relation_table = core_relation
                if relation == 'parent':
                    CORE_TYPES[self.core_name]['parent'] = relation_table
                    CORE_TYPES[relation_table.core_name]['childs'].add(self)
                elif relation == 'branch':
                    CORE_TYPES[relation_table.core_name]['branches'].add(self)
                else:
                    raise ValueError('Invalid relation type')

            tablename = 'core_%s' % self.core_name
            setattr(self, '__tablename__', tablename)
            return tablename

    @declared_attr
    @replace_core_attribute
    def key(self):
        pass

    @declared_attr
    @replace_core_attribute
    def type(self):
        pass

    @declared_attr
    @replace_core_attribute
    def start_date(self):
        pass

    @declared_attr
    @replace_core_attribute
    def end_date(self):
        pass

    @declared_attr
    @replace_core_attribute
    def updated_date(self):
        pass

    @declared_attr
    @replace_core_attribute
    def created_date(self):
        pass

    @declared_attr
    @replace_core_attribute
    def parent_id(self):
        pass

    @declared_attr
    def id_core(self):
        return Column(
            Integer, ForeignKey(Core.id),
            primary_key=True, nullable=False)

class CoreOptions(Options):
    def add_table(self, table, ignore=None, add_name=None):
        Options.add_table(self, table, ignore=ignore, add_name=add_name)

        columns = table.__dict__.keys()
        for key in columns:
            maybe_column = getattr(table, key)
            if isinstance(maybe_column, CoreColumnParent):
                if not ignore or key not in ignore:
                    if add_name:
                        key = '%s_%s' % (add_name, key)
                    self.add_column(key, maybe_column.clone())

        start_alias = table.start_date.clone()
        end_alias = table.end_date.clone()
        key = 'active'
        if add_name:
            key = '%s_%s' % (add_name, key)

        self.add_column(
            key,
            and_(or_(start_alias == None, start_alias <= func.now()),
                 or_(end_alias == None, end_alias <= func.now())).label(key))

from sqlalchemy import and_
from sqlalchemy import or_

def find_parent_tables(table):
    # Find parent tables
    tables = set()
    while True:
        core_relation = getattr(table, 'core_relation', None)
        if not core_relation:
            break
        relation, table = core_relation
        tables.add(table)
    return tables
<commit_msg>Reduce core key length to 7 chars<commit_after># -*- coding: utf-8 -*-

import datetime

from pyramid.decorator import reify
from sqlalchemy import Column
from sqlalchemy import DateTime
from sqlalchemy import ForeignKey
from sqlalchemy import func
from sqlalchemy import Integer
from sqlalchemy import Unicode
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.orm import aliased

from ines.api.database.sql import Options
from ines.api.database.sql import sql_declarative_base
from ines.exceptions import Error
from ines.utils import make_uuid_hash

class CoreTypesMissing(dict):
    def __missing__(self, key):
        self[key] = {
            'table': None,
            'parent': None,
            'childs': set(),
            'branches': set()}
        return self[key]

CORE_TYPES = CoreTypesMissing()
CORE_KEYS = set()
NOW_DATE = datetime.datetime.now
DeclarativeBase = sql_declarative_base('core')

class CoreAliased(dict):
    def __missing__(self, key):
        self[key] = aliased(Core)
        return self[key]

class Core(DeclarativeBase):
    __tablename__ = 'core'
    __key_length__ = 7

    id = Column(Integer, primary_key=True, nullable=False)
    key = Column(Unicode(70), unique=True, index=True, nullable=False)
    # If null, means this object is a relation type
    type = Column(Unicode(50), index=True)
    parent_id = Column(Integer, ForeignKey('core.id'))

    start_date = Column(DateTime)
    end_date = Column(DateTime)
    updated_date = Column(
        DateTime,
        default=func.now(), onupdate=func.now(),
        nullable=False)
    created_date = Column(DateTime, default=func.now(), nullable=False)

    def make_key(self):
        return make_uuid_hash()[:self.__key_length__]

CORE_TYPES['core']['table'] = Core

class CoreColumnParent(object):
    def __init__(self, table, attribute):
        self._table = table
        self.attribute = attribute
        self.with_label = None

    def clone(self):
        return CoreColumnParent(self._table, self.attribute)

    @reify
    def table(self):
        return self._table.__table__

    @reify
    def name(self):
        return '%s.%s' % (self._table.__tablename__, self.attribute)

    def __repr__(self):
        return self.name

    def get_core_column(self):
        return getattr(Core, self.attribute)

    def get_alias_column(self, alias):
        column = getattr(alias, self.attribute)
        if self.with_label:
            return column.label(self.with_label)
        else:
            return column

    def label(self, name):
        self.with_label = name
        return self

def replace_core_attribute(wrapped):
    def decorator(self):
        name = wrapped.__name__
        value = CoreColumnParent(self, name)
        setattr(self, name, value)
        CORE_KEYS.add(name)
        return value
    return decorator

class CoreType(object):
    core_name = None
    __key_length__ = 7

    @declared_attr
    def __tablename__(self):
        if self.core_name in CORE_TYPES:
            message = u'Core "%s" already defined' % self.core_name
            raise Error('core', message)
        else:
            CORE_TYPES[self.core_name]['table'] = self

            core_relation = getattr(self, 'core_relation', None)
            if core_relation:
                relation, relation_table = core_relation
                if relation == 'parent':
                    CORE_TYPES[self.core_name]['parent'] = relation_table
                    CORE_TYPES[relation_table.core_name]['childs'].add(self)
                elif relation == 'branch':
                    CORE_TYPES[relation_table.core_name]['branches'].add(self)
                else:
                    raise ValueError('Invalid relation type')

            tablename = 'core_%s' % self.core_name
            setattr(self, '__tablename__', tablename)
            return tablename

    @declared_attr
    @replace_core_attribute
    def key(self):
        pass

    @declared_attr
    @replace_core_attribute
    def type(self):
        pass

    @declared_attr
    @replace_core_attribute
    def start_date(self):
        pass

    @declared_attr
    @replace_core_attribute
    def end_date(self):
        pass

    @declared_attr
    @replace_core_attribute
    def updated_date(self):
        pass

    @declared_attr
    @replace_core_attribute
    def created_date(self):
        pass

    @declared_attr
    @replace_core_attribute
    def parent_id(self):
        pass

    @declared_attr
    def id_core(self):
        return Column(
            Integer, ForeignKey(Core.id),
            primary_key=True, nullable=False)

class CoreOptions(Options):
    def add_table(self, table, ignore=None, add_name=None):
        Options.add_table(self, table, ignore=ignore, add_name=add_name)

        columns = table.__dict__.keys()
        for key in columns:
            maybe_column = getattr(table, key)
            if isinstance(maybe_column, CoreColumnParent):
                if not ignore or key not in ignore:
                    if add_name:
                        key = '%s_%s' % (add_name, key)
                    self.add_column(key, maybe_column.clone())

        start_alias = table.start_date.clone()
        end_alias = table.end_date.clone()
        key = 'active'
        if add_name:
            key = '%s_%s' % (add_name, key)

        self.add_column(
            key,
            and_(or_(start_alias == None, start_alias <= func.now()),
                 or_(end_alias == None, end_alias <= func.now())).label(key))

from sqlalchemy import and_
from sqlalchemy import or_

def find_parent_tables(table):
    # Find parent tables
    tables = set()
    while True:
        core_relation = getattr(table, 'core_relation', None)
        if not core_relation:
            break
        relation, table = core_relation
        tables.add(table)
    return tables
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: iso-8859-1 -*-
# vi:ts=4:et

import pycurl
import unittest
from .util import StringIO
try:
    import cPickle
except ImportError:
    cPickle = None
import pickle
import copy

class InternalsTest(unittest.TestCase):
    def setUp(self):
        self.curl = pycurl.Curl()

    def tearDown(self):
        self.curl.close()
        del self.curl

    # /***********************************************************************
    # // test misc
    # ************************************************************************/

    def test_constant_aliasing(self):
        assert self.curl.URL is pycurl.URL

    # /***********************************************************************
    # // test handles
    # ************************************************************************/

    def test_remove_invalid_handle(self):
        m = pycurl.CurlMulti()
        try:
            m.remove_handle(self.curl)
        except pycurl.error:
            pass
        else:
            assert False, "No exception when trying to remove a handle that is not in CurlMulti"
        del m

    def test_remove_invalid_closed_handle(self):
        m = pycurl.CurlMulti()
        c = pycurl.Curl()
        c.close()
        m.remove_handle(c)
        del m, c

    def test_add_closed_handle(self):
        m = pycurl.CurlMulti()
        c = pycurl.Curl()
        c.close()
        try:
            m.add_handle(c)
        except pycurl.error:
            pass
        else:
            assert 0, "No exception when trying to add a close handle to CurlMulti"
        m.close()
        del m, c

    def test_add_handle_twice(self):
        m = pycurl.CurlMulti()
        m.add_handle(self.curl)
        try:
            m.add_handle(self.curl)
        except pycurl.error:
            pass
        else:
            assert 0, "No exception when trying to add the same handle twice"
        del m

    def test_add_handle_on_multiple_stacks(self):
        m1 = pycurl.CurlMulti()
        m2 = pycurl.CurlMulti()
        m1.add_handle(self.curl)
        try:
            m2.add_handle(self.curl)
        except pycurl.error:
            pass
        else:
            assert 0, "No exception when trying to add the same handle on multiple stacks"
        del m1, m2

    def test_move_handle(self):
        m1 = pycurl.CurlMulti()
        m2 = pycurl.CurlMulti()
        m1.add_handle(self.curl)
        m1.remove_handle(self.curl)
        m2.add_handle(self.curl)
        del m1, m2

    # /***********************************************************************
    # // test copying and pickling - copying and pickling of
    # // instances of Curl and CurlMulti is not allowed
    # ************************************************************************/

    def test_copy_curl(self):
        try:
            copy.copy(self.curl)
        # python 2 raises copy.Error, python 3 raises TypeError
        except (copy.Error, TypeError):
            pass
        else:
            assert False, "No exception when trying to copy a Curl handle"

    def test_copy_multi(self):
        m = pycurl.CurlMulti()
        try:
            copy.copy(m)
        except (copy.Error, TypeError):
            pass
        else:
            assert False, "No exception when trying to copy a CurlMulti handle"

    def test_pickle_curl(self):
        fp = StringIO()
        p = pickle.Pickler(fp, 1)
        try:
            p.dump(self.curl)
        # python 2 raises pickle.PicklingError, python 3 raises TypeError
        except (pickle.PicklingError, TypeError):
            pass
        else:
            assert 0, "No exception when trying to pickle a Curl handle"
        del fp, p

    def test_pickle_multi(self):
        m = pycurl.CurlMulti()
        fp = StringIO()
        p = pickle.Pickler(fp, 1)
        try:
            p.dump(m)
        except (pickle.PicklingError, TypeError):
            pass
        else:
            assert 0, "No exception when trying to pickle a CurlMulti handle"
        del m, fp, p

    if cPickle is not None:
        def test_cpickle_curl(self):
            fp = StringIO()
            p = cPickle.Pickler(fp, 1)
            try:
                p.dump(self.curl)
            except cPickle.PicklingError:
                pass
            else:
                assert 0, "No exception when trying to pickle a Curl handle via cPickle"
            del fp, p

        def test_cpickle_multi(self):
            m = pycurl.CurlMulti()
            fp = StringIO()
            p = cPickle.Pickler(fp, 1)
            try:
                p.dump(m)
            except cPickle.PicklingError:
                pass
            else:
                assert 0, "No exception when trying to pickle a CurlMulti handle via cPickle"
            del m, fp, p
<commit_msg>Use uniform "from . import xxx" syntax for ease of patching<commit_after>#! /usr/bin/env python
# -*- coding: iso-8859-1 -*-
# vi:ts=4:et

import pycurl
import unittest
try:
    import cPickle
except ImportError:
    cPickle = None
import pickle
import copy

from . import util

class InternalsTest(unittest.TestCase):
    def setUp(self):
        self.curl = pycurl.Curl()

    def tearDown(self):
        self.curl.close()
        del self.curl

    # /***********************************************************************
    # // test misc
    # ************************************************************************/

    def test_constant_aliasing(self):
        assert self.curl.URL is pycurl.URL

    # /***********************************************************************
    # // test handles
    # ************************************************************************/

    def test_remove_invalid_handle(self):
        m = pycurl.CurlMulti()
        try:
            m.remove_handle(self.curl)
        except pycurl.error:
            pass
        else:
            assert False, "No exception when trying to remove a handle that is not in CurlMulti"
        del m

    def test_remove_invalid_closed_handle(self):
        m = pycurl.CurlMulti()
        c = pycurl.Curl()
        c.close()
        m.remove_handle(c)
        del m, c

    def test_add_closed_handle(self):
        m = pycurl.CurlMulti()
        c = pycurl.Curl()
        c.close()
        try:
            m.add_handle(c)
        except pycurl.error:
            pass
        else:
            assert 0, "No exception when trying to add a close handle to CurlMulti"
        m.close()
        del m, c

    def test_add_handle_twice(self):
        m = pycurl.CurlMulti()
        m.add_handle(self.curl)
        try:
            m.add_handle(self.curl)
        except pycurl.error:
            pass
        else:
            assert 0, "No exception when trying to add the same handle twice"
        del m

    def test_add_handle_on_multiple_stacks(self):
        m1 = pycurl.CurlMulti()
        m2 = pycurl.CurlMulti()
        m1.add_handle(self.curl)
        try:
            m2.add_handle(self.curl)
        except pycurl.error:
            pass
        else:
            assert 0, "No exception when trying to add the same handle on multiple stacks"
        del m1, m2

    def test_move_handle(self):
        m1 = pycurl.CurlMulti()
        m2 = pycurl.CurlMulti()
        m1.add_handle(self.curl)
        m1.remove_handle(self.curl)
        m2.add_handle(self.curl)
        del m1, m2

    # /***********************************************************************
    # // test copying and pickling - copying and pickling of
    # // instances of Curl and CurlMulti is not allowed
    # ************************************************************************/

    def test_copy_curl(self):
        try:
            copy.copy(self.curl)
        # python 2 raises copy.Error, python 3 raises TypeError
        except (copy.Error, TypeError):
            pass
        else:
            assert False, "No exception when trying to copy a Curl handle"

    def test_copy_multi(self):
        m = pycurl.CurlMulti()
        try:
            copy.copy(m)
        except (copy.Error, TypeError):
            pass
        else:
            assert False, "No exception when trying to copy a CurlMulti handle"

    def test_pickle_curl(self):
        fp = util.StringIO()
        p = pickle.Pickler(fp, 1)
        try:
            p.dump(self.curl)
        # python 2 raises pickle.PicklingError, python 3 raises TypeError
        except (pickle.PicklingError, TypeError):
            pass
        else:
            assert 0, "No exception when trying to pickle a Curl handle"
        del fp, p

    def test_pickle_multi(self):
        m = pycurl.CurlMulti()
        fp = util.StringIO()
        p = pickle.Pickler(fp, 1)
        try:
            p.dump(m)
        except (pickle.PicklingError, TypeError):
            pass
        else:
            assert 0, "No exception when trying to pickle a CurlMulti handle"
        del m, fp, p

    if cPickle is not None:
        def test_cpickle_curl(self):
            fp = util.StringIO()
            p = cPickle.Pickler(fp, 1)
            try:
                p.dump(self.curl)
            except cPickle.PicklingError:
                pass
            else:
                assert 0, "No exception when trying to pickle a Curl handle via cPickle"
            del fp, p

        def test_cpickle_multi(self):
            m = pycurl.CurlMulti()
            fp = util.StringIO()
            p = cPickle.Pickler(fp, 1)
            try:
                p.dump(m)
            except cPickle.PicklingError:
                pass
            else:
                assert 0, "No exception when trying to pickle a CurlMulti handle via cPickle"
            del m, fp, p
<|endoftext|>


<commit_before>
from ggame import App, RectangleAsset, CircleAsset, ImageAsset, Sprite, LineStyle, Color, Frame

SCREEN_WIDTH = 1080
SCREEN_HEIGHT = 720
black = Color(0, 1)
lgreen = Color(0xd9ffcc, 1.0)
dgreen = Color(0x006600, 1.0)
purp = Color(0x9900cc, 1.0)
blue = Color(0x3399ff,1.0)
gold = Color(0xcc9900, 1.0)
noline = LineStyle(0, black)
topr = RectangleAsset (1080, 20, noline, dgreen)
botr = RectangleAsset (1080, 20, noline, dgreen)
sider = RectangleAsset (20, 720, noline, dgreen)
capr = RectangleAsset (2, 30, noline, dgreen)
player = RectangleAsset (20, 50, noline, purp)
prize = RectangleAsset (35, 15, noline, gold)
barrels = CircleAsset (10, noline, black)
vy=0
vx=0
ti=0

class dk(App):
    def __init__(self, width, height):
            super().__init__(width, height)
            bg_asset = RectangleAsset(width, height, noline, lgreen)
            bg = Sprite(bg_asset, (0,0))
    def step (self):
        playe.step()
        bare.step()

class top(Sprite):
    def __init__(self,position):
        super().__init__(topr, position)

class bot(Sprite):
    def __init__(self,position):
        super().__init__(botr, position)

class side(Sprite):
    def __init__(self,position):
        super().__init__(sider, position)

class cap(Sprite):
    def __init__(self,position):
        super().__init__(capr, position)

class win(Sprite):
    def __init__(self,position):
        super().__init__(prize, position)

onblock = 0
jump = 0
wub=0
go = 0

class bar(Sprite):
    def __init__(self, position, vx, vy, ti):
        super().__init__(barrels, position)
        self.vx = vx
        self.vy = vy
        self.ti = ti
        self.onblock=0
        self.go = 0
    def step(self):
        self.go=1
        if self.go == 1:
            if self.onblock == 1:
                self.ti = 0
                self.vy = 0
            self.vy =self.ti+self.vy
            self.y=self.y+self.vy
            self.x=self.x+self.vx
            self.vx = 2
            if onblock != 1:
                self.ti += 0.005
            on = self.collidingWithSprites(top)
            if len(on) > 0:
                self.ti = 0
                self.vy = 0
                self.y-=.5
                self.onblock = 1
            under = self.collidingWithSprites(bot)
            if len(under) > 0:
                self.vy=-self.vy
                self.y+=10
            csides = self.collidingWithSprites(side)
            ccap = self.collidingWithSprites(cap)
            if len(csides) > 0:
                self.vx=-self.vx
            if len(ccap) > 0:
                self.vx=-self.vx

class play(Sprite):
    def __init__(self, position, vx, vy, ti):
        super().__init__(player, position)
        dk.listenKeyEvent('keydown', 'left arrow', self.lup)
        dk.listenKeyEvent('keydown', 'right arrow', self.rup)
        dk.listenKeyEvent('keydown', 'up arrow', self.uup)
        dk.listenKeyEvent('keydown', 'w', self.wup)
        self.vx=vx
        self.vy=vy
        self.ti=ti
        self.onblock=0
        self.jump = 0
        self.wub =0
    def step(self):
        onblock=0
        if self.wub == 0:
            self.ti=self.ti+0.01
        self.vy =self.ti+self.vy
        self.y=self.y+self.vy
        self.x=self.x+self.vx
        on = self.collidingWithSprites(top)
        if len(on) > 0:
            self.ti = 0
            self.vy = 0
            self.y-=.5
            self.onblock = 1
        under = self.collidingWithSprites(bot)
        if len(under) > 0:
            self.vy=-self.vy
            self.y+=10
        csides = self.collidingWithSprites(side)
        ccap = self.collidingWithSprites(cap)
        if len(csides) > 0:
            self.vx=-self.vx
        if len(ccap) > 0:
            self.vx=-self.vx
        if self.jump == 1:
            if self.onblock ==1:
                self.vy -= 7.5
                self.jump = 0
        if self.wub == 1:
            if self.onblock == 1:
                self.vy -= 3
                self.ti+=0.0015
                self.wub = 0
    def rup(self, event):
        if self.vx<3:
            self.vx+=1
    def lup(self, event):
        if self.vx>-3:
            self.vx-=1
    def uup(self, event):
        if self.onblock==1:
            self.jump = 1
    def wup(self, event):
        self.wub = 1

myapp = dk(SCREEN_WIDTH, SCREEN_HEIGHT)

top ((0, 710))
bot ((0, -10))
side ((-10, 0))
side ((1070, 0))

top ((500, 580))
bot ((500, 590))
cap ((498, 580))
top ((-800, 580))
bot ((-800, 590))

top ((800, 450))
bot ((800, 460))
top ((-400, 450))
bot ((-400, 460))

top ((700, 320))
bot ((700, 330))
top ((-500, 320))
bot ((-500, 330))

top ((900, 190))
bot ((900, 200))
top ((-300, 190))
bot ((-300, 200))

playe = play((1000, 640), 0, 0, 0)
prizee = win((300, 175))
prizee = win((540, 10))

bare = bar ((540, 12), 0, 0, 0)

myapp.run()<commit_msg>Updated from Brython Server: 4/14/2017 2:24:40 PM<commit_after>
from ggame import App, RectangleAsset, CircleAsset, ImageAsset, Sprite, LineStyle, Color, Frame

SCREEN_WIDTH = 1080
SCREEN_HEIGHT = 720
black = Color(0, 1)
lgreen = Color(0xd9ffcc, 1.0)
dgreen = Color(0x006600, 1.0)
purp = Color(0x9900cc, 1.0)
blue = Color(0x3399ff,1.0)
gold = Color(0xcc9900, 1.0)
noline = LineStyle(0, black)
topr = RectangleAsset (1080, 20, noline, dgreen)
botr = RectangleAsset (1080, 20, noline, dgreen)
sider = RectangleAsset (20, 720, noline, dgreen)
capr = RectangleAsset (2, 30, noline, dgreen)
player = RectangleAsset (20, 50, noline, purp)
prize = RectangleAsset (35, 15, noline, gold)
barrels = CircleAsset (10, noline, black)
vy=0
vx=0
ti=0

class dk(App):
    def __init__(self, width, height):
            super().__init__(width, height)
            bg_asset = RectangleAsset(width, height, noline, lgreen)
            bg = Sprite(bg_asset, (0,0))
    def step (self):
        playe.step()
        bare.step()

class top(Sprite):
    def __init__(self,position):
        super().__init__(topr, position)

class bot(Sprite):
    def __init__(self,position):
        super().__init__(botr, position)

class side(Sprite):
    def __init__(self,position):
        super().__init__(sider, position)

class cap(Sprite):
    def __init__(self,position):
        super().__init__(capr, position)

class win(Sprite):
    def __init__(self,position):
        super().__init__(prize, position)

onblock = 0
jump = 0
wub=0
go = 0

class bar(Sprite):
    def __init__(self, position, vx, vy, ti):
        super().__init__(barrels, position)
        self.vx = vx
        self.vy = vy
        self.ti = ti
        self.onblock=0
        self.go = 0
        self.vx = 2
    def step(self):
        self.go=1
        if self.go == 1:
            if self.onblock == 1:
                self.ti = 0
                self.vy = 0
            self.vy =self.ti+self.vy
            self.y=self.y+self.vy
            self.x=self.x+self.vx
            if onblock != 1:
                self.ti += 0.005
            on = self.collidingWithSprites(top)
            if len(on) > 0:
                self.ti = 0
                self.vy = 0
                self.onblock = 1
            if len(on) == 0:
                self.onblock=0
            csides = self.collidingWithSprites(side)
            ccap = self.collidingWithSprites(cap)
            if len(csides) > 0:
                self.vx=-self.vx
            if len(ccap) > 0:
                self.vx=-self.vx

class play(Sprite):
    def __init__(self, position, vx, vy, ti):
        super().__init__(player, position)
        dk.listenKeyEvent('keydown', 'left arrow', self.lup)
        dk.listenKeyEvent('keydown', 'right arrow', self.rup)
        dk.listenKeyEvent('keydown', 'up arrow', self.uup)
        dk.listenKeyEvent('keydown', 'w', self.wup)
        self.vx=vx
        self.vy=vy
        self.ti=ti
        self.onblock=0
        self.jump = 0
        self.wub =0
    def step(self):
        onblock=0
        if self.wub == 0:
            self.ti=self.ti+0.01
        self.vy =self.ti+self.vy
        self.y=self.y+self.vy
        self.x=self.x+self.vx
        on = self.collidingWithSprites(top)
        if len(on) > 0:
            self.ti = 0
            self.vy = 0
            self.y-=.5
            self.onblock = 1
        under = self.collidingWithSprites(bot)
        if len(under) > 0:
            self.vy=-self.vy
            self.y+=10
        csides = self.collidingWithSprites(side)
        ccap = self.collidingWithSprites(cap)
        if len(csides) > 0:
            self.vx=-self.vx
        if len(ccap) > 0:
            self.vx=-self.vx
        if self.jump == 1:
            if self.onblock ==1:
                self.vy -= 7.5
                self.jump = 0
        if self.wub == 1:
            if self.onblock == 1:
                self.vy -= 3
                self.ti+=0.0012
                self.wub = 0
    def rup(self, event):
        if self.vx<3:
            self.vx+=1
    def lup(self, event):
        if self.vx>-3:
            self.vx-=1
    def uup(self, event):
        if self.onblock==1:
            self.jump = 1
    def wup(self, event):
        self.wub = 1

myapp = dk(SCREEN_WIDTH, SCREEN_HEIGHT)

top ((0, 710))
bot ((0, -10))
side ((-10, 0))
side ((1070, 0))

top ((500, 580))
bot ((500, 590))
cap ((498, 580))
top ((-800, 580))
bot ((-800, 590))

top ((800, 450))
bot ((800, 460))
top ((-400, 450))
bot ((-400, 460))

top ((700, 320))
bot ((700, 330))
top ((-500, 320))
bot ((-500, 330))

top ((900, 190))
bot ((900, 200))
top ((-300, 190))
bot ((-300, 200))

playe = play((1000, 640), 0, 0, 0)
prizee = win((300, 175))
prizee = win((540, 10))

bare = bar ((540, 12), 0, 0, 0)

myapp.run()<|endoftext|>


<commit_before># file dedicated to making the tables
# for the main document and the Supporting Information
# of the artigcle about text differentiation:
# https://github.com/ttm/artigoTextoNasRedes/ AND
# http://arxiv.org/abs/1412.7309

# tentar fazer as tabelas focadas nas medidas?

# fazer td com 100 mensagens somente de cada lista

# fazer tabela geral só com CPP
# incluir mensagens/ano na tabela
# tirar as datas para a tabela no corpo do artigo

# tentar paralelizar

# fazer tabela de caracteres soh com LAD
# para o corpo do artigo.
# renderizar 4 listas
# avaliar fazer para mais listas para tirar médias e desvios

import gmane as g, os, pickle, time as T, numpy as n
ENV=os.environ["PATH"]
import  importlib
from IPython.lib.deepreload import reload as dreload
importlib.reload(g.pca)
importlib.reload(g.loadMessages)
importlib.reload(g.listDataStructures)
dreload(g,exclude="pytz")
os.environ["PATH"]=ENV

TT=T.time()
#print("{0:.2f}".format(T.time()-TT)); TT=T.time()
def pDump(tobject,tfilename):
    with open(tfilename,"wb") as f:
        pickle.dump(tobject,f,-1)
def pRead(tfilename):
    with open(tfilename,"rb") as f:
        tobject=pickle.load(f)
    return tobject

#dl=g.DownloadGmaneData('~/.gmane3/')
#dl.downloadedStats()
PDIR="pickledir/"
#pDump(dl,"{}dl.pickle".format(PDIR))
dl=pRead("{}dl.pickle".format(PDIR))

###### DATA STRUCTURES
dss=[]; iNs=[]; nms=[]; tss=[]; nps=[]
PDIR="pickledir/"
for lid in dl.lists[:4]:
    lid=lid[0]
    print("\n",lid)
#    label=labels[lid]
    lm=g.LoadMessages(lid,500,basedir="~/.gmane3/")
    print(lid+"{0:.2f} for loading messages".format(T.time()-TT)); TT=T.time()
    ds=g.ListDataStructures(lm,text="yes")
    print(lid+"{0:.2f} for data structures".format(T.time()-TT)); TT=T.time()
    pDump(ds,"{}ds{}.pickle".format(PDIR,lid))
    dss.append(ds)

    ts=g.TimeStatistics(ds)
    print("{0:.2f} for statistics along time".format(T.time()-TT)); TT=T.time()
    pDump(ts,"{}ts{}.pickle".format(PDIR,lid))
    tss.append(ts)

    iN=g.InteractionNetwork(ds)
    print("made interaction network")
    iNs.append(iN)
    nm=g.NetworkMeasures(iN,exclude=["rich_club"])
    print("network mesaures")
    nms.append(nm)
    np2_=g.NetworkPartitioning(nm,2,"g")
    print("partitioned network")
    nps.append(np2_)

#for lid in dl.lists[:10]:
#    lid=lid[0]
#    tss.append(pRead("{}ts{}.pickle".format(PDIR,lid)))
#    print(lid+"{0:.2f} for PICKLE loading time statistics".format(T.time()-TT)); TT=T.time()

# Faz rede
# particiona rede
#for i in [0]:
#    labels_.append(labels[dl.downloaded_lists[i]])
data_=[]; count=0
for lid in dl.lists[:4]:
    lid=lid[0]
    ds=dss[count]; count+=1
    date1=ds.messages[ds.message_ids[0]][2].isoformat().split("T")[0]
    date2=ds.messages[ds.message_ids[-1]][2].isoformat().split("T")[0]
    N=ds.n_authors
    Gamma=len([i for i in ds.message_ids if ds.messages[i][1]==None])
    M_=20000-ds.n_messages
    data_.append([date1,date2,N,Gamma,M_])
#tstring=g.makeTables(labels_,data_)
#print(tstring)
#TDIR="tables/"
#TDIR="/home/r/repos/stabilityInteraction/tables/"
#FDIR="figs/"
#print(label+"{0:.2f} for making overall table".format(T.time()-TT)); TT=T.time()
#
##g.writeTex(tstring,TDIR+"tab1Geral.tex")
#

<commit_msg>complementadas medidas basicas para tabela geral<commit_after># file dedicated to making the tables
# for the main document and the Supporting Information
# of the artigcle about text differentiation:
# https://github.com/ttm/artigoTextoNasRedes/ AND
# http://arxiv.org/abs/1412.7309

# tentar fazer as tabelas focadas nas medidas?

# fazer td com 100 mensagens somente de cada lista

# fazer tabela geral só com CPP
# incluir mensagens/ano na tabela
# tirar as datas para a tabela no corpo do artigo

# tentar paralelizar

# fazer tabela de caracteres soh com LAD
# para o corpo do artigo.
# renderizar 4 listas
# avaliar fazer para mais listas para tirar médias e desvios

import gmane as g, os, pickle, time as T, numpy as n
ENV=os.environ["PATH"]
import  importlib
from IPython.lib.deepreload import reload as dreload
importlib.reload(g.pca)
importlib.reload(g.loadMessages)
importlib.reload(g.listDataStructures)
dreload(g,exclude="pytz")
os.environ["PATH"]=ENV

TT=T.time()
#print("{0:.2f}".format(T.time()-TT)); TT=T.time()
def pDump(tobject,tfilename):
    with open(tfilename,"wb") as f:
        pickle.dump(tobject,f,-1)
def pRead(tfilename):
    with open(tfilename,"rb") as f:
        tobject=pickle.load(f)
    return tobject

#dl=g.DownloadGmaneData('~/.gmane3/')
#dl.downloadedStats()
PDIR="pickledir/"
#pDump(dl,"{}dl.pickle".format(PDIR))
dl=pRead("{}dl.pickle".format(PDIR))

###### DATA STRUCTURES
TOTAL_M=500
dss=[]; iNs=[]; nms=[]; tss=[]; nps=[]
PDIR="pickledir/"
for lid in dl.lists[:4]:
    lid=lid[0]
    print("\n",lid)
#    label=labels[lid]
    lm=g.LoadMessages(lid,TOTAL_M,basedir="~/.gmane3/")
    print(lid+"{0:.2f} for loading messages".format(T.time()-TT)); TT=T.time()
    ds=g.ListDataStructures(lm,text="yes")
    print(lid+"{0:.2f} for data structures".format(T.time()-TT)); TT=T.time()
    pDump(ds,"{}ds{}.pickle".format(PDIR,lid))
    dss.append(ds)

    ts=g.TimeStatistics(ds)
    print("{0:.2f} for statistics along time".format(T.time()-TT)); TT=T.time()
    pDump(ts,"{}ts{}.pickle".format(PDIR,lid))
    tss.append(ts)

    iN=g.InteractionNetwork(ds)
    print("made interaction network")
    iNs.append(iN)
    nm=g.NetworkMeasures(iN,exclude=["rich_club"])
    print("network mesaures")
    nms.append(nm)
    np2_=g.NetworkPartitioning(nm,2,"g")
    print("partitioned network")
    nps.append(np2_)

#for lid in dl.lists[:10]:
#    lid=lid[0]
#    tss.append(pRead("{}ts{}.pickle".format(PDIR,lid)))
#    print(lid+"{0:.2f} for PICKLE loading time statistics".format(T.time()-TT)); TT=T.time()

# Faz rede
# particiona rede
#for i in [0]:
#    labels_.append(labels[dl.downloaded_lists[i]])
data_=[]; count=0
for lid in dl.lists[:4]:
    lid=lid[0]
    ds=dss[count]; np=nps[count];  count+=1
    date1=ds.messages[ds.message_ids[0]][2].isoformat().split("T")[0]
    date2=ds.messages[ds.message_ids[-1]][2].isoformat().split("T")[0]
    N=ds.n_authors
    Ns=[len(i) for i in np.sectorialized_agents__]
    Ns_=[100*len(i)/N for i in np.sectorialized_agents__]
    M_=TOTAL_M-ds.n_messages
    Mh=sum([len(ds.author_messages[author]) for author in np.sectorialized_agents__[2]])
    Mi=sum([len(ds.author_messages[author]) for author in np.sectorialized_agents__[1]])
    Mp=sum([len(ds.author_messages[author]) for author in np.sectorialized_agents__[0]])
    M=[Mh,Mi,Mp][::-1]
    M2=[100*i/ds.n_messages for i in M]
    idsh=[i[0] for j in np.sectorialized_agents__[2] for i in ds.author_messages[j] if ds.messages[i[0]][1]==None]
    idsi=[i[0] for j in np.sectorialized_agents__[1] for i in ds.author_messages[j] if ds.messages[i[0]][1]==None]
    idsp=[i[0] for j in np.sectorialized_agents__[0] for i in ds.author_messages[j] if ds.messages[i[0]][1]==None]
    idsh_=len(idsh)
    idsi_=len(idsi)
    idsp_=len(idsp)
    ids=[idsh_,idsi_,idsp_][::-1]
    Gamma=len([i for i in ds.message_ids if ds.messages[i][1]==None])
    ids_=[100*ii/Gamma for ii in ids]
    print(sum(ids),Gamma)
    data_.append([date1,date2,N,Ns,Ns_,M,M2,Gamma,ids,ids_,M_])
#tstring=g.makeTables(labels_,data_)
#print(tstring)
#TDIR="tables/"
#TDIR="/home/r/repos/stabilityInteraction/tables/"
#FDIR="figs/"
#print(label+"{0:.2f} for making overall table".format(T.time()-TT)); TT=T.time()
#
##g.writeTex(tstring,TDIR+"tab1Geral.tex")
#

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import unicode_literals

import sys
if sys.version_info[:2] == (2, 6):
    import unittest2 as unittest
else:
    import unittest

if sys.version_info[0] == 2:
    unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
    unittest.TestCase.assertRegex = unittest.TestCase.assertRegexpMatches

from maxminddb import Reader, InvalidDatabaseError

class TestReader(unittest.TestCase):

    def test_reader(self):
        for record_size in [24, 28, 32]:
            for ip_version in [4, 6]:
                file_name = ('tests/data/test-data/MaxMind-DB-test-ipv' +
                             str(ip_version) + '-' + str(record_size) +
                             '.mmdb')
                reader = Reader(file_name)

                self._check_metadata(reader, ip_version, record_size)

                if ip_version == 4:
                    self._check_ip_v4(reader, file_name)
                else:
                    self._check_ip_v6(reader, file_name)

    def test_decoder(self):
        reader = Reader('tests/data/test-data/MaxMind-DB-test-decoder.mmdb')
        record = reader.get('::1.1.1.0')

        self.assertEqual(record['array'], [1, 2, 3])
        self.assertEqual(record['boolean'], True)
        self.assertEqual(record['bytes'], bytearray(b'\x00\x00\x00*'))
        self.assertEqual(record['double'], 42.123456)
        self.assertAlmostEqual(record['float'], 1.1)
        self.assertEqual(record['int32'], -268435456)
        self.assertEqual(
            {
                'mapX': {
                    'arrayX': [7, 8, 9],
                    'utf8_stringX': 'hello'
                },
            },
            record['map']
        )

        self.assertEqual(record['uint16'], 100)
        self.assertEqual(record['uint32'], 268435456)
        self.assertEqual(record['uint64'], 1152921504606846976)
        self.assertEqual(record['utf8_string'], 'unicode! ☯ - ♫')

        self.assertEqual(
            1329227995784915872903807060280344576,
            record['uint128']
        )

    def test_broken_database(self):
        reader = Reader('tests/data/test-data/'
                        'GeoIP2-City-Test-Broken-Double-Format.mmdb')
        with self.assertRaisesRegex(InvalidDatabaseError,
                                    "Error while looking up data for "
                                    "2001:220::. The MaxMind DB file's data "
                                    "section contains bad data \(unknown data "
                                    "type or corrupt data\)"
                                    ):
            reader.get('2001:220::')

    def test_ip_validation(self):
        reader = Reader('tests/data/test-data/MaxMind-DB-test-decoder.mmdb')
        self.assertRaisesRegex(ValueError,
                               'The value "not_ip" is not a valid IP '
                               'address.',
                               reader.get, ('not_ip'))

    def test_missing_database(self):
        self.assertRaisesRegex(ValueError,
                               'The file "file-does-not-exist.mmdb" does '
                               'not exist or is not readable.',
                               Reader, ('file-does-not-exist.mmdb'))

    def test_nondatabase(self):
        self.assertRaisesRegex(InvalidDatabaseError,
                               'Error opening database file \(README.rst\). '
                               'Is this a valid MaxMind DB file\?',
                               Reader, ('README.rst'))

    def test_too_many_constructor_args(self):
        self.assertRaises(TypeError, Reader, ('README.md', 1))

    def test_no_constructor_args(self):
        self.assertRaises(TypeError, Reader)

    def test_too_many_get_args(self):
        reader = Reader(
            'tests/data/test-data/MaxMind-DB-test-decoder.mmdb'
        )
        self.assertRaises(TypeError, reader.get, ('1.1.1.1', 'blah'))

    def test_no_get_args(self):
        reader = Reader(
            'tests/data/test-data/MaxMind-DB-test-decoder.mmdb'
        )
        self.assertRaises(TypeError, reader.get)

    def test_metadata_args(self):
        reader = Reader(
            'tests/data/test-data/MaxMind-DB-test-decoder.mmdb'
        )
        self.assertRaises(TypeError, reader.metadata, ('blah'))

    def test_metadata_unknown_attribute(self):
        reader = Reader(
            'tests/data/test-data/MaxMind-DB-test-decoder.mmdb'
        )
        metadata = reader.metadata()
        with self.assertRaisesRegex(AttributeError,
                                    "'maxminddb.Metadata' object has no "
                                    "attribute 'blah'"):
            metadata.blah

    def test_close(self):
        reader = Reader(
            'tests/data/test-data/MaxMind-DB-test-decoder.mmdb'
        )
        reader.close()

    def test_double_close(self):
        reader = Reader(
            'tests/data/test-data/MaxMind-DB-test-decoder.mmdb'
        )
        reader.close()
        self.assertRaisesRegex(IOError,
                               'Attempt to close a closed MaxMind DB.',
                               reader.close)

    def test_closed_get(self):
        reader = Reader(
            'tests/data/test-data/MaxMind-DB-test-decoder.mmdb'
        )
        reader.close()
        self.assertRaisesRegex(IOError,
                               'Attempt to read from a closed MaxMind DB.',
                               reader.get, ('1.1.1.1'))

    def test_closed_metadata(self):
        reader = Reader(
            'tests/data/test-data/MaxMind-DB-test-decoder.mmdb'
        )
        reader.close()
        self.assertRaisesRegex(IOError,
                               'Attempt to read from a closed MaxMind DB.',
                               reader.metadata)

    def _check_metadata(self, reader, ip_version, record_size):
        metadata = reader.metadata()

        self.assertEqual(
            2,
            metadata.binary_format_major_version,
            'major version'
        )
        self.assertEqual(metadata.binary_format_minor_version, 0)
        self.assertEqual(metadata.build_epoch, 1373571901)
        self.assertEqual(metadata.database_type, 'Test')

        self.assertEqual(
            {'en': 'Test Database', 'zh': 'Test Database Chinese'},
            metadata.description
        )
        self.assertEqual(metadata.ip_version, ip_version)
        self.assertEqual(metadata.languages, ['en', 'zh'])
        self.assertGreater(metadata.node_count, 36)

        self.assertEqual(metadata.record_size, record_size)

    def _check_ip_v4(self, reader, file_name):
        for i in range(6):
            address = '1.1.1.' + str(pow(2, i))
            self.assertEqual(
                {'ip': address},
                reader.get(address),
                'found expected data record for '
                + address + ' in ' + file_name
            )

        pairs = {
            '1.1.1.3': '1.1.1.2',
            '1.1.1.5': '1.1.1.4',
            '1.1.1.7': '1.1.1.4',
            '1.1.1.9': '1.1.1.8',
            '1.1.1.15': '1.1.1.8',
            '1.1.1.17': '1.1.1.16',
            '1.1.1.31': '1.1.1.16'
        }
        for key_address, value_address in pairs.items():
            data = {'ip': value_address}

            self.assertEqual(
                data,
                reader.get(key_address),
                'found expected data record for ' + key_address + ' in '
                + file_name
            )

        for ip in ['1.1.1.33', '255.254.253.123']:
            self.assertIsNone(reader.get(ip))

    def _check_ip_v6(self, reader, file_name):
        subnets = ['::1:ffff:ffff', '::2:0:0',
                   '::2:0:40', '::2:0:50', '::2:0:58']

        for address in subnets:
            self.assertEqual(
                {'ip': address},
                reader.get(address),
                'found expected data record for ' + address + ' in '
                + file_name
            )

        pairs = {
            '::2:0:1': '::2:0:0',
            '::2:0:33': '::2:0:0',
            '::2:0:39': '::2:0:0',
            '::2:0:41': '::2:0:40',
            '::2:0:49': '::2:0:40',
            '::2:0:52': '::2:0:50',
            '::2:0:57': '::2:0:50',
            '::2:0:59': '::2:0:58'
        }

        for key_address, value_address in pairs.items():
            self.assertEqual(
                {'ip':  value_address},
                reader.get(key_address),
                'found expected data record for ' + key_address + ' in '
                + file_name
            )

        for ip in ['1.1.1.33', '255.254.253.123', '89fa::']:
            self.assertIsNone(reader.get(ip))
<commit_msg>Revert "Disabled IPv6 in an IPv4 database test."<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import unicode_literals

import sys
if sys.version_info[:2] == (2, 6):
    import unittest2 as unittest
else:
    import unittest

if sys.version_info[0] == 2:
    unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
    unittest.TestCase.assertRegex = unittest.TestCase.assertRegexpMatches

from maxminddb import Reader, InvalidDatabaseError

class TestReader(unittest.TestCase):

    def test_reader(self):
        for record_size in [24, 28, 32]:
            for ip_version in [4, 6]:
                file_name = ('tests/data/test-data/MaxMind-DB-test-ipv' +
                             str(ip_version) + '-' + str(record_size) +
                             '.mmdb')
                reader = Reader(file_name)

                self._check_metadata(reader, ip_version, record_size)

                if ip_version == 4:
                    self._check_ip_v4(reader, file_name)
                else:
                    self._check_ip_v6(reader, file_name)

    def test_decoder(self):
        reader = Reader('tests/data/test-data/MaxMind-DB-test-decoder.mmdb')
        record = reader.get('::1.1.1.0')

        self.assertEqual(record['array'], [1, 2, 3])
        self.assertEqual(record['boolean'], True)
        self.assertEqual(record['bytes'], bytearray(b'\x00\x00\x00*'))
        self.assertEqual(record['double'], 42.123456)
        self.assertAlmostEqual(record['float'], 1.1)
        self.assertEqual(record['int32'], -268435456)
        self.assertEqual(
            {
                'mapX': {
                    'arrayX': [7, 8, 9],
                    'utf8_stringX': 'hello'
                },
            },
            record['map']
        )

        self.assertEqual(record['uint16'], 100)
        self.assertEqual(record['uint32'], 268435456)
        self.assertEqual(record['uint64'], 1152921504606846976)
        self.assertEqual(record['utf8_string'], 'unicode! ☯ - ♫')

        self.assertEqual(
            1329227995784915872903807060280344576,
            record['uint128']
        )

    def test_ipv6_address_in_ipv4_database(self):
        reader = Reader('tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb')
        with self.assertRaisesRegex(ValueError,
                                    'The value "2001::" is not a valid IP '
                                    'address.'):
            reader.get('2001::')

    def test_broken_database(self):
        reader = Reader('tests/data/test-data/'
                        'GeoIP2-City-Test-Broken-Double-Format.mmdb')
        with self.assertRaisesRegex(InvalidDatabaseError,
                                    "Error while looking up data for "
                                    "2001:220::. The MaxMind DB file's data "
                                    "section contains bad data \(unknown data "
                                    "type or corrupt data\)"
                                    ):
            reader.get('2001:220::')

    def test_ip_validation(self):
        reader = Reader('tests/data/test-data/MaxMind-DB-test-decoder.mmdb')
        self.assertRaisesRegex(ValueError,
                               'The value "not_ip" is not a valid IP '
                               'address.',
                               reader.get, ('not_ip'))

    def test_missing_database(self):
        self.assertRaisesRegex(ValueError,
                               'The file "file-does-not-exist.mmdb" does '
                               'not exist or is not readable.',
                               Reader, ('file-does-not-exist.mmdb'))

    def test_nondatabase(self):
        self.assertRaisesRegex(InvalidDatabaseError,
                               'Error opening database file \(README.rst\). '
                               'Is this a valid MaxMind DB file\?',
                               Reader, ('README.rst'))

    def test_too_many_constructor_args(self):
        self.assertRaises(TypeError, Reader, ('README.md', 1))

    def test_no_constructor_args(self):
        self.assertRaises(TypeError, Reader)

    def test_too_many_get_args(self):
        reader = Reader(
            'tests/data/test-data/MaxMind-DB-test-decoder.mmdb'
        )
        self.assertRaises(TypeError, reader.get, ('1.1.1.1', 'blah'))

    def test_no_get_args(self):
        reader = Reader(
            'tests/data/test-data/MaxMind-DB-test-decoder.mmdb'
        )
        self.assertRaises(TypeError, reader.get)

    def test_metadata_args(self):
        reader = Reader(
            'tests/data/test-data/MaxMind-DB-test-decoder.mmdb'
        )
        self.assertRaises(TypeError, reader.metadata, ('blah'))

    def test_metadata_unknown_attribute(self):
        reader = Reader(
            'tests/data/test-data/MaxMind-DB-test-decoder.mmdb'
        )
        metadata = reader.metadata()
        with self.assertRaisesRegex(AttributeError,
                                    "'maxminddb.Metadata' object has no "
                                    "attribute 'blah'"):
            metadata.blah

    def test_close(self):
        reader = Reader(
            'tests/data/test-data/MaxMind-DB-test-decoder.mmdb'
        )
        reader.close()

    def test_double_close(self):
        reader = Reader(
            'tests/data/test-data/MaxMind-DB-test-decoder.mmdb'
        )
        reader.close()
        self.assertRaisesRegex(IOError,
                               'Attempt to close a closed MaxMind DB.',
                               reader.close)

    def test_closed_get(self):
        reader = Reader(
            'tests/data/test-data/MaxMind-DB-test-decoder.mmdb'
        )
        reader.close()
        self.assertRaisesRegex(IOError,
                               'Attempt to read from a closed MaxMind DB.',
                               reader.get, ('1.1.1.1'))

    def test_closed_metadata(self):
        reader = Reader(
            'tests/data/test-data/MaxMind-DB-test-decoder.mmdb'
        )
        reader.close()
        self.assertRaisesRegex(IOError,
                               'Attempt to read from a closed MaxMind DB.',
                               reader.metadata)

    def _check_metadata(self, reader, ip_version, record_size):
        metadata = reader.metadata()

        self.assertEqual(
            2,
            metadata.binary_format_major_version,
            'major version'
        )
        self.assertEqual(metadata.binary_format_minor_version, 0)
        self.assertEqual(metadata.build_epoch, 1373571901)
        self.assertEqual(metadata.database_type, 'Test')

        self.assertEqual(
            {'en': 'Test Database', 'zh': 'Test Database Chinese'},
            metadata.description
        )
        self.assertEqual(metadata.ip_version, ip_version)
        self.assertEqual(metadata.languages, ['en', 'zh'])
        self.assertGreater(metadata.node_count, 36)

        self.assertEqual(metadata.record_size, record_size)

    def _check_ip_v4(self, reader, file_name):
        for i in range(6):
            address = '1.1.1.' + str(pow(2, i))
            self.assertEqual(
                {'ip': address},
                reader.get(address),
                'found expected data record for '
                + address + ' in ' + file_name
            )

        pairs = {
            '1.1.1.3': '1.1.1.2',
            '1.1.1.5': '1.1.1.4',
            '1.1.1.7': '1.1.1.4',
            '1.1.1.9': '1.1.1.8',
            '1.1.1.15': '1.1.1.8',
            '1.1.1.17': '1.1.1.16',
            '1.1.1.31': '1.1.1.16'
        }
        for key_address, value_address in pairs.items():
            data = {'ip': value_address}

            self.assertEqual(
                data,
                reader.get(key_address),
                'found expected data record for ' + key_address + ' in '
                + file_name
            )

        for ip in ['1.1.1.33', '255.254.253.123']:
            self.assertIsNone(reader.get(ip))

    def _check_ip_v6(self, reader, file_name):
        subnets = ['::1:ffff:ffff', '::2:0:0',
                   '::2:0:40', '::2:0:50', '::2:0:58']

        for address in subnets:
            self.assertEqual(
                {'ip': address},
                reader.get(address),
                'found expected data record for ' + address + ' in '
                + file_name
            )

        pairs = {
            '::2:0:1': '::2:0:0',
            '::2:0:33': '::2:0:0',
            '::2:0:39': '::2:0:0',
            '::2:0:41': '::2:0:40',
            '::2:0:49': '::2:0:40',
            '::2:0:52': '::2:0:50',
            '::2:0:57': '::2:0:50',
            '::2:0:59': '::2:0:58'
        }

        for key_address, value_address in pairs.items():
            self.assertEqual(
                {'ip':  value_address},
                reader.get(key_address),
                'found expected data record for ' + key_address + ' in '
                + file_name
            )

        for ip in ['1.1.1.33', '255.254.253.123', '89fa::']:
            self.assertIsNone(reader.get(ip))
<|endoftext|>


<commit_before>import logging
import os
import subprocess
from threading import Timer, Thread

from git import Repo
from git.util import RemoteProgress
from retask import Task

from app.core.messaging import Receiver, Sender
from app.core.service_base import BaseService, BackgroundProcessServiceStart

logger = logging.getLogger(__name__)
SCRIPTS_DIR = "/home/rpi/scripts"
CODE_DIR = "/home/rpi/Code"

class PercentProgressIndicator(RemoteProgress):
    """Specialization that calls back a function with a percentage indicator"""
    def __init__(self, callback):
        self.callback = callback
        super().__init__()

    def update(self, op_code, cur_count, max_count=None, message=""):
        if max_count is not None:
            self.callback(0)
        else:
            self.callback(cur_count / float(max_count))

class Repository(object):
    """Encapsulates a Git Repository"""
    def __init__(self, path):
        """
        :param path: Path to the git repo.
        """
        self.path = path
        self.repo_name = os.path.basename(path)
        self.repo = Repo(self.path)

    def clean_untracked(self):
        """ Does git clean -dfx """
        self.repo.git.clean("-dfx")

    def reset_hard(self):
        """" Does git reset --hard """
        self.repo.git.reset('--hard')

    def needs_pull(self):
        """Does git fetch, and checks whether local tip of master is same as
        that of upstream master."""
        for remote in self.repo.remotes:
            logger.info("Pulling remote for repo: %s", self.repo_name)
            remote.fetch()

        for _ in self.repo.iter_commits('master..origin/master'):
            logger.info("Repo %s needs update", self.repo_name)
            return True
        return False

    def pull(self, progress_observer=None):
        """ Pulls from master. """
        logger.info("Updating repo: %s", self.repo_name)
        self.clean_untracked()
        self.reset_hard()
        origin = self.repo.remotes.origin
        origin.pull(progress=PercentProgressIndicator(progress_observer))

class UpdateScanner(object):
    UPDATE_CHECK_FREQ = 3600

    def __init__(self, notification_queue):
        self.notification_sender = Sender(notification_queue)
        self.timer = Timer(self.UPDATE_CHECK_FREQ, self.check_updates)

    def start(self):
        self.timer.start()

    def stop(self):
        pass

    def check_updates(self):
        res = []
        all_repos = os.listdir(CODE_DIR)

        if not all_repos:
            return []

        logger.info("Checking for updates: %s", str(all_repos))
        for count, path in enumerate(all_repos):
            repo = Repository(os.path.join(CODE_DIR, path))

            if repo.needs_pull():
                res.append(repo)
        return res

    def notify_updates(self):
        self.notification_sender.send(Task({"message": "Update available."}))

class Updater(Receiver):
    def __init__(self, queue_name):
        super().__init__(queue_name)
        self.receiver_thread = Thread(target=self.run)

    def start(self):
        self.start()
        self.receiver_thread.start()

    def on_message(self, msg):
        pass

class UpdaterService(BackgroundProcessServiceStart, BaseService):
    def __init__(self, config):
        self.update_scanner = UpdateScanner("/shell/notifications")
        self.updater = Updater("/app/services/updater")
        super().__init__()

    def get_component_name(self):
        return "updater"

    def on_service_start(self, *args, **kwargs):
        self.notify_start()

    def perform_update(self):
        logger.info("Running ansible")
        run_ansible_path = os.path.join(SCRIPTS_DIR, "run_ansible.sh")
        args = [run_ansible_path]
        with subprocess.Popen(args) as proc:
            proc.wait()
            return proc.returncode

    def on_service_stop(self):
        self.update_scanner.stop()
<commit_msg>Temporary commit.<commit_after>import logging
import os
import subprocess
from threading import Timer, Thread

from git import Repo
from git.util import RemoteProgress
from retask import Task

from app.core.messaging import Receiver, Sender
from app.core.service_base import BaseService, BackgroundProcessServiceStart

logger = logging.getLogger(__name__)
SCRIPTS_DIR = "/home/rpi/scripts"
CODE_DIR = "/home/rpi/Code"

class PercentProgressIndicator(RemoteProgress):
    """Specialization that calls back a function with a percentage indicator"""
    def __init__(self, callback):
        self.callback = callback
        super().__init__()

    def update(self, op_code, cur_count, max_count=None, message=""):
        if max_count is not None:
            self.callback(0)
        else:
            self.callback(cur_count / float(max_count))

class Repository(object):
    """Encapsulates a Git Repository"""
    def __init__(self, path):
        """
        :param path: Path to the git repo.
        """
        self.path = path
        self.repo_name = os.path.basename(path)
        self.repo = Repo(self.path)

    def clean_untracked(self):
        """ Does git clean -dfx """
        self.repo.git.clean("-dfx")

    def reset_hard(self):
        """" Does git reset --hard """
        self.repo.git.reset('--hard')

    def needs_pull(self):
        """Does git fetch, and checks whether local tip of master is same as
        that of upstream master."""
        for remote in self.repo.remotes:
            logger.info("Pulling remote for repo: %s", self.repo_name)
            remote.fetch()

        for _ in self.repo.iter_commits('master..origin/master'):
            logger.info("Repo %s needs update", self.repo_name)
            return True
        return False

    def pull(self, progress_observer=None):
        """ Pulls from master. """
        logger.info("Updating repo: %s", self.repo_name)
        self.clean_untracked()
        self.reset_hard()
        origin = self.repo.remotes.origin
        origin.pull(progress=PercentProgressIndicator(progress_observer))

class UpdateScanner(object):
    UPDATE_CHECK_FREQ = 3600

    def __init__(self, notification_queue):
        self.notification_sender = Sender(notification_queue)
        self.timer = Timer(self.UPDATE_CHECK_FREQ, self.check_updates)
        self.repos_to_update = []

    def start(self):
        self.timer.start()

    def stop(self):
        pass

    def check_updates(self):
        res = []
        all_repos = os.listdir(CODE_DIR)

        if not all_repos:
            return []

        logger.info("Checking for updates: %s", str(all_repos))
        for count, path in enumerate(all_repos):
            repo = Repository(os.path.join(CODE_DIR, path))

            if repo.needs_pull():
                res.append(repo)
        self.repos_to_update = res
        return res

    def notify_updates(self):
        self.notification_sender.send(Task({"message": "Update available."}))

    @property
    def updates(self):
        return self.repos_to_update

class Updater(Receiver):
    def __init__(self, scanner, queue_name):
        super().__init__(queue_name)
        self.scanner = scanner
        self.receiver_thread = Thread(target=self.run)

    def start(self):
        self.start()
        self.receiver_thread.start()

    def on_message(self, msg):
        repos = self.scanner.updates
        if not repos:
            self.update_status("No Updates found.")
            return

        for repo in repos:
            repo.pull(self.send_pull_progress)

        self.run_ansible()

    def run_ansible(self):
        pass

    def send_pull_progress(self, progress):
        task = Task({
            "status": "Updating..",
            "stage": 1,
            "total_stages": 2,
            "progress": progress
        })
        self.status_sender.send(task)

class UpdaterService(BackgroundProcessServiceStart, BaseService):
    def __init__(self, config):
        self.update_scanner = UpdateScanner("/shell/notifications")
        self.updater = Updater("/app/services/updater")
        super().__init__()

    def get_component_name(self):
        return "updater"

    def on_service_start(self, *args, **kwargs):
        self.notify_start()

    def perform_update(self):
        logger.info("Running ansible")
        run_ansible_path = os.path.join(SCRIPTS_DIR, "run_ansible.sh")
        args = [run_ansible_path]
        with subprocess.Popen(args) as proc:
            proc.wait()
            return proc.returncode

    def on_service_stop(self):
        self.update_scanner.stop()
<|endoftext|>


<commit_before>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Custom widgets used for form fields.
"""

__authors__ = [
  '"Pawel Solyga" <pawel.solyga@gmail.com>',
  ]

import copy

from django import forms
from django.forms import util
from django.forms import widgets
from django.utils import html
from django.utils import simplejson
from django.utils import safestring

from soc.logic import dicts

class ReadOnlyInput(forms.widgets.Input):
  """Read only input widget.
  """

  input_type = 'text'

  def render(self, name, value, attrs=None):
    """Render ReadOnlyInput widget as HTML.
    """

    attrs['readonly'] = 'readonly'
    attrs['class'] = 'plaintext'

    return super(ReadOnlyInput, self).render(name, value, attrs)

class PlainTextWidget(forms.widgets.Widget):
  """Read only input widget.
  """

  def render(self, name, value, attrs=None):
    """Render ReadOnlyInput widget as HTML.
    """

    return str(value) if value else ""

class FullTinyMCE(forms.widgets.Textarea):
  """TinyMCE widget.

  Requires to include tiny_mce_src.js in your template. Widget can be
  customized by overwriting or adding extra options to mce_settings
  dictionary

  You can set TinyMCE widget for particular form field using code below:
    class ExampleForm(helper.forms.BaseForm):
      content = forms.fields.CharField(widget=helper.widgets.TinyMCE())

  You can include tiny_mce_src.js in your template using:
    {% block scripts %}

    {% endblock %}
  """

  features1 = ("bold,italic,underline,strikethrough,|,"
               "forecolor,backcolor,blockquote,|"
              ",justifyleft,justifycenter,justifyright,justifyfull,|,"
              "fontselect, fontsizeselect,formatselect")

  features2 = ("newdocument,|,bullist,numlist,|,outdent,indent,|,undo,redo,|"
      ",link,unlink,anchor,image,cleanup,help,code,hr,removeformat,visualaid,|,"
      "sub,sup,|,charmap,"
      "")

  DEF_MCE_SETTINGS = {
      'mode': "exact",
      'theme': "advanced",
      'theme_advanced_buttons1': features1,
      'theme_advanced_buttons2': features2,
      'theme_advanced_buttons3': '',
      'theme_advanced_toolbar_location': "top",
      'theme_advanced_toolbar_align': "left",
      'relative_urls': 0,
      'remove_script_host': 0,
      }

  TINY_MCE_HTML_FMT = u'''\
<textarea %(attrs)s>%(value)s</textarea>
'''

  def __init__(self, mce_settings=None, *args, **kwargs):
    """Initialize TinyMCE widget with default or customized settings.

    Args:
      mce_settings: dict with TinyMCE widget settings
      *args, **kwargs:  passed through to parent __init__() constructor
    """

    super(forms.widgets.Textarea, self).__init__(*args, **kwargs)
    self.mce_settings = self.DEF_MCE_SETTINGS

  def render(self, name, value, attrs=None):
    """Render TinyMCE widget as HTML.
    """
    if value is None:
      value = ''
    value = util.smart_unicode(value)
    final_attrs = self.build_attrs(attrs, name=name)

    self.mce_settings['elements'] = "id_%s" % name

    # convert mce_settings from dict to JSON
    mce_json = simplejson.JSONEncoder().encode(self.mce_settings)

    return safestring.mark_safe(self.TINY_MCE_HTML_FMT %
        {'attrs': widgets.flatatt(final_attrs),
         'value': html.escape(value),
         'settings_json':  mce_json})

class TinyMCE(FullTinyMCE):
  """Regular version of TinyMce
  """

  def __init__(self, *args, **kwargs):
    """
    """

    super(TinyMCE, self).__init__(*args, **kwargs)
    keys = ['mode', 'theme', 'theme_advanced_toolbar_location',
            'theme_advanced_toolbar_align', 'relative_urls',
            'remove_script_host']
    self.mce_settings = dicts.filter(self.mce_settings, keys)

class ReferenceField(forms.CharField):
  """Widget for selecting a reference to an Entity.
  """

  def __init__(self, reference_url, filter=None, filter_fields=None,
               *args, **kwargs):
    """Initializes the widget with the specified url and filter.
    """

    self.rf = {}
    self.rf['reference_url'] = reference_url
    self.rf['filter'] = filter if filter else []
    self.rf['filter_fields'] = filter_fields if filter_fields else {}
    super(ReferenceField, self).__init__(*args, **kwargs)

class AgreementField(widgets.Widget):
  """Widget for selecting a reference to an Entity.
  """

  HTML_CODE = """

      %(text)s

  %(url)s
  """

  def __init__(self, *args, **kwargs):
    self.text = "No Agreement Text Specified"
    self.url = ""
    super(AgreementField, self).__init__(*args, **kwargs)

  def render(self, name, value, attrs=None):
    """
    """

    url = 'Full Text' % self.url if self.url else ""
    value = self.text.replace('\n', '
')
    result = self.HTML_CODE % {'url': url, 'text': value}
    return result
<commit_msg>Removed formatting from TinyMCE that use the style tag<commit_after>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Custom widgets used for form fields.
"""

__authors__ = [
  '"Pawel Solyga" <pawel.solyga@gmail.com>',
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  ]

import copy

from django import forms
from django.forms import util
from django.forms import widgets
from django.utils import html
from django.utils import simplejson
from django.utils import safestring

from soc.logic import dicts

class ReadOnlyInput(forms.widgets.Input):
  """Read only input widget.
  """

  input_type = 'text'

  def render(self, name, value, attrs=None):
    """Render ReadOnlyInput widget as HTML.
    """

    attrs['readonly'] = 'readonly'
    attrs['class'] = 'plaintext'

    return super(ReadOnlyInput, self).render(name, value, attrs)

class PlainTextWidget(forms.widgets.Widget):
  """Read only input widget.
  """

  def render(self, name, value, attrs=None):
    """Render ReadOnlyInput widget as HTML.
    """

    return str(value) if value else ""

class FullTinyMCE(forms.widgets.Textarea):
  """TinyMCE widget.

  Requires to include tiny_mce_src.js in your template. Widget can be
  customized by overwriting or adding extra options to mce_settings
  dictionary

  You can set TinyMCE widget for particular form field using code below:
    class ExampleForm(helper.forms.BaseForm):
      content = forms.fields.CharField(widget=helper.widgets.TinyMCE())

  You can include tiny_mce_src.js in your template using:
    {% block scripts %}

    {% endblock %}
  """

  features1 = ("bold,italic,underline,strikethrough,blockquote,|,"
              ",justifyleft,justifycenter,justifyright,justifyfull,|,"
              "formatselect")

  features2 = ("newdocument,|,bullist,numlist,|,outdent,indent,|,undo,redo,|"
      ",link,unlink,anchor,image,cleanup,help,code,hr,removeformat,visualaid,|,"
      "sub,sup,|,charmap,"
      "")

  DEF_MCE_SETTINGS = {
      'mode': "exact",
      'theme': "advanced",
      'theme_advanced_buttons1': features1,
      'theme_advanced_buttons2': features2,
      'theme_advanced_buttons3': '',
      'theme_advanced_toolbar_location': "top",
      'theme_advanced_toolbar_align': "left",
      'relative_urls': 0,
      'remove_script_host': 0,
      }

  TINY_MCE_HTML_FMT = u'''\
<textarea %(attrs)s>%(value)s</textarea>
'''

  def __init__(self, mce_settings=None, *args, **kwargs):
    """Initialize TinyMCE widget with default or customized settings.

    Args:
      mce_settings: dict with TinyMCE widget settings
      *args, **kwargs:  passed through to parent __init__() constructor
    """

    super(forms.widgets.Textarea, self).__init__(*args, **kwargs)
    self.mce_settings = self.DEF_MCE_SETTINGS

  def render(self, name, value, attrs=None):
    """Render TinyMCE widget as HTML.
    """
    if value is None:
      value = ''
    value = util.smart_unicode(value)
    final_attrs = self.build_attrs(attrs, name=name)

    self.mce_settings['elements'] = "id_%s" % name

    # convert mce_settings from dict to JSON
    mce_json = simplejson.JSONEncoder().encode(self.mce_settings)

    return safestring.mark_safe(self.TINY_MCE_HTML_FMT %
        {'attrs': widgets.flatatt(final_attrs),
         'value': html.escape(value),
         'settings_json':  mce_json})

class TinyMCE(FullTinyMCE):
  """Regular version of TinyMce
  """

  def __init__(self, *args, **kwargs):
    """
    """

    super(TinyMCE, self).__init__(*args, **kwargs)
    keys = ['mode', 'theme', 'theme_advanced_toolbar_location',
            'theme_advanced_toolbar_align', 'relative_urls',
            'remove_script_host']
    self.mce_settings = dicts.filter(self.mce_settings, keys)

class ReferenceField(forms.CharField):
  """Widget for selecting a reference to an Entity.
  """

  def __init__(self, reference_url, filter=None, filter_fields=None,
               *args, **kwargs):
    """Initializes the widget with the specified url and filter.
    """

    self.rf = {}
    self.rf['reference_url'] = reference_url
    self.rf['filter'] = filter if filter else []
    self.rf['filter_fields'] = filter_fields if filter_fields else {}
    super(ReferenceField, self).__init__(*args, **kwargs)

class AgreementField(widgets.Widget):
  """Widget for selecting a reference to an Entity.
  """

  HTML_CODE = """

      %(text)s

  %(url)s
  """

  def __init__(self, *args, **kwargs):
    self.text = "No Agreement Text Specified"
    self.url = ""
    super(AgreementField, self).__init__(*args, **kwargs)

  def render(self, name, value, attrs=None):
    """
    """

    url = 'Full Text' % self.url if self.url else ""
    value = self.text.replace('\n', '
')
    result = self.HTML_CODE % {'url': url, 'text': value}
    return result
<|endoftext|>