<commit_before>from rest_framework import serializers
from gwells.models.ProvinceStateCode import ProvinceStateCode
from registries.models import (
    Organization,
    ContactInfo,
    Person,
    Register,
    RegistriesApplication,
    RegistriesApplicationStatus,
    RegistriesStatusCode,
    ActivityCode,
    SubactivityCode,
    Qualification
)

class AuditModelSerializer(serializers.ModelSerializer):
    """
    Serializes AuditModel fields.
    Can be inherited into serializers for models that inherit from AuditModel
    """
    create_user = serializers.ReadOnlyField()
    create_date = serializers.ReadOnlyField()
    update_user = serializers.ReadOnlyField()
    update_date = serializers.ReadOnlyField()

class QualificationSerializer(serializers.ModelSerializer):
    """
    Serializes Qualification model
    Qualification records form a related set of a SubactivityCode object
    """

    class Meta:
        model = Qualification
        fields = (
            'well_class',
            'description'
        )

class ContactInfoSerializer(AuditModelSerializer):
    """
    Serializes ContactInfo model fields.
    """
    person_name = serializers.StringRelatedField(source="person")
    organization_name = serializers.StringRelatedField(source="org")
    street_address = serializers.StringRelatedField(source="org.street_address")
    city = serializers.StringRelatedField(source="org.city")
    postal_code = serializers.StringRelatedField(source="org.postal_code")
    website_url = serializers.StringRelatedField(source="org.website_url")

    class Meta:
        model = ContactInfo
        fields = (
            'contact_at_guid',
            'organization_name',
            'street_address',
            'city',
            'postal_code',
            'person_name',
            'person',
            'org',
            'contact_tel',
            'contact_email',
            'website_url'
        )

class SubactivitySerializer(serializers.ModelSerializer):
    """
    Serializes SubactivityCode model
    SubactivityCode records form a related set of an ActivityCode object
    """

    class Meta:
        model = SubactivityCode
        fields = (
            'registries_subactivity_code',
            'description',
        )

class ApplicationStatusSerializer(serializers.ModelSerializer):
    """
    Serializes RegistriesApplicationStatus for admin users
    ApplicationStatus objects form a related set for an Application object.
    """

    class Meta:
        model = RegistriesApplicationStatus
        fields = (
            'status',
            'status',
            'notified_date',
            'effective_date',
            'expired_date',
        )

class ApplicationListSerializer(AuditModelSerializer):
    """
    Serializes RegistryApplication model fields for anonymous users
    """

    qualifications = QualificationSerializer(many=True, read_only=True)
    status_set = ApplicationStatusSerializer(many=True, read_only=True)
    subactivity = SubactivitySerializer()

    class Meta:
        model = RegistriesApplication
        fields = (
            'application_guid',
            'file_no',
            'reason_denied',
            'qualifications',
            'status_set',
            'subactivity'
        )

class RegistrationsListSerializer(serializers.ModelSerializer):
    """
    Serializes Register model
    Register items form a related set of an Application object
    """
    status = serializers.ReadOnlyField(source='status.description')
    activity = serializers.ReadOnlyField(source='registries_activity.description')
    applications = ApplicationListSerializer(many=True, read_only=True)

    class Meta:
        model = Register
        fields = (
            'activity',
            'status',
            'registration_no',
            'applications'
        )

class PersonSerializer(AuditModelSerializer):
    """
    Serializes the Person model (public/anonymous user fields)
    """

    registrations = RegistrationsListSerializer(many=True, read_only=True)

    class Meta:
        model = Person
        fields = (
            'person_guid',
            'first_name',
            'surname',
            'registrations',
        )

class OrganizationSerializer(AuditModelSerializer):
    """
    Serializes Organization model fields (public fields list)
    """

    person_set = PersonSerializer(many=True, read_only=True)

    class Meta:
        model = Organization
        fields = (
            'org_guid',
            'name',
            'street_address',
            'city',
            'province_state',
            'postal_code',
            'main_tel',
            'fax_tel',
            'website_url',
            'certificate_authority',
            'person_set',
        )

class OrganizationAdminSerializer(AuditModelSerializer):
    """
    Serializes Organization model fields (admin fields list)
    """

    person_set = PersonSerializer(many=True, read_only=True)

    class Meta:
        model = Organization
        fields = (
            'create_user',
            'create_date',
            'update_user',
            'update_date',
            'org_guid',
            'name',
            'street_address',
            'city',
            'province_state',
            'postal_code',
            'main_tel',
            'fax_tel',
            'website_url',
            'person_set',
        )

class ActivitySerializer(serializers.ModelSerializer):
    """
    Serializes ActivityCode model
    A Register record has a foreign key relationship to an ActivityCode object
    """

    class Meta:
        model = ActivityCode
        fields = (
            'registries_activity_code',
            'description',
        )

class ApplicationAdminSerializer(AuditModelSerializer):
    """
    Serializes RegistryApplication model fields for admin users
    """

    registriesapplicationstatus_set = ApplicationStatusSerializer(many=True, read_only=True)

    class Meta:
        model = RegistriesApplication
        fields = (
            'application_guid',
            'registration',
            'file_no',
            'over19_ind',
            'registrar_notes',
            'reason_denied',
            'classificationappliedfor_set',
            'registriesapplicationstatus_set'
        )

class RegistrationAdminSerializer(serializers.ModelSerializer):
    """
    Serializes Register model for admin users
    """
    status = serializers.PrimaryKeyRelatedField(queryset=RegistriesStatusCode.objects.all())
    register_removal_reason = serializers.StringRelatedField(read_only=True)
    applications = ApplicationAdminSerializer(many=True, read_only=True)

    class Meta:
        model = Register
        fields = (
            'register_guid',
            'person',
            'registries_activity',
            'status',
            'registration_no',
            'registration_date',
            'register_removal_reason',
            'register_removal_date',
            'applications'
        )

class CityListSerializer(serializers.ModelSerializer):
    """
    Serializes city and province fields for list of cities with qualified drillers
    The queryset is limited to one record per unique city in the Person table
    The OrganizationSerializer fields are used to fill in city, province data
    """

    organization = OrganizationSerializer()

    class Meta:
        model = Person
        fields = (
            'organization',
        )

    def to_representation(self, obj):
        """
        Flattens City list response to make an array of { city: '', province_state: '' } objects
        """
        repr = super().to_representation(obj)

        # remove and store nested objects
        org = repr.pop('organization')

        # specify fields from OrganizationSerializer
        company_fields = (
            'city',
            'province_state',
        )

        for field in company_fields:
            if org and org[field]:
                repr[field] = org[field]
            else:
                repr[field] = None

        return repr

class OrganizationListSerializer(AuditModelSerializer):
    """
    Serializes Organization model fields for "list" view.
    """

    person_count = serializers.SerializerMethodField()

    class Meta:
        model = Organization
        fields = (
            'org_guid',
            'name',
            'street_address',
            'city',
            'province_state',
            'postal_code',
            'main_tel',
            'person_count'
        )

    def get_person_count(self, obj):
        return obj.person_set.count()

class PersonListSerializer(AuditModelSerializer):
    """
    Serializes the Person model for a list view (fewer fields than detail view)
    """
    registrations = RegistrationsListSerializer(many=True, read_only=True)
    organization = serializers.PrimaryKeyRelatedField(queryset=Organization.objects.all(), required=False)

    class Meta:
        model = Person
        fields = (
            'person_guid',
            'first_name',
            'surname',
            'organization',
            'registrations',
        )

class PersonAdminSerializer(AuditModelSerializer):
    """
    Serializes the Person model (admin user fields)
    """

    organization = serializers.PrimaryKeyRelatedField(queryset=Organization.objects.all(), required=False)
    registrations = RegistrationAdminSerializer(many=True, read_only=True)

    class Meta:
        model = Person
        fields = (
            'person_guid',
            'first_name',
            'surname',
            'organization',
            'registrations',
            'create_user',
            'create_date',
            'update_user',
            'update_date',
        )
<commit_msg>fix registration and application fields<commit_after>from rest_framework import serializers
from rest_framework import relations
from gwells.models.ProvinceStateCode import ProvinceStateCode
from registries.models import (
    Organization,
    ContactInfo,
    Person,
    Register,
    RegistriesApplication,
    RegistriesApplicationStatus,
    RegistriesStatusCode,
    ActivityCode,
    SubactivityCode,
    Qualification
)

class AuditModelSerializer(serializers.ModelSerializer):
    """
    Serializes AuditModel fields.
    Can be inherited into serializers for models that inherit from AuditModel
    """
    create_user = serializers.ReadOnlyField()
    create_date = serializers.ReadOnlyField()
    update_user = serializers.ReadOnlyField()
    update_date = serializers.ReadOnlyField()

class QualificationSerializer(serializers.ModelSerializer):
    """
    Serializes Qualification model
    Qualification records form a related set of a SubactivityCode object
    """

    class Meta:
        model = Qualification
        fields = (
            'registries_well_qualification_guid',
            'well_class',
            'subactivity',
            'effective_date',
            'expired_date',
        )

class ContactInfoSerializer(AuditModelSerializer):
    """
    Serializes ContactInfo model fields.
    """
    person_name = serializers.StringRelatedField(source="person")
    organization_name = serializers.StringRelatedField(source="org")
    street_address = serializers.StringRelatedField(source="org.street_address")
    city = serializers.StringRelatedField(source="org.city")
    postal_code = serializers.StringRelatedField(source="org.postal_code")
    website_url = serializers.StringRelatedField(source="org.website_url")

    class Meta:
        model = ContactInfo
        fields = (
            'contact_at_guid',
            'organization_name',
            'street_address',
            'city',
            'postal_code',
            'person_name',
            'person',
            'org',
            'contact_tel',
            'contact_email',
            'website_url'
        )

class SubactivitySerializer(serializers.ModelSerializer):
    """
    Serializes SubactivityCode model
    SubactivityCode records form a related set of an ActivityCode object
    """

    class Meta:
        model = SubactivityCode
        fields = (
            'registries_subactivity_code',
            'description',
        )

class ApplicationStatusSerializer(serializers.ModelSerializer):
    """
    Serializes RegistriesApplicationStatus for admin users
    ApplicationStatus objects form a related set for an Application object.
    """

    class Meta:
        model = RegistriesApplicationStatus
        fields = (
            'status',
            'status',
            'notified_date',
            'effective_date',
            'expired_date',
        )

class ApplicationListSerializer(AuditModelSerializer):
    """
    Serializes RegistryApplication model fields for anonymous users
    """

    qualifications = QualificationSerializer(many=True, read_only=True)
    status_set = ApplicationStatusSerializer(many=True, read_only=True)
    subactivity = SubactivitySerializer()

    class Meta:
        model = RegistriesApplication
        fields = (
            'application_guid',
            'file_no',
            'reason_denied',
            'qualifications',
            'status_set',
            'subactivity'
        )

class RegistrationsListSerializer(serializers.ModelSerializer):
    """
    Serializes Register model
    Register items form a related set of an Application object
    """
    status = serializers.ReadOnlyField(source='status.description')
    activity = serializers.ReadOnlyField(source='registries_activity.description')
    applications = ApplicationListSerializer(many=True, read_only=True)

    class Meta:
        model = Register
        fields = (
            'activity',
            'status',
            'registration_no',
            'applications'
        )

class PersonBasicSerializer(serializers.ModelSerializer):
    """
    Serializes Person model with basic fields only
    """

    class Meta:
        model = Person
        fields = ('person_guid', 'name')

class PersonSerializer(AuditModelSerializer):
    """
    Serializes the Person model (public/anonymous user fields)
    """

    registrations = RegistrationsListSerializer(many=True, read_only=True)

    class Meta:
        model = Person
        fields = (
            'person_guid',
            'first_name',
            'surname',
            'registrations',
        )

class OrganizationSerializer(AuditModelSerializer):
    """
    Serializes Organization model fields (public fields list)
    """

    person_set = PersonSerializer(many=True, read_only=True)

    class Meta:
        model = Organization
        fields = (
            'org_guid',
            'name',
            'street_address',
            'city',
            'province_state',
            'postal_code',
            'main_tel',
            'fax_tel',
            'website_url',
            'certificate_authority',
            'person_set',
        )

class OrganizationAdminSerializer(AuditModelSerializer):
    """
    Serializes Organization model fields (admin fields list)
    """

    person_set = PersonSerializer(many=True, read_only=True)

    class Meta:
        model = Organization
        fields = (
            'create_user',
            'create_date',
            'update_user',
            'update_date',
            'org_guid',
            'name',
            'street_address',
            'city',
            'province_state',
            'postal_code',
            'main_tel',
            'fax_tel',
            'website_url',
            'person_set',
        )

class ActivitySerializer(serializers.ModelSerializer):
    """
    Serializes ActivityCode model
    A Register record has a foreign key relationship to an ActivityCode object
    """

    class Meta:
        model = ActivityCode
        fields = (
            'registries_activity_code',
            'description',
        )

class ApplicationAdminSerializer(AuditModelSerializer):
    """
    Serializes RegistryApplication model fields for admin users
    """

    status_set = ApplicationStatusSerializer(many=True, read_only=True)

    class Meta:
        model = RegistriesApplication
        fields = (
            'application_guid',
            'registration',
            'file_no',
            'over19_ind',
            'registrar_notes',
            'reason_denied',
            'subactivity',
            'status_set'
        )

class RegistrationAdminSerializer(AuditModelSerializer):
    """
    Serializes Register model for admin users
    """
    status = serializers.PrimaryKeyRelatedField(queryset=RegistriesStatusCode.objects.all())
    register_removal_reason = serializers.StringRelatedField(read_only=True)
    applications = ApplicationAdminSerializer(many=True, read_only=True)
    person_name = serializers.StringRelatedField(source="person.name", read_only=True)

    class Meta:
        model = Register
        fields = (
            'create_user',
            'create_date',
            'update_user',
            'update_date',
            'register_guid',
            'person',
            'person_name',
            'registries_activity',
            'status',
            'registration_no',
            'registration_date',
            'register_removal_reason',
            'register_removal_date',
            'applications'
        )

class CityListSerializer(serializers.ModelSerializer):
    """
    Serializes city and province fields for list of cities with qualified drillers
    The queryset is limited to one record per unique city in the Person table
    The OrganizationSerializer fields are used to fill in city, province data
    """

    organization = OrganizationSerializer()

    class Meta:
        model = Person
        fields = (
            'organization',
        )

    def to_representation(self, obj):
        """
        Flattens City list response to make an array of { city: '', province_state: '' } objects
        """
        repr = super().to_representation(obj)

        # remove and store nested objects
        org = repr.pop('organization')

        # specify fields from OrganizationSerializer
        company_fields = (
            'city',
            'province_state',
        )

        for field in company_fields:
            if org and org[field]:
                repr[field] = org[field]
            else:
                repr[field] = None

        return repr

class OrganizationListSerializer(AuditModelSerializer):
    """
    Serializes Organization model fields for "list" view.
    """

    person_count = serializers.SerializerMethodField()

    class Meta:
        model = Organization
        fields = (
            'org_guid',
            'name',
            'street_address',
            'city',
            'province_state',
            'postal_code',
            'main_tel',
            'person_count'
        )

    def get_person_count(self, obj):
        return obj.person_set.count()

class PersonListSerializer(AuditModelSerializer):
    """
    Serializes the Person model for a list view (fewer fields than detail view)
    """
    registrations = RegistrationsListSerializer(many=True, read_only=True)
    organization = serializers.PrimaryKeyRelatedField(queryset=Organization.objects.all(), required=False)

    class Meta:
        model = Person
        fields = (
            'person_guid',
            'first_name',
            'surname',
            'organization',
            'registrations',
        )

class PersonAdminSerializer(AuditModelSerializer):
    """
    Serializes the Person model (admin user fields)
    """

    # organization = serializers.PrimaryKeyRelatedField(queryset=Organization.objects.all(), required=False)
    registrations = RegistrationAdminSerializer(many=True, read_only=True)

    class Meta:
        model = Person
        fields = (
            'person_guid',
            'first_name',
            'surname',
            'organization',
            'registrations',
            'create_user',
            'create_date',
            'update_user',
            'update_date',
        )
<|endoftext|>


<commit_before>from PIL import Image
import tkinter as tk
from tkinter import filedialog

root = tk.Tk()
root.withdraw()
file_path = filedialog.askopenfilename()

im = Image.open(file_path)

im = im.resize((512,512), Image.ANTIALIAS)
im = im.convert('1')

im.save('converted.bmp')
<commit_msg>With serial<commit_after>from PIL import Image
import tkinter as tk
from tkinter import filedialog

import serial
import time

ser = serial.Serial('/dev/ttyUSB0', 57600)
ser.write(b"\xF6")
rep=ser.read(2);

print('Replied')
print(rep)

if rep == 'ep' :
	root = tk.Tk()
	root.withdraw()
	file_path = filedialog.askopenfilename()

	im = Image.open(file_path)

	im = im.resize((512,512), Image.ANTIALIAS)
	im = im.convert('1')

	im.save('converted.bmp')

	ser.write(b"\xFE\xFE\xFE\xFE\xFE\xFE\xFE\xFE")
	time.sleep(3)
	ser.write(open("converted.bmp","rb").read())
	time.sleep(3)

	ser.write(b"\xF3")
	time.sleep(5)

	ser.write(b"\x10")
	time.sleep(1)

	ser.write(b"\xF4")

<|endoftext|>


<commit_before>#!/usr/bin/python

import re
import sys
import os
import tempfile
try:
    import yaml
except ImportError:
    print('Missing yaml dependency. Please install with:')
    print('pip install pyyaml')
    sys.exit(1)

CONFIG_FILE = './bankaccounts.yml'
LEDGER_FILE = './csv2journal.txt'
RE_LEDGER_FST_LINE_TRANSACTION = re.compile('^[0-9]+')
SHOW_DIFF = False

def usage():
    s = """
Usage:
    {} <account> <infile>
""".format(__file__)
    print(s)
    sys.exit(1)

def check_env():
    files = [CONFIG_FILE, LEDGER_FILE]
    for f in files:
        if (not os.path.exists(f)):
            print('Cannot find expected file {}'.format(f))
            sys.exit(1)

def ignore_transactions(lines, patterns):
    """
    Returns a) a new list of lines exluding some of the original lines and b) a
    list of excluded lines.
    """
    def match(line, patterns):
        if (patterns is not None):
            for pattern in patterns:
                if re.search(pattern, line):
                    return True
        return False

    new_lines = []
    ignored_lines = []
    for line in lines:
        if (not match(line, patterns)):
            new_lines.append(line)
        else:
            ignored_lines.append(line)
    return new_lines, ignored_lines

def modify_transactions(lines, mods):
    """
    Returns a) a new list of modified lines and b) a list of tuples of
    (original, modified) lines.
    """
    new_lines = []
    modified_lines = []
    for line in lines:
        raw = line
        if (mods is not None):
            for mod in mods:
                line = re.sub(mod[0], mod[1], line)
        new_lines.append(line)
        modified_lines.append((raw, line))
    return new_lines, modified_lines

def main(argv=None):
    check_env()
    if (argv is None or len(argv) < 3):
        usage()
    account = argv[1]
    account_colon = account.replace("___", ":")
    account_underscore = account_colon.replace(":", "___")
    csv_filename = argv[2]
    f = file(CONFIG_FILE, 'r')
    cfg = yaml.load(f)
    if account_underscore not in cfg:
        print("Cannot find accout {} in config file ({}).".
              format(account_colon, CONFIG_FILE))
        print("Did you define the correct account?")
        print("Did you use 3 underscores instead of each colon?")
        sys.exit(1)
    else:
        # Get account config.
        acfg = cfg[account_underscore]

        # Modify CSV file (delete/modify lines, add header, ...).
        _, tmp_csv_filename = tempfile.mkstemp()
        with open(csv_filename) as csv_fh:
            lines = csv_fh.readlines()
            lines = [re.sub(r'[^\x00-\x7F]+', '_', l) for l in lines]
            lines = lines[int(acfg['ignored_lines']):]
            lines, ignored_lines = \
                ignore_transactions(lines, acfg['ignored_transactions'])
            lines, modified_lines = \
                modify_transactions(lines, acfg['modify_transactions'])
            with open(tmp_csv_filename, "w") as output_fh:
                output_fh.write(acfg['convert_header'] + '\n')
                for line in lines:
                    # print(line)
                    output_fh.write(line)

        # Use Ledger to convert the modified CSV file.
        cmd = 'ledger -f {} convert {}'.format(LEDGER_FILE, tmp_csv_filename)
        cmd += ' --input-date-format "{}"'.format(acfg['date_format'])
        cmd += ' --account {}'.format(account_colon)
        cmd += ' --generated'  # pin automated transactions
        cmd += ' {}'.format(acfg['ledger_args'])
        cmd += ' | sed -e "s/\(^\s\+.*\s\+\)\([-0-9\.]\+\)$/\\1{}\\2/g"'.\
            format(acfg['currency'])
        try:
            cmd += ' | sed -e "s/Expenses:Unknown/{}/g"'.\
                format(acfg['expenses_unknown'])
        except:
            pass
        fd, tmp_journal_filename = tempfile.mkstemp()
        os.close(fd)
        cmd += ' > {}'.format(tmp_journal_filename)
        os.system(cmd)

        # For every trannsaction, add the correspinding CSV file line to the
        # generated journal file.
        new_lines = []
        with open(tmp_journal_filename, 'a+') as fh:
            i = 0
            for line in fh.readlines():
                new_lines.append(line)
                if (RE_LEDGER_FST_LINE_TRANSACTION.match(line)):
                    # Assuming that the transactions in the journal file have
                    # the same order as the transactions in the csv file, we
                    # can match modified csv lines to the journal's
                    # transactions:
                    new_lines.append('    ; CSV data:\n')
                    new_lines.append('    ; from : {}'.
                                     format(modified_lines[i][1]))
                    new_lines.append('    ; (raw): {}'.
                                     format(modified_lines[i][0]))
                    i += 1
        with open(tmp_journal_filename, 'w') as fh:
            fh.write(''.join(new_lines))

        # Append the list of ignored transactions to the generated journal
        # file.
        if(len(ignored_lines) > 0):
            with open(tmp_journal_filename, 'a+') as fh:
                fh.write('\n\n')
                fh.write('; Attention: The following lines from ')
                fh.write('{} were ignored:\n'.format(csv_filename))
                for line in ignored_lines:
                    fh.write('; {}\n'.format(line.strip()))

        # Print to stdout for later use.
        os.system('cat {}'.format(tmp_journal_filename))

        # Cleanup.
        os.remove(tmp_csv_filename)
        os.remove(tmp_journal_filename)

if __name__ == '__main__':
    main(sys.argv)
<commit_msg>Make ignored/modified transactions optional<commit_after>#!/usr/bin/python

import re
import sys
import os
import tempfile
try:
    import yaml
except ImportError:
    print('Missing yaml dependency. Please install with:')
    print('pip install pyyaml')
    sys.exit(1)

CONFIG_FILE = './bankaccounts.yml'
LEDGER_FILE = './csv2journal.txt'
RE_LEDGER_FST_LINE_TRANSACTION = re.compile('^[0-9]+')
SHOW_DIFF = False

def usage():
    s = """
Usage:
    {} <account> <infile>
""".format(__file__)
    print(s)
    sys.exit(1)

def check_env():
    files = [CONFIG_FILE, LEDGER_FILE]
    for f in files:
        if (not os.path.exists(f)):
            print('Cannot find expected file {}'.format(f))
            sys.exit(1)

def ignore_transactions(lines, patterns):
    """
    Returns a) a new list of lines exluding some of the original lines and b) a
    list of excluded lines.
    """
    def match(line, patterns):
        if (patterns is not None):
            for pattern in patterns:
                if re.search(pattern, line):
                    return True
        return False

    new_lines = []
    ignored_lines = []
    for line in lines:
        if (not match(line, patterns)):
            new_lines.append(line)
        else:
            ignored_lines.append(line)
    return new_lines, ignored_lines

def modify_transactions(lines, mods):
    """
    Returns a) a new list of modified lines and b) a list of tuples of
    (original, modified) lines.
    """
    new_lines = []
    modified_lines = []
    for line in lines:
        raw = line
        if (mods is not None):
            for mod in mods:
                line = re.sub(mod[0], mod[1], line)
        new_lines.append(line)
        modified_lines.append((raw, line))
    return new_lines, modified_lines

def main(argv=None):
    check_env()
    if (argv is None or len(argv) < 3):
        usage()
    account = argv[1]
    account_colon = account.replace("___", ":")
    account_underscore = account_colon.replace(":", "___")
    csv_filename = argv[2]
    f = file(CONFIG_FILE, 'r')
    cfg = yaml.load(f)
    if account_underscore not in cfg:
        print("Cannot find accout {} in config file ({}).".
              format(account_colon, CONFIG_FILE))
        print("Did you define the correct account?")
        print("Did you use 3 underscores instead of each colon?")
        sys.exit(1)
    else:
        # Get account config.
        acfg = cfg[account_underscore]

        # Modify CSV file (delete/modify lines, add header, ...).
        _, tmp_csv_filename = tempfile.mkstemp()
        with open(csv_filename) as csv_fh:
            lines = csv_fh.readlines()
            lines = [re.sub(r'[^\x00-\x7F]+', '_', l) for l in lines]
            lines = lines[int(acfg['ignored_header_lines']):]

            # Nothing is ignored by default.
            ignored_lines = []
            if ('ignore_transactions' in acfg):
                lines, ignored_lines = \
                    ignore_transactions(lines, acfg['ignore_transactions'])

            # Nothing is modfied by default.
            modified_lines = [(line, line) for line in lines]
            if ('modify_transactions' in acfg):
                lines, modified_lines = \
                    modify_transactions(lines, acfg['modify_transactions'])

            with open(tmp_csv_filename, "w") as output_fh:
                output_fh.write(acfg['convert_header'] + '\n')
                for line in lines:
                    # print(line)
                    output_fh.write(line)

        # Use Ledger to convert the modified CSV file.
        cmd = 'ledger -f {} convert {}'.format(LEDGER_FILE, tmp_csv_filename)
        cmd += ' --input-date-format "{}"'.format(acfg['date_format'])
        cmd += ' --account {}'.format(account_colon)
        cmd += ' --generated'  # pin automated transactions
        cmd += ' {}'.format(acfg['ledger_args'])
        cmd += ' | sed -e "s/\(^\s\+.*\s\+\)\([-0-9\.]\+\)$/\\1{}\\2/g"'.\
            format(acfg['currency'])
        try:
            cmd += ' | sed -e "s/Expenses:Unknown/{}/g"'.\
                format(acfg['expenses_unknown'])
        except:
            pass
        fd, tmp_journal_filename = tempfile.mkstemp()
        os.close(fd)
        cmd += ' > {}'.format(tmp_journal_filename)
        os.system(cmd)

        # For every trannsaction, add the correspinding CSV file line to the
        # generated journal file.
        new_lines = []
        with open(tmp_journal_filename, 'a+') as fh:
            i = 0
            for line in fh.readlines():
                new_lines.append(line)
                if (RE_LEDGER_FST_LINE_TRANSACTION.match(line)):
                    # Assuming that the transactions in the journal file have
                    # the same order as the transactions in the csv file, we
                    # can match modified csv lines to the journal's
                    # transactions:
                    new_lines.append('    ; CSV data:\n')
                    new_lines.append('    ; from : {}'.
                                     format(modified_lines[i][1]))
                    new_lines.append('    ; (raw): {}'.
                                     format(modified_lines[i][0]))
                    i += 1
        with open(tmp_journal_filename, 'w') as fh:
            fh.write(''.join(new_lines))

        # Append the list of ignored transactions to the generated journal
        # file.
        if(len(ignored_lines) > 0):
            with open(tmp_journal_filename, 'a+') as fh:
                fh.write('\n\n')
                fh.write('; Attention: The following lines from ')
                fh.write('{} were ignored:\n'.format(csv_filename))
                for line in ignored_lines:
                    fh.write('; {}\n'.format(line.strip()))

        # Print to stdout for later use.
        os.system('cat {}'.format(tmp_journal_filename))

        # Cleanup.
        os.remove(tmp_csv_filename)
        os.remove(tmp_journal_filename)

if __name__ == '__main__':
    main(sys.argv)
<|endoftext|>


<commit_before>"""
Relay plugins define 3 control points that relay can use:

    metrics - are generator functions that return values
    warmers - are functions that increase the value of the metric
    coolers - are functions that decrease the value of the metric

In a distributed environment, Relay may help you to:

    - autoscale the number of workers who are consuming from a queue
    - ensure there are enough concurrent processes running at any given time
    - generally stabilize a stream of numbers by functions that control
      increasing the values in the stream and decreasing the values in
      the stream

"""

def warmer(n):
    """A function that will increase values in your metric"""
    raise NotImplementedError()

def cooler(n):
    """A function that will decrease values in your metric"""
    raise NotImplementedError()

def metric():
    """A generator function that represents some timeseries.
    For instance this function could yield the current temperature"""
    while True:
        yield 0

######
# Example
#####

def oscillating_setpoint(_square_wave=False):
    """A basic example of a target that you may want to approximate.

    If you have a thermostat, this is a temperature setting.
    This target can't change too often
    """
    import math
    c = 0
    while 1:
        if _square_wave:
            yield ((c % 300) < 150) * 20
            c += 1
        else:
            yield 10 * math.sin(2 * 3.1415926 * c) \
                + 20 + 5 * math.sin(2 * 3.1415926 * c * 3)
            c += .001

def squarewave_setpoint():
    return oscillating_setpoint(True)

def bash_echo_metric():
    """A very basic example that monitors
    a number of currently running processes"""
    import subprocess
    import random

    # more predictable version of the metric
    cmd = (
        'pgrep -f "^bash -c sleep .*from bash: started relay launcher" |wc -l')

    # less predictable version of the metric
    # cmd = 'ps aux|wc -l'

    while True:
        yield (
            int(subprocess.check_output(cmd, shell=True))
            # + random.choice([-2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8])
        )

def bash_echo_warmer(n):
    """A very basic example of how to create n additional tasks.
    This is a warmer function with randomly delayed effects on the
    bash_echo_metric and random task lengths to make the metric less
    predictable
    """
    import subprocess
    import random
    cmd = (
        "sleep %s "
        " ; sh -c 'echo from bash: started relay launcher task && sleep %s'"
    )
    for i in range(n):
        subprocess.Popen(
            cmd % ((1 + random.random()) * 1, (1 + random.random()) * 9),
            shell=True, stdout=subprocess.PIPE, executable='bash')

def bash_echo_cooler(n):
    """A very basic example of how to destroy n running tasks
    This is a cooler function
    """
    import subprocess
    cmd = (
        'kill `pgrep -f "from bash: started relay launcher task"'
        ' |tail -n %s` 2>/dev/null' % n)
    subprocess.Popen(cmd, shell=True).wait()
<commit_msg>get square wave examplel working<commit_after>"""
Relay plugins define 3 control points that relay can use:

    metrics - are generator functions that return values
    warmers - are functions that increase the value of the metric
    coolers - are functions that decrease the value of the metric

In a distributed environment, Relay may help you to:

    - autoscale the number of workers who are consuming from a queue
    - ensure there are enough concurrent processes running at any given time
    - generally stabilize a stream of numbers by functions that control
      increasing the values in the stream and decreasing the values in
      the stream

"""

def warmer(n):
    """A function that will increase values in your metric"""
    raise NotImplementedError()

def cooler(n):
    """A function that will decrease values in your metric"""
    raise NotImplementedError()

def metric():
    """A generator function that represents some timeseries.
    For instance this function could yield the current temperature"""
    while True:
        yield 0

######
# Example
#####

def oscillating_setpoint(_square_wave=False):
    """A basic example of a target that you may want to approximate.

    If you have a thermostat, this is a temperature setting.
    This target can't change too often
    """
    import math
    c = 0
    while 1:
        if _square_wave:
            yield ((c % 300) < 150) * 30 + 20
            c += 1
        else:
            yield 10 * math.sin(2 * 3.1415926 * c) \
                + 20 + 5 * math.sin(2 * 3.1415926 * c * 3)
            c += .001

def squarewave_setpoint():
    return oscillating_setpoint(True)

def bash_echo_metric():
    """A very basic example that monitors
    a number of currently running processes"""
    import subprocess
    import random

    # more predictable version of the metric
    cmd = (
        'pgrep -f "^bash -c sleep .*from bash: started relay launcher" |wc -l')

    # less predictable version of the metric
    # cmd = 'ps aux|wc -l'

    while True:
        yield (
            int(subprocess.check_output(cmd, shell=True))
            # + random.choice([-2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8])
        )

def bash_echo_warmer(n):
    """A very basic example of how to create n additional tasks.
    This is a warmer function with randomly delayed effects on the
    bash_echo_metric and random task lengths to make the metric less
    predictable
    """
    import subprocess
    import random
    cmd = (
        "sleep %s "
        " ; sh -c 'echo from bash: started relay launcher task && sleep %s'"
    )
    for i in range(n):
        subprocess.Popen(
            cmd % ((1 + random.random()) * 1, (1 + random.random()) * 4),
            shell=True, stdout=subprocess.PIPE, executable='bash')

def bash_echo_cooler(n):
    """A very basic example of how to destroy n running tasks
    This is a cooler function
    """
    import subprocess
    cmd = (
        'kill `pgrep -f "from bash: started relay launcher task"'
        ' |tail -n %s` 2>/dev/null' % n)
    subprocess.Popen(cmd, shell=True).wait()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2001-2018, Canal TP and/or its affiliates. All rights reserved.
#
# This file is part of Navitia,
# the software to build cool stuff with public transport.
#
# Hope you'll enjoy and contribute to this project,
#     powered by Canal TP (www.canaltp.fr).
# Help us simplify mobility and open public transport:
#     a non ending quest to the responsive locomotion way of traveling!
#
# LICENCE: This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Stay tuned using
# twitter @navitia
# IRC #navitia on freenode
# https://groups.google.com/d/forum/navitia
# www.navitia.io
from __future__ import absolute_import, print_function, division
import os

os.environ['LC_ALL'] = 'en_US'
os.environ['GIT_PYTHON_TRACE'] = '1'  # can be 0 (no trace), 1 (git commands) or full (git commands + git output)

from git import *
from datetime import datetime
import subprocess
import re
from sys import exit, argv
from shutil import copyfile
from os import remove, stat
import codecs
import requests
import logging

def get_tag_name(version):
    return "v{maj}.{min}.{hf}".format(maj=version[0], min=version[1], hf=version[2])

class ReleaseManager:
    def __init__(self, release_type, remote_name="canalTP"):
        self.directory = ".."
        self.changelog_filename = self.directory + "/debian/changelog"
        self.release_type = release_type
        self.repo = Repo(self.directory)
        self.git = self.repo.git

        # we fetch latest version from remote
        self.remote_name = remote_name

        print("fetching from {}...".format(remote_name))
        self.repo.remote(remote_name).fetch("--tags")

        # and we update dev and release branches
        print("rebasing dev and release...")

        # TODO quit on error
        self.git.rebase(remote_name + "/dev", "dev")
        try:
            self.git.checkout("release")
        except Exception as e:
            print("Cannot checkout 'release':{}, creating from distant branch".format(str(e)))
            self.git.checkout("-b", "release", remote_name + "/release")
        self.git.rebase(remote_name + "/release", "release")

        print("checking that release was merged into dev...")
        unmerged = self.git.branch("--no-merged", "dev", '--no-color')
        is_release_unmerged = re.search("  release(\n|$)", unmerged)
        if is_release_unmerged:
            print(is_release_unmerged.group(0))
            print("ABORTING: {rem}/release branch was not merged in {rem}/dev".format(rem=remote_name))
            print("This is required before releasing. You may use (be careful):")
            print("git checkout dev; git submodule update --recursive")
            print("git merge release")

            exit(1)

        print("current branch: {}".format(self.repo.active_branch))

        self.version = None
        self.str_version = ""
        self.latest_tag = ""

        # if API rate limit exceeded use, get 'personal access token' on github then provide:
        # self.auth = ('user', 'pass')
        self.auth = None

    def get_new_version_number(self):
        latest_version = None
        last_tag = self.git.describe('--tags', abbrev=0)

        version = re.search('.*(\d+\.\d+\.\d+).*', last_tag)
        if version:
            latest_version = version.group(1)

        if not latest_version:
            print("no latest version found")
            exit(1)

        version_n = latest_version.split('.')
        print("latest version is {}".format(version_n))

        self.version = [int(i) for i in version_n]

        self.latest_tag = get_tag_name(self.version)
        print("last tag is " + self.latest_tag)

        if self.release_type == "major":
            self.version[0] += 1
            self.version[1] = version_n[2] = 0
        elif self.release_type == "minor":
            self.version[1] += 1
            self.version[2] = 0
        elif self.release_type == "hotfix":
            self.version[2] += 1
        else:
            exit(5)

        self.str_version = "{maj}.{min}.{hf}".format(
            maj=self.version[0], min=self.version[1], hf=self.version[2]
        )

        print("current version is {}".format(self.str_version))
        return self.str_version

    def checkout_parent_branch(self):
        parent = ""
        if self.release_type == "major" or self.release_type == "minor":
            parent = "dev"
        else:
            parent = "release"

        self.git.checkout(parent)
        self.git.submodule('update', '--recursive')

        print("current branch {}".format(self.repo.active_branch))

    def closed_pr_generator(self):
        # lazy get all closed PR ordered by last updated
        closed_pr = []
        page = 1
        while True:
            query = (
                "https://api.github.com/repos/CanalTP/navitia/"
                "pulls?state=closed&base=dev&sort=updated&direction=desc&page={page}".format(
                    latest_tag=self.latest_tag, page=page
                )
            )
            print("query github api: " + query)
            github_response = requests.get(query, auth=self.auth)

            if github_response.status_code != 200:
                message = github_response.json()['message']
                print(u'  * Impossible to retrieve PR\n  * ' + message)
                return

            closed_pr = github_response.json()
            if not closed_pr:
                print("Reached end of PR list")
                return

            for pr in closed_pr:
                yield pr

            page += 1

    def get_merged_pullrequest(self):
        lines = []
        nb_successive_merged_pr = 0
        for pr in self.closed_pr_generator():
            title = pr['title']
            url = pr['html_url']
            pr_head_sha = pr['head']['sha']
            # test if PR was merged (not simply closed)
            # and if distant/release contains HEAD of PR
            # (stops after 10 successive merged PR)
            if pr['merged_at']:
                branches = []
                try:
                    branches = self.git.branch('-r', '--contains', pr_head_sha, '--no-color') + '\n'
                except:
                    print(
                        "ERROR while searching for commit in release branch: "
                        "Following PR added to changelog, remove it if needed.\n"
                    )

                # adding separators before and after to match only branch name
                release_branch_name = '  ' + self.remote_name + '/release\n'
                if release_branch_name in branches:
                    nb_successive_merged_pr += 1
                    if nb_successive_merged_pr >= 10:
                        break
                else:
                    # doing the label search as late as possible to save api calls
                    has_excluded_label = False
                    label_query = pr['_links']['issue']['href'] + '/labels'
                    labels = requests.get(label_query, auth=self.auth).json()
                    if any(label['name'] in ("hotfix", "not_in_changelog") for label in labels):
                        has_excluded_label = True

                    if not has_excluded_label:
                        lines.append(u'  * {title}  <{url}>\n'.format(title=title, url=url))
                        print(lines[-1])
                        nb_successive_merged_pr = 0
        return lines

    def create_changelog(self):
        write_lines = [u'navitia2 (%s) unstable; urgency=low\n' % self.str_version, u'\n']

        if self.release_type != "hotfix":
            pullrequests = self.get_merged_pullrequest()
            write_lines.extend(pullrequests)
        else:
            write_lines.append(u'  * \n')

        author_name = self.git.config('user.name')
        author_mail = self.git.config('user.email')
        write_lines.extend(
            [
                u'\n',
                u' -- {name} <{mail}>  {now} +0100\n'.format(
                    name=author_name, mail=author_mail, now=datetime.now().strftime("%a, %d %b %Y %H:%m:%S")
                ),
                u'\n',
            ]
        )

        return write_lines

    def update_changelog(self):
        print("updating changelog")
        changelog = self.create_changelog()

        f_changelog = None
        back_filename = self.changelog_filename + "~"
        try:
            f_changelog = codecs.open(self.changelog_filename, 'r', 'utf-8')
        except IOError:
            print("Unable to open file: " + self.changelog_filename)
            exit(1)
        f_changelogback = codecs.open(back_filename, "w", "utf-8")

        for line in changelog:
            f_changelogback.write(line)

        for line in f_changelog:
            f_changelogback.write(line)
        f_changelog.close()
        f_changelogback.close()
        last_modified = stat(back_filename)
        (stdout, stderr) = subprocess.Popen(
            ["vim", back_filename, "--nofork"], stderr=subprocess.PIPE
        ).communicate()
        after = stat(back_filename)
        if last_modified == after:
            print("No changes made, we stop")
            remove(back_filename)
            exit(2)

        copyfile(back_filename, self.changelog_filename)

        self.git.add(self.changelog_filename)

    def get_modified_changelog(self):
        # the changelog might have been modified by the user, so we have to read it again
        f_changelog = codecs.open(self.changelog_filename, 'r', 'utf-8')

        lines = []
        nb_version = 0
        for line in f_changelog:
            # each version are separated by a line like
            # navitia2 (0.94.1) unstable; urgency=low
            if line.startswith("navitia2 "):
                nb_version += 1
                continue
            if nb_version >= 2:
                break  # we can stop
            if nb_version == 0:
                continue

            lines.append(line + u'\n')

        f_changelog.close()
        return lines

    def publish_release(self, temp_branch):
        self.git.checkout("release")
        self.git.submodule('update', '--recursive')
        # merge with the release branch
        self.git.merge(temp_branch, "release", '--no-ff')

        print("current branch {}".format(self.repo.active_branch))
        # we tag the release
        tag_message = u'Version {}\n'.format(self.str_version)

        changelog = self.get_modified_changelog()
        for change in changelog:
            tag_message += change

        print("tag: " + tag_message)

        self.repo.create_tag(get_tag_name(self.version), message=tag_message)

        # and we merge back the release branch to dev (at least for the tag in release)
        self.git.merge("release", "dev", '--no-ff')

        print("publishing the release")

        print("Check the release, you will probably want to merge release in dev:")
        print("  git checkout dev; git submodule update --recursive")
        print("  git merge release")
        print("And when you're happy do:")
        print("  git push {} release dev --tags".format(self.remote_name))
        # TODO: when we'll be confident, we will do that automaticaly

    def release_the_kraken(self):
        new_version = self.get_new_version_number()

        tmp_name = "release_%s" % new_version

        self.checkout_parent_branch()

        # we then create a new temporary branch
        print("creating temporary release branch {}".format(tmp_name))
        self.git.checkout(b=tmp_name)
        print("current branch {}".format(self.repo.active_branch))

        self.update_changelog()

        self.git.commit(m="Version %s" % self.str_version)

        if self.release_type == "hotfix":
            print("now time to do your actual hotfix! (cherry-pick commits)")
            print("PLEASE check that \"release\" COMPILES and TESTS!")
            print("Note: you'll have to merge/tag/push manually after your fix:")
            print("  git checkout release")
            print("  git merge --no-ff {tmp_branch}".format(tmp_branch=tmp_name))
            print(
                "  git tag -a {} #then add message on Version and mention concerned PRs".format(
                    get_tag_name(self.version)
                )
            )
            print("  git checkout dev")
            print("  git merge --ff release")
            print("  git push {} release dev --tags".format(self.remote_name))

            # TODO2 try to script that (put 2 hotfix param, like hotfix init and hotfix publish ?)
            exit(0)

        self.publish_release(tmp_name)

if __name__ == '__main__':

    if len(argv) < 2:
        print("mandatory argument: {major|minor|hotfix}")
        print("possible additional argument: remote (default is canalTP)")
        exit(5)

    # the git lib used bug when not in english

    logging.basicConfig(level=logging.INFO)

    r_type = argv[1]
    remote = argv[2] if len(argv) >= 3 else "CanalTP"

    manager = ReleaseManager(release_type=r_type, remote_name=remote)
    manager.release_the_kraken()
<commit_msg>User changelog abspath to avoid confusion between git's dir and CWD<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2001-2018, Canal TP and/or its affiliates. All rights reserved.
#
# This file is part of Navitia,
# the software to build cool stuff with public transport.
#
# Hope you'll enjoy and contribute to this project,
#     powered by Canal TP (www.canaltp.fr).
# Help us simplify mobility and open public transport:
#     a non ending quest to the responsive locomotion way of traveling!
#
# LICENCE: This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Stay tuned using
# twitter @navitia
# IRC #navitia on freenode
# https://groups.google.com/d/forum/navitia
# www.navitia.io
from __future__ import absolute_import, print_function, division
import os

os.environ['LC_ALL'] = 'en_US'
os.environ['GIT_PYTHON_TRACE'] = '1'  # can be 0 (no trace), 1 (git commands) or full (git commands + git output)

from git import *
from datetime import datetime
import subprocess
import re
from sys import exit, argv
from shutil import copyfile
from os import remove, stat
import codecs
import requests
import logging

def get_tag_name(version):
    return "v{maj}.{min}.{hf}".format(maj=version[0], min=version[1], hf=version[2])

class ReleaseManager:
    def __init__(self, release_type, remote_name="canalTP"):
        self.directory = ".."
        self.changelog_filename = self.directory + "/debian/changelog"
        self.release_type = release_type
        self.repo = Repo(self.directory)
        self.git = self.repo.git

        # we fetch latest version from remote
        self.remote_name = remote_name

        print("fetching from {}...".format(remote_name))
        self.repo.remote(remote_name).fetch("--tags")

        # and we update dev and release branches
        print("rebasing dev and release...")

        # TODO quit on error
        self.git.rebase(remote_name + "/dev", "dev")
        try:
            self.git.checkout("release")
        except Exception as e:
            print("Cannot checkout 'release':{}, creating from distant branch".format(str(e)))
            self.git.checkout("-b", "release", remote_name + "/release")
        self.git.rebase(remote_name + "/release", "release")

        print("checking that release was merged into dev...")
        unmerged = self.git.branch("--no-merged", "dev", '--no-color')
        is_release_unmerged = re.search("  release(\n|$)", unmerged)
        if is_release_unmerged:
            print(is_release_unmerged.group(0))
            print("ABORTING: {rem}/release branch was not merged in {rem}/dev".format(rem=remote_name))
            print("This is required before releasing. You may use (be careful):")
            print("git checkout dev; git submodule update --recursive")
            print("git merge release")

            exit(1)

        print("current branch: {}".format(self.repo.active_branch))

        self.version = None
        self.str_version = ""
        self.latest_tag = ""

        # if API rate limit exceeded use, get 'personal access token' on github then provide:
        # self.auth = ('user', 'pass')
        self.auth = None

    def get_new_version_number(self):
        latest_version = None
        last_tag = self.git.describe('--tags', abbrev=0)

        version = re.search('.*(\d+\.\d+\.\d+).*', last_tag)
        if version:
            latest_version = version.group(1)

        if not latest_version:
            print("no latest version found")
            exit(1)

        version_n = latest_version.split('.')
        print("latest version is {}".format(version_n))

        self.version = [int(i) for i in version_n]

        self.latest_tag = get_tag_name(self.version)
        print("last tag is " + self.latest_tag)

        if self.release_type == "major":
            self.version[0] += 1
            self.version[1] = version_n[2] = 0
        elif self.release_type == "minor":
            self.version[1] += 1
            self.version[2] = 0
        elif self.release_type == "hotfix":
            self.version[2] += 1
        else:
            exit(5)

        self.str_version = "{maj}.{min}.{hf}".format(
            maj=self.version[0], min=self.version[1], hf=self.version[2]
        )

        print("current version is {}".format(self.str_version))
        return self.str_version

    def checkout_parent_branch(self):
        parent = ""
        if self.release_type == "major" or self.release_type == "minor":
            parent = "dev"
        else:
            parent = "release"

        self.git.checkout(parent)
        self.git.submodule('update', '--recursive')

        print("current branch {}".format(self.repo.active_branch))

    def closed_pr_generator(self):
        # lazy get all closed PR ordered by last updated
        closed_pr = []
        page = 1
        while True:
            query = (
                "https://api.github.com/repos/CanalTP/navitia/"
                "pulls?state=closed&base=dev&sort=updated&direction=desc&page={page}".format(
                    latest_tag=self.latest_tag, page=page
                )
            )
            print("query github api: " + query)
            github_response = requests.get(query, auth=self.auth)

            if github_response.status_code != 200:
                message = github_response.json()['message']
                print(u'  * Impossible to retrieve PR\n  * ' + message)
                return

            closed_pr = github_response.json()
            if not closed_pr:
                print("Reached end of PR list")
                return

            for pr in closed_pr:
                yield pr

            page += 1

    def get_merged_pullrequest(self):
        lines = []
        nb_successive_merged_pr = 0
        for pr in self.closed_pr_generator():
            title = pr['title']
            url = pr['html_url']
            pr_head_sha = pr['head']['sha']
            # test if PR was merged (not simply closed)
            # and if distant/release contains HEAD of PR
            # (stops after 10 successive merged PR)
            if pr['merged_at']:
                branches = []
                try:
                    branches = self.git.branch('-r', '--contains', pr_head_sha, '--no-color') + '\n'
                except:
                    print(
                        "ERROR while searching for commit in release branch: "
                        "Following PR added to changelog, remove it if needed.\n"
                    )

                # adding separators before and after to match only branch name
                release_branch_name = '  ' + self.remote_name + '/release\n'
                if release_branch_name in branches:
                    nb_successive_merged_pr += 1
                    if nb_successive_merged_pr >= 10:
                        break
                else:
                    # doing the label search as late as possible to save api calls
                    has_excluded_label = False
                    label_query = pr['_links']['issue']['href'] + '/labels'
                    labels = requests.get(label_query, auth=self.auth).json()
                    if any(label['name'] in ("hotfix", "not_in_changelog") for label in labels):
                        has_excluded_label = True

                    if not has_excluded_label:
                        lines.append(u'  * {title}  <{url}>\n'.format(title=title, url=url))
                        print(lines[-1])
                        nb_successive_merged_pr = 0
        return lines

    def create_changelog(self):
        write_lines = [u'navitia2 (%s) unstable; urgency=low\n' % self.str_version, u'\n']

        if self.release_type != "hotfix":
            pullrequests = self.get_merged_pullrequest()
            write_lines.extend(pullrequests)
        else:
            write_lines.append(u'  * \n')

        author_name = self.git.config('user.name')
        author_mail = self.git.config('user.email')
        write_lines.extend(
            [
                u'\n',
                u' -- {name} <{mail}>  {now} +0100\n'.format(
                    name=author_name, mail=author_mail, now=datetime.now().strftime("%a, %d %b %Y %H:%m:%S")
                ),
                u'\n',
            ]
        )

        return write_lines

    def update_changelog(self):
        print("updating changelog")
        changelog = self.create_changelog()

        f_changelog = None
        back_filename = self.changelog_filename + "~"
        try:
            f_changelog = codecs.open(self.changelog_filename, 'r', 'utf-8')
        except IOError:
            print("Unable to open file: " + self.changelog_filename)
            exit(1)
        f_changelogback = codecs.open(back_filename, "w", "utf-8")

        for line in changelog:
            f_changelogback.write(line)

        for line in f_changelog:
            f_changelogback.write(line)
        f_changelog.close()
        f_changelogback.close()
        last_modified = stat(back_filename)
        (stdout, stderr) = subprocess.Popen(
            ["vim", back_filename, "--nofork"], stderr=subprocess.PIPE
        ).communicate()
        after = stat(back_filename)
        if last_modified == after:
            print("No changes made, we stop")
            remove(back_filename)
            exit(2)

        copyfile(back_filename, self.changelog_filename)
        self.git.add(os.path.abspath(self.changelog_filename))

    def get_modified_changelog(self):
        # the changelog might have been modified by the user, so we have to read it again
        f_changelog = codecs.open(self.changelog_filename, 'r', 'utf-8')

        lines = []
        nb_version = 0
        for line in f_changelog:
            # each version are separated by a line like
            # navitia2 (0.94.1) unstable; urgency=low
            if line.startswith("navitia2 "):
                nb_version += 1
                continue
            if nb_version >= 2:
                break  # we can stop
            if nb_version == 0:
                continue

            lines.append(line + u'\n')

        f_changelog.close()
        return lines

    def publish_release(self, temp_branch):
        self.git.checkout("release")
        self.git.submodule('update', '--recursive')
        # merge with the release branch
        self.git.merge(temp_branch, "release", '--no-ff')

        print("current branch {}".format(self.repo.active_branch))
        # we tag the release
        tag_message = u'Version {}\n'.format(self.str_version)

        changelog = self.get_modified_changelog()
        for change in changelog:
            tag_message += change

        print("tag: " + tag_message)

        self.repo.create_tag(get_tag_name(self.version), message=tag_message)

        # and we merge back the release branch to dev (at least for the tag in release)
        self.git.merge("release", "dev", '--no-ff')

        print("publishing the release")

        print("Check the release, you will probably want to merge release in dev:")
        print("  git checkout dev; git submodule update --recursive")
        print("  git merge release")
        print("And when you're happy do:")
        print("  git push {} release dev --tags".format(self.remote_name))
        # TODO: when we'll be confident, we will do that automaticaly

    def release_the_kraken(self):
        new_version = self.get_new_version_number()

        tmp_name = "release_%s" % new_version

        self.checkout_parent_branch()

        # we then create a new temporary branch
        print("creating temporary release branch {}".format(tmp_name))
        self.git.checkout(b=tmp_name)
        print("current branch {}".format(self.repo.active_branch))

        self.update_changelog()

        self.git.commit(m="Version %s" % self.str_version)

        if self.release_type == "hotfix":
            print("now time to do your actual hotfix! (cherry-pick commits)")
            print("PLEASE check that \"release\" COMPILES and TESTS!")
            print("Note: you'll have to merge/tag/push manually after your fix:")
            print("  git checkout release")
            print("  git merge --no-ff {tmp_branch}".format(tmp_branch=tmp_name))
            print(
                "  git tag -a {} #then add message on Version and mention concerned PRs".format(
                    get_tag_name(self.version)
                )
            )
            print("  git checkout dev")
            print("  git merge --ff release")
            print("  git push {} release dev --tags".format(self.remote_name))

            # TODO2 try to script that (put 2 hotfix param, like hotfix init and hotfix publish ?)
            exit(0)

        self.publish_release(tmp_name)

if __name__ == '__main__':

    if len(argv) < 2:
        print("mandatory argument: {major|minor|hotfix}")
        print("possible additional argument: remote (default is canalTP)")
        exit(5)

    # the git lib used bug when not in english

    logging.basicConfig(level=logging.INFO)

    r_type = argv[1]
    remote = argv[2] if len(argv) >= 3 else "CanalTP"

    manager = ReleaseManager(release_type=r_type, remote_name=remote)
    manager.release_the_kraken()
<|endoftext|>


<commit_before>'''
Knowledge-base class.

@author: anze.vavpetic@ijs.si
'''
from collections import defaultdict
from bitarray import bitarray
from rdflib import RDF, RDFS, URIRef

from example import Example
from predicate import UnaryPredicate
from helpers import avg, std
from settings import EXAMPLE_SCHEMA, logger, W3C, HEDWIG

class ExperimentKB:
    '''
    The knowledge base for one specific experiment.
    '''
    def __init__(self, triplets, score_fun,
                 instances_as_leaves=True):
        '''
        Initialize the knowledge base with the given triplet graph.
        The target class is given with 'target_class' - this is the
        class to be described in the induction step.
        '''
        self.score_fun = score_fun
        self.sub_class_of = defaultdict(list)
        self.super_class_of = defaultdict(list)
        self.predicates = set()
        self.binary_predicates = set()
        self.class_values = set()

        g = triplets

        # Parse the examples schema
        g.parse(EXAMPLE_SCHEMA, format='n3')

        # Extract the available examples from the graph
        ex_subjects = g.subjects(predicate=RDF.type, object=HEDWIG.Example)
        self.examples_uris = [ex for ex in ex_subjects]
        self.uri_to_idx = {}

        examples = []
        for i, ex_uri in enumerate(self.examples_uris):

            # Query for annotation link objects
            annot_objects = g.objects(subject=ex_uri,
                                           predicate=HEDWIG.annotated_with)

            annotation_links = [annot for annot in annot_objects]
            annotations = []
            weights = {}
            to_uni = lambda s: unicode(s).encode('ascii', 'ignore')

            for link in annotation_links:

                # Query for annotation objects via this link
                annot_objects = g.objects(subject=link,
                                               predicate=HEDWIG.annotation)
                annotation = [to_uni(one) for one in annot_objects][0]

                # Query for weights on this link
                weight_objects = g.objects(subject=link,
                                                predicate=HEDWIG.weight)
                weights_list = [one for one in weight_objects]

                if weights_list:
                    weights[annotation] = float(weights_list[0])

                annotations.append(annotation)

            # Scores
            score_list = list(g.objects(subject=ex_uri,
                                             predicate=HEDWIG.score))
            if score_list:
                score = float(score_list[0])
            else:
                # Classes
                score_list = list(g.objects(subject=ex_uri,
                                                 predicate=HEDWIG.class_label))
                score = str(score_list[0])
                self.class_values.add(score)

            self.uri_to_idx[ex_uri] = i

            examples.append(Example(i, str(ex_uri), score,
                                    annotations=annotations,
                                    weights=weights))

        self.examples = examples

        if not self.examples:
            raise Exception("No examples provided! Examples should be " +
                            "instances of %s." % HEDWIG)

        # Ranked or class-labeled data
        self.target_type = self.examples[0].target_type

        # Get the subClassOf hierarchy
        for sub, obj in g.subject_objects(predicate=RDFS.subClassOf):
            if self.user_defined(sub) and self.user_defined(obj):
                self.add_sub_class(sub, obj)

        # Include the instances as predicates as well
        if instances_as_leaves:
            for sub, obj in g.subject_objects(predicate=RDF.type):
                if self.user_defined(sub) and self.user_defined(obj):
                    self.add_sub_class(sub, obj)

        # Find the user-defined object predicates defined between examples
        examples_as_domain = set(g.subjects(object=HEDWIG.Example,
                                                 predicate=RDFS.domain))

        examples_as_range = set(g.subjects(object=HEDWIG.Example,
                                                predicate=RDFS.range))

        for pred in examples_as_domain.intersection(examples_as_range):
            if self.user_defined(pred):
                self.binary_predicates.add(str(pred))

        # Calculate the members for each predicate
        self.members = defaultdict(set)
        for ex in examples:
            for inst in ex.annotations:
                if instances_as_leaves:
                    self.members[inst].add(ex.id)
                else:
                    # Query for 'parents' of a given instance
                    inst_parents = list(g.objects(subject=URIRef(inst),
                                                       predicate=RDF.type))
                    inst_parents += list(g.objects(subject=URIRef(inst),
                                                    predicate=RDFS.subClassOf))
                    for obj in inst_parents:
                        self.members[str(obj)].add(ex.id)

        # Find the root classes
        roots = filter(lambda pred: not self.sub_class_of[pred],
                       self.super_class_of.keys())

        logger.debug('Detected root nodes: %s' % str(roots))

        # Add a dummy root
        self.dummy_root = 'root'
        self.predicates.add(self.dummy_root)
        for root in roots:
            self.add_sub_class(root, self.dummy_root)

        self.sub_class_of_closure = defaultdict(set)
        for pred in self.super_class_of.keys():
            self.sub_class_of_closure[pred].update(self.sub_class_of[pred])

        # Calc the closure to get the members of the subClassOf hierarchy
        def closure(pred, lvl, visited=[]):

            if pred in visited:
                raise Exception('Cycle detected in the hierarchy!')

            children = self.super_class_of[pred]
            self.levels[lvl].add(pred)

            if children:
                mems = set()
                visited.append(pred)
                for child in children:
                    parent_closure = self.sub_class_of_closure[pred]
                    self.sub_class_of_closure[child].update(parent_closure)
                    mems.update(closure(child, lvl + 1, visited=visited))
                self.members[pred].update(mems)
                visited.remove(pred)

                return self.members[pred]
            else:
                return self.members[pred]

        # Level-wise predicates
        self.levels = defaultdict(set)

        # Run the closure from root
        closure(self.dummy_root, 0)

        # Members of non-unary predicates
        self.binary_members = defaultdict(dict)
        self.reverse_binary_members = defaultdict(dict)

        for pred in self.binary_predicates:
            pairs = g.subject_objects(predicate=URIRef(pred))

            for pair in pairs:
                el1, el2 = self.uri_to_idx[pair[0]], self.uri_to_idx[pair[1]]
                if self.binary_members[pred].has_key(el1):
                    self.binary_members[pred][el1].append(el2)
                else:
                    self.binary_members[pred][el1] = [el2]

                # Add the reverse as well
                if self.reverse_binary_members[pred].has_key(el2):
                    self.reverse_binary_members[pred][el2].append(el1)
                else:
                    self.reverse_binary_members[pred][el2] = [el1]

        # Bitset of examples for input and output
        self.binary_domains = {}
        for pred in self.binary_predicates:
            self.binary_domains[pred] = (
                self.indices_to_bits(self.binary_members[pred].keys()),
                self.indices_to_bits(self.reverse_binary_members[pred].keys())
            )

        # Calc the corresponding bitsets
        self.bit_members = {}
        for pred in self.members.keys():
            self.bit_members[pred] = self.indices_to_bits(self.members[pred])

        self.bit_binary_members = defaultdict(dict)
        self.reverse_bit_binary_members = defaultdict(dict)

        for pred in self.binary_members.keys():

            for el in self.binary_members[pred].keys():
                indices = self.indices_to_bits(self.binary_members[pred][el])
                self.bit_binary_members[pred][el] = indices

            for el in self.reverse_binary_members[pred].keys():
                reverse_members = self.reverse_binary_members[pred][el]
                indices = self.indices_to_bits(reverse_members)
                self.reverse_bit_binary_members[pred][el] = indices

        # Statistics
        if self.target_type == Example.Ranked:
            self.mean = avg([ex.score for ex in self.examples])
            self.sd = std([ex.score for ex in self.examples])
        else:
            self.distribution = defaultdict(int)
            for ex in self.examples:
                self.distribution[ex.score] += 1
            logger.debug('Class distribution: %s' % str(self.distribution))

    def user_defined(self, uri):
        '''
        Is this resource user defined?
        '''
        return not uri.startswith(W3C) and not uri.startswith(HEDWIG)

    def add_sub_class(self, sub, obj):
        '''
        Adds the resource 'sub' as a subclass of 'obj'.
        '''
        to_uni = lambda s: unicode(s).encode('ascii', 'ignore')
        sub, obj = to_uni(sub), to_uni(obj)

        self.predicates.update([sub, obj])
        self.sub_class_of[sub].append(obj)
        self.super_class_of[obj].append(sub)

    def super_classes(self, pred):
        '''
        Returns all super classes of pred (with transitivity).
        '''
        return self.sub_class_of_closure[pred]

    def get_root(self):
        '''
        Root predicate, which covers all examples.
        '''
        return UnaryPredicate(self.dummy_root, self.get_full_domain(), self,
                              custom_var_name='X')

    def get_subclasses(self, predicate, producer_pred=None):
        '''
        Returns a list of subclasses (as predicate objects) for 'predicate'.
        '''
        if isinstance(predicate, UnaryPredicate):
            return self.super_class_of[predicate.label]
        else:
            return self.super_class_of[predicate]

    def get_members(self, predicate, bit=True):
        '''
        Returns the examples for this predicate,
        either as a bitset or a set of ids.
        '''
        members = None
        if predicate in self.predicates:
            if bit:
                members = self.bit_members[predicate]
            else:
                members = self.members[predicate]
        else:
            if bit:
                members = self.bit_binary_members[predicate]
            else:
                members = self.binary_members[predicate]

        return members

    def get_reverse_members(self, predicate, bit=True):
        '''
        Returns the examples for this predicate,
        either as a bitset or a set of ids.
        '''
        reverse_members = None
        if bit:
            reverse_members = self.reverse_bit_binary_members[predicate]
        else:
            reverse_members = self.reverse_binary_members[predicate]

        return reverse_members

    def n_members(self, predicate):
        return self.get_members(predicate, bit=True).count()

    def get_domains(self, predicate):
        '''
        Returns the bitsets for input and outputexamples
        of the binary predicate 'predicate'.
        '''
        return self.binary_domains[predicate]

    def get_examples(self):
        '''
        Returns all examples for this experiment.
        '''
        return self.examples

    def n_examples(self):
        '''
        Returns the number of examples.
        '''
        return len(self.examples)

    def get_full_domain(self):
        '''
        Returns a bitset covering all examples.
        '''
        return bitarray([True] * self.n_examples())

    def get_empty_domain(self):
        '''
        Returns a bitset covering no examples.
        '''
        return bitarray([False] * self.n_examples())

    def get_score(self, ex_idx):
        '''
        Returns the score for example id 'ex_idx'.
        '''
        return self.examples[ex_idx].score

    def bits_to_indices(self, bits):
        '''
        Converts the bitset to a set of indices.
        '''
        return bits.search(bitarray([1]))

    def indices_to_bits(self, indices):
        '''
        Converts the indices to a bitset.
        '''
        bits = self.get_empty_domain()
        for idx in indices:
            bits[idx] = True
        return bits

    def is_discrete_target(self):
        return bool(self.class_values)
<commit_msg>refactored the KB code a bit<commit_after>'''
Knowledge-base class.

@author: anze.vavpetic@ijs.si
'''
from collections import defaultdict
from bitarray import bitarray
from rdflib import RDF, RDFS, URIRef

from example import Example
from predicate import UnaryPredicate
from helpers import avg, std
from settings import EXAMPLE_SCHEMA, logger, W3C, HEDWIG

class ExperimentKB:
    '''
    The knowledge base for one specific experiment.
    '''
    def __init__(self, triplets, score_fun,
                 instances_as_leaves=True):
        '''
        Initialize the knowledge base with the given triplet graph.
        The target class is given with 'target_class' - this is the
        class to be described in the induction step.
        '''
        self.instances_as_leaves = instances_as_leaves
        self.score_fun = score_fun
        self.sub_class_of = defaultdict(list)
        self.super_class_of = defaultdict(list)
        self.predicates = set()
        self.binary_predicates = set()
        self.class_values = set()

        self.examples = self._build_examples(triplets)

        # Ranked or class-labeled data
        self.target_type = self.examples[0].target_type

        self._build_subclassof(triplets)
        self._calc_predicate_members(triplets)
        self._find_roots()
        self._calc_members_closure()
        self._calc_binary_members()

        # Statistics
        if self.target_type == Example.Ranked:
            self.mean = avg([ex.score for ex in self.examples])
            self.sd = std([ex.score for ex in self.examples])
        else:
            self.distribution = defaultdict(int)
            for ex in self.examples:
                self.distribution[ex.score] += 1
            logger.debug('Class distribution: %s' % str(self.distribution))

    def _build_examples(self, g):
        g.parse(EXAMPLE_SCHEMA, format='n3')

        # Extract the available examples from the graph
        ex_subjects = g.subjects(predicate=RDF.type, object=HEDWIG.Example)
        self.examples_uris = [ex for ex in ex_subjects]
        self.uri_to_idx = {}

        examples = []
        for i, ex_uri in enumerate(self.examples_uris):

            # Query for annotation link objects
            annot_objects = g.objects(subject=ex_uri,
                                           predicate=HEDWIG.annotated_with)

            annotation_links = [annot for annot in annot_objects]
            annotations = []
            weights = {}
            to_uni = lambda s: unicode(s).encode('ascii', 'ignore')

            for link in annotation_links:

                # Query for annotation objects via this link
                annot_objects = g.objects(subject=link,
                                               predicate=HEDWIG.annotation)
                annotation = [to_uni(one) for one in annot_objects][0]

                # Query for weights on this link
                weight_objects = g.objects(subject=link,
                                                predicate=HEDWIG.weight)
                weights_list = [one for one in weight_objects]

                if weights_list:
                    weights[annotation] = float(weights_list[0])

                annotations.append(annotation)

            # Scores
            score_list = list(g.objects(subject=ex_uri,
                                             predicate=HEDWIG.score))
            if score_list:
                score = float(score_list[0])
            else:
                # Classes
                score_list = list(g.objects(subject=ex_uri,
                                                 predicate=HEDWIG.class_label))
                score = str(score_list[0])
                self.class_values.add(score)

            self.uri_to_idx[ex_uri] = i

            examples.append(Example(i, str(ex_uri), score,
                                    annotations=annotations,
                                    weights=weights))

        if not examples:
            raise Exception("No examples provided! Examples should be " +
                            "instances of %s." % HEDWIG)
        return examples

    def _build_subclassof(self, g):
        for sub, obj in g.subject_objects(predicate=RDFS.subClassOf):
            if self.user_defined(sub) and self.user_defined(obj):
                self.add_sub_class(sub, obj)

        # Include the instances as predicates as well
        if self.instances_as_leaves:
            for sub, obj in g.subject_objects(predicate=RDF.type):
                if self.user_defined(sub) and self.user_defined(obj):
                    self.add_sub_class(sub, obj)

        # Find the user-defined object predicates defined between examples
        examples_as_domain = set(g.subjects(object=HEDWIG.Example,
                                                 predicate=RDFS.domain))

        examples_as_range = set(g.subjects(object=HEDWIG.Example,
                                                predicate=RDFS.range))

        for pred in examples_as_domain.intersection(examples_as_range):
            if self.user_defined(pred):
                self.binary_predicates.add(str(pred))

    def _calc_predicate_members(self, g):
        self.members = defaultdict(set)
        for ex in self.examples:
            for inst in ex.annotations:
                if self.instances_as_leaves:
                    self.members[inst].add(ex.id)
                else:
                    # Query for 'parents' of a given instance
                    inst_parents = list(g.objects(subject=URIRef(inst),
                                                       predicate=RDF.type))
                    inst_parents += list(g.objects(subject=URIRef(inst),
                                                    predicate=RDFS.subClassOf))
                    for obj in inst_parents:
                        self.members[str(obj)].add(ex.id)

    def _find_roots(self):
        roots = filter(lambda pred: not self.sub_class_of[pred],
                       self.super_class_of.keys())

        logger.debug('Detected root nodes: %s' % str(roots))

        # Add a dummy root
        self.dummy_root = 'root'
        self.predicates.add(self.dummy_root)
        for root in roots:
            self.add_sub_class(root, self.dummy_root)

    def _calc_members_closure(self):
        self.sub_class_of_closure = defaultdict(set)
        for pred in self.super_class_of.keys():
            self.sub_class_of_closure[pred].update(self.sub_class_of[pred])

        # Calc the closure to get the members of the subClassOf hierarchy
        def closure(pred, lvl, visited=[]):

            if pred in visited:
                raise Exception('Cycle detected in the hierarchy!')

            children = self.super_class_of[pred]
            self.levels[lvl].add(pred)

            if children:
                mems = set()
                visited.append(pred)
                for child in children:
                    parent_closure = self.sub_class_of_closure[pred]
                    self.sub_class_of_closure[child].update(parent_closure)
                    mems.update(closure(child, lvl + 1, visited=visited))
                self.members[pred].update(mems)
                visited.remove(pred)

                return self.members[pred]
            else:
                return self.members[pred]

        # Level-wise predicates
        self.levels = defaultdict(set)

        # Run the closure from root
        closure(self.dummy_root, 0)

    def _calc_binary_members(self):
        self.binary_members = defaultdict(dict)
        self.reverse_binary_members = defaultdict(dict)

        for pred in self.binary_predicates:
            pairs = g.subject_objects(predicate=URIRef(pred))

            for pair in pairs:
                el1, el2 = self.uri_to_idx[pair[0]], self.uri_to_idx[pair[1]]
                if self.binary_members[pred].has_key(el1):
                    self.binary_members[pred][el1].append(el2)
                else:
                    self.binary_members[pred][el1] = [el2]

                # Add the reverse as well
                if self.reverse_binary_members[pred].has_key(el2):
                    self.reverse_binary_members[pred][el2].append(el1)
                else:
                    self.reverse_binary_members[pred][el2] = [el1]

        # Bitset of examples for input and output
        self.binary_domains = {}
        for pred in self.binary_predicates:
            self.binary_domains[pred] = (
                self.indices_to_bits(self.binary_members[pred].keys()),
                self.indices_to_bits(self.reverse_binary_members[pred].keys())
            )

        # Calc the corresponding bitsets
        self.bit_members = {}
        for pred in self.members.keys():
            self.bit_members[pred] = self.indices_to_bits(self.members[pred])

        self.bit_binary_members = defaultdict(dict)
        self.reverse_bit_binary_members = defaultdict(dict)

        for pred in self.binary_members.keys():

            for el in self.binary_members[pred].keys():
                indices = self.indices_to_bits(self.binary_members[pred][el])
                self.bit_binary_members[pred][el] = indices

            for el in self.reverse_binary_members[pred].keys():
                reverse_members = self.reverse_binary_members[pred][el]
                indices = self.indices_to_bits(reverse_members)
                self.reverse_bit_binary_members[pred][el] = indices

    def user_defined(self, uri):
        '''
        Is this resource user defined?
        '''
        return not uri.startswith(W3C) and not uri.startswith(HEDWIG)

    def add_sub_class(self, sub, obj):
        '''
        Adds the resource 'sub' as a subclass of 'obj'.
        '''
        to_uni = lambda s: unicode(s).encode('ascii', 'ignore')
        sub, obj = to_uni(sub), to_uni(obj)

        self.predicates.update([sub, obj])
        self.sub_class_of[sub].append(obj)
        self.super_class_of[obj].append(sub)

    def super_classes(self, pred):
        '''
        Returns all super classes of pred (with transitivity).
        '''
        return self.sub_class_of_closure[pred]

    def get_root(self):
        '''
        Root predicate, which covers all examples.
        '''
        return UnaryPredicate(self.dummy_root, self.get_full_domain(), self,
                              custom_var_name='X')

    def get_subclasses(self, predicate, producer_pred=None):
        '''
        Returns a list of subclasses (as predicate objects) for 'predicate'.
        '''
        if isinstance(predicate, UnaryPredicate):
            return self.super_class_of[predicate.label]
        else:
            return self.super_class_of[predicate]

    def get_members(self, predicate, bit=True):
        '''
        Returns the examples for this predicate,
        either as a bitset or a set of ids.
        '''
        members = None
        if predicate in self.predicates:
            if bit:
                members = self.bit_members[predicate]
            else:
                members = self.members[predicate]
        else:
            if bit:
                members = self.bit_binary_members[predicate]
            else:
                members = self.binary_members[predicate]

        return members

    def get_reverse_members(self, predicate, bit=True):
        '''
        Returns the examples for this predicate,
        either as a bitset or a set of ids.
        '''
        reverse_members = None
        if bit:
            reverse_members = self.reverse_bit_binary_members[predicate]
        else:
            reverse_members = self.reverse_binary_members[predicate]

        return reverse_members

    def n_members(self, predicate):
        return self.get_members(predicate, bit=True).count()

    def get_domains(self, predicate):
        '''
        Returns the bitsets for input and outputexamples
        of the binary predicate 'predicate'.
        '''
        return self.binary_domains[predicate]

    def get_examples(self):
        '''
        Returns all examples for this experiment.
        '''
        return self.examples

    def n_examples(self):
        '''
        Returns the number of examples.
        '''
        return len(self.examples)

    def get_full_domain(self):
        '''
        Returns a bitset covering all examples.
        '''
        return bitarray([True] * self.n_examples())

    def get_empty_domain(self):
        '''
        Returns a bitset covering no examples.
        '''
        return bitarray([False] * self.n_examples())

    def get_score(self, ex_idx):
        '''
        Returns the score for example id 'ex_idx'.
        '''
        return self.examples[ex_idx].score

    def bits_to_indices(self, bits):
        '''
        Converts the bitset to a set of indices.
        '''
        return bits.search(bitarray([1]))

    def indices_to_bits(self, indices):
        '''
        Converts the indices to a bitset.
        '''
        bits = self.get_empty_domain()
        for idx in indices:
            bits[idx] = True
        return bits

    def is_discrete_target(self):
        return bool(self.class_values)
<|endoftext|>


<commit_before>import struct

from Crypto.Cipher import XOR
from Crypto.Cipher import AES

from dh import create_dh_key, calculate_dh_secret

class StealthConn(object):
    def __init__(self, conn, client=False, server=False, verbose=False):
        self.conn = conn
        self.cipher = None
        self.client = client
        self.server = server
        self.verbose = verbose
        self.initiate_session()

    def initiate_session(self):
        # Perform the initial connection handshake for agreeing on a shared secret

        # This can be broken into code run just on the server or just on the client
        if self.server or self.client:
            my_public_key, my_private_key = create_dh_key()
            # Send them our public key
            self.send(bytes(str(my_public_key), "ascii"))
            # Receive their public key
            their_public_key = int(self.recv())
            # Obtain our shared secret
            shared_hash = calculate_dh_secret(their_public_key, my_private_key)
            print("Shared hash: {}".format(shared_hash))

        # Default XOR algorithm can only take a key of length 32
        #self.cipher = XOR.new(shared_hash[:4])
        self.cipher = AES.new(shared_hash[:32]) # AES-256 accepts keys of 32 bytes in length

    def send(self, data):
        if self.cipher:
            # pad the data to a multiple of 16 in length
            length = 16 - (len(data) % 16)
            data += ' '.encode() * length

            encrypted_data = self.cipher.encrypt(data)
            if self.verbose:
                print("Original data: {}".format(data))
                print("Encrypted data: {}".format(repr(encrypted_data)))
                print("Sending packet of length {}".format(len(encrypted_data)))
        else:
            encrypted_data = data

        # Encode the data's length into an unsigned two byte int ('H')
        pkt_len = struct.pack('H', len(encrypted_data))
        self.conn.sendall(pkt_len)
        self.conn.sendall(encrypted_data)

    def recv(self):
        # Decode the data's length from an unsigned two byte int ('H')
        pkt_len_packed = self.conn.recv(struct.calcsize('H'))

        unpacked_contents = struct.unpack('H', pkt_len_packed)
        pkt_len = unpacked_contents[0]

        encrypted_data = self.conn.recv(pkt_len)
        if self.cipher:
            data = self.cipher.decrypt(encrypted_data)
            if self.verbose:
                print("Receiving packet of length {}".format(pkt_len))
                print("Encrypted data: {}".format(repr(encrypted_data)))
                print("Original data: {}".format(data))
        else:
            data = encrypted_data

        return data.rstrip() # remove padding

    def close(self):
        self.conn.close()<commit_msg>incorporated ECB mode of operation to test <commit_after>import struct

from Crypto.Cipher import XOR
from Crypto.Cipher import AES

from dh import create_dh_key, calculate_dh_secret

class StealthConn(object):
    def __init__(self, conn, client=False, server=False, verbose=False):
        self.conn = conn
        self.cipher = None
        self.client = client
        self.server = server
        self.verbose = verbose
        self.initiate_session()

    def initiate_session(self):
        # Perform the initial connection handshake for agreeing on a shared secret

        # This can be broken into code run just on the server or just on the client
        if self.server or self.client:
            my_public_key, my_private_key = create_dh_key()
            # Send them our public key
            self.send(bytes(str(my_public_key), "ascii"))
            # Receive their public key
            their_public_key = int(self.recv())
            # Obtain our shared secret
            shared_hash = calculate_dh_secret(their_public_key, my_private_key)
            print("Shared hash: {}".format(shared_hash))

        # Default XOR algorithm can only take a key of length 32
        #self.cipher = XOR.new(shared_hash[:4])
        self.cipher = AES.new(shared_hash[:32]) # AES-256 accepts keys of 32 bytes in length
        #self.cipher = AES.new(shared_hash[:32], AES.MODE_ECB)

    def send(self, data):
        if self.cipher:
            # pad the data to a multiple of 16 in length
            length = 16 - (len(data) % 16)
            data += ' '.encode() * length

            encrypted_data = self.cipher.encrypt(data)
            if self.verbose:
                print("Original data: {}".format(data))
                print("Encrypted data: {}".format(repr(encrypted_data)))
                print("Sending packet of length {}".format(len(encrypted_data)))
        else:
            encrypted_data = data

        # Encode the data's length into an unsigned two byte int ('H')
        pkt_len = struct.pack('H', len(encrypted_data))
        self.conn.sendall(pkt_len)
        self.conn.sendall(encrypted_data)

    def recv(self):
        # Decode the data's length from an unsigned two byte int ('H')
        pkt_len_packed = self.conn.recv(struct.calcsize('H'))

        unpacked_contents = struct.unpack('H', pkt_len_packed)
        pkt_len = unpacked_contents[0]

        encrypted_data = self.conn.recv(pkt_len)
        if self.cipher:
            data = self.cipher.decrypt(encrypted_data)
            if self.verbose:
                print("Receiving packet of length {}".format(pkt_len))
                print("Encrypted data: {}".format(repr(encrypted_data)))
                print("Original data: {}".format(data))
        else:
            data = encrypted_data

        return data.rstrip() # remove padding

    def close(self):
        self.conn.close()
<|endoftext|>