<commit_before>from django import forms
from django.conf import settings
from django.contrib import admin
from django.utils.safestring import mark_safe
from dal import autocomplete

from .models import Place

class GeonamesLookupWidget(autocomplete.Select2):
    '''Customize autocomplete select widget to display Geonames URI
    as a link and render a map for the selected location.'''
    # (inspired by )

    def render(self, name, value, attrs=None):
        widget = super(autocomplete.Select2, self).render(name, value, attrs)
        return mark_safe((u'

' +
            u'%s
%s
') % \
            (widget, value or '', value or ''))

class PlaceAdminForm(forms.ModelForm):
    '''Custom model form for Place editing, used to add geonames lookup.'''

    #: add a hidden field to pass in a mapbox access token from local settings
    mapbox_token = forms.CharField(initial=getattr(settings, 'MAPBOX_ACCESS_TOKEN', ''),
        widget=forms.HiddenInput)

    class Meta:
        model = Place
        exclude = []
        widgets = {
            'geonames_id': GeonamesLookupWidget(url='places:geonames-autocomplete',
                attrs={'data-placeholder': 'Type location name to search...',
                       'data-minimum-input-length': 3})
        }

class PlaceAdmin(admin.ModelAdmin):
    form = PlaceAdminForm
    list_display = ('name', 'geonames_id', 'has_notes')
    fields = ('name', 'geonames_id', 'latitude', 'longitude', 'notes',
        'mapbox_token')
    search_fields = ('name', 'notes', 'geonames_id')
    class Media:
        static_url = getattr(settings, 'STATIC_URL')
        css = {
            'all': ['https://unpkg.com/leaflet@1.0.2/dist/leaflet.css',
                    'admin/geonames.css']
        }
        js = ['admin/geonames-lookup.js',
            'https://unpkg.com/leaflet@1.0.2/dist/leaflet.js']

admin.site.register(Place, PlaceAdmin)
<commit_msg>Fix super call so it uses correct class (linter cleanup)<commit_after>from django import forms
from django.conf import settings
from django.contrib import admin
from django.utils.safestring import mark_safe
from dal import autocomplete

from .models import Place

class GeonamesLookupWidget(autocomplete.Select2):
    '''Customize autocomplete select widget to display Geonames URI
    as a link and render a map for the selected location.'''
    # (inspired by )

    def render(self, name, value, attrs=None):
        widget = super(GeonamesLookupWidget, self).render(name, value, attrs)
        return mark_safe((u'

' +
            u'%s
%s
') % \
            (widget, value or '', value or ''))

class PlaceAdminForm(forms.ModelForm):
    '''Custom model form for Place editing, used to add geonames lookup.'''

    #: add a hidden field to pass in a mapbox access token from local settings
    mapbox_token = forms.CharField(initial=getattr(settings, 'MAPBOX_ACCESS_TOKEN', ''),
        widget=forms.HiddenInput)

    class Meta:
        model = Place
        exclude = []
        widgets = {
            'geonames_id': GeonamesLookupWidget(url='places:geonames-autocomplete',
                attrs={'data-placeholder': 'Type location name to search...',
                       'data-minimum-input-length': 3})
        }

class PlaceAdmin(admin.ModelAdmin):
    form = PlaceAdminForm
    list_display = ('name', 'geonames_id', 'has_notes')
    fields = ('name', 'geonames_id', 'latitude', 'longitude', 'notes',
        'mapbox_token')
    search_fields = ('name', 'notes', 'geonames_id')
    class Media:
        static_url = getattr(settings, 'STATIC_URL')
        css = {
            'all': ['https://unpkg.com/leaflet@1.0.2/dist/leaflet.css',
                    'admin/geonames.css']
        }
        js = ['admin/geonames-lookup.js',
            'https://unpkg.com/leaflet@1.0.2/dist/leaflet.js']

admin.site.register(Place, PlaceAdmin)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import datetime

from django.conf import settings
from django.contrib.auth.models import AbstractUser
from django.db import models
from django.utils.translation import ugettext as _
from django.utils import timezone

# If this list is changed, remember to check that the year property on
# OnlineUser is still correct!
FIELD_OF_STUDY_CHOICES = [
    (0, _(u'Gjest')),
    (1, _(u'Bachelor i Informatikk (BIT)')),
    # master degrees take up the interval [10,30]
    (10, _(u'Software (SW)')),
    (11, _(u'Informasjonsforvaltning (DIF)')),
    (12, _(u'Komplekse Datasystemer (KDS)')),
    (13, _(u'Spillteknologi (SPT)')),
    (14, _(u'Intelligente Systemer (IRS)')),
    (15, _(u'Helseinformatikk (MSMEDTEK)')),
    (30, _(u'Annen mastergrad')),
    (80, _(u'PhD')),
    (90, _(u'International')),
    (100, _(u'Annet Onlinemedlem')),
]

class OnlineUser(AbstractUser):

    IMAGE_FOLDER = "images/profiles"
    IMAGE_EXTENSIONS = ['.jpg', '.jpeg', '.gif', '.png']

    # Online related fields
    field_of_study = models.SmallIntegerField(_(u"studieretning"), choices=FIELD_OF_STUDY_CHOICES, default=0)
    started_date = models.DateField(_(u"startet studie"), default=timezone.now().date())
    compiled = models.BooleanField(_(u"kompilert"), default=False)

    # Email
    infomail = models.BooleanField(_(u"vil ha infomail"), default=True)

    # Address
    phone_number = models.CharField(_(u"telefonnummer"), max_length=20, blank=True, null=True)
    address = models.CharField(_(u"adresse"), max_length=30, blank=True, null=True)
    zip_code = models.CharField(_(u"postnummer"), max_length=4, blank=True, null=True)

    # Other
    allergies = models.TextField(_(u"allergier"), blank=True, null=True)
    mark_rules = models.BooleanField(_(u"godtatt prikkeregler"), default=False)
    rfid = models.CharField(_(u"RFID"), max_length=50, blank=True, null=True)
    nickname = models.CharField(_(u"nickname"), max_length=50, blank=True, null=True)
    website = models.URLField(_(u"hjemmeside"), blank=True, null=True)

    image = models.ImageField(_(u"bilde"), max_length=200, upload_to=IMAGE_FOLDER, blank=True, null=True,
                              default=settings.DEFAULT_PROFILE_PICTURE_URL)

    # NTNU credentials
    ntnu_username = models.CharField(_(u"NTNU-brukernavn"), max_length=10, blank=True, null=True, unique=True)

    # TODO checkbox for forwarding of @online.ntnu.no mail

    @property
    def is_member(self):
        """
        Returns true if the User object is associated with Online.
        """
        if AllowedUsername.objects.filter(username=self.ntnu_username).filter(expiration_date__gte=timezone.now()).count() > 0:
            return True
        return False

    def get_full_name(self):
        """
        Returns the first_name plus the last_name, with a space in between.
        """
        full_name = u'%s %s' % (self.first_name, self.last_name)
        return full_name.strip()

    def get_email(self):
        return self.get_emails().filter(primary = True)[0]

    def get_emails(self):
        return Email.objects.all().filter(user = self)

    @property
    def year(self):
        today = timezone.now().date()
        started = self.started_date

        # We say that a year is 360 days incase we are a bit slower to
        # add users one year.
        year = ((today - started).days / 360) + 1

        if self.field_of_study == 0 or self.field_of_study == 100:  # others
            return 0
        # dont return a bachelor student as 4th or 5th grade
        elif self.field_of_study == 1:  # bachelor
            if year > 3:
                return 3
            return year
        elif 10 <= self.field_of_study <= 30:  # 10-29 is considered master
            if year >= 2:
                return 5
            return 4
        elif self.field_of_study == 80:  # phd
            return year + 5
        elif self.field_of_study == 90:  # international
            if year == 1:
                return 1
            return 4
        # If user's field of study is not matched by any of these tests, return -1
        else:
            return -1

    def __unicode__(self):
        return self.get_full_name()

    class Meta:
        ordering = ['first_name', 'last_name']
        verbose_name = _(u"brukerprofil")
        verbose_name_plural = _(u"brukerprofiler")

class Email(models.Model):
    user = models.ForeignKey(OnlineUser, related_name="email_user")
    email = models.EmailField(_(u"epostadresse"), unique=True)
    primary = models.BooleanField(_(u"primær"), default=False)
    verified = models.BooleanField(_(u"verifisert"), default=False, editable=False)

    def __unicode__(self):
        return self.email

    class Meta:
        verbose_name = _(u"epostadresse")
        verbose_name_plural = _(u"epostadresser")

class RegisterToken(models.Model):
    user = models.ForeignKey(OnlineUser, related_name="register_user")
    email = models.EmailField(_(u"epost"), max_length=254)
    token = models.CharField(_(u"token"), max_length=32)
    created = models.DateTimeField(_(u"opprettet dato"), editable=False, auto_now_add=True)

    @property
    def is_valid(self):
        valid_period = datetime.timedelta(days=1)
        now = timezone.now()
        return now < self.created + valid_period

class AllowedUsername(models.Model):
    """
    Holds usernames that are considered valid members of Online and the time they expire.
    """
    username = models.CharField(_(u"NTNU-brukernavn"), max_length=10, unique=True)
    registered = models.DateField(_(u"registrert"))
    note = models.CharField(_(u"notat"), max_length=100)
    description = models.TextField(_(u"beskrivelse"), blank=True, null=True)
    expiration_date = models.DateField(_(u"utløpsdato"))

    @property
    def is_active(self):
        return timezone.now().date() < self.expiration_date

    def __unicode__(self):
        return self.username

    class Meta:
        verbose_name = _(u"medlemsregister")
        verbose_name_plural = _(u"medlemsregister")
        ordering = (u"username",)

<commit_msg>Changed name of single AllowedUsername entry.<commit_after># -*- coding: utf-8 -*-

import datetime

from django.conf import settings
from django.contrib.auth.models import AbstractUser
from django.db import models
from django.utils.translation import ugettext as _
from django.utils import timezone

# If this list is changed, remember to check that the year property on
# OnlineUser is still correct!
FIELD_OF_STUDY_CHOICES = [
    (0, _(u'Gjest')),
    (1, _(u'Bachelor i Informatikk (BIT)')),
    # master degrees take up the interval [10,30]
    (10, _(u'Software (SW)')),
    (11, _(u'Informasjonsforvaltning (DIF)')),
    (12, _(u'Komplekse Datasystemer (KDS)')),
    (13, _(u'Spillteknologi (SPT)')),
    (14, _(u'Intelligente Systemer (IRS)')),
    (15, _(u'Helseinformatikk (MSMEDTEK)')),
    (30, _(u'Annen mastergrad')),
    (80, _(u'PhD')),
    (90, _(u'International')),
    (100, _(u'Annet Onlinemedlem')),
]

class OnlineUser(AbstractUser):

    IMAGE_FOLDER = "images/profiles"
    IMAGE_EXTENSIONS = ['.jpg', '.jpeg', '.gif', '.png']

    # Online related fields
    field_of_study = models.SmallIntegerField(_(u"studieretning"), choices=FIELD_OF_STUDY_CHOICES, default=0)
    started_date = models.DateField(_(u"startet studie"), default=timezone.now().date())
    compiled = models.BooleanField(_(u"kompilert"), default=False)

    # Email
    infomail = models.BooleanField(_(u"vil ha infomail"), default=True)

    # Address
    phone_number = models.CharField(_(u"telefonnummer"), max_length=20, blank=True, null=True)
    address = models.CharField(_(u"adresse"), max_length=30, blank=True, null=True)
    zip_code = models.CharField(_(u"postnummer"), max_length=4, blank=True, null=True)

    # Other
    allergies = models.TextField(_(u"allergier"), blank=True, null=True)
    mark_rules = models.BooleanField(_(u"godtatt prikkeregler"), default=False)
    rfid = models.CharField(_(u"RFID"), max_length=50, blank=True, null=True)
    nickname = models.CharField(_(u"nickname"), max_length=50, blank=True, null=True)
    website = models.URLField(_(u"hjemmeside"), blank=True, null=True)

    image = models.ImageField(_(u"bilde"), max_length=200, upload_to=IMAGE_FOLDER, blank=True, null=True,
                              default=settings.DEFAULT_PROFILE_PICTURE_URL)

    # NTNU credentials
    ntnu_username = models.CharField(_(u"NTNU-brukernavn"), max_length=10, blank=True, null=True, unique=True)

    # TODO checkbox for forwarding of @online.ntnu.no mail

    @property
    def is_member(self):
        """
        Returns true if the User object is associated with Online.
        """
        if AllowedUsername.objects.filter(username=self.ntnu_username).filter(expiration_date__gte=timezone.now()).count() > 0:
            return True
        return False

    def get_full_name(self):
        """
        Returns the first_name plus the last_name, with a space in between.
        """
        full_name = u'%s %s' % (self.first_name, self.last_name)
        return full_name.strip()

    def get_email(self):
        return self.get_emails().filter(primary = True)[0]

    def get_emails(self):
        return Email.objects.all().filter(user = self)

    @property
    def year(self):
        today = timezone.now().date()
        started = self.started_date

        # We say that a year is 360 days incase we are a bit slower to
        # add users one year.
        year = ((today - started).days / 360) + 1

        if self.field_of_study == 0 or self.field_of_study == 100:  # others
            return 0
        # dont return a bachelor student as 4th or 5th grade
        elif self.field_of_study == 1:  # bachelor
            if year > 3:
                return 3
            return year
        elif 10 <= self.field_of_study <= 30:  # 10-29 is considered master
            if year >= 2:
                return 5
            return 4
        elif self.field_of_study == 80:  # phd
            return year + 5
        elif self.field_of_study == 90:  # international
            if year == 1:
                return 1
            return 4
        # If user's field of study is not matched by any of these tests, return -1
        else:
            return -1

    def __unicode__(self):
        return self.get_full_name()

    class Meta:
        ordering = ['first_name', 'last_name']
        verbose_name = _(u"brukerprofil")
        verbose_name_plural = _(u"brukerprofiler")

class Email(models.Model):
    user = models.ForeignKey(OnlineUser, related_name="email_user")
    email = models.EmailField(_(u"epostadresse"), unique=True)
    primary = models.BooleanField(_(u"primær"), default=False)
    verified = models.BooleanField(_(u"verifisert"), default=False, editable=False)

    def __unicode__(self):
        return self.email

    class Meta:
        verbose_name = _(u"epostadresse")
        verbose_name_plural = _(u"epostadresser")

class RegisterToken(models.Model):
    user = models.ForeignKey(OnlineUser, related_name="register_user")
    email = models.EmailField(_(u"epost"), max_length=254)
    token = models.CharField(_(u"token"), max_length=32)
    created = models.DateTimeField(_(u"opprettet dato"), editable=False, auto_now_add=True)

    @property
    def is_valid(self):
        valid_period = datetime.timedelta(days=1)
        now = timezone.now()
        return now < self.created + valid_period

class AllowedUsername(models.Model):
    """
    Holds usernames that are considered valid members of Online and the time they expire.
    """
    username = models.CharField(_(u"NTNU-brukernavn"), max_length=10, unique=True)
    registered = models.DateField(_(u"registrert"))
    note = models.CharField(_(u"notat"), max_length=100)
    description = models.TextField(_(u"beskrivelse"), blank=True, null=True)
    expiration_date = models.DateField(_(u"utløpsdato"))

    @property
    def is_active(self):
        return timezone.now().date() < self.expiration_date

    def __unicode__(self):
        return self.username

    class Meta:
        verbose_name = _(u"medlem")
        verbose_name_plural = _(u"medlemsregister")
        ordering = (u"username",)

<|endoftext|>


<commit_before>from django.db import models
from django.utils.translation import ugettext_lazy as _
from filebrowser.fields import FileBrowseField

class Company(models.Model):

    IMAGE_FOLDER = "images/companies"
    IMAGE_EXTENSIONS = ['.jpg', '.jpeg', '.gif', '.png', '.tif', '.tiff']

    name = models.CharField(_(u"bedriftsnavn"), max_length=100)
    short_description = models.TextField(_(u"kort beskrivelse"), max_length=200)
    long_description = models.TextField(_(u"utdypende beskrivelse"), blank=True, null=True)
    image = FileBrowseField(_(u"bilde"),
        max_length=200, directory=IMAGE_FOLDER,
        extensions=IMAGE_EXTENSIONS, null=False, blank=False)
    site = models.URLField(_(u"hjemmeside"))
    email_address = models.EmailField(_(u"epostaddresse"), max_length=75, blank=True, null=True)
    phone_number = models.CharField(_(u"telefonnummer"), max_length=20, blank=True, null=True)

    def __unicode__(self):
        return self.name

    class Meta:
        verbose_name = _(u"Bedrift")
        verbose_name_plural = _(u"Bedrifter")
<commit_msg>Added support for svg for company logo<commit_after>from django.db import models
from django.utils.translation import ugettext_lazy as _
from filebrowser.fields import FileBrowseField

class Company(models.Model):

    IMAGE_FOLDER = "images/companies"
    IMAGE_EXTENSIONS = ['.jpg', '.jpeg', '.gif', '.png', '.tif', '.tiff', '.svg']

    name = models.CharField(_(u"bedriftsnavn"), max_length=100)
    short_description = models.TextField(_(u"kort beskrivelse"), max_length=200)
    long_description = models.TextField(_(u"utdypende beskrivelse"), blank=True, null=True)
    image = FileBrowseField(_(u"bilde"),
        max_length=200, directory=IMAGE_FOLDER,
        extensions=IMAGE_EXTENSIONS, null=False, blank=False)
    site = models.URLField(_(u"hjemmeside"))
    email_address = models.EmailField(_(u"epostaddresse"), max_length=75, blank=True, null=True)
    phone_number = models.CharField(_(u"telefonnummer"), max_length=20, blank=True, null=True)

    def __unicode__(self):
        return self.name

    class Meta:
        verbose_name = _(u"Bedrift")
        verbose_name_plural = _(u"Bedrifter")
<|endoftext|>


<commit_before>import sys
sys.path.append('../../')
from lib.cilok import urlEncode16,tokenuri,setTTL,keyuri
from lib.sampeu import getWMTS
from apps.models import calendar
from apps.templates import batik

class Controller(object):
	def home(self,uridt='null'):
		provinsi = 'gorontalo'
		provloc = '122.410923, 0.723049'
		mapzoom = '9'
		kabkotcord = [
		'122.271955, 0.718607',
		'122.465735,0.728542',
		'121.52287,0.707775',
		'123.286145, 0.580945',
		'122.966074, 0.837543',
		'121.993323, 0.843888'
		]
		listkabkot = [
		'%7501%','%7502%','%7503%','%7504%','%7505%'
		'%7571%'
		]
		batik.provinsi(provinsi,listkabkot,provloc,mapzoom,kabkotcord)
		cal = calendar.Calendar()
		dt = {}
		for kabkot in listkabkot:
			dt[kabkot]=cal.getYearCountKabKot(str(int(kabkot[1:3])),str(int(kabkot[3:5])),uridt)
		cal.close()
		dt['%WMTS%']=getWMTS()
		dt['%PERIODE%']=uridt
		dt['%LAMAN INDONESIA%']=urlEncode16(keyuri+'%peta%home'+'%'+uridt)
		dt['%TAHUN SEBELUMNYA%']=urlEncode16(keyuri+'%'+provinsi+'%home'+'%'+str(int(uridt)-1))
		dt['%TAHUN SETELAHNYA%']=urlEncode16(keyuri+'%'+provinsi+'%home'+'%'+str(int(uridt)+1))
		return dt<commit_msg>menambahkan gorontalo<commit_after>import sys
sys.path.append('../../')
from lib.cilok import urlEncode16,tokenuri,setTTL,keyuri
from lib.sampeu import getWMTS
from apps.models import calendar
from apps.templates import batik

class Controller(object):
	def home(self,uridt='null'):
		provinsi = 'gorontalo'
		provloc = '122.410923, 0.723049'
		mapzoom = '9'
		kabkotcord = [
		'122.26538870000002, 0.7013419',
		# '122.80844960000002, 0.5692733',
		# '121.71954589999996,0.7055277999999999',
		# '123.34861469999998, 0.5657884999999999',
		# '122.49200880000001, 0.9252647',
		# '121.993323, 0.843888'
		]
		listkabkot = [
		'%7501%','%7502%','%7503%','%7504%','%7505%'
		'%7571%'
		]
		batik.provinsi(provinsi,listkabkot,provloc,mapzoom,kabkotcord)
		cal = calendar.Calendar()
		dt = {}
		for kabkot in listkabkot:
			dt[kabkot]=cal.getYearCountKabKot(str(int(kabkot[1:3])),str(int(kabkot[3:5])),uridt)
		cal.close()
		dt['%WMTS%']=getWMTS()
		dt['%PERIODE%']=uridt
		dt['%LAMAN INDONESIA%']=urlEncode16(keyuri+'%peta%home'+'%'+uridt)
		dt['%TAHUN SEBELUMNYA%']=urlEncode16(keyuri+'%'+provinsi+'%home'+'%'+str(int(uridt)-1))
		dt['%TAHUN SETELAHNYA%']=urlEncode16(keyuri+'%'+provinsi+'%home'+'%'+str(int(uridt)+1))
		return dt<|endoftext|>


<commit_before>from types import StringTypes
import time
import re

from PLC.Faults import *
from PLC.Parameter import Parameter
from PLC.Debug import profile
from PLC.Table import Row, Table
from PLC.SliceInstantiations import SliceInstantiations
from PLC.Nodes import Node, Nodes
from PLC.ForeignNodes import ForeignNode, ForeignNodes
import PLC.Persons

class Slice(Row):
    """
    Representation of a row in the slices table. To use, optionally
    instantiate with a dict of values. Update as you would a
    dict. Commit to the database with sync().To use, instantiate
    with a dict of values.
    """

    table_name = 'slices'
    primary_key = 'slice_id'
    fields = {
        'slice_id': Parameter(int, "Slice identifier"),
        'site_id': Parameter(int, "Identifier of the site to which this slice belongs"),
        'name': Parameter(str, "Slice name", max = 32),
        'instantiation': Parameter(str, "Slice instantiation state"),
        'url': Parameter(str, "URL further describing this slice", max = 254, nullok = True),
        'description': Parameter(str, "Slice description", max = 2048, nullok = True),
        'max_nodes': Parameter(int, "Maximum number of nodes that can be assigned to this slice"),
        'creator_person_id': Parameter(int, "Identifier of the account that created this slice"),
        'created': Parameter(int, "Date and time when slice was created, in seconds since UNIX epoch", ro = True),
        'expires': Parameter(int, "Date and time when slice expires, in seconds since UNIX epoch"),
        'node_ids': Parameter([int], "List of nodes in this slice", ro = True),
        'person_ids': Parameter([int], "List of accounts that can use this slice", ro = True),
        'slice_attribute_ids': Parameter([int], "List of slice attributes", ro = True),
        }

    def validate_name(self, name):
        # N.B.: Responsibility of the caller to ensure that login_base
        # portion of the slice name corresponds to a valid site, if
        # desired.

        # 1. Lowercase.
        # 2. Begins with login_base (only letters).
        # 3. Then single underscore after login_base.
        # 4. Then letters, numbers, or underscores.
        good_name = r'^[a-z]+_[a-z0-9_]+$'
        if not name or \
           not re.match(good_name, name):
            raise PLCInvalidArgument, "Invalid slice name"

        conflicts = Slices(self.api, [name])
        for slice_id, slice in conflicts.iteritems():
            if 'slice_id' not in self or self['slice_id'] != slice_id:
                raise PLCInvalidArgument, "Slice name already in use"

        return name

    def validate_instantiation(self, instantiation):
        instantiations = SliceInstantiations(self.api)
        if instantiation not in instantiations:
            raise PLCInvalidArgument, "No such instantiation state"

        return instantiation

    def validate_expires(self, expires):
        # N.B.: Responsibility of the caller to ensure that expires is
        # not too far into the future.
        if expires < time.time():
            raise PLCInvalidArgument, "Expiration date must be in the future"

        return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(expires))

    def validate_creator_person_id(self, person_id):
        persons = PLC.Persons.Persons(self.api, [person_id])
        if not persons:
            raise PLCInvalidArgument, "Invalid creator"

        return person_id

    def add_person(self, person, commit = True):
        """
        Add person to existing slice.
        """

        assert 'slice_id' in self
        assert isinstance(person, PLC.Persons.Person)
        assert 'person_id' in person

        slice_id = self['slice_id']
        person_id = person['person_id']

        if person_id not in self['person_ids']:
            assert slice_id not in person['slice_ids']

            self.api.db.do("INSERT INTO slice_person (person_id, slice_id)" \
                           " VALUES(%(person_id)d, %(slice_id)d)",
                           locals())

            if commit:
                self.api.db.commit()

            self['person_ids'].append(person_id)
            person['slice_ids'].append(slice_id)

    def remove_person(self, person, commit = True):
        """
        Remove person from existing slice.
        """

        assert 'slice_id' in self
        assert isinstance(person, PLC.Persons.Person)
        assert 'person_id' in person

        slice_id = self['slice_id']
        person_id = person['person_id']

        if person_id in self['person_ids']:
            assert slice_id in person['slice_ids']

            self.api.db.do("DELETE FROM slice_person" \
                           " WHERE person_id = %(person_id)d" \
                           " AND slice_id = %(slice_id)d",
                           locals())

            if commit:
                self.api.db.commit()

            self['person_ids'].remove(person_id)
            person['slice_ids'].remove(slice_id)

    def add_node(self, node, is_foreign_node = False, commit = True):
        """
        Add node to existing slice.
        """

        assert 'slice_id' in self
        if not is_foreign_node:
            assert isinstance(node, Node)
        else:
            assert isinstance(node, ForeignNode)
        assert 'node_id' in node

        slice_id = self['slice_id']
        node_id = node['node_id']

        if node_id not in self['node_ids']:
            assert slice_id not in node['slice_ids']

            self.api.db.do("INSERT INTO slice_node (node_id, slice_id)" \
                           " VALUES(%(node_id)d, %(slice_id)d)",
                           locals())

            if commit:
                self.api.db.commit()

            self['node_ids'].append(node_id)
            node['slice_ids'].append(slice_id)

    def remove_node(self, node, commit = True):
        """
        Remove node from existing slice.
        """

        assert 'slice_id' in self
        assert isinstance(node, Node)
        assert 'node_id' in node

        slice_id = self['slice_id']
        node_id = node['node_id']

        if node_id in self['node_ids']:
            assert slice_id in node['slice_ids']

            self.api.db.do("DELETE FROM slice_node" \
                           " WHERE node_id = %(node_id)d" \
                           " AND slice_id = %(slice_id)d",
                           locals())

            if commit:
                self.api.db.commit()

            self['node_ids'].remove(node_id)
            node['slice_ids'].remove(slice_id)

    def sync(self, commit = True):
        """
        Add or update a slice.
        """

        # Before a new slice is added, delete expired slices
        if 'slice_id' not in self:
            expired = Slices(self.api, expires = -int(time.time())).values()
            for slice in expired:
                slice.delete(commit)

        Row.sync(self, commit)

    def delete(self, commit = True):
        """
        Delete existing slice.
        """

        assert 'slice_id' in self

        # Clean up miscellaneous join tables
        for table in ['slice_node', 'slice_person', 'slice_attribute']:
            self.api.db.do("DELETE FROM %s" \
                           " WHERE slice_id = %d" % \
                           (table, self['slice_id']), self)

        # Mark as deleted
        self['is_deleted'] = True
        self.sync(commit)

class Slices(Table):
    """
    Representation of row(s) from the slices table in the
    database.
    """

    def __init__(self, api, slice_id_or_name_list = None, expires = int(time.time())):
        self.api = api

        sql = "SELECT %s FROM view_slices WHERE is_deleted IS False" % \
              ", ".join(Slice.fields)

        if expires is not None:
            if expires >= 0:
                sql += " AND expires > %(expires)d"
            else:
                expires = -expires
                sql += " AND expires < %(expires)d"

        if slice_id_or_name_list:
            # Separate the list into integers and strings
            slice_ids = filter(lambda slice_id: isinstance(slice_id, (int, long)),
                               slice_id_or_name_list)
            names = filter(lambda name: isinstance(name, StringTypes),
                           slice_id_or_name_list)
            sql += " AND (False"
            if slice_ids:
                sql += " OR slice_id IN (%s)" % ", ".join(map(str, slice_ids))
            if names:
                sql += " OR name IN (%s)" % ", ".join(api.db.quote(names))
            sql += ")"

        rows = self.api.db.selectall(sql, locals())

        for row in rows:
            self[row['slice_id']] = slice = Slice(api, row)
            for aggregate in 'node_ids', 'person_ids', 'slice_attribute_ids':
                if not slice.has_key(aggregate) or slice[aggregate] is None:
                    slice[aggregate] = []
                else:
                    slice[aggregate] = map(int, slice[aggregate].split(','))
<commit_msg>- don't validate creator_person_id; if creator is deleted, then slice   cannot be updated - use Filter to select rows - psycopg2 obviates the need for manual aggregation<commit_after>from types import StringTypes
import time
import re

from PLC.Faults import *
from PLC.Parameter import Parameter
from PLC.Filter import Filter
from PLC.Debug import profile
from PLC.Table import Row, Table
from PLC.SliceInstantiations import SliceInstantiations
from PLC.Nodes import Node, Nodes
from PLC.ForeignNodes import ForeignNode, ForeignNodes
import PLC.Persons

class Slice(Row):
    """
    Representation of a row in the slices table. To use, optionally
    instantiate with a dict of values. Update as you would a
    dict. Commit to the database with sync().To use, instantiate
    with a dict of values.
    """

    table_name = 'slices'
    primary_key = 'slice_id'
    fields = {
        'slice_id': Parameter(int, "Slice identifier"),
        'site_id': Parameter(int, "Identifier of the site to which this slice belongs"),
        'name': Parameter(str, "Slice name", max = 32),
        'instantiation': Parameter(str, "Slice instantiation state"),
        'url': Parameter(str, "URL further describing this slice", max = 254, nullok = True),
        'description': Parameter(str, "Slice description", max = 2048, nullok = True),
        'max_nodes': Parameter(int, "Maximum number of nodes that can be assigned to this slice"),
        'creator_person_id': Parameter(int, "Identifier of the account that created this slice"),
        'created': Parameter(int, "Date and time when slice was created, in seconds since UNIX epoch", ro = True),
        'expires': Parameter(int, "Date and time when slice expires, in seconds since UNIX epoch"),
        'node_ids': Parameter([int], "List of nodes in this slice", ro = True),
        'person_ids': Parameter([int], "List of accounts that can use this slice", ro = True),
        'slice_attribute_ids': Parameter([int], "List of slice attributes", ro = True),
        }

    def validate_name(self, name):
        # N.B.: Responsibility of the caller to ensure that login_base
        # portion of the slice name corresponds to a valid site, if
        # desired.

        # 1. Lowercase.
        # 2. Begins with login_base (only letters).
        # 3. Then single underscore after login_base.
        # 4. Then letters, numbers, or underscores.
        good_name = r'^[a-z]+_[a-z0-9_]+$'
        if not name or \
           not re.match(good_name, name):
            raise PLCInvalidArgument, "Invalid slice name"

        conflicts = Slices(self.api, [name])
        for slice_id, slice in conflicts.iteritems():
            if 'slice_id' not in self or self['slice_id'] != slice_id:
                raise PLCInvalidArgument, "Slice name already in use"

        return name

    def validate_instantiation(self, instantiation):
        instantiations = SliceInstantiations(self.api)
        if instantiation not in instantiations:
            raise PLCInvalidArgument, "No such instantiation state"

        return instantiation

    def validate_expires(self, expires):
        # N.B.: Responsibility of the caller to ensure that expires is
        # not too far into the future.
        if expires < time.time():
            raise PLCInvalidArgument, "Expiration date must be in the future"

        return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(expires))

    def add_person(self, person, commit = True):
        """
        Add person to existing slice.
        """

        assert 'slice_id' in self
        assert isinstance(person, PLC.Persons.Person)
        assert 'person_id' in person

        slice_id = self['slice_id']
        person_id = person['person_id']

        if person_id not in self['person_ids']:
            assert slice_id not in person['slice_ids']

            self.api.db.do("INSERT INTO slice_person (person_id, slice_id)" \
                           " VALUES(%(person_id)d, %(slice_id)d)",
                           locals())

            if commit:
                self.api.db.commit()

            self['person_ids'].append(person_id)
            person['slice_ids'].append(slice_id)

    def remove_person(self, person, commit = True):
        """
        Remove person from existing slice.
        """

        assert 'slice_id' in self
        assert isinstance(person, PLC.Persons.Person)
        assert 'person_id' in person

        slice_id = self['slice_id']
        person_id = person['person_id']

        if person_id in self['person_ids']:
            assert slice_id in person['slice_ids']

            self.api.db.do("DELETE FROM slice_person" \
                           " WHERE person_id = %(person_id)d" \
                           " AND slice_id = %(slice_id)d",
                           locals())

            if commit:
                self.api.db.commit()

            self['person_ids'].remove(person_id)
            person['slice_ids'].remove(slice_id)

    def add_node(self, node, is_foreign_node = False, commit = True):
        """
        Add node to existing slice.
        """

        assert 'slice_id' in self
        if not is_foreign_node:
            assert isinstance(node, Node)
        else:
            assert isinstance(node, ForeignNode)
        assert 'node_id' in node

        slice_id = self['slice_id']
        node_id = node['node_id']

        if node_id not in self['node_ids']:
            assert slice_id not in node['slice_ids']

            self.api.db.do("INSERT INTO slice_node (node_id, slice_id)" \
                           " VALUES(%(node_id)d, %(slice_id)d)",
                           locals())

            if commit:
                self.api.db.commit()

            self['node_ids'].append(node_id)
            node['slice_ids'].append(slice_id)

    def remove_node(self, node, commit = True):
        """
        Remove node from existing slice.
        """

        assert 'slice_id' in self
        assert isinstance(node, Node)
        assert 'node_id' in node

        slice_id = self['slice_id']
        node_id = node['node_id']

        if node_id in self['node_ids']:
            assert slice_id in node['slice_ids']

            self.api.db.do("DELETE FROM slice_node" \
                           " WHERE node_id = %(node_id)d" \
                           " AND slice_id = %(slice_id)d",
                           locals())

            if commit:
                self.api.db.commit()

            self['node_ids'].remove(node_id)
            node['slice_ids'].remove(slice_id)

    def sync(self, commit = True):
        """
        Add or update a slice.
        """

        # Before a new slice is added, delete expired slices
        if 'slice_id' not in self:
            expired = Slices(self.api, expires = -int(time.time())).values()
            for slice in expired:
                slice.delete(commit)

        Row.sync(self, commit)

    def delete(self, commit = True):
        """
        Delete existing slice.
        """

        assert 'slice_id' in self

        # Clean up miscellaneous join tables
        for table in ['slice_node', 'slice_person', 'slice_attribute']:
            self.api.db.do("DELETE FROM %s" \
                           " WHERE slice_id = %d" % \
                           (table, self['slice_id']), self)

        # Mark as deleted
        self['is_deleted'] = True
        self.sync(commit)

class Slices(Table):
    """
    Representation of row(s) from the slices table in the
    database.
    """

    def __init__(self, api, slice_filter = None, expires = int(time.time())):
        Table.__init__(self, api, Slice)

        sql = "SELECT %s FROM view_slices WHERE is_deleted IS False" % \
              ", ".join(Slice.fields)

        if expires is not None:
            if expires >= 0:
                sql += " AND expires > %(expires)d"
            else:
                expires = -expires
                sql += " AND expires < %(expires)d"

        if slice_filter is not None:
            if isinstance(slice_filter, list):
                # Separate the list into integers and strings
                ints = filter(lambda x: isinstance(x, (int, long)), slice_filter)
                strs = filter(lambda x: isinstance(x, StringTypes), slice_filter)
                slice_filter = Filter(Slice.fields, {'slice_id': ints, 'name': strs})
                sql += " AND (%s)" % slice_filter.sql(api, "OR")
            elif isinstance(slice_filter, dict):
                slice_filter = Filter(Slice.fields, slice_filter)
                sql += " AND (%s)" % slice_filter.sql(api, "AND")

        self.selectall(sql, locals())
<|endoftext|>


<commit_before>from types import StringTypes
import time
import re

from PLC.Faults import *
from PLC.Parameter import Parameter
from PLC.Filter import Filter
from PLC.Debug import profile
from PLC.Table import Row, Table
from PLC.SliceInstantiations import SliceInstantiation, SliceInstantiations
from PLC.Nodes import Node
from PLC.Persons import Person, Persons

class Slice(Row):
    """
    Representation of a row in the slices table. To use, optionally
    instantiate with a dict of values. Update as you would a
    dict. Commit to the database with sync().To use, instantiate
    with a dict of values.
    """

    table_name = 'slices'
    primary_key = 'slice_id'
    join_tables = ['slice_node', 'slice_person', 'slice_attribute', 'peer_slice', 'node_slice_whitelist']
    fields = {
        'slice_id': Parameter(int, "Slice identifier"),
        'site_id': Parameter(int, "Identifier of the site to which this slice belongs"),
        'name': Parameter(str, "Slice name", max = 32),
        'instantiation': Parameter(str, "Slice instantiation state"),
        'url': Parameter(str, "URL further describing this slice", max = 254, nullok = True),
        'description': Parameter(str, "Slice description", max = 2048, nullok = True),
        'max_nodes': Parameter(int, "Maximum number of nodes that can be assigned to this slice"),
        'creator_person_id': Parameter(int, "Identifier of the account that created this slice"),
        'created': Parameter(int, "Date and time when slice was created, in seconds since UNIX epoch", ro = True),
        'expires': Parameter(int, "Date and time when slice expires, in seconds since UNIX epoch"),
        'node_ids': Parameter([int], "List of nodes in this slice", ro = True),
        'person_ids': Parameter([int], "List of accounts that can use this slice", ro = True),
        'slice_attribute_ids': Parameter([int], "List of slice attributes", ro = True),
        'peer_id': Parameter(int, "Peer to which this slice belongs", nullok = True),
        'peer_slice_id': Parameter(int, "Foreign slice identifier at peer", nullok = True),
        }
    # for Cache
    class_key = 'name'
    foreign_fields = ['instantiation', 'url', 'description', 'max_nodes', 'expires']
    foreign_xrefs = [
        {'field': 'node_ids' ,         'class': 'Node',   'table': 'slice_node' },
	{'field': 'person_ids',        'class': 'Person', 'table': 'slice_person'},
	{'field': 'creator_person_id', 'class': 'Person', 'table': 'unused-on-direct-refs'},
        {'field': 'site_id',           'class': 'Site',   'table': 'unused-on-direct-refs'},
    ]
    # forget about this one, it is read-only anyway
    # handling it causes Cache to re-sync all over again
    # 'created'

    def validate_name(self, name):
        # N.B.: Responsibility of the caller to ensure that login_base
        # portion of the slice name corresponds to a valid site, if
        # desired.

        # 1. Lowercase.
        # 2. Begins with login_base (letters or numbers).
        # 3. Then single underscore after login_base.
        # 4. Then letters, numbers, or underscores.
        good_name = r'^[a-z0-9]+_[a-zA-Z0-9_]+$'
        if not name or \
           not re.match(good_name, name):
            raise PLCInvalidArgument, "Invalid slice name"

        conflicts = Slices(self.api, [name])
        for slice in conflicts:
            if 'slice_id' not in self or self['slice_id'] != slice['slice_id']:
                raise PLCInvalidArgument, "Slice name already in use, %s"%name

        return name

    def validate_instantiation(self, instantiation):
        instantiations = [row['instantiation'] for row in SliceInstantiations(self.api)]
        if instantiation not in instantiations:
            raise PLCInvalidArgument, "No such instantiation state"

        return instantiation

    validate_created = Row.validate_timestamp

    def validate_expires(self, expires):
        # N.B.: Responsibility of the caller to ensure that expires is
        # not too far into the future.
        check_future = not ('is_deleted' in self and self['is_deleted'])
        return Row.validate_timestamp(self, expires, check_future = check_future)

    add_person = Row.add_object(Person, 'slice_person')
    remove_person = Row.remove_object(Person, 'slice_person')

    add_node = Row.add_object(Node, 'slice_node')
    remove_node = Row.remove_object(Node, 'slice_node')

    add_to_node_whitelist = Row.add_object(Node, 'node_slice_whitelist')
    delete_from_node_whitelist = Row.remove_object(Node, 'node_slice_whitelist')

    def sync(self, commit = True):
        """
        Add or update a slice.
        """

        # Before a new slice is added, delete expired slices
        if 'slice_id' not in self:
            expired = Slices(self.api, expires = -int(time.time()))
            for slice in expired:
                slice.delete(commit)

        Row.sync(self, commit)

    def delete(self, commit = True):
        """
        Delete existing slice.
        """

        assert 'slice_id' in self

        # Clean up miscellaneous join tables
        for table in self.join_tables:
            self.api.db.do("DELETE FROM %s WHERE slice_id = %d" % \
                           (table, self['slice_id']))

        # Mark as deleted
        self['is_deleted'] = True
        self.sync(commit)

class Slices(Table):
    """
    Representation of row(s) from the slices table in the
    database.
    """

    def __init__(self, api, slice_filter = None, columns = None, expires = int(time.time())):
        Table.__init__(self, api, Slice, columns)

        sql = "SELECT %s FROM view_slices WHERE is_deleted IS False" % \
              ", ".join(self.columns)

        if expires is not None:
            if expires >= 0:
                sql += " AND expires > %d" % expires
            else:
                expires = -expires
                sql += " AND expires < %d" % expires

        if slice_filter is not None:
            if isinstance(slice_filter, (list, tuple, set)):
                # Separate the list into integers and strings
                ints = filter(lambda x: isinstance(x, (int, long)), slice_filter)
                strs = filter(lambda x: isinstance(x, StringTypes), slice_filter)
                slice_filter = Filter(Slice.fields, {'slice_id': ints, 'name': strs})
                sql += " AND (%s) %s" % slice_filter.sql(api, "OR")
            elif isinstance(slice_filter, dict):
                slice_filter = Filter(Slice.fields, slice_filter)
                sql += " AND (%s) %s" % slice_filter.sql(api, "AND")
            elif isinstance (slice_filter, StringTypes):
                slice_filter = Filter(Slice.fields, {'name':[slice_filter]})
                sql += " AND (%s) %s" % slice_filter.sql(api, "AND")
            elif isinstance (slice_filter, int):
                slice_filter = Filter(Slice.fields, {'slice_id':[slice_filter]})
                sql += " AND (%s) %s" % slice_filter.sql(api, "AND")
            else:
                raise PLCInvalidArgument, "Wrong slice filter %r"%slice_filter

        self.selectall(sql)
<commit_msg>Rationalize:  added associate functions<commit_after>from types import StringTypes
import time
import re

from PLC.Faults import *
from PLC.Parameter import Parameter, Mixed
from PLC.Filter import Filter
from PLC.Debug import profile
from PLC.Table import Row, Table
from PLC.SliceInstantiations import SliceInstantiation, SliceInstantiations
from PLC.Nodes import Node
from PLC.Persons import Person, Persons
from PLC.SliceAttributes import SliceAttribute

class Slice(Row):
    """
    Representation of a row in the slices table. To use, optionally
    instantiate with a dict of values. Update as you would a
    dict. Commit to the database with sync().To use, instantiate
    with a dict of values.
    """

    table_name = 'slices'
    primary_key = 'slice_id'
    join_tables = ['slice_node', 'slice_person', 'slice_attribute', 'peer_slice', 'node_slice_whitelist']
    fields = {
        'slice_id': Parameter(int, "Slice identifier"),
        'site_id': Parameter(int, "Identifier of the site to which this slice belongs"),
        'name': Parameter(str, "Slice name", max = 32),
        'instantiation': Parameter(str, "Slice instantiation state"),
        'url': Parameter(str, "URL further describing this slice", max = 254, nullok = True),
        'description': Parameter(str, "Slice description", max = 2048, nullok = True),
        'max_nodes': Parameter(int, "Maximum number of nodes that can be assigned to this slice"),
        'creator_person_id': Parameter(int, "Identifier of the account that created this slice"),
        'created': Parameter(int, "Date and time when slice was created, in seconds since UNIX epoch", ro = True),
        'expires': Parameter(int, "Date and time when slice expires, in seconds since UNIX epoch"),
        'node_ids': Parameter([int], "List of nodes in this slice", ro = True),
        'person_ids': Parameter([int], "List of accounts that can use this slice", ro = True),
        'slice_attribute_ids': Parameter([int], "List of slice attributes", ro = True),
        'peer_id': Parameter(int, "Peer to which this slice belongs", nullok = True),
        'peer_slice_id': Parameter(int, "Foreign slice identifier at peer", nullok = True),
        }
    related_fields = {
	'persons': [Mixed(Parameter(int, "Person identifier"),
			  Parameter(str, "Email address"))],
	'nodes': [Mixed(Parameter(int, "Node identifier"),
		        Parameter(str, "Fully qualified hostname"))]
   	}
    # for Cache
    class_key = 'name'
    foreign_fields = ['instantiation', 'url', 'description', 'max_nodes', 'expires']
    foreign_xrefs = [
        {'field': 'node_ids' ,         'class': 'Node',   'table': 'slice_node' },
	{'field': 'person_ids',        'class': 'Person', 'table': 'slice_person'},
	{'field': 'creator_person_id', 'class': 'Person', 'table': 'unused-on-direct-refs'},
        {'field': 'site_id',           'class': 'Site',   'table': 'unused-on-direct-refs'},
    ]
    # forget about this one, it is read-only anyway
    # handling it causes Cache to re-sync all over again
    # 'created'

    def validate_name(self, name):
        # N.B.: Responsibility of the caller to ensure that login_base
        # portion of the slice name corresponds to a valid site, if
        # desired.

        # 1. Lowercase.
        # 2. Begins with login_base (letters or numbers).
        # 3. Then single underscore after login_base.
        # 4. Then letters, numbers, or underscores.
        good_name = r'^[a-z0-9]+_[a-zA-Z0-9_]+$'
        if not name or \
           not re.match(good_name, name):
            raise PLCInvalidArgument, "Invalid slice name"

        conflicts = Slices(self.api, [name])
        for slice in conflicts:
            if 'slice_id' not in self or self['slice_id'] != slice['slice_id']:
                raise PLCInvalidArgument, "Slice name already in use, %s"%name

        return name

    def validate_instantiation(self, instantiation):
        instantiations = [row['instantiation'] for row in SliceInstantiations(self.api)]
        if instantiation not in instantiations:
            raise PLCInvalidArgument, "No such instantiation state"

        return instantiation

    validate_created = Row.validate_timestamp

    def validate_expires(self, expires):
        # N.B.: Responsibility of the caller to ensure that expires is
        # not too far into the future.
        check_future = not ('is_deleted' in self and self['is_deleted'])
        return Row.validate_timestamp(self, expires, check_future = check_future)

    add_person = Row.add_object(Person, 'slice_person')
    remove_person = Row.remove_object(Person, 'slice_person')

    add_node = Row.add_object(Node, 'slice_node')
    remove_node = Row.remove_object(Node, 'slice_node')

    add_to_node_whitelist = Row.add_object(Node, 'node_slice_whitelist')
    delete_from_node_whitelist = Row.remove_object(Node, 'node_slice_whitelist')

    def associate_persons(self, auth, field, value):
        """
        Adds persons found in value list to this slice (using AddPersonToSlice).
	Deletes persons not found in value list from this slice (using DeletePersonFromSlice).
        """

	assert 'person_ids' in self
	assert 'slice_id' in self
        assert isinstance(value, list)

	(person_ids, emails) = self.separate_types(value)[0:2]

	# Translate emails into person_ids
	if emails:
	    persons = Persons(self.api, emails, ['person_id']).dict('person_id')
	    person_ids += persons.keys()

	# Add new ids, remove stale ids
        if self['person_ids'] != person_ids:
            from PLC.Methods.AddPersonToSlice import AddPersonToSlice
            from PLC.Methods.DeletePersonFromSlice import DeletePersonFromSlice
            new_persons = set(person_ids).difference(self['person_ids'])
            stale_persons = set(self['person_ids']).difference(person_ids)

            for new_person in new_persons:
                AddPersonToSlice.__call__(AddPersonToSlice(self.api), auth, new_person, self['slice_id'])
            for stale_person in stale_persons:
                DeletePersonFromSlice.__call__(DeletePersonFromSlice(self.api), auth, stale_person, self['slice_id'])

    def associate_nodes(self, auth, field, value):
	"""
	Adds nodes found in value list to this slice (using AddSliceToNodes).
	Deletes nodes not found in value list from this slice (using DeleteSliceFromNodes).
	"""

        from PLC.Nodes import Nodes

	assert 'node_ids' in self
	assert 'slice_id' in self
	assert isinstance(value, list)

	(node_ids, hostnames) = self.separate_types(value)[0:2]

	# Translate hostnames into node_ids
	if hostnames:
	    nodes = Nodes(self.api, hostnames, ['node_id']).dict('node_id')
	    node_ids += nodes.keys()

	# Add new ids, remove stale ids
	if self['node_ids'] != node_ids:
	    from PLC.Methods.AddSliceToNodes import AddSliceToNodes
	    from PLC.Methods.DeleteSliceFromNodes import DeleteSliceFromNodes
	    new_nodes = set(node_ids).difference(self['node_ids'])
	    stale_nodes = set(self['node_ids']).difference(node_ids)

	    if new_nodes:
		AddSliceToNodes.__call__(AddSliceToNodes(self.api), auth, self['slice_id'], list(new_nodes))
	    if stale_nodes:
		DeleteSliceFromNodes.__call__(DeleteSliceFromNodes(self.api), auth, self['slice_id'], list(stale_nodes))
    def associate_slice_attributes(self, auth, fields, value):
	"""
	Deletes slice_attribute_ids not found in value list (using DeleteSliceAttribute).
	Adds slice_attributes if slice_fields w/o slice_id is found (using AddSliceAttribute).
	Updates slice_attribute if slice_fields w/ slice_id is found (using UpdateSlceiAttribute).
	"""

	assert 'slice_attribute_ids' in self
	assert isinstance(value, list)

	(attribute_ids, blank, attributes) = self.separate_types(value)

	# There is no way to add attributes by id. They are
	# associated with a slice when they are created.
	# So we are only looking to delete here
	if self['slice_attribute_ids'] != attribute_ids:
	    from PLC.Methods.DeleteSliceAttribute import DeleteSliceAttribute
	    stale_attributes = set(self['slice_attribute_ids']).difference(attribute_ids)

	    for stale_attribute in stale_attributes:
		DeleteSliceAttribute.__call__(DeleteSliceAttribute(self.api), auth, stale_attribute['slice_attribute_id'])

	# If dictionary exists, we are either adding new
        # attributes or updating existing ones.
        if attributes:
            from PLC.Methods.AddSliceAttribute import AddSliceAttribute
            from PLC.Methods.UpdateSliceAttribute import UpdateSliceAttribute

	    added_attributes = filter(lambda x: 'slice_attribute_id' not in x, attributes)
	    updated_attributes = filter(lambda x: 'slice_attribute_id' in x, attributes)

	    for added_attribute in added_attributes:
		if 'attribute_type' in added_attribute:
		    type = added_attribute['attribute_type']
		elif 'attribute_type_id' in added_attribute:
		    type = added_attribute['attribute_type_id']
		else:
		    raise PLCInvalidArgument, "Must specify attribute_type or attribute_type_id"

		if 'value' in added_attribute:
		    value = added_attribute['value']
		else:
		    raise PLCInvalidArgument, "Must specify a value"

		if 'node_id' in added_attribute:
		    node_id = added_attribute['node_id']
		else:
		    node_id = None

		if 'nodegroup_id' in added_attribute:
		    nodegroup_id = added_attribute['nodegroup_id']
		else:
		    nodegroup_id = None

		AddSliceAttribute.__call__(AddSliceAttribute(self.api), auth, self['slice_id'], type, value, node_id, nodegroup_id)
	    for updated_attribute in updated_attributes:
		attribute_id = updated_attribute.pop('slice_attribute_id')
		if attribute_id not in self['slice_attribute_ids']:
		    raise PLCInvalidArgument, "Attribute doesnt belong to this slice"
		else:
		    UpdateSliceAttribute.__call__(UpdateSliceAttribute(self.api), auth, attribute_id, updated_attribute)

    def sync(self, commit = True):
        """
        Add or update a slice.
        """

        # Before a new slice is added, delete expired slices
        if 'slice_id' not in self:
            expired = Slices(self.api, expires = -int(time.time()))
            for slice in expired:
                slice.delete(commit)

        Row.sync(self, commit)

    def delete(self, commit = True):
        """
        Delete existing slice.
        """

        assert 'slice_id' in self

        # Clean up miscellaneous join tables
        for table in self.join_tables:
            self.api.db.do("DELETE FROM %s WHERE slice_id = %d" % \
                           (table, self['slice_id']))

        # Mark as deleted
        self['is_deleted'] = True
        self.sync(commit)

class Slices(Table):
    """
    Representation of row(s) from the slices table in the
    database.
    """

    def __init__(self, api, slice_filter = None, columns = None, expires = int(time.time())):
        Table.__init__(self, api, Slice, columns)

        sql = "SELECT %s FROM view_slices WHERE is_deleted IS False" % \
              ", ".join(self.columns)

        if expires is not None:
            if expires >= 0:
                sql += " AND expires > %d" % expires
            else:
                expires = -expires
                sql += " AND expires < %d" % expires

        if slice_filter is not None:
            if isinstance(slice_filter, (list, tuple, set)):
                # Separate the list into integers and strings
                ints = filter(lambda x: isinstance(x, (int, long)), slice_filter)
                strs = filter(lambda x: isinstance(x, StringTypes), slice_filter)
                slice_filter = Filter(Slice.fields, {'slice_id': ints, 'name': strs})
                sql += " AND (%s) %s" % slice_filter.sql(api, "OR")
            elif isinstance(slice_filter, dict):
                slice_filter = Filter(Slice.fields, slice_filter)
                sql += " AND (%s) %s" % slice_filter.sql(api, "AND")
            elif isinstance (slice_filter, StringTypes):
                slice_filter = Filter(Slice.fields, {'name':[slice_filter]})
                sql += " AND (%s) %s" % slice_filter.sql(api, "AND")
            elif isinstance (slice_filter, int):
                slice_filter = Filter(Slice.fields, {'slice_id':[slice_filter]})
                sql += " AND (%s) %s" % slice_filter.sql(api, "AND")
            else:
                raise PLCInvalidArgument, "Wrong slice filter %r"%slice_filter

        self.selectall(sql)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
 * DocumentNGramGraph.java
 *
 * Created on 17/5/2017 16:00
 *
"""

import matplotlib.pyplot as plt
import networkx as nx
import pygraphviz as pgv
from networkx.drawing.nx_agraph import graphviz_layout

"""
 *  Represents the graph of a document, with vertices n-grams of the document and edges the number
 * of the n-grams' co-occurences within a given window.
 #*
 * @author ysig
"""

class DocumentNGramGraph:
    # initialization
    def __init__(self, n=3, Dwin=2, Data=[], GPrintVerbose=True):
        # consider not having characters but lists of objects
        self._Data = []

        # data size build for reuse of len(data)
        self._dSize = 0

        # stores the ngram
        self._ngram = []

        # store the ngram graph
        self._Graph = nx.DiGraph()

        # cache of edges (set vs. list)
        self._edges = set()

        # the graph stores it's maximum and minimum weigh
        self._maxW = 0
        self._minW = float("inf")

        # data must be "listable"
        self._Dwin = abs(int(Dwin))
        # n for the n-graph
        self._n = abs(int(n))
        self.setData(Data)
        # a printing flag determining if the printing result will be stored on document or
        # be displayed on string
        self._GPrintVerbose = GPrintVerbose
        if not (self._Data == []):
            self.buildGraph()

    # we will now define @method buildGraph
    # which takes a data input
    # segments ngrams
    # and creates ngrams based on a given window
    # !notice: at this developmental stage the weighting method
    # may not be correct
    def buildGraph(self, verbose=False, d=[]):
        # set Data @class_var
        self.setData(d)
        Data = self._Data

        # build ngram
        ng = self.build_ngram()
        s = len(ng)

        win = self._Dwin

        # init graph
        # TODO: add clear function

        o = min(self._Dwin, s)
        if o >= 1:
            window = [ng[0]]
            # append the first full window
            # while adding the needed edges
            for gram in ng[1 : o + 1]:
                for w in window:
                    self.addEdgeInc(gram, w)
                window.append(gram)

            # with full window span till
            # the end.
            for gram in ng[o + 1 :]:
                for w in window:
                    self.addEdgeInc(gram, w)
                window.pop(0)
                window.append(gram)

            # print graph (optional)
            if verbose:
                self.GraphDraw(self._GPrintVerbose)
        return self._Graph

    # add's an edge if it's non existent
    # if it is increments it's weight
    # !notice: reiweighting technique may be false
    # at this developmental stage
    def addEdgeInc(self, a, b, w=1):
        # A = repr(a)#str(a)
        # B = repr(b)#str(b)
        # merging can also be done in other ways
        # add an extra class variable
        A = "".join(a)
        B = "".join(b)
        if (A, B) in self._edges:
            edata = self._Graph.get_edge_data(A, B)
            # DEBUG LINES
            # print "updating edge between (",A,B,")"
            # print "to weight",(edata['weight']+1)

            r = edata["weight"] + w
        else:
            # DEBUG LINES
            # print "adding edge between (",A,B,")"

            r = w
        # update/add edge weight
        self.setEdge(A, B, r)

    # creates ngram's of window based on @param n
    def build_ngram(self, d=[]):
        self.setData(d)
        Data = self._Data
        l = Data[0 : min(self._n, self._dSize)]
        q = []
        q.append(l[:])
        if self._n < self._dSize:
            for d in Data[min(self._n, self._dSize) :]:
                l.pop(0)
                l.append(d)
                q.append(l[:])
        self._ngram = q
        return q

    # draws a graph using math plot lib
    def GraphDraw(self, verbose=True, print_name="graph", lf=True, ns=1000, wf=True):
        pos = graphviz_layout(self._Graph)
        # pos = sring_layout(self._Graph, scale=1)
        # nx.draw(self._Graph,pos = pos,node_size=ns,with_labels = lf, node_color = 'm')
        nx.draw(
            self._Graph,
            pos=graphviz_layout(self._Graph, prog="dot"),
            node_size=ns,
            cmap=plt.cm.Blues,
            node_color=list(range(len(self._Graph))),
            with_labels=lf,
        )
        if wf:
            weight_labels = nx.get_edge_attributes(self._Graph, "weight")
            nx.draw_networkx_edge_labels(
                self._Graph, pos=pos, edge_labels=weight_labels
            )
        if verbose:
            plt.show()
        else:
            # plt.savefig('g.png')
            # or to dot
            nx.drawing.nx_pydot.write_dot(self._Graph, print_name + ".dot")
            # !!Uknown error: the produced dot file is
            # not readable by dot/xdot.

    ## set functions for structure's protected fields

    def setData(self, Data):
        if not (Data == []):
            self._Data = list(Data)
            self._dSize = len(self._Data)

    # sets an edges weight
    def setEdge(self, a, b, w=1):
        self._edges.add((a, b))  # Update cache
        self._Graph.add_edge(a, b, key="edge", weight=w)

        self._maxW = max(self._maxW, w)
        self._minW = min(self._minW, w)

    # deletes
    def delEdge(self, u, v):
        self._edges.remove((u, v))
        self._Graph.remove_edge(u, v)

    # trims the graph by removing unreached nodes
    def deleteUnreachedNodes(self):
        self._Graph.remove_nodes_from(list(nx.isolates(self._Graph)))

    def setN(self, n):
        self._n = n

    def setDwin(self, win):
        self._Dwin = win

    def size(self):
        return self._Graph.size()

    ## get functions for structures protected fields
    def getMin(self):
        return self._MinSize

    def getngram(self):
        return self._ngram

    def getGraph(self):
        return self._Graph

    def maxW(self):
        return self._maxW

    def minW(self):
        return self._minW

    def number_of_edges(self):
        return self._Graph.number_of_nodes()

# test script

# 1. construct a 2-gram graph of window_size = 2
#   from the word "abcdef"
<commit_msg>fix: replcace `"".join(...)` with `tuple(...)`<commit_after>#!/usr/bin/env python

"""
 * DocumentNGramGraph.java
 *
 * Created on 17/5/2017 16:00
 *
"""

import matplotlib.pyplot as plt
import networkx as nx
import pygraphviz as pgv
from networkx.drawing.nx_agraph import graphviz_layout

"""
 *  Represents the graph of a document, with vertices n-grams of the document and edges the number
 * of the n-grams' co-occurences within a given window.
 #*
 * @author ysig
"""

class DocumentNGramGraph:
    # initialization
    def __init__(self, n=3, Dwin=2, Data=[], GPrintVerbose=True):
        # consider not having characters but lists of objects
        self._Data = []

        # data size build for reuse of len(data)
        self._dSize = 0

        # stores the ngram
        self._ngram = []

        # store the ngram graph
        self._Graph = nx.DiGraph()

        # cache of edges (set vs. list)
        self._edges = set()

        # the graph stores it's maximum and minimum weigh
        self._maxW = 0
        self._minW = float("inf")

        # data must be "listable"
        self._Dwin = abs(int(Dwin))
        # n for the n-graph
        self._n = abs(int(n))
        self.setData(Data)
        # a printing flag determining if the printing result will be stored on document or
        # be displayed on string
        self._GPrintVerbose = GPrintVerbose
        if not (self._Data == []):
            self.buildGraph()

    # we will now define @method buildGraph
    # which takes a data input
    # segments ngrams
    # and creates ngrams based on a given window
    # !notice: at this developmental stage the weighting method
    # may not be correct
    def buildGraph(self, verbose=False, d=[]):
        # set Data @class_var
        self.setData(d)
        Data = self._Data

        # build ngram
        ng = self.build_ngram()
        s = len(ng)

        win = self._Dwin

        # init graph
        # TODO: add clear function

        o = min(self._Dwin, s)
        if o >= 1:
            window = [ng[0]]
            # append the first full window
            # while adding the needed edges
            for gram in ng[1 : o + 1]:
                for w in window:
                    self.addEdgeInc(gram, w)
                window.append(gram)

            # with full window span till
            # the end.
            for gram in ng[o + 1 :]:
                for w in window:
                    self.addEdgeInc(gram, w)
                window.pop(0)
                window.append(gram)

            # print graph (optional)
            if verbose:
                self.GraphDraw(self._GPrintVerbose)
        return self._Graph

    # add's an edge if it's non existent
    # if it is increments it's weight
    # !notice: reiweighting technique may be false
    # at this developmental stage
    def addEdgeInc(self, a, b, w=1):
        # A = repr(a)#str(a)
        # B = repr(b)#str(b)
        # merging can also be done in other ways
        # add an extra class variable
        A = tuple(a)
        B = tuple(b)
        if (A, B) in self._edges:
            edata = self._Graph.get_edge_data(A, B)
            # DEBUG LINES
            # print "updating edge between (",A,B,")"
            # print "to weight",(edata['weight']+1)

            r = edata["weight"] + w
        else:
            # DEBUG LINES
            # print "adding edge between (",A,B,")"

            r = w
        # update/add edge weight
        self.setEdge(A, B, r)

    # creates ngram's of window based on @param n
    def build_ngram(self, d=[]):
        self.setData(d)
        Data = self._Data
        l = Data[0 : min(self._n, self._dSize)]
        q = []
        q.append(l[:])
        if self._n < self._dSize:
            for d in Data[min(self._n, self._dSize) :]:
                l.pop(0)
                l.append(d)
                q.append(l[:])
        self._ngram = q
        return q

    # draws a graph using math plot lib
    def GraphDraw(self, verbose=True, print_name="graph", lf=True, ns=1000, wf=True):
        pos = graphviz_layout(self._Graph)
        # pos = sring_layout(self._Graph, scale=1)
        # nx.draw(self._Graph,pos = pos,node_size=ns,with_labels = lf, node_color = 'm')
        nx.draw(
            self._Graph,
            pos=graphviz_layout(self._Graph, prog="dot"),
            node_size=ns,
            cmap=plt.cm.Blues,
            node_color=list(range(len(self._Graph))),
            with_labels=lf,
        )
        if wf:
            weight_labels = nx.get_edge_attributes(self._Graph, "weight")
            nx.draw_networkx_edge_labels(
                self._Graph, pos=pos, edge_labels=weight_labels
            )
        if verbose:
            plt.show()
        else:
            # plt.savefig('g.png')
            # or to dot
            nx.drawing.nx_pydot.write_dot(self._Graph, print_name + ".dot")
            # !!Uknown error: the produced dot file is
            # not readable by dot/xdot.

    ## set functions for structure's protected fields

    def setData(self, Data):
        if not (Data == []):
            self._Data = list(Data)
            self._dSize = len(self._Data)

    # sets an edges weight
    def setEdge(self, a, b, w=1):
        self._edges.add((a, b))  # Update cache
        self._Graph.add_edge(a, b, key="edge", weight=w)

        self._maxW = max(self._maxW, w)
        self._minW = min(self._minW, w)

    # deletes
    def delEdge(self, u, v):
        self._edges.remove((u, v))
        self._Graph.remove_edge(u, v)

    # trims the graph by removing unreached nodes
    def deleteUnreachedNodes(self):
        self._Graph.remove_nodes_from(list(nx.isolates(self._Graph)))

    def setN(self, n):
        self._n = n

    def setDwin(self, win):
        self._Dwin = win

    def size(self):
        return self._Graph.size()

    ## get functions for structures protected fields
    def getMin(self):
        return self._MinSize

    def getngram(self):
        return self._ngram

    def getGraph(self):
        return self._Graph

    def maxW(self):
        return self._maxW

    def minW(self):
        return self._minW

    def number_of_edges(self):
        return self._Graph.number_of_nodes()

# test script

# 1. construct a 2-gram graph of window_size = 2
#   from the word "abcdef"
<|endoftext|>


<commit_before>from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.http import Request
from scrapy.selector import Selector
from sep_pdf.items import SepPdfItem
from time import time
import csv
import re
import string
import sys

class SepSpider201301(CrawlSpider):
	name = 'sep_crawler_gral'
	allowed_domains = ["sep.gob.mx"]
	def start_requests(self):
		yield Request(self.mekler)
	def parse(self, response):
		sel = Selector(response)
		items = []
		item = SepPdfItem()
		archivos = []
		sites = sel.xpath('//div[@id="content"]/p')
		if len(sites) <10:
			sites = sel.xpath('//div[@id="indiceTematico"]/div/h3')
		for site in sites:
			link = site.xpath('a/@href').extract()
			if len(link)>0:
				if link[0].find('http')>=0:
					request = Request(link[0], callback=self.parse_pdf)
				else:
					request = Request('http://www.sep.gob.mx'+link[0], callback=self.parse_pdf)
				yield request
		pass

	def parse_pdf(self, response):
		sel = Selector(response)
		items = []
		item = SepPdfItem()
		archivos = []
		sites = sel.xpath('//div[@id="content"]/p')
		item['url'] = response.url
		for site in sites:
			title = site.xpath('a/text()').extract()
			link = site.xpath('a/@href').extract()
			if len(title) >0 and len(link)>0:
				archivos.append([title[0],link[0]])
		item['archivo'] = archivos
		return item<commit_msg>cambio en scraper por html nuevo en sitio de la sep<commit_after>from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.http import Request
from scrapy.selector import Selector
from sep_pdf.items import SepPdfItem
from time import time
import csv
import re
import string
import sys

class SepSpider201301(CrawlSpider):
	name = 'sep_crawler_gral'
	allowed_domains = ["sep.gob.mx"]
	def start_requests(self):
		yield Request(self.mekler)
	def parse(self, response):
		sel = Selector(response)
		items = []
		item = SepPdfItem()
		archivos = []
		sites = sel.xpath('//div[@id="content"]/p')
		if len(sites) <10:
			sites = sel.xpath('//div[@id="indiceTematico"]/div/h3')
		for site in sites:
			link = site.xpath('a/@href').extract()
			if len(link)>0:
				if link[0].find('http')>=0:
					request = Request(link[0], callback=self.parse_pdf)
				else:
					request = Request('http://www.sep.gob.mx'+link[0], callback=self.parse_pdf)
				yield request
		pass

	def parse_pdf(self, response):
		sel = Selector(response)
		items = []
		item = SepPdfItem()
		archivos = []
		sites = sel.xpath('//div[@id="content"]/p')
		if len(sites)==0:
			sites = sel.xpath('//div[@id="interna_colIzquierda"]/p')
		item['url'] = response.url
		for site in sites:
			title = site.xpath('a/text()').extract()
			link = site.xpath('a/@href').extract()
			if len(title) >0 and len(link)>0:
				archivos.append([title[0],link[0]])
		item['archivo'] = archivos
		return item<|endoftext|>


<commit_before>'''
Copyright (c) 2014 noriah vix@noriah.dev

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software isfurnished
to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
'''

__all__ = ['PyVDF']

import re
import sys
import types
from collections import OrderedDict, deque

#############
# PyVDF #
#############
class PyVDF:
  """Parse VDFs and Valve KeyValue Files

  https://developer.valvesoftware.com/wiki/KeyValues

    >>> food = '''
    ... "Apple"
    ... {
    ...     "Pie"   "Good"
    ...     "Cobbler"        "Great"
    ... }'''

    >>> import PyVDF
    >>> apple = PyVDF(food)
    >>> apple.find('Apple.Pie')
    'Good'

    >>> apple.findMany(['Apple.Pie','Apple.Cobbler'])
    ['Good','Great']

  Load a file

    >>> apple.load(file instance/filename)

  Load a string

    >>> apple.loads(data)

  As a Static Method

    >>> apple = PyVDF.read(food)
    >>> apple['Apple']['Pie']
    'Good'

  Use OrderedDict instead of dict

    >>> PyVDF.useFastDict(False)

  Output Data as a String

    * Using Class Method on PyVDF instance
    apple.toString()
    '"Apple"\n{\n\t"Cobbler"\t\t"Great"\n\t"Pie"\t\t"Good"\n}\n'

    * Using Static Method on Dictionary
    >>> PyVDF.formatData(apple)
    '"Apple"\n{\n\t"Cobbler"\t\t"Great"\n\t"Pie"\t\t"Good"\n}\n'

  Set Condensed Output (Curly Braces on same line as Key)

    >>> PyVDF.setCondensed(True)
    >>> PyVDF.formatData(apple)
    '"Apple"{\n\t"Cobbler"\t\t"Great"\n\t"Pie"\t\t"Good"\n}\n'

  Set Output Spacing and/or Indentation

    >>> PyVDF.setIndention('\t\t\t')
    >>> PyVDF.setSpacing('What a Terrible Spacer')

    >>> PyVDF.formatData(apple)
    '"Apple"\n{\n\t\t\t"Cobbler"What a Terrible Spacer"Great"\n\t\t\t"Pie"What a Terrible Spacer"Good"\n}\n'

  Set Maximum Token length (Outline from valve says Max of 1024,
  but they broke their own rule) Default is 1200

    >>> PyVDF.setMaxTokenLength(2000)

  Write a file

    * Static Method
    >>> PyVDF.writeData(file instance/filename, apple)

    * PyVDF Instance
    >>> apple.write_file(file instance/filename)

  Get Data from Instance

    >>> apple.getData()

  Set Instance data

    >>> apple.setData(apple_new)

  """

  __version__ = "1.0.4"

  __UseDict = dict
  __OutputIndentation = "\t"
  __OutputSpacing = "\t\t"
  __CondensedOutput = False
  __MaxTokenLength = 1200

  __ERR_BadToken = "Bad token on line {}.\n Possible Newline or Token greater than {} character in length"
  __ERR_BlockNoKey = "Value block without a Key at line {}.\n Last Token: {}"
  __ERR_CompanionBrace = "Expected to get a Value, got '}}' instead.\n At line {}. Last Token: {}"
  __ERR_EODToken = "Hit End of Data while reading token"
  __ERR_EODArray = "Missing braces"

  __ERR_NotDict = "Data is not of type dict\n {}"

  __RE_Token_Quoted = re.compile(r'(?:\\.|[^"])*"')
  __RE_Token_UnQuoted = re.compile(r'^(?:\\.|[^\\"\s\{\}\[\]])*')
  __RE_Path_Seperator = re.compile(r'[^\.\[\]]+|\[[^\[\]]*\]')

  def __init__(self, data=None, infile=None):
    if data != None:
      self.__data = PyVDF.reads(data)
    elif infile != None:
      self.__data = PyVDF.read(infile)
    return

  def __getitem__(self, key):
    return self.find(key)

  def __setitem__(self, key, value):
    self.edit(key, value)

  @staticmethod
  def useFastDict(var = True):
    PyVDF.__UseDict = dict if var else OrderedDict

  @staticmethod
  def setIndentation(var = "\t"):
    PyVDF.__OutputIndentation = var

  @staticmethod
  def setSpacing(var = "\t\t"):
    PyVDF.__OutputSpacing = var

  @staticmethod
  def setCondensed(var = False):
    PyVDF.__CondensedOutput = var

  @staticmethod
  def setMaxTokenLength(var = 1024):
    PyVDF.__MaxTokenLength = var

  @staticmethod
  def read(f):
    try:
      return PyVDF.reads(f.read())
    except AttributeError:
      pass

    try:
      with open(f, 'r') as filec:
        data = PyVDF.reads(filec.read())
        filec.close()
        return data
    except IOError as e:
      print("Could not open '" + f + "' for reading.")
      print("Ignore this if you are creating a new file.")

    return PyVDF.__UseDict()

  @staticmethod
  def reads(s):
    _dict = PyVDF.__UseDict
    _len = len
    _whitespace = frozenset('\t ')
    _newline = frozenset('\n\r')

    _quote_match = PyVDF.__RE_Token_Quoted.match
    _unquote_match = PyVDF.__RE_Token_UnQuoted.match
    _max_length = PyVDF.__MaxTokenLength

    ci = 0
    line = 0
    grabKey = 1

    data = _dict()
    tree = data

    keys = deque()
    keyApp = keys.append
    keyPop = keys.pop

    try:
      while 1:

        while s[ci] in _whitespace:
          ci += 1

        char = s[ci]

        if char == '"':
          ci += 1
          string = _quote_match(s[ci:ci + _max_length]).group()[:-1]
          ci += _len(string)

          if grabKey:
            k = string
            grabKey = 0
          else:
            tree[k] = string
            grabKey = 1

        elif char == '\n':
          line += 1

        # elif char == '\r':
        #   if s[ci + 1] == '\n':
        #     ci += 1
        #   line += 1

        elif char == '{':
          if grabKey:
            raise Exception(PyVDF.__ERR_BlockNoKey.format(line, k))
          else:
            keyApp(k)
            tree[k] = _dict()
            tree = tree[k]
            grabKey = 1

        elif char == '}':
          if grabKey:
            keyPop()
            tree = data
            for key in keys:
              tree = tree[key]
          else:
            raise Exception(PyVDF.__ERR_CompanionBrace.format(line, k))

        elif char == '/' and s[ci + 1] == '/':
          ci += 1
          line += 1
          while 1:
            ci += 1
            if s[ci] in _newline:
              break

        elif char == '[':
          while 1:
            ci += 1
            if s[ci] == ']':
              break

        else:
          string = _unquote_match(s[ci:ci + _max_length]).group()
          ci += _len(string)

          if grabKey:
            k = string
            grabKey = 0
          else:
            tree[k] = string
            grabKey = 1
          continue

        ci += 1

    except IndexError:
      if _len(keys) == 0:
        return data
      raise Exception(PyVDF.__ERR_EODArray)
    except AttributeError:
      raise Exception(PyVDF.__ERR_BadToken.format(line, _max_length))

  @staticmethod
  def formatData(data):
    condensed = PyVDF.__CondensedOutput
    indentation = PyVDF.__OutputIndentation
    spacing = PyVDF.__OutputSpacing
    def loop(array, tab=''):
      string = ''
      for k, v in array.items():
        string += '{}"{}"'.format(tab,k)
        if isinstance(v, dict):
          string += '{}{{\n{}{}}}\n'.format(
            '' if condensed else '\n' + tab,
            loop(v, tab + indentation),
            tab)
        else:
            string += '{}"{}"\n'.format(spacing, v)
      return string
    return loop(data)

  @staticmethod
  def writeData(f, data):
    if not isinstance(data, dict):
      raise Exception(PyVDF.__ERR_NotDict.format(repr(data)))
    data = PyVDF.formatData(data)
    try:
      f.write(data)
    except AttributeError:
      pass

    try:
      filec = open(f, 'w')
      filec.write(data)
      filec.close()
    except IOError as e:
      print("Could not open '" + f + "' for writing.")
      print(e)

  def load(self, f):
    self.__data = PyVDF.read(f)

  def loads(self, data):
    self.__data = PyVDF.reads(data)

  def getData(self):
    try:
      return self.__data
    except AttributeError:
      return PyVDF.__UseDict()

  def setData(self, data):
    self.__data = data

  def find(self, path):
    p = [re.sub('[\[\]]', '', w) for w in PyVDF.__RE_Path_Seperator.findall(path)]
    array = self.getData()
    for c in p:
      try:
        array = array[c]
      except KeyError:
        return ''
    return array

  def edit(self, path, value):
    _dict = PyVDF.__UseDict
    p = [re.sub('[\[\]]', '', w) for w in PyVDF.__RE_Path_Seperator.findall(path)]
    array = self.getData()
    a = array
    for c in p[:-1]:
      try:
        if not isinstance(a[c], dict):
          a[c] = _dict()
      except KeyError:
        a[c] = _dict()
      a = a[c]
    if value == None:
      a.pop(p[-1], None)
    else:
      a[p[-1]] = value
    self.__data = array

  def findMany(self, paths):
    return map(self.find, paths)

  def editMany(self, paths):
    [self.edit(v[0], v[1]) for v in paths]

  def write_file(self, f):
    PyVDF.writeData(f, self.__data)

  def toString(self):
    return PyVDF.formatData(self.__data)

# TODO - Json

  # def toJson(self):

  # def fromJson(self):
  <commit_msg>DOCUMENTATION!!!!<commit_after>import re
from collections import OrderedDict, deque

#############
# PyVDF #
#############
class PyVDF:
  """Parse VDFs and Valve KeyValue Files"""

  __version__ = "1.0.4"
  __release__ = "1.0.4"

  __UseDict = dict
  __OutputIndentation = "\t"
  __OutputSpacing = "\t\t"
  __CondensedOutput = False
  __MaxTokenLength = 1200

  __ERR_BadToken = "Bad token on line {}.\n Possible Newline or Token greater than {} character in length"
  __ERR_BlockNoKey = "Value block without a Key at line {}.\n Last Token: {}"
  __ERR_CompanionBrace = "Expected to get a Value, got '}}' instead.\n At line {}. Last Token: {}"
  __ERR_EODToken = "Hit End of Data while reading token"
  __ERR_EODArray = "Missing braces"

  __ERR_NotDict = "Data is not of type dict\n {}"

  __RE_Token_Quoted = re.compile(r'(?:\\.|[^"])*"')
  __RE_Token_UnQuoted = re.compile(r'^(?:\\.|[^\\"\s\{\}\[\]])*')
  __RE_Path_Seperator = re.compile(r'[^\.\[\]]+|\[[^\[\]]*\]')

  def __init__(self, data=None, infile=None):
    if data != None:
      self.__data = PyVDF.reads(data)
    elif infile != None:
      self.__data = PyVDF.read(infile)
    return

  def __getitem__(self, key):
    return self.find(key)

  def __setitem__(self, key, value):
    self.edit(key, value)

  @staticmethod
  def useFastDict(var = True):
    """
    Use the faster built-in dict class, or the slower, OrderedDict class

    :param var: Use fast dict or not.
    :type var: :py:obj:`bool`
    """

    PyVDF.__UseDict = dict if var else OrderedDict

  @staticmethod
  def setIndentation(var = "\t"):
    """
    Set the indentation of the output

    :param var: The indentation to use
    :type var: :py:obj:`str`
    """
    PyVDF.__OutputIndentation = var

  @staticmethod
  def setSpacing(var = "\t\t"):
    """
    Set the output spacing

    :param var: The characters to use for spacing
    :type var: :py:obj:`str`
    """
    PyVDF.__OutputSpacing = var

  @staticmethod
  def setCondensed(var = False):
    """
    Set condensed output

    :param var: Use condensed output or not
    :type var: :py:obj:`bool`
    """
    PyVDF.__CondensedOutput = var

  @staticmethod
  def setMaxTokenLength(var = 1024):
    """
    Set the maxmimum token length that can be read.

    :param var: Number of characters allowed in a token
    :type var: :py:obj:`int`
    """
    PyVDF.__MaxTokenLength = var

  @staticmethod
  def read(f):
    """
    Parse a String and return the data

    :param f: The file to read
    :type f: file/str
    :returns: A dictionary object containing the parsed data from f
    :raises: SyntaxError - An error occured reading the data.
    """
    try:
      return PyVDF.reads(f.read())
    except AttributeError:
      pass

    try:
      with open(f, 'r') as filec:
        data = PyVDF.reads(filec.read())
        filec.close()
        return data
    except IOError as e:
      print("Could not open '" + f + "' for reading.")
      print("Ignore this if you are creating a new file.")

    return PyVDF.__UseDict()

  @staticmethod
  def reads(s):
    """
    Parse a String and return the data

    :param s: The string to read
    :type s: :py:obj:`str`
    :returns: A dict object containing the data from s
    :raises: SyntaxError - An error occured reading the data.
    """
    _dict = PyVDF.__UseDict
    _len = len
    _whitespace = frozenset('\t ')
    _newline = frozenset('\n\r')

    _quote_match = PyVDF.__RE_Token_Quoted.match
    _unquote_match = PyVDF.__RE_Token_UnQuoted.match
    _max_length = PyVDF.__MaxTokenLength

    ci = 0
    line = 0
    grabKey = 1

    data = _dict()
    tree = data

    keys = deque()
    keyApp = keys.append
    keyPop = keys.pop

    try:
      while 1:

        while s[ci] in _whitespace:
          ci += 1

        char = s[ci]

        if char == '"':
          ci += 1
          string = _quote_match(s[ci:ci + _max_length]).group()[:-1]
          ci += _len(string)

          if grabKey:
            k = string
            grabKey = 0
          else:
            tree[k] = string
            grabKey = 1

        elif char == '\n':
          line += 1

        # elif char == '\r':
        #   if s[ci + 1] == '\n':
        #     ci += 1
        #   line += 1

        elif char == '{':
          if grabKey:
            raise Exception(PyVDF.__ERR_BlockNoKey.format(line, k))
          else:
            keyApp(k)
            tree[k] = _dict()
            tree = tree[k]
            grabKey = 1

        elif char == '}':
          if grabKey:
            keyPop()
            tree = data
            for key in keys:
              tree = tree[key]
          else:
            raise Exception(PyVDF.__ERR_CompanionBrace.format(line, k))

        elif char == '/' and s[ci + 1] == '/':
          ci += 1
          line += 1
          while 1:
            ci += 1
            if s[ci] in _newline:
              break

        elif char == '[':
          while 1:
            ci += 1
            if s[ci] == ']':
              break

        else:
          string = _unquote_match(s[ci:ci + _max_length]).group()
          ci += _len(string)

          if grabKey:
            k = string
            grabKey = 0
          else:
            tree[k] = string
            grabKey = 1
          continue

        ci += 1

    except IndexError:
      if _len(keys) == 0:
        return data
      raise Exception(PyVDF.__ERR_EODArray)
    except AttributeError:
      raise Exception(PyVDF.__ERR_BadToken.format(line, _max_length))

  @staticmethod
  def formatData(data):
    """
    Format a dictionary object to look like a VDF file

    :param data: Data
    :type data: :py:obj:`dict`

    :returns: A vdf string representation of data
    """
    condensed = PyVDF.__CondensedOutput
    indentation = PyVDF.__OutputIndentation
    spacing = PyVDF.__OutputSpacing
    def loop(array, tab=''):
      string = ''
      for k, v in array.items():
        string += '{}"{}"'.format(tab,k)
        if isinstance(v, dict):
          string += '{}{{\n{}{}}}\n'.format(
            '' if condensed else '\n' + tab,
            loop(v, tab + indentation),
            tab)
        else:
            string += '{}"{}"\n'.format(spacing, v)
      return string
    return loop(data)

  @staticmethod
  def writeData(f, data):
    """
    Write a dictionary object to a file

    :param f: The file to write to
    :type f: file/string
    :param data: The data to write to the file
    :type data: dict
    """
    if not isinstance(data, dict):
      raise Exception(PyVDF.__ERR_NotDict.format(repr(data)))
    data = PyVDF.formatData(data)
    try:
      f.write(data)
    except AttributeError:
      pass

    try:
      filec = open(f, 'w')
      filec.write(data)
      filec.close()
    except IOError as e:
      print("Could not open '" + f + "' for writing.")
      print(e)

  def load(self, f):
    """
    Parse a file and return a dictionary object

    :param f: The file to read
    :type f: file/string
    :returns: A dict object containing the data from s
    :raises: SyntaxError - An error occured reading the data
    """
    self.__data = PyVDF.read(f)

  def loads(self, data):
    """
    Parse a string and return a dictionary object

    :param data: The data to read
    :type data: :py:obj:`str`
    :returns: A dict object containing the data from s
    :raises: SyntaxError - An error occured reading the data.
    """
    self.__data = PyVDF.reads(data)

  def getData(self):
    """
    Get the instance data
    """
    try:
      return self.__data
    except AttributeError:
      return PyVDF.__UseDict()

  def setData(self, data):
    """
    Set the data of the instance

    :param data: The data
    :type data: :py:obj:`dict`
    """
    self.__data = data

  def find(self, path):
    """
    Find a value

    :param path: The Key path to search for
    :type path: :py:obj:`str`
    :returns: The found value or an empty string if not found.
    """
    p = [re.sub('[\[\]]', '', w) for w in PyVDF.__RE_Path_Seperator.findall(path)]
    array = self.getData()
    for c in p:
      try:
        array = array[c]
      except KeyError:
        return ''
    return array

  def edit(self, path, value):
    """
    Edit a key value

    :param path: The path key for the value
    :type path: :py:obj:`str`
    :param value: The value to be set
    :type value: :py:obj:`str`
    """
    _dict = PyVDF.__UseDict
    p = [re.sub('[\[\]]', '', w) for w in PyVDF.__RE_Path_Seperator.findall(path)]
    array = self.getData()
    a = array
    for c in p[:-1]:
      try:
        if not isinstance(a[c], dict):
          a[c] = _dict()
      except KeyError:
        a[c] = _dict()
      a = a[c]
    if value == None:
      a.pop(p[-1], None)
    else:
      a[p[-1]] = value
    self.__data = array

  def findMany(self, paths):
    """
    Find multiple values

    :param paths: A list of path strings
    :type paths: :py:obj:`str`
    :returns: The found value or an empty string if not found.
    """
    return map(self.find, paths)

  def editMany(self, paths):
    """
    Edit multiple key values

    :param path: A list or tupple of lists or tupples.
    :type path: tuple
    """
    [self.edit(v[0], v[1]) for v in paths]

  def write_file(self, f):
    """
    write the instance data to a file

    :param f: The file to write to
    :type f: :py:obj:`str`
    """
    PyVDF.writeData(f, self.__data)

  def toString(self):
    """
    return the instance data as a vdf string

    :returns: A string of the instance data, in a vdf format
    :rtype: :py:obj:`str`
    """
    return PyVDF.formatData(self.__data)

# TODO - Json

  # def toJson(self):

  # def fromJson(self):
  <|endoftext|>