<commit_before># Copyright (c) 2016 Jamie Bull
# Copyright (c) 2012 Santosh Philip
# =======================================================================
#  Distributed under the MIT License.
#  (See accompanying file LICENSE or copy at
#  http://opensource.org/licenses/MIT)
# =======================================================================
"""
Monkey patches for changes to classes and functions in Eppy. These include
fixes which have not yet made it to the released version of Eppy. These will be
removed as soon as Eppy catches up. It also includes the addition of features
specific to geometry editing which may or may not be included future versions
of Eppy.

"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from geomeppy.builder import Block
from geomeppy.builder import Zone
from geomeppy.intersect_match import getidfsurfaces
from geomeppy.intersect_match import intersect_idf_surfaces
from geomeppy.intersect_match import match_idf_surfaces
from geomeppy.intersect_match import set_coords
from geomeppy.recipes import set_wwr
from geomeppy.recipes import translate
from geomeppy.recipes import translate_to_origin
from geomeppy.view_geometry import view_idf

from eppy import bunchhelpers
from eppy import iddgaps
from eppy.EPlusInterfaceFunctions import readidf
from eppy.bunch_subclass import EpBunch as BaseBunch
from eppy.idf_msequence import Idf_MSequence
from eppy.idfreader import convertallfields
from eppy.idfreader import iddversiontuple
from eppy.idfreader import idfreader1
from eppy.idfreader import makeabunch
from eppy.modeleditor import IDDNotSetError
from eppy.modeleditor import IDF as BaseIDF
from eppy.modeleditor import addthisbunch
from eppy.modeleditor import namebunch
from eppy.modeleditor import newrawobject
from eppy.modeleditor import obj2bunch
from py._log import warning

class EpBunch(BaseBunch):
    """Monkey-patched EpBunch to add the setcoords function.
    """

    def __init__(self, *args, **kwargs):
        super(EpBunch, self).__init__(*args, **kwargs)

    def setcoords(self, poly, ggr=None):
        """Set the coordinates of a surface.

        Parameters
        ----------
        poly : Polygon3D or list
             Either a Polygon3D object of a list of (x,y,z) tuples.
        ggr : EpBunch, optional
            A GlobalGeometryRules IDF object. Defaults to None.

        """
        surfaces = [
            "BUILDINGSURFACE:DETAILED",
            "WALL:DETAILED",
            "ROOFCEILING:DETAILED",
            "FLOOR:DETAILED",
            "FENESTRATIONSURFACE:DETAILED",
            "SHADING:SITE:DETAILED",
            "SHADING:BUILDING:DETAILED",
            "SHADING:ZONE:DETAILED", ]
        if self.key.upper() in surfaces:
            set_coords(self, poly, ggr)
        else:
            raise AttributeError

def idfreader1(fname, iddfile, theidf, conv=True, commdct=None, block=None):
    """read idf file and return bunches"""
    versiontuple = iddversiontuple(iddfile)
    # import pdb; pdb.set_trace()
    block, data, commdct = readidf.readdatacommdct1(
        fname,
        iddfile=iddfile,
        commdct=commdct,
        block=block)
    if conv:
        convertallfields(data, commdct)
    # fill gaps in idd
    ddtt, dtls = data.dt, data.dtls
    if versiontuple < (8,):
        skiplist = ["TABLE:MULTIVARIABLELOOKUP"]
    else:
        skiplist = None
    nofirstfields = iddgaps.missingkeys_standard(
        commdct, dtls,
        skiplist=skiplist)
    iddgaps.missingkeys_nonstandard(commdct, dtls, nofirstfields)
    # bunchdt = makebunches(data, commdct)
    bunchdt = makebunches(data, commdct, theidf)
    return bunchdt, block, data, commdct

def makebunches(data, commdct, theidf):
    """make bunches with data"""
    bunchdt = {}
    dt, dtls = data.dt, data.dtls
    for obj_i, key in enumerate(dtls):
        key = key.upper()
        objs = dt[key]
        list1 = []
        for obj in objs:
            bobj = makeabunch(commdct, obj, obj_i)
            list1.append(bobj)
        bunchdt[key] = Idf_MSequence(list1, objs, theidf)
    return bunchdt

def obj2bunch(data, commdct, obj):
    """make a new bunch object using the data object"""
    dtls = data.dtls
    key = obj[0].upper()
    key_i = dtls.index(key)
    abunch = makeabunch(commdct, obj, key_i)
    return abunch

def makeabunch(commdct, obj, obj_i):
    """make a bunch from the object"""
    objidd = commdct[obj_i]
    objfields = [comm.get('field') for comm in commdct[obj_i]]
    objfields[0] = ['key']
    objfields = [field[0] for field in objfields]
    obj_fields = [bunchhelpers.makefieldname(field) for field in objfields]
    bobj = EpBunch(obj, obj_fields, objidd)
    return bobj

class IDF(BaseIDF):
    """Monkey-patched IDF.

    Patched to add read (to add additional functionality) and to fix
    copyidfobject and newidfobject.

    """

    def __init__(self, *args, **kwargs):
        super(IDF, self).__init__(*args, **kwargs)

    def intersect_match(self):
        """Intersect all surfaces in the IDF, then set boundary conditions.
        """
        self.intersect()
        self.match()

    def intersect(self):
        """Intersect all surfaces in the IDF.
        """
        intersect_idf_surfaces(self)

    def match(self):
        """Set boundary conditions for all surfaces in the IDF.
        """
        match_idf_surfaces(self)

    def translate_to_origin(self):
        """
        Move an IDF close to the origin so that it can be viewed in SketchUp.
        """
        translate_to_origin(self)

    def translate(self, vector):
        """Move the IDF in the direction given by a vector.

        Parameters
        ----------
        vector : Vector2D, Vector3D, (x,y) or (x,y,z) list-like
            Representation of a vector to translate by.

        """
        surfaces = self.getsurfaces()
        translate(surfaces, vector)

    def getsurfaces(self):
        """Return all surfaces in the IDF.

        Returns
        -------
        list

        """
        return getidfsurfaces(self)

    def set_wwr(self, wwr):
        """Add strip windows to all external walls.

        Parameters
        ----------
        wwr : float
            Window to wall ratio in the range 0-1.

        """
        set_wwr(self, wwr)

    def view_model(self):
        """Show a zoomable, rotatable representation of the IDF.
        """
        view_idf(idf_txt=self.idfstr())

    def add_block(self, *args, **kwargs):
        """Add a block to the IDF
        """
        block = Block(*args, **kwargs)
        zoning = kwargs.get('zoning', 'by_storey')
        if zoning == 'by_storey':
            zones = [Zone('Storey %i' % storey['storey_no'], storey)
                        for storey in block.stories]
        else:
            raise ValueError('%s is not a valid zoning rule' % zoning)
        for zone in zones:
            self.add_zone(zone)
        self.intersect_match()

    def add_zone(self, zone):
        ggr = self.idfobjects['GLOBALGEOMETRYRULES']
        # add zone object
        self.newidfobject('ZONE', zone.name)
        # add wall objects
        for i, surface_coords in enumerate(zone.walls, 1):
            if not surface_coords:
                continue
            name = '%s Wall %04d' % (zone.name, i)
            s = self.newidfobject('BUILDINGSURFACE:DETAILED', name,
                    Surface_Type = 'wall',
                    Zone_Name = zone.name)
            s.setcoords(surface_coords, ggr)
        # add floor objects
        for i, surface_coords in enumerate(zone.floors, 1):
            if not surface_coords:
                continue
            name = '%s Floor %04d' % (zone.name, i)
            s = self.newidfobject('BUILDINGSURFACE:DETAILED', name,
                    Surface_Type = 'floor',
                    Zone_Name = zone.name)
            s.setcoords(surface_coords, ggr)
        # add ceiling objects
        for i, surface_coords in enumerate(zone.ceilings, 1):
            if not surface_coords:
                continue
            name = '%s Ceiling %04d' % (zone.name, i)
            s = self.newidfobject('BUILDINGSURFACE:DETAILED', name,
                    Surface_Type = 'ceiling',
                    Zone_Name = zone.name)
            s.setcoords(surface_coords, ggr)
        # add roof objects
        for i, surface_coords in enumerate(zone.roofs, 1):
            if not surface_coords:
                continue
            name = '%s Roof %04d' % (zone.name, i)
            s = self.newidfobject('BUILDINGSURFACE:DETAILED', name,
                    Surface_Type = 'roof',
                    Zone_Name = zone.name)
            s.setcoords(surface_coords, ggr)

    def read(self):
        """
        Read the IDF file and the IDD file. If the IDD file had already been
        read, it will not be read again.

        Read populates the following data structures:

        - idfobjects : list
        - model : list
        - idd_info : list

        """
        if self.getiddname() == None:
            errortxt = ("IDD file needed to read the idf file. "
                        "Set it using IDF.setiddname(iddfile)")
            raise IDDNotSetError(errortxt)
        readout = idfreader1(
            self.idfname, self.iddname, self,
            commdct=self.idd_info, block=self.block)
        self.idfobjects, block, self.model, idd_info = readout
        self.__class__.setidd(idd_info, block)

    def newidfobject(self, key, aname='', **kwargs):
        """
        Add a new idfobject to the model. If you don't specify a value for a
        field, the default value will be set.

        For example ::

            newidfobject("CONSTRUCTION")
            newidfobject("CONSTRUCTION",
                Name='Interior Ceiling_class',
                Outside_Layer='LW Concrete',
                Layer_2='soundmat')

        Parameters
        ----------
        key : str
            The type of IDF object. This must be in ALL_CAPS.
        aname : str, deprecated
            This parameter is not used. It is left there for backward
            compatibility.
        **kwargs
            Keyword arguments in the format `field=value` used to set the value
            of fields in the IDF object when it is created.

        Returns
        -------
        EpBunch object

        """
        obj = newrawobject(self.model, self.idd_info, key)
        abunch = obj2bunch(self.model, self.idd_info, obj)
        if aname:
            warning.warn("The aname parameter should no longer be used.")
            namebunch(abunch, aname)
        self.idfobjects[key].append(abunch)
        for k, v in kwargs.items():
            abunch[k] = v
        return abunch

    def copyidfobject(self, idfobject):
        """Add an IDF object to the IDF.

        This has been monkey-patched to add the return value.

        Parameters
        ----------
        idfobject : EpBunch object
            The IDF object to remove. This usually comes from another idf file,
            or it can be used to copy within this idf file.

        Returns
        -------
        EpBunch object

        """
        abunch = addthisbunch(self.idfobjects,
                             self.model,
                             self.idd_info,
                             idfobject,
                             self)

        return abunch
<commit_msg>Add addthisbunch<commit_after># Copyright (c) 2016 Jamie Bull
# Copyright (c) 2012 Santosh Philip
# =======================================================================
#  Distributed under the MIT License.
#  (See accompanying file LICENSE or copy at
#  http://opensource.org/licenses/MIT)
# =======================================================================
"""
Monkey patches for changes to classes and functions in Eppy. These include
fixes which have not yet made it to the released version of Eppy. These will be
removed as soon as Eppy catches up. It also includes the addition of features
specific to geometry editing which may or may not be included future versions
of Eppy.

"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import copy

from eppy import bunchhelpers
from eppy import iddgaps
from eppy.EPlusInterfaceFunctions import readidf
from eppy.bunch_subclass import EpBunch as BaseBunch
from eppy.idf_msequence import Idf_MSequence
from eppy.idfreader import convertallfields
from eppy.idfreader import iddversiontuple
from eppy.idfreader import idfreader1
from eppy.idfreader import makeabunch
from eppy.modeleditor import IDDNotSetError
from eppy.modeleditor import IDF as BaseIDF
from eppy.modeleditor import addthisbunch
from eppy.modeleditor import namebunch
from eppy.modeleditor import newrawobject
from eppy.modeleditor import obj2bunch
from geomeppy.builder import Block
from geomeppy.builder import Zone
from geomeppy.intersect_match import getidfsurfaces
from geomeppy.intersect_match import intersect_idf_surfaces
from geomeppy.intersect_match import match_idf_surfaces
from geomeppy.intersect_match import set_coords
from geomeppy.recipes import set_wwr
from geomeppy.recipes import translate
from geomeppy.recipes import translate_to_origin
from geomeppy.view_geometry import view_idf
from py._log import warning

class EpBunch(BaseBunch):
    """Monkey-patched EpBunch to add the setcoords function.
    """

    def __init__(self, *args, **kwargs):
        super(EpBunch, self).__init__(*args, **kwargs)

    def setcoords(self, poly, ggr=None):
        """Set the coordinates of a surface.

        Parameters
        ----------
        poly : Polygon3D or list
             Either a Polygon3D object of a list of (x,y,z) tuples.
        ggr : EpBunch, optional
            A GlobalGeometryRules IDF object. Defaults to None.

        """
        surfaces = [
            "BUILDINGSURFACE:DETAILED",
            "WALL:DETAILED",
            "ROOFCEILING:DETAILED",
            "FLOOR:DETAILED",
            "FENESTRATIONSURFACE:DETAILED",
            "SHADING:SITE:DETAILED",
            "SHADING:BUILDING:DETAILED",
            "SHADING:ZONE:DETAILED", ]
        if self.key.upper() in surfaces:
            set_coords(self, poly, ggr)
        else:
            raise AttributeError

def idfreader1(fname, iddfile, theidf, conv=True, commdct=None, block=None):
    """read idf file and return bunches"""
    versiontuple = iddversiontuple(iddfile)
    # import pdb; pdb.set_trace()
    block, data, commdct = readidf.readdatacommdct1(
        fname,
        iddfile=iddfile,
        commdct=commdct,
        block=block)
    if conv:
        convertallfields(data, commdct)
    # fill gaps in idd
    ddtt, dtls = data.dt, data.dtls
    if versiontuple < (8,):
        skiplist = ["TABLE:MULTIVARIABLELOOKUP"]
    else:
        skiplist = None
    nofirstfields = iddgaps.missingkeys_standard(
        commdct, dtls,
        skiplist=skiplist)
    iddgaps.missingkeys_nonstandard(commdct, dtls, nofirstfields)
    # bunchdt = makebunches(data, commdct)
    bunchdt = makebunches(data, commdct, theidf)
    return bunchdt, block, data, commdct

def addthisbunch(bunchdt, data, commdct, thisbunch, theidf):
    """add a bunch to model.
    abunch usually comes from another idf file
    or it can be used to copy within the idf file"""
    key = thisbunch.key.upper()
    obj = copy.copy(thisbunch.obj)
    abunch = obj2bunch(data, commdct, obj)
    bunchdt[key].append(abunch)
    return abunch

def makebunches(data, commdct, theidf):
    """make bunches with data"""
    bunchdt = {}
    dt, dtls = data.dt, data.dtls
    for obj_i, key in enumerate(dtls):
        key = key.upper()
        objs = dt[key]
        list1 = []
        for obj in objs:
            bobj = makeabunch(commdct, obj, obj_i)
            list1.append(bobj)
        bunchdt[key] = Idf_MSequence(list1, objs, theidf)
    return bunchdt

def obj2bunch(data, commdct, obj):
    """make a new bunch object using the data object"""
    dtls = data.dtls
    key = obj[0].upper()
    key_i = dtls.index(key)
    abunch = makeabunch(commdct, obj, key_i)
    return abunch

def makeabunch(commdct, obj, obj_i):
    """make a bunch from the object"""
    objidd = commdct[obj_i]
    objfields = [comm.get('field') for comm in commdct[obj_i]]
    objfields[0] = ['key']
    objfields = [field[0] for field in objfields]
    obj_fields = [bunchhelpers.makefieldname(field) for field in objfields]
    bobj = EpBunch(obj, obj_fields, objidd)
    return bobj

class IDF(BaseIDF):
    """Monkey-patched IDF.

    Patched to add read (to add additional functionality) and to fix
    copyidfobject and newidfobject.

    """

    def __init__(self, *args, **kwargs):
        super(IDF, self).__init__(*args, **kwargs)

    def intersect_match(self):
        """Intersect all surfaces in the IDF, then set boundary conditions.
        """
        self.intersect()
        self.match()

    def intersect(self):
        """Intersect all surfaces in the IDF.
        """
        intersect_idf_surfaces(self)

    def match(self):
        """Set boundary conditions for all surfaces in the IDF.
        """
        match_idf_surfaces(self)

    def translate_to_origin(self):
        """
        Move an IDF close to the origin so that it can be viewed in SketchUp.
        """
        translate_to_origin(self)

    def translate(self, vector):
        """Move the IDF in the direction given by a vector.

        Parameters
        ----------
        vector : Vector2D, Vector3D, (x,y) or (x,y,z) list-like
            Representation of a vector to translate by.

        """
        surfaces = self.getsurfaces()
        translate(surfaces, vector)

    def getsurfaces(self):
        """Return all surfaces in the IDF.

        Returns
        -------
        list

        """
        return getidfsurfaces(self)

    def set_wwr(self, wwr):
        """Add strip windows to all external walls.

        Parameters
        ----------
        wwr : float
            Window to wall ratio in the range 0-1.

        """
        set_wwr(self, wwr)

    def view_model(self):
        """Show a zoomable, rotatable representation of the IDF.
        """
        view_idf(idf_txt=self.idfstr())

    def add_block(self, *args, **kwargs):
        """Add a block to the IDF
        """
        block = Block(*args, **kwargs)
        zoning = kwargs.get('zoning', 'by_storey')
        if zoning == 'by_storey':
            zones = [Zone('Storey %i' % storey['storey_no'], storey)
                        for storey in block.stories]
        else:
            raise ValueError('%s is not a valid zoning rule' % zoning)
        for zone in zones:
            self.add_zone(zone)
        self.intersect_match()

    def add_zone(self, zone):
        ggr = self.idfobjects['GLOBALGEOMETRYRULES']
        # add zone object
        self.newidfobject('ZONE', zone.name)
        # add wall objects
        for i, surface_coords in enumerate(zone.walls, 1):
            if not surface_coords:
                continue
            name = '%s Wall %04d' % (zone.name, i)
            s = self.newidfobject('BUILDINGSURFACE:DETAILED', name,
                    Surface_Type = 'wall',
                    Zone_Name = zone.name)
            s.setcoords(surface_coords, ggr)
        # add floor objects
        for i, surface_coords in enumerate(zone.floors, 1):
            if not surface_coords:
                continue
            name = '%s Floor %04d' % (zone.name, i)
            s = self.newidfobject('BUILDINGSURFACE:DETAILED', name,
                    Surface_Type = 'floor',
                    Zone_Name = zone.name)
            s.setcoords(surface_coords, ggr)
        # add ceiling objects
        for i, surface_coords in enumerate(zone.ceilings, 1):
            if not surface_coords:
                continue
            name = '%s Ceiling %04d' % (zone.name, i)
            s = self.newidfobject('BUILDINGSURFACE:DETAILED', name,
                    Surface_Type = 'ceiling',
                    Zone_Name = zone.name)
            s.setcoords(surface_coords, ggr)
        # add roof objects
        for i, surface_coords in enumerate(zone.roofs, 1):
            if not surface_coords:
                continue
            name = '%s Roof %04d' % (zone.name, i)
            s = self.newidfobject('BUILDINGSURFACE:DETAILED', name,
                    Surface_Type = 'roof',
                    Zone_Name = zone.name)
            s.setcoords(surface_coords, ggr)

    def read(self):
        """
        Read the IDF file and the IDD file. If the IDD file had already been
        read, it will not be read again.

        Read populates the following data structures:

        - idfobjects : list
        - model : list
        - idd_info : list

        """
        if self.getiddname() == None:
            errortxt = ("IDD file needed to read the idf file. "
                        "Set it using IDF.setiddname(iddfile)")
            raise IDDNotSetError(errortxt)
        readout = idfreader1(
            self.idfname, self.iddname, self,
            commdct=self.idd_info, block=self.block)
        self.idfobjects, block, self.model, idd_info = readout
        self.__class__.setidd(idd_info, block)

    def newidfobject(self, key, aname='', **kwargs):
        """
        Add a new idfobject to the model. If you don't specify a value for a
        field, the default value will be set.

        For example ::

            newidfobject("CONSTRUCTION")
            newidfobject("CONSTRUCTION",
                Name='Interior Ceiling_class',
                Outside_Layer='LW Concrete',
                Layer_2='soundmat')

        Parameters
        ----------
        key : str
            The type of IDF object. This must be in ALL_CAPS.
        aname : str, deprecated
            This parameter is not used. It is left there for backward
            compatibility.
        **kwargs
            Keyword arguments in the format `field=value` used to set the value
            of fields in the IDF object when it is created.

        Returns
        -------
        EpBunch object

        """
        obj = newrawobject(self.model, self.idd_info, key)
        abunch = obj2bunch(self.model, self.idd_info, obj)
        if aname:
            warning.warn("The aname parameter should no longer be used.")
            namebunch(abunch, aname)
        self.idfobjects[key].append(abunch)
        for k, v in kwargs.items():
            abunch[k] = v
        return abunch

    def copyidfobject(self, idfobject):
        """Add an IDF object to the IDF.

        This has been monkey-patched to add the return value.

        Parameters
        ----------
        idfobject : EpBunch object
            The IDF object to remove. This usually comes from another idf file,
            or it can be used to copy within this idf file.

        Returns
        -------
        EpBunch object

        """
        abunch = addthisbunch(self.idfobjects,
                             self.model,
                             self.idd_info,
                             idfobject,
                             self)

        return abunch
<|endoftext|>


<commit_before>#!/usr/bin/python

import yaml, string, os, codecs, urllib, json, fnmatch, re

try:
    import droplogger
except ImportError:
    # Just until I'm able to make droplogger installable
    import sys
    sys.path.append('/home/drj/MyFiles/Programming/Python/droplogger')
    import droplogger

config = droplogger.read_config()
config['white'] = ["pile"]
config['start'] = droplogger.parse_date('min')
config['end'] = droplogger.parse_date('now')

config['files'] = droplogger.get_files(**config)
entries = droplogger.read_files(**config)

oembed_file = '_data/oembed.yml'

fp = open(oembed_file,'r')
oembed = yaml.load(fp)
fp.close()

for e in entries['pile']:
    e['layout'] = "post"
    e['timestamp'] = int(e["date"].strftime('%s'))
    if "tags" in e: e['tags'] = map(unicode, e['tags'])
    if "url" in e:
        if e["url"] is None: del e["url"]
        else:
            if e['type'] in ["link", "quote"]: e['link_url'] = e.pop("url")
            else: e["embed_url"] = e.pop("url")

    embed_html = None
    if "embed_url" in e and e["type"] not in ["image","picture"]:
        embeds = [ o for o in oembed['urls'] if o['url'] == e["embed_url"] ]
        if len(embeds) == 0:
            providers = [ p for p in oembed['providers'] if "schemes" in p['endpoints'][0] and len([ s for s in p['endpoints'][0]['schemes'] if fnmatch.fnmatch(e["embed_url"], s) ]) ]
            if len(providers) > 0:
                ep = providers[0]['endpoints'][0]['url']
                oembed_url = ep + '?' + urllib.urlencode({"format":"json","url":e['embed_url']})
                try:
                    embed = json.load(urllib.urlopen(oembed_url))
                    embed['url'] = e["embed_url"]
                    oembed["urls"].append(embed)
                    fp = open(oembed_file,'w')
                    yaml.safe_dump(oembed, fp, encoding='utf-8', canonical=False, default_flow_style=False)
                    fp.close()
                except ValueError:
                    embed = None
                embeds = [ embed ]
            else:
                embeds = [ None ]
        if embeds[0] is not None:
            embed_html = embeds[0]['html']
        else:
            # If we failed to get an embed, let's just link it
            e['link_url'] = e.pop("embed_url")
            e['type'] = "link"
    elif "embed_url" in e and e["type"] in ["image","picture"]:
        embed_html = '' % e["embed_url"]
        if "source" in e: embed_html += '\n\n%s' % e["source"]

    slug = ''.join(c for c in e['title'].replace(' ','_') if c in ("-_.%s%s" % (string.ascii_letters, string.digits)))
    filename = "%s-%s-%s-%s.md" % (e['date'].year, e['date'].month, e['date'].day, slug)
    newfile = os.path.join('_posts', filename)
    fp = codecs.open(newfile, "w", encoding="utf-8")
    fp.write('---\n')
    yaml.safe_dump(e, fp, encoding='utf-8', default_flow_style=False)
    fp.write('\n---\n')
    if embed_html is not None:
        fp.write(embed_html)
        fp.write('\n\n')
    cont = ""
    if "text" in e and e['text'] is not None:
        cont = e['text']
    elif "note" in e and e['note'] is not None:
        cont = e['note']
    elif "notes" in e and e['notes'] is not None:
        cont = e['notes']
    if e["type"] == "quote":
        cont = re.sub('^','> ', cont.strip(), flags=re.M)
        cont += '\n> \n'
        cont += '> %s' % e['source'] if "source" in e else "Unknown"
    fp.write(cont)
    fp.close()

    # Next, let's generate archive pages
    if not os.path.isdir('archives'): os.mkdir('archives')
    day_archive = "archives/date-%s.html" % e['date'].strftime('%Y-%m-%d')
    day = {"layout":"archive"}
    day['permalink'] = e['date'].strftime('%Y/%m/%d/')
    day['archive'] = 'day'
    day['title'] = 'Archive for %s' % (e['date'].strftime('%B %d, %Y'))
    day['year'] = "%s" % e['date'].year
    day['month'] = "%s" % e['date'].month
    day['day'] = "%s" % e['date'].day
    day['year-month-day'] = e['date'].strftime('%Y-%m-%d')
    with open(day_archive, 'w') as fp:
        fp.write('---\n')
        yaml.safe_dump(day, fp, encoding='utf-8', canonical=False, default_flow_style=False)
        fp.write('\n---\n')

    month_archive = "archives/date-%s.html" % e['date'].strftime('%Y-%m')
    month = {"layout":"archive"}
    month['permalink'] = e['date'].strftime('%Y/%m/')
    month['archive'] = 'month'
    month['title'] = 'Archive for %s %s' % (e['date'].strftime('%B'), e['date'].year)
    month['year'] = "%s" % e['date'].year
    month['month'] = "%s" % e['date'].month
    month['year-month'] = e['date'].strftime('%Y-%m')
    with open(month_archive, 'w') as fp:
        fp.write('---\n')
        yaml.safe_dump(month, fp, encoding='utf-8', canonical=False, default_flow_style=False)
        fp.write('\n---\n')

    year_archive = "archives/date-%s.html" % e['date'].year
    year = {"layout":"archive"}
    year['permalink'] = e['date'].strftime('%Y/')
    year['archive'] = 'year'
    year['title'] = 'Archive for %s' % e['date'].year
    year['year'] = "%s" % e['date'].year
    with open(year_archive, 'w') as fp:
        fp.write('---\n')
        yaml.safe_dump(year, fp, encoding='utf-8', canonical=False, default_flow_style=False)
        fp.write('\n---\n')

    if "tags" in e:
        for tag in e['tags']:
            tag_archive = "archives/tag-%s.html" % urllib.quote(tag)
            tago = {"layout":"archive"}
            tago['permalink'] = "/tag/%s/" % tag
            tago['archive'] = 'tag'
            tago['title'] = 'Archive for %s' % tag
            tago['tag'] = tag
            with open(tag_archive, 'w') as fp:
                fp.write('---\n')
                yaml.safe_dump(tago, fp, encoding='utf-8', canonical=False, default_flow_style=False)
                fp.write('\n---\n')

    if "type" in e:
        type_archive = "archives/type-%s.html" % urllib.quote(e["type"])
        typeo = {"layout":"archive"}
        typeo['permalink'] = "/type/%s/" % e["type"]
        typeo['archive'] = 'type'
        typeo['title'] = 'Archive for %s posts' % e["type"]
        typeo['type'] = e["type"]
        with open(type_archive, 'w') as fp:
            fp.write('---\n')
            yaml.safe_dump(typeo, fp, encoding='utf-8', canonical=False, default_flow_style=False)
            fp.write('\n---\n')
<commit_msg>[Ref]: Modifying the import to match how it should work<commit_after>#!/usr/bin/python

import yaml, string, os, codecs, urllib, json, fnmatch, re

try:
    import droplogger.droplogger as dl
except ImportError:
    # Just until I'm able to make droplogger installable
    import sys
    sys.path.append('/home/drj/MyFiles/Programming/Python')
    import droplogger.droplogger as dl

config = dl.read_config()
config['white'] = ["pile"]
config['start'] = dl.parse_date('min')
config['end'] = dl.parse_date('now')

config['files'] = dl.get_files(**config)
entries = dl.read_files(**config)

oembed_file = '_data/oembed.yml'

fp = open(oembed_file,'r')
oembed = yaml.load(fp)
fp.close()

for e in entries['pile']:
    e['layout'] = "post"
    e['timestamp'] = int(e["date"].strftime('%s'))
    if "tags" in e: e['tags'] = map(unicode, e['tags'])
    if "url" in e:
        if e["url"] is None: del e["url"]
        else:
            if e['type'] in ["link", "quote"]: e['link_url'] = e.pop("url")
            else: e["embed_url"] = e.pop("url")

    embed_html = None
    if "embed_url" in e and e["type"] not in ["image","picture"]:
        embeds = [ o for o in oembed['urls'] if o['url'] == e["embed_url"] ]
        if len(embeds) == 0:
            providers = [ p for p in oembed['providers'] if "schemes" in p['endpoints'][0] and len([ s for s in p['endpoints'][0]['schemes'] if fnmatch.fnmatch(e["embed_url"], s) ]) ]
            if len(providers) > 0:
                ep = providers[0]['endpoints'][0]['url']
                oembed_url = ep + '?' + urllib.urlencode({"format":"json","url":e['embed_url']})
                try:
                    embed = json.load(urllib.urlopen(oembed_url))
                    embed['url'] = e["embed_url"]
                    oembed["urls"].append(embed)
                    fp = open(oembed_file,'w')
                    yaml.safe_dump(oembed, fp, encoding='utf-8', canonical=False, default_flow_style=False)
                    fp.close()
                except ValueError:
                    embed = None
                embeds = [ embed ]
            else:
                embeds = [ None ]
        if embeds[0] is not None:
            embed_html = embeds[0]['html']
        else:
            # If we failed to get an embed, let's just link it
            e['link_url'] = e.pop("embed_url")
            e['type'] = "link"
    elif "embed_url" in e and e["type"] in ["image","picture"]:
        embed_html = '' % e["embed_url"]
        if "source" in e: embed_html += '\n\n%s' % e["source"]

    slug = ''.join(c for c in e['title'].replace(' ','_') if c in ("-_.%s%s" % (string.ascii_letters, string.digits)))
    filename = "%s-%s-%s-%s.md" % (e['date'].year, e['date'].month, e['date'].day, slug)
    newfile = os.path.join('_posts', filename)
    fp = codecs.open(newfile, "w", encoding="utf-8")
    fp.write('---\n')
    yaml.safe_dump(e, fp, encoding='utf-8', default_flow_style=False)
    fp.write('\n---\n')
    if embed_html is not None:
        fp.write(embed_html)
        fp.write('\n\n')
    cont = ""
    if "text" in e and e['text'] is not None:
        cont = e['text']
    elif "note" in e and e['note'] is not None:
        cont = e['note']
    elif "notes" in e and e['notes'] is not None:
        cont = e['notes']
    if e["type"] == "quote":
        cont = re.sub('^','> ', cont.strip(), flags=re.M)
        cont += '\n> \n'
        cont += '> %s' % e['source'] if "source" in e else "Unknown"
    fp.write(cont)
    fp.close()

    # Next, let's generate archive pages
    if not os.path.isdir('archives'): os.mkdir('archives')
    day_archive = "archives/date-%s.html" % e['date'].strftime('%Y-%m-%d')
    day = {"layout":"archive"}
    day['permalink'] = e['date'].strftime('%Y/%m/%d/')
    day['archive'] = 'day'
    day['title'] = 'Archive for %s' % (e['date'].strftime('%B %d, %Y'))
    day['year'] = "%s" % e['date'].year
    day['month'] = "%s" % e['date'].month
    day['day'] = "%s" % e['date'].day
    day['year-month-day'] = e['date'].strftime('%Y-%m-%d')
    with open(day_archive, 'w') as fp:
        fp.write('---\n')
        yaml.safe_dump(day, fp, encoding='utf-8', canonical=False, default_flow_style=False)
        fp.write('\n---\n')

    month_archive = "archives/date-%s.html" % e['date'].strftime('%Y-%m')
    month = {"layout":"archive"}
    month['permalink'] = e['date'].strftime('%Y/%m/')
    month['archive'] = 'month'
    month['title'] = 'Archive for %s %s' % (e['date'].strftime('%B'), e['date'].year)
    month['year'] = "%s" % e['date'].year
    month['month'] = "%s" % e['date'].month
    month['year-month'] = e['date'].strftime('%Y-%m')
    with open(month_archive, 'w') as fp:
        fp.write('---\n')
        yaml.safe_dump(month, fp, encoding='utf-8', canonical=False, default_flow_style=False)
        fp.write('\n---\n')

    year_archive = "archives/date-%s.html" % e['date'].year
    year = {"layout":"archive"}
    year['permalink'] = e['date'].strftime('%Y/')
    year['archive'] = 'year'
    year['title'] = 'Archive for %s' % e['date'].year
    year['year'] = "%s" % e['date'].year
    with open(year_archive, 'w') as fp:
        fp.write('---\n')
        yaml.safe_dump(year, fp, encoding='utf-8', canonical=False, default_flow_style=False)
        fp.write('\n---\n')

    if "tags" in e:
        for tag in e['tags']:
            tag_archive = "archives/tag-%s.html" % urllib.quote(tag)
            tago = {"layout":"archive"}
            tago['permalink'] = "/tag/%s/" % tag
            tago['archive'] = 'tag'
            tago['title'] = 'Archive for %s' % tag
            tago['tag'] = tag
            with open(tag_archive, 'w') as fp:
                fp.write('---\n')
                yaml.safe_dump(tago, fp, encoding='utf-8', canonical=False, default_flow_style=False)
                fp.write('\n---\n')

    if "type" in e:
        type_archive = "archives/type-%s.html" % urllib.quote(e["type"])
        typeo = {"layout":"archive"}
        typeo['permalink'] = "/type/%s/" % e["type"]
        typeo['archive'] = 'type'
        typeo['title'] = 'Archive for %s posts' % e["type"]
        typeo['type'] = e["type"]
        with open(type_archive, 'w') as fp:
            fp.write('---\n')
            yaml.safe_dump(typeo, fp, encoding='utf-8', canonical=False, default_flow_style=False)
            fp.write('\n---\n')
<|endoftext|>


<commit_before>import os
import sys
import glob
import ablog
import shutil
import argparse

def find_confdir():
    """Return path to current directory or its parent that contains conf.py"""

    from os.path import isfile, join, abspath
    confdir = os.getcwd()

    parent = lambda d: abspath(join(d, '..'))

    while not isfile(join(confdir, 'conf.py')) and confdir != parent(confdir):
        confdir = parent(confdir)

    conf = join(confdir, 'conf.py')

    if isfile(conf) and 'ablog' in open(conf).read():
        return confdir
    else:
        sys.exit("Current directory and its parents doesn't "
            "contain configuration file (conf.py).")

def read_conf(confdir):
    """Return conf.py file as a module."""

    sys.path.insert(0, confdir)
    conf = __import__('conf')
    sys.path.pop(0)
    return conf

parser = argparse.ArgumentParser(
    description="ABlog for blogging with Sphinx",
    epilog="See 'ablog <command> -h' for more information on a specific "
           "command.")

parser.add_argument('-v', '--version',
    help="print ABlog version and exit",
    action='version', version=ablog.__version__)

commands = ablog_commands = parser.add_subparsers(title='commands')

def cmd(func=None, **kwargs):

    if func is None:
        def cmd_inner(func):
            return cmd(func, **kwargs)
        return cmd_inner
    else:
        command = commands.add_parser(**kwargs)
        command.set_defaults(func=func)
        command.set_defaults(subparser=command)
        func.command = command
        return func

def arg(*args, **kwargs):
    if args and callable(args[0]):
        func = args[0]
        args = args[1:]
    else:
        func = None
    if func is None:
        def arg_inner(func):
            return arg(func, *args, **kwargs)
        return arg_inner
    else:
        func.command.add_argument(*args, **kwargs)
        return func

def arg_website(func):

    arg(func, '-w', dest='website', type=str,
        help="path for website, default is _website when `ablog_website` "
            "is not set in conf.py")
    return func

def arg_doctrees(func):

    arg(func, '-d', dest='doctrees', type=str, default='.doctrees',
        help="path for the cached environment and doctree files, "
            "default .doctrees when `ablog_doctrees` is not set in conf.py")
    return func

from .start import ablog_start
cmd(ablog_start, name='start', help='start a new blog project',
    description="Start a new blog project by answering a few questions. "
    "You will end up with a configuration file and sample pages.")

@arg('-P', dest='runpdb',
    action='store_true', default=False,
    help="run pdb on exception")
@arg('-T', dest='traceback',
    action='store_true', default=False,
    help="show full traceback on exception")
@arg_doctrees
@arg_website
@arg('-s', dest='sourcedir', type=str,
    help="root path for source files, "
        "default is path to the folder that contains conf.py")
@arg('-b', dest='builder', type=str,
    help="builder to use, default `ablog_builder` or dirhtml")
@cmd(name='build', help='build your blog project',
    description="Path options can be set in conf.py. "
    "Default values of paths are relative to conf.py.")
def ablog_build(builder=None, sourcedir=None, website=None, doctrees=None,
    traceback=False, runpdb=False, **kwargs):

    confdir = find_confdir()
    conf = read_conf(confdir)
    website = (website or
        os.path.join(confdir, getattr(conf, 'ablog_builddir', '_website')))
    doctrees = (doctrees or
        os.path.join(confdir, getattr(conf, 'ablog_doctrees', '.doctrees')))
    sourcedir = (sourcedir or confdir)
    argv = sys.argv[:1]
    argv.extend(['-b', builder or getattr(conf, 'ablog_builder', 'dirhtml')])
    argv.extend(['-d', doctrees])
    if traceback:
        argv.extend(['-T'])
    if runpdb:
        argv.extend(['-P'])
    argv.extend([sourcedir, website])

    from sphinx import main
    main(argv)

@arg('-D', dest='deep', action='store_true', default=False,
    help="deep clean, remove cached environment and doctree files")
@arg_doctrees
@arg_website
@cmd(name='clean', help='clean your blog build files',
    description="Path options can be set in conf.py. "
    "Default values of paths are relative to conf.py.")
def ablog_clean(website=None, doctrees=None, deep=False, **kwargs):

    confdir = find_confdir()
    conf = read_conf(confdir)

    website = (website or
        os.path.join(confdir, getattr(conf, 'ablog_builddir', '_website')))

    doctrees = (doctrees or
        os.path.join(confdir, getattr(conf, 'ablog_doctrees', '.doctrees')))

    nothing = True
    if glob.glob(os.path.join(website, '*')):
        shutil.rmtree(website)
        print('Removed {}.'.format(os.path.relpath(website)))
        nothing = False

    if deep and glob.glob(os.path.join(doctrees, '*')):
        shutil.rmtree(doctrees)
        print('Removed {}.'.format(os.path.relpath(doctrees)))
        nothing = False

    if nothing:
        print('Nothing to clean.')

@arg('--patterns', dest='rebuild', action='store_false', default='*.rst;*.txt',
    help="patterns for triggering rebuilds")
@arg('-r', dest='rebuild', action='store_true', default=False,
    help="rebuild when a file matching patterns change or get added")
@arg('-n', dest='view', action='store_false', default=True,
    help="do not open website in a new browser tab")
@arg('-p', dest='port', type=int, default=8000,
    help='port number for HTTP server; default is 8000')
@arg_website
@cmd(name='serve', help='serve and view your project',
    description="Serve options can be set in conf.py. "
    "Default values of paths are relative to conf.py.")
def ablog_serve(website=None, port=8000, view=True, rebuild=False,
    patterns='*.rst;*.txt', **kwargs):

    confdir = find_confdir()
    conf = read_conf(confdir)

    try:
        import SimpleHTTPServer as server
    except ImportError:
        from http import server
        import socketserver
    else:
        import SocketServer as socketserver

    import webbrowser

    Handler = server.SimpleHTTPRequestHandler
    httpd = socketserver.TCPServer(("", port), Handler)

    ip, port = httpd.socket.getsockname()
    print("Serving HTTP on {}:{}.".format(ip, port))
    print("Quit the server with Control-C.")

    website = (website or
        os.path.join(confdir, getattr(conf, 'ablog_builddir', '_website')))

    os.chdir(website)

    if rebuild:

        #from watchdog.watchmedo import observe_with
        from watchdog.observers import Observer
        from watchdog.tricks import ShellCommandTrick
        patterns = patterns.split(';')
        ignore_patterns = [os.path.join(website, '*')]
        handler = ShellCommandTrick(shell_command='ablog build',
                                    patterns=patterns,
                                    ignore_patterns=ignore_patterns,
                                    ignore_directories=False,
                                    wait_for_process=True,
                                    drop_during_process=False)

        observer = Observer(timeout=1)
        observer.schedule(handler, confdir, recursive=True)
        observer.start()
        try:
            if view:
                (webbrowser.open_new_tab('http://127.0.0.1:{}'.format(port)) and
                 httpd.serve_forever())
            else:
                httpd.serve_forever()
        except KeyboardInterrupt:
            observer.stop()
        observer.join()

    else:
        if view:
            (webbrowser.open_new_tab('http://127.0.0.1:{}'.format(port)) and
             httpd.serve_forever())
        else:
            httpd.serve_forever()

@arg('-t', dest='title', type=str,
    help='post title; default is formed from filename')
@arg(dest='filename', type=str,
    help='filename, e.g. my-nth-post (.rst appended)')
@cmd(name='post', help='create a blank post',)
def ablog_post(filename, title=None, **kwargs):

    POST_TEMPLATE =u'''
%(title)s
%(equal)s

.. post:: %(date)s
   :tags:
   :category:

'''
    from datetime import date
    from os import path

    #Generate basic post params.
    today = date.today()
    if not filename.lower().endswith('.rst'):
        filename += '.rst'

    today = today.strftime("%b %d, %Y")
    if not title:
        title = filename[:-4].replace('-', ' ').title()

    pars = {'date': today,
            'title': title,
            'equal': '=' * len(title)
            }

    if path.isfile(filename):
        pass
        # read the file, and add post directive
        # and save it
    else:
        with open(filename, 'w') as out:
            post_text = POST_TEMPLATE % pars
            out.write(post_text)

        print('Blog post created: %s' % filename)

@arg('--github-token', dest='github_token', type=str,
    help="environment variable name storing GitHub access token")
@arg('--push-quietly', dest='push_quietly', action='store_true', default=False,
    help="be more quiet when pushing changes")
@arg('-m', dest='message', type=str, help="commit message")
@arg('-g', dest='github_pages', type=str,
    help="GitHub username for deploying to GitHub pages")
@arg_website
@cmd(name='deploy', help='deploy your website build files',
    description="Path options can be set in conf.py. "
    "Default values of paths are relative to conf.py.")
def ablog_deploy(website, message=None, github_pages=None,
    push_quietly=False, github_token=None, **kwargs):

    confdir = find_confdir()
    conf = read_conf(confdir)

    github_pages = (github_pages or getattr(conf, 'github_pages') or None)

    website = (website or
        os.path.join(confdir, getattr(conf, 'ablog_builddir', '_website')))

    tomove = glob.glob(os.path.join(website, '*'))
    if not tomove:
        print('Nothing to deploy, build first.')
        return

    try:
        from invoke import run
    except ImportError:
        raise ImportError("invoke is required by deploy command, "
            "run `pip install invoke`")

    if github_pages:

        gitdir = os.path.join(confdir, "{0}.github.io".format(github_pages))
        if os.path.isdir(gitdir):
            os.chdir(gitdir)
            run("git pull", echo=True)
        else:
            run("git clone https://github.com/{0}/{0}.github.io.git"
                .format(github_pages), echo=True)

        git_add = []
        for tm in tomove:
            for root, dirnames, filenames in os.walk(website):
                for filename in filenames:
                    fn = os.path.join(root, filename)
                    fnnew = fn.replace(website, gitdir)
                    os.renames(fn, fnnew)
                    git_add.append(fnnew)
        print('Moved {} files to {}.github.io'
            .format(len(git_add), github_pages))

        os.chdir(gitdir)

        run("git add -f " +
            " ".join([os.path.relpath(p) for p in git_add]), echo=True)
        if not os.path.isfile('.nojekyll'):
            open('.nojekyll', 'w')
            run("git add -f .nojekyll")

        run('git commit -m "{}"'.format(message or 'Updates.', echo=True))

        if github_token:
            with open(os.path.join(gitdir, '.git/credentials'), 'w') as out:
                out.write('https://{}:@github.com'
                    .format(os.environ[github_token]))
            run('git config credential.helper "store --file=.git/credentials"')
        push = 'git push'
        if push_quietly:
            push += ' -q'
        push += ' origin master'
        run(push, echo=True)

    else:
        print('No place to deploy.')

def ablog_main():

    if len(sys.argv) == 1:
        parser.print_help()
    else:
        namespace = parser.parse_args()
        namespace.func(**namespace.__dict__)

if __name__ == '__main__':
    ablog_main()
<commit_msg>escape filenames with git add -f ...<commit_after>import os
import sys
import glob
import ablog
import shutil
import argparse

def find_confdir():
    """Return path to current directory or its parent that contains conf.py"""

    from os.path import isfile, join, abspath
    confdir = os.getcwd()

    parent = lambda d: abspath(join(d, '..'))

    while not isfile(join(confdir, 'conf.py')) and confdir != parent(confdir):
        confdir = parent(confdir)

    conf = join(confdir, 'conf.py')

    if isfile(conf) and 'ablog' in open(conf).read():
        return confdir
    else:
        sys.exit("Current directory and its parents doesn't "
            "contain configuration file (conf.py).")

def read_conf(confdir):
    """Return conf.py file as a module."""

    sys.path.insert(0, confdir)
    conf = __import__('conf')
    sys.path.pop(0)
    return conf

parser = argparse.ArgumentParser(
    description="ABlog for blogging with Sphinx",
    epilog="See 'ablog <command> -h' for more information on a specific "
           "command.")

parser.add_argument('-v', '--version',
    help="print ABlog version and exit",
    action='version', version=ablog.__version__)

commands = ablog_commands = parser.add_subparsers(title='commands')

def cmd(func=None, **kwargs):

    if func is None:
        def cmd_inner(func):
            return cmd(func, **kwargs)
        return cmd_inner
    else:
        command = commands.add_parser(**kwargs)
        command.set_defaults(func=func)
        command.set_defaults(subparser=command)
        func.command = command
        return func

def arg(*args, **kwargs):
    if args and callable(args[0]):
        func = args[0]
        args = args[1:]
    else:
        func = None
    if func is None:
        def arg_inner(func):
            return arg(func, *args, **kwargs)
        return arg_inner
    else:
        func.command.add_argument(*args, **kwargs)
        return func

def arg_website(func):

    arg(func, '-w', dest='website', type=str,
        help="path for website, default is _website when `ablog_website` "
            "is not set in conf.py")
    return func

def arg_doctrees(func):

    arg(func, '-d', dest='doctrees', type=str, default='.doctrees',
        help="path for the cached environment and doctree files, "
            "default .doctrees when `ablog_doctrees` is not set in conf.py")
    return func

from .start import ablog_start
cmd(ablog_start, name='start', help='start a new blog project',
    description="Start a new blog project by answering a few questions. "
    "You will end up with a configuration file and sample pages.")

@arg('-P', dest='runpdb',
    action='store_true', default=False,
    help="run pdb on exception")
@arg('-T', dest='traceback',
    action='store_true', default=False,
    help="show full traceback on exception")
@arg_doctrees
@arg_website
@arg('-s', dest='sourcedir', type=str,
    help="root path for source files, "
        "default is path to the folder that contains conf.py")
@arg('-b', dest='builder', type=str,
    help="builder to use, default `ablog_builder` or dirhtml")
@cmd(name='build', help='build your blog project',
    description="Path options can be set in conf.py. "
    "Default values of paths are relative to conf.py.")
def ablog_build(builder=None, sourcedir=None, website=None, doctrees=None,
    traceback=False, runpdb=False, **kwargs):

    confdir = find_confdir()
    conf = read_conf(confdir)
    website = (website or
        os.path.join(confdir, getattr(conf, 'ablog_builddir', '_website')))
    doctrees = (doctrees or
        os.path.join(confdir, getattr(conf, 'ablog_doctrees', '.doctrees')))
    sourcedir = (sourcedir or confdir)
    argv = sys.argv[:1]
    argv.extend(['-b', builder or getattr(conf, 'ablog_builder', 'dirhtml')])
    argv.extend(['-d', doctrees])
    if traceback:
        argv.extend(['-T'])
    if runpdb:
        argv.extend(['-P'])
    argv.extend([sourcedir, website])

    from sphinx import main
    main(argv)

@arg('-D', dest='deep', action='store_true', default=False,
    help="deep clean, remove cached environment and doctree files")
@arg_doctrees
@arg_website
@cmd(name='clean', help='clean your blog build files',
    description="Path options can be set in conf.py. "
    "Default values of paths are relative to conf.py.")
def ablog_clean(website=None, doctrees=None, deep=False, **kwargs):

    confdir = find_confdir()
    conf = read_conf(confdir)

    website = (website or
        os.path.join(confdir, getattr(conf, 'ablog_builddir', '_website')))

    doctrees = (doctrees or
        os.path.join(confdir, getattr(conf, 'ablog_doctrees', '.doctrees')))

    nothing = True
    if glob.glob(os.path.join(website, '*')):
        shutil.rmtree(website)
        print('Removed {}.'.format(os.path.relpath(website)))
        nothing = False

    if deep and glob.glob(os.path.join(doctrees, '*')):
        shutil.rmtree(doctrees)
        print('Removed {}.'.format(os.path.relpath(doctrees)))
        nothing = False

    if nothing:
        print('Nothing to clean.')

@arg('--patterns', dest='rebuild', action='store_false', default='*.rst;*.txt',
    help="patterns for triggering rebuilds")
@arg('-r', dest='rebuild', action='store_true', default=False,
    help="rebuild when a file matching patterns change or get added")
@arg('-n', dest='view', action='store_false', default=True,
    help="do not open website in a new browser tab")
@arg('-p', dest='port', type=int, default=8000,
    help='port number for HTTP server; default is 8000')
@arg_website
@cmd(name='serve', help='serve and view your project',
    description="Serve options can be set in conf.py. "
    "Default values of paths are relative to conf.py.")
def ablog_serve(website=None, port=8000, view=True, rebuild=False,
    patterns='*.rst;*.txt', **kwargs):

    confdir = find_confdir()
    conf = read_conf(confdir)

    try:
        import SimpleHTTPServer as server
    except ImportError:
        from http import server
        import socketserver
    else:
        import SocketServer as socketserver

    import webbrowser

    Handler = server.SimpleHTTPRequestHandler
    httpd = socketserver.TCPServer(("", port), Handler)

    ip, port = httpd.socket.getsockname()
    print("Serving HTTP on {}:{}.".format(ip, port))
    print("Quit the server with Control-C.")

    website = (website or
        os.path.join(confdir, getattr(conf, 'ablog_builddir', '_website')))

    os.chdir(website)

    if rebuild:

        #from watchdog.watchmedo import observe_with
        from watchdog.observers import Observer
        from watchdog.tricks import ShellCommandTrick
        patterns = patterns.split(';')
        ignore_patterns = [os.path.join(website, '*')]
        handler = ShellCommandTrick(shell_command='ablog build',
                                    patterns=patterns,
                                    ignore_patterns=ignore_patterns,
                                    ignore_directories=False,
                                    wait_for_process=True,
                                    drop_during_process=False)

        observer = Observer(timeout=1)
        observer.schedule(handler, confdir, recursive=True)
        observer.start()
        try:
            if view:
                (webbrowser.open_new_tab('http://127.0.0.1:{}'.format(port)) and
                 httpd.serve_forever())
            else:
                httpd.serve_forever()
        except KeyboardInterrupt:
            observer.stop()
        observer.join()

    else:
        if view:
            (webbrowser.open_new_tab('http://127.0.0.1:{}'.format(port)) and
             httpd.serve_forever())
        else:
            httpd.serve_forever()

@arg('-t', dest='title', type=str,
    help='post title; default is formed from filename')
@arg(dest='filename', type=str,
    help='filename, e.g. my-nth-post (.rst appended)')
@cmd(name='post', help='create a blank post',)
def ablog_post(filename, title=None, **kwargs):

    POST_TEMPLATE =u'''
%(title)s
%(equal)s

.. post:: %(date)s
   :tags:
   :category:

'''
    from datetime import date
    from os import path

    #Generate basic post params.
    today = date.today()
    if not filename.lower().endswith('.rst'):
        filename += '.rst'

    today = today.strftime("%b %d, %Y")
    if not title:
        title = filename[:-4].replace('-', ' ').title()

    pars = {'date': today,
            'title': title,
            'equal': '=' * len(title)
            }

    if path.isfile(filename):
        pass
        # read the file, and add post directive
        # and save it
    else:
        with open(filename, 'w') as out:
            post_text = POST_TEMPLATE % pars
            out.write(post_text)

        print('Blog post created: %s' % filename)

@arg('--github-token', dest='github_token', type=str,
    help="environment variable name storing GitHub access token")
@arg('--push-quietly', dest='push_quietly', action='store_true', default=False,
    help="be more quiet when pushing changes")
@arg('-m', dest='message', type=str, help="commit message")
@arg('-g', dest='github_pages', type=str,
    help="GitHub username for deploying to GitHub pages")
@arg_website
@cmd(name='deploy', help='deploy your website build files',
    description="Path options can be set in conf.py. "
    "Default values of paths are relative to conf.py.")
def ablog_deploy(website, message=None, github_pages=None,
    push_quietly=False, github_token=None, **kwargs):

    confdir = find_confdir()
    conf = read_conf(confdir)

    github_pages = (github_pages or getattr(conf, 'github_pages') or None)

    website = (website or
        os.path.join(confdir, getattr(conf, 'ablog_builddir', '_website')))

    tomove = glob.glob(os.path.join(website, '*'))
    if not tomove:
        print('Nothing to deploy, build first.')
        return

    try:
        from invoke import run
    except ImportError:
        raise ImportError("invoke is required by deploy command, "
            "run `pip install invoke`")

    if github_pages:

        gitdir = os.path.join(confdir, "{0}.github.io".format(github_pages))
        if os.path.isdir(gitdir):
            os.chdir(gitdir)
            run("git pull", echo=True)
        else:
            run("git clone https://github.com/{0}/{0}.github.io.git"
                .format(github_pages), echo=True)

        git_add = []
        for tm in tomove:
            for root, dirnames, filenames in os.walk(website):
                for filename in filenames:
                    fn = os.path.join(root, filename)
                    fnnew = fn.replace(website, gitdir)
                    os.renames(fn, fnnew)
                    git_add.append(fnnew)
        print('Moved {} files to {}.github.io'
            .format(len(git_add), github_pages))

        os.chdir(gitdir)

        run("git add -f " + " ".join(['"{}"'.format(os.path.relpath(p))
                                      for p in git_add]), echo=True)
        if not os.path.isfile('.nojekyll'):
            open('.nojekyll', 'w')
            run("git add -f .nojekyll")

        run('git commit -m "{}"'.format(message or 'Updates.', echo=True))

        if github_token:
            with open(os.path.join(gitdir, '.git/credentials'), 'w') as out:
                out.write('https://{}:@github.com'
                    .format(os.environ[github_token]))
            run('git config credential.helper "store --file=.git/credentials"')
        push = 'git push'
        if push_quietly:
            push += ' -q'
        push += ' origin master'
        run(push, echo=True)

    else:
        print('No place to deploy.')

def ablog_main():

    if len(sys.argv) == 1:
        parser.print_help()
    else:
        namespace = parser.parse_args()
        namespace.func(**namespace.__dict__)

if __name__ == '__main__':
    ablog_main()
<|endoftext|>


<commit_before># coding: utf-8

from django.forms import CharField, ModelChoiceField, ModelMultipleChoiceField
from django.forms.widgets import CheckboxSelectMultiple
from registration.forms import RegistrationFormUniqueEmail
from django.utils.translation import ugettext_lazy as _
from .models import StudentProfile
from django.contrib.auth.models import User, Group
from crispy_forms.helper import FormHelper
from crispy_forms.layout import Layout, Submit, Field, Reset, Fieldset
from crispy_forms.bootstrap import PrependedText, FormActions
from django.contrib.sites.models import get_current_site
from django.template.loader import render_to_string

def get_professors():
    return User.objects.filter(is_superuser=True)

def get_groups():
    return Group.objects.all()

class UserField(ModelChoiceField):
    def label_from_instance(self, obj):
        return obj.get_full_name() or unicode(u)

class UserRegistrationForm(RegistrationFormUniqueEmail):
    first_name = CharField(label=_(u'Prénom(s)'))
    last_name = CharField(label=_('Nom'))
    professor = UserField(queryset=get_professors(), label=_('Professeur'))
    groups = ModelMultipleChoiceField(queryset=get_groups(),
                             widget=CheckboxSelectMultiple, label=_('Groupes'))

    def __init__(self, *args, **kwargs):
        self.helper = FormHelper()
        self.helper.form_class = 'form-horizontal'
        self.helper.layout = Layout(
            Fieldset(
                _(u'Général'),
                Field('first_name', 'last_name', css_class='input-xlarge'),
                PrependedText('email', '', active=True),
            ),
            Fieldset(
                _('Utilisateur'),
                PrependedText('username', '', active=True),
                'password1', 'password2',
            ),
            Fieldset(
                _(u'Étudiant'),
                'professor',
                'groups',
            ),
            FormActions(
                Submit('save_changes', _('Enregistrer'), css_class="btn-primary"),
                Reset('reset', _(u'Réinitialiser')),
            ),
        )
        super(UserRegistrationForm, self).__init__(*args, **kwargs)

    def save(self, request, user):
        data = self.cleaned_data

        user.first_name = data['first_name']
        user.last_name = data['last_name']
        user.groups = data['groups']
        user.save()

        professor = data['professor']
        profile = StudentProfile.objects.create(
            user=user,
            professor=professor)
        profile.save()

        site_url = 'http://' + get_current_site(request).domain
        email_content = render_to_string(
            'accounts/grant_to_admin_demand_email.txt',
            {'user': user, 'site_url': site_url})
        professor.email_user(_(u'Demande d’accès étudiant'),
            email_content)
<commit_msg>Autorise plusieurs comptes sur une adresse mail.<commit_after># coding: utf-8

from django.forms import CharField, ModelChoiceField, ModelMultipleChoiceField
from django.forms.widgets import CheckboxSelectMultiple
# FIXME: Remplacer ceci par RegistrationFormUniqueEmail quand Joann aura fini
# de faire mumuse.
from registration.forms import RegistrationForm
from django.utils.translation import ugettext_lazy as _
from .models import StudentProfile
from django.contrib.auth.models import User, Group
from crispy_forms.helper import FormHelper
from crispy_forms.layout import Layout, Submit, Field, Reset, Fieldset
from crispy_forms.bootstrap import PrependedText, FormActions
from django.contrib.sites.models import get_current_site
from django.template.loader import render_to_string

def get_professors():
    return User.objects.filter(is_superuser=True)

def get_groups():
    return Group.objects.all()

class UserField(ModelChoiceField):
    def label_from_instance(self, obj):
        return obj.get_full_name() or unicode(u)

class UserRegistrationForm(RegistrationForm):
    first_name = CharField(label=_(u'Prénom(s)'))
    last_name = CharField(label=_('Nom'))
    professor = UserField(queryset=get_professors(), label=_('Professeur'))
    groups = ModelMultipleChoiceField(queryset=get_groups(),
                             widget=CheckboxSelectMultiple, label=_('Groupes'))

    def __init__(self, *args, **kwargs):
        self.helper = FormHelper()
        self.helper.form_class = 'form-horizontal'
        self.helper.layout = Layout(
            Fieldset(
                _(u'Général'),
                Field('first_name', 'last_name', css_class='input-xlarge'),
                PrependedText('email', '', active=True),
            ),
            Fieldset(
                _('Utilisateur'),
                PrependedText('username', '', active=True),
                'password1', 'password2',
            ),
            Fieldset(
                _(u'Étudiant'),
                'professor',
                'groups',
            ),
            FormActions(
                Submit('save_changes', _('Enregistrer'), css_class="btn-primary"),
                Reset('reset', _(u'Réinitialiser')),
            ),
        )
        super(UserRegistrationForm, self).__init__(*args, **kwargs)

    def save(self, request, user):
        data = self.cleaned_data

        user.first_name = data['first_name']
        user.last_name = data['last_name']
        user.groups = data['groups']
        user.save()

        professor = data['professor']
        profile = StudentProfile.objects.create(
            user=user,
            professor=professor)
        profile.save()

        site_url = 'http://' + get_current_site(request).domain
        email_content = render_to_string(
            'accounts/grant_to_admin_demand_email.txt',
            {'user': user, 'site_url': site_url})
        professor.email_user(_(u'Demande d’accès étudiant'),
            email_content)
<|endoftext|>


<commit_before>import os
import random
import traceback
from collections import defaultdict
from datetime import date
import time

import vk_api

from vk_text_likeness.lda_maker import LdaMaker
from vk_text_likeness.tools import cache_by_entity_id
from vk_text_likeness.logs import log_method_begin, log_method_end

class RawUsersData:
    def __init__(self, group_id, vk_session):
        self.group_id = group_id
        self.vk_session = vk_session
        self.vk = self.vk_session.get_api()
        self.vk_tools = vk_api.VkTools(self.vk_session)

        self.members = None
        self.member_friends = None

        self.member_fields = 'sex,bdate,country'
        self.group_fields = 'description'

        self.fetch_groups_mark_file = 'RawUsersData._fetch_groups_error{}'.format(self.group_id)

    def fetch(self):
        self._fetch_members()

    def fetch_more(self, liked_users_set, reposted_users_set):
        self._fetch_member_friends(reposted_users_set)
        self._fetch_groups(liked_users_set | self._sample_user_ids(len(liked_users_set) * 2, without=liked_users_set))

    def _fetch_members(self):
        if self.members is not None:
            return
        log_method_begin()

        self.members = self.vk_tools.get_all(
            'groups.getMembers', 1000, {'group_id': self.group_id, 'fields': self.member_fields}
        )['items']
        print('{} members'.format(len(self.members)))

        for member in self.members:
            member['is_member'] = True

        log_method_end()

    def _fetch_member_friends(self, user_subset):
        if self.member_friends is not None:
            return
        log_method_begin()

        members = [member for member in self.members if member['id'] in user_subset]
        print('{} users to fetch'.format(len(members)))

        pool_results = []

        with vk_api.VkRequestsPool(self.vk_session) as pool:
            for member in members:
                pool_results.append(
                    (member['id'], pool.method('friends.get', {'user_id': member['id'], 'fields': 'photo'}))
                )

        self.member_friends = defaultdict(list)
        for member_id, friend_request in pool_results:
            if friend_request.ok:
                for friend in friend_request.result['items']:
                    if friend['id'] not in user_subset:
                        friend['is_member'] = False
                        self.member_friends[member_id].append(friend)

        log_method_end()

    def _fetch_groups(self, user_subset):
        log_method_begin()

        users = [user for user in self.get_all_users() if user['id'] in user_subset]
        print('{} users to fetch'.format(len(users)))

        do_fetch = True
        last_error_time = -1
        while do_fetch:
            try:
                pool_results = []
                with vk_api.VkRequestsPool(self.vk_session) as pool:
                    for user in users:
                        if 'groups' not in user:
                            pool_results.append(
                                (user, pool.method('groups.get', {'user_id': user['id'], 'count': 1000, 'extended': 1, 'fields': self.group_fields}))
                            )
                do_fetch = False
                self.unmark_fetch_groups_error()
            except Exception as e:
                print('Can\'t fetch groups because of', e)
                traceback.print_exc()
                if time.time() - last_error_time < 120:
                    print('Can\'t do anything, exit. Restart will reuse fetched users')
                    do_fetch = False
                    self.mark_fetch_groups_error()
                else:
                    print('Trying again in 1 minute')
                    time.sleep(60)
                last_error_time = time.time()
            finally:
                for user, groups_request in pool_results:
                    if groups_request.ok and groups_request.ready:
                        user['groups'] = groups_request.result['items']

        log_method_end()

    def was_fetch_groups_error(self):
        return os.path.isfile(self.fetch_groups_mark_file)

    def mark_fetch_groups_error(self):
        with open(self.fetch_groups_mark_file, 'w') as f:
            f.write('True')

    def unmark_fetch_groups_error(self):
        if self.was_fetch_groups_error():
            os.remove(self.fetch_groups_mark_file)

    def find_user(self, user_id):
        for user in self.members:
            if user['id'] == user_id:
                return user

        for users in self.member_friends.values():
            for user in users:
                if user['id'] == user_id:
                    return user

        return None

    def get_all_users(self):
        everybody = []
        ids = set()
        for users in [self.members] + list(self.member_friends.values()):
            for user in users:
                if user['id'] not in ids:
                    everybody.append(user)
                    ids.add(user['id'])
        return everybody

    def _sample_user_ids(self, n, without=set()):
        ids = set()
        for users in [self.members] + list(self.member_friends.values()):
            for user in users:
                if user['id'] not in ids and user['id'] not in without:
                    ids.add(user['id'])
        return set(random.sample(ids, n))

class TableUsersData:
    def __init__(self, raw_users_data, num_topics=15):
        self.raw_users_data = raw_users_data
        self.num_topics = num_topics

    def fit(self):
        log_method_begin()
        self.lda_maker = LdaMaker(self._get_corpora_for_lda(), self.num_topics)
        log_method_end()

    @cache_by_entity_id
    def get_row(self, user):
        return [self._user_is_woman(user), self._user_is_man(user), self._user_age(user),
                self._user_is_in_russia(user), self._user_is_in_ukraine(user), self._user_is_in_byelorussia(user),
                self._user_is_in_kazakstan(user)] + \
               self._user_lda_by_groups(user)

    def get_labels(self):
        return (['is_woman', 'is_man', 'age',
                 'is_in_russia', 'is_in_ukraine', 'is_in_byelorussia', 'is_in_kazakstan'] +
                ['user_lda' + str(i) for i in range(self.lda_maker.num_topics)])

    def _get_corpora_for_lda(self):
        corpora = set()
        for user in self.raw_users_data.get_all_users():
            if 'groups' in user:
                for group in user['groups']:
                    try:
                        doc = group['description']
                        corpora.add(doc)
                    except KeyError:
                        pass
        return list(corpora)

    @staticmethod
    def _user_is_woman(user):
        if 'sex' not in user:
            return False
        return user['sex'] == 1

    @staticmethod
    def _user_is_man(user):
        if 'sex' not in user:
            return False
        return user['sex'] == 2

    @staticmethod
    def _user_age(user):
        if 'bdate' in user:
            bdate_parts = user['bdate'].split('.')
            if len(bdate_parts) == 3:
                year = int(bdate_parts[-1])
                return date.today().year - year
            else:
                return -1
        else:
            return -1

    @staticmethod
    def _user_is_in_russia(user):
        if 'country' not in user:
            return False
        return user['country']['id'] == 1

    @staticmethod
    def _user_is_in_ukraine(user):
        if 'country' not in user:
            return False
        return user['country']['id'] == 2

    @staticmethod
    def _user_is_in_byelorussia(user):
        if 'country' not in user:
            return False
        return user['country']['id'] == 3

    @staticmethod
    def _user_is_in_kazakstan(user):
        if 'country' not in user:
            return False
        return user['country']['id'] == 4

    def _user_lda_by_groups(self, user):
        result = [0] * self.lda_maker.num_topics
        lda_count = 0
        if 'groups' in user:
            for group in user['groups']:
                try:
                    lda_desc = self.lda_maker.get(group['description'])
                    for k, v in lda_desc:
                        result[k] += v
                    lda_count += 1
                except KeyError:
                    pass
            if lda_count != 0:
                for i in range(len(result)):
                    result[i] /= lda_count
        return result
<commit_msg>Change<commit_after>import os
import random
import traceback
from collections import defaultdict
from datetime import date
import time

import vk_api

from vk_text_likeness.lda_maker import LdaMaker
from vk_text_likeness.tools import cache_by_entity_id
from vk_text_likeness.logs import log_method_begin, log_method_end

class RawUsersData:
    def __init__(self, group_id, vk_session):
        self.group_id = group_id
        self.vk_session = vk_session
        self.vk = self.vk_session.get_api()
        self.vk_tools = vk_api.VkTools(self.vk_session)

        self.members = None
        self.member_friends = None

        self.member_fields = 'sex,bdate,country'
        self.group_fields = 'description'

        self.fetch_groups_mark_file = 'RawUsersData._fetch_groups_error{}'.format(self.group_id)

    def fetch(self):
        self._fetch_members()

    def fetch_more(self, liked_users_set, reposted_users_set):
        self._fetch_member_friends(reposted_users_set)
        self._fetch_groups(liked_users_set | self._sample_user_ids(len(liked_users_set), without=liked_users_set))

    def _fetch_members(self):
        if self.members is not None:
            return
        log_method_begin()

        self.members = self.vk_tools.get_all(
            'groups.getMembers', 1000, {'group_id': self.group_id, 'fields': self.member_fields}
        )['items']
        print('{} members'.format(len(self.members)))

        for member in self.members:
            member['is_member'] = True

        log_method_end()

    def _fetch_member_friends(self, user_subset):
        if self.member_friends is not None:
            return
        log_method_begin()

        members = [member for member in self.members if member['id'] in user_subset]
        print('{} users to fetch'.format(len(members)))

        pool_results = []

        with vk_api.VkRequestsPool(self.vk_session) as pool:
            for member in members:
                pool_results.append(
                    (member['id'], pool.method('friends.get', {'user_id': member['id'], 'fields': 'photo'}))
                )

        self.member_friends = defaultdict(list)
        for member_id, friend_request in pool_results:
            if friend_request.ok:
                for friend in friend_request.result['items']:
                    if friend['id'] not in user_subset:
                        friend['is_member'] = False
                        self.member_friends[member_id].append(friend)

        log_method_end()

    def _fetch_groups(self, user_subset):
        log_method_begin()

        users = [user for user in self.get_all_users() if user['id'] in user_subset]
        print('{} users to fetch'.format(len(users)))

        do_fetch = True
        last_error_time = -1
        while do_fetch:
            try:
                pool_results = []
                with vk_api.VkRequestsPool(self.vk_session) as pool:
                    for user in users:
                        if 'groups' not in user:
                            pool_results.append(
                                (user, pool.method('groups.get', {'user_id': user['id'], 'count': 1000, 'extended': 1, 'fields': self.group_fields}))
                            )
                do_fetch = False
                self.unmark_fetch_groups_error()
            except Exception as e:
                print('Can\'t fetch groups because of', e)
                traceback.print_exc()
                if time.time() - last_error_time < 120:
                    print('Can\'t do anything, exit. Restart will reuse fetched users')
                    do_fetch = False
                    self.mark_fetch_groups_error()
                else:
                    print('Trying again in 1 minute')
                    time.sleep(60)
                last_error_time = time.time()
            finally:
                for user, groups_request in pool_results:
                    if groups_request.ok and groups_request.ready:
                        user['groups'] = groups_request.result['items']

        log_method_end()

    def was_fetch_groups_error(self):
        return os.path.isfile(self.fetch_groups_mark_file)

    def mark_fetch_groups_error(self):
        with open(self.fetch_groups_mark_file, 'w') as f:
            f.write('True')

    def unmark_fetch_groups_error(self):
        if self.was_fetch_groups_error():
            os.remove(self.fetch_groups_mark_file)

    def find_user(self, user_id):
        for user in self.members:
            if user['id'] == user_id:
                return user

        for users in self.member_friends.values():
            for user in users:
                if user['id'] == user_id:
                    return user

        return None

    def get_all_users(self):
        everybody = []
        ids = set()
        for users in [self.members] + list(self.member_friends.values()):
            for user in users:
                if user['id'] not in ids:
                    everybody.append(user)
                    ids.add(user['id'])
        return everybody

    def _sample_user_ids(self, n, without=set()):
        ids = set()
        for users in [self.members] + list(self.member_friends.values()):
            for user in users:
                if user['id'] not in ids and user['id'] not in without:
                    ids.add(user['id'])
        return set(random.sample(ids, n))

class TableUsersData:
    def __init__(self, raw_users_data, num_topics=15):
        self.raw_users_data = raw_users_data
        self.num_topics = num_topics

    def fit(self):
        log_method_begin()
        self.lda_maker = LdaMaker(self._get_corpora_for_lda(), self.num_topics)
        log_method_end()

    @cache_by_entity_id
    def get_row(self, user):
        return [self._user_is_woman(user), self._user_is_man(user), self._user_age(user),
                self._user_is_in_russia(user), self._user_is_in_ukraine(user), self._user_is_in_byelorussia(user),
                self._user_is_in_kazakstan(user)] + \
               self._user_lda_by_groups(user)

    def get_labels(self):
        return (['is_woman', 'is_man', 'age',
                 'is_in_russia', 'is_in_ukraine', 'is_in_byelorussia', 'is_in_kazakstan'] +
                ['user_lda' + str(i) for i in range(self.lda_maker.num_topics)])

    def _get_corpora_for_lda(self):
        corpora = set()
        for user in self.raw_users_data.get_all_users():
            if 'groups' in user:
                for group in user['groups']:
                    try:
                        doc = group['description']
                        corpora.add(doc)
                    except KeyError:
                        pass
        return list(corpora)

    @staticmethod
    def _user_is_woman(user):
        if 'sex' not in user:
            return False
        return user['sex'] == 1

    @staticmethod
    def _user_is_man(user):
        if 'sex' not in user:
            return False
        return user['sex'] == 2

    @staticmethod
    def _user_age(user):
        if 'bdate' in user:
            bdate_parts = user['bdate'].split('.')
            if len(bdate_parts) == 3:
                year = int(bdate_parts[-1])
                return date.today().year - year
            else:
                return -1
        else:
            return -1

    @staticmethod
    def _user_is_in_russia(user):
        if 'country' not in user:
            return False
        return user['country']['id'] == 1

    @staticmethod
    def _user_is_in_ukraine(user):
        if 'country' not in user:
            return False
        return user['country']['id'] == 2

    @staticmethod
    def _user_is_in_byelorussia(user):
        if 'country' not in user:
            return False
        return user['country']['id'] == 3

    @staticmethod
    def _user_is_in_kazakstan(user):
        if 'country' not in user:
            return False
        return user['country']['id'] == 4

    def _user_lda_by_groups(self, user):
        result = [0] * self.lda_maker.num_topics
        lda_count = 0
        if 'groups' in user:
            for group in user['groups']:
                try:
                    lda_desc = self.lda_maker.get(group['description'])
                    for k, v in lda_desc:
                        result[k] += v
                    lda_count += 1
                except KeyError:
                    pass
            if lda_count != 0:
                for i in range(len(result)):
                    result[i] /= lda_count
        return result
<|endoftext|>


<commit_before>
from bokeh.charts import Scatter, output_file, show
from bokeh.sampledata.autompg import autompg as df

p = Scatter(df, x='displ', y='hp', color='cyl',
            title="HP vs DISPL (shaded by CYL)", legend="top_left",
            sort_legend=[("color", False)], xlabel="Displacement",
            ylabel="Horsepower")

output_file("scatter.html")

show(p)<commit_msg>fix documentation example<commit_after>
from bokeh.charts import Scatter, output_file, show
from bokeh.sampledata.autompg import autompg as df

p = Scatter(df, x='displ', y='hp', color='cyl',
            title="HP vs DISPL (shaded by CYL)", legend="top_left",
            legend_sort_field = 'color',
            legend_sort_direction = 'ascending',
            xlabel="Displacement",
            ylabel="Horsepower")

output_file("scatter.html")

show(p)<|endoftext|>