<commit_before># -*- coding: utf-8 -*-

# sysdata

# Sysdata is a module uses great Py3status (i3status wrapper) to
# display system information (RAM usage) in i3bar (Linux systems).
# For more information read:
# i3wm homepage: http://i3wm.org
# py3status homepage: https://github.com/ultrabug/py3status

# NOTE: If you want py3status to show you your CPU temperature, change value of CPUTEMP into True
# in Py3status class - CPUInfo function
# and REMEMBER that you must install lm_sensors if you want CPU temp!

# Copyright (C) <2013> <Shahin Azad [ishahinism at Gmail]>

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import subprocess
from time import time

class GetData:
    """Get system status

    """
    def execCMD(self, cmd, arg):
        """Take a system command and its argument, then return the result.

        Arguments:
        - `cmd`: system command.
        - `arg`: argument.
        """
        result = subprocess.check_output([cmd, arg])
        return result

    def cpu(self):
        """Get the cpu usage data from /proc/stat :
        cpu  2255 34 2290 22625563 6290 127 456 0 0
        - user: normal processes executing in user mode
        - nice: niced processes executing in user mode
        - system: processes executing in kernel mode
        - idle: twiddling thumbs
        - iowait: waiting for I/O to complete
        - irq: servicing interrupts
        - softirq: servicing softirqs
        - steal: involuntary wait
        - guest: running a normal guest
        - guest_nice: running a niced guest
        These numbers identify the amount of time the CPU has spent performing
        different kinds of work.  Time units are in USER_HZ (typically hundredths of a
        second)
        """
        with open('/proc/stat', 'r') as fd:
            line = fd.readline()
        cpu_data = line.split()
        total_cpu_time = sum(map(int, cpu_data[1:]))
        cpu_idle_time = int(cpu_data[4])

        #return the cpu total&idle time
        return total_cpu_time, cpu_idle_time

    def memory(self):
        """Execute 'free -m' command, grab the memory capacity and used size
        then return; Memory size 'total_mem', Used_mem, and percentage
        of used memory.

        """
        # Run 'free -m' command and make a list from output.
        mem_data = self.execCMD('free', '-m').split()
        total_mem = int(mem_data[7]) / 1024.
        used_mem = int(mem_data[15]) / 1024.
        # Caculate percentage
        used_mem_percent = int(used_mem / (total_mem / 100))

        # Results are in kilobyte.
        return total_mem, used_mem, used_mem_percent

class Py3status:
    """
    System status in i3bar
    """
    def __init__(self):
        self.data = GetData()
        self.cpu_total = 0
        self.cpu_idle = 0

    def cpuInfo(self, json, i3status_config):
        """calculate the CPU status and return it.

        """
        response = {'full_text': '', 'name': 'cpu_usage'}
        cpu_total, cpu_idle = self.data.cpu()
        used_cpu_percent = 1 - float(cpu_idle-self.cpu_idle)/float(cpu_total-self.cpu_total)
        self.cpu_total = cpu_total
        self.cpu_idle = cpu_idle

        if used_cpu_percent <= 40:
            response['color'] = i3status_config['color_good']
        elif used_cpu_percent <= 75:
            response['color'] = i3status_config['color_degraded']
        else:
            response['color'] = i3status_config['color_bad']
        #cpu temp
        CPUTEMP=False
        if CPUTEMP:
                cputemp=subprocess.check_output('sensors | grep "CPU Temp" | cut -f 2 -d "+" | cut -f 1 -d " "',shell=True)
                cputemp=cputemp[:-1].decode('utf-8')
                response['full_text'] = "CPU: %.2f%%" % (used_cpu_percent*100) +" "+cputemp
        else:
             	response['full_text'] = "CPU: %.2f%%" % (used_cpu_percent*100)

        #cache the status for 10 seconds
        response['cached_until'] = time() + 10

        return (0, response)

    def ramInfo(self, json, i3status_config):
        """calculate the memory (RAM) status and return it.

        """
        response = {'full_text': '', 'name': 'ram_info'}
        total_mem, used_mem, used_mem_percent = self.data.memory()

        if used_mem_percent <= 40:
            response['color'] = i3status_config['color_good']
        elif used_mem_percent <= 75:
            response['color'] = i3status_config['color_degraded']
        else:
            response['color'] = i3status_config['color_bad']

        response['full_text'] = "RAM: %.2f/%.2f GB (%d%%)" % \
                                (used_mem, total_mem, used_mem_percent)
        response['cached_until'] = time()

        return (0, response)
<commit_msg>Update sysdata.py<commit_after># -*- coding: utf-8 -*-

# sysdata

# Sysdata is a module uses great Py3status (i3status wrapper) to
# display system information (RAM usage) in i3bar (Linux systems).
# For more information read:
# i3wm homepage: http://i3wm.org
# py3status homepage: https://github.com/ultrabug/py3status

# NOTE: If you want py3status to show you your CPU temperature, change value of CPUTEMP into True
# in Py3status class - CPUInfo function
# and REMEMBER that you must install lm_sensors if you want CPU temp!

# Copyright (C) <2013> <Shahin Azad [ishahinism at Gmail]>

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import subprocess
from time import time

class GetData:
    """Get system status

    """
    def execCMD(self, cmd, arg):
        """Take a system command and its argument, then return the result.

        Arguments:
        - `cmd`: system command.
        - `arg`: argument.
        """
        result = subprocess.check_output([cmd, arg])
        return result

    def cpu(self):
        """Get the cpu usage data from /proc/stat :
        cpu  2255 34 2290 22625563 6290 127 456 0 0
        - user: normal processes executing in user mode
        - nice: niced processes executing in user mode
        - system: processes executing in kernel mode
        - idle: twiddling thumbs
        - iowait: waiting for I/O to complete
        - irq: servicing interrupts
        - softirq: servicing softirqs
        - steal: involuntary wait
        - guest: running a normal guest
        - guest_nice: running a niced guest
        These numbers identify the amount of time the CPU has spent performing
        different kinds of work.  Time units are in USER_HZ (typically hundredths of a
        second)
        """
        with open('/proc/stat', 'r') as fd:
            line = fd.readline()
        cpu_data = line.split()
        total_cpu_time = sum(map(int, cpu_data[1:]))
        cpu_idle_time = int(cpu_data[4])

        #return the cpu total&idle time
        return total_cpu_time, cpu_idle_time

    def memory(self):
        """Execute 'free -m' command, grab the memory capacity and used size
        then return; Memory size 'total_mem', Used_mem, and percentage
        of used memory.

        """
        # Run 'free -m' command and make a list from output.
        mem_data = self.execCMD('free', '-m').split()
        total_mem = int(mem_data[7]) / 1024.
        used_mem = int(mem_data[15]) / 1024.
        # Caculate percentage
        used_mem_percent = int(used_mem / (total_mem / 100))

        # Results are in kilobyte.
        return total_mem, used_mem, used_mem_percent

class Py3status:
    """
    System status in i3bar
    """
    def __init__(self):
        self.data = GetData()
        self.cpu_total = 0
        self.cpu_idle = 0

    def cpuInfo(self, json, i3status_config):
        """calculate the CPU status and return it.

        """
        response = {'full_text': '', 'name': 'cpu_usage'}
        cpu_total, cpu_idle = self.data.cpu()
        used_cpu_percent = 1 - float(cpu_idle-self.cpu_idle)/float(cpu_total-self.cpu_total)
        self.cpu_total = cpu_total
        self.cpu_idle = cpu_idle

        if used_cpu_percent <= 40/100.0:
            response['color'] = i3status_config['color_good']
        elif used_cpu_percent <= 75/100.0:
            response['color'] = i3status_config['color_degraded']
        else:
            response['color'] = i3status_config['color_bad']
        #cpu temp
        CPUTEMP=False
        if CPUTEMP:
                cputemp=subprocess.check_output('sensors | grep "CPU Temp" | cut -f 2 -d "+" | cut -f 1 -d " "',shell=True)
                cputemp=cputemp[:-1].decode('utf-8')
                response['full_text'] = "CPU: %.2f%%" % (used_cpu_percent*100) +" "+cputemp
        else:
             	response['full_text'] = "CPU: %.2f%%" % (used_cpu_percent*100)

        #cache the status for 10 seconds
        response['cached_until'] = time() + 10

        return (0, response)

    def ramInfo(self, json, i3status_config):
        """calculate the memory (RAM) status and return it.

        """
        response = {'full_text': '', 'name': 'ram_info'}
        total_mem, used_mem, used_mem_percent = self.data.memory()

        if used_mem_percent <= 40:
            response['color'] = i3status_config['color_good']
        elif used_mem_percent <= 75:
            response['color'] = i3status_config['color_degraded']
        else:
            response['color'] = i3status_config['color_bad']

        response['full_text'] = "RAM: %.2f/%.2f GB (%d%%)" % \
                                (used_mem, total_mem, used_mem_percent)
        response['cached_until'] = time()

        return (0, response)
<|endoftext|>


<commit_before>from __future__ import division, print_function
"""
Various helpful snippets by Erik Tollerud (erik.tollerud@gmail.com)
"""

import numpy as np

__all__ = ['write_to_mac_clipboard', 'show_imagelist']

def write_to_mac_clipboard(s):
    from subprocess import Popen, PIPE

    p = Popen('pbcopy', stdin=PIPE)
    return p.communicate(s)

SDSS_IMAGE_LIST_URL = 'http://skyserver.sdss3.org/dr10/en/tools/chart/list.aspx'
_imglist_post_templ = """

<title>SDSS sampled_imagelist form</title>

 SDSS sampled_imagelist form

Using URL {url}

<form action="{url}"
method="post">
<TEXTAREA name="paste">
{text}
</TEXTAREA>

Scale: <input class="in" type="text" value="0.4" name="scale">

Opt: <input class="in" type="text" value="" name="opt">

<input type="submit">
</form>

"""
def show_imagelist(scs, names=None, url=SDSS_IMAGE_LIST_URL, posttoimglist=3.):
    """
    Returns the text to be pasted into the sdss image list page.  Also opens
    the page (if `url` is not None) and copies the text to the clipboard if on
    a mac or linux.

    Parameters
    ----------
    ras : SkyCoord
        SkyCoord of objects to show
    url : str or None
        The URL to the SDSS image list page or None to not open in a web
        browser.
    copytoclipboard : bool
        If True, copies the list of images to the clipboard for use on the SDSS
        web site
    posttoimglist : bool or float
        If True, makes a form to post to the URL site. If a float, gives the
        number of seconds to wait until deleting the temporary file (to gives
        the browser time to load).

    Returns
    -------
    text : str
        The table to be pasted into the image list text box

    """
    import webbrowser
    import tempfile
    import time
    import os

    if hasattr(scs, 'ra') and hasattr(scs, 'dec'):
        decs = scs.dec
        ras = scs.ra
    else:
        # assume it's a Table
        decs = ras['dec']
        ras = ras['ra']

    if len(ras) != len(decs):
        raise ValueError('ras and decs not the same size!')

    ras = np.array(ras, copy=False)
    decs = np.array(decs, copy=False)

    if names is None:
        names = [str(i) for i in range(len(ras))]

    text = ['name ra dec']
    for nmi, rai, deci in zip(names, ras, decs):
        text.append('{0} {1} {2}'.format(nmi, rai, deci))
    text = '\n'.join(text)

    if url:
        if posttoimglist:
            page = _imglist_post_templ.format(url=url, text=text)
            tf = tempfile.NamedTemporaryFile(delete=False)
            tf.write(page)
            tf.flush()
            fiurl = 'file://' + os.path.abspath(tf.name)
            webbrowser.open(fiurl)
            if isinstance(posttoimglist, float):
                time.sleep(posttoimglist)
        else:
            webbrowser.open(url)

    return text

def show_skycoord_sdss(sc, openinbrowser=True,
                       baseurl='http://skyserver.sdss3.org/dr12/en/tools/chart/navi.aspx?ra={ra}&dec={dec}',
                       **kwargs):
    """
    Shows a `SkyCoord` in the SDSS Navigate interface.

    Parameters
    ----------
    sc : SkyCoord
        The coordinate to view
    openinbrowser : bool
        If True, opens the page in a web browser
    baseurl : str
        The URL to use, with '{ra}' and '{dec}' indicating the coordinates in
        degrees
    kwargs
        Additional string keywords to add to the query

    Returns
    -------
    url : str
        The url to show
    """
    from urllib import urlencode
    import webbrowser

    url = baseurl.format(ra=sc.ra.degree, dec=sc.dec.degree)
    if kwargs:
        url = url + '&' + urlencode(kwargs)
    if openinbrowser:
        webbrowser.open(url)
    return url

def get_deimos_slits_inorder(maskfn):
    """
    Takes slits from a DEIMOS dsimulator fits file and returns them sorted in
    order of increasing x.  This should be the same orientation as they appear
    on the DEIMOS science images (so the far-left/lowest x object is the 0th
    entry in the returned lists)

    return scs, desislits, objcats
    """
    from astropy import units as u
    from astropy.coordinates import SkyCoord
    from astropy.io import fits

    f = fits.open(maskfn)
    xorder = np.argsort(f['BluSlits'].data['slitX1'])

    ddesi = f['DesiSlits'].data
    scsinorder = SkyCoord(ddesi['slitRa']*u.deg, ddesi['slitDec']*u.deg)[xorder]

    somdata = f['SlitObjMap'].data
    cdata = f['ObjectCat'].data
    cats = cdata[np.searchsorted(cdata['ObjectId'], somdata['ObjectId'][xorder])]

    return scsinorder, ddesi[xorder], cats

def to_clipboard(data):
    import subprocess

    p = subprocess.Popen('pbcopy', stdin=subprocess.PIPE)
    p.communicate(data)
    return p.wait()
<commit_msg>added keola_html_to_ascii_log<commit_after>from __future__ import division, print_function
"""
Various helpful snippets by Erik Tollerud (erik.tollerud@gmail.com)
"""

import numpy as np

__all__ = ['write_to_mac_clipboard', 'show_imagelist']

def write_to_mac_clipboard(s):
    from subprocess import Popen, PIPE

    p = Popen('pbcopy', stdin=PIPE)
    return p.communicate(s)

SDSS_IMAGE_LIST_URL = 'http://skyserver.sdss3.org/dr10/en/tools/chart/list.aspx'
_imglist_post_templ = """

<title>SDSS sampled_imagelist form</title>

 SDSS sampled_imagelist form

Using URL {url}

<form action="{url}"
method="post">
<TEXTAREA name="paste">
{text}
</TEXTAREA>

Scale: <input class="in" type="text" value="0.4" name="scale">

Opt: <input class="in" type="text" value="" name="opt">

<input type="submit">
</form>

"""
def show_imagelist(scs, names=None, url=SDSS_IMAGE_LIST_URL, posttoimglist=3.):
    """
    Returns the text to be pasted into the sdss image list page.  Also opens
    the page (if `url` is not None) and copies the text to the clipboard if on
    a mac or linux.

    Parameters
    ----------
    ras : SkyCoord
        SkyCoord of objects to show
    url : str or None
        The URL to the SDSS image list page or None to not open in a web
        browser.
    copytoclipboard : bool
        If True, copies the list of images to the clipboard for use on the SDSS
        web site
    posttoimglist : bool or float
        If True, makes a form to post to the URL site. If a float, gives the
        number of seconds to wait until deleting the temporary file (to gives
        the browser time to load).

    Returns
    -------
    text : str
        The table to be pasted into the image list text box

    """
    import webbrowser
    import tempfile
    import time
    import os

    if hasattr(scs, 'ra') and hasattr(scs, 'dec'):
        decs = scs.dec
        ras = scs.ra
    else:
        # assume it's a Table
        decs = ras['dec']
        ras = ras['ra']

    if len(ras) != len(decs):
        raise ValueError('ras and decs not the same size!')

    ras = np.array(ras, copy=False)
    decs = np.array(decs, copy=False)

    if names is None:
        names = [str(i) for i in range(len(ras))]

    text = ['name ra dec']
    for nmi, rai, deci in zip(names, ras, decs):
        text.append('{0} {1} {2}'.format(nmi, rai, deci))
    text = '\n'.join(text)

    if url:
        if posttoimglist:
            page = _imglist_post_templ.format(url=url, text=text)
            tf = tempfile.NamedTemporaryFile(delete=False)
            tf.write(page)
            tf.flush()
            fiurl = 'file://' + os.path.abspath(tf.name)
            webbrowser.open(fiurl)
            if isinstance(posttoimglist, float):
                time.sleep(posttoimglist)
        else:
            webbrowser.open(url)

    return text

def show_skycoord_sdss(sc, openinbrowser=True,
                       baseurl='http://skyserver.sdss3.org/dr12/en/tools/chart/navi.aspx?ra={ra}&dec={dec}',
                       **kwargs):
    """
    Shows a `SkyCoord` in the SDSS Navigate interface.

    Parameters
    ----------
    sc : SkyCoord
        The coordinate to view
    openinbrowser : bool
        If True, opens the page in a web browser
    baseurl : str
        The URL to use, with '{ra}' and '{dec}' indicating the coordinates in
        degrees
    kwargs
        Additional string keywords to add to the query

    Returns
    -------
    url : str
        The url to show
    """
    from urllib import urlencode
    import webbrowser

    url = baseurl.format(ra=sc.ra.degree, dec=sc.dec.degree)
    if kwargs:
        url = url + '&' + urlencode(kwargs)
    if openinbrowser:
        webbrowser.open(url)
    return url

def get_deimos_slits_inorder(maskfn):
    """
    Takes slits from a DEIMOS dsimulator fits file and returns them sorted in
    order of increasing x.  This should be the same orientation as they appear
    on the DEIMOS science images (so the far-left/lowest x object is the 0th
    entry in the returned lists)

    return scs, desislits, objcats
    """
    from astropy import units as u
    from astropy.coordinates import SkyCoord
    from astropy.io import fits

    f = fits.open(maskfn)
    xorder = np.argsort(f['BluSlits'].data['slitX1'])

    ddesi = f['DesiSlits'].data
    scsinorder = SkyCoord(ddesi['slitRa']*u.deg, ddesi['slitDec']*u.deg)[xorder]

    somdata = f['SlitObjMap'].data
    cdata = f['ObjectCat'].data
    cats = cdata[np.searchsorted(cdata['ObjectId'], somdata['ObjectId'][xorder])]

    return scsinorder, ddesi[xorder], cats

def to_clipboard(data):
    import subprocess

    p = subprocess.Popen('pbcopy', stdin=subprocess.PIPE)
    p.communicate(data)
    return p.wait()

def keola_html_to_ascii_log(htmlfn, outfn=None):
    from astropy.io import ascii

    if not htmlfn.endswith('.html'):
        raise ValueError('need the html log from KeOLA')
    if outfn is None:
        outfn = htmlfn[:-5] + '.log'

    header = ascii.read(htmlfn, format='html', htmldict={'table_id': 1})
    observations = ascii.read(htmlfn, format='html', htmldict={'table_id': 2})

    with open(outfn, 'w') as f:
        header.write(f, format='ascii.no_header')
        f.write('\n\n')
        observations.write(f, format='ascii')

    return outfn
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
youtube.py - Jenni YouTube Module
Copyright 2012, Dimitri Molenaars, Tyrope.nl.
Licensed under the Eiffel Forum License 2.

More info:
 * Jenni: https://github.com/myano/jenni/
 * Phenny: http://inamidst.com/phenny/

This module will respond to .yt and .youtube commands and searches the youtubes.
"""

import urllib2, re

def ytsearch(jenni, input):
    """YouTube search module"""
    #modified from ytinfo: Copyright 2010-2011, Michael Yanovich, yanovich.net, Kenneth Sham.

    #Right now, this uses a parsing script from rscript.org. Eventually, I'd
    #like to use the YouTube API directly.

    #Before actually loading this in, let's see what input actually is so we can parse it right.

    #Grab info from rscript
    uri = 'http://rscript.org/lookup.php?type=ytinfo&term=' + input.group(2).encode('utf-8')
    uri = uri.replace(' ', '%20')
    redirects = 0
    while True:
        req = urllib2.Request(uri, headers={'Accept':'text/html'})
        req.add_header('User-Agent', 'OpenAnything/1.0 +http://diveintopython.org/')
        try: u = urllib2.urlopen(req, None, 0.5)
        except:
            jenni.say('Something went wrong when accessing the rscript.org parser.')
            return
            info = u.info()
            u.close()
            # info = web.head(uri)
            if not isinstance(info, list):
                status = '200'
            else:
                status = str(info[1])
                info = info[0]
            if status.startswith('3'):
                uri = urlparse.urljoin(uri, info['Location'])
            else: break
            redirects += 1
            if redirects >= 50:
                return "Too many re-directs."
    try: mtype = info['content-type']
    except:
        return
    if not (('/html' in mtype) or ('/xhtml' in mtype)):
        return
    u = urllib2.urlopen(req)
    bytes = u.read(262144)
    u.close()

    #Parse rscript info.
    rtitle = re.search('(TITLE: )(.*)', bytes)
    title = rtitle.group(2)

    rauthor = re.search('(AUTHOR: )(\S*) (20\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d).*', bytes)
    author = rauthor.group(2)
    if not author:
        jenni.say('I couldn\'t find information for that video.')
        return

    duration = int(re.search('(DURATION: )(.*)', bytes).group(2))
    if duration < 1: length = 'LIVE'
    else:
        hours = duration / (60 * 60)
        minutes = duration / 60 - (hours * 60)
        seconds = duration % 60

        length = ''
        if hours:
            length = str(hours) + 'hours'
            if minutes or seconds:
                length = length + ' '
        if minutes:
            length = length + str(minutes) + 'mins'
            if seconds:
                length = length + ' '
        if seconds: length = length + str(seconds) + 'secs'

    views = str('{:20,d}'.format(int(re.search('(VIEWS: )(.*)', bytes).group(2)))).lstrip(' ')

    rurl = re.search('(URL: )(.*)', bytes)
    url = rurl.group(2)

    message = '[YT Search] Title: ' +title+ \
              ' | Author: ' +author+ \
              ' | Duration: ' +length+ \
              ' | Views: ' +views+ \
              ' | Link: ' +url

    jenni.say(message)
ytsearch.commands = ['yt','youtube']
ytsearch.example = '.yt how to be a nerdfighter FAQ'

def ytinfo(jenni, input):
    #Grab info from YT API
    uri = 'http://gdata.youtube.com/feeds/api/videos/' + input.group(2) + '?v=2'
    redirects = 0
    while True:
        req = urllib2.Request(uri, headers={'Accept':'*/*', 'User-Agent':'OpenAnything/1.0 +http://diveintopython.org/'})
        try: u = urllib2.urlopen(req, None, 0.5)
        except:
            jenni.say('Something went wrong when accessing the YouTube API.')
            return
        info = u.info()
        u.close()
        # info = web.head(uri)
        if not isinstance(info, list):
            status = '200'
        else:
            status = str(info[1])
            try: info = info[0]
            except AttributeError as e: jenni.msg(input.devchan,"[DEVMSG]Line 120: info= "+type(info))
        jenni.msg(input.devchan,"[DEVMSG]YT API Result: ["+status+"]"+info)
        if status.startswith('3'):
            uri = urlparse.urljoin(uri, info['Location'])
        else: break
        redirects += 1
        if redirects >= 50:
            return "Too many re-directs."
    try: mtype = info['content-type']
    except: return
    if not (('/html' in mtype) or ('/xhtml' in mtype)):
        return
    try: u = urllib2.urlopen(req, None, 0.5)
    except:
        jenni.say('Something went wrong when accessing the YouTube API.')
        return
    bytes = u.read(262144)
    u.close()

    #Parse YouTube API info (XML)

    #get title
    title_result = re.search('(?:<media:title type=\'plain\'>)(.*)(?:</media:title>)', bytes)
    title = title_result.group(1)

    #get youtube channel
    uploader_result = re.search('(?:<author><name>)(.*)(?:</name>)', bytes)
    uploader = uploader_result.group(1)

    #get upload time in format: yyyy-MM-ddThh:mm:ss.sssZ
    uploaded_result = re.search('(?:<yt:uploaded>)(.*)(?:</yt:uploaded>)', bytes)
    upraw = uploaded_result.group(1)
    #parse from current format to output format: DD/MM/yyyy, hh:mm
    uploaded = upraw[8:10]+"/"+upraw[5:7]+"/"+upraw[0:4]+", "+upraw[11:13]+":"+upraw[14:16]

    #get duration in seconds
    length_result = re.search('(?:<yt:duration seconds=\')(.*)(?:\'/>)', bytes)
    duration = length_result.group(1)

    #Detect liveshow + parse duration into proper time format.
    if duration < 1: length = 'LIVE'
    else:
        hours = duration / (60 * 60)
        minutes = duration / 60 - (hours * 60)
        seconds = duration % 60

        length = ''
        if hours:
            length = str(hours) + 'hours'
            if minutes or seconds:
                length = length + ' '
        if minutes:
            length = length + str(minutes) + 'mins'
            if seconds:
                length = length + ' '
        if seconds: length = length + str(seconds) + 'secs'

    #get views
    views_result = re.search('(?:<yt:statistics favoriteCount=\')([0-9]*)(?:\' viewCount=\')([0-9]*)(?:\'/>)', bytes)
    views = views_result.group(2)

    #get favourites (for future use?)
    favs = views_result.group(1)

    #get comment count
    comments_result = re.search('(?:<gd:comments><gd:feedLink)(?:.*)(?:countHint=\')(.*)(?:\'/></gd:comments>)', bytes)
    comments = comments_result.group(1)

    #get likes & dislikes
    liking_result = re.search('(?:<yt:rating numDislikes=\')(.*)(?:\' numLikes=\')(.*)(?:\'/>)',bytes)
    likes = liking_result.group(2)
    dislikes = liking_result.group(1)

    #combine variables and print
    message = '[YouTube] Title: ' + title + ' | Uploader: ' + uploader + \
              ' | Uploaded: ' + uploaded + ' | Length: ' + length + \
              ' | Views: ' + views + ' | Comments: ' + comments + ' | Likes: '\
              + likes + ' | Dislikes: ' + dislikes

    jenni.say(message)
ytinfo.rule = '.*(youtube.com/watch\S*v=|youtu.be/)([\w-]+).*'

if __name__ == '__main__':
    print __doc__.strip()
<commit_msg>TypeError.<commit_after>#!/usr/bin/env python
"""
youtube.py - Jenni YouTube Module
Copyright 2012, Dimitri Molenaars, Tyrope.nl.
Licensed under the Eiffel Forum License 2.

More info:
 * Jenni: https://github.com/myano/jenni/
 * Phenny: http://inamidst.com/phenny/

This module will respond to .yt and .youtube commands and searches the youtubes.
"""

import urllib2, re

def ytsearch(jenni, input):
    """YouTube search module"""
    #modified from ytinfo: Copyright 2010-2011, Michael Yanovich, yanovich.net, Kenneth Sham.

    #Right now, this uses a parsing script from rscript.org. Eventually, I'd
    #like to use the YouTube API directly.

    #Before actually loading this in, let's see what input actually is so we can parse it right.

    #Grab info from rscript
    uri = 'http://rscript.org/lookup.php?type=ytinfo&term=' + input.group(2).encode('utf-8')
    uri = uri.replace(' ', '%20')
    redirects = 0
    while True:
        req = urllib2.Request(uri, headers={'Accept':'text/html'})
        req.add_header('User-Agent', 'OpenAnything/1.0 +http://diveintopython.org/')
        try: u = urllib2.urlopen(req, None, 0.5)
        except:
            jenni.say('Something went wrong when accessing the rscript.org parser.')
            return
            info = u.info()
            u.close()
            # info = web.head(uri)
            if not isinstance(info, list):
                status = '200'
            else:
                status = str(info[1])
                info = info[0]
            if status.startswith('3'):
                uri = urlparse.urljoin(uri, info['Location'])
            else: break
            redirects += 1
            if redirects >= 50:
                return "Too many re-directs."
    try: mtype = info['content-type']
    except:
        return
    if not (('/html' in mtype) or ('/xhtml' in mtype)):
        return
    u = urllib2.urlopen(req)
    bytes = u.read(262144)
    u.close()

    #Parse rscript info.
    rtitle = re.search('(TITLE: )(.*)', bytes)
    title = rtitle.group(2)

    rauthor = re.search('(AUTHOR: )(\S*) (20\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d).*', bytes)
    author = rauthor.group(2)
    if not author:
        jenni.say('I couldn\'t find information for that video.')
        return

    duration = int(re.search('(DURATION: )(.*)', bytes).group(2))
    if duration < 1: length = 'LIVE'
    else:
        hours = duration / (60 * 60)
        minutes = duration / 60 - (hours * 60)
        seconds = duration % 60

        length = ''
        if hours:
            length = str(hours) + 'hours'
            if minutes or seconds:
                length = length + ' '
        if minutes:
            length = length + str(minutes) + 'mins'
            if seconds:
                length = length + ' '
        if seconds: length = length + str(seconds) + 'secs'

    views = str('{:20,d}'.format(int(re.search('(VIEWS: )(.*)', bytes).group(2)))).lstrip(' ')

    rurl = re.search('(URL: )(.*)', bytes)
    url = rurl.group(2)

    message = '[YT Search] Title: ' +title+ \
              ' | Author: ' +author+ \
              ' | Duration: ' +length+ \
              ' | Views: ' +views+ \
              ' | Link: ' +url

    jenni.say(message)
ytsearch.commands = ['yt','youtube']
ytsearch.example = '.yt how to be a nerdfighter FAQ'

def ytinfo(jenni, input):
    #Grab info from YT API
    uri = 'http://gdata.youtube.com/feeds/api/videos/' + input.group(2) + '?v=2'
    redirects = 0
    while True:
        req = urllib2.Request(uri, headers={'Accept':'*/*', 'User-Agent':'OpenAnything/1.0 +http://diveintopython.org/'})
        try: u = urllib2.urlopen(req, None, 0.5)
        except:
            jenni.say('Something went wrong when accessing the YouTube API.')
            return
        info = u.info()
        u.close()
        # info = web.head(uri)
        if not isinstance(info, list):
            status = '200'
        else:
            status = str(info[1])
            try: info = info[0]
            except AttributeError as e: jenni.msg(input.devchan,"[DEVMSG]Line 120: info= "+type(info))
        jenni.msg(input.devchan,"[DEVMSG]YT API Result: ["+str(status)+"]"+str(info))
        if status.startswith('3'):
            uri = urlparse.urljoin(uri, info['Location'])
        else: break
        redirects += 1
        if redirects >= 50:
            return "Too many re-directs."
    try: mtype = info['content-type']
    except: return
    if not (('/html' in mtype) or ('/xhtml' in mtype)):
        return
    try: u = urllib2.urlopen(req, None, 0.5)
    except:
        jenni.say('Something went wrong when accessing the YouTube API.')
        return
    bytes = u.read(262144)
    u.close()

    #Parse YouTube API info (XML)

    #get title
    title_result = re.search('(?:<media:title type=\'plain\'>)(.*)(?:</media:title>)', bytes)
    title = title_result.group(1)

    #get youtube channel
    uploader_result = re.search('(?:<author><name>)(.*)(?:</name>)', bytes)
    uploader = uploader_result.group(1)

    #get upload time in format: yyyy-MM-ddThh:mm:ss.sssZ
    uploaded_result = re.search('(?:<yt:uploaded>)(.*)(?:</yt:uploaded>)', bytes)
    upraw = uploaded_result.group(1)
    #parse from current format to output format: DD/MM/yyyy, hh:mm
    uploaded = upraw[8:10]+"/"+upraw[5:7]+"/"+upraw[0:4]+", "+upraw[11:13]+":"+upraw[14:16]

    #get duration in seconds
    length_result = re.search('(?:<yt:duration seconds=\')(.*)(?:\'/>)', bytes)
    duration = length_result.group(1)

    #Detect liveshow + parse duration into proper time format.
    if duration < 1: length = 'LIVE'
    else:
        hours = duration / (60 * 60)
        minutes = duration / 60 - (hours * 60)
        seconds = duration % 60

        length = ''
        if hours:
            length = str(hours) + 'hours'
            if minutes or seconds:
                length = length + ' '
        if minutes:
            length = length + str(minutes) + 'mins'
            if seconds:
                length = length + ' '
        if seconds: length = length + str(seconds) + 'secs'

    #get views
    views_result = re.search('(?:<yt:statistics favoriteCount=\')([0-9]*)(?:\' viewCount=\')([0-9]*)(?:\'/>)', bytes)
    views = views_result.group(2)

    #get favourites (for future use?)
    favs = views_result.group(1)

    #get comment count
    comments_result = re.search('(?:<gd:comments><gd:feedLink)(?:.*)(?:countHint=\')(.*)(?:\'/></gd:comments>)', bytes)
    comments = comments_result.group(1)

    #get likes & dislikes
    liking_result = re.search('(?:<yt:rating numDislikes=\')(.*)(?:\' numLikes=\')(.*)(?:\'/>)',bytes)
    likes = liking_result.group(2)
    dislikes = liking_result.group(1)

    #combine variables and print
    message = '[YouTube] Title: ' + title + ' | Uploader: ' + uploader + \
              ' | Uploaded: ' + uploaded + ' | Length: ' + length + \
              ' | Views: ' + views + ' | Comments: ' + comments + ' | Likes: '\
              + likes + ' | Dislikes: ' + dislikes

    jenni.say(message)
ytinfo.rule = '.*(youtube.com/watch\S*v=|youtu.be/)([\w-]+).*'

if __name__ == '__main__':
    print __doc__.strip()
<|endoftext|>


<commit_before>"""
Encapsulate interaction with Solr
"""

# TODO: use pysolr where appropriate

import os.path
from decimal import Decimal
from json import dumps, loads, JSONEncoder

import requests

class Solr:
    def __init__(self, url, core, chunk_size=1000):
        self.url = url
        self.core = core
        self.chunk_size = chunk_size
        self.fields = { }

    # Built-in JSON can't serialize Decimals
    # Use like this: json.dumps(o, cls=__JsonDecimalEncoder)
    class JsonDecimalEncoder(JSONEncoder):
        def default(self, o):
            if isinstance(o, Decimal):
                return float(o)
            return super(Solr.JsonDecimalEncoder, self).default(o)

    def postall(self, mols):
        """Post all molecules in a sequence"""
        chunk = []
        self._get_fields()
        new_fields = []
        for mol in mols:
            chunk.append(mol)
            new_fields += self._new_fields(mol)
            if len(chunk) == self.chunk_size:
                self._post_fields(new_fields)
                new_fields = []
                self._post_chunk(chunk)
                chunk = []
        if len(chunk) != 0:
                self._post_chunk(chunk)

    def _post_chunk(self, chunk):
        url = os.path.join(self.url, self.core, 'update') + '?commit=true'
        headers = {'Content-Type' : 'application/json'}
        data = dumps(chunk, cls=Solr.JsonDecimalEncoder)
        print("Posting {0} molecules to Solr at {1}".format(len(chunk), url))
        requests.post(url, headers=headers, data=data)

    def _get_fields(self):
        url = os.path.join(self.url, self.core, 'schema/fields')
        response = requests.get(url, params={'wt': 'json'})
        fields = loads(response.text)['fields']
        self.fields = {f['name']: f for f in fields}
        print("Solr has {0} fields defined.".format(len(self.fields)))

    def _new_fields(self, mol):
        new_fields = []
        for key, val in mol.items():
            if key not in self.fields.keys():
                type = self._guess_type(val)
                if type is None:
                    # Skip this one; hopefully, another record will have a value for this field.
                    continue
                field = {
                    'name': key,
                    'indexed': True,
                    'stored': True,
                    'type': type
                }
                self.fields[key] = field
                new_fields.append(field)
        return new_fields

    def _guess_type(self, o):
        if o is None:
            return None
        if isinstance(o, list):
            subtype = self._guess_type(o[0])
            if subtype != 'text_general':
                return subtype + 's'
            return subtype
        if isinstance(o, bool):
            return 'boolean'
        if isinstance(o, int):
            # Use trie-encoded integers for range filtering
            return 'tlong'
        if isinstance(o, float):
            # Use trie-encoded floats for range filtering
            return 'tdouble'
        if isinstance(o, Decimal):
            if (o - int(o)) == 0:
                return 'tlong'
            return 'tdouble'
        # For short strings, use "string", because they will be used mainly as labels.
        # For long strings, use "text_general", because they will be used mainly for search.
        # '64' is a somewhat arbitrary cut-off between 'short' and 'long'
        if len(o) < 64:
            return 'string'
        return 'text_general'

    def _post_fields(self, new_fields):
        url = os.path.join(self.url, self.core, 'schema/fields') + '?commit=true'
        headers = {'Content-Type' : 'application/json'}
        # As of Solr version 5.4.1:
        # The documentation at https://cwiki.apache.org/confluence/display/solr/Schema+API is wrong.
        # It says we need to send something like:
        #    { "add-field": {"name": ..., "type": ..., ...} }
        # In reality, we need to send:
        #    [ {"name": ..., "type": ..., ...}, {"name": ..., "type": ..., ...}, ...]
        # Apparently, "add-field" is implied.
        data = dumps(new_fields, cls=Solr.JsonDecimalEncoder)
        requests.post(url, headers=headers, data=data)
        print("Posting {0} fields to Solr at {1}".format(len(new_fields), url))
        # Update our local field cache
        self._get_fields()

def test_guess_type():
    solr = Solr("#", 'core')
    assert(solr._guess_type(None) is None)
    assert(solr._guess_type(True) == 'boolean')
    assert(solr._guess_type([True, False]) == 'booleans')
    assert(solr._guess_type(1) == 'tlong')
    assert(solr._guess_type([1, 2, 3]) == 'tlongs')
    assert(solr._guess_type(1.0) == 'tdouble')
    assert(solr._guess_type([1.0, 2.0, 3.0]) == 'tdoubles')
    assert(solr._guess_type(Decimal('1.0')) == 'tlong')
    assert(solr._guess_type(Decimal('1.1')) == 'tdouble')
    assert(solr._guess_type('helios') == 'string')
    assert(solr._guess_type(['helios', 'centauri', 'betelgeuse']) == 'strings')
    assert(solr._guess_type('A ford is a fine kind of truck, whether it be a small, pickup style or a large semi-trailer. Either will serve you reliably for many years to come.')
           == 'text_general')
    assert(solr._guess_type(['A ford is a fine kind of truck, whether it be a small, pickup style or a large semi-trailer. Either will serve you reliably for many years to come.'])
           == 'text_general')

def test_postall():
    mols = [{
                "id": Decimal(i),
                'beauty':True,
                'beauties':[True, False],
                'ingot': 11,
                'ingots': [12, 13, 14],
                'fish': 3.14,
                'fishes': [1.1, 2.2, 3.3],
                'sun': 'helios',
                'suns': ['centauri', 'betelgeuse', 'sirius'],
                'truck': 'A ford is a fine kind of truck, whether it be a small, pickup style or a large semi-trailer. Either will serve you reliably for many years to come.'
            } for i in range(1500)]
    solr = Solr('http://localhost:8983/solr/', 'test')
    solr.postall(mols)

if __name__ == '__main__':
    test_guess_type()
    test_postall()
<commit_msg>Short circuit posting new fields of there aren't any.<commit_after>"""
Encapsulate interaction with Solr
"""

# TODO: use pysolr where appropriate

import os.path
from decimal import Decimal
from json import dumps, loads, JSONEncoder

import requests

class Solr:
    def __init__(self, url, core, chunk_size=1000):
        self.url = url
        self.core = core
        self.chunk_size = chunk_size
        self.fields = { }

    # Built-in JSON can't serialize Decimals
    # Use like this: json.dumps(o, cls=__JsonDecimalEncoder)
    class JsonDecimalEncoder(JSONEncoder):
        def default(self, o):
            if isinstance(o, Decimal):
                return float(o)
            return super(Solr.JsonDecimalEncoder, self).default(o)

    def postall(self, mols):
        """Post all molecules in a sequence"""
        chunk = []
        self._get_fields()
        new_fields = []
        for mol in mols:
            chunk.append(mol)
            new_fields += self._new_fields(mol)
            if len(chunk) == self.chunk_size:
                if new_fields:
                    self._post_fields(new_fields)
                    new_fields = []
                self._post_chunk(chunk)
                chunk = []
        if len(chunk) != 0:
                self._post_chunk(chunk)

    def _post_chunk(self, chunk):
        url = os.path.join(self.url, self.core, 'update') + '?commit=true'
        headers = {'Content-Type' : 'application/json'}
        data = dumps(chunk, cls=Solr.JsonDecimalEncoder)
        print("Posting {0} molecules to Solr at {1}".format(len(chunk), url))
        requests.post(url, headers=headers, data=data)

    def _get_fields(self):
        url = os.path.join(self.url, self.core, 'schema/fields')
        response = requests.get(url, params={'wt': 'json'})
        fields = loads(response.text)['fields']
        self.fields = {f['name']: f for f in fields}
        print("Solr has {0} fields defined.".format(len(self.fields)))

    def _new_fields(self, mol):
        new_fields = []
        for key, val in mol.items():
            if key not in self.fields.keys():
                type = self._guess_type(val)
                if type is None:
                    # Skip this one; hopefully, another record will have a value for this field.
                    continue
                field = {
                    'name': key,
                    'indexed': True,
                    'stored': True,
                    'type': type
                }
                self.fields[key] = field
                new_fields.append(field)
        return new_fields

    def _guess_type(self, o):
        if o is None:
            return None
        if isinstance(o, list):
            subtype = self._guess_type(o[0])
            if subtype != 'text_general':
                return subtype + 's'
            return subtype
        if isinstance(o, bool):
            return 'boolean'
        if isinstance(o, int):
            # Use trie-encoded integers for range filtering
            return 'tlong'
        if isinstance(o, float):
            # Use trie-encoded floats for range filtering
            return 'tdouble'
        if isinstance(o, Decimal):
            if (o - int(o)) == 0:
                return 'tlong'
            return 'tdouble'
        # For short strings, use "string", because they will be used mainly as labels.
        # For long strings, use "text_general", because they will be used mainly for search.
        # '64' is a somewhat arbitrary cut-off between 'short' and 'long'
        if len(o) < 64:
            return 'string'
        return 'text_general'

    def _post_fields(self, new_fields):
        url = os.path.join(self.url, self.core, 'schema/fields') + '?commit=true'
        headers = {'Content-Type' : 'application/json'}
        # As of Solr version 5.4.1:
        # The documentation at https://cwiki.apache.org/confluence/display/solr/Schema+API is wrong.
        # It says we need to send something like:
        #    { "add-field": {"name": ..., "type": ..., ...} }
        # In reality, we need to send:
        #    [ {"name": ..., "type": ..., ...}, {"name": ..., "type": ..., ...}, ...]
        # Apparently, "add-field" is implied.
        data = dumps(new_fields, cls=Solr.JsonDecimalEncoder)
        requests.post(url, headers=headers, data=data)
        print("Posting {0} fields to Solr at {1}".format(len(new_fields), url))
        # Update our local field cache
        self._get_fields()

def test_guess_type():
    solr = Solr("#", 'core')
    assert(solr._guess_type(None) is None)
    assert(solr._guess_type(True) == 'boolean')
    assert(solr._guess_type([True, False]) == 'booleans')
    assert(solr._guess_type(1) == 'tlong')
    assert(solr._guess_type([1, 2, 3]) == 'tlongs')
    assert(solr._guess_type(1.0) == 'tdouble')
    assert(solr._guess_type([1.0, 2.0, 3.0]) == 'tdoubles')
    assert(solr._guess_type(Decimal('1.0')) == 'tlong')
    assert(solr._guess_type(Decimal('1.1')) == 'tdouble')
    assert(solr._guess_type('helios') == 'string')
    assert(solr._guess_type(['helios', 'centauri', 'betelgeuse']) == 'strings')
    assert(solr._guess_type('A ford is a fine kind of truck, whether it be a small, pickup style or a large semi-trailer. Either will serve you reliably for many years to come.')
           == 'text_general')
    assert(solr._guess_type(['A ford is a fine kind of truck, whether it be a small, pickup style or a large semi-trailer. Either will serve you reliably for many years to come.'])
           == 'text_general')

def test_postall():
    mols = [{
                "id": Decimal(i),
                'beauty':True,
                'beauties':[True, False],
                'ingot': 11,
                'ingots': [12, 13, 14],
                'fish': 3.14,
                'fishes': [1.1, 2.2, 3.3],
                'sun': 'helios',
                'suns': ['centauri', 'betelgeuse', 'sirius'],
                'truck': 'A ford is a fine kind of truck, whether it be a small, pickup style or a large semi-trailer. Either will serve you reliably for many years to come.'
            } for i in range(1500)]
    solr = Solr('http://localhost:8983/solr/', 'test')
    solr.postall(mols)

if __name__ == '__main__':
    test_guess_type()
    test_postall()
<|endoftext|>


<commit_before>import random

from datetime import datetime

from celery import task
from time import strptime

from django.conf import settings
from django.core.mail import EmailMessage
from django.template.loader import render_to_string
from django.core import management

from molo.core.content_import import api
from molo.core.models import (
    ArticlePage, Main, SectionIndexPage, SiteLanguage, SectionPage)

from wagtail.contrib.settings.context_processors import SettingsProxy
from wagtail.wagtailcore.models import Site
from django.utils import timezone

IMPORT_EMAIL_TEMPLATE = "core/content_import/import_email.html"
VALIDATE_EMAIL_TEMPLATE = "core/content_import/validate_email.html"

@task(ignore_result=True)
def rotate_content(day=None):
    """ this method gets the parameters that are needed for rotate_latest
    and rotate_featured_in_homepage methods, and calls them both"""
    # getting the content rotation settings from site settings
    main_lang = SiteLanguage.objects.filter(is_main_language=True).first()
    main = Main.objects.all().first()
    index = SectionIndexPage.objects.live().first()
    site = Site.objects.get(is_default_site=True)
    settings = SettingsProxy(site)
    site_settings = settings['core']['SiteSettings']
    if day is None:
        day = datetime.today().weekday()
    # creates a days of the week list

    # calls the two rotate methods with the necessary params
    if main and index:
        rotate_latest(main_lang, index, main, site_settings, day)
        rotate_featured_in_homepage(main_lang, day)

@task(ignore_result=True)
def publish_scheduled_pages():
    management.call_command(
        'publish_scheduled_pages', verbosity=0, interactive=False)

@task(ignore_result=True)
def clearsessions():
    # Expired sessions will only be cleared roughly once an hour - randomly
    if random.randint(0, 59) == 0:
        management.call_command(
            'clearsessions', verbosity=0, interactive=False)

@task(ignore_result=True)
def demote_articles():
    ArticlePage.objects.live().filter(
        featured_in_latest_end_date__lte=datetime.now()).update(
            featured_in_latest=False,
            featured_in_latest_start_date=None,
            featured_in_latest_end_date=None)
    ArticlePage.objects.live().filter(
        featured_in_section_end_date__lte=datetime.now()).update(
            featured_in_section=False,
            featured_in_section_start_date=None,
            featured_in_section_end_date=None)
    ArticlePage.objects.live().filter(
        featured_in_homepage_end_date__lte=datetime.now()).update(
            featured_in_homepage=False,
            featured_in_homepage_start_date=None,
            featured_in_homepage_end_date=None)

@task(ignore_result=True)
def promote_articles():
    ArticlePage.objects.live().filter(
        featured_in_latest_start_date__lte=datetime.now()).update(
        featured_in_latest=True)
    ArticlePage.objects.live().filter(
        featured_in_section_start_date__lte=datetime.now()).update(
        featured_in_section=True)
    ArticlePage.objects.live().filter(
        featured_in_homepage_start_date__lte=datetime.now()).update(
        featured_in_homepage=True)

def get_days_section(section=None):
        return [
            section.monday_rotation, section.tuesday_rotation,
            section.wednesday_rotation, section.thursday_rotation,
            section.friday_rotation, section.saturday_rotation,
            section.sunday_rotation]

def get_days_site_settings(site_settings):
        return [
            site_settings.monday_rotation, site_settings.tuesday_rotation,
            site_settings.wednesday_rotation, site_settings.thursday_rotation,
            site_settings.friday_rotation, site_settings.saturday_rotation,
            site_settings.sunday_rotation]

def rotate_latest(main_lang, index, main, site_settings, day):
    """This rotates all the articles that have been marked as
    featured_in_latest. It checks whether current date falls within the set
    date range for content rotation. It then checks whether the current weekday
    is set to rotate, and then rotates an articles for each hour the admin has
    set."""

    def demote_last_featured_article():
        # set the last featured_in_latest article to false
        if main.latest_articles().live().count() >= 2:
            article = main.latest_articles().live().last()
            article.featured_in_latest = False
            article.featured_in_latest_start_date = None
            article.featured_in_latest_end_date = None
            article.save_revision().publish()

    days = get_days_site_settings(site_settings)
    # checks if the current date is within the content rotation range
    if site_settings.content_rotation_start_date and \
            site_settings.content_rotation_end_date:
        if site_settings.content_rotation_start_date < timezone.now() \
                < site_settings.content_rotation_end_date:

            # checks if the current weekday is set to rotate
            if days[day]:
                for time in site_settings.time:
                    time = strptime(str(time), '%H:%M:%S')
                    if time.tm_hour == datetime.now().hour:
                        # get a random article
                        random_article = ArticlePage.objects.live().filter(
                            featured_in_latest=False,
                            languages__language__id=main_lang.id
                        ).descendant_of(index).order_by('?').exact_type(
                            ArticlePage).first()
                        # set random article to feature in latest
                        if random_article:
                            random_article.featured_in_latest_start_date = \
                                datetime.now()
                            random_article.save_revision().publish()
                            promote_articles()
                            demote_last_featured_article()

def rotate_featured_in_homepage(main_lang, day):
    def demote_last_featured_article_in_homepage():
            articles = ArticlePage.objects.live().filter(
                featured_in_homepage=True,
                languages__language__id=main_lang.id
            ).order_by(
                '-featured_in_homepage_start_date')
            if articles.count() >= 2:
                article = articles.last()
                article.featured_in_homepage = False
                article.featured_in_homepage_start_date = None
                article.featured_in_homepage_end_date = None
                article.save_revision().publish()

    for section in SectionPage.objects.all():
        days = get_days_section(section)
        # checks if current date is within the rotation date range
        if section.content_rotation_start_date and \
                section.content_rotation_end_date:
            if section.content_rotation_start_date < timezone.now() \
                    < section.content_rotation_end_date:

                # checks if the current weekday is set to rotate
                if days[day]:
                    for time in section.time:
                        time = strptime(str(time), '%H:%M:%S')
                        if time.tm_hour == datetime.now().hour:
                            random_article = ArticlePage.objects.live().filter(
                                featured_in_homepage=False,
                                languages__language__id=main_lang.id
                            ).descendant_of(section).order_by('?').exact_type(
                                ArticlePage).first()

                            # promotes an article and bumps last one off list
                            if random_article:
                                random_article. \
                                    featured_in_homepage_start_date = \
                                    datetime.now()
                                random_article.save_revision().publish()
                                promote_articles()
                                demote_last_featured_article_in_homepage()

def send_import_email(to_email, context):
    from_email = settings.FROM_EMAIL
    subject = settings.CONTENT_IMPORT_SUBJECT
    body = render_to_string(IMPORT_EMAIL_TEMPLATE, context)
    email_message = EmailMessage(subject, body, from_email, [to_email])
    email_message.send()

def send_validate_email(to_email, context):
    from_email = settings.FROM_EMAIL
    subject = settings.CONTENT_IMPORT_SUBJECT
    body = render_to_string(VALIDATE_EMAIL_TEMPLATE, context)
    email_message = EmailMessage(subject, body, from_email, [to_email])
    email_message.send()

@task(ignore_result=True)
def import_content(data, locales, username, email, host):
    repos = api.get_repos(data)
    result = api.validate_content(repos, locales)

    if not result['errors']:
        api.import_content(repos, locales)

    send_import_email(email, {
        'name': username,
        'host': host,
        'errors': result['errors'],
        'warnings': result['warnings']
    })

@task(ignore_result=True)
def validate_content(data, locales, username, email, host):
    repos = api.get_repos(data)
    result = api.validate_content(repos, locales)

    send_import_email(email, {
        'name': username,
        'host': host,
        'type': 'import_failure',
        'errors': result['errors'],
        'warnings': result['warnings']
    })

@task(ignore_result=True)
def molo_consolidated_minute_task():
    """
    Purpose: To reduce the number of db connections, we run all
    tasks in 1 call
    """

    demote_articles()
    promote_articles()
    publish_scheduled_pages()
    clearsessions()
<commit_msg>insure content demotion happens for each section individually<commit_after>import random

from datetime import datetime

from celery import task
from time import strptime

from django.conf import settings
from django.core.mail import EmailMessage
from django.template.loader import render_to_string
from django.core import management

from molo.core.content_import import api
from molo.core.models import (
    ArticlePage, Main, SectionIndexPage, SiteLanguage, SectionPage)

from wagtail.contrib.settings.context_processors import SettingsProxy
from wagtail.wagtailcore.models import Site
from django.utils import timezone

IMPORT_EMAIL_TEMPLATE = "core/content_import/import_email.html"
VALIDATE_EMAIL_TEMPLATE = "core/content_import/validate_email.html"

@task(ignore_result=True)
def rotate_content(day=None):
    """ this method gets the parameters that are needed for rotate_latest
    and rotate_featured_in_homepage methods, and calls them both"""
    # getting the content rotation settings from site settings
    main_lang = SiteLanguage.objects.filter(is_main_language=True).first()
    main = Main.objects.all().first()
    index = SectionIndexPage.objects.live().first()
    site = Site.objects.get(is_default_site=True)
    settings = SettingsProxy(site)
    site_settings = settings['core']['SiteSettings']
    if day is None:
        day = datetime.today().weekday()
    # creates a days of the week list

    # calls the two rotate methods with the necessary params
    if main and index:
        rotate_latest(main_lang, index, main, site_settings, day)
        rotate_featured_in_homepage(main_lang, day)

@task(ignore_result=True)
def publish_scheduled_pages():
    management.call_command(
        'publish_scheduled_pages', verbosity=0, interactive=False)

@task(ignore_result=True)
def clearsessions():
    # Expired sessions will only be cleared roughly once an hour - randomly
    if random.randint(0, 59) == 0:
        management.call_command(
            'clearsessions', verbosity=0, interactive=False)

@task(ignore_result=True)
def demote_articles():
    ArticlePage.objects.live().filter(
        featured_in_latest_end_date__lte=datetime.now()).update(
            featured_in_latest=False,
            featured_in_latest_start_date=None,
            featured_in_latest_end_date=None)
    ArticlePage.objects.live().filter(
        featured_in_section_end_date__lte=datetime.now()).update(
            featured_in_section=False,
            featured_in_section_start_date=None,
            featured_in_section_end_date=None)
    ArticlePage.objects.live().filter(
        featured_in_homepage_end_date__lte=datetime.now()).update(
            featured_in_homepage=False,
            featured_in_homepage_start_date=None,
            featured_in_homepage_end_date=None)

@task(ignore_result=True)
def promote_articles():
    ArticlePage.objects.live().filter(
        featured_in_latest_start_date__lte=datetime.now()).update(
        featured_in_latest=True)
    ArticlePage.objects.live().filter(
        featured_in_section_start_date__lte=datetime.now()).update(
        featured_in_section=True)
    ArticlePage.objects.live().filter(
        featured_in_homepage_start_date__lte=datetime.now()).update(
        featured_in_homepage=True)

def get_days_section(section=None):
        return [
            section.monday_rotation, section.tuesday_rotation,
            section.wednesday_rotation, section.thursday_rotation,
            section.friday_rotation, section.saturday_rotation,
            section.sunday_rotation]

def get_days_site_settings(site_settings):
        return [
            site_settings.monday_rotation, site_settings.tuesday_rotation,
            site_settings.wednesday_rotation, site_settings.thursday_rotation,
            site_settings.friday_rotation, site_settings.saturday_rotation,
            site_settings.sunday_rotation]

def rotate_latest(main_lang, index, main, site_settings, day):
    """This rotates all the articles that have been marked as
    featured_in_latest. It checks whether current date falls within the set
    date range for content rotation. It then checks whether the current weekday
    is set to rotate, and then rotates an articles for each hour the admin has
    set."""

    def demote_last_featured_article():
        # set the last featured_in_latest article to false
        if main.latest_articles().live().count() >= 2:
            article = main.latest_articles().live().last()
            article.featured_in_latest = False
            article.featured_in_latest_start_date = None
            article.featured_in_latest_end_date = None
            article.save_revision().publish()

    days = get_days_site_settings(site_settings)
    # checks if the current date is within the content rotation range
    if site_settings.content_rotation_start_date and \
            site_settings.content_rotation_end_date:
        if site_settings.content_rotation_start_date < timezone.now() \
                < site_settings.content_rotation_end_date:

            # checks if the current weekday is set to rotate
            if days[day]:
                for time in site_settings.time:
                    time = strptime(str(time), '%H:%M:%S')
                    if time.tm_hour == datetime.now().hour:
                        # get a random article
                        random_article = ArticlePage.objects.live().filter(
                            featured_in_latest=False,
                            languages__language__id=main_lang.id
                        ).descendant_of(index).order_by('?').exact_type(
                            ArticlePage).first()
                        # set random article to feature in latest
                        if random_article:
                            random_article.featured_in_latest_start_date = \
                                datetime.now()
                            random_article.save_revision().publish()
                            promote_articles()
                            demote_last_featured_article()

def rotate_featured_in_homepage(main_lang, day):
    def demote_last_featured_article_in_homepage(section):
            articles = ArticlePage.objects.live().filter(
                featured_in_homepage=True,
                languages__language__id=main_lang.id
            ).descendant_of(section).order_by(
                '-featured_in_homepage_start_date')
            if articles.count() >= 2:
                article = articles.last()
                article.featured_in_homepage = False
                article.featured_in_homepage_start_date = None
                article.featured_in_homepage_end_date = None
                article.save_revision().publish()

    for section in SectionPage.objects.all():
        days = get_days_section(section)
        # checks if current date is within the rotation date range
        if section.content_rotation_start_date and \
                section.content_rotation_end_date:
            if section.content_rotation_start_date < timezone.now() \
                    < section.content_rotation_end_date:

                # checks if the current weekday is set to rotate
                if days[day]:
                    for time in section.time:
                        time = strptime(str(time), '%H:%M:%S')
                        if time.tm_hour == datetime.now().hour:
                            random_article = ArticlePage.objects.live().filter(
                                featured_in_homepage=False,
                                languages__language__id=main_lang.id
                            ).descendant_of(section).order_by('?').exact_type(
                                ArticlePage).first()

                            # promotes an article and bumps last one off list
                            if random_article:
                                random_article. \
                                    featured_in_homepage_start_date = \
                                    datetime.now()
                                random_article.save_revision().publish()
                                promote_articles()
                                demote_last_featured_article_in_homepage(section)

def send_import_email(to_email, context):
    from_email = settings.FROM_EMAIL
    subject = settings.CONTENT_IMPORT_SUBJECT
    body = render_to_string(IMPORT_EMAIL_TEMPLATE, context)
    email_message = EmailMessage(subject, body, from_email, [to_email])
    email_message.send()

def send_validate_email(to_email, context):
    from_email = settings.FROM_EMAIL
    subject = settings.CONTENT_IMPORT_SUBJECT
    body = render_to_string(VALIDATE_EMAIL_TEMPLATE, context)
    email_message = EmailMessage(subject, body, from_email, [to_email])
    email_message.send()

@task(ignore_result=True)
def import_content(data, locales, username, email, host):
    repos = api.get_repos(data)
    result = api.validate_content(repos, locales)

    if not result['errors']:
        api.import_content(repos, locales)

    send_import_email(email, {
        'name': username,
        'host': host,
        'errors': result['errors'],
        'warnings': result['warnings']
    })

@task(ignore_result=True)
def validate_content(data, locales, username, email, host):
    repos = api.get_repos(data)
    result = api.validate_content(repos, locales)

    send_import_email(email, {
        'name': username,
        'host': host,
        'type': 'import_failure',
        'errors': result['errors'],
        'warnings': result['warnings']
    })

@task(ignore_result=True)
def molo_consolidated_minute_task():
    """
    Purpose: To reduce the number of db connections, we run all
    tasks in 1 call
    """

    demote_articles()
    promote_articles()
    publish_scheduled_pages()
    clearsessions()
<|endoftext|>


<commit_before>#  Copyright (c) 2009-2010, Cloud Matrix Pty. Ltd.
#  All rights reserved; available under the terms of the BSD License.
"""

  esky.util:  misc utility functions for esky

"""

from __future__ import with_statement

import os
import re
import sys
import shutil
import zipfile
import errno
from itertools import tee, izip
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

from distutils.util import get_platform as _distutils_get_platform

from esky.bootstrap import get_best_version, get_all_versions,\
                           is_version_dir, is_installed_version_dir,\
                           is_uninstalled_version_dir,\
                           split_app_version, join_app_version, parse_version,\
                           get_original_filename, lock_version_dir,\
                           unlock_version_dir, fcntl
from esky.bootstrap import appdir_from_executable as _bs_appdir_from_executable

def pairwise(iterable):
    """Iterator over pairs of elements from the given iterable."""
    a,b = tee(iterable)
    try:
        b.next()
    except StopIteration:
        pass
    return izip(a,b)

def common_prefix(iterables):
    """Find the longest common prefix of a series of iterables."""
    iterables = iter(iterables)
    try:
        prefix = iterables.next()
    except StopIteration:
        raise ValueError("at least one iterable is required")
    for item in iterables:
        count = 0
        for (c1,c2) in izip(prefix,item):
            if c1 != c2:
                break
            count += 1
        prefix = prefix[:count]
    return prefix

def appdir_from_executable(exepath):
    """Find the top-level application directory, given sys.executable."""
    vdir = _bs_appdir_from_executable(exepath)
    appdir = os.path.dirname(vdir)
    if os.path.exists(os.path.join(appdir,"esky-bootstrap.txt")):
        appdir = os.path.dirname(appdir)
    return appdir

def extract_zipfile(source,target,name_filter=None):
    """Extract the contents of a zipfile into a target directory.

    The argument 'source' names the zipfile to read, while 'target' names
    the directory into which to extract.  If given, the optional argument
    'name_filter' must be a function mapping names from the zipfile to names
    in the target directory.
    """
    zf = zipfile.ZipFile(source,"r")
    try:
        if hasattr(zf,"open"):
            zf_open = zf.open
        else:
            def zf_open(nm,mode):
                return StringIO(zf.read(nm))
        for nm in zf.namelist():
            if nm.endswith("/"):
                continue
            if name_filter:
                outfilenm = os.path.join(target,name_filter(nm))
            else:
                outfilenm = os.path.join(target,nm)
            if not os.path.isdir(os.path.dirname(outfilenm)):
                os.makedirs(os.path.dirname(outfilenm))
            infile = zf_open(nm,"r")
            try:
                outfile = open(outfilenm,"wb")
                try:
                    shutil.copyfileobj(infile,outfile)
                finally:
                    outfile.close()
            finally:
                infile.close()
            mode = zf.getinfo(nm).external_attr >> 16L
            if mode:
                os.chmod(outfilenm,mode)
    finally:
        zf.close()

def zipfile_common_prefix_dir(source):
    """Find the common prefix directory of all files in a zipfile."""
    zf = zipfile.ZipFile(source)
    prefix = common_prefix(zf.namelist())
    if "/" in prefix:
        return prefix.rsplit("/",1)[0] + "/"
    else:
        return ""

def deep_extract_zipfile(source,target,name_filter=None):
    """Extract the deep contents of a zipfile into a target directory.

    This is just like extract_zipfile() except that any common prefix dirs
    are removed.  For example, if everything in the zipfile is under the
    directory "example.app" then that prefix will be removed during unzipping.

    This is useful to allow distribution of "friendly" zipfiles that don't
    overwrite files in the current directory when extracted by hand.
    """
    prefix = zipfile_common_prefix_dir(source)
    if prefix:
        def new_name_filter(nm):
            if not nm.startswith(prefix):
                return None
            if name_filter is not None:
                return name_filter(nm[len(prefix):])
            return nm[len(prefix):]
    else:
         new_name_filter = name_filter
    return extract_zipfile(source,target,new_name_filter)

def create_zipfile(source,target,get_zipinfo=None,members=None,compress=None):
    """Bundle the contents of a given directory into a zipfile.

    The argument 'source' names the directory to read, while 'target' names
    the zipfile to be written.

    If given, the optional argument 'get_zipinfo' must be a function mapping
    filenames to ZipInfo objects.  It may also return None to indicate that
    defaults should be used, or a string to indicate that defaults should be
    used with a new archive name.

    If given, the optional argument 'members' must be an iterable yielding
    names or ZipInfo objects.  Files will be added to the archive in the
    order specified by this function.

    If the optional argument 'compress' is given, it must be a bool indicating
    whether to compress the files by default.  The default is no compression.
    """
    if not compress:
        compress_type = zipfile.ZIP_STORED
    else:
        compress_type = zipfile.ZIP_DEFLATED
    zf = zipfile.ZipFile(target,"w",compression=compress_type)
    if members is None:
        def gen_members():
            for (dirpath,dirnames,filenames) in os.walk(source):
                for fn in filenames:
                    yield os.path.join(dirpath,fn)[len(source)+1:]
        members = gen_members()
    for fpath in members:
        if isinstance(fpath,zipfile.ZipInfo):
            zinfo = fpath
            fpath = os.path.join(source,zinfo.filename)
        else:
            if get_zipinfo:
                zinfo = get_zipinfo(fpath)
            else:
                zinfo = None
            fpath = os.path.join(source,fpath)
        if zinfo is None:
            zf.write(fpath,fpath[len(source)+1:])
        elif isinstance(zinfo,basestring):
            zf.write(fpath,zinfo)
        else:
            with open(fpath,"rb") as f:
                zf.writestr(zinfo,f.read())
    zf.close()

def get_platform():
    """Get the platform identifier for the current platform.

    This is similar to the function distutils.util.get_platform(); it returns
    a string identifying the types of platform on which binaries built on this
    machine can reasonably be expected to run.

    Unlike distutils.util.get_platform(), the value returned by this function
    is guaranteed not to contain any periods. This makes it much easier to
    parse out of filenames.
    """
    return _distutils_get_platform().replace(".","_")

def is_core_dependency(filenm):
    """Check whether than named file is a core python dependency.

    If it is, then it's required for any frozen program to run (even the
    bootstrapper).  Currently this includes only the python DLL and the
    MSVCRT private assembly.
    """
    if re.match("^(lib)?python\\d[\\d\\.]*\\.[a-z\\.]*$",filenm):
        return True
    if filenm.startswith("Microsoft.") and filenm.endswith(".CRT"):
        return True
    return False

def copy_ownership_info(src,dst,cur="",default=None):
    """Copy file ownership from src onto dst, as much as possible."""
    # TODO: how on win32?
    source = os.path.join(src,cur)
    target = os.path.join(dst,cur)
    if default is None:
        default = os.stat(src)
    if os.path.exists(source):
        info = os.stat(source)
    else:
        info = default
    if sys.platform != "win32":
        os.chown(target,info.st_uid,info.st_gid)
    if os.path.isdir(target):
        for nm in os.listdir(target):
            copy_ownership_info(src,dst,os.path.join(cur,nm),default)

def get_backup_filename(filename):
    """Get the name to which a backup of the given file can be written.

    This will typically the filename with ".old" inserted at an appropriate
    location.  We try to preserve the file extension where possible.
    """
    parent = os.path.dirname(filename)
    parts = os.path.basename(filename).split(".")
    parts.insert(-1,"old")
    backname = os.path.join(parent,".".join(parts))
    while os.path.exists(backname):
        parts.insert(-1,"old")
        backname = os.path.join(parent,".".join(parts))
    return backname

def is_locked_version_dir(vdir):
    """Check whether the given version dir is locked."""
    if sys.platform == "win32":
        lockfile = os.path.join(vdir,"esky-bootstrap.txt")
        try:
            os.rename(lockfile,lockfile)
        except EnvironmentError:
            return True
        else:
            return False
    else:
        lockfile = os.path.join(vdir,"esky-lockfile.txt")
        f = open(lockfile,"r")
        try:
            fcntl.flock(f,fcntl.LOCK_EX|fcntl.LOCK_NB)
        except EnvironmentError, e:
            if e.errno not in (errno.EACCES,errno.EAGAIN,):
                raise
            return True
        else:
            return False
        finally:
            f.close()

<commit_msg>extract_zip_file:  let name_filter return None == don't extract that file<commit_after>#  Copyright (c) 2009-2010, Cloud Matrix Pty. Ltd.
#  All rights reserved; available under the terms of the BSD License.
"""

  esky.util:  misc utility functions for esky

"""

from __future__ import with_statement

import os
import re
import sys
import shutil
import zipfile
import errno
from itertools import tee, izip
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

from distutils.util import get_platform as _distutils_get_platform

from esky.bootstrap import get_best_version, get_all_versions,\
                           is_version_dir, is_installed_version_dir,\
                           is_uninstalled_version_dir,\
                           split_app_version, join_app_version, parse_version,\
                           get_original_filename, lock_version_dir,\
                           unlock_version_dir, fcntl
from esky.bootstrap import appdir_from_executable as _bs_appdir_from_executable

def pairwise(iterable):
    """Iterator over pairs of elements from the given iterable."""
    a,b = tee(iterable)
    try:
        b.next()
    except StopIteration:
        pass
    return izip(a,b)

def common_prefix(iterables):
    """Find the longest common prefix of a series of iterables."""
    iterables = iter(iterables)
    try:
        prefix = iterables.next()
    except StopIteration:
        raise ValueError("at least one iterable is required")
    for item in iterables:
        count = 0
        for (c1,c2) in izip(prefix,item):
            if c1 != c2:
                break
            count += 1
        prefix = prefix[:count]
    return prefix

def appdir_from_executable(exepath):
    """Find the top-level application directory, given sys.executable."""
    vdir = _bs_appdir_from_executable(exepath)
    appdir = os.path.dirname(vdir)
    if os.path.exists(os.path.join(appdir,"esky-bootstrap.txt")):
        appdir = os.path.dirname(appdir)
    return appdir

def extract_zipfile(source,target,name_filter=None):
    """Extract the contents of a zipfile into a target directory.

    The argument 'source' names the zipfile to read, while 'target' names
    the directory into which to extract.  If given, the optional argument
    'name_filter' must be a function mapping names from the zipfile to names
    in the target directory.
    """
    zf = zipfile.ZipFile(source,"r")
    try:
        if hasattr(zf,"open"):
            zf_open = zf.open
        else:
            def zf_open(nm,mode):
                return StringIO(zf.read(nm))
        for nm in zf.namelist():
            if nm.endswith("/"):
                continue
            if name_filter:
                outfilenm = name_filter(nm)
                if outfilenm is None:
                    continue
                outfilenm = os.path.join(target,outfilenm)
            else:
                outfilenm = os.path.join(target,nm)
            if not os.path.isdir(os.path.dirname(outfilenm)):
                os.makedirs(os.path.dirname(outfilenm))
            infile = zf_open(nm,"r")
            try:
                outfile = open(outfilenm,"wb")
                try:
                    shutil.copyfileobj(infile,outfile)
                finally:
                    outfile.close()
            finally:
                infile.close()
            mode = zf.getinfo(nm).external_attr >> 16L
            if mode:
                os.chmod(outfilenm,mode)
    finally:
        zf.close()

def zipfile_common_prefix_dir(source):
    """Find the common prefix directory of all files in a zipfile."""
    zf = zipfile.ZipFile(source)
    prefix = common_prefix(zf.namelist())
    if "/" in prefix:
        return prefix.rsplit("/",1)[0] + "/"
    else:
        return ""

def deep_extract_zipfile(source,target,name_filter=None):
    """Extract the deep contents of a zipfile into a target directory.

    This is just like extract_zipfile() except that any common prefix dirs
    are removed.  For example, if everything in the zipfile is under the
    directory "example.app" then that prefix will be removed during unzipping.

    This is useful to allow distribution of "friendly" zipfiles that don't
    overwrite files in the current directory when extracted by hand.
    """
    prefix = zipfile_common_prefix_dir(source)
    if prefix:
        def new_name_filter(nm):
            if not nm.startswith(prefix):
                return None
            if name_filter is not None:
                return name_filter(nm[len(prefix):])
            return nm[len(prefix):]
    else:
         new_name_filter = name_filter
    return extract_zipfile(source,target,new_name_filter)

def create_zipfile(source,target,get_zipinfo=None,members=None,compress=None):
    """Bundle the contents of a given directory into a zipfile.

    The argument 'source' names the directory to read, while 'target' names
    the zipfile to be written.

    If given, the optional argument 'get_zipinfo' must be a function mapping
    filenames to ZipInfo objects.  It may also return None to indicate that
    defaults should be used, or a string to indicate that defaults should be
    used with a new archive name.

    If given, the optional argument 'members' must be an iterable yielding
    names or ZipInfo objects.  Files will be added to the archive in the
    order specified by this function.

    If the optional argument 'compress' is given, it must be a bool indicating
    whether to compress the files by default.  The default is no compression.
    """
    if not compress:
        compress_type = zipfile.ZIP_STORED
    else:
        compress_type = zipfile.ZIP_DEFLATED
    zf = zipfile.ZipFile(target,"w",compression=compress_type)
    if members is None:
        def gen_members():
            for (dirpath,dirnames,filenames) in os.walk(source):
                for fn in filenames:
                    yield os.path.join(dirpath,fn)[len(source)+1:]
        members = gen_members()
    for fpath in members:
        if isinstance(fpath,zipfile.ZipInfo):
            zinfo = fpath
            fpath = os.path.join(source,zinfo.filename)
        else:
            if get_zipinfo:
                zinfo = get_zipinfo(fpath)
            else:
                zinfo = None
            fpath = os.path.join(source,fpath)
        if zinfo is None:
            zf.write(fpath,fpath[len(source)+1:])
        elif isinstance(zinfo,basestring):
            zf.write(fpath,zinfo)
        else:
            with open(fpath,"rb") as f:
                zf.writestr(zinfo,f.read())
    zf.close()

def get_platform():
    """Get the platform identifier for the current platform.

    This is similar to the function distutils.util.get_platform(); it returns
    a string identifying the types of platform on which binaries built on this
    machine can reasonably be expected to run.

    Unlike distutils.util.get_platform(), the value returned by this function
    is guaranteed not to contain any periods. This makes it much easier to
    parse out of filenames.
    """
    return _distutils_get_platform().replace(".","_")

def is_core_dependency(filenm):
    """Check whether than named file is a core python dependency.

    If it is, then it's required for any frozen program to run (even the
    bootstrapper).  Currently this includes only the python DLL and the
    MSVCRT private assembly.
    """
    if re.match("^(lib)?python\\d[\\d\\.]*\\.[a-z\\.]*$",filenm):
        return True
    if filenm.startswith("Microsoft.") and filenm.endswith(".CRT"):
        return True
    return False

def copy_ownership_info(src,dst,cur="",default=None):
    """Copy file ownership from src onto dst, as much as possible."""
    # TODO: how on win32?
    source = os.path.join(src,cur)
    target = os.path.join(dst,cur)
    if default is None:
        default = os.stat(src)
    if os.path.exists(source):
        info = os.stat(source)
    else:
        info = default
    if sys.platform != "win32":
        os.chown(target,info.st_uid,info.st_gid)
    if os.path.isdir(target):
        for nm in os.listdir(target):
            copy_ownership_info(src,dst,os.path.join(cur,nm),default)

def get_backup_filename(filename):
    """Get the name to which a backup of the given file can be written.

    This will typically the filename with ".old" inserted at an appropriate
    location.  We try to preserve the file extension where possible.
    """
    parent = os.path.dirname(filename)
    parts = os.path.basename(filename).split(".")
    parts.insert(-1,"old")
    backname = os.path.join(parent,".".join(parts))
    while os.path.exists(backname):
        parts.insert(-1,"old")
        backname = os.path.join(parent,".".join(parts))
    return backname

def is_locked_version_dir(vdir):
    """Check whether the given version dir is locked."""
    if sys.platform == "win32":
        lockfile = os.path.join(vdir,"esky-bootstrap.txt")
        try:
            os.rename(lockfile,lockfile)
        except EnvironmentError:
            return True
        else:
            return False
    else:
        lockfile = os.path.join(vdir,"esky-lockfile.txt")
        f = open(lockfile,"r")
        try:
            fcntl.flock(f,fcntl.LOCK_EX|fcntl.LOCK_NB)
        except EnvironmentError, e:
            if e.errno not in (errno.EACCES,errno.EAGAIN,):
                raise
            return True
        else:
            return False
        finally:
            f.close()

<|endoftext|>