<commit_before>import psutil
import SystemInfo
import logging

# log file
logger=logging.getLogger()
handler=logging.FileHandler("SystemInfo.log")
logger.addHandler(handler)
logger.setLevel(logging.NOTSET)

# alert limit define
# region

# system info get
systemInfo = SystemInfo.SystemInfo(pids=None, processName=None)

# alert limit configure
cpuAlertCount = int(systemInfo.cpu_wait_time)
cpuAlertCounter = 0

memAlertCount = int(systemInfo.mem_wait_time)
memAlertCounter = 0

netAlertCount = int(systemInfo.net_wait_time)
netAlertCounter = 0

# endregion

def cpuInfoCheck(cpuInfo) :
    count = 0
    for rate in cpuInfo :
        if rate > systemInfo.cpu_upper_limit or rate < systemInfo.cpu_lower_limit :\
            count += 1
    if count == len(cpuInfo) :
        return True
    return False

def memInfoCheck(memInfo) :
    if memInfo.percent > systemInfo.mem_upper_limit or memInfo.percent < systemInfo.mem_upper_limit:
        return True
    return False

def diskInfoCheck(diskInfo) :
    if diskInfo.percent > systemInfo.disk_alert_limit :
        return True
    return False

def netInfoCheck(netInfo, interface) :
    try :
        speed = netInfo[interface]
    except :
        return True
    if (speed[0] > systemInfo.net_upload_upper_limit or speed[0] < systemInfo.net_upload_lower_limit) and (speed[1] < systemInfo.net_down_lower_limit or speed[1] > systemInfo.net_down_upper_limit) :
        return True
    return False

def deBugOutPut():
    pids = psutil.pids()
    systemInfo.getMemInfo(pids)
    print systemInfo.processInfo_mem

while True :
    # check cpu usage rate
    systemInfo.getCpuInfo()
    if cpuAlertCounter > cpuAlertCount :
        # call the alert moduel
        print 'cpu alert'
        cpuAlertCounter = 0
    if cpuInfoCheck(systemInfo.sysInfo_cpu[1]) :
        cpuAlertCounter += 1
    else :
        cpuAlertCounter = 0

    # check mem usage rate
    systemInfo.getMemInfo()
    if memAlertCounter > memAlertCount :
        # calthe alert moduel
        # report the top 10 process
        print 'mem alert'
        memAlertCounter = 0
    if memInfoCheck(systemInfo.sysInfo_mem[0]) :
        memAlertCounter += 1
    else :
        memAlertCounter = 0

    # check disk usage rate
    systemInfo.getDiskInfo()
    if diskInfoCheck(systemInfo.sysInfo_disk[0]) :
        # call alert moduel
        # report the detail usage
        print 'disk alert'

    # check net usage rate
    systemInfo.getNetInfo()
    if netAlertCounter > netAlertCount :
        # call alert moduel
        # report the up/download speed
        print 'net alert'
    if netInfoCheck(systemInfo.sysInfo_net, 'lo') :
        netAlertCounter += 1
    else :
        netAlertCounter = 0

    # deBugOutPut()

    break
<commit_msg>-fill the monitor alert method<commit_after>import psutil
import SystemInfo
import logging
import Alert

# log file
logger=logging.getLogger()
handler=logging.FileHandler("SystemInfo.log")
logger.addHandler(handler)
logger.setLevel(logging.NOTSET)

# system info get
systemInfo = SystemInfo.SystemInfo(pids=None, processName=None)
# alert center
alertcenter = Alert.Alert()

# alert limit define
# region

# alert limit configure
cpuAlertCount = int(systemInfo.cpu_wait_time)
cpuAlertCounter = 0

memAlertCount = int(systemInfo.mem_wait_time)
memAlertCounter = 0

netAlertCount = int(systemInfo.net_wait_time)
netAlertCounter = 0

# endregion

def cpuInfoCheck(cpuInfo) :
    count = 0
    for rate in cpuInfo :
        if rate > systemInfo.cpu_upper_limit or rate < systemInfo.cpu_lower_limit :\
            count += 1
    if count == len(cpuInfo) :
        return True
    return False

def memInfoCheck(memInfo) :
    if memInfo.percent > systemInfo.mem_upper_limit or memInfo.percent < systemInfo.mem_upper_limit:
        return True
    return False

def diskInfoCheck(diskInfo) :
    if diskInfo.percent > systemInfo.disk_alert_limit :
        return True
    return False

def netInfoCheck(netInfo, interface) :
    try :
        speed = netInfo[interface]
    except :
        return True
    if (speed[0] > systemInfo.net_upload_upper_limit or speed[0] < systemInfo.net_upload_lower_limit) and (speed[1] < systemInfo.net_down_lower_limit or speed[1] > systemInfo.net_down_upper_limit) :
        return True
    return False

def deBugOutPut():
    pids = psutil.pids()
    systemInfo.getMemInfo(pids)
    print systemInfo.processInfo_mem

while True :
    # check cpu usage rate
    systemInfo.getCpuInfo()
    if cpuAlertCounter > cpuAlertCount :
        # call the alert moduel
        print 'cpu alert'
        alertcenter.cpuAlert(systemInfo.sysInfo_cpu)
        cpuAlertCounter = 0
    if cpuInfoCheck(systemInfo.sysInfo_cpu[1]) :
        cpuAlertCounter += 1
    else :
        cpuAlertCounter = 0

    # check mem usage rate
    systemInfo.getMemInfo()
    if memAlertCounter > memAlertCount :
        # calthe alert moduel
        # report the top 10 process
        print 'mem alert'
        alertcenter.memAlert(systemInfo.sysInfo_mem)
        memAlertCounter = 0
    if memInfoCheck(systemInfo.sysInfo_mem[0]) :
        memAlertCounter += 1
    else :
        memAlertCounter = 0

    # check disk usage rate
    systemInfo.getDiskInfo()
    if diskInfoCheck(systemInfo.sysInfo_disk[0]) :
        # call alert moduel
        # report the detail usage
        print 'disk alert'
        alertcenter.diskAlert(systemInfo.sysInfo_disk)
    # check net usage rate
    systemInfo.getNetInfo()
    if netAlertCounter > netAlertCount :
        # call alert moduel
        # report the up/download speed
        print 'net alert'
        alertcenter.netAlert(interface='lo', netInfo=systemInfo.sysInfo_net)
    if netInfoCheck(systemInfo.sysInfo_net, 'lo') :
        netAlertCounter += 1
    else :
        netAlertCounter = 0

    # deBugOutPut()

    break
<|endoftext|>


<commit_before>""" Functions for determinig best-fit template parameters by convolution with a
grid """

from dem import Hillshade
import WindowedTemplate as wt
import numpy as np
import matplotlib.pyplot as plt
import pyfftw
from pyfftw.interfaces.numpy_fft import fft2, ifft2, fftshift

from progressbar import ProgressBar, Bar, Percentage, ETA

eps = np.spacing(1)
pyfftw.interfaces.cache.enable()

def calculate_amplitude(dem, Template, d, age, alpha):

    ny, nx = dem._griddata.shape
    de = dem._georef_info.dx
    t = Template(d, age, alpha, nx, ny, de)
    template = t.template()

    curv = dem._calculate_directional_laplacian(alpha)

    amp, snr = match_template(curv, template)
    mask = t.get_window_limits()
    amp[mask] = 0
    snr[mask] = 0

    return amp, snr

#@profile
def calculate_best_fit_parameters(dem, Template, **kwargs):

    #args = parse_args(**kwargs)
    d = 100
    de = 1

    age_max = 3
    age_min = 0
    age_stepsize = 0.5
    ang_stepsize = 2

    num_angles = 180/ang_stepsize + 1
    num_ages = (age_max - age_min)/age_stepsize
    orientations = np.linspace(-np.pi/2, np.pi/2, num_angles)
    ages = np.linspace(age_min, age_max, num_ages)

    ny, nx = dem._griddata.shape
    best_amp = np.zeros((ny, nx))
    best_age = np.zeros((ny, nx))
    best_alpha = np.zeros((ny, nx))
    best_snr = np.zeros((ny, nx))

    pbar = ProgressBar(widgets=[Percentage(), Bar(left='[', right=']'), ' ', ETA()], maxval=len(ages)*len(orientations)).start()

    for i, this_alpha in enumerate(orientations):
        for j, this_age in enumerate(ages):

            this_age = 10**this_age

            #args['kt'] = this_age
            #args['alpha'] = this_alpha
            #t = Template(args)
            t = Template(d, this_age, this_alpha, nx, ny, de)
            template = t.template()

            curv = dem._calculate_directional_laplacian(this_alpha)

            this_amp, this_snr = match_template(curv, template)
            mask = t.get_window_limits()
            this_amp[mask] = 0
            this_snr[mask] = 0

            best_amp = (best_snr > this_snr)*best_amp + (best_snr < this_snr)*this_amp
            best_age = (best_snr > this_snr)*best_age + (best_snr < this_snr)*this_age
            best_alpha = (best_snr > this_snr)*best_alpha + (best_snr < this_snr)*this_alpha
            best_snr = (best_snr > this_snr)*best_snr + (best_snr < this_snr)*this_snr

        pbar.update((i+1)*len(ages))

    best_snr = ParameterGrid(dem, best_snr, d, name='SNR')
    best_amp = ParameterGrid(dem, best_amp, d, name='Amplitude', units='m')
    best_age = ParameterGrid(dem, best_age, d, name='Morphologic age', units='m^2')
    best_alpha = ParameterGrid(dem, best_alpha, d, name='Orientation', units='deg.')

    return best_amp, best_age, best_alpha, best_snr

def compare_fits(grids):

    s = grids[0].amplitude.shape
    best_snr = np.zeros(s)
    best_amp = np.zeros(s)
    best_age = np.zeros(s)
    best_alpha = np.zeros(s)

    for fit in grids:
        best_amp = (best_snr > fit.snr)*best_amp + (best_snr < fit.snr)*fit.amplitude
        best_age = (best_snr > fit.snr)*best_age + (best_snr < fit.snr)*fit.age
        best_alpha = (best_snr > fit.snr)*best_alpha + (best_snr < fit.snr)*fit.alpha
        best_snr = (best_snr > fit.snr)*best_snr + (best_snr < fit.snr)*fit.snr

    best_snr = ParameterGrid(dem, best_snr, grids[0].d, name='SNR')
    best_amp = ParameterGrid(dem, best_amp, grids[0].d, name='Amplitude', units='m')
    best_age = ParameterGrid(dem, best_age, grids[0].d, name='Morphologic age', units='m^2')
    best_alpha = ParameterGrid(dem, best_alpha, grids[0].d, name='Orientation', units='deg.')

    return best_amp, best_age, best_alpha, best_snr

#@profile
def match_template(data, template):

    #template = template_function(template_args)

    if data.ndim < template.ndim:
        raise ValueError("Dimensions of template must be less than or equal to dimensions of data matrix")
    if np.any(np.less(data.shape, template.shape)):
        raise ValueError("Size of template must be less than or equal to size of data matrix")

    pad_width = tuple((wid, wid) for wid in template.shape)

    #data = np.pad(data, pad_width=pad_width, mode='symmetric')

    M = template != 0
    fc = fft2(data)
    ft = fft2(template)
    fc2 = fft2(data**2)
    fm2 = fft2(M)

    #xcorr = signal.fftconvolve(data, template, mode='same')
    xcorr = np.real(fftshift(ifft2(ft*fc)))
    template_sum = np.sum(template**2)

    amp = xcorr/template_sum

    n = np.sum(M) + eps
    T1 = template_sum*(amp**2)
    T2 = -2*amp*xcorr
    T3 = fftshift(ifft2(fc2*fm2))
    error = (1/n)*(T1 + T2 + T3) + eps # avoid small-magnitude dvision
    #error = (1/n)*(amp**2*template_sum - 2*amp*fftshift(ifft2(fc*ft)) + fftshift(ifft2(fc2*fm2))) + eps
    error = np.real(error)
    snr = T1/error
    snr = np.real(snr)

    return amp, snr

def plot_results(dem, amp, age, alpha, snr):

    results = [amp, age, alpha, snr]

    for data in results:
        plt.figure()
        hs = Hillshade(dem)
        hs.plot()
        data.plot()
        plt.savefig(data.name + '_' + dem.label + '.png', dpi=300, bbox_inches='tight')
        plt.show()

def save_results(dem, amp, age, alpha, snr, base_dir=''):

    results = [amp, age, alpha, snr] # as ParameterGrid objects

    labels = {'Amplitude' : 'm',
            'Morphologic age' : 'm^2',
            'Orientation' : 'deg.',
            'Signal-to-noise ratio' : ''}

    for data, param in zip(results, labels):
        #grid = ParameterGrid(dem, data, d, name=param, units=labels[param])
        filename = '_'.join(param.split(' ')) + '_' + dem.label + '.tif'
        filename = base_dir + filename
        data.save(filename)

class ParameterGrid(BaseSpatialGrid):

    def __init__(self, dem, data, d, name='', units=''):

        self._georef_info = dem._georef_info
        self._griddata = data
        self.d = d
        self.name = name
        self.units = units

    def plot(self, alpha=0.5, colormap='viridis'):

        plt.imshow(self._griddata, alpha=alpha, cmap=colormap)
        cb = plt.colorbar(orientation='horizontal', extend='both')
        label = self.name + ' [' + self.units + ']'
        cb.set_label(label)

# XXX: necessary?
class TemplateFit(object):

    def __init__(d, age, alpha, amplitude, snr):

        self.d = d
        self.age = age
        self.alpha = alpha
        self.amplitude = amplitude
        self.snr = snr

<commit_msg>Add run all and SNR masking functions, plotting cleanup<commit_after>""" Functions for determinig best-fit template parameters by convolution with a
grid """

from dem import BaseSpatialGrid, Hillshade
import WindowedTemplate as wt

import numpy as np
import matplotlib.pyplot as plt
import pyfftw
from pyfftw.interfaces.numpy_fft import fft2, ifft2, fftshift

from progressbar import ProgressBar, Bar, Percentage, ETA

eps = np.spacing(1)
pyfftw.interfaces.cache.enable()

def calculate_amplitude(dem, Template, d, age, alpha):

    ny, nx = dem._griddata.shape
    de = dem._georef_info.dx
    t = Template(d, age, alpha, nx, ny, de)
    template = t.template()

    curv = dem._calculate_directional_laplacian(alpha)

    amp, snr = match_template(curv, template)
    mask = t.get_window_limits()
    amp[mask] = 0
    snr[mask] = 0

    return amp, snr

#@profile
def calculate_best_fit_parameters(dem, Template, **kwargs):

    #args = parse_args(**kwargs)
    d = 100
    de = 1

    age_max = 3
    age_min = 0
    age_stepsize = 0.5
    ang_stepsize = 2

    num_angles = 180/ang_stepsize + 1
    num_ages = (age_max - age_min)/age_stepsize
    orientations = np.linspace(-np.pi/2, np.pi/2, num_angles)
    ages = np.linspace(age_min, age_max, num_ages)

    ny, nx = dem._griddata.shape
    best_amp = np.zeros((ny, nx))
    best_age = np.zeros((ny, nx))
    best_alpha = np.zeros((ny, nx))
    best_snr = np.zeros((ny, nx))

    pbar = ProgressBar(widgets=[Percentage(), Bar(left='[', right=']'), ' ', ETA()], maxval=len(ages)*len(orientations)).start()

    for i, this_alpha in enumerate(orientations):
        for j, this_age in enumerate(ages):

            this_age = 10**this_age

            #args['kt'] = this_age
            #args['alpha'] = this_alpha
            #t = Template(args)
            t = Template(d, this_age, this_alpha, nx, ny, de)
            template = t.template()

            curv = dem._calculate_directional_laplacian(this_alpha)

            this_amp, this_snr = match_template(curv, template)
            mask = t.get_window_limits()
            this_amp[mask] = 0
            this_snr[mask] = 0

            best_amp = (best_snr > this_snr)*best_amp + (best_snr < this_snr)*this_amp
            best_age = (best_snr > this_snr)*best_age + (best_snr < this_snr)*this_age
            best_alpha = (best_snr > this_snr)*best_alpha + (best_snr < this_snr)*this_alpha
            best_snr = (best_snr > this_snr)*best_snr + (best_snr < this_snr)*this_snr

        pbar.update((i+1)*len(ages))

    best_snr = ParameterGrid(dem, best_snr, d, name='SNR')
    best_amp = ParameterGrid(dem, best_amp, d, name='Amplitude', units='m')
    best_age = ParameterGrid(dem, best_age, d, name='Morphologic age', units='m^2')
    best_alpha = ParameterGrid(dem, best_alpha, d, name='Orientation', units='deg.')

    return best_amp, best_age, best_alpha, best_snr

def compare_fits(grids):

    s = grids[0].amplitude.shape
    best_snr = np.zeros(s)
    best_amp = np.zeros(s)
    best_age = np.zeros(s)
    best_alpha = np.zeros(s)

    for fit in grids:
        best_amp = (best_snr > fit.snr)*best_amp + (best_snr < fit.snr)*fit.amplitude
        best_age = (best_snr > fit.snr)*best_age + (best_snr < fit.snr)*fit.age
        best_alpha = (best_snr > fit.snr)*best_alpha + (best_snr < fit.snr)*fit.alpha
        best_snr = (best_snr > fit.snr)*best_snr + (best_snr < fit.snr)*fit.snr

    best_snr = ParameterGrid(dem, best_snr, grids[0].d, name='SNR')
    best_amp = ParameterGrid(dem, best_amp, grids[0].d, name='Amplitude', units='m')
    best_age = ParameterGrid(dem, best_age, grids[0].d, name='Morphologic age', units='m^2')
    best_alpha = ParameterGrid(dem, best_alpha, grids[0].d, name='Orientation', units='deg.')

    return best_amp, best_age, best_alpha, best_snr

def mask_by_snr(amp, age, alpha, snr, thresh=None):

    if thresh is None:
        thresh = np.nanmean(snr._griddata)

    mask = snr._griddata < thresh

    amp._griddata[mask] = np.nan
    age._griddata[mask] = np.nan
    alpha._griddata[mask] = np.nan
    snr._griddata[mask] = np.nan

    return amp, age, alpha, snr

#@profile
def match_template(data, template):

    #template = template_function(template_args)

    if data.ndim < template.ndim:
        raise ValueError("Dimensions of template must be less than or equal to dimensions of data matrix")
    if np.any(np.less(data.shape, template.shape)):
        raise ValueError("Size of template must be less than or equal to size of data matrix")

    pad_width = tuple((wid, wid) for wid in template.shape)

    #data = np.pad(data, pad_width=pad_width, mode='symmetric')

    M = template != 0
    fc = fft2(data)
    ft = fft2(template)
    fc2 = fft2(data**2)
    fm2 = fft2(M)

    #xcorr = signal.fftconvolve(data, template, mode='same')
    xcorr = np.real(fftshift(ifft2(ft*fc)))
    template_sum = np.sum(template**2)

    amp = xcorr/template_sum

    # TODO  remove intermediate terms to make more memory efficent
    n = np.sum(M) + eps
    T1 = template_sum*(amp**2)
    T2 = -2*amp*xcorr
    T3 = fftshift(ifft2(fc2*fm2))
    error = (1/n)*(T1 + T2 + T3) + eps # avoid small-magnitude dvision
    #error = (1/n)*(amp**2*template_sum - 2*amp*fftshift(ifft2(fc*ft)) + fftshift(ifft2(fc2*fm2))) + eps
    error = np.real(error)
    snr = T1/error
    snr = np.real(snr)

    return amp, snr

def plot_results(dem, amp, age, alpha, snr, colormap='viridis'):

    results = [amp, age, alpha, snr]

    for data in results:
        plt.figure()
        hs = Hillshade(dem)
        hs.plot()
        data.plot(colormap=colormap)
        plt.savefig(data.name + '_' + dem.label + '.png', dpi=300, bbox_inches='tight')
        plt.show()

def run_on_grid(dem):

    amp, age, alpha, snr = calculate_best_fit_parameters(dem, wt.Scarp)
    amp, age, alpha, snr = mask_by_snr(amp, age, alpha, snr, thresh=1)
    plot_results(dem, amp, age, alpha, snr)

    return amp, age, alpha, snr

def save_results(dem, amp, age, alpha, snr, base_dir=''):

    results = [amp, age, alpha, snr] # as ParameterGrid objects

    labels = {'Amplitude' : 'm',
            'Morphologic age' : 'm^2',
            'Orientation' : 'deg.',
            'Signal-to-noise ratio' : ''}

    for data, param in zip(results, labels):
        #grid = ParameterGrid(dem, data, d, name=param, units=labels[param])
        filename = '_'.join(param.split(' ')) + '_' + dem.label + '.tif'
        filename = base_dir + filename
        data.save(filename)

class ParameterGrid(BaseSpatialGrid):

    def __init__(self, dem, data, d, name='', units=''):

        self._georef_info = dem._georef_info
        self._griddata = data
        self.d = d
        self.name = name
        self.units = units

    def plot(self, alpha=0.5, colormap='viridis'):

        plt.imshow(self._griddata, alpha=alpha, cmap=colormap)
        cb = plt.colorbar(orientation='horizontal', extend='both')
        label = self.name + ' [' + self.units + ']'
        cb.set_label(label)

# XXX: necessary?
class TemplateFit(object):

    def __init__(d, age, alpha, amplitude, snr):

        self.d = d
        self.age = age
        self.alpha = alpha
        self.amplitude = amplitude
        self.snr = snr

<|endoftext|>


<commit_before>#!/bin/python3

from telnetlib import Telnet
import re
from hashlib import sha512
from typing import *
from time import sleep
from os import stat, path
import yaml
import argparse
import datetime
prognameMatch = re.compile("\(\(\( filename: (.+) \)\)\)")
progDependencyMatch = re.compile("\(\(\( dependsOn: (.+) \)\)\)")
progIncludeMatch = re.compile('\(\(\( includes: (.+) as (\.\.[a-zA-Z0-9-]+) \)\)\)')
programFinder = "@find {}\n"
programFinderRegex = re.compile(b"(.+)([0-9]+):.+")
programFinderTerminator = re.compile(b'\*\*\*End of List\*\*\*')
ProgramId = re.compile(b"Program .+ created with number ([0-9]+)")
ProgramId2 = re.compile(
    b'Entering editor for .+\(#([0-9]+).+\)\.')
# Command to list content of a program, showing line numbers
programListCommand = "@list {}\n"
programListMatch = re.compile(b"\s*([0-9]+):(.+)\r\n")
programListTerminator = re.compile(b"[0-9]+ lines displayed\.")

editorInsertExitMatch = [re.compile(b"Exiting insert mode\.")]
editorCompilerStringMatch = [re.compile(b"Compiler done\."), re.compile(b"^Error in line")]
editorExitStringMatch = [re.compile(b"Editor exited\.")]

objectModifiedStringFieldMatch = \
    [
        re.compile(b"Modified: (.+) by (.+)$"),
        re.compile(b"I don't see that there\.$")
    ]
objectModificationCommand = "ex {}\n"

functionListCommand = "@listfunc {}\n"
functionListRegex = re.compile("\x1b\[[^m]*m")
# Goals:
#    Manage Dependencies:
#        Upload changed files in necessary order
#            Replacing special tokens with the correct program reference
#            Send minimal line-by-line diff in format accepted by @edit
#             and @prog
#            Provide Server-Client code synchronization when requested
#             (This will be hard to do properly in a very noisy server)
#   Provide cleanup functionality for things that totally fuck up the system

# Current stuff that needs doing
# [x] Determine if file needs to be updated due to the program being modified
#     since it was last retrieved.
# [ ] Better latency handling for the editor commands.
#    a. expect/error loop until match is found
#    b. Maybe the telnet class could do with a wrapper class
#       for handling this automatically.
# 3.

class SyncException(Exception):
    def __init__(self, filename, remoteid):
        super(Exception, self).__init__(filename, remoteid)
        self.message = "The object with id {} associated with {}".\
            format(remoteid, filename) + \
            " could not be found"

def because_I_cant_understand_strptime(s:str):
    months = {
        "Jan": 1,
        "Feb": 2,
        "Mar": 3,
        "Apr": 4,
        "May": 5,
        "Jun": 6,
        "Jul": 7,
        "Aug": 8,
        "Sep": 9,
        "Oct": 10,
        "Nov": 11,
        "Dec": 12
    }
    m = re.compile("(Sat|Sun|Mon|Tue|Wed|Thu|Fri) " +
                   "(Jan|Feb|Mar|Apr|May|Jun|Jul" +
                   "|Aug|Sep|Oct|Nov|Dec) " +
                   "([123 ][0-9]) " +
                   "([012 ][0-9]):" +
                   "([0-5][0-9]):" +
                   "([0-5][0-9]) " +
                   "(CST|CDT) " +
                   "([0-9]+)").match(s)
    month = months[m.group(2)]
    monthday = int(m.group(3))
    hour = int(m.group(4))
    minute = int(m.group(5))
    second = int(m.group(6))
    year = int(m.group(7))
    dt = datetime.datetime(year, month, day, hour, minute, second)
    return dt

class MufFile():
    def __init__(self, filename, depth=0, parent=None, send_method="name",id=None,
                 regname=None):
        self.dependencies = []
        self.transformedname = ""
        self.filename = filename
        self.hash = sha512()
        self.length = 0
        self.parent = parent
        self.includes = {}
        self.id = id
        self.regname = regname
        self.send_method = send_method
        with open(filename) as file:
            for z in file.readlines():
                pnmatch = prognameMatch.match(z)
                if pnmatch is not None:
                    self.transformedname = pnmatch.group(1)
                    continue
                pdepmatch = progDependencyMatch.match(z)
                if pdepmatch is not None:
                    self.dependencies.append(pdepmatch.group(1))
                    continue
                pincMatch = progIncludeMatch.match(z)
                if pincMatch is not None:
                    self.includes[pincMatch.group(2)] = pincMatch.group(1)
                self.hash.update(z.encode())
                self.length += 1
        self.hash = self.hash.hexdigest()

    def send(self, tc: Telnet):
        let_be = False
        while True:
            if self.send_method == "name":
                tc.write("@prog {}\n".format(self.transformedname).encode())
            elif self.send_method == "id":
                tc.write("@prog {}\n".format(self.id).encode())
            elif self.send_method == "regname":
                print("Using regname:{0}".format(self.regname))
                tc.write("@prog {}\n".format(self.regname).encode())
            mindex, match, _ = tc.expect([ProgramId, ProgramId2], timeout=3)
            if match is not None:
                self.id = int(match.group(1))
                break
        tc.write("1 {} delete\n".format(self.length * 10).encode())
        tc.write("i\n".encode())
        counter = 0
        with open(self.filename) as fi:
            lines = fi.readlines()
            if len(lines[-1]) > 0:
                lines.append('')
            for i in lines:
                tc.write("{}".format(i).encode())
#                sleep(0.05)
                counter += 1
                print("{: =4.2}%".format(100 * counter / len(lines)),
                      end='\r', flush=True)
        print("\n", end="", flush=True)
        print("finished sending")
        while True:
            tc.write('.\n'.encode())
            index, m, _ = tc.expect(editorInsertExitMatch,
                                    timeout=5)
            if m is not None:
                break
        print("compiling program")
        while True:
            tc.write("c\n".encode())
            index, m, _ = tc.expect(editorCompilerStringMatch,
                                    timeout=7)
            if index != None and index != 1:
                let_be = True
            if m is not None:
                break
        print("quitting")
        while True:
            if let_be:
                tc.write("q\n".encode())
            else:
                tc.write("x\n".encode())
            index, m, _ = tc.expect(editorExitStringMatch,
                                    timeout=7)
            if m is not None:
                break

    @staticmethod
    def check_last_modified(filename, remoteid, tc: Telnet):
        tc.send(objectModificationCommand.format(remoteid).encode())
        idx, match, _ = tc.expect(objectModifiedStringFieldMatch)
        if idx == 1:
            raise SyncException(filename, remoteid)
        #mod_date = datetime.datetime.strptime(match.group(1),
        #                                      "%a %b %d %H:%M:%S %Z %Y")
        mod_date = because_I_cant_understand_strptime(match.group(1))
        local_stuff = path.getmtime(filename)
        return mod_date >= local_stuff

    @staticmethod
    def sync(filename, remoteid, tc: Telnet):
        tc.read_very_eager()
        tc.write(b"@set me=H\n")
        tc.write(b"pub #alloff\n")
        sleep(2)
        tc.read_very_eager()
        tc.write(programListCommand.format(remoteid).encode())
        print(programListCommand.format(remoteid))
        with open(filename, 'w') as output:
            lines = tc.read_until(b" lines displayed.").decode().split('\r\n')
            for i in lines[:-1]:
                output.write(i + '\n')
        tc.write(b"@set me=!H\n")
        tc.write(b"pub #allon\n")
        tc.read_very_eager()
#            mindex = 0
#            while mindex < 1:
#                mindex, match, _ = tc.expect([programListMatch,
#                                               programListTerminator])
#                if mindex >= 1 \
#                   or match is None:
#                    break
#                output.write(match.group(2).decode()+'\n')

# Keep track of whether or not files are up to date on the server.
class Cache():
    def __init__(self, path):
        import pickle
        self.newfiles = {}
        self.oldfiles = {}
        try:
            self = pickle.load(path + ".cache")
        except IOError:
            # probably doesn't exist
            pass

    def addFile(self, file: MufFile):
        fname = file.filename
        if fname in self.oldfiles.keys():
            if self.newfiles[fname].hash != file.hash:
                self.oldfiles[fname] = self.newfiles[fname]
                self.newfiles[fname] = file

    def syncOld(self, file: MufFile, tc: Telnet):
        tc.write(programFinder.format(file.filename))
        mindex, match, _ = tc.expect([programFinderRegex,
                                      programFinderTerminator])
        fn = None
        while match is not None and mindex != 1:
            if match.group(1) == file.transformedname:
                fn = match.group(1)
                break
            else:
                mindex, match, _ = tc.expect([programFinderRegex,
                                              programFinderTerminator])
        tc.write(programListCommand.format(fn))
        mindex = 0
        lines = []
        lastindex = 0
        while mindex != 1:
            mindex, match, _ = tc.expect([programListMatch,
                                          programListTerminator])
            if mindex != 1:
                if int(math.group(2)) != lastindex + 1:
                    print("Hmm. There might be a problem.")
                else:
                    lastindex = int(match.group(1))
                lines.append(match.group(2))

class DepGraph():
    def __init__(self):
        self.nodes = {}
        self.edges = {}
        self.depths = {}
        self.validstarts = set()

    def addFile(self, file: MufFile, depth=0):
        self.nodes[file.filename] = file
        if file.filename not in self.edges.keys():
            self.edges[file.filename] = set()
            self.depths[file.filename] = depth
            if depth == 0:
                self.validstarts.add(file.filename)
        for fn in file.dependencies:
            self.edges[file.filename].add(fn)
            if fn not in self.nodes.keys():
                self.addFile(MufFile(fn, depth=depth + 1), depth + 1)

    def send(self, tc: Telnet):
        stack = list()
        path = []
        sent = set()
        for i in self.validstarts:
            stack.append(i)
            while len(stack) > 0:
                cn = stack.pop()
                if cn not in path:
                    path.append(cn)
                else:
                    continue
                for n in self.edges[cn]:
                    path.append(n)
                    stack.append(n)
            for n in reversed(path):
                print("Updating program {}".format(n))
                self.nodes[n].send(tc)

# TODO: Use a cache to check if the file needs to be uploaded again,
#  I.E. it's hash has changed.
# TODO: define a macro on the copy sent to the server such that each
#  program refers to the correct id at runtime.

# argInterpret = argparse.ArgumentParser()
# argInterpret.add_argument()
# tc = Telnet(host="localhost", port=2001)
# tc.write(b"connect one potrzebie\n")
# dg = DepGraph()
# dg.addFile(MufFile("Channel/Channel.muf"))
# dg.send(tc)
parser = argparse.ArgumentParser("Manage files on the MUCK")
parser.add_argument("--send", dest='files', action='append',
                    help='Files to send', default=[])
parser.add_argument('--sync', dest='sync', action='store_const',
                    help='Sync files?', const=True, default=False)
parser.add_argument('--send-all', dest='send_all', action='store_const',
                    help='send all files', const=True, default=False)
args = parser.parse_args()
with open('project.yaml') as projfile:
    project = yaml.load(projfile)
    print(project)
    project = project['project']
    tc = Telnet(host=project['connect']['host'],
                port=int(project['connect']['port']))
    tc.read_some()
    tc.write("connect {} {}\n".format(project['connect']['username'],
                                      project['connect']['password']).encode())
    print("connect {} {}".format(project['connect']['username'],
                                 project['connect']['password']))
    sleep(2)

    if args.sync:
        for i in project['sync']:
            if 'no_exist' in i['file'].keys() and i['file']['no_exist']:
                try:
                    stat(i['file']['name'])
                    print('skipping {}'.format(i['file']['name']))
                    continue
                except FileNotFoundError:
                    print('need to get {}'.format(i['file']['name']))
            MufFile.sync(i['file']['name'], i['file']['id'], tc)
    if args.send_all:
        for i in project['send']:
            f = None
            if 'send_method' in i['file'].keys():
                id = None
                regname = None
                print("Send method:"+i['file']['send_method'])
                if 'id' in i['file'].keys():
                    id = i['file']['id']
                if 'regname' in i['file'].keys():
                    regname = i['file']['regname']
                f = MufFile(i['file']['name'], send_method=i['file']['send_method'],
                        id=id,regname=regname)
            else:
                print("No send method found")
                f = MufFile(i['file']['name'])
            f.transformedname = i['file']['gamename']
            print("Sending " + f.transformedname)
            f.send(tc)
            sleep(1)
            print("\a")
    else:
        for i in project['send']:
            if i['file']['name'] not in args.files:
                continue
            send_with_id = False
            f = None
            if 'send_method' in i['file'].keys():
                id = None
                regname = None
                print("Send method:"+i['file']['send_method'])
                if 'id' in i['file'].keys():
                    id = i['file']['id']
                if 'regname' in i['file'].keys():
                    regname = i['file']['regname']
                f = MufFile(i['file']['name'], send_method=i['file']['send_method'],
                        id=id,regname=regname)
            else:
                f = MufFile(i['file']['name'])
            f.transformedname = i['file']['gamename']
            print("Sending " + f.transformedname)
            f.send(tc)
            sleep(1)
<commit_msg>Misc changes<commit_after>#!/bin/python3

from telnetlib import Telnet
import re
from hashlib import sha512
from typing import *
from time import sleep
from os import stat, path
import yaml
import argparse
import datetime
prognameMatch = re.compile("\(\(\( filename: (.+) \)\)\)")
progDependencyMatch = re.compile("\(\(\( dependsOn: (.+) \)\)\)")
progIncludeMatch = re.compile(
    '\(\(\( includes: (.+) as (\.\.[a-zA-Z0-9-]+) \)\)\)')
programFinder = "@find {}\n"
programFinderRegex = re.compile(b"(.+)([0-9]+):.+")
programFinderTerminator = re.compile(b'\*\*\*End of List\*\*\*')
ProgramId = re.compile(b"Program .+ created with number ([0-9]+)")
ProgramId2 = re.compile(
    b'Entering editor for .+\(#([0-9]+).+\)\.')
# Command to list content of a program, showing line numbers
programListCommand = "@list {}\n"
programListMatch = re.compile(b"\s*([0-9]+):(.+)\r\n")
programListTerminator = re.compile(b"[0-9]+ lines displayed\.")

editorInsertExitMatch = [re.compile(b"Exiting insert mode\.")]
editorCompilerStringMatch = [re.compile(
    b"Compiler done\."), re.compile(b"^Error in line")]
editorExitStringMatch = [re.compile(b"Editor exited\.")]

objectModifiedStringFieldMatch = \
    [
        re.compile(b"Modified: (.+) by (.+)$"),
        re.compile(b"I don't see that there\.$")
    ]
objectModificationCommand = "ex {}\n"

functionListCommand = "@listfunc {}\n"
functionListRegex = re.compile("\x1b\[[^m]*m")
# Goals:
#    Manage Dependencies:
#        Upload changed files in necessary order
#            Replacing special tokens with the correct program reference
#            Send minimal line-by-line diff in format accepted by @edit
#             and @prog
#            Provide Server-Client code synchronization when requested
#             (This will be hard to do properly in a very noisy server)
#   Provide cleanup functionality for things that totally fuck up the system

# Current stuff that needs doing
# [x] Determine if file needs to be updated due to the program being modified
#     since it was last retrieved.
# [ ] Better latency handling for the editor commands.
#    a. expect/error loop until match is found
#    b. Maybe the telnet class could do with a wrapper class
#       for handling this automatically.
# 3.

class SyncException(Exception):
    def __init__(self, filename, remoteid):
        super(Exception, self).__init__(filename, remoteid)
        self.message = "The object with id {} associated with {}".\
            format(remoteid, filename) + \
            " could not be found"

def because_I_cant_understand_strptime(s: str):
    months = {
        "Jan": 1,
        "Feb": 2,
        "Mar": 3,
        "Apr": 4,
        "May": 5,
        "Jun": 6,
        "Jul": 7,
        "Aug": 8,
        "Sep": 9,
        "Oct": 10,
        "Nov": 11,
        "Dec": 12
    }
    m = re.compile("(Sat|Sun|Mon|Tue|Wed|Thu|Fri) " +
                   "(Jan|Feb|Mar|Apr|May|Jun|Jul" +
                   "|Aug|Sep|Oct|Nov|Dec) " +
                   "([123 ][0-9]) " +
                   "([012 ][0-9]):" +
                   "([0-5][0-9]):" +
                   "([0-5][0-9]) " +
                   "(CST|CDT) " +
                   "([0-9]+)").match(s)
    month = months[m.group(2)]
    monthday = int(m.group(3))
    hour = int(m.group(4))
    minute = int(m.group(5))
    second = int(m.group(6))
    year = int(m.group(7))
    dt = datetime.datetime(year, month, day, hour, minute, second)
    return dt

class MufFile():
    def __init__(self, filename, depth=0, parent=None, send_method="name", id=None,
                 regname=None):
        self.dependencies = []
        self.transformedname = ""
        self.filename = filename
        self.hash = sha512()
        self.length = 0
        self.parent = parent
        self.includes = {}
        self.id = id
        self.regname = regname
        self.send_method = send_method
        with open(filename) as file:
            for z in file.readlines():
                pnmatch = prognameMatch.match(z)
                if pnmatch is not None:
                    self.transformedname = pnmatch.group(1)
                    continue
                pdepmatch = progDependencyMatch.match(z)
                if pdepmatch is not None:
                    self.dependencies.append(pdepmatch.group(1))
                    continue
                pincMatch = progIncludeMatch.match(z)
                if pincMatch is not None:
                    self.includes[pincMatch.group(2)] = pincMatch.group(1)
                self.hash.update(z.encode())
                self.length += 1
        self.hash = self.hash.hexdigest()

    def send(self, tc: Telnet):
        let_be = False
        while True:
            if self.send_method == "name":
                tc.write("@prog {}\n".format(self.transformedname).encode())
            elif self.send_method == "id":
                tc.write("@prog {}\n".format(self.id).encode())
            elif self.send_method == "regname":
                print("Using regname:{0}".format(self.regname))
                tc.write("@prog {}\n".format(self.regname).encode())
            mindex, match, _ = tc.expect([ProgramId, ProgramId2], timeout=3)
            if match is not None:
                self.id = int(match.group(1))
                break
        tc.write("1 {} delete\n".format(self.length * 10).encode())
        tc.write("i\n".encode())
        counter = 0
        with open(self.filename) as fi:
            lines = fi.readlines()
            if len(lines[-1]) > 0:
                lines.append('')
            for i in lines:
                tc.write("{}".format(i).encode())
#                sleep(0.05)
                counter += 1
                print("{: =4.2}%".format(100 * counter / len(lines)),
                      end='\r', flush=True)
        print("\n", end="", flush=True)
        print("finished sending")
        while True:
            tc.write('.\n'.encode())
            index, m, _ = tc.expect(editorInsertExitMatch,
                                    timeout=5)
            if m is not None:
                break
        print("compiling program")
        while True:
            tc.write("c\n".encode())
            index, m, line = tc.expect(editorCompilerStringMatch,
                                       timeout=7)
            if index != None and index != 1:
                print("Message Recieved")
                print(line.decode("ascii"))
                let_be = True
            if m is not None:
                break
        print("quitting")
        while True:
            if let_be:
                tc.write("q\n".encode())
            else:
                tc.write("x\n".encode())
            index, m, _ = tc.expect(editorExitStringMatch,
                                    timeout=7)
            if m is not None:
                break

    @staticmethod
    def check_last_modified(filename, remoteid, tc: Telnet):
        tc.send(objectModificationCommand.format(remoteid).encode())
        idx, match, _ = tc.expect(objectModifiedStringFieldMatch)
        if idx == 1:
            raise SyncException(filename, remoteid)
        # mod_date = datetime.datetime.strptime(match.group(1),
        #                                      "%a %b %d %H:%M:%S %Z %Y")
        mod_date = because_I_cant_understand_strptime(match.group(1))
        local_stuff = path.getmtime(filename)
        return mod_date >= local_stuff

    @staticmethod
    def sync(filename, remoteid, tc: Telnet):
        tc.read_very_eager()
        tc.write(b"@set me=H\n")
        tc.write(b"pub #alloff\n")
        sleep(2)
        tc.read_very_eager()
        tc.write(programListCommand.format(remoteid).encode())
        print(programListCommand.format(remoteid))
        with open(filename, 'w') as output:
            lines = tc.read_until(b" lines displayed.").decode().split('\r\n')
            for i in lines[:-1]:
                output.write(i + '\n')
        tc.write(b"@set me=!H\n")
        tc.write(b"pub #allon\n")
        tc.read_very_eager()
#            mindex = 0
#            while mindex < 1:
#                mindex, match, _ = tc.expect([programListMatch,
#                                               programListTerminator])
#                if mindex >= 1 \
#                   or match is None:
#                    break
#                output.write(match.group(2).decode()+'\n')

# Keep track of whether or not files are up to date on the server.
class Cache():
    def __init__(self, path):
        import pickle
        self.newfiles = {}
        self.oldfiles = {}
        try:
            self = pickle.load(path + ".cache")
        except IOError:
            # probably doesn't exist
            pass

    def addFile(self, file: MufFile):
        fname = file.filename
        if fname in self.oldfiles.keys():
            if self.newfiles[fname].hash != file.hash:
                self.oldfiles[fname] = self.newfiles[fname]
                self.newfiles[fname] = file

    def syncOld(self, file: MufFile, tc: Telnet):
        tc.write(programFinder.format(file.filename))
        mindex, match, _ = tc.expect([programFinderRegex,
                                      programFinderTerminator])
        fn = None
        while match is not None and mindex != 1:
            if match.group(1) == file.transformedname:
                fn = match.group(1)
                break
            else:
                mindex, match, _ = tc.expect([programFinderRegex,
                                              programFinderTerminator])
        tc.write(programListCommand.format(fn))
        mindex = 0
        lines = []
        lastindex = 0
        while mindex != 1:
            mindex, match, _ = tc.expect([programListMatch,
                                          programListTerminator])
            if mindex != 1:
                if int(math.group(2)) != lastindex + 1:
                    print("Hmm. There might be a problem.")
                else:
                    lastindex = int(match.group(1))
                lines.append(match.group(2))

class DepGraph():
    def __init__(self):
        self.nodes = {}
        self.edges = {}
        self.depths = {}
        self.validstarts = set()

    def addFile(self, file: MufFile, depth=0):
        self.nodes[file.filename] = file
        if file.filename not in self.edges.keys():
            self.edges[file.filename] = set()
            self.depths[file.filename] = depth
            if depth == 0:
                self.validstarts.add(file.filename)
        for fn in file.dependencies:
            self.edges[file.filename].add(fn)
            if fn not in self.nodes.keys():
                self.addFile(MufFile(fn, depth=depth + 1), depth + 1)

    def send(self, tc: Telnet):
        stack = list()
        path = []
        sent = set()
        for i in self.validstarts:
            stack.append(i)
            while len(stack) > 0:
                cn = stack.pop()
                if cn not in path:
                    path.append(cn)
                else:
                    continue
                for n in self.edges[cn]:
                    path.append(n)
                    stack.append(n)
            for n in reversed(path):
                print("Updating program {}".format(n))
                self.nodes[n].send(tc)

# TODO: Use a cache to check if the file needs to be uploaded again,
#  I.E. it's hash has changed.
# TODO: define a macro on the copy sent to the server such that each
#  program refers to the correct id at runtime.

# argInterpret = argparse.ArgumentParser()
# argInterpret.add_argument()
# tc = Telnet(host="localhost", port=2001)
# tc.write(b"connect one potrzebie\n")
# dg = DepGraph()
# dg.addFile(MufFile("Channel/Channel.muf"))
# dg.send(tc)
parser = argparse.ArgumentParser("Manage files on the MUCK")
parser.add_argument("--send", dest='files', action='append',
                    help='Files to send', default=[])
parser.add_argument('--sync', dest='sync', action='store_const',
                    help='Sync files?', const=True, default=False)
parser.add_argument('--force-sync', default=[],
                    dest='needsync',
                    action='append', help='Force a file to be synced')
parser.add_argument('--send-all', dest='send_all', action='store_const',
                    help='send all files', const=True, default=False)
args = parser.parse_args()
with open('project.yaml') as projfile:
    project = yaml.load(projfile)
    print(project)
    project = project['project']
    tc = Telnet(host=project['connect']['host'],
                port=int(project['connect']['port']))
    tc.read_some()
    tc.write("connect {} {}\n".format(project['connect']['username'],
                                      project['connect']['password']).encode())
    print("connect {} {}".format(project['connect']['username'],
                                 project['connect']['password']))
    sleep(2)

    if args.sync:
        for i in project['sync']:
            if 'no_exist' in i['file'].keys() and i['file']['no_exist']:
                try:
                    stat(i['file']['name'])
                    print('skipping {}'.format(i['file']['name']))
                    continue
                except FileNotFoundError:
                    print('need to get {}'.format(i['file']['name']))
            MufFile.sync(i['file']['name'], i['file']['id'], tc)
    for i in project['sync']:
        if i['file']['name'] in args.needsync:
            MufFile.sync(i['file']['name'], i['file']['id'], tc)
    if args.send_all:
        for i in project['send']:
            f = None
            if 'send_method' in i['file'].keys():
                id = None
                regname = None
                print("Send method:" + i['file']['send_method'])
                if 'id' in i['file'].keys():
                    id = i['file']['id']
                if 'regname' in i['file'].keys():
                    regname = i['file']['regname']
                f = MufFile(i['file']['name'], send_method=i['file']['send_method'],
                            id=id, regname=regname)
            else:
                print("No send method found")
                f = MufFile(i['file']['name'])
            f.transformedname = i['file']['gamename']
            print("Sending " + f.transformedname)
            f.send(tc)
            sleep(1)
            print("\a")
    else:
        for i in project['send']:
            if i['file']['name'] not in args.files:
                continue
            send_with_id = False
            f = None
            if 'send_method' in i['file'].keys():
                id = None
                regname = None
                print("Send method:" + i['file']['send_method'])
                if 'id' in i['file'].keys():
                    id = i['file']['id']
                if 'regname' in i['file'].keys():
                    regname = i['file']['regname']
                f = MufFile(i['file']['name'], send_method=i['file']['send_method'],
                            id=id, regname=regname)
            else:
                f = MufFile(i['file']['name'])
            f.transformedname = i['file']['gamename']
            print("Sending " + f.transformedname)
            f.send(tc)
            sleep(1)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
# Copyright 2012 Google, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
:module: watchdog.observers.inotify
:synopsis: ``inotify(7)`` based emitter implementation.
:author: Sebastien Martini <seb@dbzteam.org>
:author: Luke McCarthy <luke@iogopro.co.uk>
:author: yesudeep@google.com (Yesudeep Mangalapilly)
:author: Tim Cuthbertson <tim+github@gfxmonk.net>
:platforms: Linux 2.6.13+.

.. ADMONITION:: About system requirements

    Recommended minimum kernel version: 2.6.25.

    Quote from the inotify(7) man page:

        "Inotify was merged into the 2.6.13 Linux kernel. The required library
        interfaces were added to glibc in version 2.4. (IN_DONT_FOLLOW,
        IN_MASK_ADD, and IN_ONLYDIR were only added in version 2.5.)"

    Therefore, you must ensure the system is running at least these versions
    appropriate libraries and the kernel.

.. ADMONITION:: About recursiveness, event order, and event coalescing

    Quote from the inotify(7) man page:

        If successive output inotify events produced on the inotify file
        descriptor are identical (same wd, mask, cookie, and name) then they
        are coalesced into a single event if the older event has not yet been
        read (but see BUGS).

        The events returned by reading from an inotify file descriptor form
        an ordered queue. Thus, for example, it is guaranteed that when
        renaming from one directory to another, events will be produced in
        the correct order on the inotify file descriptor.

        ...

        Inotify monitoring of directories is not recursive: to monitor
        subdirectories under a directory, additional watches must be created.

    This emitter implementation therefore automatically adds watches for
    sub-directories if running in recursive mode.

Some extremely useful articles and documentation:

.. _inotify FAQ: http://inotify.aiken.cz/?section=inotify&page=faq&lang=en
.. _intro to inotify: http://www.linuxjournal.com/article/8478

"""

from __future__ import with_statement

import os
import threading
from .inotify_c import Inotify

from watchdog.observers.api import (
    EventEmitter,
    BaseObserver,
    DEFAULT_EMITTER_TIMEOUT,
    DEFAULT_OBSERVER_TIMEOUT
)

from watchdog.events import (
    DirDeletedEvent,
    DirModifiedEvent,
    DirMovedEvent,
    DirCreatedEvent,
    FileDeletedEvent,
    FileModifiedEvent,
    FileMovedEvent,
    FileCreatedEvent,
    EVENT_TYPE_MODIFIED,
    EVENT_TYPE_CREATED,
    EVENT_TYPE_DELETED,
    EVENT_TYPE_MOVED
)

ACTION_EVENT_MAP = {
    (True, EVENT_TYPE_MODIFIED): DirModifiedEvent,
    (True, EVENT_TYPE_CREATED): DirCreatedEvent,
    (True, EVENT_TYPE_DELETED): DirDeletedEvent,
    (True, EVENT_TYPE_MOVED): DirMovedEvent,
    (False, EVENT_TYPE_MODIFIED): FileModifiedEvent,
    (False, EVENT_TYPE_CREATED): FileCreatedEvent,
    (False, EVENT_TYPE_DELETED): FileDeletedEvent,
    (False, EVENT_TYPE_MOVED): FileMovedEvent,
}

class InotifyEmitter(EventEmitter):
    """
    inotify(7)-based event emitter.

    :param event_queue:
        The event queue to fill with events.
    :param watch:
        A watch object representing the directory to monitor.
    :type watch:
        :class:`watchdog.observers.api.ObservedWatch`
    :param timeout:
        Read events blocking timeout (in seconds).
    :type timeout:
        ``float``
    """

    def __init__(self, event_queue, watch, timeout=DEFAULT_EMITTER_TIMEOUT):
        EventEmitter.__init__(self, event_queue, watch, timeout)
        self._lock = threading.Lock()
        self._inotify = Inotify(watch.path, watch.is_recursive)

    def on_thread_stop(self):
        self._inotify.close()

    def queue_events(self, timeout):
        with self._lock:
            inotify_events = self._inotify.read_events()
            if not any([event.is_moved_from or event.is_moved_to for event in inotify_events]):
                self._inotify.clear_move_records()
            for event in inotify_events:
                if event.is_moved_to:
                    # TODO: Sometimes this line will bomb even when a previous
                    # moved_from event with the same cookie has fired. I have
                    # yet to figure out why this is the case, so we're
                    # temporarily swallowing the exception and the move event.
                    # This happens only during massively quick file movement
                    # for example, when you execute `git gc` in a monitored
                    # directory.
                    try:
                        src_path = self._inotify.source_for_move(event)
                        to_event = event
                        dest_path = to_event.src_path
                        klass = ACTION_EVENT_MAP[(to_event.is_directory, EVENT_TYPE_MOVED)]
                        event = klass(src_path, dest_path)
                        self.queue_event(event)
                        # Generate sub events for the directory if recursive.
                        if event.is_directory and self.watch.is_recursive:
                            for sub_event in event.sub_moved_events():
                                self.queue_event(sub_event)
                    except KeyError:
                        pass
                elif event.is_attrib:
                    klass = ACTION_EVENT_MAP[(event.is_directory, EVENT_TYPE_MODIFIED)]
                    self.queue_event(klass(event.src_path))
                elif event.is_modify:
                    klass = ACTION_EVENT_MAP[(event.is_directory, EVENT_TYPE_MODIFIED)]
                    self.queue_event(klass(event.src_path))
                elif event.is_delete_self:
                    klass = ACTION_EVENT_MAP[(event.is_directory, EVENT_TYPE_DELETED)]
                    self.queue_event(klass(event.src_path))
                elif event.is_delete:
                    klass = ACTION_EVENT_MAP[(event.is_directory, EVENT_TYPE_DELETED)]
                    self.queue_event(klass(event.src_path))
                    self.queue_event(DirModifiedEvent(os.path.dirname(event.src_path)))
                elif event.is_create:
                    klass = ACTION_EVENT_MAP[(event.is_directory, EVENT_TYPE_CREATED)]
                    self.queue_event(klass(event.src_path))
                    self.queue_event(DirModifiedEvent(os.path.dirname(event.src_path)))

class InotifyObserver(BaseObserver):
    """
    Observer thread that schedules watching directories and dispatches
    calls to event handlers.
    """

    def __init__(self, timeout=DEFAULT_OBSERVER_TIMEOUT):
        BaseObserver.__init__(self, emitter_class=InotifyEmitter,
                              timeout=timeout)
<commit_msg>update emitter to use buffered inotify<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
# Copyright 2012 Google, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
:module: watchdog.observers.inotify
:synopsis: ``inotify(7)`` based emitter implementation.
:author: Sebastien Martini <seb@dbzteam.org>
:author: Luke McCarthy <luke@iogopro.co.uk>
:author: yesudeep@google.com (Yesudeep Mangalapilly)
:author: Tim Cuthbertson <tim+github@gfxmonk.net>
:platforms: Linux 2.6.13+.

.. ADMONITION:: About system requirements

    Recommended minimum kernel version: 2.6.25.

    Quote from the inotify(7) man page:

        "Inotify was merged into the 2.6.13 Linux kernel. The required library
        interfaces were added to glibc in version 2.4. (IN_DONT_FOLLOW,
        IN_MASK_ADD, and IN_ONLYDIR were only added in version 2.5.)"

    Therefore, you must ensure the system is running at least these versions
    appropriate libraries and the kernel.

.. ADMONITION:: About recursiveness, event order, and event coalescing

    Quote from the inotify(7) man page:

        If successive output inotify events produced on the inotify file
        descriptor are identical (same wd, mask, cookie, and name) then they
        are coalesced into a single event if the older event has not yet been
        read (but see BUGS).

        The events returned by reading from an inotify file descriptor form
        an ordered queue. Thus, for example, it is guaranteed that when
        renaming from one directory to another, events will be produced in
        the correct order on the inotify file descriptor.

        ...

        Inotify monitoring of directories is not recursive: to monitor
        subdirectories under a directory, additional watches must be created.

    This emitter implementation therefore automatically adds watches for
    sub-directories if running in recursive mode.

Some extremely useful articles and documentation:

.. _inotify FAQ: http://inotify.aiken.cz/?section=inotify&page=faq&lang=en
.. _intro to inotify: http://www.linuxjournal.com/article/8478

"""

from __future__ import with_statement

import os
import threading
from .inotify_buffered import InotifyBuffered

from watchdog.observers.api import (
    EventEmitter,
    BaseObserver,
    DEFAULT_EMITTER_TIMEOUT,
    DEFAULT_OBSERVER_TIMEOUT
)

from watchdog.events import (
    DirDeletedEvent,
    DirModifiedEvent,
    DirMovedEvent,
    DirCreatedEvent,
    FileDeletedEvent,
    FileModifiedEvent,
    FileMovedEvent,
    FileCreatedEvent,
    EVENT_TYPE_MODIFIED,
    EVENT_TYPE_CREATED,
    EVENT_TYPE_DELETED,
    EVENT_TYPE_MOVED
)

ACTION_EVENT_MAP = {
    (True, EVENT_TYPE_MODIFIED): DirModifiedEvent,
    (True, EVENT_TYPE_CREATED): DirCreatedEvent,
    (True, EVENT_TYPE_DELETED): DirDeletedEvent,
    (True, EVENT_TYPE_MOVED): DirMovedEvent,
    (False, EVENT_TYPE_MODIFIED): FileModifiedEvent,
    (False, EVENT_TYPE_CREATED): FileCreatedEvent,
    (False, EVENT_TYPE_DELETED): FileDeletedEvent,
    (False, EVENT_TYPE_MOVED): FileMovedEvent,
}

class InotifyEmitter(EventEmitter):
    """
    inotify(7)-based event emitter.

    :param event_queue:
        The event queue to fill with events.
    :param watch:
        A watch object representing the directory to monitor.
    :type watch:
        :class:`watchdog.observers.api.ObservedWatch`
    :param timeout:
        Read events blocking timeout (in seconds).
    :type timeout:
        ``float``
    """

    def __init__(self, event_queue, watch, timeout=DEFAULT_EMITTER_TIMEOUT):
        EventEmitter.__init__(self, event_queue, watch, timeout)
        self._lock = threading.Lock()
        self._inotify = InotifyBuffered(watch.path, watch.is_recursive)

    def on_thread_stop(self):
        self._inotify.close()

    def queue_events(self, timeout):
        with self._lock:
            event = self._inotify.read_event()

            try:
                move_from, move_to = event
                klass = ACTION_EVENT_MAP[(move_from.is_directory, EVENT_TYPE_MOVED)]
                event = klass(move_from.src_path, move_to.src_path)
                self.queue_event(event)

                #TODO: remove all record keeping code from inotify_c
                #if not any([event.is_moved_from or event.is_moved_to for event in inotify_events]):
                #    self._inotify.clear_move_records()
                #for event in inotify_events:

                if move_from.is_directory and self.watch.is_recursive:
                    for sub_event in event.sub_moved_events():
                        self.queue_event(sub_event)

            except TypeError:
                if event.is_attrib:
                    klass = ACTION_EVENT_MAP[(event.is_directory, EVENT_TYPE_MODIFIED)]
                    self.queue_event(klass(event.src_path))
                elif event.is_modify:
                    klass = ACTION_EVENT_MAP[(event.is_directory, EVENT_TYPE_MODIFIED)]
                    self.queue_event(klass(event.src_path))
                elif event.is_delete_self:
                    klass = ACTION_EVENT_MAP[(event.is_directory, EVENT_TYPE_DELETED)]
                    self.queue_event(klass(event.src_path))
                elif event.is_delete or event.is_moved_from:
                    klass = ACTION_EVENT_MAP[(event.is_directory, EVENT_TYPE_DELETED)]
                    self.queue_event(klass(event.src_path))
                    self.queue_event(DirModifiedEvent(os.path.dirname(event.src_path)))
                elif event.is_create or event.is_moved_to:
                    klass = ACTION_EVENT_MAP[(event.is_directory, EVENT_TYPE_CREATED)]
                    self.queue_event(klass(event.src_path))
                    self.queue_event(DirModifiedEvent(os.path.dirname(event.src_path)))

class InotifyObserver(BaseObserver):
    """
    Observer thread that schedules watching directories and dispatches
    calls to event handlers.
    """

    def __init__(self, timeout=DEFAULT_OBSERVER_TIMEOUT):
        BaseObserver.__init__(self, emitter_class=InotifyEmitter,
                              timeout=timeout)
<|endoftext|>


<commit_before># Poll NextBus for real-time bus locations and predictions

import datetime
import os
import pickle
import urllib
from lxml import etree

try:
    import MySQLdb
except ImportError:
    import pymysql as MySQLdb

from TransitAgency import TransitAgency
from Bus import Bus
from Direction import Direction
from Line import Line
from Prediction import Prediction
from Stop import Stop

class NextBus (TransitAgency):

    # default local data directory
    sDirectory = None

    # HTTP interface for NextBus
    sUrlNextbus = 'http://webservices.nextbus.com/service/publicXMLFeed?'
    sAgency = '&a=mbta'
    sRoute = '&r='
    sStopId = '&stopId='
    sStop = '&stops='
    sFlags = '&terse'
    sTime = '&t='
    sCommandGetStops = 'command=routeConfig'
    sCommandPredictions = 'command=predictions'
    sCommandMultiplePredictions = 'command=predictionsForMultiStops'
    sCommandVehicleLocations = 'command=vehicleLocations'

    # initialization
    def __init__(self, sDirectory=None, sAgency=None):

        # call initializers for super classes
        TransitAgency.__init__(self)

        if sDirectory:
            self.sDirectory = sDirectory
        else:
            self.sDirectory = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/')

        if sAgency:
            self.sAgency = str('&a=' + sAgency)

    # update predictions for a particular line/direction
    #
    # inputs
    #   nRouteNumber: the line name that NextBus uses to identify the desired route
    #   sRouteDirection: the route direction to return predictions for
    #
    # outputs
    #   returns an array of predictions for each direction
    def _getPredictions(self, nLine, sDirection):

        line = self.lines[nLine]
        dStops = line.getStopsFor(sDirection)

        # create URL string for multiple stops
        sStops = ''

        # iterate through a list of stop numbers to create the NextBus query
        # and clear any existing predictions
        for stopID, data in dStops.iteritems():
            sStops = sStops + '&stops=' + nLine + '|' + data.id

        output = {}

        try:
            fhPredictions = urllib.urlopen(self.sUrlNextbus
                                           + self.sCommandMultiplePredictions
                                           + self.sAgency
                                           + sStops)

            xml = fhPredictions.read()
            fhPredictions.close()

            root = etree.fromstring(xml)

            # process stop predictions
            for element in root.findall('predictions'):

                try:
                    stopTag = element.attrib['stopTag']
                    # routeTag = element.attrib['routeTag']

                    output[stopTag] = []

                    for elementA in element.findall('direction'):
                        for elementB in elementA.findall('prediction'):

                            try:
                                # direction = elementB.attrib['dirTag']
                                vehicle = elementB.attrib['vehicle']

                                arrivalInEpochTime = int(float(elementB.attrib['epochTime']) / 1000)
                                tripTag = elementB.attrib['tripTag']

                                try:
                                    prediction = Prediction(
                                        bus=line.buses[vehicle],
                                        tripID=tripTag, arrivalTime=arrivalInEpochTime
                                    )

                                    output[stopTag].append(prediction)

                                except:
                                    # print "Could not add prediction for bus %s" % vehicle

                            except (KeyError, AttributeError) as e:
                                print "Could not get attribute: %s" % e
                                continue

                except KeyError:
                    continue

        except IOError:
            print("Could not open " + self.sUrlNextbus + self.sCommandMultiplePredictions + self.sAgency + sStops)

        return output

    # Get route configuration data
    #
    #   inputs
    #       nRouteNumber: the line name that NextBus uses to identify the desired route
    #
    #   outputs
    #       returns a Transit::Line object
    def _getLineConfiguration(self, nLine):

        # if the data directory does not exist, create it
        if not os.path.exists(self.sDirectory):
            os.makedirs(self.sDirectory)

        sFilename = ('route_' + str(nLine) + '_directionsTable.txt')
        sFilename = os.path.join(self.sDirectory, sFilename)

        try:
            bFileExists = os.path.isfile(sFilename)
            bUpdatedToday = (datetime.date.today() == datetime.date.fromtimestamp(os.path.getmtime(sFilename)))

        except OSError:
            bFileExists = False
            bUpdatedToday = False

        # configuration files are only updated once a day
        if True or bFileExists is False or bUpdatedToday is False:

            output = Line(id=nLine)

            sRoute = '&r=' + str(nLine)
            try:
                urlHandle = urllib.urlopen(self.sUrlNextbus + self.sCommandGetStops
                                           + self.sAgency + sRoute + self.sFlags)
                xml = urlHandle.read()
                urlHandle.close()

            except urllib.error.URLError as e:
                print "Could not load configuration: %s" % e
                urlHandle.close()
                return

            lStops = {}

            root = etree.fromstring(xml)
            for elementA in root:
                if elementA.tag == 'route':
                    for elementB in elementA:

                        if elementB.tag == 'stop':
                            stopID = elementB.attrib['tag']

                            lStops[stopID] = Stop(
                                id=elementB.attrib['tag'],
                                name=elementB.attrib['title'],
                                latitude=elementB.attrib['lat'],
                                longitude=elementB.attrib['lon']
                            )

                        if elementB.tag == 'direction':
                            sBusDirection = elementB.attrib['tag']
                            direction = Direction(line=nLine, id=sBusDirection, title=elementB.attrib['title'])

                            for elementC in elementB:
                                direction.addStop(lStops[elementC.attrib['tag']])

                            output.addDirection(direction)

            # Write out direction "variables" table to a file
            fhDirections = open(sFilename, 'wb')
            pickle.dump(output, fhDirections)
            fhDirections.close()

            return output

        # route information is cached, so just restore it
        else:
            fhDirections = open(sFilename, 'rb')
            output = pickle.load(fhDirections)
            fhDirections.close()
            return output

    # poll NextBus for vehicle locations
    #
    # inputs
    #   nLine: the line name that NextBus uses to identify the desired route
    #
    # outputs
    #   return an hash of Bus objects, keyed by their IDs
    def _pollNextBusLocations(self, nLine):

        # Set epochTime to zero so that NextBus gives the last 15 minutes worth of updates
        # (the 15 minute window is a NextBus default parameter when nHistory = 0...)
        nHistory = 0

        # Get vehicle locations
        try:
            urlHandle = urllib.urlopen(self.sUrlNextbus + self.sCommandVehicleLocations + self.sAgency
                                   + self.sRoute + str(nLine)
                                   + self.sTime + str(nHistory))

            xml = urlHandle.read()
            urlHandle.close()

        except urllib.error.URLError, e:
            print e.code
            urlHandle.close()
            return

        output = {}

        root = etree.fromstring(xml)
        for element in root.findall('vehicle'):

            try:
                output[element.attrib['id']] = Bus(
                    id=element.attrib['id'],
                    line=element.attrib['routeTag'],
                    direction=element.attrib['dirTag'],
                    latitude=element.attrib['lat'],
                    longitude=element.attrib['lon'],
                    secondsSinceLastUpdate=element.attrib['secsSinceReport'],
                    heading=element.attrib['heading']
                )

            except KeyError as e:
                continue

        return output

    # pollNextBus
    # this function polls NextBus for stop predictions for a specific route and direction
    #
    # inputs
    #    nLine: the bus line number to query NextBus for
    def poll(self, nLine):

        # get bus (nLine)'s directions
        self.lines[nLine] = self._getLineConfiguration(nLine)

        # get (nLine)'s vehicle locations
        self.lines[nLine].buses = self._pollNextBusLocations(nLine)

        # update predictions
        for sDirection in self.lines[nLine].getAvailableDirections():

            # clear any existing predictions
            for stop in self.lines[nLine].directions[sDirection].stops.itervalues():
                stop.clearPredictions()

            predictions = self._getPredictions(nLine, sDirection)
            for stopID, data in predictions.iteritems():
                self.lines[nLine].directions[sDirection].stops[stopID].predictions = data<commit_msg>Do nothing on error<commit_after># Poll NextBus for real-time bus locations and predictions

import datetime
import os
import pickle
import urllib
from lxml import etree

try:
    import MySQLdb
except ImportError:
    import pymysql as MySQLdb

from TransitAgency import TransitAgency
from Bus import Bus
from Direction import Direction
from Line import Line
from Prediction import Prediction
from Stop import Stop

class NextBus (TransitAgency):

    # default local data directory
    sDirectory = None

    # HTTP interface for NextBus
    sUrlNextbus = 'http://webservices.nextbus.com/service/publicXMLFeed?'
    sAgency = '&a=mbta'
    sRoute = '&r='
    sStopId = '&stopId='
    sStop = '&stops='
    sFlags = '&terse'
    sTime = '&t='
    sCommandGetStops = 'command=routeConfig'
    sCommandPredictions = 'command=predictions'
    sCommandMultiplePredictions = 'command=predictionsForMultiStops'
    sCommandVehicleLocations = 'command=vehicleLocations'

    # initialization
    def __init__(self, sDirectory=None, sAgency=None):

        # call initializers for super classes
        TransitAgency.__init__(self)

        if sDirectory:
            self.sDirectory = sDirectory
        else:
            self.sDirectory = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/')

        if sAgency:
            self.sAgency = str('&a=' + sAgency)

    # update predictions for a particular line/direction
    #
    # inputs
    #   nRouteNumber: the line name that NextBus uses to identify the desired route
    #   sRouteDirection: the route direction to return predictions for
    #
    # outputs
    #   returns an array of predictions for each direction
    def _getPredictions(self, nLine, sDirection):

        line = self.lines[nLine]
        dStops = line.getStopsFor(sDirection)

        # create URL string for multiple stops
        sStops = ''

        # iterate through a list of stop numbers to create the NextBus query
        # and clear any existing predictions
        for stopID, data in dStops.iteritems():
            sStops = sStops + '&stops=' + nLine + '|' + data.id

        output = {}

        try:
            fhPredictions = urllib.urlopen(self.sUrlNextbus
                                           + self.sCommandMultiplePredictions
                                           + self.sAgency
                                           + sStops)

            xml = fhPredictions.read()
            fhPredictions.close()

            root = etree.fromstring(xml)

            # process stop predictions
            for element in root.findall('predictions'):

                try:
                    stopTag = element.attrib['stopTag']
                    # routeTag = element.attrib['routeTag']

                    output[stopTag] = []

                    for elementA in element.findall('direction'):
                        for elementB in elementA.findall('prediction'):

                            try:
                                # direction = elementB.attrib['dirTag']
                                vehicle = elementB.attrib['vehicle']

                                arrivalInEpochTime = int(float(elementB.attrib['epochTime']) / 1000)
                                tripTag = elementB.attrib['tripTag']

                                try:
                                    prediction = Prediction(
                                        bus=line.buses[vehicle],
                                        tripID=tripTag, arrivalTime=arrivalInEpochTime
                                    )

                                    output[stopTag].append(prediction)

                                except:
                                    # print "Could not add prediction for bus %s" % vehicle
                                    pass

                            except (KeyError, AttributeError) as e:
                                print "Could not get attribute: %s" % e
                                continue

                except KeyError:
                    continue

        except IOError:
            print("Could not open " + self.sUrlNextbus + self.sCommandMultiplePredictions + self.sAgency + sStops)

        return output

    # Get route configuration data
    #
    #   inputs
    #       nRouteNumber: the line name that NextBus uses to identify the desired route
    #
    #   outputs
    #       returns a Transit::Line object
    def _getLineConfiguration(self, nLine):

        # if the data directory does not exist, create it
        if not os.path.exists(self.sDirectory):
            os.makedirs(self.sDirectory)

        sFilename = ('route_' + str(nLine) + '_directionsTable.txt')
        sFilename = os.path.join(self.sDirectory, sFilename)

        try:
            bFileExists = os.path.isfile(sFilename)
            bUpdatedToday = (datetime.date.today() == datetime.date.fromtimestamp(os.path.getmtime(sFilename)))

        except OSError:
            bFileExists = False
            bUpdatedToday = False

        # configuration files are only updated once a day
        if True or bFileExists is False or bUpdatedToday is False:

            output = Line(id=nLine)

            sRoute = '&r=' + str(nLine)
            try:
                urlHandle = urllib.urlopen(self.sUrlNextbus + self.sCommandGetStops
                                           + self.sAgency + sRoute + self.sFlags)
                xml = urlHandle.read()
                urlHandle.close()

            except urllib.error.URLError as e:
                print "Could not load configuration: %s" % e
                urlHandle.close()
                return

            lStops = {}

            root = etree.fromstring(xml)
            for elementA in root:
                if elementA.tag == 'route':
                    for elementB in elementA:

                        if elementB.tag == 'stop':
                            stopID = elementB.attrib['tag']

                            lStops[stopID] = Stop(
                                id=elementB.attrib['tag'],
                                name=elementB.attrib['title'],
                                latitude=elementB.attrib['lat'],
                                longitude=elementB.attrib['lon']
                            )

                        if elementB.tag == 'direction':
                            sBusDirection = elementB.attrib['tag']
                            direction = Direction(line=nLine, id=sBusDirection, title=elementB.attrib['title'])

                            for elementC in elementB:
                                direction.addStop(lStops[elementC.attrib['tag']])

                            output.addDirection(direction)

            # Write out direction "variables" table to a file
            fhDirections = open(sFilename, 'wb')
            pickle.dump(output, fhDirections)
            fhDirections.close()

            return output

        # route information is cached, so just restore it
        else:
            fhDirections = open(sFilename, 'rb')
            output = pickle.load(fhDirections)
            fhDirections.close()
            return output

    # poll NextBus for vehicle locations
    #
    # inputs
    #   nLine: the line name that NextBus uses to identify the desired route
    #
    # outputs
    #   return an hash of Bus objects, keyed by their IDs
    def _pollNextBusLocations(self, nLine):

        # Set epochTime to zero so that NextBus gives the last 15 minutes worth of updates
        # (the 15 minute window is a NextBus default parameter when nHistory = 0...)
        nHistory = 0

        # Get vehicle locations
        try:
            urlHandle = urllib.urlopen(self.sUrlNextbus + self.sCommandVehicleLocations + self.sAgency
                                   + self.sRoute + str(nLine)
                                   + self.sTime + str(nHistory))

            xml = urlHandle.read()
            urlHandle.close()

        except urllib.error.URLError, e:
            print e.code
            urlHandle.close()
            return

        output = {}

        root = etree.fromstring(xml)
        for element in root.findall('vehicle'):

            try:
                output[element.attrib['id']] = Bus(
                    id=element.attrib['id'],
                    line=element.attrib['routeTag'],
                    direction=element.attrib['dirTag'],
                    latitude=element.attrib['lat'],
                    longitude=element.attrib['lon'],
                    secondsSinceLastUpdate=element.attrib['secsSinceReport'],
                    heading=element.attrib['heading']
                )

            except KeyError as e:
                continue

        return output

    # pollNextBus
    # this function polls NextBus for stop predictions for a specific route and direction
    #
    # inputs
    #    nLine: the bus line number to query NextBus for
    def poll(self, nLine):

        # get bus (nLine)'s directions
        self.lines[nLine] = self._getLineConfiguration(nLine)

        # get (nLine)'s vehicle locations
        self.lines[nLine].buses = self._pollNextBusLocations(nLine)

        # update predictions
        for sDirection in self.lines[nLine].getAvailableDirections():

            # clear any existing predictions
            for stop in self.lines[nLine].directions[sDirection].stops.itervalues():
                stop.clearPredictions()

            predictions = self._getPredictions(nLine, sDirection)
            for stopID, data in predictions.iteritems():
                self.lines[nLine].directions[sDirection].stops[stopID].predictions = data<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""

ZUGBRUECKE
Calling routines in Windows DLLs from Python scripts running on unixlike systems
https://github.com/pleiszenburg/zugbruecke

    src/zugbruecke/core/dll_client.py: Classes for managing the access to DLLs

    Required to run on platform / side: [UNIX]

    Copyright (C) 2017-2021 Sebastian M. Ernst <ernst@pleiszenburg.de>

<LICENSE_BLOCK>
The contents of this file are subject to the GNU Lesser General Public License
Version 2.1 ("LGPL" or "License"). You may not use this file except in
compliance with the License. You may obtain a copy of the License at
https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt
https://github.com/pleiszenburg/zugbruecke/blob/master/LICENSE

Software distributed under the License is distributed on an "AS IS" basis,
WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the
specific language governing rights and limitations under the License.
</LICENSE_BLOCK>

"""

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# IMPORT
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

from typing import Union

from .abc import DataABC, DllClientABC, LogABC, RoutineClientABC, RpcClientABC
from .routine_client import RoutineClient
from .typeguard import typechecked

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# DLL CLIENT CLASS
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

@typechecked
class DllClient(DllClientABC):
    """
    Representing one idividual dll to be called into, returned by LoadLibrary
    """

    def __init__(
        self,
        name: str,
        hash_id: str,
        convention: str,
        log: LogABC,
        rpc_client: RpcClientABC,
        data: DataABC,
    ):

        self._name = name
        self._hash_id = hash_id
        self._convention = convention

        self._log = log
        self._rpc_client = rpc_client
        self._data = data

        self._routines = {}

        for name in (
            "get_repr",
            "register_routine",
        ):
            setattr(
                self,
                "_{NAME:s}_on_server".format(NAME=name),
                getattr(
                    self._rpc_client,
                    "{HASH_ID:s}_{NAME:s}".format(HASH_ID=self._hash_id, NAME=name),
                ),
            )

    def __getattr__(self, name: str) -> RoutineClientABC:

        if name in ["__objclass__", "__name__"]:
            raise AttributeError(name)

        if name not in self._routines.keys():
            self._register_routine(name)

        return self._routines[name]

    def __getitem__(self, name_or_ordinal: Union[str, int]) -> RoutineClientABC:

        if name_or_ordinal not in self._routines.keys():
            self._register_routine(name_or_ordinal)

        return self._routines[name_or_ordinal]

    def __repr__(self) -> str:

        return self._get_repr_on_server()

    def _register_routine(self, name: Union[str, int]):

        self._log.out(
            '[dll-client] Trying to attach to register routine "{NAME:s}" in DLL file "{FN:s}" ...'.format(
                NAME=str(name),
                FN=self._name,
            )
        )

        try:
            self._register_routine_on_server(name)
        except AttributeError as e:
            self._log.out("[dll-client] ... failed!")
            raise e

        self._routines[name] = RoutineClient(
            name,
            self._hash_id,
            self._convention,
            self._name,
            self._log,
            self._rpc_client,
            self._data,
        )

        self._log.out("[dll-client] ... registered (unconfigured).")
<commit_msg>grammar<commit_after># -*- coding: utf-8 -*-

"""

ZUGBRUECKE
Calling routines in Windows DLLs from Python scripts running on unixlike systems
https://github.com/pleiszenburg/zugbruecke

    src/zugbruecke/core/dll_client.py: Classes for managing the access to DLLs

    Required to run on platform / side: [UNIX]

    Copyright (C) 2017-2021 Sebastian M. Ernst <ernst@pleiszenburg.de>

<LICENSE_BLOCK>
The contents of this file are subject to the GNU Lesser General Public License
Version 2.1 ("LGPL" or "License"). You may not use this file except in
compliance with the License. You may obtain a copy of the License at
https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt
https://github.com/pleiszenburg/zugbruecke/blob/master/LICENSE

Software distributed under the License is distributed on an "AS IS" basis,
WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the
specific language governing rights and limitations under the License.
</LICENSE_BLOCK>

"""

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# IMPORT
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

from typing import Union

from .abc import DataABC, DllClientABC, LogABC, RoutineClientABC, RpcClientABC
from .routine_client import RoutineClient
from .typeguard import typechecked

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# DLL CLIENT CLASS
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

@typechecked
class DllClient(DllClientABC):
    """
    Representing one idividual dll to be called into, returned by LoadLibrary
    """

    def __init__(
        self,
        name: str,
        hash_id: str,
        convention: str,
        log: LogABC,
        rpc_client: RpcClientABC,
        data: DataABC,
    ):

        self._name = name
        self._hash_id = hash_id
        self._convention = convention

        self._log = log
        self._rpc_client = rpc_client
        self._data = data

        self._routines = {}

        for name in (
            "get_repr",
            "register_routine",
        ):
            setattr(
                self,
                "_{NAME:s}_on_server".format(NAME=name),
                getattr(
                    self._rpc_client,
                    "{HASH_ID:s}_{NAME:s}".format(HASH_ID=self._hash_id, NAME=name),
                ),
            )

    def __getattr__(self, name: str) -> RoutineClientABC:

        if name in ["__objclass__", "__name__"]:
            raise AttributeError(name)

        if name not in self._routines.keys():
            self._register_routine(name)

        return self._routines[name]

    def __getitem__(self, name_or_ordinal: Union[str, int]) -> RoutineClientABC:

        if name_or_ordinal not in self._routines.keys():
            self._register_routine(name_or_ordinal)

        return self._routines[name_or_ordinal]

    def __repr__(self) -> str:

        return self._get_repr_on_server()

    def _register_routine(self, name: Union[str, int]):

        self._log.out(
            '[dll-client] Trying to register routine "{NAME:s}" in DLL file "{FN:s}" ...'.format(
                NAME=str(name),
                FN=self._name,
            )
        )

        try:
            self._register_routine_on_server(name)
        except AttributeError as e:
            self._log.out("[dll-client] ... failed!")
            raise e

        self._routines[name] = RoutineClient(
            name,
            self._hash_id,
            self._convention,
            self._name,
            self._log,
            self._rpc_client,
            self._data,
        )

        self._log.out("[dll-client] ... registered (unconfigured).")
<|endoftext|>