<commit_before>from epyper.enum import enum
from epyper import displayHardwareDriver as epd

#-------------------------------------------------------------------------------

class COG_Parameters():
    def __init__(self, channelSelect, voltageLevel, horizontal, vertical,
                                          dataLineSize, frameTime, stageTime):
        self.channelSelect = channelSelect
        self.voltageLevel = voltageLevel
        self.horizontal = horizontal
        self.vertical = vertical
        self.dataLineSize = dataLineSize
        self.frameTime = frameTime
        self.stageTime = stageTime

#-------------------------------------------------------------------------------

class Display():

    BLACK0 = 0x03
    BLACK1 = 0x0C
    BLACK2 = 0x30
    BLACK3 = 0xc0
    WHITE0 = 0x02
    WHITE1 = 0x08
    WHITE2 = 0x20
    WHITE3 = 0x80
    NOTHING = 0x00

    PATTERNS = [
        [WHITE3, WHITE2, WHITE1, WHITE0],
        [NOTHING, NOTHING, NOTHING, NOTHING],
        [BLACK3, BLACK2, BLACK1, BLACK0]
    ]

    TESTS = [
        [0x80, 0x20, 0x08, 0x02],
        [0x01, 0x04, 0x10, 0x40]
    ]

    EPD_TYPE_144 = 0
    EPD_TYPE_200 = 1
    EPD_TYPE_270 = 2

    scanTable = [0xC0, 0x30, 0x0C, 0x03]

#-------------------------------------------------------------------------------

    def __init__(self, typeIndex):

        self.EPD_Type_Index = typeIndex

        if typeIndex == self.EPD_TYPE_144:
            self.even = [self.NOTHING] * 16
            self.scan = [self.NOTHING] * 24
            self.odd = [self.NOTHING] * 16

            self.COG_Params = COG_Parameters( #for 1.44"
                    [0x00,0x00,0x00,0x00,0x00,0x0f,0xff,0x00],
                    0x03,
                    128/8,
                    96,
                    ((128+96)*2)/8,
                    43,
                    480
                )

        elif typeIndex == self.EPD_TYPE_200:
            self.even = [self.NOTHING] * 25
            self.scan = [self.NOTHING] * 24
            self.odd = [self.NOTHING] * 25

            self.COG_Params = COG_Parameters( #for 2.0"
                    [0x00,0x00,0x00,0x00,0x01,0xFF,0xE0,0x00],
                    0x03,
                    200/8,
                    96,
                    (((200+96)*2)/8)+1,
                    53,
                    480
                )

        elif typeIndex == self.EPD_TYPE_270:
            self.even = [self.NOTHING] * 33
            self.scan = [self.NOTHING] * 44
            self.odd = [self.NOTHING] * 33

            self.COG_Params = COG_Parameters( #for 2.7"
                    [0x00,0x00,0x00,0x7F,0xFF,0xFE,0x00,0x00],
                    0x00,
                    264/8,
                    176,
                    (((264+176)*2)/8)+1,
                    105,
                    630
                )

#-------------------------------------------------------------------------------
# Local functions
#-------------------------------------------------------------------------------

    def setTemperature_Factor(self):
        temperature = epd.get_temperature()

        print "temperature = %s" % temperature

        if temperature < -10:
            self.stageTime = self.COG_Params.stageTime * 17
        elif temperature < -5:
            self.stageTime = self.COG_Params.stageTime * 12
        elif temperature < 5:
            self.stageTime = self.COG_Params.stageTime * 8
        elif temperature < 10:
            self.stageTime = self.COG_Params.stageTime * 4
        elif temperature < 15:
            self.stageTime = self.COG_Params.stageTime * 3
        elif temperature < 20:
            self.stageTime = self.COG_Params.stageTime * 2
        elif temperature < 40:
            self.stageTime = self.COG_Params.stageTime * 1
        else:
            self.stageTime = (self.COG_Params.stageTime * 7)/10
        print "stageTime = %s" % self.stageTime

#-------------------------------------------------------------------------------

    def sendData(self, data):
        voltageLevel = self.COG_Params.voltageLevel
        frameTime = self.COG_Params.frameTime
        currentframe = int(self.COG_Params.frameTime)
        startClock = epd.getCurrentTimeTick()

        while True:
            for line in data:
                epd.spi_send_byte(0x04, voltageLevel)
                epd.spi_send(0x0a, line)
                epd.spi_send_byte(0x02, 0x2F)
            currentframe = epd.getCurrentTimeTick() - startClock + frameTime
            if stageTime > currentframe:
                break

        while stageTime > (epd.getCurrentTimeTick() - startClock):
            continue

#-------------------------------------------------------------------------------

    def displayImg(self, previousPicture, stage):
        dataBuffer = []
        idx = 0
        pre = self.PATTERNS[stage-1]
        post = self.PATTERNS[(stage+1) % len(self.PATTERNS)]
        for i in range(self.COG_Params.vertical): # for every line
            k = self.COG_Params.horizontal - 1
            for j in range (self.COG_Params.horizontal): # check every bit in the line
                tempByte = previousPicture[idx]
                idx += 1
                self.odd[j] = 0
                self.even[k] = 0
                for s in range(len(pre)):
                    self.odd[j] |= pre[s] if (tempByte & self.TESTS[0][s]) else post[s]
                    self.even[k] |= pre[s] if (tempByte & self.TESTS[1][s]) else post[s]
                k -= 1

            self.scan[i>>2] = self.scanTable[i%4]
            data = self.even + self.scan + self.odd
            data += [self.NOTHING] * (self.COG_Params.dataLineSize - len(data))
            dataBuffer.append(str(bytearray(data)))
            self.scan[i>>2] = self.NOTHING
        return dataBuffer

#-------------------------------------------------------------------------------
# Public functions
#-------------------------------------------------------------------------------

    def hwInit(self):
        epd.initDisplayHardware()

#-------------------------------------------------------------------------------

    def powerOn(self):

        epd.discharge_low()
        epd.rst_low()
        epd.cs_low()
        epd.spi_init()
        epd.pwm_active(5)
        epd.panelon_on()
        epd.pwm_active(10)
        epd.cs_high()
        epd.rst_high()
        epd.pwm_active(5)
        epd.rst_low()
        epd.pwm_active(5)
        epd.rst_high()
        epd.pwm_active(5)

#-------------------------------------------------------------------------------

    def initializeDriver(self):
        sendBuffer = [self.NOTHING] * 2
        self.setTemperature_Factor()

        epd.spi_send(0x01, str(bytearray(self.COG_Params.channelSelect)))

        epd.spi_send_byte(0x06, 0xff)
        epd.spi_send_byte(0x07, 0x9d)
        epd.spi_send_byte(0x08, 0x00)

        sendBuffer[0] = 0xd0
        sendBuffer[1] = 0x00

        epd.spi_send(0x09, str(bytearray(sendBuffer)))
        epd.spi_send_byte(0x04, self.COG_Params.voltageLevel)
        epd.spi_send_byte(0x03, 0x01)
        epd.spi_send_byte(0x03, 0x00)
        epd.pwm_active(5)
        epd.spi_send_byte(0x05, 0x01)
        epd.pwm_active(30)
        epd.spi_send_byte(0x05, 0x03)
        epd.delay_ms(30)
        epd.spi_send_byte(0x05, 0x0f)
        epd.delay_ms(30)
        epd.spi_send_byte(0x02, 0x24)

#-------------------------------------------------------------------------------

    def display(self, newImg, prevImg):
        data = []
        data.append(self.displayImg(prevImg, 1))
        data.append(self.displayImg(prevImg, 2))
        data.append(self.displayImg(newImg, 3))
        data.append(self.displayImg(newImg, 1))
        data.append(self.displayImg(newImg, 1))
        data.append(self.displayImg(newImg, 1))
        data.append(self.displayImg(newImg, 1))
        for chunk in data:
            self.sendData(chunk)

#-------------------------------------------------------------------------------

    def powerOff(self):

        epd.spi_send_byte(0x03, 0x01)
        epd.spi_send_byte(0x02, 0x05)
        epd.spi_send_byte(0x05, 0x0E)
        epd.spi_send_byte(0x05, 0x02)
        epd.spi_send_byte(0x04, 0x0C)
        epd.delay_ms(120)
        epd.spi_send_byte(0x05, 0x00)
        epd.spi_send_byte(0x07, 0x0D)
        epd.spi_send_byte(0x04, 0x50)
        epd.delay_ms(40)
        epd.spi_send_byte(0x04, 0xA0)
        epd.delay_ms(40)
        epd.spi_send_byte(0x04, 0x00)

        epd.rst_low()
        epd.cs_low()
        epd.spi_detach()
        epd.panelon_off()

        epd.discharge_high()
        epd.delay_ms(150)
        epd.discharge_low()

#-------------------------------------------------------------------------------
<commit_msg>bugfix<commit_after>from epyper.enum import enum
from epyper import displayHardwareDriver as epd

#-------------------------------------------------------------------------------

class COG_Parameters():
    def __init__(self, channelSelect, voltageLevel, horizontal, vertical,
                                          dataLineSize, frameTime, stageTime):
        self.channelSelect = channelSelect
        self.voltageLevel = voltageLevel
        self.horizontal = horizontal
        self.vertical = vertical
        self.dataLineSize = dataLineSize
        self.frameTime = frameTime
        self.stageTime = stageTime

#-------------------------------------------------------------------------------

class Display():

    BLACK0 = 0x03
    BLACK1 = 0x0C
    BLACK2 = 0x30
    BLACK3 = 0xc0
    WHITE0 = 0x02
    WHITE1 = 0x08
    WHITE2 = 0x20
    WHITE3 = 0x80
    NOTHING = 0x00

    PATTERNS = [
        [WHITE3, WHITE2, WHITE1, WHITE0],
        [NOTHING, NOTHING, NOTHING, NOTHING],
        [BLACK3, BLACK2, BLACK1, BLACK0]
    ]

    TESTS = [
        [0x80, 0x20, 0x08, 0x02],
        [0x01, 0x04, 0x10, 0x40]
    ]

    EPD_TYPE_144 = 0
    EPD_TYPE_200 = 1
    EPD_TYPE_270 = 2

    scanTable = [0xC0, 0x30, 0x0C, 0x03]

#-------------------------------------------------------------------------------

    def __init__(self, typeIndex):

        self.EPD_Type_Index = typeIndex

        if typeIndex == self.EPD_TYPE_144:
            self.even = [self.NOTHING] * 16
            self.scan = [self.NOTHING] * 24
            self.odd = [self.NOTHING] * 16

            self.COG_Params = COG_Parameters( #for 1.44"
                    [0x00,0x00,0x00,0x00,0x00,0x0f,0xff,0x00],
                    0x03,
                    128/8,
                    96,
                    ((128+96)*2)/8,
                    43,
                    480
                )

        elif typeIndex == self.EPD_TYPE_200:
            self.even = [self.NOTHING] * 25
            self.scan = [self.NOTHING] * 24
            self.odd = [self.NOTHING] * 25

            self.COG_Params = COG_Parameters( #for 2.0"
                    [0x00,0x00,0x00,0x00,0x01,0xFF,0xE0,0x00],
                    0x03,
                    200/8,
                    96,
                    (((200+96)*2)/8)+1,
                    53,
                    480
                )

        elif typeIndex == self.EPD_TYPE_270:
            self.even = [self.NOTHING] * 33
            self.scan = [self.NOTHING] * 44
            self.odd = [self.NOTHING] * 33

            self.COG_Params = COG_Parameters( #for 2.7"
                    [0x00,0x00,0x00,0x7F,0xFF,0xFE,0x00,0x00],
                    0x00,
                    264/8,
                    176,
                    (((264+176)*2)/8)+1,
                    105,
                    630
                )

#-------------------------------------------------------------------------------
# Local functions
#-------------------------------------------------------------------------------

    def setTemperature_Factor(self):
        temperature = epd.get_temperature()

        print "temperature = %s" % temperature

        if temperature < -10:
            self.stageTime = self.COG_Params.stageTime * 17
        elif temperature < -5:
            self.stageTime = self.COG_Params.stageTime * 12
        elif temperature < 5:
            self.stageTime = self.COG_Params.stageTime * 8
        elif temperature < 10:
            self.stageTime = self.COG_Params.stageTime * 4
        elif temperature < 15:
            self.stageTime = self.COG_Params.stageTime * 3
        elif temperature < 20:
            self.stageTime = self.COG_Params.stageTime * 2
        elif temperature < 40:
            self.stageTime = self.COG_Params.stageTime * 1
        else:
            self.stageTime = (self.COG_Params.stageTime * 7)/10
        print "stageTime = %s" % self.stageTime

#-------------------------------------------------------------------------------

    def sendData(self, data):
        voltageLevel = self.COG_Params.voltageLevel
        frameTime = self.COG_Params.frameTime
        currentframe = int(self.COG_Params.frameTime)
        startClock = epd.getCurrentTimeTick()

        while True:
            for line in data:
                epd.spi_send_byte(0x04, voltageLevel)
                epd.spi_send(0x0a, line)
                epd.spi_send_byte(0x02, 0x2F)
            currentframe = epd.getCurrentTimeTick() - startClock + frameTime
            if self.stageTime > currentframe:
                break

        while self.stageTime > (epd.getCurrentTimeTick() - startClock):
            continue

#-------------------------------------------------------------------------------

    def displayImg(self, previousPicture, stage):
        dataBuffer = []
        idx = 0
        pre = self.PATTERNS[stage-1]
        post = self.PATTERNS[(stage+1) % len(self.PATTERNS)]
        for i in range(self.COG_Params.vertical): # for every line
            k = self.COG_Params.horizontal - 1
            for j in range (self.COG_Params.horizontal): # check every bit in the line
                tempByte = previousPicture[idx]
                idx += 1
                self.odd[j] = 0
                self.even[k] = 0
                for s in range(len(pre)):
                    self.odd[j] |= pre[s] if (tempByte & self.TESTS[0][s]) else post[s]
                    self.even[k] |= pre[s] if (tempByte & self.TESTS[1][s]) else post[s]
                k -= 1

            self.scan[i>>2] = self.scanTable[i%4]
            data = self.even + self.scan + self.odd
            data += [self.NOTHING] * (self.COG_Params.dataLineSize - len(data))
            dataBuffer.append(str(bytearray(data)))
            self.scan[i>>2] = self.NOTHING
        return dataBuffer

#-------------------------------------------------------------------------------
# Public functions
#-------------------------------------------------------------------------------

    def hwInit(self):
        epd.initDisplayHardware()

#-------------------------------------------------------------------------------

    def powerOn(self):

        epd.discharge_low()
        epd.rst_low()
        epd.cs_low()
        epd.spi_init()
        epd.pwm_active(5)
        epd.panelon_on()
        epd.pwm_active(10)
        epd.cs_high()
        epd.rst_high()
        epd.pwm_active(5)
        epd.rst_low()
        epd.pwm_active(5)
        epd.rst_high()
        epd.pwm_active(5)

#-------------------------------------------------------------------------------

    def initializeDriver(self):
        sendBuffer = [self.NOTHING] * 2
        self.setTemperature_Factor()

        epd.spi_send(0x01, str(bytearray(self.COG_Params.channelSelect)))

        epd.spi_send_byte(0x06, 0xff)
        epd.spi_send_byte(0x07, 0x9d)
        epd.spi_send_byte(0x08, 0x00)

        sendBuffer[0] = 0xd0
        sendBuffer[1] = 0x00

        epd.spi_send(0x09, str(bytearray(sendBuffer)))
        epd.spi_send_byte(0x04, self.COG_Params.voltageLevel)
        epd.spi_send_byte(0x03, 0x01)
        epd.spi_send_byte(0x03, 0x00)
        epd.pwm_active(5)
        epd.spi_send_byte(0x05, 0x01)
        epd.pwm_active(30)
        epd.spi_send_byte(0x05, 0x03)
        epd.delay_ms(30)
        epd.spi_send_byte(0x05, 0x0f)
        epd.delay_ms(30)
        epd.spi_send_byte(0x02, 0x24)

#-------------------------------------------------------------------------------

    def display(self, newImg, prevImg):
        data = []
        data.append(self.displayImg(prevImg, 1))
        data.append(self.displayImg(prevImg, 2))
        data.append(self.displayImg(newImg, 3))
        data.append(self.displayImg(newImg, 1))
        data.append(self.displayImg(newImg, 1))
        data.append(self.displayImg(newImg, 1))
        data.append(self.displayImg(newImg, 1))
        for chunk in data:
            self.sendData(chunk)

#-------------------------------------------------------------------------------

    def powerOff(self):

        epd.spi_send_byte(0x03, 0x01)
        epd.spi_send_byte(0x02, 0x05)
        epd.spi_send_byte(0x05, 0x0E)
        epd.spi_send_byte(0x05, 0x02)
        epd.spi_send_byte(0x04, 0x0C)
        epd.delay_ms(120)
        epd.spi_send_byte(0x05, 0x00)
        epd.spi_send_byte(0x07, 0x0D)
        epd.spi_send_byte(0x04, 0x50)
        epd.delay_ms(40)
        epd.spi_send_byte(0x04, 0xA0)
        epd.delay_ms(40)
        epd.spi_send_byte(0x04, 0x00)

        epd.rst_low()
        epd.cs_low()
        epd.spi_detach()
        epd.panelon_off()

        epd.discharge_high()
        epd.delay_ms(150)
        epd.discharge_low()

#-------------------------------------------------------------------------------
<|endoftext|>


<commit_before>#!/bin/sh
# edit on 06122017

echo "Removing any old CB installers, in case they exist"
rm -f cloudbolt_installer_*.tgz

echo "Opening up incoming traffic on TCP ports 80 & 443"
iptables -I INPUT 2 -m state --state NEW -m tcp -p tcp --dport 80 -j ACCEPT
iptables -I INPUT 2 -m state --state NEW -m tcp -p tcp --dport 443 -j ACCEPT
service iptables save
service iptables restart

echo "Downloading CloudBolt"
wget --no-verbose http://downloads.cloudbolt.io/cloudbolt-installer-latest.tgz

echo "Extracting CloudBolt Archive"
tar xfz cloudbolt_installer_*.tgz
cd cloudbolt_installer*

echo "Installing CloudBolt"
./install_cloudbolt.sh force
<commit_msg>Create download_and_install_cloudbolt.py<commit_after>#!/bin/sh

echo "Removing any old CB installers, in case they exist"
rm -f cloudbolt_installer_*.tgz

echo "Opening up incoming traffic on TCP ports 80 & 443"
iptables -I INPUT 2 -m state --state NEW -m tcp -p tcp --dport 80 -j ACCEPT
iptables -I INPUT 2 -m state --state NEW -m tcp -p tcp --dport 443 -j ACCEPT
service iptables save
service iptables restart

echo "Downloading CloudBolt"
wget --no-verbose http://downloads.cloudbolt.io/cloudbolt-installer-latest.tgz

echo "Extracting CloudBolt Archive"
tar xfz cloudbolt_installer_*.tgz
cd cloudbolt_installer*

echo "Installing CloudBolt"
./install_cloudbolt.sh force
<|endoftext|>


<commit_before>import sys
from ibex.geometry import ib3shapes
from ibex.utilities.constants import *

class MetaData:
    def __init__(self, prefix):
        # initialize the prefix variable
        self.prefix = prefix

        # backwards compatability variable defaults
        self.affinity_filename = None
        self.boundary_filename = None
        self.gold_filename = None
        self.image_filename = None
        self.mask_filename = None
        self.rhoana_filename = None
        self.synapse = None

        # open the meta data txt file
        filename = 'meta_data/{}.meta'.format(prefix)
        with open(filename, 'r') as fd:
            lines = fd.readlines()

            for ix in range(0, len(lines), 2):
                # get the comment and the corresponding value
                comment = lines[ix].strip()
                value = lines[ix + 1].strip()

                if comment == '# resolution in nm':
                    # separate into individual dimensions
                    samples = value.split('x')
                    self.resolution = (int(samples[2]), int(samples[1]), int(samples[0]))
                elif comment == '# affinity filename':
                    self.affinity_filename = value
                elif comment == '# boundary filename':
                    self.boundary_filename = value
                elif comment == '# gold filename':
                    self.gold_filename = value
                elif comment == '# image filename':
                    self.image_filename = value
                elif comment == '# mask filename':
                    self.mask_filename = value
                elif comment == '# rhoana filename':
                    self.rhoana_filename = value
                elif comment == '# synapse filename':
                    self.synapse_filename = value
                elif comment == '# world bounding box':
                    if value == 'None':
                        self.bounding_box = ib3shapes.IBBox((0, 0, 0), (-1, -1, -1))
                        continue
                    # separate into minimums and maximums
                    minmax = value.split('-')
                    mins = (minmax[0].strip('()')).split(',')
                    maxs = (minmax[1].strip('()')).split(',')

                    # split into individual mins
                    for iv in range(NDIMS):
                        mins[iv] = int(mins[iv])
                        maxs[iv] = int(maxs[iv])

                    # swap x and z
                    tmp = mins[0]
                    mins[0] = mins[2]
                    mins[2] = tmp

                    tmp = maxs[0]
                    maxs[0] = maxs[2]
                    maxs[2] = tmp

                    # save the bounding box
                    self.bounding_box = ib3shapes.IBBox(mins, maxs)
                else:
                    sys.stderr.write('Unrecognized attribute in {}: {}\n'.format(prefix, comment))
                    sys.exit()

    def WorldBBox(self):
        return self.bounding_box

    def Resolution(self):
        return self.resolution

    def GoldFilename(self):
        if self.gold_filename == None:
            return 'gold/{}_gold.h5'.format(self.prefix), 'main'
        else:
            return self.gold_filename.split()[0], self.gold_filename.split()[1]

    def ImageFilename(self):
        if self.image_filename == None:
            return 'images/{}_image.h5'.format(self.prefix), 'main'
        else:
            return self.image_filename.split()[0], self.image_filename.split()[1]

    def SegmentationFilename(self):
        if self.rhoana_filename == None:
            return 'rhoana/{}_rhoana.h5'.format(self.prefix), 'main'
        else:
            return self.rhoana_filename.split()[0], self.rhoana_filename.split()[1]

    def AffinityFilename(self):
        if self.affinity_filename == None:
            return 'affinities/{}_affinities.h5'.format(self.prefix), 'main'
        else:
            return self.affinity_filename.split()[0], self.affinity_filename.split()[1]<commit_msg>meta_data: changed directory structure for meta data<commit_after>import sys
from ibex.geometry import ib3shapes
from ibex.utilities.constants import *

class MetaData:
    def __init__(self, prefix):
        # initialize the prefix variable
        self.prefix = prefix

        # backwards compatability variable defaults
        self.affinity_filename = None
        self.boundary_filename = None
        self.gold_filename = None
        self.image_filename = None
        self.mask_filename = None
        self.rhoana_filename = None
        self.synapse = None

        # open the meta data txt file
        filename = 'meta/{}.meta'.format(prefix)
        with open(filename, 'r') as fd:
            lines = fd.readlines()

            for ix in range(0, len(lines), 2):
                # get the comment and the corresponding value
                comment = lines[ix].strip()
                value = lines[ix + 1].strip()

                if comment == '# resolution in nm':
                    # separate into individual dimensions
                    samples = value.split('x')
                    self.resolution = (int(samples[2]), int(samples[1]), int(samples[0]))
                elif comment == '# affinity filename':
                    self.affinity_filename = value
                elif comment == '# boundary filename':
                    self.boundary_filename = value
                elif comment == '# gold filename':
                    self.gold_filename = value
                elif comment == '# image filename':
                    self.image_filename = value
                elif comment == '# mask filename':
                    self.mask_filename = value
                elif comment == '# rhoana filename':
                    self.rhoana_filename = value
                elif comment == '# synapse filename':
                    self.synapse_filename = value
                elif comment == '# world bounding box':
                    if value == 'None':
                        self.bounding_box = ib3shapes.IBBox((0, 0, 0), (-1, -1, -1))
                        continue
                    # separate into minimums and maximums
                    minmax = value.split('-')
                    mins = (minmax[0].strip('()')).split(',')
                    maxs = (minmax[1].strip('()')).split(',')

                    # split into individual mins
                    for iv in range(NDIMS):
                        mins[iv] = int(mins[iv])
                        maxs[iv] = int(maxs[iv])

                    # swap x and z
                    tmp = mins[0]
                    mins[0] = mins[2]
                    mins[2] = tmp

                    tmp = maxs[0]
                    maxs[0] = maxs[2]
                    maxs[2] = tmp

                    # save the bounding box
                    self.bounding_box = ib3shapes.IBBox(mins, maxs)
                else:
                    sys.stderr.write('Unrecognized attribute in {}: {}\n'.format(prefix, comment))
                    sys.exit()

    def WorldBBox(self):
        return self.bounding_box

    def Resolution(self):
        return self.resolution

    def GoldFilename(self):
        if self.gold_filename == None:
            return 'gold/{}_gold.h5'.format(self.prefix), 'main'
        else:
            return self.gold_filename.split()[0], self.gold_filename.split()[1]

    def ImageFilename(self):
        if self.image_filename == None:
            return 'images/{}_image.h5'.format(self.prefix), 'main'
        else:
            return self.image_filename.split()[0], self.image_filename.split()[1]

    def SegmentationFilename(self):
        if self.rhoana_filename == None:
            return 'rhoana/{}_rhoana.h5'.format(self.prefix), 'main'
        else:
            return self.rhoana_filename.split()[0], self.rhoana_filename.split()[1]

    def AffinityFilename(self):
        if self.affinity_filename == None:
            return 'affinities/{}_affinities.h5'.format(self.prefix), 'main'
        else:
            return self.affinity_filename.split()[0], self.affinity_filename.split()[1]<|endoftext|>


<commit_before># coding: utf-8

"""
LangaugesApi.py
Copyright 2016 SmartBear Software

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
"""

from __future__ import absolute_import

import sys
import os

# python 2 and python 3 compatibility library
from six import iteritems

from ..configuration import Configuration
from ..api_client import ApiClient

class LangaugesApi(object):
    """
    NOTE: This class is auto generated by the swagger code generator program.
    Do not edit the class manually.
    Ref: https://github.com/swagger-api/swagger-codegen
    """

    def __init__(self, api_client=None):
        config = Configuration()
        if api_client:
            self.api_client = api_client
        else:
            if not config.api_client:
                config.api_client = ApiClient()
            self.api_client = config.api_client

    def get_all_languages(self, **kwargs):
        """

        Get all languages

        This method makes a synchronous HTTP request by default. To make an
        asynchronous HTTP request, please define a `callback` function
        to be invoked when receiving the response.
        >>> def callback_function(response):
        >>>     pprint(response)
        >>>
        >>> thread = api.get_all_languages(callback=callback_function)

        :param callback function: The callback function
            for asynchronous request. (optional)
        :param str oauth_consumer_key: The access token required to access the Naakh API
        :return: LanguagesResource
                 If the method is called asynchronously,
                 returns the request thread.
        """

        all_params = ['oauth_consumer_key']
        all_params.append('callback')

        params = locals()
        for key, val in iteritems(params['kwargs']):
            if key not in all_params:
                raise TypeError(
                    "Got an unexpected keyword argument '%s'"
                    " to method get_all_languages" % key
                )
            params[key] = val
        del params['kwargs']

        resource_path = '/langauges/'.replace('{format}', 'json')
        method = 'GET'

        path_params = {}

        query_params = {}
        if 'oauth_consumer_key' in params:
            query_params['oauth_consumer_key'] = params['oauth_consumer_key']

        header_params = {}

        form_params = {}
        files = {}

        body_params = None

        # HTTP header `Accept`
        header_params['Accept'] = self.api_client.\
            select_header_accept(['application/json'])
        if not header_params['Accept']:
            del header_params['Accept']

        # HTTP header `Content-Type`
        header_params['Content-Type'] = self.api_client.\
            select_header_content_type([])

        # Authentication setting
        auth_settings = []

        response = self.api_client.call_api(resource_path, method,
                                            path_params,
                                            query_params,
                                            header_params,
                                            body=body_params,
                                            post_params=form_params,
                                            files=files,
                                            response_type='LanguagesResource',
                                            auth_settings=auth_settings,
                                            callback=params.get('callback'))
        return response
<commit_msg>Corrected typo in the URL path<commit_after># coding: utf-8

"""
LangaugesApi.py
Copyright 2016 SmartBear Software

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
"""

from __future__ import absolute_import

import sys
import os

# python 2 and python 3 compatibility library
from six import iteritems

from ..configuration import Configuration
from ..api_client import ApiClient

class LangaugesApi(object):
    """
    NOTE: This class is auto generated by the swagger code generator program.
    Do not edit the class manually.
    Ref: https://github.com/swagger-api/swagger-codegen
    """

    def __init__(self, api_client=None):
        config = Configuration()
        if api_client:
            self.api_client = api_client
        else:
            if not config.api_client:
                config.api_client = ApiClient()
            self.api_client = config.api_client

    def get_all_languages(self, **kwargs):
        """

        Get all languages

        This method makes a synchronous HTTP request by default. To make an
        asynchronous HTTP request, please define a `callback` function
        to be invoked when receiving the response.
        >>> def callback_function(response):
        >>>     pprint(response)
        >>>
        >>> thread = api.get_all_languages(callback=callback_function)

        :param callback function: The callback function
            for asynchronous request. (optional)
        :param str oauth_consumer_key: The access token required to access the Naakh API
        :return: LanguagesResource
                 If the method is called asynchronously,
                 returns the request thread.
        """

        all_params = ['oauth_consumer_key']
        all_params.append('callback')

        params = locals()
        for key, val in iteritems(params['kwargs']):
            if key not in all_params:
                raise TypeError(
                    "Got an unexpected keyword argument '%s'"
                    " to method get_all_languages" % key
                )
            params[key] = val
        del params['kwargs']

        resource_path = '/languages/'.replace('{format}', 'json')
        method = 'GET'

        path_params = {}

        query_params = {}
        if 'oauth_consumer_key' in params:
            query_params['oauth_consumer_key'] = params['oauth_consumer_key']

        header_params = {}

        form_params = {}
        files = {}

        body_params = None

        # HTTP header `Accept`
        header_params['Accept'] = self.api_client.\
            select_header_accept(['application/json'])
        if not header_params['Accept']:
            del header_params['Accept']

        # HTTP header `Content-Type`
        header_params['Content-Type'] = self.api_client.\
            select_header_content_type([])

        # Authentication setting
        auth_settings = []

        response = self.api_client.call_api(resource_path, method,
                                            path_params,
                                            query_params,
                                            header_params,
                                            body=body_params,
                                            post_params=form_params,
                                            files=files,
                                            response_type='LanguagesResource',
                                            auth_settings=auth_settings,
                                            callback=params.get('callback'))
        return response
<|endoftext|>


<commit_before>import config

BROKER_URL = config.BROKER_URL

CELERY_SEND_EVENTS = True
CELERY_TASK_RESULT_EXPIRES = None

CELERY_RESULT_BACKEND = config.CELERY_RESULT_BACKEND

CELERY_MONGODB_BACKEND_SETTINGS = config.CELERY_MONGODB_BACKEND_SETTINGS

<commit_msg>celeryconfig<commit_after>import config

BROKER_URL = config.BROKER_URL

CELERY_SEND_EVENTS = True
CELERY_TASK_RESULT_EXPIRES = None

CELERY_RESULT_BACKEND = config.CELERY_RESULT_BACKEND

CELERY_MONGODB_BACKEND_SETTINGS = config.CELERY_MONGODB_BACKEND_SETTINGS

<|endoftext|>


<commit_before>"""Testing framework."""

from __future__ import print_function

import functools
import json
import itertools

from double import BaseTestDouble

import helpers

import models

import pytest

def list_dicts_diff(prv, nxt):
    """Compare two lists of dicts."""
    result = []
    for prv_element, nxt_element in itertools.izip_longest(prv, nxt, fillvalue={}):
        intermediate_result = dict_diff(prv_element, nxt_element)
        if intermediate_result:
            result.append(intermediate_result)
    return result

def dict_diff(prv, nxt):
    """Return a dict of keys that differ with another config object."""
    keys = set(prv.keys() + nxt.keys())
    result = {}

    for k in keys:
        if isinstance(prv.get(k), dict):
            if isinstance(nxt.get(k), dict):
                "If both are dicts we do a recursive call."
                diff = dict_diff(prv.get(k), nxt.get(k))
                if diff:
                    result[k] = diff
            else:
                "If only one is a dict they are clearly different"
                result[k] = {'result': prv.get(k), 'expected': nxt.get(k)}
        else:
            "Ellipsis is a wildcard."""
            if prv.get(k) != nxt.get(k) and nxt.get(k) != "...":
                result[k] = {'result': prv.get(k), 'expected': nxt.get(k)}
    return result

def wrap_test_cases(func):
    """Wrap test cases."""
    @functools.wraps(func)
    def wrapper(cls, test_case):
        cls.device.device.current_test = func.__name__
        cls.device.device.current_test_case = test_case

        try:
            result = func(cls)
            not_implemented = False

            if isinstance(cls.device.device, BaseTestDouble):
                if isinstance(result, list):
                    diff = list_dicts_diff(result, cls.device.device.expected_result)
                else:
                    diff = dict_diff(result, cls.device.device.expected_result)
                if diff:
                    print("Resulting JSON object was: {}".format(json.dumps(result)))
                    raise AssertionError("Expected result varies on some keys {}".format(diff))

        except NotImplementedError:
            not_implemented = True

        cls.device.device.current_test = ''
        cls.device.device.current_test_case = ''

        if not_implemented:
            pytest.skip("Method not implemented")
        else:
            return result

    return wrapper

class BaseTestGetters:
    """Base class for testing drivers."""

    @wrap_test_cases
    def test_get_facts(self):
        """Test get_facts method."""
        facts = self.device.get_facts()
        assert helpers.test_model(models.facts, facts)
        return facts

    @wrap_test_cases
    def test_get_interfaces(self):
        """Test get_interfaces."""
        get_interfaces = self.device.get_interfaces()
        result = len(get_interfaces) > 0

        for interface, interface_data in get_interfaces.iteritems():
            result = result and helpers.test_model(models.interface, interface_data)

        assert result
        return get_interfaces

    @wrap_test_cases
    def test_get_lldp_neighbors(self):
        """Test get_lldp_neighbors."""
        get_lldp_neighbors = self.device.get_lldp_neighbors()
        result = len(get_lldp_neighbors) > 0

        for interface, neighbor_list in get_lldp_neighbors.iteritems():
            for neighbor in neighbor_list:
                result = result and helpers.test_model(models.lldp_neighbors, neighbor)

        assert result
        return get_lldp_neighbors

    @wrap_test_cases
    def test_get_interfaces_counters(self):
        """Test get_interfaces_counters."""
        get_interfaces_counters = self.device.get_interfaces_counters()
        result = len(self.device.get_interfaces_counters()) > 0

        for interface, interface_data in get_interfaces_counters.iteritems():
            result = result and helpers.test_model(models.interface_counters, interface_data)

        assert result
        return get_interfaces_counters

    @wrap_test_cases
    def test_get_environment(self):
        """Test get_environment."""
        environment = self.device.get_environment()
        result = len(environment) > 0

        for fan, fan_data in environment['fans'].iteritems():
            result = result and helpers.test_model(models.fan, fan_data)

        for power, power_data in environment['power'].iteritems():
            result = result and helpers.test_model(models.power, power_data)

        for temperature, temperature_data in environment['temperature'].iteritems():
            result = result and helpers.test_model(models.temperature, temperature_data)

        for cpu, cpu_data in environment['cpu'].iteritems():
            result = result and helpers.test_model(models.cpu, cpu_data)

        result = result and helpers.test_model(models.memory, environment['memory'])

        assert result
        return environment

    @wrap_test_cases
    def test_get_bgp_neighbors(self):
        """Test get_bgp_neighbors."""
        get_bgp_neighbors = self.device.get_bgp_neighbors()
        result = 'global' in get_bgp_neighbors.keys()

        if not result:
            print('global is not part of the returned vrfs')
        else:
            for vrf, vrf_data in get_bgp_neighbors.iteritems():
                result = result and isinstance(vrf_data['router_id'], unicode)
                if not result:
                    print('router_id is not unicode')

                for peer, peer_data in vrf_data['peers'].iteritems():
                    result = result and helpers.test_model(models.peer, peer_data)

                    for af, af_data in peer_data['address_family'].iteritems():
                        result = result and helpers.test_model(models.af, af_data)

            assert result
            return get_bgp_neighbors

    @wrap_test_cases
    def test_get_lldp_neighbors_detail(self):
        """Test get_lldp_neighbors_detail."""
        get_lldp_neighbors_detail = self.device.get_lldp_neighbors_detail()
        result = len(get_lldp_neighbors_detail) > 0

        for interface, neighbor_list in get_lldp_neighbors_detail.iteritems():
            for neighbor in neighbor_list:
                result = result and helpers.test_model(models.lldp_neighbors_detail, neighbor)

        assert result
        return get_lldp_neighbors_detail

    @wrap_test_cases
    def test_get_bgp_config(self):
        """Test get_bgp_config."""
        get_bgp_config = self.device.get_bgp_config()
        result = len(get_bgp_config) > 0

        for bgp_group in get_bgp_config.values():
            result = result and helpers.test_model(models.bgp_config_group, bgp_group)
            for bgp_neighbor in bgp_group.get('neighbors', {}).values():
                result = result and helpers.test_model(models.bgp_config_neighbor, bgp_neighbor)

        assert result
        return get_bgp_config

    @wrap_test_cases
    def test_get_bgp_neighbors_detail(self):
        """Test get_bgp_neighbors_detail."""
        get_bgp_neighbors_detail = self.device.get_bgp_neighbors_detail()

        result = len(get_bgp_neighbors_detail) > 0

        for vrf, vrf_ases in get_bgp_neighbors_detail.iteritems():
            result = result and isinstance(vrf, unicode)
            for remote_as, neighbor_list in vrf_ases.iteritems():
                result = result and isinstance(remote_as, int)
                for neighbor in neighbor_list:
                    result = result and helpers.test_model(models.peer_details, neighbor)

        assert result
        return get_bgp_neighbors_detail

    @wrap_test_cases
    def test_get_arp_table(self):
        """Test get_arp_table."""
        get_arp_table = self.device.get_arp_table()
        result = len(get_arp_table) > 0

        for arp_entry in get_arp_table:
            result = result and helpers.test_model(models.arp_table, arp_entry)

        assert result
        return get_arp_table

    @wrap_test_cases
    def test_get_ntp_peers(self):
        """Test get_ntp_peers."""
        get_ntp_peers = self.device.get_ntp_peers()
        result = len(get_ntp_peers) > 0

        for peer, peer_details in get_ntp_peers.iteritems():
            result = result and isinstance(peer, unicode)
            result = result and helpers.test_model(models.ntp_peer, peer_details)

        assert result
        return get_ntp_peers

    @wrap_test_cases
    def test_get_ntp_stats(self):
        """Test get_ntp_stats."""
        get_ntp_stats = self.device.get_ntp_stats()
        result = len(get_ntp_stats) > 0

        for ntp_peer_details in get_ntp_stats:
            result = result and helpers.test_model(models.ntp_stats, ntp_peer_details)

        assert result
        return get_ntp_stats

    @wrap_test_cases
    def test_get_interfaces_ip(self):
        """Test get_interfaces_ip."""
        get_interfaces_ip = self.device.get_interfaces_ip()
        result = len(get_interfaces_ip) > 0

        for interface, interface_details in get_interfaces_ip.iteritems():
            ipv4 = interface_details.get('ipv4', {})
            ipv6 = interface_details.get('ipv6', {})
            for ip, ip_details in ipv4.iteritems():
                result = result and helpers.test_model(models.interfaces_ip, ip_details)
            for ip, ip_details in ipv6.iteritems():
                result = result and helpers.test_model(models.interfaces_ip, ip_details)

        assert result
        return get_interfaces_ip

    @wrap_test_cases
    def test_get_mac_address_table(self):
        """Test get_mac_address_table."""
        get_mac_address_table = self.device.get_mac_address_table()
        assert len(get_mac_address_table) > 0

        for mac_table_entry in get_mac_address_table:
            assert helpers.test_model(models.mac_address_table, mac_table_entry)

        return get_mac_address_table

    @wrap_test_cases
    def test_get_route_to(self):
        """Test get_route_to."""
        destination = '1.0.4.0/24'
        protocol = 'bgp'
        get_route_to = self.device.get_route_to(destination=destination, protocol=protocol)

        result = len(get_route_to) > 0

        for prefix, routes in get_route_to.iteritems():
            for route in routes:
                result = result and helpers.test_model(models.route, route)

        assert result
        return get_route_to

    @wrap_test_cases
    def test_get_snmp_information(self):
        """Test get_snmp_information."""
        get_snmp_information = self.device.get_snmp_information()

        result = len(get_snmp_information) > 0

        for snmp_entry in get_snmp_information:
            result = result and helpers.test_model(models.snmp, get_snmp_information)

        for community, community_data in get_snmp_information['community'].iteritems():
            result = result and helpers.test_model(models.snmp_community, community_data)

        assert result
        return get_snmp_information

    @wrap_test_cases
    def test_get_probes_config(self):
        """Test get_probes_config."""
        get_probes_config = self.device.get_probes_config()

        result = len(get_probes_config) > 0

        for probe_name, probe_tests in get_probes_config.iteritems():
            for test_name, test_config in probe_tests.iteritems():
                result = result and helpers.test_model(models.probe_test, test_config)

        assert result
        return get_probes_config

    @wrap_test_cases
    def test_get_probes_results(self):
        """Test get_probes_results."""
        get_probes_results = self.device.get_probes_results()
        result = len(get_probes_results) > 0

        for probe_name, probe_tests in get_probes_results.iteritems():
            for test_name, test_results in probe_tests.iteritems():
                result = result and helpers.test_model(models.probe_test_results, test_results)

        assert result
        return get_probes_results

    @wrap_test_cases
    def test_ping(self):
        """Test ping."""
        destination = '8.8.8.8'
        get_ping = self.device.ping(destination)
        result = isinstance(get_ping.get('success'), dict)
        ping_results = get_ping.get('success', {})

        result = result and helpers.test_model(models.ping, ping_results)

        for ping_result in ping_results.get('results', []):
            result = result and helpers.test_model(models.ping_result, ping_result)

        assert result
        return get_ping

    @wrap_test_cases
    def test_traceroute(self):
        """Test traceroute."""
        destination = '8.8.8.8'
        get_traceroute = self.device.traceroute(destination)
        result = isinstance(get_traceroute.get('success'), dict)
        traceroute_results = get_traceroute.get('success', {})

        for hope_id, hop_result in traceroute_results.iteritems():
            for probe_id, probe_result in hop_result.get('probes', {}).iteritems():
                result = result and helpers.test_model(models.traceroute, probe_result)

        assert result
        return get_traceroute

    @wrap_test_cases
    def test_get_users(self):
        """Test get_users."""
        get_users = self.device.get_users()
        result = len(get_users)

        for user, user_details in get_users.iteritems():
            result = result and helpers.test_model(models.users, user_details)
            result = result and (0 <= user_details.get('level') <= 15)

        assert result
        return get_users

    @wrap_test_cases
    def test_get_optics(self):
        """Test get_optics."""
        get_optics = self.device.get_optics()
        assert isinstance(get_optics, dict)

        for iface, iface_data in get_optics.iteritems():
            assert isinstance(iface, unicode)
            for channel in iface_data['physical_channels']['channel']:
                assert len(channel) == 2
                assert isinstance(channel['index'], int)
                for field in ['input_power', 'output_power',
                              'laser_bias_current']:

                    assert len(channel['state'][field]) == 4
                    assert isinstance(channel['state'][field]['instant'],
                                      float)
                    assert isinstance(channel['state'][field]['avg'], float)
                    assert isinstance(channel['state'][field]['min'], float)
                    assert isinstance(channel['state'][field]['max'], float)

        return get_optics
<commit_msg>Better comparison<commit_after>"""Testing framework."""

from __future__ import print_function

import functools
import json
import itertools

from double import BaseTestDouble

import helpers

import models

import pytest

def list_dicts_diff(prv, nxt):
    """Compare two lists of dicts."""
    result = []
    for prv_element, nxt_element in itertools.izip_longest(prv, nxt, fillvalue={}):
        intermediate_result = dict_diff(prv_element, nxt_element)
        if intermediate_result:
            result.append(intermediate_result)
    return result

def dict_diff(prv, nxt):
    """Return a dict of keys that differ with another config object."""
    keys = set(prv.keys() + nxt.keys())
    result = {}

    for k in keys:
        if isinstance(prv.get(k), dict):
            if isinstance(nxt.get(k), dict):
                "If both are dicts we do a recursive call."
                diff = dict_diff(prv.get(k), nxt.get(k))
                if diff:
                    result[k] = diff
            else:
                "If only one is a dict they are clearly different"
                result[k] = {'result': prv.get(k), 'expected': nxt.get(k)}
        else:
            "Ellipsis is a wildcard."""
            if prv.get(k) != nxt.get(k) and nxt.get(k) != "...":
                result[k] = {'result': prv.get(k), 'expected': nxt.get(k)}
    return result

def wrap_test_cases(func):
    """Wrap test cases."""
    @functools.wraps(func)
    def wrapper(cls, test_case):
        cls.device.device.current_test = func.__name__
        cls.device.device.current_test_case = test_case

        try:
            # This is an ugly, ugly, ugly hack because some python objects don't load
            # as expected. For example, dicts where integers are strings
            result = json.loads(json.dumps(func(cls)))
            not_implemented = False

            if isinstance(cls.device.device, BaseTestDouble):
                try:
                    expected_result = cls.device.device.expected_result
                except IOError as e:
                    raise Exception("{}. Actual result was: {}".format(e, json.dumps(result)))
                if isinstance(result, list):
                    diff = list_dicts_diff(result, expected_result)
                else:
                    diff = dict_diff(result, expected_result)
                if diff:
                    print("Resulting JSON object was: {}".format(json.dumps(result)))
                    raise AssertionError("Expected result varies on some keys {}".format(
                                                                                json.dumps(diff)))

        except NotImplementedError:
            not_implemented = True

        cls.device.device.current_test = ''
        cls.device.device.current_test_case = ''

        if not_implemented:
            pytest.skip("Method not implemented")
        else:
            return result

    return wrapper

class BaseTestGetters:
    """Base class for testing drivers."""

    @wrap_test_cases
    def test_get_facts(self):
        """Test get_facts method."""
        facts = self.device.get_facts()
        assert helpers.test_model(models.facts, facts)
        return facts

    @wrap_test_cases
    def test_get_interfaces(self):
        """Test get_interfaces."""
        get_interfaces = self.device.get_interfaces()
        result = len(get_interfaces) > 0

        for interface, interface_data in get_interfaces.iteritems():
            result = result and helpers.test_model(models.interface, interface_data)

        assert result
        return get_interfaces

    @wrap_test_cases
    def test_get_lldp_neighbors(self):
        """Test get_lldp_neighbors."""
        get_lldp_neighbors = self.device.get_lldp_neighbors()
        result = len(get_lldp_neighbors) > 0

        for interface, neighbor_list in get_lldp_neighbors.iteritems():
            for neighbor in neighbor_list:
                result = result and helpers.test_model(models.lldp_neighbors, neighbor)

        assert result
        return get_lldp_neighbors

    @wrap_test_cases
    def test_get_interfaces_counters(self):
        """Test get_interfaces_counters."""
        get_interfaces_counters = self.device.get_interfaces_counters()
        result = len(self.device.get_interfaces_counters()) > 0

        for interface, interface_data in get_interfaces_counters.iteritems():
            result = result and helpers.test_model(models.interface_counters, interface_data)

        assert result
        return get_interfaces_counters

    @wrap_test_cases
    def test_get_environment(self):
        """Test get_environment."""
        environment = self.device.get_environment()
        result = len(environment) > 0

        for fan, fan_data in environment['fans'].iteritems():
            result = result and helpers.test_model(models.fan, fan_data)

        for power, power_data in environment['power'].iteritems():
            result = result and helpers.test_model(models.power, power_data)

        for temperature, temperature_data in environment['temperature'].iteritems():
            result = result and helpers.test_model(models.temperature, temperature_data)

        for cpu, cpu_data in environment['cpu'].iteritems():
            result = result and helpers.test_model(models.cpu, cpu_data)

        result = result and helpers.test_model(models.memory, environment['memory'])

        assert result
        return environment

    @wrap_test_cases
    def test_get_bgp_neighbors(self):
        """Test get_bgp_neighbors."""
        get_bgp_neighbors = self.device.get_bgp_neighbors()
        result = 'global' in get_bgp_neighbors.keys()

        if not result:
            print('global is not part of the returned vrfs')
        else:
            for vrf, vrf_data in get_bgp_neighbors.iteritems():
                result = result and isinstance(vrf_data['router_id'], unicode)
                if not result:
                    print('router_id is not unicode')

                for peer, peer_data in vrf_data['peers'].iteritems():
                    result = result and helpers.test_model(models.peer, peer_data)

                    for af, af_data in peer_data['address_family'].iteritems():
                        result = result and helpers.test_model(models.af, af_data)

            assert result
            return get_bgp_neighbors

    @wrap_test_cases
    def test_get_lldp_neighbors_detail(self):
        """Test get_lldp_neighbors_detail."""
        get_lldp_neighbors_detail = self.device.get_lldp_neighbors_detail()
        result = len(get_lldp_neighbors_detail) > 0

        for interface, neighbor_list in get_lldp_neighbors_detail.iteritems():
            for neighbor in neighbor_list:
                result = result and helpers.test_model(models.lldp_neighbors_detail, neighbor)

        assert result
        return get_lldp_neighbors_detail

    @wrap_test_cases
    def test_get_bgp_config(self):
        """Test get_bgp_config."""
        get_bgp_config = self.device.get_bgp_config()
        result = len(get_bgp_config) > 0

        for bgp_group in get_bgp_config.values():
            result = result and helpers.test_model(models.bgp_config_group, bgp_group)
            for bgp_neighbor in bgp_group.get('neighbors', {}).values():
                result = result and helpers.test_model(models.bgp_config_neighbor, bgp_neighbor)

        assert result
        return get_bgp_config

    @wrap_test_cases
    def test_get_bgp_neighbors_detail(self):
        """Test get_bgp_neighbors_detail."""
        get_bgp_neighbors_detail = self.device.get_bgp_neighbors_detail()

        result = len(get_bgp_neighbors_detail) > 0

        for vrf, vrf_ases in get_bgp_neighbors_detail.iteritems():
            result = result and isinstance(vrf, unicode)
            for remote_as, neighbor_list in vrf_ases.iteritems():
                result = result and isinstance(remote_as, int)
                for neighbor in neighbor_list:
                    result = result and helpers.test_model(models.peer_details, neighbor)

        assert result
        return get_bgp_neighbors_detail

    @wrap_test_cases
    def test_get_arp_table(self):
        """Test get_arp_table."""
        get_arp_table = self.device.get_arp_table()
        result = len(get_arp_table) > 0

        for arp_entry in get_arp_table:
            result = result and helpers.test_model(models.arp_table, arp_entry)

        assert result
        return get_arp_table

    @wrap_test_cases
    def test_get_ntp_peers(self):
        """Test get_ntp_peers."""
        get_ntp_peers = self.device.get_ntp_peers()
        result = len(get_ntp_peers) > 0

        for peer, peer_details in get_ntp_peers.iteritems():
            result = result and isinstance(peer, unicode)
            result = result and helpers.test_model(models.ntp_peer, peer_details)

        assert result
        return get_ntp_peers

    @wrap_test_cases
    def test_get_ntp_stats(self):
        """Test get_ntp_stats."""
        get_ntp_stats = self.device.get_ntp_stats()
        result = len(get_ntp_stats) > 0

        for ntp_peer_details in get_ntp_stats:
            result = result and helpers.test_model(models.ntp_stats, ntp_peer_details)

        assert result
        return get_ntp_stats

    @wrap_test_cases
    def test_get_interfaces_ip(self):
        """Test get_interfaces_ip."""
        get_interfaces_ip = self.device.get_interfaces_ip()
        result = len(get_interfaces_ip) > 0

        for interface, interface_details in get_interfaces_ip.iteritems():
            ipv4 = interface_details.get('ipv4', {})
            ipv6 = interface_details.get('ipv6', {})
            for ip, ip_details in ipv4.iteritems():
                result = result and helpers.test_model(models.interfaces_ip, ip_details)
            for ip, ip_details in ipv6.iteritems():
                result = result and helpers.test_model(models.interfaces_ip, ip_details)

        assert result
        return get_interfaces_ip

    @wrap_test_cases
    def test_get_mac_address_table(self):
        """Test get_mac_address_table."""
        get_mac_address_table = self.device.get_mac_address_table()
        assert len(get_mac_address_table) > 0

        for mac_table_entry in get_mac_address_table:
            assert helpers.test_model(models.mac_address_table, mac_table_entry)

        return get_mac_address_table

    @wrap_test_cases
    def test_get_route_to(self):
        """Test get_route_to."""
        destination = '1.0.4.0/24'
        protocol = 'bgp'
        get_route_to = self.device.get_route_to(destination=destination, protocol=protocol)

        result = len(get_route_to) > 0

        for prefix, routes in get_route_to.iteritems():
            for route in routes:
                result = result and helpers.test_model(models.route, route)

        assert result
        return get_route_to

    @wrap_test_cases
    def test_get_snmp_information(self):
        """Test get_snmp_information."""
        get_snmp_information = self.device.get_snmp_information()

        result = len(get_snmp_information) > 0

        for snmp_entry in get_snmp_information:
            result = result and helpers.test_model(models.snmp, get_snmp_information)

        for community, community_data in get_snmp_information['community'].iteritems():
            result = result and helpers.test_model(models.snmp_community, community_data)

        assert result
        return get_snmp_information

    @wrap_test_cases
    def test_get_probes_config(self):
        """Test get_probes_config."""
        get_probes_config = self.device.get_probes_config()

        result = len(get_probes_config) > 0

        for probe_name, probe_tests in get_probes_config.iteritems():
            for test_name, test_config in probe_tests.iteritems():
                result = result and helpers.test_model(models.probe_test, test_config)

        assert result
        return get_probes_config

    @wrap_test_cases
    def test_get_probes_results(self):
        """Test get_probes_results."""
        get_probes_results = self.device.get_probes_results()
        result = len(get_probes_results) > 0

        for probe_name, probe_tests in get_probes_results.iteritems():
            for test_name, test_results in probe_tests.iteritems():
                result = result and helpers.test_model(models.probe_test_results, test_results)

        assert result
        return get_probes_results

    @wrap_test_cases
    def test_ping(self):
        """Test ping."""
        destination = '8.8.8.8'
        get_ping = self.device.ping(destination)
        result = isinstance(get_ping.get('success'), dict)
        ping_results = get_ping.get('success', {})

        result = result and helpers.test_model(models.ping, ping_results)

        for ping_result in ping_results.get('results', []):
            result = result and helpers.test_model(models.ping_result, ping_result)

        assert result
        return get_ping

    @wrap_test_cases
    def test_traceroute(self):
        """Test traceroute."""
        destination = '8.8.8.8'
        get_traceroute = self.device.traceroute(destination)
        result = isinstance(get_traceroute.get('success'), dict)
        traceroute_results = get_traceroute.get('success', {})

        for hope_id, hop_result in traceroute_results.iteritems():
            for probe_id, probe_result in hop_result.get('probes', {}).iteritems():
                result = result and helpers.test_model(models.traceroute, probe_result)

        assert result
        return get_traceroute

    @wrap_test_cases
    def test_get_users(self):
        """Test get_users."""
        get_users = self.device.get_users()
        result = len(get_users)

        for user, user_details in get_users.iteritems():
            result = result and helpers.test_model(models.users, user_details)
            result = result and (0 <= user_details.get('level') <= 15)

        assert result
        return get_users

    @wrap_test_cases
    def test_get_optics(self):
        """Test get_optics."""
        get_optics = self.device.get_optics()
        assert isinstance(get_optics, dict)

        for iface, iface_data in get_optics.iteritems():
            assert isinstance(iface, unicode)
            for channel in iface_data['physical_channels']['channel']:
                assert len(channel) == 2
                assert isinstance(channel['index'], int)
                for field in ['input_power', 'output_power',
                              'laser_bias_current']:

                    assert len(channel['state'][field]) == 4
                    assert isinstance(channel['state'][field]['instant'],
                                      float)
                    assert isinstance(channel['state'][field]['avg'], float)
                    assert isinstance(channel['state'][field]['min'], float)
                    assert isinstance(channel['state'][field]['max'], float)

        return get_optics
<|endoftext|>


<commit_before>import glob
import os

from lettuce import *
from splinter import Browser
from django.core.management import call_command

from django.template.defaultfilters import slugify

@before.each_scenario
def flush_database(step):
    call_command('flush', interactive=False)

@before.all
def clear_screenshots():
    screenshots = glob.glob('./screenshots/*.png')
    for screenshot in screenshots:
        os.remove(screenshot)
    open_browser()

def open_browser():
    world.browser = Browser("phantomjs")
    world.browser.driver.set_window_size(1024, 720)

@after.each_scenario
def take_screenshot(scenario):
    if scenario.failed:
        world.browser.driver.save_screenshot('screenshots/%s.png' % slugify(scenario.name))

@after.each_scenario
def clear_cookies(scenario):
    world.browser.cookies.delete()

@after.all
def close_browser(total):
    world.browser.quit()
<commit_msg>end to end ft for extract functionality<commit_after>import glob
import os

from lettuce import *
from splinter import Browser
from django.core.management import call_command

from django.template.defaultfilters import slugify

@before.each_scenario
def flush_database(step):
    call_command('flush', interactive=False)

@before.all
def clear_screen_shots():
    screen_shots = glob.glob('./screenshots/*.png')
    for screen_shot in screen_shots:
        os.remove(screen_shot)
    open_browser()

def open_browser():
    world.browser = Browser("phantomjs")
    world.browser.driver.set_window_size(1024, 720)

@after.each_scenario
def take_screen_shot(scenario):
    if scenario.failed:
        world.browser.driver.save_screenshot('screenshots/%s.png' % slugify(scenario.name))

@after.each_scenario
def clear_cookies(scenario):
    world.browser.cookies.delete()

@after.all
def close_browser(total):
    world.browser.quit()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Feb 28 22:41:26 2017

@author: lamahamadeh
"""

'''
Case Study about Language Processing
'''

#counting words
#---------------

text = "This is a test text. We're keping this text short to keep things manageable." #test text

#Using loops
#-----------

def count_words(text):
    """count the number of times each word occurs in text (str).
    Return dictionary where keys are unique words and values are
    word counts. skip punctuations"""
    text = text.lower() #lowercase for the counting letters so the function can cont the same words whether it's capatilised or not
    skips = [".", ",", ";", ":", "'", '"'] #skipping all the punctuations to not be counted with the words that come bfore them
    for ch in skips:
        text = text.replace(ch,"")
    word_counts = {}
    for word in text.split(" "):
        if word in word_counts: #known word case
            word_counts[word] += 1
        else:
            word_counts[word] = 1 #unknown word case
    return word_counts

print(count_words(text))
print(len(count_words("This comprehension check is to check for comprehension.")))#first quiz question

#------------------------------------------------------------------------------

#using collections module
#-------------------------

from collections import Counter

def count_words_fast(text):
    """count the number of times each word occurs in text (str).
    Return dictionary where keys are unique words and values are
    word counts. skip punctuations"""
    text = text.lower() #lowercase for the counting letters so the function can cont the same words whether it's capatilised or not
    skips = [".", ",", ";", ":", "'", '"'] #skipping all the punctuations to not be counted with the words that come bfore them
    for ch in skips:
        text = text.replace(ch,"")
    word_counts = Counter(text.split(" "))
    return word_counts

print(count_words_fast)

print(count_words(text)==count_words_fast(text))
print(count_words(text) is count_words_fast(text))#second quiz question

#------------------------------------------------------------------------------

#read a book
#-------------

def read_book(title_path):
    """Read a book and return it as a string"""
    with open(title_path, "r", encoding = "utf8") as current_file: #encoding = "utf8" causes a problem when running the code in Python 2.7. However, it runs normally when using Python 3.5.
        text = current_file.read()
        text = text.replace("\n","").replace("\r","")
    return text

text = read_book('/Users/ADB3HAMADL/Desktop/Movies/English/Nora Ephron/You Have Got Mail.txt')#read a book from its path

print(len(text))#number of charatcers in the book

#if there is a famous/wanted line in the book we can use the 'find' method to find it
ind = text.find("go to the mattresses")
print(ind)#print the index number of the famous/wanted sentence
sample_text = text[ind : ind + 953]#slice the paragraph that contains the famous line
print(sample_text)#print the whole chosen paragraph

#------------------------------------------------------------------------------

#Counting the number of unique words
#------------------------------------

def word_stats(word_counts):
   """return the number of unique words and word frequencies"""
   num_unique = len(word_counts) #calculate the number of unique words in the text
   counts = word_counts.values() #calculate the frequency of each word in the text
   return(num_unique,counts)

text = read_book('/Users/ADB3HAMADL/Desktop/Movies/English/Nora Ephron/You Have Got Mail.txt')

word_counts = count_words(text)

(num_unique, counts) = word_stats(word_counts)

print(num_unique) #print the number of unique number of words in the text
print(sum(counts)) #print the total number of words in the text

#------------------------------------------------------------------------------

#Reading multiple files
#-----------------------

import os #to read directories

movie_dir = "/Users/ADB3HAMADL/Desktop/movies" #tells us how many directories in the book directory

import pandas as pd

'''
Pandas example of how to create a dataframe:
--------------------------------------------
import pandas as pd

table = pd.DataFrame(coloums = ("name" , "age"))
table.loc[1] = "James", 22
table.loc[2] = "Jess", 32

print(table)
'''

stats = pd.DataFrame(columns = ("language" , "director" , "title" , "length" , "unique")) #this creates an empty dataframe
#with empty table elements with 5 columns

#To put data in the table
title_num =1
for language in os.listdir(movie_dir):
    for director in os.listdir(movie_dir + "/" + language):
        for title in os.listdir(movie_dir + "/" + language + "/" + director):
            inputfile = movie_dir + "/" + language + "/" + director + "/" + title
            print(inputfile)
            text = read_book(inputfile)
            (num_unique, counts) = word_stats(count_words(text))
            stats.loc[title_num ] = language , director.capitalize(), title.replace(".txt", " ") , sum(counts) , num_unique
            title_num += 1

print(stats) #print the created dataframe
print(stats.head()) #print the top 5 lines
print(stats.tail()) #print the last 5 lines

print(stats[stats.language == "English"]) #print the number of entries for language English (a subset from the whole dataframe)

#------------------------------------------------------------------------------

#Plotting Book Statistics
#-------------------------

import matplotlib.pyplot as plt

plt.plot(stats.length, stats.unique, "bo")
#OR we can write plt.plot(stats['length'], stats['unique'])
plt.loglog(stats.length, stats.unique, "bo") #it is a straight line which suggest data modelling strategies that we might use

plt.figure(figsize = (10,10))

subset = stats[stats.language == "English"] #extract a subset that has only the rows with English Language
plt.loglog(subset.length, subset.unique, "o", label = "English", color = "blue")

subset = stats[stats.language == "French"] #extract a subset that has only the rows with French Language
plt.loglog(subset.length, subset.unique, "o", label = "English", color = "red")

plt.legend()
plt.xlabel("Movie Length")
plt.ylabel("Number of unique words")
plt.savefig("lang_plot.pdf")

#------------------------------------------------------------------------------

#
<commit_msg>Update Language_Processing.py<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Feb 28 22:41:26 2017

@author: lamahamadeh
"""

'''
Case Study about Language Processing
'''

#counting words
#---------------

text = "This is a test text. We're keping this text short to keep things manageable." #test text

#Using loops
#-----------

def count_words(text):
    """count the number of times each word occurs in text (str).
    Return dictionary where keys are unique words and values are
    word counts. skip punctuations"""
    text = text.lower() #lowercase for the counting letters so the function can cont the same words whether it's capatilised or not
    skips = [".", ",", ";", ":", "'", '"'] #skipping all the punctuations to not be counted with the words that come bfore them
    for ch in skips:
        text = text.replace(ch,"")
    word_counts = {}
    for word in text.split(" "):
        if word in word_counts: #known word case
            word_counts[word] += 1
        else:
            word_counts[word] = 1 #unknown word case
    return word_counts

print(count_words(text))
print(len(count_words("This comprehension check is to check for comprehension.")))#first quiz question

#------------------------------------------------------------------------------

#using collections module
#-------------------------

from collections import Counter

def count_words_fast(text):
    """count the number of times each word occurs in text (str).
    Return dictionary where keys are unique words and values are
    word counts. skip punctuations"""
    text = text.lower() #lowercase for the counting letters so the function can cont the same words whether it's capatilised or not
    skips = [".", ",", ";", ":", "'", '"'] #skipping all the punctuations to not be counted with the words that come bfore them
    for ch in skips:
        text = text.replace(ch,"")
    word_counts = Counter(text.split(" "))
    return word_counts

print(count_words_fast)

print(count_words(text)==count_words_fast(text))
print(count_words(text) is count_words_fast(text))#second quiz question

#------------------------------------------------------------------------------

#read a book
#-------------

def read_book(title_path):
    """Read a book and return it as a string"""
    with open(title_path, "r", encoding = "utf8") as current_file: #encoding = "utf8" causes a problem when running the code in Python 2.7. However, it runs normally when using Python 3.5.
        text = current_file.read()
        text = text.replace("\n","").replace("\r","")
    return text

text = read_book('/Users/ADB3HAMADL/Desktop/Movies/English/Nora Ephron/You Have Got Mail.txt')#read a book from its path

print(len(text))#number of charatcers in the book

#if there is a famous/wanted line in the book we can use the 'find' method to find it
ind = text.find("go to the mattresses")
print(ind)#print the index number of the famous/wanted sentence
sample_text = text[ind : ind + 953]#slice the paragraph that contains the famous line
print(sample_text)#print the whole chosen paragraph

#------------------------------------------------------------------------------

#Counting the number of unique words
#------------------------------------

def word_stats(word_counts):
   """return the number of unique words and word frequencies"""
   num_unique = len(word_counts) #calculate the number of unique words in the text
   counts = word_counts.values() #calculate the frequency of each word in the text
   return(num_unique,counts)

text = read_book('/Users/ADB3HAMADL/Desktop/Movies/English/Nora Ephron/You Have Got Mail.txt')

word_counts = count_words(text)

(num_unique, counts) = word_stats(word_counts)

print(num_unique) #print the number of unique number of words in the text
print(sum(counts)) #print the total number of words in the text

#------------------------------------------------------------------------------

#Reading multiple files
#-----------------------

import os #to read directories

movie_dir = "/Users/ADB3HAMADL/Desktop/movies" #tells us how many directories in the book directory

import pandas as pd

'''
Pandas example of how to create a dataframe:
--------------------------------------------
import pandas as pd

table = pd.DataFrame(coloums = ("name" , "age"))
table.loc[1] = "James", 22
table.loc[2] = "Jess", 32

print(table)
'''

stats = pd.DataFrame(columns = ("language" , "director" , "title" , "length" , "unique")) #this creates an empty dataframe
#with empty table elements with 5 columns

#To put data in the table
title_num =1
for language in os.listdir(movie_dir):
    for director in os.listdir(movie_dir + "/" + language):
        for title in os.listdir(movie_dir + "/" + language + "/" + director):
            inputfile = movie_dir + "/" + language + "/" + director + "/" + title
            print(inputfile)
            text = read_book(inputfile)
            (num_unique, counts) = word_stats(count_words(text))
            stats.loc[title_num ] = language , director.title(), title.replace(".txt", " ") , sum(counts) , num_unique #.title() here capitalises the first letter from the first and last name of the director. If we want to capitalise only the first letter, we can use .capitalize().
            title_num += 1

print(stats) #print the created dataframe
print(stats.head()) #print the top 5 lines
print(stats.tail()) #print the last 5 lines

print(stats[stats.language == "English"]) #print the number of entries for language English (a subset from the whole dataframe)

#------------------------------------------------------------------------------

#Plotting Book Statistics
#-------------------------

import matplotlib.pyplot as plt

plt.plot(stats.length, stats.unique, "bo")
#OR we can write plt.plot(stats['length'], stats['unique'])
plt.loglog(stats.length, stats.unique, "bo") #it is a straight line which suggest data modelling strategies that we might use

plt.figure(figsize = (10,10))

subset = stats[stats.language == "English"] #extract a subset that has only the rows with English Language
plt.loglog(subset.length, subset.unique, "o", label = "English", color = "blue")

subset = stats[stats.language == "French"] #extract a subset that has only the rows with French Language
plt.loglog(subset.length, subset.unique, "o", label = "English", color = "red")

plt.legend()
plt.xlabel("Movie Length")
plt.ylabel("Number of unique words")
plt.savefig("lang_plot.pdf")

#------------------------------------------------------------------------------

#
<|endoftext|>


<commit_before>"""
Django settings for qblog project.

For more information on this file, see
https://docs.djangoproject.com/en/dev/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/dev/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/dev/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'mpjn+z)t_w0%nttcu$web9*@*br)^5qzgb@(gy$v%es23rs(%a'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = ['*']

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'blog',
    'django_markdown',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'qblog.urls'

WSGI_APPLICATION = 'qblog.wsgi.application'

# Database
# https://docs.djangoproject.com/en/dev/ref/settings/#databases

# DATABASES = {
#     'default': {
#         'ENGINE': 'django.db.backends.sqlite3',
#         'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
#     }
# }

import dj_database_url
DATABASES = {
    'default':  dj_database_url.config()
}

SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')

# Internationalization
# https://docs.djangoproject.com/en/dev/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/dev/howto/static-files/
STATIC_ROOT = os.path.join(BASE_DIR, 'staticfile')

STATIC_URL = '/static/'
TEMPLATE_DIRS = (os.path.join(BASE_DIR, "templates"), )
STATICFILES_DIRS = (os.path.join(BASE_DIR, "static"), )
# STATICFILES_STORAGE = 'whitenoise.django.GzipManifestStaticFilesStorage'
<commit_msg>add django herokuapp<commit_after>"""
Django settings for qblog project.

For more information on this file, see
https://docs.djangoproject.com/en/dev/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/dev/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/dev/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'mpjn+z)t_w0%nttcu$web9*@*br)^5qzgb@(gy$v%es23rs(%a'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False

TEMPLATE_DEBUG = False

ALLOWED_HOSTS = ['*']

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'blog',
    'django_markdown',
    'herokuapp',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'qblog.urls'

WSGI_APPLICATION = 'qblog.wsgi.application'

# Database
# https://docs.djangoproject.com/en/dev/ref/settings/#databases

# DATABASES = {
#     'default': {
#         'ENGINE': 'django.db.backends.sqlite3',
#         'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
#     }
# }

import dj_database_url
DATABASES = {
    'default':  dj_database_url.config()
}

SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')

# Internationalization
# https://docs.djangoproject.com/en/dev/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/dev/howto/static-files/
STATIC_ROOT = os.path.join(BASE_DIR, 'staticfile')

STATIC_URL = '/static/'
TEMPLATE_DIRS = (os.path.join(BASE_DIR, "templates"), )
STATICFILES_DIRS = (os.path.join(BASE_DIR, "static"), )
# STATICFILES_STORAGE = 'whitenoise.django.GzipManifestStaticFilesStorage'

LOGGING = {
    "version": 1,
    "disable_existing_loggers": False,
    "handlers": {
        "console": {
            "level": "INFO",
            "class": "logging.StreamHandler",
        },
    },
    "loggers": {
        "django": {
            "handlers": ["console"],
        }
    }
}
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Feb 28 22:41:26 2017

@author: lamahamadeh
"""

'''
Case Study about Language Processing
'''

#counting words
#---------------

text = "This is a test text. We're keping this text short to keep things manageable." #test text

#Using loops
#-----------

def count_words(text):
    """count the number of times each word occurs in text (str).
    Return dictionary where keys are unique words and values are
    word counts. skip punctuations"""
    text = text.lower() #lowercase for the counting letters so the function can cont the same words whether it's capatilised or not
    skips = [".", ",", ";", ":", "'", '"'] #skipping all the punctuations to not be counted with the words that come bfore them
    for ch in skips:
        text = text.replace(ch,"")
    word_counts = {}
    for word in text.split(" "):
        if word in word_counts: #known word case
            word_counts[word] += 1
        else:
            word_counts[word] = 1 #unknown word case
    return word_counts

print(count_words(text))
print(len(count_words("This comprehension check is to check for comprehension.")))#first quiz question

#------------------------------------------------------------------------------

#using collections module
#-------------------------

from collections import Counter

def count_words_fast(text):
    """count the number of times each word occurs in text (str).
    Return dictionary where keys are unique words and values are
    word counts. skip punctuations"""
    text = text.lower() #lowercase for the counting letters so the function can cont the same words whether it's capatilised or not
    skips = [".", ",", ";", ":", "'", '"'] #skipping all the punctuations to not be counted with the words that come bfore them
    for ch in skips:
        text = text.replace(ch,"")
    word_counts = Counter(text.split(" "))
    return word_counts

print(count_words_fast)

print(count_words(text)==count_words_fast(text))
print(count_words(text) is count_words_fast(text))#second quiz question

#------------------------------------------------------------------------------

#read a book
#-------------

def read_book(title_path):
    """Read a book and return it as a string"""
    with open(title_path, "r", encoding = "utf8") as current_file:
        text = current_file.read()
        text = text.replace("\n","").replace("\r","")
    return text

text = read_book('/Users/ADB3HAMADL/Desktop/Movies/English/Nora Ephron/You Have Got Mail.txt')#read a book from its path

print(len(text))#number of charatcers in the book

#if there is a famous/wanted line in the book we can use the 'find' method to find it
ind = text.find("go to the mattresses")
print(ind)#print the index number of the famous/wanted sentence
sample_text = text[ind : ind + 953]#slice the paragraph that contains the famous line
print(sample_text)#print the whole chosen paragraph

#------------------------------------------------------------------------------

#Counting the number of unique words
#------------------------------------

def word_stats(word_counts):
   """return the number of unique words and word frequencies"""
   num_unique = len(word_counts) #calculate the number of unique words in the text
   counts = word_counts.values() #calculate the frequency of each word in the text
   return(num_unique,counts)

text = read_book('/Users/ADB3HAMADL/Desktop/Movies/English/Nora Ephron/You Have Got Mail.txt')

word_counts = count_words(text)

(num_unique, counts) = word_stats(word_counts)

print(num_unique) #print the number of unique number of words in the text
print(sum(counts)) #print the total number of words in the text

#------------------------------------------------------------------------------

#Reading multiple files
#-----------------------

import os #to read directories

movie_dir = "/Users/ADB3HAMADL/Desktop/movies" #tells us how many directories in the book directory

import pandas as pd

'''
Pandas example of how to create a dataframe:
--------------------------------------------
import pandas as pd

table = pd.DataFrame(coloums = ("name" , "age"))
table.loc[1] = "James", 22
table.loc[2] = "Jess", 32

print(table)
'''

stats = pd.DataFrame(columns = ("language" , "director" , "title" , "length" , "unique")) #this creates an empty dataframe
#with empty table elements with 5 columns

#To put data in the table
title_num =1
for language in os.listdir(movie_dir):
    for director in os.listdir(movie_dir + "/" + language):
        for title in os.listdir(movie_dir + "/" + language + "/" + director):
            inputfile = movie_dir + "/" + language + "/" + director + "/" + title
            print(inputfile)
            text = read_book(inputfile)
            (num_unique, counts) = word_stats(count_words(text))
            stats.loc[title_num ] = language , director.capitalize(), title.replace(".txt", " ") , sum(counts) , num_unique
            title_num += 1

print(stats) #print the created dataframe
print(stats.head()) #print the top 5 lines
print(stats.tail()) #print the last 5 lines

print(stats[stats.language == "English"]) #print the number of entries for language English (a subset from the whole dataframe)

#------------------------------------------------------------------------------

#Plotting Book Statistics
#-------------------------

import matplotlib.pyplot as plt

plt.plot(stats.length, stats.unique, "bo")
#OR we can write plt.plot(stats['length'], stats['unique'])
plt.loglog(stats.length, stats.unique, "bo") #it is a straight line which suggest data modelling strategies that we might use

plt.figure(figsize = (10,10))

subset = stats[stats.language == "English"] #extract a subset that has only the rows with English Language
plt.loglog(subset.length, subset.unique, "o", label = "English", color = "blue")

subset = stats[stats.language == "French"] #extract a subset that has only the rows with French Language
plt.loglog(subset.length, subset.unique, "o", label = "English", color = "red")

plt.legend()
plt.xlabel("Movie Length")
plt.ylabel("Number of unique words")
plt.savefig("lang_plot.pdf")

#------------------------------------------------------------------------------

#
<commit_msg>Update Language_Processing.py<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Feb 28 22:41:26 2017

@author: lamahamadeh
"""

'''
Case Study about Language Processing
'''

#counting words
#---------------

text = "This is a test text. We're keping this text short to keep things manageable." #test text

#Using loops
#-----------

def count_words(text):
    """count the number of times each word occurs in text (str).
    Return dictionary where keys are unique words and values are
    word counts. skip punctuations"""
    text = text.lower() #lowercase for the counting letters so the function can cont the same words whether it's capatilised or not
    skips = [".", ",", ";", ":", "'", '"'] #skipping all the punctuations to not be counted with the words that come bfore them
    for ch in skips:
        text = text.replace(ch,"")
    word_counts = {}
    for word in text.split(" "):
        if word in word_counts: #known word case
            word_counts[word] += 1
        else:
            word_counts[word] = 1 #unknown word case
    return word_counts

print(count_words(text))
print(len(count_words("This comprehension check is to check for comprehension.")))#first quiz question

#------------------------------------------------------------------------------

#using collections module
#-------------------------

from collections import Counter

def count_words_fast(text):
    """count the number of times each word occurs in text (str).
    Return dictionary where keys are unique words and values are
    word counts. skip punctuations"""
    text = text.lower() #lowercase for the counting letters so the function can cont the same words whether it's capatilised or not
    skips = [".", ",", ";", ":", "'", '"'] #skipping all the punctuations to not be counted with the words that come bfore them
    for ch in skips:
        text = text.replace(ch,"")
    word_counts = Counter(text.split(" "))
    return word_counts

print(count_words_fast)

print(count_words(text)==count_words_fast(text))
print(count_words(text) is count_words_fast(text))#second quiz question

#------------------------------------------------------------------------------

#read a book
#-------------

def read_book(title_path):
    """Read a book and return it as a string"""
    with open(title_path, "r", encoding = "utf8") as current_file: #encoding = "utf8" causes a problem when running the code in Python 2.7. However, it runs normally when using Python 3.5.
        text = current_file.read()
        text = text.replace("\n","").replace("\r","")
    return text

text = read_book('/Users/ADB3HAMADL/Desktop/Movies/English/Nora Ephron/You Have Got Mail.txt')#read a book from its path

print(len(text))#number of charatcers in the book

#if there is a famous/wanted line in the book we can use the 'find' method to find it
ind = text.find("go to the mattresses")
print(ind)#print the index number of the famous/wanted sentence
sample_text = text[ind : ind + 953]#slice the paragraph that contains the famous line
print(sample_text)#print the whole chosen paragraph

#------------------------------------------------------------------------------

#Counting the number of unique words
#------------------------------------

def word_stats(word_counts):
   """return the number of unique words and word frequencies"""
   num_unique = len(word_counts) #calculate the number of unique words in the text
   counts = word_counts.values() #calculate the frequency of each word in the text
   return(num_unique,counts)

text = read_book('/Users/ADB3HAMADL/Desktop/Movies/English/Nora Ephron/You Have Got Mail.txt')

word_counts = count_words(text)

(num_unique, counts) = word_stats(word_counts)

print(num_unique) #print the number of unique number of words in the text
print(sum(counts)) #print the total number of words in the text

#------------------------------------------------------------------------------

#Reading multiple files
#-----------------------

import os #to read directories

movie_dir = "/Users/ADB3HAMADL/Desktop/movies" #tells us how many directories in the book directory

import pandas as pd

'''
Pandas example of how to create a dataframe:
--------------------------------------------
import pandas as pd

table = pd.DataFrame(coloums = ("name" , "age"))
table.loc[1] = "James", 22
table.loc[2] = "Jess", 32

print(table)
'''

stats = pd.DataFrame(columns = ("language" , "director" , "title" , "length" , "unique")) #this creates an empty dataframe
#with empty table elements with 5 columns

#To put data in the table
title_num =1
for language in os.listdir(movie_dir):
    for director in os.listdir(movie_dir + "/" + language):
        for title in os.listdir(movie_dir + "/" + language + "/" + director):
            inputfile = movie_dir + "/" + language + "/" + director + "/" + title
            print(inputfile)
            text = read_book(inputfile)
            (num_unique, counts) = word_stats(count_words(text))
            stats.loc[title_num ] = language , director.capitalize(), title.replace(".txt", " ") , sum(counts) , num_unique
            title_num += 1

print(stats) #print the created dataframe
print(stats.head()) #print the top 5 lines
print(stats.tail()) #print the last 5 lines

print(stats[stats.language == "English"]) #print the number of entries for language English (a subset from the whole dataframe)

#------------------------------------------------------------------------------

#Plotting Book Statistics
#-------------------------

import matplotlib.pyplot as plt

plt.plot(stats.length, stats.unique, "bo")
#OR we can write plt.plot(stats['length'], stats['unique'])
plt.loglog(stats.length, stats.unique, "bo") #it is a straight line which suggest data modelling strategies that we might use

plt.figure(figsize = (10,10))

subset = stats[stats.language == "English"] #extract a subset that has only the rows with English Language
plt.loglog(subset.length, subset.unique, "o", label = "English", color = "blue")

subset = stats[stats.language == "French"] #extract a subset that has only the rows with French Language
plt.loglog(subset.length, subset.unique, "o", label = "English", color = "red")

plt.legend()
plt.xlabel("Movie Length")
plt.ylabel("Number of unique words")
plt.savefig("lang_plot.pdf")

#------------------------------------------------------------------------------

#
<|endoftext|>