<commit_before>import logging
from setuptools.command.build_ext import if_dl
from numpy.core.multiarray import zeros

__author__ = 'blaz'

import sys
import os
import time
import threading
import select
import sys
import pybonjour
import settings
import socket
import pygame
import pygame.midi
from logger import *

class VirtualInstrument(threading.Thread):
    """
    Class for playing virtual instrument bonded via socket
    """
    """ midi instrument codes
    0   GRAND_PIANO
    19  CHURCH_ORGAN
    37  SLAP_BASS_1
    41  VIOLIN
    115 STEEL_DRUMS
    """

    instrument = 0
    hostname = ""
    channel = 0
    midi_out = None
    keyPressed = [False for _ in range(128)]
    def __init__(self, hostname, midi_out, instrument=0, channel=0):
        try:
            self.instrument = int(instrument.split(':')[1])
        except Exception, err:
            logging.error("Instrument ni pravilnega tipa, Details: %s", err.message)
        self.hostname = hostname
        self.channel = channel
        self.midi_out = midi_out
        try:
            self.midi_out.set_instrument(self.instrument, self.channel)
        except Exception, err:
            logging.error(err.message)
            #print err.message

    def processData(self, data):
        if "init:" in data:
            pass
        else:
            try:
                notes = [True if "True" in x else False for x in data[1:-1].split(', ')]
                for i in notes:
                    a = i + settings.FIRST_NOTE
                    self.keyPressed[a] = notes[i]

                    if self.keyPressed[a]:
                        self.midi_out.note_on(a, 127, channel=self.channel)
                        #logging.info("Note %d on channel %d is ON", a, self.channel)
                    else:
                        self.midi_out.note_off(a, 127, channel=self.channel)
                        #logging.info("Note %d on channel %d is OFF", a, self.channel)
            except Exception:
                logging.error("Received data is not in correct form.")

class AdvertiseService(threading.Thread):
    """
    Class for advertising service using Bonjour (MDNS). It runs in a seperate thread in the background
    """
    def __init__(self, name='rpiSpeaker01', regType='_VirtualSpeaker._udp', port=3333):
        threading.Thread.__init__(self)
        self.name = name
        self.regType = regType
        self.port = port
    def run(self):
        logging.info("Starting Bonjour service advertising thread")
        self.register_service(self.name, self.regType, self.port)

    def register_callback(self, sdRef, flags, errorCode, name, regtype, domain):
        if errorCode == pybonjour.kDNSServiceErr_NoError:
            logging.info('Started VirtualSpeaker Service:')
            logging.info('  name    =%s', name)
            logging.info('  regtype =%s', regtype)
            logging.info('  domain  =%s', domain)

    def register_service(self, name='rpiSpeaker01', regtype='_VirtualSpeaker._udp', port=3333):
        sdRef = pybonjour.DNSServiceRegister(name = name,
                                             regtype = regtype,
                                             port = port,
                                             callBack = self.register_callback)

        try:
            try:
                while True:
                    ready = select.select([sdRef], [], [])
                    if sdRef in ready[0]:
                        pybonjour.DNSServiceProcessResult(sdRef)
            except KeyboardInterrupt:
                pass
        finally:
            sdRef.close()

class InitTimidity(threading.Thread):
    """
    Class for software synthesizer timidity. It runs in a seperate thread in the background
    """
    def __init__(self):
        threading.Thread.__init__(self)

    def run(self):
        logging.info("Starting TiMidity in ALSA server mode")
        try:
            os.system("timidity -iA")
        except Exception as err:
            logging.error("Cant start softsynth, %s", err)
            exit()

#####
#MAIN
#####

#Start background Service Advertisment
register = AdvertiseService(name=settings.SPEAKER_NAME, port=settings.PORT)
logging.info("Starting VirtualDevice named: %s", settings.SPEAKER_NAME)
register.daemon = True
register.start()

logging.info("Initializing pygame midi")
try:
    pygame.init()
    pygame.midi.init()
    #port = pygame.midi.get_default_output_id()
    port = settings.MIDI_DEVICE
    logging.info("Using midi output_id :%s:", port)
    midi_out = pygame.midi.Output(port, 0)
except Exception as err:
    logging.error(err)
    exit()

#Initialize Timidity software synthesizer to output music to your soundcard
#timidity = InitTimidity()
#timidity.daemon = True
#timidity.start()

#Initialize UDP SERVER
port = settings.PORT
host = settings.HOST

# Datagram (udp) socket
try :
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    logging.info('Socket created')
except socket.error, msg :
    logging.error('Failed to create socket. Error Code : %s message: %s',str(msg[0]), msg[1])
    sys.exit()

# Bind socket to local host and port
try:
    s.bind((host, port))
except socket.error , msg:
    logging.fatal('Bind failed. Error Code : %s message: %s', str(msg[0]),msg[1])
    sys.exit()

logging.info('Socket bind complete')

logging.info("Waiting on port: %s", port)

#Check for connection loss and remove virtual instrument accordingly
longtimeNoSee = list()

#Virtual instrument list
virtualInstruments = dict()

newChannel = 0
"""
Loop forever
"""
while(1):

    #Read from socket
    data, addr = s.recvfrom(1024)

    #Virtual instrument missing from client list
    if addr not in virtualInstruments.keys():
        if "init:" in data:
            print("Added instument:", addr, " ", data, " on channel: ", newChannel)
            virtualInstruments.setdefault(addr, VirtualInstrument(hostname=addr, midi_out=midi_out,  instrument=data, channel=newChannel))
            logging.info("Virtual instrument with instrument_id: %s added.", data.split(':')[1])
            newChannel += 1
    else:
        virtualInstruments[addr].processData(data)
    if False:
        if not virtualInstruments[addr].processData(data):
            del virtualInstruments[addr]

    #print addr, ": ", data, "    dict ", virtualInstruments

s.close()

<commit_msg>Speaker fix<commit_after>import logging
from setuptools.command.build_ext import if_dl
from numpy.core.multiarray import zeros

__author__ = 'blaz'

import sys
import os
import time
import threading
import select
import sys
import pybonjour
import settings
import socket
import pygame
import pygame.midi
from logger import *

class VirtualInstrument(threading.Thread):
    """
    Class for playing virtual instrument bonded via socket
    """
    """ midi instrument codes
    0   GRAND_PIANO
    19  CHURCH_ORGAN
    37  SLAP_BASS_1
    41  VIOLIN
    115 STEEL_DRUMS
    """

    instrument = 0
    hostname = ""
    channel = 0
    midi_out = None
    keyPressed = [False for _ in range(128)]
    def __init__(self, hostname, midi_out, instrument=0, channel=0):
        try:
            self.instrument = int(instrument.split(':')[1])
        except Exception, err:
            logging.error("Instrument ni pravilnega tipa, Details: %s", err.message)
        self.hostname = hostname
        self.channel = channel
        self.midi_out = midi_out
        try:
            self.midi_out.set_instrument(self.instrument, self.channel)
        except Exception, err:
            logging.error(err.message)
            #print err.message

    def processData(self, data):
        if "init:" in data:
            pass
        else:
            try:
                notes = [True if "True" in x else False for x in data[1:-1].split(', ')]
                print notes
                for i in notes:
                    a = i + settings.FIRST_NOTE
                    self.keyPressed[a] = notes[i]

                    if notes[i]:
                        self.midi_out.note_on(a, 127, channel=self.channel)
                        logging.info("Note %d on channel %d is ON", a, self.channel)
                    else:
                        self.midi_out.note_off(a, 127, channel=self.channel)
                        logging.info("Note %d on channel %d is OFF", a, self.channel)
            except Exception:
                logging.error("Received data is not in correct form.")

class AdvertiseService(threading.Thread):
    """
    Class for advertising service using Bonjour (MDNS). It runs in a seperate thread in the background
    """
    def __init__(self, name='rpiSpeaker01', regType='_VirtualSpeaker._udp', port=3333):
        threading.Thread.__init__(self)
        self.name = name
        self.regType = regType
        self.port = port
    def run(self):
        logging.info("Starting Bonjour service advertising thread")
        self.register_service(self.name, self.regType, self.port)

    def register_callback(self, sdRef, flags, errorCode, name, regtype, domain):
        if errorCode == pybonjour.kDNSServiceErr_NoError:
            logging.info('Started VirtualSpeaker Service:')
            logging.info('  name    =%s', name)
            logging.info('  regtype =%s', regtype)
            logging.info('  domain  =%s', domain)

    def register_service(self, name='rpiSpeaker01', regtype='_VirtualSpeaker._udp', port=3333):
        sdRef = pybonjour.DNSServiceRegister(name = name,
                                             regtype = regtype,
                                             port = port,
                                             callBack = self.register_callback)

        try:
            try:
                while True:
                    ready = select.select([sdRef], [], [])
                    if sdRef in ready[0]:
                        pybonjour.DNSServiceProcessResult(sdRef)
            except KeyboardInterrupt:
                pass
        finally:
            sdRef.close()

class InitTimidity(threading.Thread):
    """
    Class for software synthesizer timidity. It runs in a seperate thread in the background
    """
    def __init__(self):
        threading.Thread.__init__(self)

    def run(self):
        logging.info("Starting TiMidity in ALSA server mode")
        try:
            os.system("timidity -iA")
        except Exception as err:
            logging.error("Cant start softsynth, %s", err)
            exit()

#####
#MAIN
#####

#Start background Service Advertisment
register = AdvertiseService(name=settings.SPEAKER_NAME, port=settings.PORT)
logging.info("Starting VirtualDevice named: %s", settings.SPEAKER_NAME)
register.daemon = True
register.start()

logging.info("Initializing pygame midi")
try:
    pygame.init()
    pygame.midi.init()
    #port = pygame.midi.get_default_output_id()
    port = settings.MIDI_DEVICE
    logging.info("Using midi output_id :%s:", port)
    midi_out = pygame.midi.Output(port, 0)
except Exception as err:
    logging.error(err)
    exit()

#Initialize Timidity software synthesizer to output music to your soundcard
#timidity = InitTimidity()
#timidity.daemon = True
#timidity.start()

#Initialize UDP SERVER
port = settings.PORT
host = settings.HOST

# Datagram (udp) socket
try :
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    logging.info('Socket created')
except socket.error, msg :
    logging.error('Failed to create socket. Error Code : %s message: %s',str(msg[0]), msg[1])
    sys.exit()

# Bind socket to local host and port
try:
    s.bind((host, port))
except socket.error , msg:
    logging.fatal('Bind failed. Error Code : %s message: %s', str(msg[0]),msg[1])
    sys.exit()

logging.info('Socket bind complete')

logging.info("Waiting on port: %s", port)

#Check for connection loss and remove virtual instrument accordingly
longtimeNoSee = list()

#Virtual instrument list
virtualInstruments = dict()

newChannel = 0
"""
Loop forever
"""
while(1):

    #Read from socket
    data, addr = s.recvfrom(1024)

    #Virtual instrument missing from client list
    if addr not in virtualInstruments.keys():
        if "init:" in data:
            print("Added instument:", addr, " ", data, " on channel: ", newChannel)
            virtualInstruments.setdefault(addr, VirtualInstrument(hostname=addr, midi_out=midi_out,  instrument=data, channel=newChannel))
            logging.info("Virtual instrument with instrument_id: %s added.", data.split(':')[1])
            newChannel += 1
    else:
        virtualInstruments[addr].processData(data)
    if False:
        if not virtualInstruments[addr].processData(data):
            del virtualInstruments[addr]

    #print addr, ": ", data, "    dict ", virtualInstruments

s.close()

<|endoftext|>


<commit_before>#!/usr/bin/env python
# XIA2DPAVersion.py
#   Copyright (C) 2006 CCLRC, Graeme Winter
#
#   This code is distributed under the terms and conditions of the
#   CCP4 Program Suite Licence Agreement as a CCP4 Library.
#   A copy of the CCP4 licence can be obtained by writing to the
#   CCP4 Secretary, Daresbury Laboratory, Warrington WA4 4AD, UK.
#
# 6th June 2006
#
# A file containing the version number of the current xia2dpa.
#
# History:
#
# 0.0.1 - initial build 06/06/06 - incomplete implementations of most
#         things but wanted to get the ball rolling on the version
#         numbering.
#
# 0.0.2 - 21/JUN/06 added first application xia2scan for analysing
#         humidifier images, added many useful types in Schema,
#         handlers in Handler etc and Wrappers.
#
# 0.0.3 - 16/AUG/06 getting something usefil in eventually, which
#         will integrate data using only mosflm - xia2process.
#
# 0.0.4 - 25/AUG/06 beginning work on a scaling module, will allow
#         reasonably complete processing from start to end
#
# 0.0.5 - 06/SEP/06 moved all text in programs from camelCase to tidy_style
#         and also implemented connection between XSweep and Integrater
#         factory
#
# 0.1.0 - 27/SEP/06 can successfully integrate & scale multiwavelength
#         data though this is not yet perfected!
#         25/OCT/06 can now run with JUST ccp4-6.0.1 through all 4 TS
#         examples from the .xinfo files, so this is version 0.1.0.
#         Tagging... - oh that's 0.1.1 then.
#
# 0.1.1.1 27/OCT/06 added new functionality to xia2scan.
#
# 0.2.0 - 08/NOV/06 this is it - it will now refine the resolution, refine
#         standard deviation parameters, feed back from scaling to integration
#         and all that jazz...
#
# 0.2.1 - 15/NOV/06 release candidate to go with CCP4 6.0.2.
#

VersionNumber = "0.2.1"
Version = "XIA2DPA %s" % VersionNumber
CVSTag = "xia2dpa-%s" % VersionNumber.replace('.', '_')
Directory = "xia2dpa-%s" % VersionNumber

if __name__ == '__main__':
    print 'This is XIA 2 DPA version %s' % VersionNumber
    print 'This should be in a directory called "%s"' % Directory
    print 'And should be CVS tagged as "%s"' % CVSTag

<commit_msg>Updating dpa version.<commit_after>#!/usr/bin/env python
# XIA2DPAVersion.py
#   Copyright (C) 2006 CCLRC, Graeme Winter
#
#   This code is distributed under the terms and conditions of the
#   CCP4 Program Suite Licence Agreement as a CCP4 Library.
#   A copy of the CCP4 licence can be obtained by writing to the
#   CCP4 Secretary, Daresbury Laboratory, Warrington WA4 4AD, UK.
#
# 6th June 2006
#
# A file containing the version number of the current xia2dpa.
#
# History:
#
# 0.0.1 - initial build 06/06/06 - incomplete implementations of most
#         things but wanted to get the ball rolling on the version
#         numbering.
#
# 0.0.2 - 21/JUN/06 added first application xia2scan for analysing
#         humidifier images, added many useful types in Schema,
#         handlers in Handler etc and Wrappers.
#
# 0.0.3 - 16/AUG/06 getting something usefil in eventually, which
#         will integrate data using only mosflm - xia2process.
#
# 0.0.4 - 25/AUG/06 beginning work on a scaling module, will allow
#         reasonably complete processing from start to end
#
# 0.0.5 - 06/SEP/06 moved all text in programs from camelCase to tidy_style
#         and also implemented connection between XSweep and Integrater
#         factory
#
# 0.1.0 - 27/SEP/06 can successfully integrate & scale multiwavelength
#         data though this is not yet perfected!
#         25/OCT/06 can now run with JUST ccp4-6.0.1 through all 4 TS
#         examples from the .xinfo files, so this is version 0.1.0.
#         Tagging... - oh that's 0.1.1 then.
#
# 0.1.1.1 27/OCT/06 added new functionality to xia2scan.
#
# 0.2.0 - 08/NOV/06 this is it - it will now refine the resolution, refine
#         standard deviation parameters, feed back from scaling to integration
#         and all that jazz...
#
# 0.2.1 - 15/NOV/06 release candidate to go with CCP4 6.0.2.
#
# 0.2.2 - 21/NOV/06 second release candidate - this one includes stuff for
#         estimating number of molecules in ASU, solvent fraction, spacegroup.
#

VersionNumber = "0.2.2"
Version = "XIA2DPA %s" % VersionNumber
CVSTag = "xia2dpa-%s" % VersionNumber.replace('.', '_')
Directory = "xia2dpa-%s" % VersionNumber

if __name__ == '__main__':
    print 'This is XIA 2 DPA version %s' % VersionNumber
    print 'This should be in a directory called "%s"' % Directory
    print 'And should be CVS tagged as "%s"' % CVSTag

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""

@author: Adam Reinhold Von Fisher - https://www.linkedin.com/in/adamrvfisher/

"""

#This is a HTML scraper, I/O, and formatting tool that requests OHLC data from Yahoo!

#Define function
def YahooGrabberII(ticker):

    #Import modules
    import requests
    import pandas as pd
    import time as t
    import json

    #Start timer
    starttime = t.time()

    #Variable assignment
    Asset = pd.DataFrame()
    #Assign URL for post request
    url = ('http://finance.yahoo.com/quote/' + ticker + '/history?period1=-2552017587' +
           '&period2=2552397400&interval=1d&filter=history&frequency=1d')
    #Make post request
    page = requests.post(url)
    print(page)
    #Format response to text
    textI = page.text

    #Locate text in response
    markerI = textI.find('[{"date":')
    #Assign trimmed response
    textI = textI[markerI:]

    #Locate text in trimmed response
    markerII = textI.find(',"isPending":false')
    #Assign trimmed response
    textI = textI[:markerII]
    #text to json
    textI = json.loads(textI)

    for i in textI:
        Asset = Asset.append(i, ignore_index = True)

    #Rename columns
    Asset = Asset.rename(columns={"adjclose": "Adj Close", "close": "Close", "date": "Date",
                       "high": "High", "low": "Low", "open": "Open", "volume": "Volume"})

    #Format datetime index
    Asset['Date'] = pd.to_datetime(Asset['Date'], unit = 's')
    Asset['Date'] = Asset['Date'].dt.date

    #Set index to Date
    Asset = Asset.set_index('Date')

    #End timer
    endtime = t.time()
    #Time calculation
    duration = endtime - starttime
    #Display time taken for request and formatting
    print("Data request for " + ticker + " took " + str(duration) +" seconds.")

    return Asset
    <commit_msg>Delete YahooGrabberII.py<commit_after><|endoftext|>


<commit_before>import unittest
from mockito_classmethods_test import *
from mockito_matchers_test import *
from mockito_staticmethods_test import *
from mockito_stubbing_test import *
from mockito_verification_test import *
from mockito_modulefunctions_test import *
from mockito_demo_test import *
from mockito_verification_errors_test import *

#TODO: can this be smarter - i can forget to put a test class here:
tests = [MockitoClassMethodsTest, MockitoMatchersTest, MockitoStaticMethodsTest,
         MockitoStubbingTest, MockitoVerificationTest, MockitoModuleFunctionsTest,
         MockitoDemoTest, MockitoVerificationErrorsTest]

all = unittest.TestSuite()
for test in tests: all.addTest(unittest.makeSuite(test))
unittest.TextTestRunner(verbosity=2).run(all)<commit_msg>Loading and running tests automatically from *_test.py files by all_tests.py. No more manual updates required!<commit_after>import unittest
import os
import re

pattern = re.compile('([a-z]+_)+test\.py$')

loader = unittest.TestLoader()
suite = unittest.TestSuite()

names = [f.replace('.py', '') for f in os.listdir('.') if pattern.match(f, 1)]
for name in names: suite.addTests(loader.loadTestsFromName(name))
unittest.TextTestRunner(verbosity=2).run(suite)<|endoftext|>


<commit_before># Licensed under an MIT open source license - see LICENSE

'''
Load in data sets for tests. 32^2 pixels portions of two data sets
are loaded in (one Design and one fiducial run).
Only the channels with signal were kept. Additional channels the match the
original spectral axis are added on and filled with noise centered around the
limit.
'''

# Need to create the property arrays
from ..data_reduction import Mask_and_Moments

from spectral_cube import SpectralCube, LazyMask, Projection

import os
import warnings
import numpy as np
import numpy.random as ra
from astropy.io.fits import getheader
from astropy.wcs import WCS
from astropy.io import fits
import astropy.units as u

# Set seed for adding noise.
ra.seed(121212)

turb_path = os.path.dirname(__file__)

# Open header for both
hdr_path = os.path.join(turb_path, "data/header.fits")
header = getheader(hdr_path)

keywords = ["centroid", "centroid_error", "integrated_intensity",
            "integrated_intensity_error", "linewidth",
            "linewidth_error", "moment0", "moment0_error", "cube"]

path1 = os.path.join(turb_path, "data/dataset1.npz")

dataset1 = np.load(path1)

cube1 = np.empty((500, 32, 32))

count = 0
for posn, kept in zip(*dataset1["channels"]):
    posn = int(posn)
    if kept:
        cube1[posn, :, :] = dataset1["cube"][count, :, :]
        count += 1
    else:
        cube1[posn, :, :] = ra.normal(0.005, 0.005, (32, 32))

sc1 = SpectralCube(data=cube1, wcs=WCS(header))
mask = LazyMask(np.isfinite, sc1)
sc1 = sc1.with_mask(mask)
# Set the scale for the purposes of the tests
props1 = Mask_and_Moments(sc1, scale=0.003031065017916262 * u.Unit(""))
props1.make_mask(mask=mask)
props1.make_moments()
props1.make_moment_errors()

dataset1 = props1.to_dict()

moment0_hdu1 = fits.PrimaryHDU(dataset1["moment0"][0],
                               header=dataset1["moment0"][1])

moment0_proj = Projection.from_hdu(moment0_hdu1)

##############################################################################

path2 = os.path.join(turb_path, "data/dataset2.npz")

dataset2 = np.load(path2)

cube2 = np.empty((500, 32, 32))

count = 0
for posn, kept in zip(*dataset2["channels"]):
    posn = int(posn)
    if kept:
        cube2[posn, :, :] = dataset2["cube"][count, :, :]
        count += 1
    else:
        cube2[posn, :, :] = ra.normal(0.005, 0.005, (32, 32))

sc2 = SpectralCube(data=cube2, wcs=WCS(header))
mask = LazyMask(np.isfinite, sc2)
sc2 = sc2.with_mask(mask)
# Set the scale for the purposes of the tests
props2 = Mask_and_Moments(sc2, scale=0.003029658694658428 * u.Unit(""))
props2.make_moments()
props2.make_moment_errors()

dataset2 = props2.to_dict()

##############################################################################

# Load in saved comparison data.
try:
    computed_data = np.load(os.path.join(turb_path, "data/checkVals.npz"))

    computed_distances = np.load(os.path.join(turb_path,
                                              "data/computed_distances.npz"))
except IOError:
    warnings.warn("No checkVals or computed_distances files.")

# Define ways to make 1D and 2D profiles (typically Gaussian for ease)
def twoD_gaussian(shape=(201, 201), x_std=10., y_std=10., theta=0, amp=1.,
                  bkg=0.):

    from astropy.modeling.models import Gaussian2D, Const2D

    centre = tuple([val // 2 for val in shape])

    mod = Gaussian2D(x_mean=centre[0], y_mean=centre[1],
                     x_stddev=x_std, y_stddev=y_std,
                     amplitude=amp, theta=theta) + \
        Const2D(amplitude=bkg)

    return mod

def generate_2D_array(shape=(201, 201), curve_type='gaussian', **kwargs):

    if curve_type == "gaussian":

        mod = twoD_gaussian(shape=shape, **kwargs)

    else:
        raise ValueError("curve_type must be 'gaussian'.")

    ygrid, xgrid = np.mgrid[:shape[0], :shape[1]]

    return mod(xgrid, ygrid)

def oneD_gaussian(shape=200, mean=0., std=10., amp=1., bkg=0.):

    from astropy.modeling.models import Gaussian1D, Const1D

    mod = Gaussian1D(mean=mean, stddev=std, amplitude=amp) + \
        Const1D(amplitude=bkg)

    return mod

def generate_1D_array(shape=200, curve_type='gaussian', **kwargs):

    if curve_type == "gaussian":

        mod = oneD_gaussian(shape=shape, **kwargs)

    else:
        raise ValueError("curve_type must be 'gaussian'.")

    xgrid = np.arange(shape)

    return mod(xgrid)

def assert_between(value, lower, upper):
    '''
    Check if a value is between two values.
    '''

    within_lower = value >= lower
    within_upper = value <= upper

    if within_lower and within_upper:
        return
    else:
        raise AssertionError("{0} not within {1} and {2}".format(value, lower,
                                                                 upper))
<commit_msg>Function for making powerlaw images, optionally with elliptical profiles<commit_after># Licensed under an MIT open source license - see LICENSE

'''
Load in data sets for tests. 32^2 pixels portions of two data sets
are loaded in (one Design and one fiducial run).
Only the channels with signal were kept. Additional channels the match the
original spectral axis are added on and filled with noise centered around the
limit.
'''

# Need to create the property arrays
from ..data_reduction import Mask_and_Moments

from spectral_cube import SpectralCube, LazyMask, Projection

import os
import warnings
import numpy as np
import numpy.random as ra
from astropy.io.fits import getheader
from astropy.wcs import WCS
from astropy.io import fits
import astropy.units as u

# Set seed for adding noise.
ra.seed(121212)

turb_path = os.path.dirname(__file__)

# Open header for both
hdr_path = os.path.join(turb_path, "data/header.fits")
header = getheader(hdr_path)

keywords = ["centroid", "centroid_error", "integrated_intensity",
            "integrated_intensity_error", "linewidth",
            "linewidth_error", "moment0", "moment0_error", "cube"]

path1 = os.path.join(turb_path, "data/dataset1.npz")

dataset1 = np.load(path1)

cube1 = np.empty((500, 32, 32))

count = 0
for posn, kept in zip(*dataset1["channels"]):
    posn = int(posn)
    if kept:
        cube1[posn, :, :] = dataset1["cube"][count, :, :]
        count += 1
    else:
        cube1[posn, :, :] = ra.normal(0.005, 0.005, (32, 32))

sc1 = SpectralCube(data=cube1, wcs=WCS(header))
mask = LazyMask(np.isfinite, sc1)
sc1 = sc1.with_mask(mask)
# Set the scale for the purposes of the tests
props1 = Mask_and_Moments(sc1, scale=0.003031065017916262 * u.Unit(""))
props1.make_mask(mask=mask)
props1.make_moments()
props1.make_moment_errors()

dataset1 = props1.to_dict()

moment0_hdu1 = fits.PrimaryHDU(dataset1["moment0"][0],
                               header=dataset1["moment0"][1])

moment0_proj = Projection.from_hdu(moment0_hdu1)

##############################################################################

path2 = os.path.join(turb_path, "data/dataset2.npz")

dataset2 = np.load(path2)

cube2 = np.empty((500, 32, 32))

count = 0
for posn, kept in zip(*dataset2["channels"]):
    posn = int(posn)
    if kept:
        cube2[posn, :, :] = dataset2["cube"][count, :, :]
        count += 1
    else:
        cube2[posn, :, :] = ra.normal(0.005, 0.005, (32, 32))

sc2 = SpectralCube(data=cube2, wcs=WCS(header))
mask = LazyMask(np.isfinite, sc2)
sc2 = sc2.with_mask(mask)
# Set the scale for the purposes of the tests
props2 = Mask_and_Moments(sc2, scale=0.003029658694658428 * u.Unit(""))
props2.make_moments()
props2.make_moment_errors()

dataset2 = props2.to_dict()

##############################################################################

# Load in saved comparison data.
try:
    computed_data = np.load(os.path.join(turb_path, "data/checkVals.npz"))

    computed_distances = np.load(os.path.join(turb_path,
                                              "data/computed_distances.npz"))
except IOError:
    warnings.warn("No checkVals or computed_distances files.")

# Define ways to make 1D and 2D profiles (typically Gaussian for ease)
def twoD_gaussian(shape=(201, 201), x_std=10., y_std=10., theta=0, amp=1.,
                  bkg=0.):

    from astropy.modeling.models import Gaussian2D, Const2D

    centre = tuple([val // 2 for val in shape])

    mod = Gaussian2D(x_mean=centre[0], y_mean=centre[1],
                     x_stddev=x_std, y_stddev=y_std,
                     amplitude=amp, theta=theta) + \
        Const2D(amplitude=bkg)

    return mod

def generate_2D_array(shape=(201, 201), curve_type='gaussian', **kwargs):

    if curve_type == "gaussian":

        mod = twoD_gaussian(shape=shape, **kwargs)

    else:
        raise ValueError("curve_type must be 'gaussian'.")

    ygrid, xgrid = np.mgrid[:shape[0], :shape[1]]

    return mod(xgrid, ygrid)

def oneD_gaussian(shape=200, mean=0., std=10., amp=1., bkg=0.):

    from astropy.modeling.models import Gaussian1D, Const1D

    mod = Gaussian1D(mean=mean, stddev=std, amplitude=amp) + \
        Const1D(amplitude=bkg)

    return mod

def generate_1D_array(shape=200, curve_type='gaussian', **kwargs):

    if curve_type == "gaussian":

        mod = oneD_gaussian(shape=shape, **kwargs)

    else:
        raise ValueError("curve_type must be 'gaussian'.")

    xgrid = np.arange(shape)

    return mod(xgrid)

def assert_between(value, lower, upper):
    '''
    Check if a value is between two values.
    '''

    within_lower = value >= lower
    within_upper = value <= upper

    if within_lower and within_upper:
        return
    else:
        raise AssertionError("{0} not within {1} and {2}".format(value, lower,
                                                                 upper))

def make_extended(imsize, imsize2=None, powerlaw=2.0, theta=0., ellip=1.):
    '''
    Adapted from https://github.com/keflavich/image_registration. Added ability
    to make the power spectra elliptical.

    Parameters
    ----------
    imsize : int
        Array size.
    imsize2 : int, optional
        Array size in 2nd dimension.
    powerlaw : float, optional
        Powerlaw index.
    theta : float, optional
        Position angle of major axis in radians. Has no effect when ellip==1.
    ellip : float, optional
        Ratio of the minor to major axis. Must be > 0 and <= 1. Defaults to
        the circular case (ellip=1).

    Returns
    -------
    newmap : np.ndarray
        Two-dimensional array with the given power-law properties.
    '''
    imsize = int(imsize)
    if imsize2 is None:
        imsize2 = imsize

    if ellip > 1 or ellip <= 0:
        raise ValueError("ellip must be > 0 and <= 1.")

    yy, xx = np.indices((imsize2, imsize), dtype='float')
    xcen = imsize / 2. - (1. - imsize % 2)
    ycen = imsize2 / 2. - (1. - imsize2 % 2)
    yy -= ycen
    xx -= xcen

    if ellip < 1:
        # Apply a rotation and scale the x-axis (ellip).
        costheta = np.cos(theta)
        sintheta = np.sin(theta)

        rr2 = xx**2 * (ellip**2 * costheta**2 + sintheta**2) + \
            (1 - ellip**2) * 2 * xx * yy * sintheta * costheta + \
            yy**2 * (ellip**2 * sintheta**2 + sintheta**2)

        rr = rr2**0.5
    else:
        # Circular whenever ellip == 1
        rr = (xx**2 + yy**2)**0.5

    # flag out the bad point to avoid warnings
    rr[rr == 0] = np.nan

    powermap = (np.random.randn(imsize2, imsize) * rr**(-powerlaw) +
                np.random.randn(imsize2, imsize) * rr**(-powerlaw) * 1j)
    powermap[powermap != powermap] = 0

    newmap = np.abs(np.fft.fftshift(np.fft.fft2(powermap)))

    return newmap
<|endoftext|>


<commit_before>from twisted.plugin import IPlugin
from twisted.words.protocols import irc
from txircd.config import ConfigValidationError
from txircd.module_interface import Command, ICommand, IModuleData, ModuleData
from txircd.utils import durationToSeconds, ipAddressToShow, ircLower, now
from zope.interface import implementer
from datetime import datetime, timedelta
from typing import Any, Callable, Dict, List, Optional, Tuple

irc.RPL_WHOWASIP = "379"

@implementer(IPlugin, IModuleData, ICommand)
class WhowasCommand(ModuleData, Command):
	name = "WhowasCommand"
	core = True

	def actions(self) -> List[Tuple[str, int, Callable]]:
		return [ ("quit", 10, self.addUserToWhowas),
		         ("remotequit", 10, self.addUserToWhowas),
		         ("localquit", 10, self.addUserToWhowas) ]

	def userCommands(self) -> List[Tuple[str, int, Command]]:
		return [ ("WHOWAS", 1, self) ]

	def load(self) -> None:
		if "whowas" not in self.ircd.storage:
			self.ircd.storage["whowas"] = {}

	def verifyConfig(self, config: Dict[str, Any]) -> None:
		if "whowas_duration" in config and not isinstance(config["whowas_duration"], str) and not isinstance(config["whowas_duration"], int):
			raise ConfigValidationError("whowas_duration", "value must be an integer or a duration string")
		if "whowas_max_entries" in config and (not isinstance(config["whowas_max_entries"], int) or config["whowas_max_entries"] < 0):
			raise  ConfigValidationError("whowas_max_entries", "invalid number")

	def removeOldEntries(self, whowasEntries: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
		expireDuration = durationToSeconds(self.ircd.config.get("whowas_duration", "1d"))
		maxCount = self.ircd.config.get("whowas_max_entries", 10)
		while whowasEntries and len(whowasEntries) > maxCount:
			whowasEntries.pop(0)
		expireDifference = timedelta(seconds=expireDuration)
		expireTime = now() - expireDifference
		while whowasEntries and whowasEntries[0]["when"] < expireTime:
			whowasEntries.pop(0)
		return whowasEntries

	def addUserToWhowas(self, user: "IRCUser", reason: str, fromServer: "IRCServer" = None) -> None:
		if not user.isRegistered():
			# user never registered a nick, so no whowas entry to add
			return
		lowerNick = ircLower(user.nick)
		allWhowas = self.ircd.storage["whowas"]
		if lowerNick in allWhowas:
			whowasEntries = allWhowas[lowerNick]
		else:
			whowasEntries = []
		serverName = self.ircd.name
		if user.uuid[:3] != self.ircd.serverID:
			serverName = self.ircd.servers[user.uuid[:3]].name
		whowasEntries.append({
			"nick": user.nick,
			"ident": user.ident,
			"host": user.host(),
			"realhost": user.realHost,
			"ip": ipAddressToShow(user.ip),
			"gecos": user.gecos,
			"server": serverName,
			"when": now()
		})
		whowasEntries = self.removeOldEntries(whowasEntries)
		if whowasEntries:
			allWhowas[lowerNick] = whowasEntries
		elif lowerNick in allWhowas:
			del allWhowas[lowerNick]

	def parseParams(self, user: "IRCUser", params: List[str], prefix: str, tags: Dict[str, Optional[str]]) -> Optional[Dict[Any, Any]]:
		if not params:
			user.sendSingleError("WhowasCmd", irc.ERR_NEEDMOREPARAMS, "WHOWAS", "Not enough parameters")
			return None
		lowerParam = ircLower(params[0])
		if lowerParam not in self.ircd.storage["whowas"]:
			user.sendSingleError("WhowasNick", irc.ERR_WASNOSUCHNICK, params[0], "There was no such nickname")
			return None
		return {
			"nick": lowerParam,
			"param": params[0]
		}

	def execute(self, user: "IRCUser", data: Dict[Any, Any]) -> bool:
		nick = data["nick"]
		allWhowas = self.ircd.storage["whowas"]
		whowasEntries = allWhowas[nick]
		whowasEntries = self.removeOldEntries(whowasEntries)
		if not whowasEntries:
			del allWhowas[nick]
			self.ircd.storage["whowas"] = allWhowas
			user.sendMessage(irc.ERR_WASNOSUCHNICK, data["param"], "There was no such nickname")
			return True
		allWhowas[nick] = whowasEntries # Save back to the list excluding the removed entries
		self.ircd.storage["whowas"] = allWhowas
		for entry in whowasEntries:
			entryNick = entry["nick"]
			user.sendMessage(irc.RPL_WHOWASUSER, entryNick, entry["ident"], entry["host"], "*", entry["gecos"])
			if self.ircd.runActionUntilValue("userhasoperpermission", user, "whowas-host", users=[user]):
				user.sendMessage(irc.RPL_WHOWASIP, entryNick, "was connecting from {}@{} {}".format(entry["ident"], entry["realhost"], entry["ip"]))
			user.sendMessage(irc.RPL_WHOISSERVER, entryNick, entry["server"], str(datetime.utcfromtimestamp(entry["when"])))
		user.sendMessage(irc.RPL_ENDOFWHOWAS, nick, "End of WHOWAS")
		return True

whowasCmd = WhowasCommand()<commit_msg>We've started saving datetimes instead of timestamps for whowas data, so let's read them like that as well<commit_after>from twisted.plugin import IPlugin
from twisted.words.protocols import irc
from txircd.config import ConfigValidationError
from txircd.module_interface import Command, ICommand, IModuleData, ModuleData
from txircd.utils import durationToSeconds, ipAddressToShow, ircLower, now
from zope.interface import implementer
from datetime import datetime, timedelta
from typing import Any, Callable, Dict, List, Optional, Tuple

irc.RPL_WHOWASIP = "379"

@implementer(IPlugin, IModuleData, ICommand)
class WhowasCommand(ModuleData, Command):
	name = "WhowasCommand"
	core = True

	def actions(self) -> List[Tuple[str, int, Callable]]:
		return [ ("quit", 10, self.addUserToWhowas),
		         ("remotequit", 10, self.addUserToWhowas),
		         ("localquit", 10, self.addUserToWhowas) ]

	def userCommands(self) -> List[Tuple[str, int, Command]]:
		return [ ("WHOWAS", 1, self) ]

	def load(self) -> None:
		if "whowas" not in self.ircd.storage:
			self.ircd.storage["whowas"] = {}

	def verifyConfig(self, config: Dict[str, Any]) -> None:
		if "whowas_duration" in config and not isinstance(config["whowas_duration"], str) and not isinstance(config["whowas_duration"], int):
			raise ConfigValidationError("whowas_duration", "value must be an integer or a duration string")
		if "whowas_max_entries" in config and (not isinstance(config["whowas_max_entries"], int) or config["whowas_max_entries"] < 0):
			raise  ConfigValidationError("whowas_max_entries", "invalid number")

	def removeOldEntries(self, whowasEntries: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
		expireDuration = durationToSeconds(self.ircd.config.get("whowas_duration", "1d"))
		maxCount = self.ircd.config.get("whowas_max_entries", 10)
		while whowasEntries and len(whowasEntries) > maxCount:
			whowasEntries.pop(0)
		expireDifference = timedelta(seconds=expireDuration)
		expireTime = now() - expireDifference
		while whowasEntries and whowasEntries[0]["when"] < expireTime:
			whowasEntries.pop(0)
		return whowasEntries

	def addUserToWhowas(self, user: "IRCUser", reason: str, fromServer: "IRCServer" = None) -> None:
		if not user.isRegistered():
			# user never registered a nick, so no whowas entry to add
			return
		lowerNick = ircLower(user.nick)
		allWhowas = self.ircd.storage["whowas"]
		if lowerNick in allWhowas:
			whowasEntries = allWhowas[lowerNick]
		else:
			whowasEntries = []
		serverName = self.ircd.name
		if user.uuid[:3] != self.ircd.serverID:
			serverName = self.ircd.servers[user.uuid[:3]].name
		whowasEntries.append({
			"nick": user.nick,
			"ident": user.ident,
			"host": user.host(),
			"realhost": user.realHost,
			"ip": ipAddressToShow(user.ip),
			"gecos": user.gecos,
			"server": serverName,
			"when": now()
		})
		whowasEntries = self.removeOldEntries(whowasEntries)
		if whowasEntries:
			allWhowas[lowerNick] = whowasEntries
		elif lowerNick in allWhowas:
			del allWhowas[lowerNick]

	def parseParams(self, user: "IRCUser", params: List[str], prefix: str, tags: Dict[str, Optional[str]]) -> Optional[Dict[Any, Any]]:
		if not params:
			user.sendSingleError("WhowasCmd", irc.ERR_NEEDMOREPARAMS, "WHOWAS", "Not enough parameters")
			return None
		lowerParam = ircLower(params[0])
		if lowerParam not in self.ircd.storage["whowas"]:
			user.sendSingleError("WhowasNick", irc.ERR_WASNOSUCHNICK, params[0], "There was no such nickname")
			return None
		return {
			"nick": lowerParam,
			"param": params[0]
		}

	def execute(self, user: "IRCUser", data: Dict[Any, Any]) -> bool:
		nick = data["nick"]
		allWhowas = self.ircd.storage["whowas"]
		whowasEntries = allWhowas[nick]
		whowasEntries = self.removeOldEntries(whowasEntries)
		if not whowasEntries:
			del allWhowas[nick]
			self.ircd.storage["whowas"] = allWhowas
			user.sendMessage(irc.ERR_WASNOSUCHNICK, data["param"], "There was no such nickname")
			return True
		allWhowas[nick] = whowasEntries # Save back to the list excluding the removed entries
		self.ircd.storage["whowas"] = allWhowas
		for entry in whowasEntries:
			entryNick = entry["nick"]
			user.sendMessage(irc.RPL_WHOWASUSER, entryNick, entry["ident"], entry["host"], "*", entry["gecos"])
			if self.ircd.runActionUntilValue("userhasoperpermission", user, "whowas-host", users=[user]):
				user.sendMessage(irc.RPL_WHOWASIP, entryNick, "was connecting from {}@{} {}".format(entry["ident"], entry["realhost"], entry["ip"]))
			user.sendMessage(irc.RPL_WHOISSERVER, entryNick, entry["server"], str(entry["when"]))
		user.sendMessage(irc.RPL_ENDOFWHOWAS, nick, "End of WHOWAS")
		return True

whowasCmd = WhowasCommand()<|endoftext|>


<commit_before>#
# Copyright (c) SAS Institute, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
Module for interacting with packages defined by pom files
"""

from functools import wraps
import logging

from lxml import objectify
import artifactory

from updatebot.pkgsource.common import BasePackageSource

XMLParser = objectify.makeparser(recover=True, remove_comments=True)

log = logging.getLogger('updatebot.pkgsource')

def loaded(func):
    @wraps(func)
    def wrapper(other, *args, **kwargs):
        if other._loaded:
            return
        return func(other, *args, **kwargs)
    return wrapper

class Package(object):
    __slots__ = ('_pom', '_json', 'arch', 'fullVersion', 'name',
                 'epoch', 'version', 'release', 'location')

    def __init__(self, pom, arch, json=None):
        self._json = json
        self._pom = pom
        self.epoch = '1'
        try:
            self.fullVersion = str(pom.version).strip()
        except AttributeError:
            log.debug("No version set, trying parent project")
            self.fullVersion = str(pom.parent.version).strip()
        self.name = str(pom.artifactId)
        self.arch = arch and arch or ''
        head, _, tail = self.fullVersion.rpartition('-')
        if head and tail:
            self.version = head
            self.release = tail
        elif tail:
            self.version = tail
            self.release = ''
        else:
            raise ValueError("Could not parse maven version into"
                             " version and release")
        self.location = json['downloadUri'] if json else None

    def __repr__(self):
        return '<pomsource.Package(%r, %r, %r)>' % (
            self.name, self.version, self.arch)

    def getConaryVersion(self):
        assert self.arch == 'src'
        return self.fullVersion.replace('-', '_')

    def getBuildRequires(self):
        build_requires = []
        if hasattr(self._pom, 'parent'):
            # add parent projects as a build req
            build_requires.append(str(self._pom.parent.artifactId))

        if not hasattr(self._pom, 'dependencies'):
            if not hasattr(self._pom, 'dependencyManagement'):
                return build_requires
            dependencies = self._pom.dependencyManagement.dependencies
        else:
            dependencies = self._pom.dependencies

        for dep in dependencies.iterchildren():
            if hasattr(dep, 'optional') and dep.optional:
                # ignore opitonal deps, these are deps a dependent
                # project may want to include
                continue

            if hasattr(dep, 'scope') and str(dep.scope) != 'compile':
                # we only want compile deps (the default)
                continue

            build_requires.append(str(dep.artifactId))

        return build_requires

    def getNevra(self):
        return self.name, self.epoch, self.version, self.release, self.arch

class PomSource(BasePackageSource):
    """
    Creates a map of packages from a pom file
    """

    def __init__(self, cfg, ui):
        BasePackageSource.__init__(self, cfg, ui)

    @loaded
    def load(self):
        auth = self._cfg.artifactoryUser.find(self._cfg.repositoryUrl)

        for repo in self._cfg.repositoryPaths:
            log.info('loading repository data %s' % repo)
            client = artifactory.Client(self._cfg.repositoryUrl, auth)
            archStr = self._cfg.repositoryArch.get(repo, None)
            self.loadFromClient(client, repo, archStr=archStr)

        self._loaded = True

    def loadFromClient(self, client, repo, archStr=None):
        poms = [pom for pom in client.quick_search('pom', repos=repo)
                if pom.get('mimeType') == 'application/x-maven-pom+xml']
        artifacts = client.retrieve_artifact(
            '%(repo)s:%(path)s' % p for p in poms)

        for pom, artifact in zip(poms, artifacts):
            pomObj = objectify.fromstring(artifact.text.encode('utf-8'),
                                          XMLParser)
            if hasattr(pomObj, 'getroot'):
                pomObj = pomObj.getroot()

            artifacts = client.gavc_search(
                group=(str(pomObj.groupId) if hasattr(pomObj, 'groupId') else
                       str(pomObj.parent.groupId)),
                artifact=str(pomObj.artifactId),
                version=(str(pomObj.version) if hasattr(pomObj, 'version') else
                         str(pomObj.parent.version)),
            )
            if not artifacts:
                log.info("No artifacts associated with this pom:"
                          " %(repo)s:%(path)s" % pom)

            srcPkg = Package(pomObj, 'src')
            binPkgs = [Package(pomObj, archStr, a) for a in artifacts]

            self.srcNameMap.setdefault(srcPkg.name, set()).add(srcPkg)
            self.locationMap[srcPkg.location] = srcPkg
            self.srcPkgMap[srcPkg] = binPkgs

            trvSpec = (srcPkg.name, srcPkg.getConaryVersion(), ('x86_64',))
            self.useMap.setdefault(trvSpec, set()).update(set(['x86_64']))

            for binPkg in binPkgs:
                self.binNameMap.setdefault(binPkg.name, set()).add(binPkg)
                self.locationMap[binPkg.location] = binPkg
                self.binPkgMap[binPkg] = srcPkg
<commit_msg>Do not process dependencyManagemnt nodes<commit_after>#
# Copyright (c) SAS Institute, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
Module for interacting with packages defined by pom files
"""

from functools import wraps
import logging

from lxml import objectify
import artifactory

from updatebot.pkgsource.common import BasePackageSource

XMLParser = objectify.makeparser(recover=True, remove_comments=True)

log = logging.getLogger('updatebot.pkgsource')

def loaded(func):
    @wraps(func)
    def wrapper(other, *args, **kwargs):
        if other._loaded:
            return
        return func(other, *args, **kwargs)
    return wrapper

class Package(object):
    __slots__ = ('_pom', '_json', 'arch', 'fullVersion', 'name',
                 'epoch', 'version', 'release', 'location')

    def __init__(self, pom, arch, json=None):
        self._json = json
        self._pom = pom
        self.epoch = '1'
        try:
            self.fullVersion = str(pom.version).strip()
        except AttributeError:
            log.debug("No version set, trying parent project")
            self.fullVersion = str(pom.parent.version).strip()
        self.name = str(pom.artifactId)
        self.arch = arch and arch or ''
        head, _, tail = self.fullVersion.rpartition('-')
        if head and tail:
            self.version = head
            self.release = tail
        elif tail:
            self.version = tail
            self.release = ''
        else:
            raise ValueError("Could not parse maven version into"
                             " version and release")
        self.location = json['downloadUri'] if json else None

    def __repr__(self):
        return '<pomsource.Package(%r, %r, %r)>' % (
            self.name, self.version, self.arch)

    def getConaryVersion(self):
        assert self.arch == 'src'
        return self.fullVersion.replace('-', '_')

    def getBuildRequires(self):
        buildRequires = set()
        if hasattr(self._pom, 'parent'):
            # add parent projects as a build req
            buildRequires.add(str(self._pom.parent.artifactId))

        if hasattr(self._pom, 'dependencies'):
            for dep in self._pom.dependencies.iterchildren():
                if hasattr(dep, 'optional') and dep.optional:
                    # ignore opitonal deps, these are deps a dependent
                    # project may want to include
                    continue

                if hasattr(dep, 'scope') and str(dep.scope) != 'compile':
                    # we only want compile deps (the default)
                    continue

                buildRequires.add(str(dep.artifactId))

        return list(buildRequires)

    def getNevra(self):
        return self.name, self.epoch, self.version, self.release, self.arch

class PomSource(BasePackageSource):
    """
    Creates a map of packages from a pom file
    """

    def __init__(self, cfg, ui):
        BasePackageSource.__init__(self, cfg, ui)

    @loaded
    def load(self):
        auth = self._cfg.artifactoryUser.find(self._cfg.repositoryUrl)

        for repo in self._cfg.repositoryPaths:
            log.info('loading repository data %s' % repo)
            client = artifactory.Client(self._cfg.repositoryUrl, auth)
            archStr = self._cfg.repositoryArch.get(repo, None)
            self.loadFromClient(client, repo, archStr=archStr)

        self._loaded = True

    def loadFromClient(self, client, repo, archStr=None):
        poms = [pom for pom in client.quick_search('pom', repos=repo)
                if pom.get('mimeType') == 'application/x-maven-pom+xml']
        artifacts = client.retrieve_artifact(
            '%(repo)s:%(path)s' % p for p in poms)

        for pom, artifact in zip(poms, artifacts):
            pomObj = objectify.fromstring(artifact.text.encode('utf-8'),
                                          XMLParser)
            if hasattr(pomObj, 'getroot'):
                pomObj = pomObj.getroot()

            artifacts = client.gavc_search(
                group=(str(pomObj.groupId) if hasattr(pomObj, 'groupId') else
                       str(pomObj.parent.groupId)),
                artifact=str(pomObj.artifactId),
                version=(str(pomObj.version) if hasattr(pomObj, 'version') else
                         str(pomObj.parent.version)),
            )
            if not artifacts:
                log.info("No artifacts associated with this pom:"
                          " %(repo)s:%(path)s" % pom)

            srcPkg = Package(pomObj, 'src')
            binPkgs = [Package(pomObj, archStr, a) for a in artifacts]

            self.srcNameMap.setdefault(srcPkg.name, set()).add(srcPkg)
            self.locationMap[srcPkg.location] = srcPkg
            self.srcPkgMap[srcPkg] = binPkgs

            trvSpec = (srcPkg.name, srcPkg.getConaryVersion(), ('x86_64',))
            self.useMap.setdefault(trvSpec, set()).update(set(['x86_64']))

            for binPkg in binPkgs:
                self.binNameMap.setdefault(binPkg.name, set()).add(binPkg)
                self.locationMap[binPkg.location] = binPkg
                self.binPkgMap[binPkg] = srcPkg
<|endoftext|>


<commit_before>import discord
from discord import Forbidden
from discord.ext import commands
import random
from dict import DictionaryReader
from botkey import Key
from subprocess import call
import sys
from priestLogger import PriestLogger
import logging

logging.basicConfig(level=logging.INFO)

description = '''I'm PriestBot, your robot friend for links and quick info!

Below you'll find my basic commands.

You can find my full list of commands at https://github.com/lgkern/discord.py/blob/async/examples/dictEntries.txt'''
bot = commands.Bot(command_prefix='!', description=description)

client = discord.Client()

prefix = Key().prefix()

logger = PriestLogger

@client.event
async def on_ready():
    print('Logged in as')
    print(client.user.name)
    print(client.user.id)
    print('------')

@client.event
async def on_message(message):
	r = DictionaryReader()
    # we do not want the bot to reply to itself
	if message.author == client.user:
		return

	if message.channel.name in r.logChannels():
		logger.log(message)
	if message.content.startswith(prefix):
		await messageHandler(message)

@client.event
async def on_member_join(member):
	await sendWelcomeMessage(member)

async def messageHandler(message):
	try:
		await client.send_message(client.get_channel('220534135947526154'), '{0.server.name} - {0.channel.name} - {0.author} invoked {0.content}'.format(message))
	except:
		try:
			await client.send_message(client.get_channel('220534135947526154'), 'PM - PM - {0.author} invoked {0.content}'.format(message))
		except:
			print('error while writing log')

	if message.content.startswith(prefix+'fullupdate') or message.content.startswith(prefix+'update'):
		await maintenanceMessages(message)

	elif message.content.startswith(prefix+'send'):
		await forwardMessage(message)

	elif message.content.startswith(prefix+'item'):
		await itemMessage(message)

	elif message.content.startswith(prefix+'pin') or message.content.startswith(prefix+'pins'):
		await sendPinMessages(message)

	elif message.content.startswith(prefix+'channel'):
		await client.send_message(message.channel, message.channel.id)

	else:
		await generalMessage(message)

async def maintenanceMessages(message):
	p = DictionaryReader()
	if message.content.startswith(prefix+'fullupdate'):
		if message.author.id not in p.admins():
			await client.send_message(message.channel, 'You\'re not my dad, {0.mention}!'.format(message.author))
			return
		call(["git","pull"])
		call(["cmdhere.bat"])
		sys.exit()
	elif message.content.startswith('!update'):
		call(["git","pull"])

async def forwardMessage(message):
	p = DictionaryReader()
	roles = message.author.roles
	canSend = False
	for role in roles:
		canSend = canSend or (role.name in p.roles())
	if not canSend:
		print('{0.author.name} can\'t send whispers'.format(message))
		return
	entries = message.content.split(' ')
	target = message.mentions[0]
	if target != None:
		entry = ' '.join(entries[2::])
		msg = p.commandReader(entry)
		if msg != None:
			await client.send_message(target, msg)
			await client.delete_message(message)
			await client.send_message(message.author, 'Message sent to {0.mention}'.format(target))
		else:
			await client.send_message(message.channel, 'Invalid Message, {0.mention}'.format(message.author))

async def itemMessage(message):
	p = DictionaryReader()
	msg = p.itemReader(message.content[1::])
	await client.send_message(message.channel, msg)

async def sendWelcomeMessage(member):
	p = DictionaryReader()
	msg = p.commandReader('help')
	await client.send_message(member, msg))

async def sendPinMessages(message):
	pins = await client.pins_from(message.channel)
	size = 10
	count = 0
	command = message.content.split(' ')
	if len(command) > 1:
		size = command[1]
		if not isinstance(size, int):
			size = 10
	for msg in pins:
		if count > size:
			return
		await client.send_message(message.author, msg.content)
		count += 1

	try:
		await client.delete_message(message)
	except Exception:
		print('Error deleting message, probably from whisper')

async def generalMessage(message):
	p = DictionaryReader()
	try:
		roles = message.author.roles
	except Exception:
		roles = 10
	command = message.content[1::].split(' ')[0].lower()
	msg = p.commandReader(message.content[1::])
	if msg != None:
		if command in p.whisperCommands():
			if command == 'pub' and len(roles) > 1 and 'help' not in message.content:
				await client.send_message(message.channel, msg)
			else:
				await client.send_message(message.author, msg)
				try:
					await client.delete_message(message)
				except Exception:
					print('Error deleting message, probably from whisper')
		else:
			await client.send_message(message.channel, msg)

#client.run('MjI4NTQzMzIwNzE5MDk3ODU2.CsWPFw.Hz3E67ykKwVRt6Dy14qnEJUiSUw')
client.run(Key().value())
#bot.run(Key().value())

<commit_msg>Fixes<commit_after>import discord
from discord import Forbidden
from discord.ext import commands
import random
from dict import DictionaryReader
from botkey import Key
from subprocess import call
import sys
from priestLogger import PriestLogger
import logging

logging.basicConfig(level=logging.INFO)

description = '''I'm PriestBot, your robot friend for links and quick info!

Below you'll find my basic commands.

You can find my full list of commands at https://github.com/lgkern/discord.py/blob/async/examples/dictEntries.txt'''
bot = commands.Bot(command_prefix='!', description=description)

client = discord.Client()

prefix = Key().prefix()

logger = PriestLogger

@client.event
async def on_ready():
    print('Logged in as')
    print(client.user.name)
    print(client.user.id)
    print('------')

@client.event
async def on_message(message):
	r = DictionaryReader()
    # we do not want the bot to reply to itself
	if message.author == client.user:
		return

	if message.channel.name in r.logChannels():
		logger.log(message)

	if message.content.startswith(prefix):
		await messageHandler(message)

@client.event
async def on_member_join(member):
	await sendWelcomeMessage(member)

async def messageHandler(message):
	try:
		await client.send_message(client.get_channel('220534135947526154'), '{0.server.name} - {0.channel.name} - {0.author} invoked {0.content}'.format(message))
	except:
		try:
			await client.send_message(client.get_channel('220534135947526154'), 'PM - PM - {0.author} invoked {0.content}'.format(message))
		except:
			print('error while writing log')

	if message.content.startswith(prefix+'fullupdate') or message.content.startswith(prefix+'update'):
		await maintenanceMessages(message)

	elif message.content.startswith(prefix+'send'):
		await forwardMessage(message)

	elif message.content.startswith(prefix+'item'):
		await itemMessage(message)

	elif message.content.startswith(prefix+'pin') or message.content.startswith(prefix+'pins'):
		await sendPinMessages(message)

	elif message.content.startswith(prefix+'channel'):
		await client.send_message(message.channel, message.channel.id)

	else:
		await generalMessage(message)

async def maintenanceMessages(message):
	p = DictionaryReader()
	if message.content.startswith(prefix+'fullupdate'):
		if message.author.id not in p.admins():
			await client.send_message(message.channel, 'You\'re not my dad, {0.mention}!'.format(message.author))
			return
		call(["git","pull"])
		call(["cmdhere.bat"])
		sys.exit()
	elif message.content.startswith('!update'):
		call(["git","pull"])

async def forwardMessage(message):
	p = DictionaryReader()
	roles = message.author.roles
	canSend = False
	for role in roles:
		canSend = canSend or (role.name in p.roles())
	if not canSend:
		print('{0.author.name} can\'t send whispers'.format(message))
		return
	entries = message.content.split(' ')
	target = message.mentions[0]
	if target != None:
		entry = ' '.join(entries[2::])
		msg = p.commandReader(entry)
		if msg != None:
			await client.send_message(target, msg)
			await client.delete_message(message)
			await client.send_message(message.author, 'Message sent to {0.mention}'.format(target))
		else:
			await client.send_message(message.channel, 'Invalid Message, {0.mention}'.format(message.author))

async def itemMessage(message):
	p = DictionaryReader()
	msg = p.itemReader(message.content[1::])
	await client.send_message(message.channel, msg)

async def sendWelcomeMessage(member):
	p = DictionaryReader()
	msg = p.commandReader('help')
	await client.send_message(member, msg)

async def sendPinMessages(message):
	pins = await client.pins_from(message.channel)
	size = 10
	count = 0
	command = message.content.split(' ')
	if len(command) > 1:
		size = command[1]
		if not isinstance(size, int):
			size = 10
	for msg in pins:
		if count > size:
			return
		await client.send_message(message.author, msg.content)
		count += 1

	try:
		await client.delete_message(message)
	except Exception:
		print('Error deleting message, probably from whisper')

async def generalMessage(message):
	p = DictionaryReader()
	try:
		roles = message.author.roles
	except Exception:
		roles = 10
	command = message.content[1::].split(' ')[0].lower()
	msg = p.commandReader(message.content[1::])
	if msg != None:
		if command in p.whisperCommands():
			if command == 'pub' and len(roles) > 1 and 'help' not in message.content:
				await client.send_message(message.channel, msg)
			else:
				await client.send_message(message.author, msg)
				try:
					await client.delete_message(message)
				except Exception:
					print('Error deleting message, probably from whisper')
		else:
			await client.send_message(message.channel, msg)

#client.run('MjI4NTQzMzIwNzE5MDk3ODU2.CsWPFw.Hz3E67ykKwVRt6Dy14qnEJUiSUw')
client.run(Key().value())
#bot.run(Key().value())

<|endoftext|>


<commit_before>from celery import Celery
from celery.schedules import crontab
from kombu import Exchange, Queue
from upol_search_engine import settings

class Config(object):
    broker_url = 'amqp://guest:guest@localhost:5672//'
    result_backend = 'amqp://guest:guest@localhost:5672//'

    task_queues = (
        Queue(
            'crawler',
            exchange=Exchange('crawler'),
            routing_key='crawler'
        ),
        Queue(
            'search_engine',
            exchange=Exchange('search_engine'),
            routing_key='search_engine'
        ),
    )

    enable_utc = False
    timezone = 'Europe/Prague'
    include = ['upol_search_engine.upol_crawler.tasks',
               'upol_search_engine.tasks']

    log_file = settings.CONFIG.get('Settings', 'log_dir')
    task_acks_late = True

    beat_schedule = {
        'run-search-engine': {
            'task': 'upol_search_engine.tasks.main_task',
            'schedule': 20,
        }
    }

app = Celery('celery_app')
app.config_from_object(Config)

if __name__ == '__main__':
    app.start()
<commit_msg>Schedule<commit_after>from celery import Celery
from celery.schedules import crontab
from kombu import Exchange, Queue
from upol_search_engine import settings

class Config(object):
    broker_url = 'amqp://guest:guest@localhost:5672//'
    result_backend = 'amqp://guest:guest@localhost:5672//'

    task_queues = (
        Queue(
            'crawler',
            exchange=Exchange('crawler'),
            routing_key='crawler'
        ),
        Queue(
            'search_engine',
            exchange=Exchange('search_engine'),
            routing_key='search_engine'
        ),
    )

    enable_utc = False
    timezone = 'Europe/Prague'
    include = ['upol_search_engine.upol_crawler.tasks',
               'upol_search_engine.tasks']

    log_file = settings.CONFIG.get('Settings', 'log_dir')
    task_acks_late = True

    beat_schedule = {
        'run-search-engine': {
            'task': 'upol_search_engine.tasks.main_task',
            'schedule': 60 * 60 + 60 * 10,
        }
    }

app = Celery('celery_app')
app.config_from_object(Config)

if __name__ == '__main__':
    app.start()
<|endoftext|>


<commit_before>from functools import reduce
from itertools import zip_longest, filterfalse
from re import search, findall
from nltk import ngrams, word_tokenize, pos_tag, FreqDist, ne_chunk, Tree
from nltk.corpus import opinion_lexicon
from nltk.corpus import wordnet, cmudict
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from nltk.stem import WordNetLemmatizer
from nltk.tokenize.treebank import TreebankWordTokenizer

# Ignore twython library missing, we aren't using it's functionality
# Must use nltk.download() and get the Opinion Lexicon and Vader Lexicon

PUNCTUATION_RE = "[\'\!\"\#\$\%\&\/\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\]\^\_\`\{\}\|\~\\u2026\\u2018\\u2019]"
TWEET_LINK_RE = "https://t.co/(\w)+"
TWEET_HANDLE_RE = "@(\w)+"

class nlp:
    lemma = WordNetLemmatizer()
    negWords = frozenset(opinion_lexicon.negative())
    posWords = frozenset(opinion_lexicon.positive())
    suffixes = list(line.rstrip() for line in open("suffixes.txt"))
    vader = SentimentIntensityAnalyzer()
    d = cmudict.dict()

    chunk = lambda self, posTagged: ne_chunk(posTagged, binary=True)
    freq = lambda self, grams: FreqDist(grams)
    lemmatize = lambda self, posTokens: [self.processPosTagsAndLemmatize(*wordPos) for wordPos in posTokens]
    ngrams = lambda self, tokens, n: list(ngrams(tokens, n))
    pos = lambda self, tokens: pos_tag(tokens)
    posTagOnly = lambda self, tagged: [tag for word, tag in tagged]
    tokenize = lambda self, text: word_tokenize(text)
    wordCases = lambda self, ulls: [self.wordCase(*ull) for ull in ulls]

    def capLetterFreq(self, ull):
        return reduce(lambda i, u: i + u[0], ull, 0) / reduce(lambda i, l: i + l[1], ull, 0)

    def processPosTagsAndLemmatize(self, word, pos):
        return self.lemma.lemmatize(word, self.treebankToWordnetPOS(pos))

    def removeNamedEntities(self, chunked, removeNumbers=True):
        def rec(t, r):
            if type(t) == Tree:
                if t.label() == 'NE':
                    r.append(('[NE]', 'NE'))
                else:
                    for child in t:
                        r.extend(rec(child, []))
            else:
                if removeNumbers:
                    r.append(('[CD]', 'CD')) if t[1] == "CD" else r.append(t)
                else:
                    r.append(t)
            return r
        return rec(chunked, [])

    def sentimentGrams(self, grams):
        return [{"LiuHu": self.sentimentLiuHu(gram), "Vader": self.sentimentVader(gram)} for gram in grams]

    def sentimentLiuHu(self, gram):
        posWord = lambda word: word in self.posWords
        negWord = lambda word: word in self.negWords
        countPosNeg = lambda pn, word: (pn[0] + 1,
                                        pn[1],
                                        pn[2]) if posWord(word) else (pn[0],
                                                                      pn[1] + 1,
                                                                      pn[2]) if negWord(word) else (pn[0],
                                                                                pn[1],
                                                                                pn[2] + 1)
        p, n, u = reduce(countPosNeg, gram, [0, 0, 0])
        l = p + n + u
        return {"compound": round((p - n) / l, 3),
                "neg": round(n / l, 4),
                "neu": round(u / l, 4),
                "pos": round(p / l, 4)} if l > 0 else {"compound": 0,
                                                       "neg": 0,
                                                       "neu": 0,
                                                       "pos": 0}

    def sentimentVader(self, gram):
        return self.vader.polarity_scores(" ".join(gram))

    def tokenSuffixes(self, tokens):
        return [max(s2, key=len) for s2 in
                [s1 for s1 in
                 [
                     [s0 for s0 in self.suffixes if word.endswith(s0)]
                     for word in tokens]
                 if s1]]

    def treebankToWordnetPOS(self, treebankPosTag):
        return {'J': wordnet.ADJ,
                'V': wordnet.VERB,
                'N': wordnet.NOUN,
                'R': wordnet.ADV}.get(treebankPosTag[0], wordnet.NOUN)

    def upperLowerLen(self, tokensOriginalCase):
        return [
            (
                sum([1 if letter.isupper() else 0 for letter in token]),
                sum([1 if letter.islower() else 0 for letter in token]),
                len(token),
                token
            )for token in tokensOriginalCase]

    def wordCase(self, upper, lower, length, token):
        if upper == 0:
            return "NC"  # No Caps
        elif upper == length:
            return "AC"  # All Caps
        elif upper == 1 and token[0].isupper():
            return "FC"  # First Cap
        else:
            return "MC"  # Mixed Caps

    def numSyllables(self, word):
        return [len(list(y for y in x if y[-1].isdigit()))for x in self.d[word.lower()]][0]

    def hasVowel(self, word):
        for char in word:
            if char in ['a','e','i','o','u']:
                return 1
        return 0

    def hasPunctuation(self, word):
        return search(PUNCTUATION_RE, word) is not None

    def inDict(self, word):
        try:
            self.d[word.lower()]
            return True
        except KeyError:
            return False

    def punctuationFeatures(self, s):
        """
        s: input string
        returns {punctuation_mark: (raw #, % of length of s, % of total # of punctuation marks found in s)}

        example:
        punctuation_features("Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed consequat magna eu facilisis!!?")
        {'!': (2, 0.0217, 0.4),
         ',': (1, 0.0109, 0.2),
         '.': (1, 0.0109, 0.2),
         '?': (1, 0.0109, 0.2)}
         """

        punctuation_found_list = findall(PUNCTUATION_RE, s)
        return {p: (punctuation_found_list.count(p),
                    round(punctuation_found_list.count(p)/len(s), 4),
                    round(punctuation_found_list.count(p)/len(punctuation_found_list), 4)) for p in punctuation_found_list}
<commit_msg>Fix letter case bug<commit_after>from functools import reduce
from itertools import zip_longest, filterfalse
from re import search, findall
from nltk import ngrams, word_tokenize, pos_tag, FreqDist, ne_chunk, Tree
from nltk.corpus import opinion_lexicon
from nltk.corpus import wordnet, cmudict
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from nltk.stem import WordNetLemmatizer
from nltk.tokenize.treebank import TreebankWordTokenizer

# Ignore twython library missing, we aren't using it's functionality
# Must use nltk.download() and get the Opinion Lexicon and Vader Lexicon

PUNCTUATION_RE = "[\'\!\"\#\$\%\&\/\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\]\^\_\`\{\}\|\~\\u2026\\u2018\\u2019]"
TWEET_LINK_RE = "https://t.co/(\w)+"
TWEET_HANDLE_RE = "@(\w)+"

class nlp:
    lemma = WordNetLemmatizer()
    negWords = frozenset(opinion_lexicon.negative())
    posWords = frozenset(opinion_lexicon.positive())
    suffixes = list(line.rstrip() for line in open("suffixes.txt"))
    vader = SentimentIntensityAnalyzer()
    d = cmudict.dict()

    chunk = lambda self, posTagged: ne_chunk(posTagged, binary=True)
    freq = lambda self, grams: FreqDist(grams)
    lemmatize = lambda self, posTokens: [self.processPosTagsAndLemmatize(*wordPos) for wordPos in posTokens]
    ngrams = lambda self, tokens, n: list(ngrams(tokens, n))
    pos = lambda self, tokens: pos_tag(tokens)
    posTagOnly = lambda self, tagged: [tag for word, tag in tagged]
    tokenize = lambda self, text: word_tokenize(text)
    wordCases = lambda self, ulls: [self.wordCase(*ull) for ull in ulls]

    def capLetterFreq(self, ull):
        return reduce(lambda i, u: i + u[0], ull, 0) / reduce(lambda i, l: i + l[1], ull, 0)

    def processPosTagsAndLemmatize(self, word, pos):
        return self.lemma.lemmatize(word, self.treebankToWordnetPOS(pos))

    def removeNamedEntities(self, chunked, removeNumbers=True):
        def rec(t, r):
            if type(t) == Tree:
                if t.label() == 'NE':
                    r.append(('[NE]', 'NE'))
                else:
                    for child in t:
                        r.extend(rec(child, []))
            else:
                if removeNumbers:
                    r.append(('[CD]', 'CD')) if t[1] == "CD" else r.append(t)
                else:
                    r.append(t)
            return r
        return rec(chunked, [])

    def sentimentGrams(self, grams):
        return [{"LiuHu": self.sentimentLiuHu(gram), "Vader": self.sentimentVader(gram)} for gram in grams]

    def sentimentLiuHu(self, gram):
        posWord = lambda word: word in self.posWords
        negWord = lambda word: word in self.negWords
        countPosNeg = lambda pn, word: (pn[0] + 1,
                                        pn[1],
                                        pn[2]) if posWord(word) else (pn[0],
                                                                      pn[1] + 1,
                                                                      pn[2]) if negWord(word) else (pn[0],
                                                                                pn[1],
                                                                                pn[2] + 1)
        p, n, u = reduce(countPosNeg, gram, [0, 0, 0])
        l = p + n + u
        return {"compound": round((p - n) / l, 3),
                "neg": round(n / l, 4),
                "neu": round(u / l, 4),
                "pos": round(p / l, 4)} if l > 0 else {"compound": 0,
                                                       "neg": 0,
                                                       "neu": 0,
                                                       "pos": 0}

    def sentimentVader(self, gram):
        return self.vader.polarity_scores(" ".join(gram))

    def tokenSuffixes(self, tokens):
        return [max(s2, key=len) for s2 in
                [s1 for s1 in
                 [
                     [s0 for s0 in self.suffixes if word.endswith(s0)]
                     for word in tokens]
                 if s1]]

    def treebankToWordnetPOS(self, treebankPosTag):
        return {'J': wordnet.ADJ,
                'V': wordnet.VERB,
                'N': wordnet.NOUN,
                'R': wordnet.ADV}.get(treebankPosTag[0], wordnet.NOUN)

    def upperLowerLen(self, tokensOriginalCase):
        return [
            (
                sum([1 if letter.isupper() else 0 for letter in token]),
                sum([1 if letter.islower() else 0 for letter in token]),
                len(token),
                token
            )for token in tokensOriginalCase]

    def wordCase(self, upper, lower, length, token):
        if upper == 0:
            return "NC"  # No Caps
        elif upper == length:
            return "AC"  # All Caps
        elif upper == 1 and token[0].isupper():
            return "FC"  # First Cap
        else:
            return "MC"  # Mixed Caps

    def numSyllables(self, word):
        return [len(list(y for y in x if y[-1].isdigit()))for x in self.d[word.lower()]][0]

    def hasVowel(self, word):
        for char in word.lower():
            if char in ['a','e','i','o','u']:
                return 1
        return 0

    def hasPunctuation(self, word):
        return search(PUNCTUATION_RE, word) is not None

    def inDict(self, word):
        try:
            self.d[word.lower()]
            return True
        except KeyError:
            return False

    def punctuationFeatures(self, s):
        """
        s: input string
        returns {punctuation_mark: (raw #, % of length of s, % of total # of punctuation marks found in s)}

        example:
        punctuation_features("Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed consequat magna eu facilisis!!?")
        {'!': (2, 0.0217, 0.4),
         ',': (1, 0.0109, 0.2),
         '.': (1, 0.0109, 0.2),
         '?': (1, 0.0109, 0.2)}
         """

        punctuation_found_list = findall(PUNCTUATION_RE, s)
        return {p: (punctuation_found_list.count(p),
                    round(punctuation_found_list.count(p)/len(s), 4),
                    round(punctuation_found_list.count(p)/len(punctuation_found_list), 4)) for p in punctuation_found_list}
<|endoftext|>


<commit_before>import os, sys, subprocess, re, zlib, tempfile

## Binary Analysis Tool
## Copyright 2011-2012 Armijn Hemel for Tjaldur Software Governance Solutions
## Licensed under Apache 2.0, see LICENSE file for details

def readJFFS2Inodes(path):
	unpackenv = os.environ.copy()
	unpackenv['PATH'] = unpackenv['PATH'] + ":/usr/sbin"

	p = subprocess.Popen(['jffs2dump', '-cv', path], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, env=unpackenv)
	(stanout, stanerr) = p.communicate()
	if p.returncode != 0:
		return ([], [])
	st = stanout.split("\n")

	## (offset, size, parent, inode, name)
	direntries = {}

	## (offset, size, inode, name)
	nodeentries = []
	maxoffset = 0

	for s in st:
		if re.match("\s+Dirent", s) != None:
			(dirent, size, parentinode, version, inode, namesize, name) = s.split(',', 6)
			res = re.search("\s+Dirent\s*node\sat\s(\w+)", dirent)
			if res != None:
				offset = int(res.groups()[0], 16)
			res = re.search("\s+#pino\s*(\d+)", parentinode)
			if res != None:
				pinodenr = int(res.groups()[0])
			res = re.search("\s+#ino\s*(\d+)", inode)
			if res != None:
				inodenr = int(res.groups()[0])
			## use the namesize to get the name.
			res = re.search("\s+nsize\s*(\d+)", namesize)
			if res != None:
				namesize = int(res.groups()[0])
			nodename = name[-namesize:]
			direntries[inodenr] = {'offset': offset, 'size': 0, 'parent': pinodenr, 'name': nodename}
			if offset > maxoffset:
				maxoffset = offset
		elif re.match("\s+Inode", s) != None:
			(inodeent, size, inode, version, inodesize, csize, dsize, decompressedoffset) = s.split(',', 7)
			res = re.search("\s+dsize\s*(\d+)", dsize)
			if res != None:
				decompressedsize = int(res.groups()[0])
				if decompressedsize == 0: continue
			res = re.search("\s+csize\s*(\d+)", csize)
			if res != None:
				compressedsize = int(res.groups()[0])
			res = re.search("\s+Inode\s*node\sat\s(\w+)", inodeent)
			if res != None:
				offset = int(res.groups()[0], 16)
			res = re.search("\s+totlen\s(\w+)", size)
			if res != None:
				size = int(res.groups()[0], 16)
			res = re.search("\s+#ino\s*(\d+)", inode)
			if res != None:
				inodenr = int(res.groups()[0])
			nodeentries.append({'offset': offset, 'size': size, 'inode': inodenr, 'compressedsize': compressedsize, 'decompressedsize': decompressedsize})
			if offset > maxoffset:
				maxoffset = offset
	return (direntries, nodeentries, maxoffset)

def unpackJFFS2(path, tempdir=None):
	if tempdir == None:
		tmpdir = tempfile.mkdtemp()
	else:
		tmpdir = tempdir
	res = readJFFS2Inodes(path)
	if res == ({}, [], 0):
		## cleanup
		if tempdir == None:
			os.rmdir(tmpdir)
		return None
	(direntries, nodeentries, maxoffset) = res

	## first get all the entries for the direntries (a misnomer)
	directories = []
	for d in direntries.keys():
		directories.append(direntries[d]['parent'])
	directories = list(set(directories))
	directories.sort()

	pathinodes = {1: ''}

	data = open(path).read()

	jffs2size = maxoffset

	for n in direntries.keys():
		## create directory structure
		if n in directories:
			parentdirs = direntries[n]['name']
			parent = direntries[n]['parent']
			while parent != 1:
				parentdirs = direntries[parent]['name'] + "/" + parentdirs
				parent = direntries[parent]['parent']
			##
			pathinodes[n] = parentdirs
			parentdirs = tmpdir + "/" +  parentdirs
			os.makedirs(parentdirs)
		## we have a leaf node, so we need to unpack data here
		else:
			unzfiledata = ""
			for node in nodeentries:
				if node['inode'] == n:
					filedata = data[node['offset'] + 0x44: node['offset'] + node['size']]
					if node['offset'] == maxoffset:
						jffs2size += node['size']
					try:
						unzfiledata = unzfiledata + zlib.decompress(filedata)
					except Exception, e:
						## TODO: handle symlinks
						## a symlink is written as an ASCII file with the target of the symlink as the content of the file
						unzfiledata = unzfiledata + filedata
			datafile = open('%s/%s/%s' % (tmpdir, pathinodes[direntries[n]['parent']], direntries[n]['name']), 'w')
			datafile.write(unzfiledata)
			datafile.close()
	return (tmpdir, jffs2size)
<commit_msg>doc fixes<commit_after>import os, sys, subprocess, re, zlib, tempfile

## Binary Analysis Tool
## Copyright 2011-2012 Armijn Hemel for Tjaldur Software Governance Solutions
## Licensed under Apache 2.0, see LICENSE file for details

## method to process output of jffs2dump and read all the inodes from a JFFS2 file system.
def readJFFS2Inodes(path):
	## quick hack for systems that don't have /usr/sbin in $PATH (such as Debian and Ubuntu)
	unpackenv = os.environ.copy()
	unpackenv['PATH'] = unpackenv['PATH'] + ":/usr/sbin"

	p = subprocess.Popen(['jffs2dump', '-cv', path], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, env=unpackenv)
	(stanout, stanerr) = p.communicate()
	if p.returncode != 0:
		return ([], [])
	st = stanout.split("\n")

	## (offset, size, parent, inode, name)
	direntries = {}

	## (offset, size, inode, name)
	nodeentries = []
	maxoffset = 0

	for s in st:
		if re.match("\s+Dirent", s) != None:
			(dirent, size, parentinode, version, inode, namesize, name) = s.split(',', 6)
			res = re.search("\s+Dirent\s*node\sat\s(\w+)", dirent)
			if res != None:
				offset = int(res.groups()[0], 16)
			res = re.search("\s+#pino\s*(\d+)", parentinode)
			if res != None:
				pinodenr = int(res.groups()[0])
			res = re.search("\s+#ino\s*(\d+)", inode)
			if res != None:
				inodenr = int(res.groups()[0])
			## use the namesize to get the name.
			res = re.search("\s+nsize\s*(\d+)", namesize)
			if res != None:
				namesize = int(res.groups()[0])
			nodename = name[-namesize:]
			direntries[inodenr] = {'offset': offset, 'size': 0, 'parent': pinodenr, 'name': nodename}
			if offset > maxoffset:
				maxoffset = offset
		elif re.match("\s+Inode", s) != None:
			(inodeent, size, inode, version, inodesize, csize, dsize, decompressedoffset) = s.split(',', 7)
			res = re.search("\s+dsize\s*(\d+)", dsize)
			if res != None:
				decompressedsize = int(res.groups()[0])
				if decompressedsize == 0: continue
			res = re.search("\s+csize\s*(\d+)", csize)
			if res != None:
				compressedsize = int(res.groups()[0])
			res = re.search("\s+Inode\s*node\sat\s(\w+)", inodeent)
			if res != None:
				offset = int(res.groups()[0], 16)
			res = re.search("\s+totlen\s(\w+)", size)
			if res != None:
				size = int(res.groups()[0], 16)
			res = re.search("\s+#ino\s*(\d+)", inode)
			if res != None:
				inodenr = int(res.groups()[0])
			nodeentries.append({'offset': offset, 'size': size, 'inode': inodenr, 'compressedsize': compressedsize, 'decompressedsize': decompressedsize})
			if offset > maxoffset:
				maxoffset = offset
	return (direntries, nodeentries, maxoffset)

def unpackJFFS2(path, tempdir=None):
	if tempdir == None:
		tmpdir = tempfile.mkdtemp()
	else:
		tmpdir = tempdir
	res = readJFFS2Inodes(path)
	if res == ({}, [], 0):
		## cleanup
		if tempdir == None:
			os.rmdir(tmpdir)
		return None
	(direntries, nodeentries, maxoffset) = res

	## first get all the entries for the direntries (a misnomer)
	directories = []
	for d in direntries.keys():
		directories.append(direntries[d]['parent'])
	directories = list(set(directories))
	directories.sort()

	pathinodes = {1: ''}

	data = open(path).read()

	jffs2size = maxoffset

	for n in direntries.keys():
		## recreate directory structure
		if n in directories:
			parentdirs = direntries[n]['name']
			parent = direntries[n]['parent']
			while parent != 1:
				parentdirs = direntries[parent]['name'] + "/" + parentdirs
				parent = direntries[parent]['parent']
			pathinodes[n] = parentdirs
			parentdirs = tmpdir + "/" +  parentdirs
			os.makedirs(parentdirs)
		## we have a leaf node, so we need to unpack data here. Data is zlib compressed per inode.
		else:
			unzfiledata = ""
			for node in nodeentries:
				if node['inode'] == n:
					filedata = data[node['offset'] + 0x44: node['offset'] + node['size']]
					if node['offset'] == maxoffset:
						jffs2size += node['size']
					try:
						unzfiledata = unzfiledata + zlib.decompress(filedata)
					except Exception, e:
						## TODO: handle symlinks
						## a symlink is written as an ASCII file with the target of the symlink as the content of the file
						unzfiledata = unzfiledata + filedata
			datafile = open('%s/%s/%s' % (tmpdir, pathinodes[direntries[n]['parent']], direntries[n]['name']), 'w')
			datafile.write(unzfiledata)
			datafile.close()
	return (tmpdir, jffs2size)
<|endoftext|>