<commit_before>
import sys
import numpy as np

from os.path import join
from seisflows.tools import unix
from seisflows.tools.tools import exists
from seisflows.config import ParameterError

PAR = sys.modules['seisflows_parameters']
PATH = sys.modules['seisflows_paths']

system = sys.modules['seisflows_system']
solver = sys.modules['seisflows_solver']

class base(object):
    """ Gradient postprocessing class
    """

    def check(self):
        """ Checks parameters and paths
        """
        # check parameters
        if 'CLIP' not in PAR:
            setattr(PAR, 'CLIP', 0.)

        if 'SMOOTH' not in PAR:
            setattr(PAR, 'SMOOTH', 0.)

        if 'KERNELTYPE' not in PAR:
            setattr(PAR, 'KERNELTYPE', 'Relative')

        if 'PRECOND' not in PAR:
            setattr(PAR, 'PRECOND', False)

        # check paths
        if 'MASK' not in PATH:
            setattr(PATH, 'MASK', None)

        if 'PRECOND' not in PATH:
            setattr(PATH, 'PRECOND', None)

        if PATH.MASK:
            assert exists(PATH.MASK)

    def setup(self):
        """ Can be used to perform any required initialization or setup tasks
        """
        pass

    def write_gradient(self, path):
        """ Processes and combines contributions to the gradient from
          individual sources
        """
        if not exists(path):
            raise Exception

        system.run('postprocess', 'process_kernels',
                 hosts='head',
                 path=path+'/kernels',
                 parameters=solver.parameters)

        g = solver.merge(solver.load(
                 path +'/'+ 'kernels/sum',
                 suffix='_kernel'))

        self.save(g, path)

        if PAR.KERNELTYPE=='Relative':
            # convert from relative to absolute perturbations
            g *= solver.merge(solver.load(path +'/'+ 'model'))
            self.save(g, path, backup='relative')

        if PATH.MASK:
            # apply mask
            g *= solver.merge(solver.load(PATH.MASK))
            self.save(g, path, backup='nomask')

    def process_kernels(self, path='', parameters=[]):
        """ Combines contributions from individual sources and performs any
         required processing steps

          INPUT
              PATH - directory containing sensitivity kernels
              PARAMETERS - list of material parameters e.g. ['vp','vs']
        """
        if not exists(path):
            raise Exception

        if not parameters:
            parameters = solver.parameters

        if PAR.SMOOTH > 0:
            suffix = '_nosmooth'

        solver.combine(
               input_path=path,
               output_path=path+'/'+'sum'+suffix,
               parameters=parameters)

        if PAR.SMOOTH > 0.:
            solver.smooth(
                   input_path=path+'/'+'sum'+suffix,
                   output_path=path+'/'+'sum',
                   parameters=parameters,
                   span=PAR.SMOOTH)

    def save(self, g, path='', parameters=[], backup=None):
        """ Utility for saving dictionary representation of gradient
        """
        if not exists(path):
            raise Exception

        if not parameters:
            parameters = solver.parameters

        if backup:
            src = path +'/'+ 'gradient'
            dst = path +'/'+ 'gradient_'+backup
            unix.mv(src, dst)

        solver.save(solver.split(g),
                    path +'/'+ 'gradient',
                    parameters=parameters,
                    suffix='_kernel')

<commit_msg>refactored postprocess.base<commit_after>
import sys
import numpy as np

from os.path import join
from seisflows.tools import unix
from seisflows.tools.tools import exists
from seisflows.config import ParameterError

PAR = sys.modules['seisflows_parameters']
PATH = sys.modules['seisflows_paths']

system = sys.modules['seisflows_system']
solver = sys.modules['seisflows_solver']

class base(object):
    """ Gradient postprocessing class
    """

    def check(self):
        """ Checks parameters and paths
        """
        # check parameters
        if 'CLIP' not in PAR:
            setattr(PAR, 'CLIP', 0.)

        if 'SMOOTH' not in PAR:
            setattr(PAR, 'SMOOTH', 0.)

        if 'KERNELTYPE' not in PAR:
            setattr(PAR, 'KERNELTYPE', 'Relative')

        if 'PRECOND' not in PAR:
            setattr(PAR, 'PRECOND', False)

        # check paths
        if 'MASK' not in PATH:
            setattr(PATH, 'MASK', None)

        if 'PRECOND' not in PATH:
            setattr(PATH, 'PRECOND', None)

        if PATH.MASK:
            assert exists(PATH.MASK)

    def setup(self):
        """ Can be used to perform any required initialization or setup tasks
        """
        pass

    def write_gradient(self, path):
        """ Processes and combines contributions to the gradient from
          individual sources
        """
        if not exists(path):
            raise Exception

        system.run('postprocess', 'process_kernels',
                 hosts='head',
                 path=path+'/kernels',
                 parameters=solver.parameters)

        g = solver.merge(solver.load(
                 path +'/'+ 'kernels/sum',
                 suffix='_kernel'))

        self.save(g, path)

        if PAR.KERNELTYPE=='Relative':
            # convert from relative to absolute perturbations
            g *= solver.merge(solver.load(path +'/'+ 'model'))
            self.save(g, path, backup='relative')

        if PATH.MASK:
            # apply mask
            g *= solver.merge(solver.load(PATH.MASK))
            self.save(g, path, backup='nomask')

    def process_kernels(self, path='', parameters=[]):
        """ Combines contributions from individual sources and performs any
         required processing steps

          INPUT
              PATH - directory containing sensitivity kernels
              PARAMETERS - list of material parameters e.g. ['vp','vs']
        """
        if not exists(path):
            raise Exception

        if not parameters:
            parameters = solver.parameters

        if PAR.SMOOTH > 0:
            solver.combine(
                   input_path=path,
                   output_path=path+'/'+'sum_nosmooth',
                   parameters=parameters)

            solver.smooth(
                   input_path=path+'/'+'sum_nosmooth',
                   output_path=path+'/'+'sum',
                   parameters=parameters,
                   span=PAR.SMOOTH)
        else:
            solver.combine(
                   input_path=path,
                   output_path=path+'/'+'sum',
                   parameters=parameters)

    def save(self, g, path='', parameters=[], backup=None):
        """ Utility for saving dictionary representation of gradient
        """
        if not exists(path):
            raise Exception

        if not parameters:
            parameters = solver.parameters

        if backup:
            src = path +'/'+ 'gradient'
            dst = path +'/'+ 'gradient_'+backup
            unix.mv(src, dst)

        solver.save(solver.split(g),
                    path +'/'+ 'gradient',
                    parameters=parameters,
                    suffix='_kernel')

<|endoftext|>


<commit_before>
import numpy as np

import scipy.signal as signal

def sbandpass(s, h, freqlo, freqhi):
    nr = h.nr
    dt = h.dt
    fs = 1/dt

    for ir in range(0, nr):
        s[:,ir] = bandpass(s[:,ir],freqlo,freqhi,fs)
    return s

def slowpass(s, h, freq):
    raise NotImplementedError

def shighpass(s, h, freq):
    raise NotImplementedError

def smute(s, h, vel, toff, xoff=0, constant_spacing=False):
    nt = h.nt
    dt = h.dt
    nr = h.nr

    # construct tapered window
    length = 400
    win = np.sin(np.linspace(0, 1, 2*length))
    win = win[0:length]

    for ir in range(0, nr):

        # calculate slope
        if vel!=0:
            slope = 1./vel
        else:
            slope = 0

        # calculate offsets
        if constant_spacing:
            itoff = toff/dt
            ixoff = (ir-xoff)/dt
        else:
            itoff = toff/dt
            ixoff = (h.rx[ir]-h.sx[0]-xoff)/dt

        itmin = int(np.ceil(slope*abs(ixoff)+itoff)) - length/2
        itmax = itmin + length

        # apply window
        if 1 < itmin and itmax < nt:
            s[0:itmin,ir] = 0.
            s[itmin:itmax,ir] = win*s[itmin:itmax,ir]
        elif itmin < 1 <= itmax:
            s[1:itmax,ir] = win[length-itmax+1:length]*s[1:itmax,ir]
        elif itmin < nt < itmax:
            s[0:itmin,ir] = 0.
            s[itmin:nt,ir] = win[1:nt-itmin+1]*s[itmin:nt,ir]
        elif itmin > nt:
            s[:,ir] = 0.

    return s

def swindow(s, h, tmin, tmax, alpha=0.05, units='samples'):
    nt = h.nt
    dt = h.dt
    nr = h.nr
    t0 = h.t0

    if units == 'time':
        itmin = int((tmin-t0)/dt)
        itmax = int((tmax-t0)/dt)

    elif units == 'samples':
        itmin = int(tmin)
        itmax = int(tmax)

    else:
        raise ValueError

    win = tukeywin(nt, itmin, itmax, alpha)
    for ir in range(0,nr):
        s[:,ir] = win*s[:,ir]

    return s

# --

def correlate(u, v):
    w = np.convolve(u, np.flipud(v))
    return

def bandpass(w, freqlo, freqhi, fs, npass=2):
    wn = [2*freqlo/fs,2*freqhi/fs]
    (b,a) = signal.butter(npass, wn)
    w = signal.filtfilt(b,a,w)
    return w

def highpass(w, freq, df, npass=2):
    raise Exception('Not yet implemented.')

def lowpass(w, freq, df, npass=2):
    raise Exception('Not yet implemented.')

def window(nt, type='sine', **kwargs):
    if type == 'sine':
        return np.sin(np.linspace(0, 1, 2*length))
    elif type == 'tukey':
        return tukeywin

def tukeywin(nt, imin, imax, alpha=0.05):
    t = np.linspace(0,1,imax-imin)
    w = np.zeros(imax-imin)
    p = alpha/2.
    lo = np.floor(p*(imax-imin-1))+1
    hi = imax-imin-lo
    w[:lo] = (1+np.cos(np.pi/p*(t[:lo]-p)))/2
    w[lo:hi] = np.ones((hi-lo))
    w[hi:] = (1+np.cos(np.pi/p*(t[hi:]-p)))/2
    win = np.zeros(nt)
    win[imin:imax] = w
    return win

<commit_msg>added signal.sconvolve utility<commit_after>
import numpy as np

import scipy.signal as signal

def sbandpass(s, h, freqlo, freqhi):
    nr = h.nr
    dt = h.dt
    fs = 1/dt

    for ir in range(0, nr):
        s[:,ir] = bandpass(s[:,ir],freqlo,freqhi,fs)
    return s

def sconvolve(s, h, w, inplace=True):
    nt = h.nt
    nr = h.nr

    if inplace:
        for ir in range(nr):
            s[:,ir] = np.convolve(s[:,ir], w, 'same')
        return s
    else:
        s2 = np.zeros((nt,nr))
        for ir in range(nr):
            s2[:,ir] = np.convolve(s[:,ir], w, 'same')
        return s2

def slowpass(s, h, freq):
    raise NotImplementedError

def shighpass(s, h, freq):
    raise NotImplementedError

def smute(s, h, vel, toff, xoff=0, constant_spacing=False):
    nt = h.nt
    dt = h.dt
    nr = h.nr

    # construct tapered window
    length = 400
    win = np.sin(np.linspace(0, 1, 2*length))
    win = win[0:length]

    for ir in range(0, nr):

        # calculate slope
        if vel!=0:
            slope = 1./vel
        else:
            slope = 0

        # calculate offsets
        if constant_spacing:
            itoff = toff/dt
            ixoff = (ir-xoff)/dt
        else:
            itoff = toff/dt
            ixoff = (h.rx[ir]-h.sx[0]-xoff)/dt

        itmin = int(np.ceil(slope*abs(ixoff)+itoff)) - length/2
        itmax = itmin + length

        # apply window
        if 1 < itmin and itmax < nt:
            s[0:itmin,ir] = 0.
            s[itmin:itmax,ir] = win*s[itmin:itmax,ir]
        elif itmin < 1 <= itmax:
            s[1:itmax,ir] = win[length-itmax+1:length]*s[1:itmax,ir]
        elif itmin < nt < itmax:
            s[0:itmin,ir] = 0.
            s[itmin:nt,ir] = win[1:nt-itmin+1]*s[itmin:nt,ir]
        elif itmin > nt:
            s[:,ir] = 0.

    return s

def swindow(s, h, tmin, tmax, alpha=0.05, units='samples'):
    nt = h.nt
    dt = h.dt
    nr = h.nr
    t0 = h.t0

    if units == 'time':
        itmin = int((tmin-t0)/dt)
        itmax = int((tmax-t0)/dt)

    elif units == 'samples':
        itmin = int(tmin)
        itmax = int(tmax)

    else:
        raise ValueError

    win = tukeywin(nt, itmin, itmax, alpha)
    for ir in range(0,nr):
        s[:,ir] = win*s[:,ir]

    return s

# --

def correlate(u, v):
    w = np.convolve(u, np.flipud(v))
    return

def bandpass(w, freqlo, freqhi, fs, npass=2):
    wn = [2*freqlo/fs,2*freqhi/fs]
    (b,a) = signal.butter(npass, wn)
    w = signal.filtfilt(b,a,w)
    return w

def highpass(w, freq, df, npass=2):
    raise Exception('Not yet implemented.')

def lowpass(w, freq, df, npass=2):
    raise Exception('Not yet implemented.')

def window(nt, type='sine', **kwargs):
    if type == 'sine':
        return np.sin(np.linspace(0, 1, 2*length))
    elif type == 'tukey':
        return tukeywin

def tukeywin(nt, imin, imax, alpha=0.05):
    t = np.linspace(0,1,imax-imin)
    w = np.zeros(imax-imin)
    p = alpha/2.
    lo = np.floor(p*(imax-imin-1))+1
    hi = imax-imin-lo
    w[:lo] = (1+np.cos(np.pi/p*(t[:lo]-p)))/2
    w[lo:hi] = np.ones((hi-lo))
    w[hi:] = (1+np.cos(np.pi/p*(t[hi:]-p)))/2
    win = np.zeros(nt)
    win[imin:imax] = w
    return win

<|endoftext|>


<commit_before>import os
import re
import socket
import random
import logging
import threading
import subprocess as sp
from os.path import expanduser

from unidecode import unidecode
from contextlib import closing
from datetime import datetime, timedelta, timezone

from indra.resources.greek_alphabet import greek_alphabet
from indra.tools.reading.readers.core import Reader

from indra.sources.trips import client, process_xml

logger = logging.getLogger(__name__)

startup_path = '/sw/drum/bin/startup.sh'
service_endpoint = 'drum'
DRUM_DOCKER = '292075781285.dkr.ecr.us-east-1.amazonaws.com/drum'

def find_free_ports():
    """Find ports that are unused.

    The order is randomized to minimize the chances of race-condition overlaps.
    """
    ports = list(range(1, 65536))
    random.shuffle(ports)
    for port in ports:
        with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sok:
            res = sok.connect_ex(('localhost', port))
            if res != 0:
                yield port

def _wait_for_trips_startup(p):
    # Wait for the service to be ready
    for log_line in _tail_trips(p):
        if 'can\'t bind to port' in 'log_line':
            port_failure = True
        if log_line == 'Ready':
            # TRIPS is ready to read and we can continue on.
            break
    else:
        if port_failure:
            return "PORT"
        else:
            return "UNKNOWN"
    return None

def _start_trips():
    """Start up an instance of TRIPS."""
    for port in find_free_ports():
        if os.environ.get("IN_TRIPS_DOCKER", 'false') == 'true':
            for trips_port in find_free_ports():
                if trips_port == port:
                    # Obviously don't try the same port for both.
                    continue
                logger.info("Attempting to starting up a TRIPS service "
                            "from within the docker on outward facing "
                            "port %d and internal port %d."
                            % (port, trips_port))
                p = sp.Popen([expanduser('~/startup_trips.sh'), str(port),
                              str(trips_port)],
                             stdout=sp.PIPE, stderr=sp.STDOUT)

                # Wait for TRIPS to start.
                res = _wait_for_trips_startup(p)
                if res == "PORT":
                    # External port failed.
                    logger.error("In-docker TRIPS failed to start service "
                                 "with external port %s" % port)
                    break
                elif res is not None:
                    # Something else failed. (Try another internal port)
                    logger.error("In-docker TRIPS failed to start.")
                    continue

                # Everything seems to have worked as expected.
                break
            else:
                # We tried every internal port, without success or explicit
                # external port error.
                raise TripsStartupError("Trips failed to start up on any "
                                        "internal port, no apparent "
                                        "external port errors.")
        else:
            logger.info("Starting up a TRIPS service using drum docker.")
            p = sp.Popen(['docker', 'run', '-it', '-p', '%d:80' % port,
                          '--entrypoint', '/sw/drum/bin/startup.sh',
                          DRUM_DOCKER],
                         stdout=sp.PIPE, stderr=sp.STDOUT)
            res = _wait_for_trips_startup(p)
            if res == "PORT":
                logger.error("Service failed to start on port %s." % port)
                continue
            elif res:
                raise TripsStartupError("Trips failed to start up. "
                                        "Reason: %s" % res)
        service_host = 'http://localhost:%d/cgi/' % port

        # The above for-loop existed without going to else, indicating
        # successful startup.
        break
    else:
        # We exhausted all possible ports.
        raise TripsStartupError("Could not start TRIPS, all ports appear "
                                "to be busy.")
    logger.info("Service has started up.")
    return p, service_host

class TripsStartupError(Exception):
    pass

class TripsReader(Reader):
    """A stand-in for TRIPS reading.

    Currently, we do not run TRIPS (more specifically DRUM) regularly at large
    scales, however on occasion we have outputs from TRIPS that were generated
    a while ago.
    """
    name = 'TRIPS'
    result_format = 'xml'

    def __init__(self, *args, **kwargs):
        self.version = self.get_version()
        super(TripsReader, self).__init__(*args, **kwargs)
        self.running = False
        self.stopping = False
        return

    def _monitor_trips_service(self, proc):
        self.running = True
        for _ in _tail_trips(proc):
            if self.stopping:
                break
        proc.communicate()
        if proc.returncode:
            logger.error("TRIPS ended badly.")
        self.running = False

    def _read(self, content_iter, verbose=False, log=False, n_per_proc=None):
        # Start trips running
        p, service_host = _start_trips()

        # Set up the trips monitor
        th = threading.Thread(target=self._monitor_trips_service, args=[p])
        th.start()

        # Process all the content.
        for content in content_iter:
            if not self.running:
                logger.error("Breaking loop: trips is down.")
                break

            # Clean up the text string a bit.
            # - remove all excess white space.
            # - remove special greek letters
            # - remove all special unicode, replace with ascii
            raw_text = content.get_text()
            raw_text = re.sub('\s+', ' ', raw_text)
            for greek_letter, spelled_letter in greek_alphabet.items():
                raw_text = raw_text.replace(greek_letter, spelled_letter)
            text = unidecode(raw_text)

            # Process the text
            html = client.send_query(text, service_host=service_host,
                                     service_endpoint=service_endpoint)
            xml = client.get_xml(html)
            self.add_result(content.get_id(), xml)

        # Stop TRIPS if it hasn't stopped already.
        p.kill()  # Send signal to the process to stop
        self.stopping = True  # Sends signal to the loop to stop
        th.join()

        return self.results

    @classmethod
    def get_version(cls):
        git_date_cmd = ['git', 'log', '-1', '--format=%cd']
        if os.environ.get("IN_TRIPS_DOCKER", "false") == "true":
            curdir = os.getcwd()
            try:
                # Get the date of the last commit from git in drum.
                os.chdir('/sw/drum')
                res = sp.run(git_date_cmd, stdout=sp.PIPE)
            finally:
                os.chdir(curdir)
        else:
            res = sp.run(['docker', 'run', DRUM_DOCKER, '-c',
                          ' '.join(['cd', '/sw/drum;'] + git_date_cmd)],
                         stdout=sp.PIPE)

        # Format that string into a datetime and standardize to utc.
        d = datetime.strptime(res.stdout.decode('utf-8').strip(),
                              '%a %b %d %H:%M:%S %Y %z')
        d.astimezone(tz=timezone(timedelta(0)))

        # Create a formatted string as the version.
        version = d.strftime('%Y%b%d')

        return version

    @staticmethod
    def get_processor(reading_content):
        return process_xml(reading_content)

def _tail_trips(proc):
    for line in iter(proc.stdout.readline, b''):
        log_line = line.strip().decode('utf-8')
        logger.info('TRIPS: ' + log_line)
        yield log_line
<commit_msg>Intensify logging around TRIPS shutdown.<commit_after>import os
import re
import socket
import random
import logging
import threading
import subprocess as sp
from os.path import expanduser

from unidecode import unidecode
from contextlib import closing
from datetime import datetime, timedelta, timezone

from indra.resources.greek_alphabet import greek_alphabet
from indra.tools.reading.readers.core import Reader

from indra.sources.trips import client, process_xml

logger = logging.getLogger(__name__)

startup_path = '/sw/drum/bin/startup.sh'
service_endpoint = 'drum'
DRUM_DOCKER = '292075781285.dkr.ecr.us-east-1.amazonaws.com/drum'

def find_free_ports():
    """Find ports that are unused.

    The order is randomized to minimize the chances of race-condition overlaps.
    """
    ports = list(range(1, 65536))
    random.shuffle(ports)
    for port in ports:
        with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sok:
            res = sok.connect_ex(('localhost', port))
            if res != 0:
                yield port

def _wait_for_trips_startup(p):
    # Wait for the service to be ready
    for log_line in _tail_trips(p):
        if 'can\'t bind to port' in 'log_line':
            port_failure = True
        if log_line == 'Ready':
            # TRIPS is ready to read and we can continue on.
            break
    else:
        if port_failure:
            return "PORT"
        else:
            return "UNKNOWN"
    return None

def _start_trips():
    """Start up an instance of TRIPS."""
    for port in find_free_ports():
        if os.environ.get("IN_TRIPS_DOCKER", 'false') == 'true':
            for trips_port in find_free_ports():
                if trips_port == port:
                    # Obviously don't try the same port for both.
                    continue
                logger.info("Attempting to starting up a TRIPS service "
                            "from within the docker on outward facing "
                            "port %d and internal port %d."
                            % (port, trips_port))
                p = sp.Popen([expanduser('~/startup_trips.sh'), str(port),
                              str(trips_port)],
                             stdout=sp.PIPE, stderr=sp.STDOUT)

                # Wait for TRIPS to start.
                res = _wait_for_trips_startup(p)
                if res == "PORT":
                    # External port failed.
                    logger.error("In-docker TRIPS failed to start service "
                                 "with external port %s" % port)
                    break
                elif res is not None:
                    # Something else failed. (Try another internal port)
                    logger.error("In-docker TRIPS failed to start.")
                    continue

                # Everything seems to have worked as expected.
                break
            else:
                # We tried every internal port, without success or explicit
                # external port error.
                raise TripsStartupError("Trips failed to start up on any "
                                        "internal port, no apparent "
                                        "external port errors.")
        else:
            logger.info("Starting up a TRIPS service using drum docker.")
            p = sp.Popen(['docker', 'run', '-it', '-p', '%d:80' % port,
                          '--entrypoint', '/sw/drum/bin/startup.sh',
                          DRUM_DOCKER],
                         stdout=sp.PIPE, stderr=sp.STDOUT)
            res = _wait_for_trips_startup(p)
            if res == "PORT":
                logger.error("Service failed to start on port %s." % port)
                continue
            elif res:
                raise TripsStartupError("Trips failed to start up. "
                                        "Reason: %s" % res)
        service_host = 'http://localhost:%d/cgi/' % port

        # The above for-loop existed without going to else, indicating
        # successful startup.
        break
    else:
        # We exhausted all possible ports.
        raise TripsStartupError("Could not start TRIPS, all ports appear "
                                "to be busy.")
    logger.info("Service has started up.")
    return p, service_host

class TripsStartupError(Exception):
    pass

class TripsReader(Reader):
    """A stand-in for TRIPS reading.

    Currently, we do not run TRIPS (more specifically DRUM) regularly at large
    scales, however on occasion we have outputs from TRIPS that were generated
    a while ago.
    """
    name = 'TRIPS'
    result_format = 'xml'

    def __init__(self, *args, **kwargs):
        self.version = self.get_version()
        super(TripsReader, self).__init__(*args, **kwargs)
        self.running = False
        self.stopping = False
        return

    def _monitor_trips_service(self, proc):
        self.running = True
        for _ in _tail_trips(proc):
            if self.stopping:
                logger.info("Got stop signal. Stopping.")
                break
        logger.info("Waiting for TRIPS process to join.")
        proc.communicate()
        if proc.returncode:
            logger.error("TRIPS ended with return code %d." % proc.returncode)
        else:
            logger.info("TRIPS ended without error.")
        logger.info("TRIPS is no longer running.")
        self.running = False

    def _read(self, content_iter, verbose=False, log=False, n_per_proc=None):
        # Start trips running
        p, service_host = _start_trips()

        # Set up the trips monitor
        th = threading.Thread(target=self._monitor_trips_service, args=[p])
        th.start()

        # Process all the content.
        for content in content_iter:
            if not self.running:
                logger.error("Breaking loop: trips is down.")
                break

            # Clean up the text string a bit.
            # - remove all excess white space.
            # - remove special greek letters
            # - remove all special unicode, replace with ascii
            raw_text = content.get_text()
            raw_text = re.sub('\s+', ' ', raw_text)
            for greek_letter, spelled_letter in greek_alphabet.items():
                raw_text = raw_text.replace(greek_letter, spelled_letter)
            text = unidecode(raw_text)

            # Process the text
            html = client.send_query(text, service_host=service_host,
                                     service_endpoint=service_endpoint)
            if html:
                xml = client.get_xml(html)
                self.add_result(content.get_id(), xml)
            else:
                self.add_result(content.get_id(), None)

        # Stop TRIPS if it hasn't stopped already.
        logger.info("Killing TRIPS")
        p.kill()  # Send signal to the process to stop

        logger.info("Signalling observation loop to stop.")
        self.stopping = True  # Sends signal to the loop to stop

        logger.info("Waiting for observation loop thread to join.")
        th.join()

        return self.results

    @classmethod
    def get_version(cls):
        git_date_cmd = ['git', 'log', '-1', '--format=%cd']
        if os.environ.get("IN_TRIPS_DOCKER", "false") == "true":
            curdir = os.getcwd()
            try:
                # Get the date of the last commit from git in drum.
                os.chdir('/sw/drum')
                res = sp.run(git_date_cmd, stdout=sp.PIPE)
            finally:
                os.chdir(curdir)
        else:
            res = sp.run(['docker', 'run', DRUM_DOCKER, '-c',
                          ' '.join(['cd', '/sw/drum;'] + git_date_cmd)],
                         stdout=sp.PIPE)

        # Format that string into a datetime and standardize to utc.
        d = datetime.strptime(res.stdout.decode('utf-8').strip(),
                              '%a %b %d %H:%M:%S %Y %z')
        d.astimezone(tz=timezone(timedelta(0)))

        # Create a formatted string as the version.
        version = d.strftime('%Y%b%d')

        return version

    @staticmethod
    def get_processor(reading_content):
        return process_xml(reading_content)

def _tail_trips(proc):
    for line in iter(proc.stdout.readline, b''):
        log_line = line.strip().decode('utf-8')
        logger.info('TRIPS: ' + log_line)
        yield log_line
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
#  This file is part of Sequana software
#
#  Copyright (c) 2016 - Sequana Development Team
#
#  File author(s):
#      Thomas Cokelaer <thomas.cokelaer@pasteur.fr>
#      Dimitri Desvillechabrol <dimitri.desvillechabrol@pasteur.fr>,
#          <d.desvillechabrol@gmail.com>
#
#  Distributed under the terms of the 3-clause BSD license.
#  The full license is in the LICENSE file, distributed with this software.
#
#  website: https://github.com/sequana/sequana
#  documentation: http://sequana.readthedocs.io
#
##############################################################################

import easydev
import os
import glob
import json

from .report_main import BaseReport

# a utility from external reports package
from reports import HTMLTable

import pandas as pd

class FastQStatsReport(BaseReport):
    """

    """
    def __init__(self, input_directory, output_filename="fastq_stats.html",
                 directory="report", **kargs):
        """

        :param jinja_template: name of a directory (either local) or
            from sequana/share/templates where JINJA files are available. A file
            named index.html is required but may be renamed (with
            **output_filename** parameter).
        :param output_filename: name of the final HTML file.
        :param directory: name of the output directory (defaults to report)

        Parameters accepted by :class:`reports.Report` are also accepted.

        """
        super(FastQStatsReport, self).__init__(
                jinja_filename="fastq_stats/index.html",
                directory=directory,
                output_filename=output_filename, **kargs)
        self.input_directory = input_directory
        self.jinja['title'] = "FastQ Stats Report"

    def parse(self):
        # Add the canvas js in the header
        from sequana.resources.canvas import bar

        files = glob.glob("%s/*json" % self.input_directory)
        acgt = [
            {"name":"A", "data": {}},
            {"name":"C", "data": {}},
            {"name":"G", "data": {}},
            {"name":"T", "data": {}}]

        for filename in files:
            try:
                # if the mapped file is empty, the file is empty
                thisdata = json.load(open(filename))
            except:
                thisdata= {"A":0, "C":0, "G":0, "T":0}
            if "R2.unmapped" in filename:
                key = "R2.unmapped"
            elif "R1.unmapped" in filename:
                key = "R1.unmapped"
            elif "R1.mapped" in filename:
                key = "R1.mapped"
            elif "R2.mapped" in filename:
                key = "R2.mapped"
            elif "_R1_" in filename:
                key = "R1"
            elif "_R2_" in filename:
                key = "R2"
            elif "_R1." in filename:
                key = "R1"
            elif "_R2." in filename:
                key = "R2"

            acgt[0]["data"][key] = thisdata['A']
            acgt[1]["data"][key] = thisdata['C']
            acgt[2]["data"][key] = thisdata['G']
            acgt[3]["data"][key] = thisdata['T']

            # { "n_reads": 1627, "GC content": 0.45281800009750867}

        # First argument is just an Identifier
        data = bar.stacked_bar("Test", "ACGT distribution", acgt)

        data += """ """
        self.jinja["canvas"] = data

        html = """

        """

        # Assuming in fastq directory, we figure out the HTML files and
        # create a table accordingly.
        links = glob.glob("%s/*png" % self.input_directory)
        links = [x.split("/",1)[1] for x in links]

        from sequana.htmltools import galleria
        html += galleria(links)

        self.jinja["content"] = html

<commit_msg>add more info on the web page<commit_after># -*- coding: utf-8 -*-
#
#  This file is part of Sequana software
#
#  Copyright (c) 2016 - Sequana Development Team
#
#  File author(s):
#      Thomas Cokelaer <thomas.cokelaer@pasteur.fr>
#      Dimitri Desvillechabrol <dimitri.desvillechabrol@pasteur.fr>,
#          <d.desvillechabrol@gmail.com>
#
#  Distributed under the terms of the 3-clause BSD license.
#  The full license is in the LICENSE file, distributed with this software.
#
#  website: https://github.com/sequana/sequana
#  documentation: http://sequana.readthedocs.io
#
##############################################################################

import easydev
import os
import glob
import json

from .report_main import BaseReport

# a utility from external reports package
from reports import HTMLTable

import pandas as pd

class FastQStatsReport(BaseReport):
    """

    """
    def __init__(self, input_directory, output_filename="fastq_stats.html",
                 directory="report", **kargs):
        """

        :param jinja_template: name of a directory (either local) or
            from sequana/share/templates where JINJA files are available. A file
            named index.html is required but may be renamed (with
            **output_filename** parameter).
        :param output_filename: name of the final HTML file.
        :param directory: name of the output directory (defaults to report)

        Parameters accepted by :class:`reports.Report` are also accepted.

        """
        super(FastQStatsReport, self).__init__(
                jinja_filename="fastq_stats/index.html",
                directory=directory,
                output_filename=output_filename, **kargs)
        self.input_directory = input_directory
        self.jinja['title'] = "FastQ Stats Report"

    def parse(self):
        # Add the canvas js in the header
        from sequana.resources.canvas import bar

        files = glob.glob("%s/*json" % self.input_directory)
        acgt = [
            {"name":"A", "data": {}},
            {"name":"C", "data": {}},
            {"name":"G", "data": {}},
            {"name":"T", "data": {}}]

        dfsum = None
        for filename in files:
            try:
                # if the mapped file is empty, the file is empty
                thisdata = json.load(open(filename))
            except:
                thisdata= {"A":0, "C":0, "G":0, "T":0, "n_reads":0, "GC content":0}

            if "R2.unmapped" in filename:
                key = "R2.unmapped"
            elif "R1.unmapped" in filename:
                key = "R1.unmapped"
            elif "R1.mapped" in filename:
                key = "R1.mapped"
            elif "R2.mapped" in filename:
                key = "R2.mapped"
            elif "_R1_" in filename:
                key = "R1"
            elif "_R2_" in filename:
                key = "R2"
            elif "_R1." in filename:
                key = "R1"
            elif "_R2." in filename:
                key = "R2"

            if dfsum is None:
                dfsum = pd.Series(thisdata).to_frame()
                dfsum.columns = [key]
            else:
                dfsum[key] = pd.Series(thisdata)

            acgt[0]["data"][key] = thisdata['A']
            acgt[1]["data"][key] = thisdata['C']
            acgt[2]["data"][key] = thisdata['G']
            acgt[3]["data"][key] = thisdata['T']

            # { "n_reads": 1627, "GC content": 0.45281800009750867}

        # First argument is just an Identifier
        data = bar.stacked_bar("Test", "ACGT distribution", acgt)

        data += """ """
        self.jinja["canvas"] = data

        dfsum = dfsum[sorted(dfsum.columns)]
        dfsum = dfsum.ix[['A', 'C', 'G', 'T', 'GC content', 'n_reads']]
        S = dfsum.ix[['A', 'C', 'G', 'T']].sum()
        dfsum.ix[['A', 'C', 'G', 'T']] /= S

        html = HTMLTable(dfsum).to_html(index=True)

        html += """

        """

        # Assuming in fastq directory, we figure out the HTML files and
        # create a table accordingly.
        sources = glob.glob("%s/*png" % self.input_directory)
        import shutil
        targets = []
        for source in sources:
            print(source)
            filename = source.rsplit("/", 1)[1]
            print(filename)
            target = "report/images/" + filename
            print(target)
            shutil.copy(source, target)
            targets.append(target.replace("report/", ""))

        from sequana.htmltools import galleria
        html += galleria(targets)

        self.jinja["content"] = html

<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
##############################################################################
#
#    Intercompany Trade - Purchase & Sale Margin module for Odoo
#    Copyright (C) 2015-Today GRAP (http://www.grap.coop)
#    @author Sylvain LE GAL (https://twitter.com/legalsylvain)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
{
    'name': 'Intercompany Trade - Purchase & Sale Margin',
    'version': '0.1',
    'category': 'Sale',
    'description': """
Module for Intercompany Trade for Purchase and Sale Margin Module
=================================================================

This module fixes a bug in V7 that make that computed field sale_order.margin
is not recomputed when we update only a sale.order.

TODO: check if v8 fixes the bug.

Copyright, Authors and Licence:
-------------------------------
    * Copyright: 2014, GRAP: Groupement Régional Alimentaire de Proximité;
    * Author:
        * Sylvain LE GAL (https://twitter.com/legalsylvain);
    * Licence: AGPL-3 (http://www.gnu.org/licenses/);""",
    'author': 'GRAP',
    'website': 'http://www.grap.coop',
    'license': 'AGPL-3',
    'depends': [
        'intercompany_trade_purchase_sale',
        'sale_margin',
    ],
    'auto_install': True,
}
<commit_msg>[WIP] travis;<commit_after># -*- encoding: utf-8 -*-
##############################################################################
#
#    Intercompany Trade - Purchase & Sale Margin module for Odoo
#    Copyright (C) 2015-Today GRAP (http://www.grap.coop)
#    @author Sylvain LE GAL (https://twitter.com/legalsylvain)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
{
    'name': 'Intercompany Trade - Purchase & Sale Margin',
    'version': '0.1',
    'category': 'Sale',
    'description': """
Module for Intercompany Trade for Purchase and Sale Margin Module
=================================================================

This module fixes a bug in V7 that make that computed field sale_order.margin
is not recomputed when we update only a sale.order.

TODO: check if v8 fixes the bug.

Copyright, Authors and Licence:
-------------------------------
    * Copyright: 2014, GRAP: Groupement Régional Alimentaire de Proximité;
    * Author:
        * Sylvain LE GAL (https://twitter.com/legalsylvain);
    * Licence: AGPL-3 (http://www.gnu.org/licenses/);""",
    'author': 'GRAP',
    'website': 'http://www.grap.coop',
    'license': 'AGPL-3',
    'depends': [
        'intercompany_trade_purchase_sale',
        'sale_margin',
    ],
    'auto_install': True,
    'installable': False,
}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Thu Apr 18 12:29:52 2013

@author: aitor
"""

import re
import urllib2
#import pyPdf
from topia.termextract import extract

import os
from os import listdir
from os.path import isfile, join

import codecs
from cStringIO import StringIO
from pdfminer.converter import TextConverter
from pdfminer.pdfinterp import PDFResourceManager, process_pdf
from pdfminer.layout import LAParams
from pdfminer.pdfparser import PDFSyntaxError
from pdfminer.pdfinterp import PDFTextExtractionNotAllowed

verbose = True

def get_pdfs(include_spanish = False):
    spanish = ['pcuriel_CISTI12', 'ablago2011entorno', 'CAIS2011-07', 'pretel2010framework',
               'jornadasrfid2008_v1.0', 'WebLabTAEE2006']
    url = 'http://www.morelab.deusto.es/'
    path_base = 'publications/'
    years = range(2006, 2014)
    if verbose:
        print "Downloading files"
    for y in years:
        path = path_base + str(y) + '/'
        d = os.path.dirname("./pdf/" + path)
        if not os.path.exists(d):
            os.makedirs(d)
        response = urllib2.urlopen(url+path).read()
        if verbose:
            print response
            print "\n************FILES**********\n"
        for filename in re.findall('"\S+.pdf"', response):
            if not include_spanish and filename in spanish:
                continue
            filename = filename.replace('"', '')
            if verbose:
                print "Filename: " + filename
                print "URL: " + url + path + filename
            f = urllib2.urlopen(url + path + filename)
            data = f.read()
            with open("./pdf/" + path + filename, "wb") as code:
                code.write(data)

def get_pdf_content(path):
    laparams = LAParams()
    rsrc = PDFResourceManager()
    outfp = StringIO()
    try:
        #TODO: detect the encoding of the PDF
        device = TextConverter(rsrc, outfp, codec="cp1252", laparams=laparams)
        process_pdf(rsrc, device, codecs.open(path))
    except (PDFSyntaxError, PDFTextExtractionNotAllowed):
        print "Error processing PDF file: " + path
    return outfp.getvalue()

def get_terms_topia(text):
    extractor = extract.TermExtractor()
    extractor.filter = extract.DefaultFilter(singleStrengthMinOccur=2)
    terms = sorted(extractor(text))
    return terms

def process_pdfs():
    years = range(2006, 2014)
    relations = []
    for y in years:
        file_path = './pdf/publications/' + str(y) + '/'
        filenames = [ f for f in listdir(file_path) if isfile(join(file_path,f)) ]
        for name in filenames:
            if verbose:
                print "Processing" + file_path + name
            content = get_pdf_content(file_path + name)
            terms = get_terms_topia(content)
            #paper_terms = [t[0] for t in terms if t[1] > 2 and not '\\x' in repr(t[0])]
            #still having problems with the encodings of the pdf file, the repr
            #is hopefully a temporal "workaround
            paper_terms = [t[0].strip() for t in terms if t[1] > 4 and not is_bad(t[0].strip()) and not '\\' in repr(t[0])]
            if verbose:
                print "  ->  Total terms: " + str(len(paper_terms))
            relations.append(paper_terms)

    if verbose:
        print "\n\n" + str(len(relations)) + " papers analized"
    return relations

#SUSPICIOUS = []

#curating the terms extracted with topia to delete some incorrect ones
def is_bad(term):
    bad = False
    invalid_terms = ['\\x', '.,', '(cid', '/', '/ i n', '/ f', '</', '><!--',
                       '),', '--></', '.:', '].', '[<', '[</', '</', '|k', '=0.6).',
                       'http ://www', '/ / www', '://drupal', '://www', '="http',
                       '="x', '(?', '),', '->', '2),', '(?', '),', '.,', '(%)', '(1',
                       '|', ',i', '(c', ',5',
                       'W m', 'r e', '2.0-',
                       'Fig', 'Figure', 'figure', 'e.g.', 'i.e.', 'e.g', 'e.g .,',
                       'i.e', 'i.e.,', 'i.e .,',
                       '-2006', '(2006).', '2011)',
                       'ontolo', 're', 'gy', 'http ://www http ://www DRAFT', 'et',
                       'al', '/her', 'Elect', 'tion', '2let', 'st',
                       '08-09', '10-11',
                       'Ordu', 'Garc', 'Universidades 24, 48007, Bilbao', 'tion',
                       'del.icio.u', 'del.icio.u', 'user \x92s', '\xf3pez', '2Object',
                       'EMI 2Object', 'EMI 2Objects', 'EMI 2let', 'EMI 2let Player',
                       'EMI 2let Server', 'EMI 2lets', 'EMI 2lets platform', 'L \xf3pez',
                       '\xe1zquez', '\xf1a', '\xf3pez', 'Web 2.0-', '</ action',
                       '</ argumentList', '>actionName', 'name >actionName',
                       '2.Control', '2Button', '2Peer', 'EMI 2Peer', 'EMI 2Peers',
                       'EMI 2let', 'EMI 2let Player', 'EMI 2let Server', 'EMI 2lets',
                       'EMI 2lets platform', '\xf3pez', '\xf1a', '@eside', '\xeda',
                       '="urn', '2Object', '2Protocol', 'EMI 2Object', 'EMI 2Objects',
                       'EMI 2let', 'EMI 2let Player', 'EMI 2let Server', 'EMI 2let server-side',
                       'EMI 2lets', 'EMI 2lets platform', '\xf1a', '\xf3pez', '\xe1ctica',
                       '\xf1o', '1, MARCH 2005 DRAFT', 'Diego L \xb4 opez', 'Ipi \x98',
                       '/cm', '\xe6a', '09, 2006 Florianopolis', '\xe1ndez', '\xeda',
                       '\xf1a', 'Web 2.0 technologies', '\x92t', ':name', ':type',
                       '://deusto', ':type', 'http ://deusto', 'Web 2.0-', '\x98na',
                       'estaci \xf3n base', 'informaci \xf3n', 'interacci \xf3n',
                       '\xe1mbrica', '\xe1n', '\xe9n', '\xe9tica', '\xe9tico',
                       '\xeda', '\xedda', '\xf1o', '\xf3lo', '\xf3n', '\xfan','2onto',
                       'WWW 2006, Edinburgh', 'del.icio.u', 'folk 2onto',
                       '1-1569228491-2409\xa9SoftCOM', '(URI', '(URI space',
                       '<ITriple', ':Ciudad', 'user \x92s', 'ElderCare \x92s',
                       '\x1berent', ':Pablo', ':Person', ':SecretClas', ':fullName',
                       ':name', 'b 1', 'b 1 injection :name', 'injection :Pablo injection :fullName',
                       'injection :Person', 'injection :SecretClass', 'name 1',
                       'p 1', 'p 1 injection :fullName', 'p 2', '/IEC', '>test',
                       'aplicaci \xf3n', 'evaluaci \xf3n', '\xd3N', '\xe1lculo',
                       '\xe1lisi', '\xe1ndar', '\xe9n', '\xe9trica', '\xeda',
                       '\xedculo', '\xf3dulo', '\xf3n', '\xf3vil', '\xf3vile',
                       '/IEC', '\xe1n', '\xeda', '\xedgrafo', '\xf1o', '\xf3dulo',
                       '\xf3n', '\xf1a', '\xf3pez', '\xf3pez', 'CAIS 2011', "EMS ',",
                       'extracci \xf3n', 'precisi \xf3n', 't \xe9rminos', '\x92,\x92fem',
                       '\x92,\x92sg', '\xe1gina', '\xe1lisi', '\xe1tica', '\xe9dico',
                       '\xe9rmino', '\xe9tera', '\xe9todo', '\xeda', '\xedan', '\xeddo',
                       '\xf3n', '\xf3nica', '\xfamero', '\xfc\xedstica', '(w', 'room 0.11',
                       '(sp', '://nodeuri', 'http ://nodeuri', '\x98na', '\xb4omez',
                       '\x98na', '\xb4omez', '://nodeuri', 'FoxG 20', '\xf3pez', '/IEC',
                       '?How', '\xf3n', "'2011ISBN", '\x92,\x91fem', '\x92,\x91sg', '\xb4a',
                       '\xb4o', '\xb4on', '\xb4onica', '/10/$25.00 \xa92011 IEEE April 4',
                       '6, 2011, Amman', 'Jordan IEEE EDUCON Education Engineering 2011 \x96 Learning Environments',
                       '\xf1a', '/IEEE', '15, 2011, Rapid City', '@ieec',
                       'SD 978-1-61284-469-5/11/$26.00 \xa92011 IEEE 41 st ASEE /IEEE Frontiers',
                       '/IEEE', '15, 2011, Rapid City', '@deusto',
                       'SD 978-1-61284-469-5/11/$26.00 \xa92011 IEEE 41 st ASEE /IEEE Frontiers',
                       '\xeda', '\xf1a', '2009-2010 course', ':1829)(cid', ':1832)(cid',
                       ':1842)(cid', ':1865)(cid', ':4666)(cid', 'cid :1829)(cid',
                       'cid :1845)', 'cid :1865)(cid', 'cid :3397) 0.95', 'cid :3400)',
                       'cid :3404)', 'cid :4667)', '://nodeuri', 'FoxG 20', '+Jena',
                       'W 3C', 'n p 0', 'p 0', 'S \xb3OiA', '\xb3OiA', '\xf3pez',
                       ':10.1016/j', ':10.3758/BF', 'J .,', 'R .,', 'S .,', '\xe1rcena',
                       '\xeda', '\xf1o', '/IP', '\xb4opez', '/XML', 'gesti \xf3n',
                       'gesti \xf3n del contexto', 'informaci \xf3n', '\xe1n',
                       '\xe1ntica', '\xe1ntico', '\xe9todo', '\xeda', '\xedncrona',
                       '\xedstica', '\xf1adir', '\xf1o', '\xf3dulo', '\xf3n', '\xf3vile',
                       '\xfan', 'K authS ,A', 'K encS ,A', '_time', '3 DU Blocks',
                       '3 DU Blocks Music', '3 DU Blocks library', '3 DU Blocks',
                       '3 DU Blocks Music', '3 DU Blocks library', 'pp', 'bol',
                       'compartici', 'digitale', 'educacio', 'patr', 'virtuale',
                       'electr', 'pr', '/IP', 'ices', 'f', 'satisfacci', 'metodolog',
                       'nominale', 'precisi', 'presi', 'sintagm', 'Se', 'SN', 'Smorph',
                       'aqu', 'autom', 'cc', 'enumeraci', 'enumeracione', 'espa', 'extracci']

    non_significant = ['copy', 'cada', 'muestra', 'gracia', 'funcionamiento', 'pueden'
                       'punto', 'externa', 'tener', 'tecla', 'Ejemplo', 'example',
                       'Tabla',]

    if len(re.findall('^,[A-Za-z]*',term)) > 0:
        bad  = True
    elif len(re.findall('^,\d',term)) > 0:
        bad  = True
    elif len(re.findall('^\[\d+\]',term)) > 0:
        bad  = True
    elif len(re.findall('^\d+.$',term)) > 0:
        bad  = True
    elif len(re.findall('^[A-Z]$',term)) > 0:
        bad = True
    elif len(re.findall('^[a-z]$',term)) > 0:
        bad = True
    elif len(re.findall('^\d+$',term)) > 0:
        bad = True
    elif len(re.findall('^\d+.\d+$',term)) > 0:
        bad = True
    elif len(re.findall('^[^A-Za-z]*$',term)) > 0:
        bad = True
    elif term in invalid_terms:
        bad = True
    elif term in non_significant:
        bad = True
#    elif len(re.findall('^[A-Za-z -]*$',term)) == 0:
#        print "Sospechoso:", term, repr(term)
#        SUSPICIOUS.append(term)

    return bad

def export_csv_undirected(relations):
    if verbose:
        print "\nExporting CSV"
    total_rels = 0
    with open('./data/termRelations.csv', 'wb') as file:
        #writer = csv.writer(csvfile, delimiter=';')
        for terms in relations:
            for term in terms:
                ind = terms.index(term)
                for i in range (ind+1, len(terms)):
                    file.write(term + ';' + terms[i] + '\n')
                    total_rels+=1
    if verbose:
        print "File exported, total relations: " + str(total_rels)

if __name__ == "__main__":
    # Run once this line to get the pdfs
    get_pdfs()
#

#    content = get_pdf_content('./pdf/publications/2006/WebLabEWME2006.pdf')
#    print content
#    keywords = get_terms_topia(content)
#    print keywords
    rels = process_pdfs()
    export_csv_undirected(rels)
#    import pprint
#    pprint.pprint(SUSPICIOUS)<commit_msg>expanded the invalid filter<commit_after># -*- coding: utf-8 -*-
"""
Created on Thu Apr 18 12:29:52 2013

@author: aitor
"""

import re
import urllib2
#import pyPdf
from topia.termextract import extract

import os
from os import listdir
from os.path import isfile, join

import codecs
from cStringIO import StringIO
from pdfminer.converter import TextConverter
from pdfminer.pdfinterp import PDFResourceManager, process_pdf
from pdfminer.layout import LAParams
from pdfminer.pdfparser import PDFSyntaxError
from pdfminer.pdfinterp import PDFTextExtractionNotAllowed

verbose = True

def get_pdfs(include_spanish = False):
    spanish = ['pcuriel_CISTI12', 'ablago2011entorno', 'CAIS2011-07', 'pretel2010framework',
               'jornadasrfid2008_v1.0', 'WebLabTAEE2006']
    url = 'http://www.morelab.deusto.es/'
    path_base = 'publications/'
    years = range(2006, 2014)
    if verbose:
        print "Downloading files"
    for y in years:
        path = path_base + str(y) + '/'
        d = os.path.dirname("./pdf/" + path)
        if not os.path.exists(d):
            os.makedirs(d)
        response = urllib2.urlopen(url+path).read()
        if verbose:
            print response
            print "\n************FILES**********\n"
        for filename in re.findall('"\S+.pdf"', response):
            if not include_spanish and filename in spanish:
                continue
            filename = filename.replace('"', '')
            if verbose:
                print "Filename: " + filename
                print "URL: " + url + path + filename
            f = urllib2.urlopen(url + path + filename)
            data = f.read()
            with open("./pdf/" + path + filename, "wb") as code:
                code.write(data)

def get_pdf_content(path):
    laparams = LAParams()
    rsrc = PDFResourceManager()
    outfp = StringIO()
    try:
        #TODO: detect the encoding of the PDF
        device = TextConverter(rsrc, outfp, codec="cp1252", laparams=laparams)
        process_pdf(rsrc, device, codecs.open(path))
    except (PDFSyntaxError, PDFTextExtractionNotAllowed):
        print "Error processing PDF file: " + path
    return outfp.getvalue()

def get_terms_topia(text):
    extractor = extract.TermExtractor()
    extractor.filter = extract.DefaultFilter(singleStrengthMinOccur=2)
    terms = sorted(extractor(text))
    return terms

def process_pdfs():
    years = range(2006, 2014)
    relations = []
    for y in years:
        file_path = './pdf/publications/' + str(y) + '/'
        filenames = [ f for f in listdir(file_path) if isfile(join(file_path,f)) ]
        for name in filenames:
            if verbose:
                print "Processing" + file_path + name
            content = get_pdf_content(file_path + name)
            terms = get_terms_topia(content)
            #paper_terms = [t[0] for t in terms if t[1] > 2 and not '\\x' in repr(t[0])]
            #still having problems with the encodings of the pdf file, the repr
            #is hopefully a temporal "workaround
            paper_terms = [t[0].strip().lower() for t in terms if t[1] > 4 and not is_bad(t[0].strip()) and not '\\' in repr(t[0])]
            if verbose:
                print "  ->  Total terms: " + str(len(paper_terms))
            relations.append(paper_terms)

    if verbose:
        print "\n\n" + str(len(relations)) + " papers analized"
    return relations

#SUSPICIOUS = []

#curating the terms extracted with topia to delete some incorrect ones
def is_bad(term):
    bad = False
    invalid_terms = ['\\x', '.,', '(cid', '/', '/ i n', '/ f', '</', '><!--',
                       '),', '--></', '.:', '].', '[<', '[</', '</', '|k', '=0.6).',
                       'http ://www', '/ / www', '://drupal', '://www', '="http',
                       '="x', '(?', '),', '->', '2),', '(?', '),', '.,', '(%)', '(1',
                       '|', ',i', '(c', ',5',
                       'W m', 'r e', '2.0-',
                       'Fig', 'Figure', 'figure', 'e.g.', 'i.e.', 'e.g', 'e.g .,',
                       'i.e', 'i.e.,', 'i.e .,',
                       '-2006', '(2006).', '2011)',
                       'ontolo', 're', 'gy', 'http ://www http ://www DRAFT', 'et',
                       'al', '/her', 'Elect', 'tion', '2let', 'st',
                       '08-09', '10-11',
                       'Ordu', 'Garc', 'Universidades 24, 48007, Bilbao', 'tion',
                       'del.icio.u', 'del.icio.u', 'user \x92s', '\xf3pez', '2Object',
                       'EMI 2Object', 'EMI 2Objects', 'EMI 2let', 'EMI 2let Player',
                       'EMI 2let Server', 'EMI 2lets', 'EMI 2lets platform', 'L \xf3pez',
                       '\xe1zquez', '\xf1a', '\xf3pez', 'Web 2.0-', '</ action',
                       '</ argumentList', '>actionName', 'name >actionName',
                       '2.Control', '2Button', '2Peer', 'EMI 2Peer', 'EMI 2Peers',
                       'EMI 2let', 'EMI 2let Player', 'EMI 2let Server', 'EMI 2lets',
                       'EMI 2lets platform', '\xf3pez', '\xf1a', '@eside', '\xeda',
                       '="urn', '2Object', '2Protocol', 'EMI 2Object', 'EMI 2Objects',
                       'EMI 2let', 'EMI 2let Player', 'EMI 2let Server', 'EMI 2let server-side',
                       'EMI 2lets', 'EMI 2lets platform', '\xf1a', '\xf3pez', '\xe1ctica',
                       '\xf1o', '1, MARCH 2005 DRAFT', 'Diego L \xb4 opez', 'Ipi \x98',
                       '/cm', '\xe6a', '09, 2006 Florianopolis', '\xe1ndez', '\xeda',
                       '\xf1a', 'Web 2.0 technologies', '\x92t', ':name', ':type',
                       '://deusto', ':type', 'http ://deusto', 'Web 2.0-', '\x98na',
                       'estaci \xf3n base', 'informaci \xf3n', 'interacci \xf3n',
                       '\xe1mbrica', '\xe1n', '\xe9n', '\xe9tica', '\xe9tico',
                       '\xeda', '\xedda', '\xf1o', '\xf3lo', '\xf3n', '\xfan','2onto',
                       'WWW 2006, Edinburgh', 'del.icio.u', 'folk 2onto',
                       '1-1569228491-2409\xa9SoftCOM', '(URI', '(URI space',
                       '<ITriple', ':Ciudad', 'user \x92s', 'ElderCare \x92s',
                       '\x1berent', ':Pablo', ':Person', ':SecretClas', ':fullName',
                       ':name', 'b 1', 'b 1 injection :name', 'injection :Pablo injection :fullName',
                       'injection :Person', 'injection :SecretClass', 'name 1',
                       'p 1', 'p 1 injection :fullName', 'p 2', '/IEC', '>test',
                       'aplicaci \xf3n', 'evaluaci \xf3n', '\xd3N', '\xe1lculo',
                       '\xe1lisi', '\xe1ndar', '\xe9n', '\xe9trica', '\xeda',
                       '\xedculo', '\xf3dulo', '\xf3n', '\xf3vil', '\xf3vile',
                       '/IEC', '\xe1n', '\xeda', '\xedgrafo', '\xf1o', '\xf3dulo',
                       '\xf3n', '\xf1a', '\xf3pez', '\xf3pez', 'CAIS 2011', "EMS ',",
                       'extracci \xf3n', 'precisi \xf3n', 't \xe9rminos', '\x92,\x92fem',
                       '\x92,\x92sg', '\xe1gina', '\xe1lisi', '\xe1tica', '\xe9dico',
                       '\xe9rmino', '\xe9tera', '\xe9todo', '\xeda', '\xedan', '\xeddo',
                       '\xf3n', '\xf3nica', '\xfamero', '\xfc\xedstica', '(w', 'room 0.11',
                       '(sp', '://nodeuri', 'http ://nodeuri', '\x98na', '\xb4omez',
                       '\x98na', '\xb4omez', '://nodeuri', 'FoxG 20', '\xf3pez', '/IEC',
                       '?How', '\xf3n', "'2011ISBN", '\x92,\x91fem', '\x92,\x91sg', '\xb4a',
                       '\xb4o', '\xb4on', '\xb4onica', '/10/$25.00 \xa92011 IEEE April 4',
                       '6, 2011, Amman', 'Jordan IEEE EDUCON Education Engineering 2011 \x96 Learning Environments',
                       '\xf1a', '/IEEE', '15, 2011, Rapid City', '@ieec',
                       'SD 978-1-61284-469-5/11/$26.00 \xa92011 IEEE 41 st ASEE /IEEE Frontiers',
                       '/IEEE', '15, 2011, Rapid City', '@deusto',
                       'SD 978-1-61284-469-5/11/$26.00 \xa92011 IEEE 41 st ASEE /IEEE Frontiers',
                       '\xeda', '\xf1a', '2009-2010 course', ':1829)(cid', ':1832)(cid',
                       ':1842)(cid', ':1865)(cid', ':4666)(cid', 'cid :1829)(cid',
                       'cid :1845)', 'cid :1865)(cid', 'cid :3397) 0.95', 'cid :3400)',
                       'cid :3404)', 'cid :4667)', '://nodeuri', 'FoxG 20', '+Jena',
                       'W 3C', 'n p 0', 'p 0', 'S \xb3OiA', '\xb3OiA', '\xf3pez',
                       ':10.1016/j', ':10.3758/BF', 'J .,', 'R .,', 'S .,', '\xe1rcena',
                       '\xeda', '\xf1o', '/IP', '\xb4opez', '/XML', 'gesti \xf3n',
                       'gesti \xf3n del contexto', 'informaci \xf3n', '\xe1n',
                       '\xe1ntica', '\xe1ntico', '\xe9todo', '\xeda', '\xedncrona',
                       '\xedstica', '\xf1adir', '\xf1o', '\xf3dulo', '\xf3n', '\xf3vile',
                       '\xfan', 'K authS ,A', 'K encS ,A', '_time', '3 DU Blocks',
                       '3 DU Blocks Music', '3 DU Blocks library', '3 DU Blocks',
                       '3 DU Blocks Music', '3 DU Blocks library', 'pp', 'bol',
                       'compartici', 'digitale', 'educacio', 'patr', 'virtuale',
                       'electr', 'pr', '/IP', 'ices', 'f', 'satisfacci', 'metodolog',
                       'nominale', 'precisi', 'presi', 'sintagm', 'Se', 'SN', 'Smorph',
                       'aqu', 'autom', 'cc', 'enumeraci', 'enumeracione', 'espa', 'extracci',
                       'ject', 'nod', 'Ubiquitou', 'ing', 'dene', 'ob', 'Ipi', 'San',
                       'Ed', 'dened', 'specic', 'Comput', 'ment', 'Twit-er', 'nally',
                       'opez', 'topologie', 'denition', 'i', 'jects', 'Capabilitie',
                       'n', 'ed', 't', 'e', 'Intl', 'Ob', 'folksonomie', 'Commun',
                       'Wirel', 'environ', 'oC', 'tation0', 'aj', 'p', 'b', 'c',
                       'codication', 'l', 'el', 'cl', 'cr', 'insuciencia', 'jur', '/IP',
                       'x', 'J', 'di', 'TalismanPlu', '++', 'ion', 'ice', 'part',
                       'consumer', 'log', 'men', 'ide', 'cca', 'con', 'figu', 'int',
                       'th', 'ab', 'acco', 'agen', 'algo', 'app', 'arch', 'cen', 'compu',
                       'concep', 'cus', 'dep', 'dev', 'ices', 'eb', 'eman', 'eng',
                       'env', 'ironmen', 'ference', 'fo', 'iat', 'ibut', 'ic', 'ical',
                       'icat', 'ider', 'ie', 'ierarchy', 'ifferen', 'ime', 'ine',
                       'ing', 'ines', 'ional', 'itectu', 'ity', 'iv', 'ld', 'lem',
                       '', 'llow', 'locat', 'ies', '', 'logy', 'lt', 'ly', '', 'mo',
                       'mp', 'netwo', 'ou', 'peer-to', 'rd', 'rder', 'rea', 'rep',
                       'rithm', 'rk', 'rmat', 'roach', 'roce', 'roces', 'ropo', 'rov',
                       'te', 'ted', 'tem', 'tere', 'the', 'tho', 'tribu', 'ty', 'umer',
                       'ure', 'xt', 'ystem']

    non_significant = ['copy', 'cada', 'muestra', 'gracia', 'funcionamiento', 'pueden'
                       'punto', 'externa', 'tener', 'tecla', 'Ejemplo', 'example',
                       'Tabla', 'Tim', 'Diego', 'Ignacio', 'Juan', 'Vazquez', 'John',
                       'other', 'Such', 'one', 'May', 'April', 'Matute', 'Reip', 'Vadillo',
                       'Joshi', 'Chen', 'Finin', 'Harry', 'MARCH', 'Roy', 'Zhang',
                       'Davis', 'Springer', 'Avda', 'Iker', 'Larizgoitia',
                       'December', 'Platero', 'derecho', 'frecuencia', 'la', 'Lassila',
                       'Ivan', 'Aitor', 'Pretel', 'Almeida', 'Goiri']

    if len(re.findall('^,[A-Za-z]*',term)) > 0:
        bad  = True
    elif len(re.findall('^,\d',term)) > 0:
        bad  = True
    elif len(re.findall('^\[\d+\]',term)) > 0:
        bad  = True
    elif len(re.findall('^\d+.$',term)) > 0:
        bad  = True
    elif len(re.findall('^[A-Z]$',term)) > 0:
        bad = True
    elif len(re.findall('^[a-z]$',term)) > 0:
        bad = True
    elif len(re.findall('^\d+$',term)) > 0:
        bad = True
    elif len(re.findall('^\d+.\d+$',term)) > 0:
        bad = True
    elif len(re.findall('^[^A-Za-z]*$',term)) > 0:
        bad = True
    elif term in invalid_terms:
        bad = True
    elif term in non_significant:
        bad = True
#    elif len(re.findall('^[A-Za-z -]*$',term)) == 0:
#        print "Sospechoso:", term, repr(term)
#        SUSPICIOUS.append(term)

    return bad

def export_csv_undirected(relations):
    if verbose:
        print "\nExporting CSV"
    total_rels = 0
    with open('./data/termRelations.csv', 'wb') as file:
        #writer = csv.writer(csvfile, delimiter=';')
        for terms in relations:
            for term in terms:
                ind = terms.index(term)
                for i in range (ind+1, len(terms)):
                    file.write(term + ';' + terms[i] + '\n')
                    total_rels+=1
    if verbose:
        print "File exported, total relations: " + str(total_rels)

if __name__ == "__main__":
    #Run once this line to get the pdfs
    #get_pdfs()
#

#    content = get_pdf_content('./pdf/publications/2006/WebLabEWME2006.pdf')
#    print content
#    keywords = get_terms_topia(content)
#    print keywords
    rels = process_pdfs()
    export_csv_undirected(rels)
#    import pprint
#    pprint.pprint(SUSPICIOUS)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Thu Apr 18 12:29:52 2013

@author: aitor
"""

import re
import urllib2
#import pyPdf
from topia.termextract import extract

import os
from os import listdir
from os.path import isfile, join

import codecs
from cStringIO import StringIO
from pdfminer.converter import TextConverter
from pdfminer.pdfinterp import PDFResourceManager, process_pdf
from pdfminer.layout import LAParams
from pdfminer.pdfparser import PDFSyntaxError
from pdfminer.pdfinterp import PDFTextExtractionNotAllowed

import csv

verbose = True

def getPdfs():
    url = 'http://www.morelab.deusto.es/'
    path_base = 'publications/'
    years = range(2006, 2014)
    if verbose:
        print "Downloading files"
    for y in years:
        path = path_base + str(y) + '/'
        d = os.path.dirname("./pdf/" + path)
        if not os.path.exists(d):
            os.makedirs(d)
        response = urllib2.urlopen(url+path).read()
        if verbose:
            print response
            print "\n************FILES**********\n"
        for filename in re.findall('"\S+.pdf"', response):
            filename = filename.replace('"', '')
            if verbose:
                print "Filename: " + filename
                print "URL: " + url + path + filename
            f = urllib2.urlopen(url + path + filename)
            data = f.read()
            with open("./pdf/" + path + filename, "wb") as code:
                code.write(data)

#def get_pdf_content(path):
#    content = ""
#    p = open(path, "rb")
#    pdf = pyPdf.PdfFileReader(p)
#    num_pages = pdf.getNumPages()
#    for i in range(0, num_pages):
#        content += pdf.getPage(i).extractText() + u"\n"
#    # content = " ".join(content.replace(u"\xa0", " ").strip().split())
#    new_content = u""
#    for c in content:
#        if '\\u' not in repr(c):
#            new_content += c
#    return new_content

def get_pdf_content(path):
    laparams = LAParams()
    rsrc = PDFResourceManager()
    outfp = StringIO()
    try:
        #TODO: detect the encoding of the PDF
        device = TextConverter(rsrc, outfp, codec="cp1252", laparams=laparams)
        process_pdf(rsrc, device, codecs.open(path))
    except (PDFSyntaxError, PDFTextExtractionNotAllowed):
        print "Error processing PDF file: " + path
    return outfp.getvalue()

def get_terms_topia(text):
    extractor = extract.TermExtractor()
    extractor.filter = extract.DefaultFilter(singleStrengthMinOccur=2)
    terms = sorted(extractor(text))
    return terms

def process_pdfs():
    years = range(2006, 2014)
    relations = []
    for y in years:
        file_path = './pdf/publications/' + str(y) + '/'
        filenames = [ f for f in listdir(file_path) if isfile(join(file_path,f)) ]
        for name in filenames:
            if verbose:
                print "Procesing" + file_path + name
            content = get_pdf_content(file_path + name)
            terms = get_terms_topia(content)
            #still having problems with the encodings of the pdf file, the second
            #part of the conditons is hopefully a temporal "workaround"
            paper_terms = [t[0] for t in terms if t[1] > 2 and not '\\x' in repr(t[0])]
            if verbose:
                print "  ->  Total terms: " + str(len(paper_terms))
            relations.append(paper_terms)

    if verbose:
        print "\n\n" + str(len(relations)) + " papers analized"
    return relations

def export_csv_undirected(relations):
    if verbose:
        print "\nExporting CSV"
    with open('./data/termRelations.csv', 'wb') as csvfile:
        writer = csv.writer(csvfile, delimiter=';')
        for terms in relations:
            for term in terms:
                ind = terms.index(term)
                for i in range (ind+1, len(terms)):
                    row = [term, terms[i]]
                    writer.writerow(row)

if __name__ == "__main__":
#    content = get_pdf_content('./pdf/publications/2006/WebLabEWME2006.pdf')
#    print content
#    keywords = get_terms_topia(content)
#    print keywords
    rels = process_pdfs()
    export_csv_undirected(rels)<commit_msg>added filter for invalid terms<commit_after># -*- coding: utf-8 -*-
"""
Created on Thu Apr 18 12:29:52 2013

@author: aitor
"""

import re
import urllib2
#import pyPdf
from topia.termextract import extract

import os
from os import listdir
from os.path import isfile, join

import codecs
from cStringIO import StringIO
from pdfminer.converter import TextConverter
from pdfminer.pdfinterp import PDFResourceManager, process_pdf
from pdfminer.layout import LAParams
from pdfminer.pdfparser import PDFSyntaxError
from pdfminer.pdfinterp import PDFTextExtractionNotAllowed

import csv

verbose = True

def getPdfs():
    url = 'http://www.morelab.deusto.es/'
    path_base = 'publications/'
    years = range(2006, 2014)
    if verbose:
        print "Downloading files"
    for y in years:
        path = path_base + str(y) + '/'
        d = os.path.dirname("./pdf/" + path)
        if not os.path.exists(d):
            os.makedirs(d)
        response = urllib2.urlopen(url+path).read()
        if verbose:
            print response
            print "\n************FILES**********\n"
        for filename in re.findall('"\S+.pdf"', response):
            filename = filename.replace('"', '')
            if verbose:
                print "Filename: " + filename
                print "URL: " + url + path + filename
            f = urllib2.urlopen(url + path + filename)
            data = f.read()
            with open("./pdf/" + path + filename, "wb") as code:
                code.write(data)

def get_pdf_content(path):
    laparams = LAParams()
    rsrc = PDFResourceManager()
    outfp = StringIO()
    try:
        #TODO: detect the encoding of the PDF
        device = TextConverter(rsrc, outfp, codec="cp1252", laparams=laparams)
        process_pdf(rsrc, device, codecs.open(path))
    except (PDFSyntaxError, PDFTextExtractionNotAllowed):
        print "Error processing PDF file: " + path
    return outfp.getvalue()

def get_terms_topia(text):
    extractor = extract.TermExtractor()
    extractor.filter = extract.DefaultFilter(singleStrengthMinOccur=2)
    terms = sorted(extractor(text))
    return terms

def process_pdfs():
    years = range(2006, 2014)
    relations = []
    for y in years:
        file_path = './pdf/publications/' + str(y) + '/'
        filenames = [ f for f in listdir(file_path) if isfile(join(file_path,f)) ]
        for name in filenames:
            if verbose:
                print "Procesing" + file_path + name
            content = get_pdf_content(file_path + name)
            terms = get_terms_topia(content)
            #paper_terms = [t[0] for t in terms if t[1] > 2 and not '\\x' in repr(t[0])]
            #still having problems with the encodings of the pdf file, the repr
            #is hopefully a temporal "workaround
            paper_terms = [t[0] for t in terms if t[1] > 2 and not is_bad(t[0]) and not '\\' in repr(t[0])]
            if verbose:
                print "  ->  Total terms: " + str(len(paper_terms))
            relations.append(paper_terms)

    if verbose:
        print "\n\n" + str(len(relations)) + " papers analized"
    return relations

#curating the terms extracted with topia to delete some incorrect ones
def is_bad(term):
    invalid_terms = ['\\x', '.,', '(cid', '/', '/ i n', '/ f', '</', '><!--',
                       '),', '--></', '.:', '].', '[<', '[</', '</', '|k', '=0.6).',
                       'http ://www', '/ / www', '://drupal', '://www', '="http',
                       '="x', '(?', '),', '->', '2),', '(?', '),', '.,',
                       'T', 'P', 'V', 'U', 'l', 'J', 'F', 'L', 'D', 'W', 'S',
                       'b', 'm', 'w', 't', 'W m', 'n', 'r', 'r e', 'g', 'k',
                       '1.', '2.', '3.', '4.', '5.' '6.', '7.', '8.', '2.0-', '[4].',
                       '[1]', '[2]', '[3]', '[4]', '[5]', '[6]', '[7]', '[8]',
                       '[9]', '[10]', '[11]', '[12]', '[13]', '[14]', '[15]',
                       '[16]', '[17]', '[18]', '[19]', '[24]', '[40]',
                       'Fig', 'Figure', 'figure', 'e.g.', 'i.e.',
                       '-2006', '(2006).', '2011)',
                       'ontolo', 're', 'gy', 'http ://www http ://www DRAFT', 'et',
                       'al', '/her',
                       'Ordu', 'Garc']

    if verbose:
        if term in invalid_terms:
            print "Invalid term: " + term
    return term in invalid_terms

def export_csv_undirected(relations):
    if verbose:
        print "\nExporting CSV"
    total_rels = 0
    with open('./data/termRelations.csv', 'wb') as file:
        #writer = csv.writer(csvfile, delimiter=';')
        for terms in relations:
            for term in terms:
                ind = terms.index(term)
                for i in range (ind+1, len(terms)):
                    file.write(term + ';' + terms[i] + '\n')
                    total_rels+=1
    if verbose:
        print "File exported, total relations: " + str(total_rels)

if __name__ == "__main__":
#    content = get_pdf_content('./pdf/publications/2006/WebLabEWME2006.pdf')
#    print content
#    keywords = get_terms_topia(content)
#    print keywords
    rels = process_pdfs()
    export_csv_undirected(rels)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Nicolas Bessi
#            Guewen Baconnier
#    Copyright 2013-2014 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from openerp import models, fields, api

class AccountInvoice(models.Model):
    _inherit = 'account.invoice'

    condition_template1_id = fields.Many2one('base.condition_template',
                                             string='Top conditions')
    condition_template2_id = fields.Many2one('base.condition_template',
                                             string='Bottom conditions')
    note1 = fields.Html('Top conditions')
    note2 = fields.Html('Bottom conditions')

    @api.onchange('condition_template1_id')
    def _set_note1(self):
        condition = self.condition_template1_id
        if condition:
            self.note1 = condition.get_value(self.partner_id.id)

    @api.onchange('condition_template2_id')
    def _set_note2(self):
        condition = self.condition_template2_id
        if condition:
            self.note2 = condition.get_value(self.partner_id.id)
<commit_msg>The model has been renamed<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Nicolas Bessi
#            Guewen Baconnier
#    Copyright 2013-2014 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from openerp import models, fields, api

class AccountInvoice(models.Model):
    _inherit = 'account.invoice'

    condition_template1_id = fields.Many2one('base.condition.template',
                                             string='Top conditions')
    condition_template2_id = fields.Many2one('base.condition.template',
                                             string='Bottom conditions')
    note1 = fields.Html('Top conditions')
    note2 = fields.Html('Bottom conditions')

    @api.onchange('condition_template1_id')
    def _set_note1(self):
        condition = self.condition_template1_id
        if condition:
            self.note1 = condition.get_value(self.partner_id.id)

    @api.onchange('condition_template2_id')
    def _set_note2(self):
        condition = self.condition_template2_id
        if condition:
            self.note2 = condition.get_value(self.partner_id.id)
<|endoftext|>


<commit_before>## This script contains the main logic of the code implementing OpenCV
## It is simple and to the point, outlining all the essential features of the application

import cv2
import numpy as np

def plot(imgs,titles = []):
    num = len(imgs)
    for i in range(num):
        plt.figure(i)
        plt.imshow(imgs[i])
        if len(titles)>0:
            plt.title(titles[i])
    plt.show()

def importImage(filepath):
	"""Imports the img at the filepath to an RGB ndarray."""
	return cv2.imread(fileName,0)

def saveImage(img, filepath, name):
    """Saves the img at the filepath as the given name + filetype."""
    raise IOError

def filterImage(img):
    """Filters the img that the border is easily and uniquely visible."""
	img = img.copy()
	gray = cv2.cvtColor(img,cv2.CV_RGB2GRAY);		#Convert Image captured from Image Input to GrayScale
	edges = cv2.Canny(gray, 100 , 200, 3);			#Apply Canny edge detection on the gray image

	contours, heirarchy = cv2.findContours(edges, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) # Find contours with hierarchy

	#Return approximate polygons
	polys = []
	for c in contours:
		temp = cv2.approxPolyDP(c,0.02,True):
		polys.append(temp[:,0])
	return polys, heirarchy

#Code copied from https://github.com/bharathp666/opencv_qr
def findCorners(img):
	"""Returns Border Poly. Checks that the QR info matches the border found."""
	img = img.copy()
	mark = 0;	#Reset all detected marker count for this frame

	# Get Moments for all Contours and the mass centers
	mu = np.zeros(len(contours))
  	mc = np.zeros(len(contours))

	for i in range(len(contours)):
		mu[i] = cv2.moments( contours[i], false );
		mc[i] = cv2.Point2f( mu[i].m10/mu[i].m00 , mu[i].m01/mu[i].m00 )

	# Start processing the contour data

	# Find Three repeatedly enclosed contours A,B,C
	# NOTE: 1. Contour enclosing other contours is assumed to be the three Alignment markings of the QR code.
	# 2. Alternately, the Ratio of areas of the "concentric" squares can also be used for identifying base Alignment markers.
	# The below demonstrates the first method

	A, B, C = 0, 0, 0
	for i in range(len(contours)):
		k, c = i, 0;

		while hierarchy[k][2] != -1:
			k = hierarchy[k][2] ;
			c = c+1;

		if hierarchy[k][2] != -1: c = c+1;

		if c >= 5:
			if (mark == 0):			A = i;
			else if  (mark == 1)	B = i;		# i.e., A is already found, assign current contour to B
			else if  (mark == 2)	C = i;		# i.e., A and B are already found, assign current contour to C
			mark = mark + 1

	ax = 1
	if mark == 3: return [np.mean(contours[A], axis = ax),np.mean(contours[B], axis = ax),np.mean(contours[C], axis = ax)]
	elif mark > 3: raise Exception("Too many found!")
	else: raise Exception("Not enough markers found!")

def dist(a,b):
	"""Simple 2D Distance Formula"""
	return np.sqrt((a[0]-b[0])**2+(a[1]-b[1])**2)

def fixPerspective(img, border):
	"""Returns img skewed to the border."""
	out = img.copy()
	#Get order of corners
	B = border[1]
	if dst(border[1],border[0])>dst(border[1],border[2]): A,C = border[0],border[1]
	else: A,C = border[2],border[0]

	#Set dst order to match the length ratios of previous (assume 8.5x11 page)
	src = np.array([A,B,C])
	dst = np.array([[0,1100],[0,0],[850,0]])

	#Return Affine Transform
	M = cv2.getAffineTransform(src, dst)
	out = cv2.warpAffine(out, M)
	return out, dst

def drawBorder(img, border):
	"""Returns img with the border drawn overlay."""
	out = img.copy()
	cv2.drawContours(out, border, color=, thickness=, lineType=)
	return out

def cropImage(img, border):
	pass

#User Input
filename = raw_input("Where is the file? ")
savedir = raw_input("Save Directory? ")
name = raw_input("Name (no type)? ")

#Import Image
IMG = importImage(filename)

#Crop and Skew
filtered = filterImage(IMG)
skew, skewBorder = fixPerspective(Filtered,Border)
crop = cropImage(skew,skewBorder)

#Output Information
plot(drawBorder(IMG,Border))
plot(crop)
#saveImage(Crop,savedir,name)
#saveImage(drawBorder(IMG,Border),savedir,name+"(Border)")
<commit_msg>Testing detection with webcam.<commit_after>## This script contains the main logic of the code implementing OpenCV
## It is simple and to the point, outlining all the essential features of the application

import cv2
import numpy as np

def plot(imgs,titles = []):
	num = len(imgs)
	for i in range(num):
		plt.figure(i)
		plt.imshow(imgs[i])
		if len(titles)>0:
			plt.title(titles[i])
	plt.show()

def importImage(filepath):
	"""Imports the img at the filepath to an RGB ndarray."""
	return cv2.imread(fileName,0)

def saveImage(img, filepath, name):
	"""Saves the img at the filepath as the given name + filetype."""
	raise IOError

def filterImage(img):
	"""Filters the img that the border is easily and uniquely visible."""
	img = img.copy()
	gray = cv2.cvtColor(img,cv2.CV_RGB2GRAY);		#Convert Image captured from Image Input to GrayScale
	edges = cv2.Canny(gray, 100 , 200, 3);			#Apply Canny edge detection on the gray image

	contours, heirarchy = cv2.findContours(edges, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) # Find contours with hierarchy

	#Return approximate polygons
	polys = []
	for c in contours:
		temp = cv2.approxPolyDP(c,0.02,True)
		polys.append(temp[:,0])
	return polys, heirarchy

#Code copied and modified from https://github.com/bharathp666/opencv_qr
def findFocusPoints(polys, heirarchy, poly1 = 4, poly2 = 4, num = 4):
	"""Returns Border Poly. Checks that the QR info matches the border found."""
	img = img.copy()

	# Start processing the contour data

	# Find Three repeatedly enclosed contours A,B,C
	# NOTE: 1. Contour enclosing other contours is assumed to be the three Alignment markings of the QR code.
	# 2. Alternately, the Ratio of areas of the "concentric" squares can also be used for identifying base Alignment markers.
	# The below demonstrates the first method

	focus = []
	for i in range(len(contours)):
		k, obj = i, []

		#Add all k indexes to obj
		while hierarchy[k][2] != -1:
			k = hierarchy[k][2]
			obj.append(k)
		if hierarchy[k][2] != -1: obj.append(k)

		#Check if the polys are of the right shape if there are enough of them
		if c >= 5 and checkConvex(obj[-1]) and checkConvex(obj[-3]) and len(obj[-1])==poly2 and len(obj[-3])==poly1:
			focus.append(obj)

	#if mark == num: return [np.mean(i, axis = 1) for i in icnts]
	if len(focus) == num: return focus
	elif len(focus) > num: raise Exception("Too many found!")
	else: raise Exception("Not enough markers found!")

def dist(a,b):
	"""Simple 2D Distance Formula"""
	return np.sqrt((a[0]-b[0])**2+(a[1]-b[1])**2)

def fixPerspective(img, border):
	"""Returns img skewed to the border."""
	out = img.copy()
	#Get order of corners
	B = border[1]
	if dst(border[1],border[0])>dst(border[1],border[2]): A,C = border[0],border[1]
	else: A,C = border[2],border[0]

	#Set dst order to match the length ratios of previous (assume 8.5x11 page)
	src = np.array([A,B,C])
	dst = np.array([[0,1100],[0,0],[850,0]])

	#Return Affine Transform
	M = cv2.getAffineTransform(src, dst)
	out = cv2.warpAffine(out, M)
	return out, dst

def drawBorder(img, border):
	"""Returns img with the border drawn overlay."""
	out = img.copy()
	cv2.drawContours(out, border)
	return out

def cropImage(img, border):
	pass

# -------------------- Working Methods -----------------------
def webcamTest():
	cap = cv2.VideoCapture(0) #Setup webcam

	while(True):
		ret, frame = cap.read() #Capture frame by frame
		out = detect(frame) #Detection method
		cv2.imshow('frame',out) #Display
		if cv2.waitKey(1) & 0xFF == ord('q'): #Exit
			break

	#When everything done, release the capture
	cap.release()
	cv2.destroyAllWindows()

def detect(img):
	polys, heir = findContours(img)
	fp = findFocusPoints(polys, heir)
	for f in fp:
		out = drawBorder(out,f[-3])
	return out

if __name__ == "__main__":
	webcamTest()
<|endoftext|>


<commit_before>from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
import random
import time

class Preferences:
    click_on_results = True # set to False if you don't want to follow links
    email = "" # leave empty if you don't wish to sign in
    max_delay_time = 10 # in seconds
    password = "" # leave empty if you don't wish to sign in
    total_searches = 1000 # so script won't run forever if you forget about it

def setup_browser():
    firefox_profile = FirefoxProfile()
    # Reduce bandwidth consumption and increase speed by disabling images, Flash
    firefox_profile.set_preference('permissions.default.image', 2)
    firefox_profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so',
                                  'false')
    return webdriver.Firefox(firefox_profile)

def perform_search(driver, search_terms, options):
    search_box = driver.find_element_by_id("gbqfq")
    search_box.clear()
    search_box.send_keys(search_terms)
    search_box.send_keys(Keys.RETURN)
    random_delay(options)

def randomly_perform_action():
    return random.randrange(2)

def random_delay(options):
    current_delay = random.randrange(1, options.max_delay_time + 1)
    time.sleep(current_delay)

def press_back(driver, options):
    if randomly_perform_action() == True:
        driver.back()
    else:
        go_to_google_main_page(driver)
    random_delay(options)

def go_to_google_main_page(driver):
    driver.get("https://www.google.com/")

def click_on_result(driver, options):
    if randomly_perform_action() == True and options.click_on_results == True:
        driver.find_element_by_xpath("//*[@id='rso']//h3/a").click()
        random_delay(options)
        press_back(driver, options)
    else:
        random_delay(options)

def login_to_google_account(driver, options):
    if options.email != "" and options.password != "":
        driver.find_element_by_id("gb_70").click()
        driver.find_element_by_id("Email").send_keys(options.email)
        driver.find_element_by_id("Passwd").send_keys(options.password)
        driver.find_element_by_id("signIn").click()

def main():
    options = Preferences()
    driver = setup_browser()
    i = 0
    go_to_google_main_page(driver)
    login_to_google_account(driver, options)

    while(i < options.total_searches):
        search_terms = str(i)
        perform_search(driver, search_terms, options)
        click_on_result(driver, options)
        i+=1

if __name__ == "__main__":
    main()
<commit_msg>Now actually clicks on random results<commit_after>from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
import random
import time

class Preferences:
    click_on_results = True # set to False if you don't want to follow links
    email = "" # leave empty if you don't wish to sign in
    max_delay_time = 15 # in seconds
    password = "" # leave empty if you don't wish to sign in
    total_searches = 1000 # so script won't run forever if you forget about it

def setup_browser():
    firefox_profile = FirefoxProfile()
    # Reduce bandwidth consumption and increase speed by disabling images, Flash
    firefox_profile.set_preference('permissions.default.image', 2)
    firefox_profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so',
                                  'false')
    return webdriver.Firefox(firefox_profile)

def perform_search(driver, search_terms, options):
    search_box = driver.find_element_by_id("gbqfq")
    search_box.clear()
    search_box.send_keys(search_terms)
    search_box.send_keys(Keys.RETURN)
    random_delay(options)

def randomly_perform_action():
    return random.randrange(2)

def random_delay(options):
    current_delay = random.randrange(1, options.max_delay_time + 1)
    time.sleep(current_delay)

def press_back(driver, options):
    if randomly_perform_action() == True:
        driver.back()
    else:
        go_to_google_main_page(driver)
    random_delay(options)

def go_to_google_main_page(driver):
    driver.get("https://www.google.com/")

def click_on_result(driver, options):
    if randomly_perform_action() == True and options.click_on_results == True:
        all_results = driver.find_elements_by_xpath("//div/h3/a")
        result_to_click = random.randrange(0, len(all_results))
        print result_to_click
        all_results[result_to_click].click()
        random_delay(options)
        press_back(driver, options)
    else:
        random_delay(options)

def login_to_google_account(driver, options):
    if options.email != "" and options.password != "":
        driver.find_element_by_id("gb_70").click()
        driver.find_element_by_id("Email").send_keys(options.email)
        driver.find_element_by_id("Passwd").send_keys(options.password)
        driver.find_element_by_id("signIn").click()

def main():
    options = Preferences()
    driver = setup_browser()
    i = 0
    go_to_google_main_page(driver)
    login_to_google_account(driver, options)

    while(i < options.total_searches):
        search_terms = str(i)
        perform_search(driver, search_terms, options)
        click_on_result(driver, options)
        i+=1

if __name__ == "__main__":
    main()
<|endoftext|>