<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2010-2014, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""
Scenario calculator core functionality
"""
import collections
import random

from openquake.nrmllib.hazard.parsers import RuptureModelParser

# HAZARDLIB
from openquake.hazardlib.calc import filters
from openquake.hazardlib.calc.gmf import GmfComputer
from openquake.hazardlib.imt import from_string
import openquake.hazardlib.gsim

from openquake.commonlib.general import split_on_blocks, distinct
from openquake.commonlib import source

from openquake.engine.calculators.hazard import general as haz_general
from openquake.engine.utils import tasks, config
from openquake.engine.db import models
from openquake.engine import writer
from openquake.engine.performance import EnginePerformanceMonitor

AVAILABLE_GSIMS = openquake.hazardlib.gsim.get_available_gsims()

@tasks.oqtask
def gmfs(job_id, ses_ruptures, sitecol, gmf_id, task_no):
    """
    :param int job_id: the current job ID
    :param ses_ruptures: a set of `SESRupture` instances
    :param sitecol: a `SiteCollection` instance
    :param int gmf_id: the ID of a `Gmf` instance
    :param int task_no: the task number
    """
    hc = models.HazardCalculation.objects.get(oqjob=job_id)
    # distinct is here to make sure that IMTs such as
    # SA(0.8) and SA(0.80) are considered the same
    imts = distinct(from_string(x) for x in hc.intensity_measure_types)
    gsim = AVAILABLE_GSIMS[hc.gsim]()  # instantiate the GSIM class
    correlation_model = hc.get_correl_model()

    cache = collections.defaultdict(list)  # {site_id, imt -> gmvs}
    inserter = writer.CacheInserter(models.GmfData, 1000)
    # insert GmfData in blocks of 1000 sites

    # NB: ses_ruptures a non-empty list produced by the block_splitter
    rupture = ses_ruptures[0].rupture  # ProbabilisticRupture instance
    with EnginePerformanceMonitor('computing gmfs', job_id, gmfs):
        gmf = GmfComputer(rupture, sitecol, imts, [gsim], hc.truncation_level,
                          correlation_model)
        gname = gsim.__class__.__name__
        for ses_rup in ses_ruptures:
            gmf_dict = gmf.compute(ses_rup.seed)
            for gname, imt in gmf_dict:
                for site_id, gmv in zip(sitecol.sids, gmf_dict[gname, imt]):
                    # float may be needed below to convert 1x1 matrices
                    cache[site_id, imt].append((gmv, ses_rup.id))

    with EnginePerformanceMonitor('saving gmfs', job_id, gmfs):
        for (site_id, imt), data in cache.iteritems():
            gmvs, rup_ids = zip(*data)
            inserter.add(
                models.GmfData(
                    gmf_id=gmf_id,
                    task_no=task_no,
                    imt=imt[0],
                    sa_period=imt[1],
                    sa_damping=imt[2],
                    site_id=site_id,
                    rupture_ids=rup_ids,
                    gmvs=gmvs))
        inserter.flush()

class ScenarioHazardCalculator(haz_general.BaseHazardCalculator):
    """
    Scenario hazard calculator. Computes ground motion fields.
    """

    core_calc_task = gmfs
    output = None  # defined in pre_execute

    def __init__(self, *args, **kwargs):
        super(ScenarioHazardCalculator, self).__init__(*args, **kwargs)
        self.gmf = None
        self.rupture = None

    def initialize_sources(self):
        """
        Get the rupture_model file from the job.ini file, and set the
        attribute self.rupture.
        """
        nrml = RuptureModelParser(self.hc.inputs['rupture_model']).parse()
        self.rupture = source.NrmlHazardlibConverter(
            self.hc.investigation_time,
            self.hc.rupture_mesh_spacing,
            self.hc.width_of_mfd_bin,
            self.hc.area_source_discretization,
        )(nrml)

    def initialize_realizations(self):
        """There are no realizations for the scenario calculator"""
        pass

    def pre_execute(self):
        """
        Do pre-execution work. At the moment, this work entails:
        parsing and initializing sources, parsing and initializing the
        site model (if there is one), parsing vulnerability and
        exposure files, and generating logic tree realizations. (The
        latter piece basically defines the work to be done in the
        `execute` phase.)
        """
        self.parse_risk_models()
        self.initialize_sources()
        self.initialize_site_model()
        self.create_ruptures()

    def create_ruptures(self):
        # check filtering
        hc = self.hc
        if hc.maximum_distance:
            self.sites = filters.filter_sites_by_distance_to_rupture(
                self.rupture, hc.maximum_distance, hc.site_collection)
            if self.sites is None:
                raise RuntimeError(
                    'All sites where filtered out! '
                    'maximum_distance=%s km' % hc.maximum_distance)

        # create ses output
        output = models.Output.objects.create(
            oq_job=self.job,
            display_name='SES Collection',
            output_type='ses')
        self.ses_coll = models.SESCollection.objects.create(
            output=output, lt_model=None, ordinal=0)

        # create gmf output
        output = models.Output.objects.create(
            oq_job=self.job,
            display_name="GMF",
            output_type="gmf_scenario")
        self.gmf = models.Gmf.objects.create(output=output)

        # creating seeds
        rnd = random.Random()
        rnd.seed(self.hc.random_seed)
        all_seeds = [
            rnd.randint(0, models.MAX_SINT_32)
            for _ in xrange(self.hc.number_of_ground_motion_fields)]

        with self.monitor('saving ruptures'):
            # in order to save a ProbabilisticRupture, a TrtModel is needed;
            # here we generate a fake one, corresponding to the tectonic
            # region type NA i.e. Not Available
            trt_model = models.TrtModel.objects.create(
                tectonic_region_type='NA',
                num_sources=0,
                num_ruptures=len(all_seeds),
                min_mag=self.rupture.mag,
                max_mag=self.rupture.mag,
                gsims=[self.hc.gsim])
            prob_rup = models.ProbabilisticRupture.create(
                self.rupture, self.ses_coll, trt_model)
            inserter = writer.CacheInserter(models.SESRupture, 100000)
            for ses_idx, seed in enumerate(all_seeds):
                inserter.add(
                    models.SESRupture(
                        ses_id=1, rupture=prob_rup,
                        tag='scenario-%010d' % ses_idx,  seed=seed))
            inserter.flush()

    def task_arg_gen(self):
        """
        Yield a tuple of the form (job_id, sitecol, rupture_id, gmf_id,
        task_seed, num_realizations). `task_seed` will be used to seed
        numpy for temporal occurence sampling. Only a single task
        will be generated which is fine since the computation is fast
        anyway.
        """
        ses_ruptures = models.SESRupture.objects.filter(
            rupture__ses_collection=self.ses_coll.id)
        for task_no, ruptures in enumerate(
                split_on_blocks(ses_ruptures, self.concurrent_tasks)):
            yield self.job.id, ruptures, self.sites, self.gmf.id, task_no

    def task_completed(self, result):
        """Do nothing"""
<commit_msg>Fixed a misprint<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2010-2014, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""
Scenario calculator core functionality
"""
import collections
import random

from openquake.nrmllib.hazard.parsers import RuptureModelParser

# HAZARDLIB
from openquake.hazardlib.calc import filters
from openquake.hazardlib.calc.gmf import GmfComputer
from openquake.hazardlib.imt import from_string
import openquake.hazardlib.gsim

from openquake.commonlib.general import split_in_blocks, distinct
from openquake.commonlib import source

from openquake.engine.calculators.hazard import general as haz_general
from openquake.engine.utils import tasks, config
from openquake.engine.db import models
from openquake.engine import writer
from openquake.engine.performance import EnginePerformanceMonitor

AVAILABLE_GSIMS = openquake.hazardlib.gsim.get_available_gsims()

@tasks.oqtask
def gmfs(job_id, ses_ruptures, sitecol, gmf_id, task_no):
    """
    :param int job_id: the current job ID
    :param ses_ruptures: a set of `SESRupture` instances
    :param sitecol: a `SiteCollection` instance
    :param int gmf_id: the ID of a `Gmf` instance
    :param int task_no: the task number
    """
    hc = models.HazardCalculation.objects.get(oqjob=job_id)
    # distinct is here to make sure that IMTs such as
    # SA(0.8) and SA(0.80) are considered the same
    imts = distinct(from_string(x) for x in hc.intensity_measure_types)
    gsim = AVAILABLE_GSIMS[hc.gsim]()  # instantiate the GSIM class
    correlation_model = hc.get_correl_model()

    cache = collections.defaultdict(list)  # {site_id, imt -> gmvs}
    inserter = writer.CacheInserter(models.GmfData, 1000)
    # insert GmfData in blocks of 1000 sites

    # NB: ses_ruptures a non-empty list produced by the block_splitter
    rupture = ses_ruptures[0].rupture  # ProbabilisticRupture instance
    with EnginePerformanceMonitor('computing gmfs', job_id, gmfs):
        gmf = GmfComputer(rupture, sitecol, imts, [gsim], hc.truncation_level,
                          correlation_model)
        gname = gsim.__class__.__name__
        for ses_rup in ses_ruptures:
            gmf_dict = gmf.compute(ses_rup.seed)
            for gname, imt in gmf_dict:
                for site_id, gmv in zip(sitecol.sids, gmf_dict[gname, imt]):
                    # float may be needed below to convert 1x1 matrices
                    cache[site_id, imt].append((gmv, ses_rup.id))

    with EnginePerformanceMonitor('saving gmfs', job_id, gmfs):
        for (site_id, imt), data in cache.iteritems():
            gmvs, rup_ids = zip(*data)
            inserter.add(
                models.GmfData(
                    gmf_id=gmf_id,
                    task_no=task_no,
                    imt=imt[0],
                    sa_period=imt[1],
                    sa_damping=imt[2],
                    site_id=site_id,
                    rupture_ids=rup_ids,
                    gmvs=gmvs))
        inserter.flush()

class ScenarioHazardCalculator(haz_general.BaseHazardCalculator):
    """
    Scenario hazard calculator. Computes ground motion fields.
    """

    core_calc_task = gmfs
    output = None  # defined in pre_execute

    def __init__(self, *args, **kwargs):
        super(ScenarioHazardCalculator, self).__init__(*args, **kwargs)
        self.gmf = None
        self.rupture = None

    def initialize_sources(self):
        """
        Get the rupture_model file from the job.ini file, and set the
        attribute self.rupture.
        """
        nrml = RuptureModelParser(self.hc.inputs['rupture_model']).parse()
        self.rupture = source.NrmlHazardlibConverter(
            self.hc.investigation_time,
            self.hc.rupture_mesh_spacing,
            self.hc.width_of_mfd_bin,
            self.hc.area_source_discretization,
        )(nrml)

    def initialize_realizations(self):
        """There are no realizations for the scenario calculator"""
        pass

    def pre_execute(self):
        """
        Do pre-execution work. At the moment, this work entails:
        parsing and initializing sources, parsing and initializing the
        site model (if there is one), parsing vulnerability and
        exposure files, and generating logic tree realizations. (The
        latter piece basically defines the work to be done in the
        `execute` phase.)
        """
        self.parse_risk_models()
        self.initialize_sources()
        self.initialize_site_model()
        self.create_ruptures()

    def create_ruptures(self):
        # check filtering
        hc = self.hc
        if hc.maximum_distance:
            self.sites = filters.filter_sites_by_distance_to_rupture(
                self.rupture, hc.maximum_distance, hc.site_collection)
            if self.sites is None:
                raise RuntimeError(
                    'All sites where filtered out! '
                    'maximum_distance=%s km' % hc.maximum_distance)

        # create ses output
        output = models.Output.objects.create(
            oq_job=self.job,
            display_name='SES Collection',
            output_type='ses')
        self.ses_coll = models.SESCollection.objects.create(
            output=output, lt_model=None, ordinal=0)

        # create gmf output
        output = models.Output.objects.create(
            oq_job=self.job,
            display_name="GMF",
            output_type="gmf_scenario")
        self.gmf = models.Gmf.objects.create(output=output)

        # creating seeds
        rnd = random.Random()
        rnd.seed(self.hc.random_seed)
        all_seeds = [
            rnd.randint(0, models.MAX_SINT_32)
            for _ in xrange(self.hc.number_of_ground_motion_fields)]

        with self.monitor('saving ruptures'):
            # in order to save a ProbabilisticRupture, a TrtModel is needed;
            # here we generate a fake one, corresponding to the tectonic
            # region type NA i.e. Not Available
            trt_model = models.TrtModel.objects.create(
                tectonic_region_type='NA',
                num_sources=0,
                num_ruptures=len(all_seeds),
                min_mag=self.rupture.mag,
                max_mag=self.rupture.mag,
                gsims=[self.hc.gsim])
            prob_rup = models.ProbabilisticRupture.create(
                self.rupture, self.ses_coll, trt_model)
            inserter = writer.CacheInserter(models.SESRupture, 100000)
            for ses_idx, seed in enumerate(all_seeds):
                inserter.add(
                    models.SESRupture(
                        ses_id=1, rupture=prob_rup,
                        tag='scenario-%010d' % ses_idx,  seed=seed))
            inserter.flush()

    def task_arg_gen(self):
        """
        Yield a tuple of the form (job_id, sitecol, rupture_id, gmf_id,
        task_seed, num_realizations). `task_seed` will be used to seed
        numpy for temporal occurence sampling. Only a single task
        will be generated which is fine since the computation is fast
        anyway.
        """
        ses_ruptures = models.SESRupture.objects.filter(
            rupture__ses_collection=self.ses_coll.id)
        for task_no, ruptures in enumerate(
                split_in_blocks(ses_ruptures, self.concurrent_tasks)):
            yield self.job.id, ruptures, self.sites, self.gmf.id, task_no

    def task_completed(self, result):
        """Do nothing"""
<|endoftext|>


<commit_before>"""Computes the least-squares optimal Keplerian elements for a sequence of
   cartesian position observations.
"""

import math
#import numpy as np
#Thinly-wrapped numpy
import autograd.numpy as np
import matplotlib.pyplot as plt
from autograd import grad
from autograd import jacobian
from autograd import elementwise_grad as egrad

# convention:
# a: semi-major axis
# e: eccentricity
# eps: mean longitude at epoch
# Euler angles:
# I: inclination
# Omega: longitude of ascending node
# omega: argument of pericenter

#rotation about the z-axis about an angle `ang`
def rotz(ang):
    cos_ang = np.cos(ang)
    sin_ang = np.sin(ang)
    return np.array(((cos_ang,-sin_ang,0.0), (sin_ang, cos_ang,0.0), (0.0,0.0,1.0)))

#rotation about the x-axis about an angle `ang`
def rotx(ang):
    cos_ang = np.cos(ang)
    sin_ang = np.sin(ang)
    return np.array(((1.0,0.0,0.0), (0.0,cos_ang,-sin_ang), (0.0,sin_ang,cos_ang)))

#rotation from the orbital plane to the inertial frame
#it is composed of the following rotations, in that order:
#1) rotation about the z axis about an angle `omega` (argument of pericenter)
#2) rotation about the x axis about an angle `I` (inclination)
#3) rotation about the z axis about an angle `Omega` (longitude of ascending node)
def orbplane2frame_(omega,I,Omega):
    P2_mul_P3 = np.matmul(rotx(I),rotz(omega))
    return np.matmul(rotz(Omega),P2_mul_P3)

def orbplane2frame(x):
    return orbplane2frame_(x[0],x[1],x[2])

def kep_r_(a, e, f):
    return a*(1.0-e**2)/(1.0+e*np.cos(f))

def kep_r(x):
    return kep_r_(x[0],x[1],x[2])

def xyz_orbplane_(a, e, f):
    r = kep_r_(a, e, f)
    return np.array((r*np.cos(f),r*np.sin(f),0.0))

def xyz_orbplane(x):
    return xyz_orbplane_(x[0],x[1],x[2])

def xyz_frame_(a,e,f,omega,I,Omega):
    return np.matmul( orbplane2frame_(omega,I,Omega) , xyz_orbplane_(a, e, f) )

def xyz_frame(x):
    return np.matmul( orbplane2frame(x[3:6]) , xyz_orbplane(x[0:3]) )

def meanmotion(mu,a):
    return np.sqrt(mu/(a**3))

# # TODO:
# # write function to compute true anomaly as a function of time-of-fly
# # write function to compute range as a function of orbital elements
# # write function which takes observed values and computes the difference wrt expected-to-be-observed values as a function of unknown orbital elements (to be fitted)
# # compute Q as a function of unknown orbital elements (to be fitted)
# # optimize Q -> return fitted orbital elements (requires an ansatz: take input from minimalistic Gibb's?)

# NOTES:
# matrix multiplication of numpy's 2-D arrays is done through `np.matmul`

omega = np.radians(31.124)
I = np.radians(75.0)
Omega = np.radians(60.0)

# rotation matrix from orbital plane to inertial frame
# two ways to compute it; result should be the same
P_1 = rotz(omega) #rotation about z axis by an angle `omega`
P_2 = rotx(I) #rotation about x axis by an angle `I`
P_3 = rotz(Omega) #rotation about z axis by an angle `Omega`

Rot1 = np.matmul(P_3,np.matmul(P_2,P_1))
Rot2 = orbplane2frame_(omega,I,Omega)

v = np.array((3.0,-2.0,1.0))

print(I)
print(omega)
print(Omega)

print(Rot1)

print(np.matmul(Rot1,v))

print(Rot2)

# #import autograd.numpy as np  # Thinly-wrapped numpy
# #from autograd import grad    # The only autograd function you may ever need
# def tanh(x):                 # Define a function
#     y = np.exp(-2.0 * x)
#     return (1.0 - y) / (1.0 + y)

# grad_tanh = grad(tanh)       # Obtain its gradient function
# print(grad_tanh(1.0))

# def mycos(x):
#     return np.cos(x)

# drotz = grad(mycos)

# print(drotz(0.1))

drotz = jacobian(orbplane2frame_)
drotz2 = jacobian(orbplane2frame)

print('op2f=',orbplane2frame_(np.radians(10.0),np.radians(-31.124),np.radians(200.001)))

# print('drotz=',drotz(np.radians(10.0),np.radians(-31.124),np.radians(200.001)))

bbb= np.array( (np.radians(10.0),np.radians(-31.124),np.radians(200.001)) )
#bbb= np.array( (np.radians(0.0),np.radians(0.0),np.radians(0.0)) )
print('drotz2(bbb)=',drotz2( bbb ))

#print(dkep_r)
#print(kep_r_(1.0,0.1,np.radians(-29.99)))
#print(dkep_r(1.0,0.1,np.radians(-29.99)))

aaa = np.array((1.0,0.45,np.radians(1.0)))
# def printz(x):
#     return print(str(x)+' = ',x)
# printz(aaa)

print('aaa = ', aaa)
print('kep_r(aaa) = ',kep_r(aaa))

dkep_r = grad(kep_r)

print('dkep_r(aaa) = ',dkep_r(aaa))

print('xyz_orbplane(aaa) = ',xyz_orbplane(aaa))

dxyz_orbplane = jacobian(xyz_orbplane)

print('dxyz_orbplane(aaa) = ',dxyz_orbplane(aaa))

dxyz_orbplane_ = jacobian(xyz_orbplane_)

print('dxyz_orbplane_(aaa) = ',dxyz_orbplane_(aaa[0],aaa[1],aaa[2]))

ccc = np.array((aaa[0],aaa[1],aaa[2],bbb[0],bbb[1],bbb[2]))

print('xyz_frame(ccc) = ',xyz_frame(ccc))

print('xyz_frame_((aaa[0],aaa[1],aaa[2],bbb[0],bbb[1],bbb[2])) = ',xyz_frame((aaa[0],aaa[1],aaa[2],bbb[0],bbb[1],bbb[2])))

dxyz_frame = jacobian(xyz_frame)

print('dxyz_frame(ccc) = ',dxyz_frame(ccc))

print('meanmotion(1.0,1.1) = ', meanmotion(1.0,1.1))
<commit_msg>update TODOs<commit_after>"""Computes the least-squares optimal Keplerian elements for a sequence of
   cartesian position observations.
"""

import math
#import numpy as np
#Thinly-wrapped numpy
import autograd.numpy as np
import matplotlib.pyplot as plt
from autograd import grad
from autograd import jacobian
from autograd import elementwise_grad as egrad

# convention:
# a: semi-major axis
# e: eccentricity
# eps: mean longitude at epoch
# taup: time of pericenter passage
# Euler angles:
# I: inclination
# Omega: longitude of ascending node
# omega: argument of pericenter

#rotation about the z-axis about an angle `ang`
def rotz(ang):
    cos_ang = np.cos(ang)
    sin_ang = np.sin(ang)
    return np.array(((cos_ang,-sin_ang,0.0), (sin_ang, cos_ang,0.0), (0.0,0.0,1.0)))

#rotation about the x-axis about an angle `ang`
def rotx(ang):
    cos_ang = np.cos(ang)
    sin_ang = np.sin(ang)
    return np.array(((1.0,0.0,0.0), (0.0,cos_ang,-sin_ang), (0.0,sin_ang,cos_ang)))

#rotation from the orbital plane to the inertial frame
#it is composed of the following rotations, in that order:
#1) rotation about the z axis about an angle `omega` (argument of pericenter)
#2) rotation about the x axis about an angle `I` (inclination)
#3) rotation about the z axis about an angle `Omega` (longitude of ascending node)
def orbplane2frame_(omega,I,Omega):
    P2_mul_P3 = np.matmul(rotx(I),rotz(omega))
    return np.matmul(rotz(Omega),P2_mul_P3)

def orbplane2frame(x):
    return orbplane2frame_(x[0],x[1],x[2])

def kep_r_(a, e, f):
    return a*(1.0-e**2)/(1.0+e*np.cos(f))

def kep_r(x):
    return kep_r_(x[0],x[1],x[2])

def xyz_orbplane_(a, e, f):
    r = kep_r_(a, e, f)
    return np.array((r*np.cos(f),r*np.sin(f),0.0))

def xyz_orbplane(x):
    return xyz_orbplane_(x[0],x[1],x[2])

def xyz_frame_(a,e,f,omega,I,Omega):
    return np.matmul( orbplane2frame_(omega,I,Omega) , xyz_orbplane_(a, e, f) )

def xyz_frame(x):
    return np.matmul( orbplane2frame(x[3:6]) , xyz_orbplane(x[0:3]) )

def meanmotion(mu,a):
    return np.sqrt(mu/(a**3))

# # TODO:
# # write function to compute range as a function of orbital elements: DONE
# # write function to compute true anomaly as a function of time-of-fly
# # the following transformation is needed: from time t, to mean anomaly M,
# # to eccentric anomaly E, to true anomaly f
# # t -> M=n*(t-taup) -> M=E-e*sin(E) (invert) ->
# # -> f = 2*atan(  sqrt((1+e)/(1-e))*tan(E/2)  )
# # write function which takes observed values and computes the difference wrt expected-to-be-observed values as a function of unknown orbital elements (to be fitted)
# # compute Q as a function of unknown orbital elements (to be fitted)
# # optimize Q -> return fitted orbital elements (requires an ansatz: take input from minimalistic Gibb's?)

# NOTES:
# matrix multiplication of numpy's 2-D arrays is done through `np.matmul`

omega = np.radians(31.124)
I = np.radians(75.0)
Omega = np.radians(60.0)

# rotation matrix from orbital plane to inertial frame
# two ways to compute it; result should be the same
P_1 = rotz(omega) #rotation about z axis by an angle `omega`
P_2 = rotx(I) #rotation about x axis by an angle `I`
P_3 = rotz(Omega) #rotation about z axis by an angle `Omega`

Rot1 = np.matmul(P_3,np.matmul(P_2,P_1))
Rot2 = orbplane2frame_(omega,I,Omega)

v = np.array((3.0,-2.0,1.0))

print(I)
print(omega)
print(Omega)

print(Rot1)

print(np.matmul(Rot1,v))

print(Rot2)

# #import autograd.numpy as np  # Thinly-wrapped numpy
# #from autograd import grad    # The only autograd function you may ever need
# def tanh(x):                 # Define a function
#     y = np.exp(-2.0 * x)
#     return (1.0 - y) / (1.0 + y)

# grad_tanh = grad(tanh)       # Obtain its gradient function
# print(grad_tanh(1.0))

# def mycos(x):
#     return np.cos(x)

# drotz = grad(mycos)

# print(drotz(0.1))

drotz = jacobian(orbplane2frame_)
drotz2 = jacobian(orbplane2frame)

print('op2f=',orbplane2frame_(np.radians(10.0),np.radians(-31.124),np.radians(200.001)))

# print('drotz=',drotz(np.radians(10.0),np.radians(-31.124),np.radians(200.001)))

bbb= np.array( (np.radians(10.0),np.radians(-31.124),np.radians(200.001)) )
#bbb= np.array( (np.radians(0.0),np.radians(0.0),np.radians(0.0)) )
print('drotz2(bbb)=',drotz2( bbb ))

#print(dkep_r)
#print(kep_r_(1.0,0.1,np.radians(-29.99)))
#print(dkep_r(1.0,0.1,np.radians(-29.99)))

aaa = np.array((1.0,0.45,np.radians(1.0)))
# def printz(x):
#     return print(str(x)+' = ',x)
# printz(aaa)

print('aaa = ', aaa)
print('kep_r(aaa) = ',kep_r(aaa))

dkep_r = grad(kep_r)

print('dkep_r(aaa) = ',dkep_r(aaa))

print('xyz_orbplane(aaa) = ',xyz_orbplane(aaa))

dxyz_orbplane = jacobian(xyz_orbplane)

print('dxyz_orbplane(aaa) = ',dxyz_orbplane(aaa))

dxyz_orbplane_ = jacobian(xyz_orbplane_)

print('dxyz_orbplane_(aaa) = ',dxyz_orbplane_(aaa[0],aaa[1],aaa[2]))

ccc = np.array((aaa[0],aaa[1],aaa[2],bbb[0],bbb[1],bbb[2]))

print('xyz_frame(ccc) = ',xyz_frame(ccc))

print('xyz_frame_((aaa[0],aaa[1],aaa[2],bbb[0],bbb[1],bbb[2])) = ',xyz_frame((aaa[0],aaa[1],aaa[2],bbb[0],bbb[1],bbb[2])))

dxyz_frame = jacobian(xyz_frame)

print('dxyz_frame(ccc) = ',dxyz_frame(ccc))

print('meanmotion(1.0,1.1) = ', meanmotion(1.0,1.1))
<|endoftext|>


<commit_before>'''
Created on Aug 17, 2013

@author: family
'''
import shutil
import logging
import fnmatch
import os
import time
from datetime import date, timedelta
import multiprocessing
import glob

_logger = logging.getLogger(__name__)

SECONDS = 1
HOURS = 60 * 60 * SECONDS

MAX_AGE_TO_COPY = 12 * HOURS
MIN_AGE_TO_COPY = 60 * SECONDS
HISTORY_FILE_AGE_THRESHOLD = 12 * HOURS

COPY_POOL_SIZE = 10

class CopyRules(object):
    def __init__(self,src_path,dst_path,is_history_path=False):
        self.src_path = src_path
        self.dst_path = dst_path
        self.is_history_path = is_history_path

    def file_should_be_copied(self):
        print 'file_should_be_copied called'
        return (
                self._file_does_not_exist(self.dst_path) and
                self._file_is_not_too_old(self.src_path) and
                self._file_is_not_too_young(self.src_path) and
                self._history_file_does_not_exist()
                )

    def _file_does_not_exist(self,path):
        retval = not os.path.exists(path)
        print '_file_does_not_exist ', retval
        return retval

    def _file_is_not_too_old(self,path):
        file_modified_time = os.path.getmtime(path)
        time_now = time.time()
        retval = file_modified_time > (time_now - MAX_AGE_TO_COPY)
        print '_file_is_not_too_old ', retval
        return retval

    def _file_is_not_too_young(self,path):
        file_modified_time = os.path.getmtime(path)
        time_now = time.time()
        retval = file_modified_time < (time_now - MIN_AGE_TO_COPY)
        print '_file_is_not_too_young', file_modified_time < (time_now - MIN_AGE_TO_COPY)
        return retval

    def _path_in_yesterdays_history_directory(self,path_in_todays_directory):
        yesterday = date.today() - timedelta(1)
        hist_dir_name_yesterday = yesterday.strftime('%Y%m%d')
        history_path,filename = os.path.split(self.dst_path)
        history_base,dummy = os.path.split(history_path)
        return os.path.join(history_base,hist_dir_name_yesterday,filename)

    def _history_file_does_not_exist(self):
        if not self.is_history_path:
            retval = True
        else:
            path_in_yesterdays_history = self._path_in_yesterdays_history_directory(self.dst_path)
            if not os.path.exists(path_in_yesterdays_history):
                retval = True #History file does not exist in yesterday's directory
            else:
                file_modified_time = os.path.getmtime(path_in_yesterdays_history)
                time_now = time.time()
                retval = file_modified_time < (time_now - HISTORY_FILE_AGE_THRESHOLD)
        print '_history_file_does_not_exist ',retval # History file effectively doesn't exist if it is older than threshold
        return retval

class Destination(object):

    def __init__(self,source_spec,source_path,dest_spec,dest_is_history_path):
        self.source_spec = source_spec
        self.source_path = source_path
        self.dest_spec = dest_spec
        self.dest_is_history_path = dest_is_history_path
        self.path = self._path()

    def file_should_be_copied(self):
#        print 'self.destspec: %s,self.path: %s' % (self.dest_spec,self.path)
        rules = CopyRules(self.source_path,self.path,self.dest_is_history_path)
        return rules.file_should_be_copied()

    def _path(self):
        path = self._path_after_wildcard_substitution()
        if self.dest_is_history_path:
            path = self._make_path_in_dated_directory(path)
        return path

    def _path_after_wildcard_substitution(self):
        '''
        Return the destination path for the copied file

        Characters that match wildcards in the source spec are substituted for wildcards in the dest
        spec.
        E.g. source spec: a*.txt
        actual file: a123.txt
        dest spec: b*.txt
        returned value: b123.txt

        Special rules apply if the destination path is for the history location. In this case the
        characters that match the wildcards are appended to the end of the filename, but only if
        the dest spec contains no wildcard.
        '''
        # If dest spec has no wildcards and this isn't the history location
        # then no substitution is required so just return the spec
        dest_contains_wildcard = (self.dest_spec.count('*') != 0)
        if not dest_contains_wildcard and not self.dest_is_history_path:
            dest = self.dest_spec
        else:
            # Get the characters making up the first * in the source spec
            star_chars = self._characters_matching_wildcard(self.source_spec, self.source_path)
            if dest_contains_wildcard:
                dest = self.dest_spec.replace('*',star_chars)
            else:
                dest = self.dest_spec + star_chars
        return dest

    def _characters_matching_wildcard(self, source_spec, actual_filename):
        star_pos = source_spec.find('*')
        chars_after_star = len(source_spec) - star_pos - 1
        star_chars = actual_filename[star_pos:len(actual_filename) - chars_after_star]
        return star_chars

    def _make_path_in_dated_directory(self,path):
        '''
        Create a dated directory if one doesn't already exist and return the path modified to point to it

        e.g. if path is /a_directory/a_file
        and date is 2013-10-29
        then return value is /a_directory/20131029/a_file
        '''
        (dirname,fname) = os.path.split(path)
        dated_subdir = date.today().strftime('%Y%m%d')
        dated_path = os.path.join(dirname,dated_subdir)
        if not os.path.exists(dated_path):
            os.mkdir(dated_path)
        return os.path.join(dirname,dated_subdir,fname)

def copy_file_task(src,dst,queue):
    try:
        shutil.copyfile(src, dst)
        queue.put(['success',src,dst])
    except Exception as e:
        queue.put(['failure',e])
    finally:
        time.sleep(0.01)

class FileCopier(object):
    '''
    TODO:
    - if dest file already exists then delete before copying
    '''

    def __init__(self,file_copy_spec,file_copied_callback = None):
        '''
        file_copy_spec is dictionary indexed by full source path containing list of full
        destination paths
        '''
        self.file_copy_spec = file_copy_spec
        self.file_copied_callback = file_copied_callback
        self.pool = multiprocessing.Pool(processes=COPY_POOL_SIZE)
        self.copy_processes_active = 0
        mgr = multiprocessing.Manager()
        self.queue = mgr.Queue()

    def _copy_file(self,src,dst):
        _logger.info('Copying from %s to %s',src,dst)
#        shutil.copyfile(src,dst)
#        self.pool.apply_async(shutil.copyfile, (src,dst))
        self.copy_processes_active += 1
        self.pool.apply_async(copy_file_task, (src, dst, self.queue))
#        self._copy_file_task(src, dst)

    def _wildcard_match(self,spec,target):
        '''
        Does the target match the spec?

        '''
        return fnmatch.fnmatch(target, spec)

    def _find_copy_spec(self,source_path):
        for source_spec in self.file_copy_spec:
            if self._wildcard_match(source_spec,source_path):
                return self.file_copy_spec[source_spec], source_spec
        return None, None

    def _process_one_destination(self,dest_path,source_path,source_spec,dest_is_history_path):
        if dest_path is not None:
            _logger.debug('Processing destination path: %s', dest_path)
            destination = Destination(source_spec,source_path,dest_path,dest_is_history_path)
            if destination.file_should_be_copied():
                _logger.debug('Copying %s to %s',source_path,destination.path)
                self._copy_file(source_path,destination.path)

    def copy(self,source_path):
        '''
        Copy the file at source_path to all specified destinations

        Destinations are specified when a FileCopier instance is created.
        Destinations are identified by a wildcard search on the spec keys
        '''
        copy_spec, source_spec = self._find_copy_spec(source_path)
        if copy_spec is None:
            return
        _logger.debug('Spec found for %s',source_path)
        self._copy(source_path, source_spec, copy_spec)

    def _copy(self,source_path,source_spec,copy_spec):
        for idx, dest_path in enumerate(copy_spec):
            dest_is_history_path = idx == 1
            self._process_one_destination(dest_path,source_path,source_spec,dest_is_history_path)

    def check_copy_status(self):
        while not self.queue.empty():
            result = self.queue.get()
            print 'Result retrieved: ', result
            self.copy_processes_active -= 1
            if result[0] == 'success' and self.file_copied_callback is not None:
                dst = result[2]
                self.file_copied_callback(dst)

    def flush(self):
        '''
        Wait for all copy processes to complete
        '''
        while self.copy_processes_active > 0:
            self.check_copy_status()
            time.sleep(0.01)

    def poll(self):
        '''
        Check whether there are files to copy and if so copy them
        '''
        for destination in self.file_copy_spec:
            matching_files = glob.glob(destination)
            for path in matching_files:
                self._copy(path,destination,self.file_copy_spec[destination])

    def close(self):
        '''
        Wait for any in-progress work to complete. This is needed for unit tests.
        '''
        _logger.debug('FileCopier.close()')
        self.pool.close()
        self.pool.join()<commit_msg>Define message values. Remove unnecessary timeout.<commit_after>'''
Created on Aug 17, 2013

@author: family
'''
import shutil
import logging
import fnmatch
import os
import time
from datetime import date, timedelta
import multiprocessing
import glob

_logger = logging.getLogger(__name__)

SECONDS = 1
HOURS = 60 * 60 * SECONDS

MAX_AGE_TO_COPY = 12 * HOURS
MIN_AGE_TO_COPY = 60 * SECONDS
HISTORY_FILE_AGE_THRESHOLD = 12 * HOURS

MSG_SUCCESS = 'success'
MSG_FAILURE = 'failure'

COPY_POOL_SIZE = 10

class CopyRules(object):
    def __init__(self,src_path,dst_path,is_history_path=False):
        self.src_path = src_path
        self.dst_path = dst_path
        self.is_history_path = is_history_path

    def file_should_be_copied(self):
        print 'file_should_be_copied called'
        return (
                self._file_does_not_exist(self.dst_path) and
                self._file_is_not_too_old(self.src_path) and
                self._file_is_not_too_young(self.src_path) and
                self._history_file_does_not_exist()
                )

    def _file_does_not_exist(self,path):
        retval = not os.path.exists(path)
        print '_file_does_not_exist ', retval
        return retval

    def _file_is_not_too_old(self,path):
        file_modified_time = os.path.getmtime(path)
        time_now = time.time()
        retval = file_modified_time > (time_now - MAX_AGE_TO_COPY)
        print '_file_is_not_too_old ', retval
        return retval

    def _file_is_not_too_young(self,path):
        file_modified_time = os.path.getmtime(path)
        time_now = time.time()
        retval = file_modified_time < (time_now - MIN_AGE_TO_COPY)
        print '_file_is_not_too_young', file_modified_time < (time_now - MIN_AGE_TO_COPY)
        return retval

    def _path_in_yesterdays_history_directory(self,path_in_todays_directory):
        yesterday = date.today() - timedelta(1)
        hist_dir_name_yesterday = yesterday.strftime('%Y%m%d')
        history_path,filename = os.path.split(self.dst_path)
        history_base,dummy = os.path.split(history_path)
        return os.path.join(history_base,hist_dir_name_yesterday,filename)

    def _history_file_does_not_exist(self):
        if not self.is_history_path:
            retval = True
        else:
            path_in_yesterdays_history = self._path_in_yesterdays_history_directory(self.dst_path)
            if not os.path.exists(path_in_yesterdays_history):
                retval = True #History file does not exist in yesterday's directory
            else:
                file_modified_time = os.path.getmtime(path_in_yesterdays_history)
                time_now = time.time()
                retval = file_modified_time < (time_now - HISTORY_FILE_AGE_THRESHOLD)
        print '_history_file_does_not_exist ',retval # History file effectively doesn't exist if it is older than threshold
        return retval

class Destination(object):

    def __init__(self,source_spec,source_path,dest_spec,dest_is_history_path):
        self.source_spec = source_spec
        self.source_path = source_path
        self.dest_spec = dest_spec
        self.dest_is_history_path = dest_is_history_path
        self.path = self._path()

    def file_should_be_copied(self):
#        print 'self.destspec: %s,self.path: %s' % (self.dest_spec,self.path)
        rules = CopyRules(self.source_path,self.path,self.dest_is_history_path)
        return rules.file_should_be_copied()

    def _path(self):
        path = self._path_after_wildcard_substitution()
        if self.dest_is_history_path:
            path = self._make_path_in_dated_directory(path)
        return path

    def _path_after_wildcard_substitution(self):
        '''
        Return the destination path for the copied file

        Characters that match wildcards in the source spec are substituted for wildcards in the dest
        spec.
        E.g. source spec: a*.txt
        actual file: a123.txt
        dest spec: b*.txt
        returned value: b123.txt

        Special rules apply if the destination path is for the history location. In this case the
        characters that match the wildcards are appended to the end of the filename, but only if
        the dest spec contains no wildcard.
        '''
        # If dest spec has no wildcards and this isn't the history location
        # then no substitution is required so just return the spec
        dest_contains_wildcard = (self.dest_spec.count('*') != 0)
        if not dest_contains_wildcard and not self.dest_is_history_path:
            dest = self.dest_spec
        else:
            # Get the characters making up the first * in the source spec
            star_chars = self._characters_matching_wildcard(self.source_spec, self.source_path)
            if dest_contains_wildcard:
                dest = self.dest_spec.replace('*',star_chars)
            else:
                dest = self.dest_spec + star_chars
        return dest

    def _characters_matching_wildcard(self, source_spec, actual_filename):
        star_pos = source_spec.find('*')
        chars_after_star = len(source_spec) - star_pos - 1
        star_chars = actual_filename[star_pos:len(actual_filename) - chars_after_star]
        return star_chars

    def _make_path_in_dated_directory(self,path):
        '''
        Create a dated directory if one doesn't already exist and return the path modified to point to it

        e.g. if path is /a_directory/a_file
        and date is 2013-10-29
        then return value is /a_directory/20131029/a_file
        '''
        (dirname,fname) = os.path.split(path)
        dated_subdir = date.today().strftime('%Y%m%d')
        dated_path = os.path.join(dirname,dated_subdir)
        if not os.path.exists(dated_path):
            os.mkdir(dated_path)
        return os.path.join(dirname,dated_subdir,fname)

def copy_file_task(src,dst,queue):
    try:
        shutil.copyfile(src, dst)
        queue.put([MSG_SUCCESS,src,dst])
    except Exception as e:
        queue.put([MSG_FAILURE,e])

class FileCopier(object):
    '''
    TODO:
    - if dest file already exists then delete before copying
    '''

    def __init__(self,file_copy_spec,file_copied_callback = None):
        '''
        file_copy_spec is dictionary indexed by full source path containing list of full
        destination paths
        '''
        self.file_copy_spec = file_copy_spec
        self.file_copied_callback = file_copied_callback
        self.pool = multiprocessing.Pool(processes=COPY_POOL_SIZE)
        self.copy_processes_active = 0
        mgr = multiprocessing.Manager()
        self.queue = mgr.Queue()

    def _copy_file(self,src,dst):
        _logger.info('Copying from %s to %s',src,dst)
#        shutil.copyfile(src,dst)
#        self.pool.apply_async(shutil.copyfile, (src,dst))
        self.copy_processes_active += 1
        self.pool.apply_async(copy_file_task, (src, dst, self.queue))
#        self._copy_file_task(src, dst)

    def _wildcard_match(self,spec,target):
        '''
        Does the target match the spec?

        '''
        return fnmatch.fnmatch(target, spec)

    def _find_copy_spec(self,source_path):
        for source_spec in self.file_copy_spec:
            if self._wildcard_match(source_spec,source_path):
                return self.file_copy_spec[source_spec], source_spec
        return None, None

    def _process_one_destination(self,dest_path,source_path,source_spec,dest_is_history_path):
        if dest_path is not None:
            _logger.debug('Processing destination path: %s', dest_path)
            destination = Destination(source_spec,source_path,dest_path,dest_is_history_path)
            if destination.file_should_be_copied():
                _logger.debug('Copying %s to %s',source_path,destination.path)
                self._copy_file(source_path,destination.path)

    def copy(self,source_path):
        '''
        Copy the file at source_path to all specified destinations

        Destinations are specified when a FileCopier instance is created.
        Destinations are identified by a wildcard search on the spec keys
        '''
        copy_spec, source_spec = self._find_copy_spec(source_path)
        if copy_spec is None:
            return
        _logger.debug('Spec found for %s',source_path)
        self._copy(source_path, source_spec, copy_spec)

    def _copy(self,source_path,source_spec,copy_spec):
        for idx, dest_path in enumerate(copy_spec):
            dest_is_history_path = idx == 1
            self._process_one_destination(dest_path,source_path,source_spec,dest_is_history_path)

    def check_copy_status(self):
        while not self.queue.empty():
            result = self.queue.get()
            print 'Result retrieved: ', result
            self.copy_processes_active -= 1
            if result[0] == MSG_SUCCESS and self.file_copied_callback is not None:
                dst = result[2]
                self.file_copied_callback(dst)

    def flush(self):
        '''
        Wait for all copy processes to complete
        '''
        while self.copy_processes_active > 0:
            self.check_copy_status()
            time.sleep(0.01)

    def poll(self):
        '''
        Check whether there are files to copy and if so copy them
        '''
        for destination in self.file_copy_spec:
            matching_files = glob.glob(destination)
            for path in matching_files:
                self._copy(path,destination,self.file_copy_spec[destination])

    def close(self):
        '''
        Wait for any in-progress work to complete. This is needed for unit tests.
        '''
        _logger.debug('FileCopier.close()')
        self.pool.close()
        self.pool.join()<|endoftext|>


<commit_before>import ipaddress
from typing import Union

async def fetch_from_database(conn, ip):
    messages = await conn.fetchrow(
        "SELECT * FROM ip_storage WHERE ip = $1", ip
    )

    return messages

async def fetch_formattor(ip_dict):
    formatted_ip_dict = {"ip": await ip_formattor(ip_dict["ip"]),
                         "country": ip_dict["country_code"],
                         "last_updated": ip_dict["last_updated"],
                         "amount_checked": ip_dict["accessed"]}

    return formatted_ip_dict

async def ip_formattor(ip_address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address]):
    if ip_address.version == 4:
        return await ipv4_formattor(ip_address)
    else:
        return await ipv6_formattor(ip_address)

async def ipv4_formattor(ip_address: ipaddress.IPv4Address):
    formatted_ipv4 = {"version": ip_address.version,
                      "max_prefixlen": ip_address.max_prefixlen,
                      "compressed": ip_address.compressed,
                      "exploded": ip_address.exploded,
                      "reverse_pointer": ip_address.reverse_pointer,
                      "is_multicast": ip_address.is_multicast,
                      "is_private": ip_address.is_private,
                      "is_global": ip_address.is_global,
                      "is_unspecified": ip_address.is_unspecified,
                      "is_reserved": ip_address.is_reserved,
                      "is_loopback": ip_address.is_loopback,
                      "is_link_local": ip_address.is_link_local}
    return formatted_ipv4

async def ipv6_formattor(ip_address: ipaddress.IPv6Address):
    formatted_ipv6 = {"version": ip_address.version,
                      "max_prefixlen": ip_address.max_prefixlen,
                      "compressed": ip_address.compressed,
                      "exploded": ip_address.exploded,

                      "reverse_pointer": ip_address.reverse_pointer,
                      "is_multicast": ip_address.is_multicast,
                      "is_private": ip_address.is_private,
                      "is_global": ip_address.is_global,
                      "is_unspecified": ip_address.is_unspecified,
                      "is_reserved": ip_address.is_reserved,
                      "is_loopback": ip_address.is_loopback,
                      "is_link_local": ip_address.is_link_local,
                      "is_site_local": ip_address.is_site_local,

                      "ipv4_mapped": ip_address.ipv4_mapped,
                      "sixtofour": ip_address.sixtofour,
                      "teredo": ip_address.teredo}

    return formatted_ipv6

async def update_entry(conn, ip, current_total):
    await conn.execute("""
    UPDATE ip_storage
        SET accessed = accessed + 1
    WHERE ip = $1
        AND accessed = $2
    """, ip, current_total)

async def insert_into_database(conn, ip, country):
    await conn.execute("""
    INSERT INTO ip_storage(ip, country_code) VALUES($1, $2)""",
                       ip, country)

async def country_count_stats(conn, country=None):
    if country is None:
        return await conn.fetch("""
        SELECT COUNT(country_code), country_code FROM ip_storage GROUP BY country_code ORDER BY 1 DESC""")
    else:
        return await conn.fetchval("""
        SELECT COUNT(country_code) FROM ip_storage WHERE country_code = $1""", country)
<commit_msg>enable support for single and multi country requests<commit_after>import ipaddress
from typing import Union

async def fetch_from_database(conn, ip):
    messages = await conn.fetchrow(
        "SELECT * FROM ip_storage WHERE ip = $1", ip
    )

    return messages

async def fetch_formattor(ip_dict):
    formatted_ip_dict = {"ip": await ip_formattor(ip_dict["ip"]),
                         "country": ip_dict["country_code"],
                         "last_updated": ip_dict["last_updated"],
                         "amount_checked": ip_dict["accessed"]}

    return formatted_ip_dict

async def ip_formattor(ip_address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address]):
    if ip_address.version == 4:
        return await ipv4_formattor(ip_address)
    else:
        return await ipv6_formattor(ip_address)

async def ipv4_formattor(ip_address: ipaddress.IPv4Address):
    formatted_ipv4 = {"version": ip_address.version,
                      "max_prefixlen": ip_address.max_prefixlen,
                      "compressed": ip_address.compressed,
                      "exploded": ip_address.exploded,
                      "reverse_pointer": ip_address.reverse_pointer,
                      "is_multicast": ip_address.is_multicast,
                      "is_private": ip_address.is_private,
                      "is_global": ip_address.is_global,
                      "is_unspecified": ip_address.is_unspecified,
                      "is_reserved": ip_address.is_reserved,
                      "is_loopback": ip_address.is_loopback,
                      "is_link_local": ip_address.is_link_local}
    return formatted_ipv4

async def ipv6_formattor(ip_address: ipaddress.IPv6Address):
    formatted_ipv6 = {"version": ip_address.version,
                      "max_prefixlen": ip_address.max_prefixlen,
                      "compressed": ip_address.compressed,
                      "exploded": ip_address.exploded,

                      "reverse_pointer": ip_address.reverse_pointer,
                      "is_multicast": ip_address.is_multicast,
                      "is_private": ip_address.is_private,
                      "is_global": ip_address.is_global,
                      "is_unspecified": ip_address.is_unspecified,
                      "is_reserved": ip_address.is_reserved,
                      "is_loopback": ip_address.is_loopback,
                      "is_link_local": ip_address.is_link_local,
                      "is_site_local": ip_address.is_site_local,

                      "ipv4_mapped": ip_address.ipv4_mapped,
                      "sixtofour": ip_address.sixtofour,
                      "teredo": ip_address.teredo}

    return formatted_ipv6

async def update_entry(conn, ip, current_total):
    await conn.execute("""
    UPDATE ip_storage
        SET accessed = accessed + 1
    WHERE ip = $1
        AND accessed = $2
    """, ip, current_total)

async def insert_into_database(conn, ip, country):
    await conn.execute("""
    INSERT INTO ip_storage(ip, country_code) VALUES($1, $2)""",
                       ip, country)

async def list_data_into_str(data_list: list):
    base_string = ""
    for item in data_list:
        base_string = base_string + item + ", "
    base_string = base_string[:-2]
    return base_string

async def country_count_stats(conn, country=None):
    if country is None:
        return await conn.fetch("""
        SELECT COUNT(country_code),country_code
        FROM ip_storage
        GROUP BY country_code
        ORDER BY 1 DESC""")
    else:
        if len(country) == 1:
            return await conn.fetchval("""
            SELECT COUNT(country_code)
            FROM ip_storage
            WHERE country_code = $1""", country)
        else:
            return await conn.fetch("""
            SELECT COUNT(country_code), country_code
            FROM ip_storage
            WHERE country_code in ($1)
            GROUP BY country_code
            ORDER BY 1 DESC""", await list_data_into_str(country))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Generates fingerprints of text document.
"""
import string
import sys

__author__ = 'kailash.buki@gmail.com (Kailash Budhathoki)'
__all__ = [
    'Fingerprint',
    'FingerprintException',
]

guid = lambda x: ord(x)

class FingerprintException(Exception):

    def __init__(self, *args, **kwargs):
        Exception.__init__(self, *args, **kwargs)

class Fingerprint(object):
    """
    Generates fingerprints of the input text file or plain string. Please consider taking a look at http://theory.stanford.edu/~aiken/publications/papers/sigmod03.pdf for detailed description on how fingerprints are computed.

    Attributes:
        kgram_len (Optional[int]): length of the contiguous substring. Defaults to 50.
        base (Optional[int]): base required for computing the rolling hash function. Defaults to 101.
        modulo (Optional[int]): hash values cannot exceed this value. Defaults to sys.maxint.
        window_len (Optional[len]): length of the windows when computing fingerprints. Defaults to 100.
        kgrams (List(str)): k-grams extracted from the text
        hashes (List(int)): hash values of the k-grams
        fingerprints (List(tuple(int))): selected representative hash values along with their positions.
    """

    def __init__(self, kgram_len=None, base=None, modulo=None, window_len=None):
        self.kgram_len = kgram_len or 50
        self.base = base or 101
        self.modulo = modulo or sys.maxsize
        self.window_len = window_len or 100

    def get_min_with_pos(self, sequence):
        min_val = sys.maxsize
        min_pos = 0
        for pos, val in enumerate(sequence):
            if val <= min_val:
                min_val = val
                min_pos = pos
        return min_val, min_pos

    def normal_hash(self, kgram):
        hash = 0
        for i, c in enumerate(kgram):
            hash += guid(c) * self.base ** (self.kgram_len - 1 - i)
        hash = hash % self.modulo
        return hash

    def rolling_hash(self, old_hash, del_char, new_char):
        # more powerful version of rolling hash
        hash = ((old_hash - guid(del_char) * self.base **
                 self.kgram_len) + guid(new_char)) * self.base
        hash = hash % self.modulo
        return hash

    def prepare_storage(self):
        self.kgrams = []
        self.hashes = []
        self.fingerprints = []
        self.str = ""

    def generate_kgrams(self):
        self.kgrams = [self.str[i:i + self.kgram_len]
                       for i in range(len(self.str) - self.kgram_len + 1)]

    def hash_kgrams(self):
        prev_kgram = self.kgrams[0]
        prev_hash = self.normal_hash(prev_kgram)
        self.hashes.append(prev_hash)

        for cur_kgram in self.kgrams[1:]:
            prev_hash = self.rolling_hash(
                prev_hash, prev_kgram[0], cur_kgram[-1])
            self.hashes.append(prev_hash)
            prev_kgram = cur_kgram

    def generate_fingerprints(self):
        windows = [self.hashes[i:i + self.window_len]
                   for i in range(len(self.hashes) - self.window_len + 1)]

        cur_min_hash, cur_min_pos = self.get_min_with_pos(windows[0])
        self.fingerprints.append((cur_min_hash, cur_min_pos))

        for i, window in enumerate(windows[1:]):
            min_hash, min_pos = self.get_min_with_pos(window)
            min_pos += i + 1
            if min_hash != cur_min_hash or min_hash == cur_min_hash and min_pos > cur_min_pos:
                cur_min_hash, cur_min_pos = min_hash, min_pos
                self.fingerprints.append((min_hash, min_pos))

    def validate_config(self):
        if len(self.str) < self.window_len:
            raise FingerprintException(
                "Length of the string is smaller than the length of the window.")

    def sanitize(self, str):
        sanitized = ""
        exclude = string.punctuation
        for c in str:
            if c not in exclude and c not in ('\n', '\r', ' '):
                sanitized += c
        return sanitized

    def generate(self, str=None, fpath=None):
        """generates fingerprints of the input. Either provide `str` to compute fingerprint directly from your string or `fpath` to compute fingerprint from the text of the file. Make sure to have your text decoded in `utf-8` format if you pass the input string.

        Args:
            str (Optional(str)): string whose fingerprint is to be computed.
            fpath (Optional(str)): absolute path of the text file whose fingerprint is to be computed.

        Returns:
            List(int): fingerprints of the input.

        Raises:
            FingerprintException: If the input string do not meet the requirements of parameters provided for fingerprinting.
        """
        self.prepare_storage()
        self.str = self.load_file(fpath) if fpath else self.sanitize(str)
        self.validate_config()
        self.generate_kgrams()
        self.hash_kgrams()
        self.generate_fingerprints()
        return self.fingerprints

    def load_file(self, fpath):
        with open(fpath, 'r') as fp:
            data = fp.read()
        data = data.encode().decode('utf-8')
        return data
<commit_msg>Fix incorrect rolling hash calculation<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Generates fingerprints of text document.
"""
import string
import sys

__author__ = 'kailash.buki@gmail.com (Kailash Budhathoki)'
__all__ = [
    'Fingerprint',
    'FingerprintException',
]

guid = lambda x: ord(x)

class FingerprintException(Exception):

    def __init__(self, *args, **kwargs):
        Exception.__init__(self, *args, **kwargs)

class Fingerprint(object):
    """
    Generates fingerprints of the input text file or plain string. Please consider taking a look at http://theory.stanford.edu/~aiken/publications/papers/sigmod03.pdf for detailed description on how fingerprints are computed.

    Attributes:
        kgram_len (Optional[int]): length of the contiguous substring. Defaults to 50.
        base (Optional[int]): base required for computing the rolling hash function. Defaults to 101.
        modulo (Optional[int]): hash values cannot exceed this value. Defaults to sys.maxint.
        window_len (Optional[len]): length of the windows when computing fingerprints. Defaults to 100.
        kgrams (List(str)): k-grams extracted from the text
        hashes (List(int)): hash values of the k-grams
        fingerprints (List(tuple(int))): selected representative hash values along with their positions.
    """

    def __init__(self, kgram_len=None, base=None, modulo=None, window_len=None):
        self.kgram_len = kgram_len or 50
        self.base = base or 101
        self.modulo = modulo or sys.maxsize
        self.window_len = window_len or 100

    def get_min_with_pos(self, sequence):
        min_val = sys.maxsize
        min_pos = 0
        for pos, val in enumerate(sequence):
            if val <= min_val:
                min_val = val
                min_pos = pos
        return min_val, min_pos

    def normal_hash(self, kgram):
        hash = 0
        for i, c in enumerate(kgram):
            hash += guid(c) * self.base ** (self.kgram_len - 1 - i)
        hash = hash % self.modulo
        return hash

    def rolling_hash(self, old_hash, del_char, new_char):
        # more powerful version of rolling hash
        hash = (old_hash - guid(del_char) * self.base **
                 (self.kgram_len-1)) * self.base + guid(new_char)
        hash = hash % self.modulo
        return hash

    def prepare_storage(self):
        self.kgrams = []
        self.hashes = []
        self.fingerprints = []
        self.str = ""

    def generate_kgrams(self):
        self.kgrams = [self.str[i:i + self.kgram_len]
                       for i in range(len(self.str) - self.kgram_len + 1)]

    def hash_kgrams(self):
        prev_kgram = self.kgrams[0]
        prev_hash = self.normal_hash(prev_kgram)
        self.hashes.append(prev_hash)

        for cur_kgram in self.kgrams[1:]:
            prev_hash = self.rolling_hash(
                prev_hash, prev_kgram[0], cur_kgram[-1])
            self.hashes.append(prev_hash)
            prev_kgram = cur_kgram

    def generate_fingerprints(self):
        windows = [self.hashes[i:i + self.window_len]
                   for i in range(len(self.hashes) - self.window_len + 1)]

        cur_min_hash, cur_min_pos = self.get_min_with_pos(windows[0])
        self.fingerprints.append((cur_min_hash, cur_min_pos))

        for i, window in enumerate(windows[1:]):
            min_hash, min_pos = self.get_min_with_pos(window)
            min_pos += i + 1
            if min_hash != cur_min_hash or min_hash == cur_min_hash and min_pos > cur_min_pos:
                cur_min_hash, cur_min_pos = min_hash, min_pos
                self.fingerprints.append((min_hash, min_pos))

    def validate_config(self):
        if len(self.str) < self.window_len:
            raise FingerprintException(
                "Length of the string is smaller than the length of the window.")

    def sanitize(self, str):
        sanitized = ""
        exclude = string.punctuation
        for c in str:
            if c not in exclude and c not in ('\n', '\r', ' '):
                sanitized += c
        return sanitized

    def generate(self, str=None, fpath=None):
        """generates fingerprints of the input. Either provide `str` to compute fingerprint directly from your string or `fpath` to compute fingerprint from the text of the file. Make sure to have your text decoded in `utf-8` format if you pass the input string.

        Args:
            str (Optional(str)): string whose fingerprint is to be computed.
            fpath (Optional(str)): absolute path of the text file whose fingerprint is to be computed.

        Returns:
            List(int): fingerprints of the input.

        Raises:
            FingerprintException: If the input string do not meet the requirements of parameters provided for fingerprinting.
        """
        self.prepare_storage()
        self.str = self.load_file(fpath) if fpath else self.sanitize(str)
        self.validate_config()
        self.generate_kgrams()
        self.hash_kgrams()
        self.generate_fingerprints()
        return self.fingerprints

    def load_file(self, fpath):
        with open(fpath, 'r') as fp:
            data = fp.read()
        data = data.encode().decode('utf-8')
        return data
<|endoftext|>


<commit_before># system imports
import datetime
import numpy as np
import pdb

# local imports
import Maybe

class FoldResult(object):
    '''Computations on a cross validation fold.'''

    def __init__(self):
        # vectors of results
        self.actuals = np.array([])
        self.estimates = np.array([])
        # dictionaries of results
        self.fitted = {}
        self.predictor_names = {}
        self.num_test = {}
        self.num_train = {}

    def save_num_test(self, date, num_test):
        self.num_test[date] = num_test

    def get_num_test(self):
        return self.num_test

    def save_num_train(self, date, num_train):
        self.num_train[date] = num_train

    def get_num_train(self):
        return self.num_train

    def save_fitted(self, date, fitted):
        self.fitted[date] = fitted

    def get_fitted(self):
        return self.fitted

    def save_predictor_names(self, date, predictor_names):
        self.predictor_names[date] = predictor_names

    def get_predictor_names(self):
        return self.predictor_names

    def extend(self, actuals, estimates):
        '''Extend the collection of values.

        Args
        actuals      1D np.array
        estimates    1D np.array

        Returns: None
        '''
        if len(actuals) != len(estimates):
            print 'len(actuals)', len(actuals)
            print 'len(estimates)', len(estimates)
            raise ValueError('lengths differ')
        self.actuals = np.append(self.actuals, actuals)
        self.estimates = np.append(self.estimates, estimates)

    def maybe_errors(self):
        '''Return Maybe(vector of errors, without nans).'''
        errors = self.actuals - self.estimates
        if np.isnan(errors).all():
            # return NoValue, if
            # - errors.size == 0 OR
            # - every element of errors is nan
            # NOTE: The one test does it all
            return Maybe.NoValue()
        else:
            errors_without_nans = errors[~np.isnan(errors)]
            return Maybe.Maybe(errors_without_nans)

if __name__ == '__main__':
    import unittest

    if False:
        pdb.set_trace()  # avoid warning

    class Test(unittest.TestCase):

        def setUp(self):
            self.verbose = False
            self.group1_actuals = np.array([1, 2, np.nan, 4])
            self.group1_estimates = np.array([2, 3, 4, np.nan])

            self.group2_actuals = np.array([10])
            self.group2_estimates = np.array([20])

            # build the FoldResult object
            fr = FoldResult()
            fr.extend(self.group1_actuals,
                      self.group1_estimates)
            fr.extend(self.group2_actuals,
                      self.group2_estimates)
            self.fr = fr

        def test_fitted(self):
            # test methods .save_fitted() and .get_fitted()
            fr = self.fr
            fitted1 = ['a', 'b']
            date1 = datetime.date(2015, 05, 18)
            fr.save_fitted(date1, fitted1)
            fitted2 = ['aa', 'bb']
            date2 = datetime.date(2015, 05, 19)
            fr.save_fitted(date2, fitted2)
            fitted = fr.get_fitted()  # return a dict
            self.assertEqual(len(fitted), 2)

        def test_predictor_names(self):
            # test methods .save_preictor_names() and .get_predictor_names()
            fr = self.fr
            names1 = ['a', 'b']
            date1 = datetime.date(2015, 05, 18)
            fr.save_predictor_names(date1, names1)
            names2 = ['aa']
            date2 = datetime.date(2015, 05, 19)
            fr.save_predictor_names(date2, names2)
            predictor_names = fr.get_predictor_names()
            self.assertEqual(len(predictor_names), 2)

        def test_num_test(self):
            fr = self.fr
            fr.save_num_test(datetime.date(2015, 05, 18), 10)
            fr.save_num_test(datetime.date(2015, 05, 19), 20)
            num_tests = fr.get_num_test()
            self.assertEqual(len(num_tests), 2)

        def test_num_train(self):
            fr = self.fr
            fr.save_num_train(datetime.date(2015, 05, 18), 10)
            fr.save_num_train(datetime.date(2015, 05, 19), 20)
            num_trains = fr.get_num_train()
            self.assertEqual(len(num_trains), 2)

        def test_has_elements(self):
            me = self.fr.maybe_errors()
            self.assertTrue(me.has_value)
            value = me.value
            expected_errors = np.array([-1, -1, -10])
            for i in xrange(3):
                self.assertEqual(value[i], expected_errors[i])

        def test_has_no_elements(self):
            fr = FoldResult()
            fr.extend(np.array([1, np.nan]),
                      np.array([np.nan, 2]))
            me = fr.maybe_errors()
            self.assertTrue(not me.has_value)

        def test_extend_one(self):
            a = np.array([1, 2, 3])
            b = np.array([4, 5, 6])
            fr = FoldResult()
            fr.extend(a, b)
            if self.verbose:
                print fr.actuals
                print fr.estimates
            self.assertEqual(fr.actuals.size, 3)
            self.assertEqual(fr.actuals[1], 2)

        def test_extend_two(self):
            a = np.array([1, 2, 3])
            b = np.array([4, 5, 6])
            fr = FoldResult()
            fr.extend(a, b)
            fr.extend(np.array([10, 20]),
                      np.array([40, 50]))
            if self.verbose:
                print fr.actuals
                print fr.estimates
            self.assertEqual(fr.estimates.size, 5)
            self.assertEqual(fr.estimates[4], 50)

#        def test_mean_exists(self):
#            fr = FoldResult()
#            fr.extend(np.array([1, 2, np.nan, 10, 20]),
#                      np.array([2, 3, 4,      20, np.nan]))
#            value = fr.mean_error_ignore_nans()
#            expected = -(1 + 1 + 10) / 3.0
#            self.assertTrue(value.has_value)
#            self.assertAlmostEqual(value.value, expected)

#        def test_mean_doesnt_exist(self):
#            actuals = np.array([np.nan, 2])
#            estimates = np.array([1, np.nan])
#            fr = FoldResult()
#            fr.extend(actuals, estimates)
#            mean_error = fr.mean_error_ignore_nans()
#            self.assertTrue(not mean_error.has_value)
#
#        def test_median_exists(self):
#            value = self.fr.median_error_ignore_nans()
#            expected = -1
#            self.assertTrue(value.has_value)
#            self.assertAlmostEqual(value.value, expected)
#
#        def test_median_doesnt_exists(self):
#            fr = FoldResult()
#            fr.extend(np.array([]), np.array([]))
#            value = fr.median_error_ignore_nans()
#            self.assertTrue(not value.has_value)
#
    unittest.main()
<commit_msg>refactor: move Test class to outer scope<commit_after># system imports
import datetime
import numpy as np
import pdb
import unittest

# local imports
import Maybe

class FoldResult(object):
    '''Computations on a cross validation fold.'''

    def __init__(self):
        # vectors of results
        self.actuals = np.array([])
        self.estimates = np.array([])
        # dictionaries of results
        self.fitted = {}
        self.predictor_names = {}
        self.num_test = {}
        self.num_train = {}

    def save_num_test(self, date, num_test):
        self.num_test[date] = num_test

    def get_num_test(self):
        return self.num_test

    def save_num_train(self, date, num_train):
        self.num_train[date] = num_train

    def get_num_train(self):
        return self.num_train

    def save_fitted(self, date, fitted):
        self.fitted[date] = fitted

    def get_fitted(self):
        return self.fitted

    def save_predictor_names(self, date, predictor_names):
        self.predictor_names[date] = predictor_names

    def get_predictor_names(self):
        return self.predictor_names

    def extend(self, actuals, estimates):
        '''Extend the collection of values.

        Args
        actuals      1D np.array
        estimates    1D np.array

        Returns: None
        '''
        if len(actuals) != len(estimates):
            print 'len(actuals)', len(actuals)
            print 'len(estimates)', len(estimates)
            raise ValueError('lengths differ')
        self.actuals = np.append(self.actuals, actuals)
        self.estimates = np.append(self.estimates, estimates)

    def maybe_errors(self):
        '''Return Maybe(vector of errors, without nans).'''
        errors = self.actuals - self.estimates
        if np.isnan(errors).all():
            # return NoValue, if
            # - errors.size == 0 OR
            # - every element of errors is nan
            # NOTE: The one test does it all
            return Maybe.NoValue()
        else:
            errors_without_nans = errors[~np.isnan(errors)]
            return Maybe.Maybe(errors_without_nans)

class Test(unittest.TestCase):

    def setUp(self):
        self.verbose = False
        self.group1_actuals = np.array([1, 2, np.nan, 4])
        self.group1_estimates = np.array([2, 3, 4, np.nan])

        self.group2_actuals = np.array([10])
        self.group2_estimates = np.array([20])

        # build the FoldResult object
        fr = FoldResult()
        fr.extend(self.group1_actuals,
                  self.group1_estimates)
        fr.extend(self.group2_actuals,
                  self.group2_estimates)
        self.fr = fr

    def test_fitted(self):
        # test methods .save_fitted() and .get_fitted()
        fr = self.fr
        fitted1 = ['a', 'b']
        date1 = datetime.date(2015, 05, 18)
        fr.save_fitted(date1, fitted1)
        fitted2 = ['aa', 'bb']
        date2 = datetime.date(2015, 05, 19)
        fr.save_fitted(date2, fitted2)
        fitted = fr.get_fitted()  # return a dict
        self.assertEqual(len(fitted), 2)

    def test_predictor_names(self):
        # test methods .save_preictor_names() and .get_predictor_names()
        fr = self.fr
        names1 = ['a', 'b']
        date1 = datetime.date(2015, 05, 18)
        fr.save_predictor_names(date1, names1)
        names2 = ['aa']
        date2 = datetime.date(2015, 05, 19)
        fr.save_predictor_names(date2, names2)
        predictor_names = fr.get_predictor_names()
        self.assertEqual(len(predictor_names), 2)

    def test_num_test(self):
        fr = self.fr
        fr.save_num_test(datetime.date(2015, 05, 18), 10)
        fr.save_num_test(datetime.date(2015, 05, 19), 20)
        num_tests = fr.get_num_test()
        self.assertEqual(len(num_tests), 2)

    def test_num_train(self):
        fr = self.fr
        fr.save_num_train(datetime.date(2015, 05, 18), 10)
        fr.save_num_train(datetime.date(2015, 05, 19), 20)
        num_trains = fr.get_num_train()
        self.assertEqual(len(num_trains), 2)

    def test_has_elements(self):
        me = self.fr.maybe_errors()
        self.assertTrue(me.has_value)
        value = me.value
        expected_errors = np.array([-1, -1, -10])
        for i in xrange(3):
            self.assertEqual(value[i], expected_errors[i])

    def test_has_no_elements(self):
        fr = FoldResult()
        fr.extend(np.array([1, np.nan]),
                  np.array([np.nan, 2]))
        me = fr.maybe_errors()
        self.assertTrue(not me.has_value)

    def test_extend_one(self):
        a = np.array([1, 2, 3])
        b = np.array([4, 5, 6])
        fr = FoldResult()
        fr.extend(a, b)
        if self.verbose:
            print fr.actuals
            print fr.estimates
            self.assertEqual(fr.actuals.size, 3)
            self.assertEqual(fr.actuals[1], 2)

    def test_extend_two(self):
        a = np.array([1, 2, 3])
        b = np.array([4, 5, 6])
        fr = FoldResult()
        fr.extend(a, b)
        fr.extend(np.array([10, 20]),
                  np.array([40, 50]))
        if self.verbose:
            print fr.actuals
            print fr.estimates
            self.assertEqual(fr.estimates.size, 5)
            self.assertEqual(fr.estimates[4], 50)

if __name__ == '__main__':
    import unittest

    if False:
        pdb.set_trace()  # avoid warning

    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# json2csv.py
# Convert JSON output (from parse_all.py/CoreNLP) to .csv format
# to enable manual review of extracted relations.
#
# Kiri Wagstaff
# August 28, 2017

import sys, os, shutil
import json
from ioutils import read_jsonlines
from progressbar import ProgressBar, ETA, Bar, Percentage

def usage():
    print './json2csv.py <JSON file>'
    sys.exit(1)

def convert_json_to_csv(jsonfile):
    # Get the number of lines (docs) to process
    # Do this before re-opening the file because read_jsonlines()
    # returns a generator.
    with open(jsonfile) as f:
        l = f.readlines()
        ndocs = len(l)
        f.close()

    # Read in the JSON file
    docs = read_jsonlines(jsonfile)

    # Open the output CSV file
    outfn = jsonfile[:jsonfile.rfind('.')] + '.csv'
    outf = open(outfn, 'w')
    print 'Writing to', outfn
    # Header
    outf.write('Judgment, Docid, Target, Component, Sentence\n')

    widgets = ['Docs (of %d): ' % ndocs, Percentage(), ' ',
               Bar('='), ' ', ETA()]
    pbar = ProgressBar(widgets=widgets, maxval=ndocs).start()
    # Iterate over documents
    i=1
    for d in docs:
        if 'rel' not in d['metadata']:
            continue

        docid = d['metadata']['resourceName']

        # Output relations into the .csv file
        rels = d['metadata']['rel']
        ners = d['metadata']['ner']
        skip_inds = []
        for (t, r) in enumerate(rels):
            # Special merging step for adjacent Target tokens
            # If this matches a multi-word NER,
            # expand the target name and skip the next relation
            start_target = int(r['target_ids'][0].split('_')[1])
            end_target   = int(r['target_ids'][0].split('_')[2])
            targ_name    = r['target_names'][0]
            if start_target in skip_inds:
                continue
            next_rels = [r2 for r2 in rels if
                         int(r2['target_ids'][0].split('_')[1]) > end_target]
            if len(next_rels) > 0:
                next_rels.sort(key=lambda x:
                               int(x['target_ids'][0].split('_')[1]))
                next_rel = next_rels[0]
                start_next_target = int(next_rel['target_ids'][0].split('_')[1])
                end_next_target   = int(next_rel['target_ids'][0].split('_')[2])
                ner_matches = [n for n in ners if \
                               n['text'].startswith(targ_name) and
                               n['begin'] == start_target and
                               n['end'] == end_next_target]
                if len(ner_matches) > 0:
                    print('Merging %s and %s' % (targ_name,
                                                 next_rel['target_names'][0]))
                    targ_name += ' ' + next_rel['target_names'][0]
                    skip_inds.append(start_next_target)

            outf.write(',%s,%s,%s,"%s"\n' %
                       (docid,
                        # These are arrays, but for auto-annotations,
                        # they will only ever have one item
                        #r['target_names'][0],
                        targ_name,
                        r['cont_names'][0],
                        r['sentence']))
                       # build URL manually?
        pbar.update(i)
        i += 1

    print
    outf.close()

def main():
    if len(sys.argv) != 2:
        usage()

    if not os.path.exists(sys.argv[1]):
        print 'Error: could not find JSON input file %s.' % sys.argv[1]
        usage()

    convert_json_to_csv(sys.argv[1])

if __name__ == '__main__':
    main()
<commit_msg>Truncate hyphenated components (like 'Fe-rich') to just 'Fe'.<commit_after>#!/usr/bin/env python
# json2csv.py
# Convert JSON output (from parse_all.py/CoreNLP) to .csv format
# to enable manual review of extracted relations.
#
# Kiri Wagstaff
# August 28, 2017

import sys, os, shutil
import json
from ioutils import read_jsonlines
from progressbar import ProgressBar, ETA, Bar, Percentage

def usage():
    print './json2csv.py <JSON file>'
    sys.exit(1)

def convert_json_to_csv(jsonfile):
    # Get the number of lines (docs) to process
    # Do this before re-opening the file because read_jsonlines()
    # returns a generator.
    with open(jsonfile) as f:
        l = f.readlines()
        ndocs = len(l)
        f.close()

    # Read in the JSON file
    docs = read_jsonlines(jsonfile)

    # Open the output CSV file
    outfn = jsonfile[:jsonfile.rfind('.')] + '.csv'
    outf = open(outfn, 'w')
    print 'Writing to', outfn
    # Header
    outf.write('Judgment, Docid, Target, Component, Sentence\n')

    widgets = ['Docs (of %d): ' % ndocs, Percentage(), ' ',
               Bar('='), ' ', ETA()]
    pbar = ProgressBar(widgets=widgets, maxval=ndocs).start()
    # Iterate over documents
    i=1
    for d in docs:
        if 'rel' not in d['metadata']:
            continue

        docid = d['metadata']['resourceName']

        # Output relations into the .csv file
        rels = d['metadata']['rel']
        ners = d['metadata']['ner']
        skip_inds = []
        for (t, r) in enumerate(rels):
            # Special merging step for adjacent Target tokens
            # If this matches a multi-word NER,
            # expand the target name and skip the next relation
            start_target = int(r['target_ids'][0].split('_')[1])
            end_target   = int(r['target_ids'][0].split('_')[2])
            # These are arrays, but for auto-annotations,
            # they will only ever have one item
            targ_name    = r['target_names'][0]
            cont_name    = r['cont_names'][0]

            if start_target in skip_inds:
                continue
            next_rels = [r2 for r2 in rels if
                         int(r2['target_ids'][0].split('_')[1]) > end_target]
            if len(next_rels) > 0:
                next_rels.sort(key=lambda x:
                               int(x['target_ids'][0].split('_')[1]))
                next_rel = next_rels[0]
                start_next_target = int(next_rel['target_ids'][0].split('_')[1])
                end_next_target   = int(next_rel['target_ids'][0].split('_')[2])
                ner_matches = [n for n in ners if \
                               n['text'].startswith(targ_name) and
                               n['begin'] == start_target and
                               n['end'] == end_next_target]
                if len(ner_matches) > 0:
                    print('Merging %s and %s' % (targ_name,
                                                 next_rel['target_names'][0]))
                    targ_name += ' ' + next_rel['target_names'][0]
                    skip_inds.append(start_next_target)

            # If cont_name is something like Fe-rich or Mg_sulfate,
            # only keep the first bit.
            if '-' in cont_name:
                cont_name = cont_name[:cont_name.find('-')]
            elif '_' in cont_name:
                cont_name = cont_name[:cont_name.find('_')]

            outf.write(',%s,%s,%s,"%s"\n' %
                       (docid,
                        #r['target_names'][0],
                        targ_name,
                        cont_name,
                        r['sentence']))
                       # build URL manually?
        pbar.update(i)
        i += 1

    print
    outf.close()

def main():
    if len(sys.argv) != 2:
        usage()

    if not os.path.exists(sys.argv[1]):
        print 'Error: could not find JSON input file %s.' % sys.argv[1]
        usage()

    convert_json_to_csv(sys.argv[1])

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from os.path import exists

import sublime, sublime_plugin

class ForceCloseByIndexCommand(sublime_plugin.WindowCommand):

    def run(self, group = -1, index = -1):
        '''Force closes a view.

        If the view has an underlying file, it is saved, then closed. If it doesn't, the file is cleared then closed. Losing all progress.
        If either the view group or index is not specified, the active view is used.'''

        if group == -1 or index == -1:
            the_view = self.window.active_view()
            group, index = self.window.get_view_index(the_view)
        else:
            the_view = next((view for view in self.window.views_in_group(group) if self.window.get_view_index(view) == (group, index)), None)

        if the_view != None:

            settings = sublime.load_settings('force_close.sublime-settings')

            has_file = the_view.file_name() != None
            should_save = the_view.is_dirty() and has_file and settings.get('save_when_possible')
            is_orphan = has_file and not exists(the_view.file_name())

            if (should_save and not is_orphan) or (is_orphan and settings.get('save_orphaned_views')):
                the_view.run_command('save')
            else:
                the_view.set_scratch(True)

            self.window.run_command('close_by_index', {"group": group, "index": index})
<commit_msg>Fixed issue #1<commit_after>from os.path import exists

import sublime, sublime_plugin

class ForceCloseByIndexCommand(sublime_plugin.WindowCommand):

    def run(self, group = -1, index = -1):
        """Force closes a view.

        If the view has an underlying file, it is saved, then closed. If it doesn't, the file is cleared then closed. Losing all progress.
        If either the view group or index is not specified, the active view is used.
        """

        if group == -1 or index == -1:
            the_view = self.window.active_view()
            group, index = self.window.get_view_index(the_view)
        else:
            the_view = next((view for view in self.window.views_in_group(group) if self.window.get_view_index(view) == (group, index)), None)

        if the_view != None:

            settings = sublime.load_settings('force_close.sublime-settings')

            has_file = the_view.file_name() != None
            should_save = the_view.is_dirty() and has_file and settings.get('save_when_possible')
            is_orphan = has_file and not exists(the_view.file_name())

            if (should_save and not is_orphan) or (is_orphan and settings.get('save_orphaned_views')):
                the_view.run_command('save')
            else:
                the_view.set_scratch(True)

            self.window.run_command('close_by_index', {"group": group, "index": index})
<|endoftext|>