<commit_before>#!/usr/bin/env python

"""Part of the container that manages ION processes etc."""

__author__ = 'Michael Meisinger'

from zope.interface import providedBy, implementedBy
from zope.interface import Interface, implements

from pyon.core.bootstrap import CFG
from pyon.ion.process import IonProcessSupervisor
from pyon.net.channel import PubSub
from pyon.net.endpoint import BinderListener, ProcessRPCServer, ProcessRPCClient, Subscriber
from pyon.net.messaging import IDPool
from pyon.service.service import BaseService, get_service_by_name
from pyon.util.containers import DictModifier, DotDict, for_name
from pyon.util.log import log

class ProcManager(object):
    def __init__(self, container):
        self.container = container

        # Define the callables that can be added to Container public API
        self.container_api = [self.spawn_process]

        # Add the public callables to Container
        for call in self.container_api:
            setattr(self.container, call.__name__, call)

        self.proc_id_pool = IDPool()

        # Temporary registry of running processes
        self.procs = {}

        # The pyon worker process supervisor
        self.proc_sup = IonProcessSupervisor(heartbeat_secs=CFG.cc.timeout.heartbeat)

    def start(self):
        log.debug("ProcManager: start")

        self.proc_sup.start()

    def stop(self):
        log.debug("ProcManager: stop")

        # Call quit on procs to give them ability to clean up
        for procname, proc in self.procs.iteritems():
            try:
                # These are service processes with full life cycle
                proc.quit()
            except Exception, ex:
                log.exception("Process %s quit failed" % procname)

        # TODO: Have a choice of shutdown behaviors for waiting on children, timeouts, etc
        self.proc_sup.shutdown(CFG.cc.timeout.shutdown)

    def spawn_process(self, name=None, module=None, cls=None, config=None):
        """
        Spawn a process within the container.
        """
        log.debug("AppManager.spawn_process(name=%s, module=%s)" % (name, module))

        if config is None:
            config = DictModifier(CFG)

        errcause = "instantiating service"
        try:
            log.debug("AppManager.spawn_process: for_name(mod=%s, cls=%s)" % (module, cls))
            process_instance = for_name(module, cls)
            assert isinstance(process_instance, BaseService), "Instantiated process not a BaseService %r" % process_instance

            # Prepare process instance
            process_instance.id = "%s.%s" % (self.container.id, self.proc_id_pool.get_id())
            process_instance.container = self.container
            process_instance.CFG = config

            # Set dependencies (clients)
            errcause = "setting service dependencies"
            process_instance.clients = DotDict()
            log.debug("AppManager.spawn_process dependencies: %s" % process_instance.dependencies)
            # TODO: Service dependency != process dependency
            for dependency in process_instance.dependencies:
                dependency_service = get_service_by_name(dependency)
                dependency_interface = list(implementedBy(dependency_service))[0]

                # @TODO: start_client call instead?
                client = ProcessRPCClient(node=self.container.node, name=dependency, iface=dependency_interface, process=process_instance)
                process_instance.clients[dependency] = client

            # Init process
            errcause = "initializing service"
            process_instance.init()

            # Add to process dict
            self.procs[name] = process_instance
            errcause = "setting process messaging endpoints"

            process_type = config.get("process",{}).get("type", "service")
            listen_name = config.get("process",{}).get("listen_name", name)

            if process_type == "service":
                # Service RPC endpoint
                rsvc = ProcessRPCServer(node=self.container.node, name=name, service=process_instance, process=process_instance)
                # Start an ION process with the right kind of endpoint factory
                listener = BinderListener(self.container.node, listen_name, rsvc, None, None)
                self.proc_sup.spawn((CFG.cc.proctype or 'green', None), listener=listener)
                # Wait for app to spawn
                listener.get_ready_event().get()
                log.debug("Process %s service listener ready: %s", name, listen_name)

            elif process_type == "stream_process":
                # Start pubsub listener
                sub = Subscriber(node=self.container.node, name=listen_name,
                                 callback=lambda m,h: process_instance.process(m))
                listener = BinderListener(self.container.node, listen_name, sub, PubSub, None)
                self.proc_sup.spawn((CFG.cc.proctype or 'green', None), listener=listener)
                # Wait for app to spawn
                listener.get_ready_event().get()
                log.debug("Process %s stream listener ready: %s", name, listen_name)

            elif process_type == "agent":
                # TODO: Determine special treatment of agents
                pass

            elif process_type == "immediate":
                # One off process. No extra listeners
                pass

            elif process_type == "simple":
                # In this case we don't want any extra listeners
                pass

            else:
                raise Exception("Unknown process type: %s" % process_type)

            # Process exclusive RPC endpoint
            rsvc_proc = ProcessRPCServer(node=self.container.node, name=process_instance.id, service=process_instance, process=process_instance)
            # Start an ION process with the right kind of endpoint factory
            # TODO: Don't start in its own Greenlet!
            listener1 = BinderListener(self.container.node, process_instance.id, rsvc_proc, None, None)
            self.proc_sup.spawn((CFG.cc.proctype or 'green', None), listener=listener1)
            # Wait for app to spawn
            listener1.get_ready_event().get()
            log.debug("Process %s pid listener ready: %s", name, process_instance.id)

            errcause = "registering"
            self.container.directory.register("/Containers/%s/Processes" % self.container.id, process_instance.id, name=name)

            errcause = "starting service"
            process_instance.start()

            log.info("AppManager.spawn_process: %s.%s -> id=%s OK" % (module, cls, process_instance.id))
            return True
        except Exception as ex:
            log.exception("Error %s for process: %s.%s" % (errcause, module, cls))
            return False
<commit_msg>Refactored process spawn to be more modular and flexible for different proc types<commit_after>#!/usr/bin/env python

"""Part of the container that manages ION processes etc."""

__author__ = 'Michael Meisinger'

from zope.interface import providedBy, implementedBy
from zope.interface import Interface, implements

from pyon.core.bootstrap import CFG
from pyon.ion.process import IonProcessSupervisor
from pyon.net.channel import PubSub
from pyon.net.endpoint import BinderListener, ProcessRPCServer, ProcessRPCClient, Subscriber
from pyon.net.messaging import IDPool
from pyon.service.service import BaseService, get_service_by_name
from pyon.util.containers import DictModifier, DotDict, for_name
from pyon.util.log import log

class ProcManager(object):
    def __init__(self, container):
        self.container = container

        # Define the callables that can be added to Container public API
        self.container_api = [self.spawn_process]

        # Add the public callables to Container
        for call in self.container_api:
            setattr(self.container, call.__name__, call)

        self.proc_id_pool = IDPool()

        # Temporary registry of running processes
        self.procs_by_name = {}
        self.procs = {}

        # The pyon worker process supervisor
        self.proc_sup = IonProcessSupervisor(heartbeat_secs=CFG.cc.timeout.heartbeat)

    def start(self):
        log.debug("ProcManager starting ...")
        self.proc_sup.start()
        log.debug("ProcManager started, OK.")

    def stop(self):
        log.debug("ProcManager stopping ...")

        # Call quit on procs to give them ability to clean up
        # TODO: This can be done concurrently
        for procid, proc in self.procs.iteritems():
            try:
                # These are service processes with full life cycle
                proc.quit()
            except Exception, ex:
                log.exception("Process %s quit failed" % procid)

        # TODO: Have a choice of shutdown behaviors for waiting on children, timeouts, etc
        self.proc_sup.shutdown(CFG.cc.timeout.shutdown)
        log.debug("ProcManager stopped, OK.")

    def spawn_process(self, name=None, module=None, cls=None, config=None, process_type=None):
        """
        Spawn a process within the container. Processes can be of different type.
        """
        # Generate a new process id
        # TODO: Ensure it is system-wide unique
        process_id = str(self.proc_id_pool.get_id())
        log.debug("ProcManager.spawn_process(name=%s, module.cls=%s.%s) as pid=%s" % (name, module, cls, process_id))

        if config is None:
            config = DictModifier(CFG)

        log.debug("spawn_process() pid=%s config=%s", process_id, config)

        # PROCESS TYPE.
        # One of: service, stream_process, agent, simple, immediate
        process_type = process_type or config.get("process",{}).get("type", "service")

        service_instance = None
        try:
            # spawn service by type
            if process_type == "service":
                service_instance = self._spawn_service_process(process_id, name, module, cls, config)

            elif process_type == "stream_process":
                service_instance = self._spawn_stream_process(process_id, name, module, cls, config)

            elif process_type == "agent":
                service_instance = self._spawn_agent_process(process_id, name, module, cls, config)

            elif process_type == "immediate":
                service_instance = self._spawn_immediate_process(process_id, name, module, cls, config)

            elif process_type == "simple":
                service_instance = self._spawn_simple_process(process_id, name, module, cls, config)

            else:
                raise Exception("Unknown process type: %s" % process_type)

            service_instance._proc_type = process_type
            self._register_process(service_instance, name)

            service_instance.errcause = "OK"
            log.info("AppManager.spawn_process: %s.%s -> pid=%s OK" % (module, cls, process_id))
            return True

        except Exception as ex:
            errcause = service_instance.errcause if service_instance else "instantiating service"
            log.exception("Error spawning process %s: %s" % (process_id, errcause))
            return False

    def _spawn_service_process(self, process_id, name, module, cls, config):
        """
        Spawn a process acting as a service worker.
        Attach to service queue with service definition, attach to service pid
        """
        service_instance = self._create_service_instance(process_id, name, module, cls, config)
        self._service_init(service_instance)
        listen_name = config.get("process",{}).get("listen_name", name)
        print "**********", name, listen_name
        self._set_service_endpoint(service_instance, listen_name)
        self._set_service_endpoint(service_instance, service_instance.id)
        self._service_start(service_instance)
        return service_instance

    def _spawn_stream_process(self, process_id, name, module, cls, config):
        """
        Spawn a process acting as a data stream process.
        Attach to subscription queue with process function.
        """
        service_instance = self._create_service_instance(process_id, name, module, cls, config)
        self._service_init(service_instance)
        listen_name = config.get("process",{}).get("listen_name", name)
        self._set_subscription_endpoint(service_instance, listen_name)
        self._service_start(service_instance)
        return service_instance

    def _spawn_agent_process(self, process_id, name, module, cls, config):
        """
        Spawn a process acting as agent process.
        Attach to service pid.
        """
        service_instance = self._create_service_instance(process_id, name, module, cls, config)
        self._service_init(service_instance)
        self._set_service_endpoint(service_instance, service_instance.id)
        self._service_start(service_instance)
        return service_instance

    def _spawn_simple_process(self, process_id, name, module, cls, config):
        """
        Spawn a process acting as simple process.
        No attachments.
        """
        service_instance = self._create_service_instance(process_id, name, module, cls, config)
        self._service_init(service_instance)
        self._service_start(service_instance)
        return service_instance

    def _spawn_immediate_process(self, process_id, name, module, cls, config):
        """
        Spawn a process acting as immediate one off process.
        No attachments.
        """
        service_instance = self._create_service_instance(process_id, name, module, cls, config)
        self._service_init(service_instance)
        self._service_start(service_instance)
        return service_instance

    def _create_service_instance(self, process_id, name, module, cls, config):
        # SERVICE INSTANCE.
        service_instance = for_name(module, cls)
        assert isinstance(service_instance, BaseService), "Instantiated service not a BaseService %r" % service_instance

        # Prepare service instance
        service_instance.errcause = ""
        service_instance.id = "%s.%s" % (self.container.id, process_id)
        service_instance.container = self.container
        service_instance.CFG = config
        service_instance._proc_name = name

        # Set service dependencies (RPC clients)
        self._set_service_dependencies(service_instance)

        return service_instance

    def _set_service_dependencies(self, service_instance):
        service_instance.errcause = "setting service dependencies"
        service_instance.clients = DotDict()
        log.debug("spawn_process dependencies: %s" % service_instance.dependencies)
        # TODO: Service dependency != process dependency
        for dependency in service_instance.dependencies:
            dependency_service = get_service_by_name(dependency)
            dependency_interface = list(implementedBy(dependency_service))[0]

            # @TODO: start_client call instead?
            client = ProcessRPCClient(node=self.container.node, name=dependency, iface=dependency_interface, process=service_instance)
            service_instance.clients[dependency] = client

    def _service_init(self, service_instance):
        # Init process
        service_instance.errcause = "initializing service"
        service_instance.init()

    def _service_start(self, service_instance):
        # Start process
        # TODO: Check for timeout
        service_instance.errcause = "starting service"
        service_instance.start()

    def _set_service_endpoint(self, service_instance, listen_name):
        service_instance.errcause = "setting process service endpoint"

        # Service RPC endpoint
        rsvc = ProcessRPCServer(node=self.container.node,
                                name=listen_name,
                                service=service_instance,
                                process=service_instance)
        # Start an ION process with the right kind of endpoint factory
        listener = BinderListener(self.container.node, listen_name, rsvc, None, None)
        self.proc_sup.spawn((CFG.cc.proctype or 'green', None), listener=listener, name=listen_name)
        # Wait for app to spawn
        listener.get_ready_event().get()
        log.debug("Process %s service listener ready: %s", service_instance.id, listen_name)

    def _set_subscription_endpoint(self, service_instance, listen_name):
        service_instance.errcause = "setting process subscription endpoint"

        # Start pubsub listener
        # TODO: Use Refactored ION endpoint for streams once ready.
        sub = Subscriber(node=self.container.node,
                         name=listen_name,
                         callback=lambda m,h: service_instance.process(m))
        listener = BinderListener(self.container.node, listen_name, sub, PubSub, None)
        self.proc_sup.spawn((CFG.cc.proctype or 'green', None), listener=listener, name=listen_name)
        # Wait for app to spawn
        listener.get_ready_event().get()
        log.debug("Process %s stream listener ready: %s", service_instance.id, listen_name)

    def _register_process(self, service_instance, name):
        # Add to local process dict
        self.procs_by_name[name] = service_instance
        self.procs[service_instance.id] = service_instance

        # Add to directory
        service_instance.errcause = "registering"
        self.container.directory.register("/Containers/%s/Processes" % self.container.id,
                                          service_instance.id, name=name)

    def terminate_process(self, process_id):
        service_instance = self.procs.get(process_id, None)
        if not service_instance: return

        service_instance.quit()
        # TODO: Cleanup messaging attachments

        del self.procs[process_id]
<|endoftext|>


<commit_before>from __future__ import division

import sys
import os
import logging
import math
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from datetime import datetime, timedelta
from collections import defaultdict, OrderedDict

import pandas as PD

from pyglow.pyglow import Point

"""
NOTE: This code assumes that the input and output data are 1-minute interval.
"""

HEADER_TEMPLATE = """\
 Format                 IAGA-2002                                    |
 Source of Data         SuperMAG                                     |
 IAGA CODE              {stn}                                          |
 Geodetic Latitude      {lat:<8.3f}                                     |
 Geodetic Longitude     {lon:<8.3f}                                     |
 Elevation              {el}                                          |
 Reported               {reported}                                         |
 Digital Sampling       1-Minute                                     |
DATE       TIME         DOY     {stn}{C1}       {stn}{C2}     {stn}Z      {stn}F   |
"""

def read_sm_csv(csv_fname):
    """
    """
    df = PD.read_csv(csv_fname,
                     header=0,
                     parse_dates=[0],
                     date_parser=lambda x: datetime.strptime(x, '%Y-%m-%d %H:%M:%S'))
    return {name: group for name, group in df.groupby('IAGA')}

def dataframe2iaga(fname,
                   stn,
                   date,
                   df,
                   lat,
                   lon,
                   elevation=None,
                   nez=False,
                   header_template=HEADER_TEMPLATE):
    """
    ASSUMES DF SPANS ONLY A SINGLE DAY
    """
    #reported, coord_map = get_reported(df)
    # build map of data records
    data_map = {}
    for _, row in df.iterrows():
        N = row['N']
        E = row['E']
        Z = row['Z']
        F = math.hypot(math.hypot(N, E), Z)
        data_map[row.Date_UTC] = (N, E, Z, F)
    # fill missing records
    filled_data = []
    dts = list(PD.date_range(start=date,
                             end=date + timedelta(days=1),
                             freq='min'))[:-1]
    for dt in dts:
        filled_data.append(data_map.get(dt,
                                        (88888,) * 4))
    if not nez:
        # get magnetic declination angle for local magnetic (NEZ) to
        # geographic (XYZ) conversion
        point = Point(dts[0], lat, lon, elevation / 1e3 if elevation else 0)
        point.run_igrf()
        dec_deg = point.dec
        dec_rad = math.radians(dec_deg)
        cos_dec = math.cos(dec_rad)
        sin_dec = math.sin(dec_rad)

    # write data records to file
    with open(fname, 'w') as fid:
        fid.write(header_template.format(stn=stn,
                                         reported='NEZF' if nez else 'XYZF',
                                         lat=lat,
                                         lon=lon,
                                         el='XXX' if elevation is None else elevation,
                                         C1='N' if nez else 'X',
                                         C2='E' if nez else 'Y'))
        for dt, (N, E, Z, F) in zip(dts, filled_data):
            if nez:
                C1 = N
                C2 = E
            else:
                if N in [88888, 99999] or E in [88888, 99999]:
                    C1 = 88888
                    C2 = 88888
                else:
                    C1 = cos_dec * N - sin_dec * E
                    C2 = sin_dec * N + cos_dec * E
            fid.write('{date:%Y-%m-%d %H:%M:%S.000} {date:%j}'
                      '    {C1:>9.2f} {C2:>9.2f} {Z:>9.2f} {F:>9.2f}\n'.format(date=dt,
                                                                               C1=C1,
                                                                               C2=C2,
                                                                               Z=Z,
                                                                               F=F))
    return fname

def iaga_fname(path, stn, date, cutoff_date=datetime(2007, 1, 1)):
    """
    """
    if date >= cutoff_date.date():
        ext = 'dmin.min'
    else:
        ext = 'd.min'
    return os.path.join(path, '{stn}{date:%Y%m%d}{ext}'.format(stn=stn.lower(),
                                                               date=date,
                                                               ext=ext))

def df_map2iaga(path, df_map, lat, lon, elevation=None, nez=False):
    """
    """
    output_map = defaultdict(dict)
    for stn, df in df_map.iteritems():
        for date, df_date in df.groupby(df['Date_UTC'].map(lambda x: x.date())):
            output_fname = iaga_fname(path, stn, date)
            dataframe2iaga(output_fname,
                           stn,
                           date,
                           df_date,
                           lat,
                           lon,
                           elevation=elevation,
                           nez=nez)
            output_map[stn][date] = output_fname
    return output_map

def main(argv=None):
    if argv is None:
        argv = sys.argv

    parser = ArgumentParser('Convert SuperMAG CSV format data to daily IAGA2002 format.',
                            formatter_class=ArgumentDefaultsHelpFormatter)
    parser.add_argument('output_path',
                        type=str,
                        help='path to store daily IAGA2002 format files')
    parser.add_argument('csv_fname',
                        type=str,
                        help='SuperMAG CSV file')
    parser.add_argument('latitude',
                        type=float,
                        help='site geodetic latitude [deg]')
    parser.add_argument('longitude',
                        type=float,
                        help='site longitude [deg]')
    parser.add_argument('--elevation',
                        '-e',
                        type=float,
                        default=None,
                        help='site elevation [m]')
    parser.add_argument('--nez',
                        action='store_true',
                        help='store (raw) HEZ components (aligned to local magnetic field) instead of XYZ components')
    args = parser.parse_args(argv[1:])

    df_map = read_sm_csv(args.csv_fname)
    df_map2iaga(args.output_path,
                df_map,
                args.latitude,
                args.longitude,
                elevation=args.elevation,
                nez=args.nez)

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    sys.exit(main())
<commit_msg>Now user SuperMAG provided site locations.<commit_after>from __future__ import division

import sys
import os
import logging
import math
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from datetime import datetime, timedelta
from collections import defaultdict, OrderedDict

import pandas as PD

from pyglow.pyglow import Point

from sm_stations import STATION_MAP

"""
NOTE: This code assumes that the input and output data are 1-minute interval.
"""

HEADER_TEMPLATE = """\
 Format                 IAGA-2002                                    |
 Source of Data         SuperMAG                                     |
 IAGA CODE              {stn}                                          |
 Geodetic Latitude      {lat:<8.3f}                                     |
 Geodetic Longitude     {lon:<8.3f}                                     |
 Elevation              {el}                                          |
 Reported               {reported}                                         |
 Digital Sampling       1-Minute                                     |
DATE       TIME         DOY     {stn}{C1}       {stn}{C2}     {stn}Z      {stn}F   |
"""

def read_sm_csv(csv_fname):
    """
    """
    df = PD.read_csv(csv_fname,
                     header=0,
                     parse_dates=[0],
                     date_parser=lambda x: datetime.strptime(x, '%Y-%m-%d %H:%M:%S'))
    return {name: group for name, group in df.groupby('IAGA')}

def dataframe2iaga(fname,
                   stn,
                   date,
                   df,
                   lat,
                   lon,
                   elevation=None,
                   nez=False,
                   header_template=HEADER_TEMPLATE):
    """
    ASSUMES DF SPANS ONLY A SINGLE DAY
    """
    #reported, coord_map = get_reported(df)
    # build map of data records
    data_map = {}
    for _, row in df.iterrows():
        N = row['N']
        E = row['E']
        Z = row['Z']
        F = math.hypot(math.hypot(N, E), Z)
        data_map[row.Date_UTC] = (N, E, Z, F)
    # fill missing records
    filled_data = []
    dts = list(PD.date_range(start=date,
                             end=date + timedelta(days=1),
                             freq='min'))[:-1]
    for dt in dts:
        filled_data.append(data_map.get(dt,
                                        (88888,) * 4))
    if not nez:
        # get magnetic declination angle for local magnetic (NEZ) to
        # geographic (XYZ) conversion
        point = Point(dts[0], lat, lon, elevation / 1e3 if elevation else 0)
        point.run_igrf()
        dec_deg = point.dec
        dec_rad = math.radians(dec_deg)
        cos_dec = math.cos(dec_rad)
        sin_dec = math.sin(dec_rad)

    # write data records to file
    with open(fname, 'w') as fid:
        fid.write(header_template.format(stn=stn,
                                         reported='NEZF' if nez else 'XYZF',
                                         lat=lat,
                                         lon=lon,
                                         el='XXX' if elevation is None else elevation,
                                         C1='N' if nez else 'X',
                                         C2='E' if nez else 'Y'))
        for dt, (N, E, Z, F) in zip(dts, filled_data):
            if nez:
                C1 = N
                C2 = E
            else:
                if N in [88888, 99999] or E in [88888, 99999]:
                    C1 = 88888
                    C2 = 88888
                else:
                    C1 = cos_dec * N - sin_dec * E
                    C2 = sin_dec * N + cos_dec * E
            fid.write('{date:%Y-%m-%d %H:%M:%S.000} {date:%j}'
                      '    {C1:>9.2f} {C2:>9.2f} {Z:>9.2f} {F:>9.2f}\n'.format(date=dt,
                                                                               C1=C1,
                                                                               C2=C2,
                                                                               Z=Z,
                                                                               F=F))
    return fname

def iaga_fname(path, stn, date, cutoff_date=datetime(2007, 1, 1)):
    """
    """
    if date >= cutoff_date.date():
        ext = 'dmin.min'
    else:
        ext = 'd.min'
    return os.path.join(path, '{stn}{date:%Y%m%d}{ext}'.format(stn=stn.lower(),
                                                               date=date,
                                                               ext=ext))

def df_map2iaga(path, df_map, nez=False):
    """
    """
    output_map = defaultdict(dict)
    for stn, df in df_map.iteritems():
        station_info = STATION_MAP[stn.upper()]
        lat = station_info.glat
        lon = station_info.glon
        for date, df_date in df.groupby(df['Date_UTC'].map(lambda x: x.date())):
            output_fname = iaga_fname(path, stn, date)
            dataframe2iaga(output_fname,
                           stn,
                           date,
                           df_date,
                           lat,
                           lon,
                           elevation=0,
                           nez=nez)
            output_map[stn][date] = output_fname
    return output_map

def main(argv=None):
    if argv is None:
        argv = sys.argv

    parser = ArgumentParser('Convert SuperMAG CSV format data to daily IAGA2002 format.',
                            formatter_class=ArgumentDefaultsHelpFormatter)
    parser.add_argument('output_path',
                        type=str,
                        help='path to store daily IAGA2002 format files')
    parser.add_argument('csv_fname',
                        type=str,
                        help='SuperMAG CSV file')
    parser.add_argument('--nez',
                        action='store_true',
                        help='store (raw) HEZ components (aligned to local magnetic field) instead of XYZ components')
    args = parser.parse_args(argv[1:])

    df_map = read_sm_csv(args.csv_fname)
    df_map2iaga(args.output_path,
                df_map,
                nez=args.nez)

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    sys.exit(main())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
from abc import ABCMeta, abstractmethod

import ply.lex as lex
import ply.yacc as yacc

"""
Base classes for parsers.

It contains some interfaces, along a base parser for the parsing library being
used.
"""

__author__ = 'Bernardo Martínez Garrido'
__license__ = 'MIT'

class Parser(object, metaclass=ABCMeta):
    """
    Interface for implementing parsers.

    It just contains a single method, 'parse', which will receive a value
    and take care of parsing it into another.
    """

    def __init__(self):
        pass

    @abstractmethod
    def parse(self, value):
        raise NotImplementedError('The parse method must be implemented')

class PlyParser(Parser):
    """
    Base class for a lexer/parser that has the rules defined as methods.

    It makes use of Ply for the parsing.
    """

    tokens = ()
    precedence = ()

    def __init__(self, **kw):
        super(PlyParser, self).__init__()
        self.debug = kw.get('debug', 0)
        self.names = {}
        try:
            modname = os.path.split(os.path.splitext(__file__)[0])[
                          1] + "_" + self.__class__.__name__
        except:
            modname = "parser" + "_" + self.__class__.__name__
        self.debugfile = modname + ".dbg"
        self.tabmodule = modname + "_" + "parsetab"
        # print self.debugfile, self.tabmodule

        # Builds the lexer and parser
        lex.lex(module=self, debug=self.debug)
        yacc.yacc(module=self,
                  debug=self.debug,
                  debugfile=self.debugfile,
                  tabmodule=self.tabmodule)

    def parse(self, value):
        return yacc.parse(value)
<commit_msg>Defined exception being caught<commit_after># -*- coding: utf-8 -*-

import os
from abc import ABCMeta, abstractmethod

import ply.lex as lex
import ply.yacc as yacc

"""
Base classes for parsers.

It contains some interfaces, along a base parser for the parsing library being
used.
"""

__author__ = 'Bernardo Martínez Garrido'
__license__ = 'MIT'

class Parser(object, metaclass=ABCMeta):
    """
    Interface for implementing parsers.

    It just contains a single method, 'parse', which will receive a value
    and take care of parsing it into another.
    """

    def __init__(self):
        pass

    @abstractmethod
    def parse(self, value):
        raise NotImplementedError('The parse method must be implemented')

class PlyParser(Parser):
    """
    Base class for a lexer/parser that has the rules defined as methods.

    It makes use of Ply for the parsing.
    """

    tokens = ()
    precedence = ()

    def __init__(self, **kw):
        super(PlyParser, self).__init__()
        self.debug = kw.get('debug', 0)
        self.names = {}
        try:
            modname = os.path.split(os.path.splitext(__file__)[0])[
                          1] + "_" + self.__class__.__name__
        except Exception:
            modname = "parser" + "_" + self.__class__.__name__
        self.debugfile = modname + ".dbg"
        self.tabmodule = modname + "_" + "parsetab"
        # print self.debugfile, self.tabmodule

        # Builds the lexer and parser
        lex.lex(module=self, debug=self.debug)
        yacc.yacc(module=self,
                  debug=self.debug,
                  debugfile=self.debugfile,
                  tabmodule=self.tabmodule)

    def parse(self, value):
        return yacc.parse(value)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""Part of the container that manages ION processes etc."""

__author__ = 'Michael Meisinger'

from zope.interface import implementedBy

from pyon.core.bootstrap import CFG
from pyon.ion.endpoint import ProcessRPCServer, ProcessRPCClient, ProcessSubscriber
from pyon.ion.endpoint import StreamSubscriberRegistrar, StreamSubscriberRegistrarError, StreamPublisher, StreamPublisherRegistrar
from pyon.ion.process import IonProcessSupervisor
from pyon.net.messaging import IDPool
from pyon.service.service import BaseService, get_service_by_name
from pyon.util.containers import DictModifier, DotDict, for_name
from pyon.util.log import log

class ProcManager(object):
    def __init__(self, container):
        self.container = container

        # Define the callables that can be added to Container public API
        self.container_api = [self.spawn_process]

        # Add the public callables to Container
        for call in self.container_api:
            setattr(self.container, call.__name__, call)

        self.proc_id_pool = IDPool()

        # Temporary registry of running processes
        self.procs_by_name = {}
        self.procs = {}

        # The pyon worker process supervisor
        self.proc_sup = IonProcessSupervisor(heartbeat_secs=CFG.cc.timeout.heartbeat)

    def start(self):
        log.debug("ProcManager starting ...")
        self.proc_sup.start()
        log.debug("ProcManager started, OK.")

    def stop(self):
        log.debug("ProcManager stopping ...")

        # Call quit on procs to give them ability to clean up
        # TODO: This can be done concurrently
        for procid, proc in self.procs.iteritems():
            try:
                # These are service processes with full life cycle
                proc.quit()
            except Exception:
                log.exception("Process %s quit failed" % procid)

        # TODO: Have a choice of shutdown behaviors for waiting on children, timeouts, etc
        self.proc_sup.shutdown(CFG.cc.timeout.shutdown)
        log.debug("ProcManager stopped, OK.")

    def spawn_process(self, name=None, module=None, cls=None, config=None, process_type=None):
        """
        Spawn a process within the container. Processes can be of different type.
        """
        # Generate a new process id
        # TODO: Ensure it is system-wide unique
        process_id = str(self.proc_id_pool.get_id())
        log.debug("ProcManager.spawn_process(name=%s, module.cls=%s.%s, proc_type=%s) as pid=%s", name, module, cls, process_type, process_id)

        if config is None:
            config = DictModifier(CFG)

        log.debug("spawn_process() pid=%s config=%s", process_id, config)

        # PROCESS TYPE.
        # One of: service, stream_process, agent, simple, immediate
        process_type = process_type or config.get("process", {}).get("type", "service")

        service_instance = None
        try:
            # spawn service by type
            if process_type == "service":
                service_instance = self._spawn_service_process(process_id, name, module, cls, config)

            elif process_type == "stream_process":
                service_instance = self._spawn_stream_process(process_id, name, module, cls, config)

            elif process_type == "agent":
                service_instance = self._spawn_agent_process(process_id, name, module, cls, config)

            elif process_type == "immediate":
                service_instance = self._spawn_immediate_process(process_id, name, module, cls, config)

            elif process_type == "simple":
                service_instance = self._spawn_simple_process(process_id, name, module, cls, config)

            else:
                raise Exception("Unknown process type: %s" % process_type)

            service_instance._proc_type = process_type
            self._register_process(service_instance, name)

            service_instance.errcause = "OK"
            log.info("AppManager.spawn_process: %s.%s -> pid=%s OK" % (module, cls, process_id))
            return process_id

        except Exception:
            errcause = service_instance.errcause if service_instance else "instantiating service"
            log.exception("Error spawning %s %s process (process_id: %s): %s" % (name, process_type, process_id, errcause))
            raise

    def _spawn_service_process(self, process_id, name, module, cls, config):
        """
        Spawn a process acting as a service worker.
        Attach to service queue with service definition, attach to service pid
        """
        service_instance = self._create_service_instance(process_id, name, module, cls, config)
        self._service_init(service_instance)
        listen_name = config.get("process", {}).get("listen_name", name)
        self._set_service_endpoint(service_instance, listen_name)
        self._set_service_endpoint(service_instance, service_instance.id)
        self._service_start(service_instance)
        return service_instance

    def _spawn_stream_process(self, process_id, name, module, cls, config):
        """
        Spawn a process acting as a data stream process.
        Attach to subscription queue with process function.
        """
        service_instance = self._create_service_instance(process_id, name, module, cls, config)
        self._service_init(service_instance)

        listen_name = config.get("process", {}).get("listen_name", name)
        # Throws an exception if no listen name is given!
        self._set_subscription_endpoint(service_instance, listen_name)

        # Spawn the greenlet
        self._service_start(service_instance)

        publish_streams = config.get("process", {}).get("publish_streams", None)
        # Add the publishers if any...
        self._set_publisher_endpoints(service_instance, publish_streams)

        return service_instance

    def _spawn_agent_process(self, process_id, name, module, cls, config):
        """
        Spawn a process acting as agent process.
        Attach to service pid.
        """
        service_instance = self._create_service_instance(process_id, name, module, cls, config)
        self._service_init(service_instance)
        self._set_service_endpoint(service_instance, service_instance.id)
        self._service_start(service_instance)
        return service_instance

    def _spawn_simple_process(self, process_id, name, module, cls, config):
        """
        Spawn a process acting as simple process.
        No attachments.
        """
        service_instance = self._create_service_instance(process_id, name, module, cls, config)
        self._service_init(service_instance)
        self._service_start(service_instance)
        return service_instance

    def _spawn_immediate_process(self, process_id, name, module, cls, config):
        """
        Spawn a process acting as immediate one off process.
        No attachments.
        """
        service_instance = self._create_service_instance(process_id, name, module, cls, config)
        self._service_init(service_instance)
        self._service_start(service_instance)
        return service_instance

    def _create_service_instance(self, process_id, name, module, cls, config):
        # SERVICE INSTANCE.
        service_instance = for_name(module, cls)
        assert isinstance(service_instance, BaseService), "Instantiated service not a BaseService %r" % service_instance

        # Prepare service instance
        service_instance.errcause = ""
        service_instance.id = "%s.%s" % (self.container.id, process_id)
        service_instance.container = self.container
        service_instance.CFG = config
        service_instance._proc_name = name

        # start service dependencies (RPC clients)
        self._start_service_dependencies(service_instance)

        return service_instance

    def _start_service_dependencies(self, service_instance):
        service_instance.errcause = "setting service dependencies"
        log.debug("spawn_process dependencies: %s" % service_instance.dependencies)
        # TODO: Service dependency != process dependency
        for dependency in service_instance.dependencies:
            client = getattr(service_instance.clients, dependency)
            assert client, "Client for dependency not found: %s" % dependency

            # @TODO: should be in a start_client in RPCClient chain
            client.process  = service_instance
            client.node     = self.container.node

    def _service_init(self, service_instance):
        # Init process
        service_instance.errcause = "initializing service"
        service_instance.init()

    def _service_start(self, service_instance):
        # Start process
        # TODO: Check for timeout
        service_instance.errcause = "starting service"
        service_instance.start()

    def _set_service_endpoint(self, service_instance, listen_name):
        service_instance.errcause = "setting process service endpoint"

        # Service RPC endpoint
        rsvc = ProcessRPCServer(node=self.container.node,
                                name=listen_name,
                                service=service_instance,
                                process=service_instance)
        # Start an ION process with the right kind of endpoint factory
        self.proc_sup.spawn((CFG.cc.proctype or 'green', None), listener=rsvc, name=listen_name)
        rsvc.get_ready_event().wait(timeout=10)
        log.debug("Process %s service listener ready: %s", service_instance.id, listen_name)

    def _set_subscription_endpoint(self, service_instance, listen_name):
        service_instance.errcause = "setting process subscription endpoint"

        service_instance.stream_subscriber_registrar = StreamSubscriberRegistrar(process=service_instance, node=self.container.node)

        sub = service_instance.stream_subscriber_registrar.create_subscriber(exchange_name=listen_name,callback=lambda m,h: service_instance.process(m))

        self.proc_sup.spawn((CFG.cc.proctype or 'green', None), listener=sub, name=listen_name)
        sub.get_ready_event().wait(timeout=10)

        log.debug("Process %s stream listener ready: %s", service_instance.id, listen_name)

    def _set_publisher_endpoints(self, service_instance, publisher_streams=None):

        service_instance.stream_publisher_registrar = StreamPublisherRegistrar(process=service_instance, node=self.container.node)

        publisher_streams = publisher_streams or {}

        for name, stream_id in publisher_streams.iteritems():
            # problem is here
            pub = service_instance.stream_publisher_registrar.create_publisher(stream_id)

            setattr(service_instance, name, pub)

    def _register_process(self, service_instance, name):
        # Add to local process dict
        self.procs_by_name[name] = service_instance
        self.procs[service_instance.id] = service_instance

        # Add to directory
        service_instance.errcause = "registering"
        self.container.directory.register("/Containers/%s/Processes" % self.container.id,
                                          service_instance.id, name=name)

    def terminate_process(self, process_id):
        service_instance = self.procs.get(process_id, None)
        if not service_instance: return

        service_instance.quit()
        # TODO: Cleanup messaging attachments

        del self.procs[process_id]
<commit_msg>Fix service_name/listen_name crossover in app startup<commit_after>#!/usr/bin/env python

"""Part of the container that manages ION processes etc."""

__author__ = 'Michael Meisinger'

from zope.interface import implementedBy

from pyon.core.bootstrap import CFG
from pyon.ion.endpoint import ProcessRPCServer, ProcessRPCClient, ProcessSubscriber
from pyon.ion.endpoint import StreamSubscriberRegistrar, StreamSubscriberRegistrarError, StreamPublisher, StreamPublisherRegistrar
from pyon.ion.process import IonProcessSupervisor
from pyon.net.messaging import IDPool
from pyon.service.service import BaseService, get_service_by_name
from pyon.util.containers import DictModifier, DotDict, for_name
from pyon.util.log import log

class ProcManager(object):
    def __init__(self, container):
        self.container = container

        # Define the callables that can be added to Container public API
        self.container_api = [self.spawn_process]

        # Add the public callables to Container
        for call in self.container_api:
            setattr(self.container, call.__name__, call)

        self.proc_id_pool = IDPool()

        # Temporary registry of running processes
        self.procs_by_name = {}
        self.procs = {}

        # The pyon worker process supervisor
        self.proc_sup = IonProcessSupervisor(heartbeat_secs=CFG.cc.timeout.heartbeat)

    def start(self):
        log.debug("ProcManager starting ...")
        self.proc_sup.start()
        log.debug("ProcManager started, OK.")

    def stop(self):
        log.debug("ProcManager stopping ...")

        # Call quit on procs to give them ability to clean up
        # TODO: This can be done concurrently
        for procid, proc in self.procs.iteritems():
            try:
                # These are service processes with full life cycle
                proc.quit()
            except Exception:
                log.exception("Process %s quit failed" % procid)

        # TODO: Have a choice of shutdown behaviors for waiting on children, timeouts, etc
        self.proc_sup.shutdown(CFG.cc.timeout.shutdown)
        log.debug("ProcManager stopped, OK.")

    def spawn_process(self, name=None, module=None, cls=None, config=None, process_type=None):
        """
        Spawn a process within the container. Processes can be of different type.
        """
        # Generate a new process id
        # TODO: Ensure it is system-wide unique
        process_id = str(self.proc_id_pool.get_id())
        log.debug("ProcManager.spawn_process(name=%s, module.cls=%s.%s, proc_type=%s) as pid=%s", name, module, cls, process_type, process_id)

        if config is None:
            config = DictModifier(CFG)

        log.debug("spawn_process() pid=%s config=%s", process_id, config)

        # PROCESS TYPE.
        # One of: service, stream_process, agent, simple, immediate
        process_type = process_type or config.get("process", {}).get("type", "service")

        service_instance = None
        try:
            # spawn service by type
            if process_type == "service":
                service_instance = self._spawn_service_process(process_id, name, module, cls, config)

            elif process_type == "stream_process":
                service_instance = self._spawn_stream_process(process_id, name, module, cls, config)

            elif process_type == "agent":
                service_instance = self._spawn_agent_process(process_id, name, module, cls, config)

            elif process_type == "immediate":
                service_instance = self._spawn_immediate_process(process_id, name, module, cls, config)

            elif process_type == "simple":
                service_instance = self._spawn_simple_process(process_id, name, module, cls, config)

            else:
                raise Exception("Unknown process type: %s" % process_type)

            service_instance._proc_type = process_type
            self._register_process(service_instance, name)

            service_instance.errcause = "OK"
            log.info("AppManager.spawn_process: %s.%s -> pid=%s OK" % (module, cls, process_id))
            return process_id

        except Exception:
            errcause = service_instance.errcause if service_instance else "instantiating service"
            log.exception("Error spawning %s %s process (process_id: %s): %s" % (name, process_type, process_id, errcause))
            raise

    def _spawn_service_process(self, process_id, name, module, cls, config):
        """
        Spawn a process acting as a service worker.
        Attach to service queue with service definition, attach to service pid
        """
        service_instance = self._create_service_instance(process_id, name, module, cls, config)
        self._service_init(service_instance)

        listen_name = config.get("process", {}).get("listen_name", service_instance.name)
        log.debug("Service Process (%s) listen_name: %s", name, listen_name)

        self._set_service_endpoint(service_instance, listen_name)
        self._set_service_endpoint(service_instance, service_instance.id)
        self._service_start(service_instance)
        return service_instance

    def _spawn_stream_process(self, process_id, name, module, cls, config):
        """
        Spawn a process acting as a data stream process.
        Attach to subscription queue with process function.
        """
        service_instance = self._create_service_instance(process_id, name, module, cls, config)
        self._service_init(service_instance)

        listen_name = config.get("process", {}).get("listen_name", name)
        # Throws an exception if no listen name is given!
        self._set_subscription_endpoint(service_instance, listen_name)

        # Spawn the greenlet
        self._service_start(service_instance)

        publish_streams = config.get("process", {}).get("publish_streams", None)
        # Add the publishers if any...
        self._set_publisher_endpoints(service_instance, publish_streams)

        return service_instance

    def _spawn_agent_process(self, process_id, name, module, cls, config):
        """
        Spawn a process acting as agent process.
        Attach to service pid.
        """
        service_instance = self._create_service_instance(process_id, name, module, cls, config)
        self._service_init(service_instance)
        self._set_service_endpoint(service_instance, service_instance.id)
        self._service_start(service_instance)
        return service_instance

    def _spawn_simple_process(self, process_id, name, module, cls, config):
        """
        Spawn a process acting as simple process.
        No attachments.
        """
        service_instance = self._create_service_instance(process_id, name, module, cls, config)
        self._service_init(service_instance)
        self._service_start(service_instance)
        return service_instance

    def _spawn_immediate_process(self, process_id, name, module, cls, config):
        """
        Spawn a process acting as immediate one off process.
        No attachments.
        """
        service_instance = self._create_service_instance(process_id, name, module, cls, config)
        self._service_init(service_instance)
        self._service_start(service_instance)
        return service_instance

    def _create_service_instance(self, process_id, name, module, cls, config):
        # SERVICE INSTANCE.
        service_instance = for_name(module, cls)
        assert isinstance(service_instance, BaseService), "Instantiated service not a BaseService %r" % service_instance

        # Prepare service instance
        service_instance.errcause = ""
        service_instance.id = "%s.%s" % (self.container.id, process_id)
        service_instance.container = self.container
        service_instance.CFG = config
        service_instance._proc_name = name

        # start service dependencies (RPC clients)
        self._start_service_dependencies(service_instance)

        return service_instance

    def _start_service_dependencies(self, service_instance):
        service_instance.errcause = "setting service dependencies"
        log.debug("spawn_process dependencies: %s" % service_instance.dependencies)
        # TODO: Service dependency != process dependency
        for dependency in service_instance.dependencies:
            client = getattr(service_instance.clients, dependency)
            assert client, "Client for dependency not found: %s" % dependency

            # @TODO: should be in a start_client in RPCClient chain
            client.process  = service_instance
            client.node     = self.container.node

    def _service_init(self, service_instance):
        # Init process
        service_instance.errcause = "initializing service"
        service_instance.init()

    def _service_start(self, service_instance):
        # Start process
        # TODO: Check for timeout
        service_instance.errcause = "starting service"
        service_instance.start()

    def _set_service_endpoint(self, service_instance, listen_name):
        service_instance.errcause = "setting process service endpoint"

        # Service RPC endpoint
        rsvc = ProcessRPCServer(node=self.container.node,
                                name=listen_name,
                                service=service_instance,
                                process=service_instance)
        # Start an ION process with the right kind of endpoint factory
        self.proc_sup.spawn((CFG.cc.proctype or 'green', None), listener=rsvc, name=listen_name)
        rsvc.get_ready_event().wait(timeout=10)
        log.debug("Process %s service listener ready: %s", service_instance.id, listen_name)

    def _set_subscription_endpoint(self, service_instance, listen_name):
        service_instance.errcause = "setting process subscription endpoint"

        service_instance.stream_subscriber_registrar = StreamSubscriberRegistrar(process=service_instance, node=self.container.node)

        sub = service_instance.stream_subscriber_registrar.create_subscriber(exchange_name=listen_name,callback=lambda m,h: service_instance.process(m))

        self.proc_sup.spawn((CFG.cc.proctype or 'green', None), listener=sub, name=listen_name)
        sub.get_ready_event().wait(timeout=10)

        log.debug("Process %s stream listener ready: %s", service_instance.id, listen_name)

    def _set_publisher_endpoints(self, service_instance, publisher_streams=None):

        service_instance.stream_publisher_registrar = StreamPublisherRegistrar(process=service_instance, node=self.container.node)

        publisher_streams = publisher_streams or {}

        for name, stream_id in publisher_streams.iteritems():
            # problem is here
            pub = service_instance.stream_publisher_registrar.create_publisher(stream_id)

            setattr(service_instance, name, pub)

    def _register_process(self, service_instance, name):
        # Add to local process dict
        self.procs_by_name[name] = service_instance
        self.procs[service_instance.id] = service_instance

        # Add to directory
        service_instance.errcause = "registering"
        self.container.directory.register("/Containers/%s/Processes" % self.container.id,
                                          service_instance.id, name=name)

    def terminate_process(self, process_id):
        service_instance = self.procs.get(process_id, None)
        if not service_instance: return

        service_instance.quit()
        # TODO: Cleanup messaging attachments

        del self.procs[process_id]
<|endoftext|>


<commit_before>#########################################################################
#This module intergrates Open ERP with the magento core                 #
#Core settings are stored here                                          #
#########################################################################
#                                                                       #
# Copyright (C) 2009  Sharoon Thomas                                    #
#                                                                       #
#This program is free software: you can redistribute it and/or modify   #
#it under the terms of the GNU General Public License as published by   #
#the Free Software Foundation, either version 3 of the License, or      #
#(at your option) any later version.                                    #
#                                                                       #
#This program is distributed in the hope that it will be useful,        #
#but WITHOUT ANY WARRANTY; without even the implied warranty of         #
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the          #
#GNU General Public License for more details.                           #
#                                                                       #
#You should have received a copy of the GNU General Public License      #
#along with this program.  If not, see <http://www.gnu.org/licenses/>.  #
#########################################################################

from osv import osv, fields
import magerp_osv
from base_external_referentials import external_osv

DEBUG = True
TIMEOUT = 2

class external_referential(magerp_osv.magerp_osv):
    #This class stores instances of magento to which the ERP will connect, the concept of multi website, multistore integration?
    _inherit = "external.referential"

    _columns = {
        'attribute_sets':fields.one2many('magerp.product_attribute_set', 'instance', 'Attribute Sets'),
        'default_pro_cat':fields.many2one('product.category','Default Product Category',required=True, help="Products imported from magento may have many categories.\nOpen ERP requires a specific category for a product to facilitate invoicing etc.")
    }

    def connect(self, cr, uid, ids, ctx={}):#TODO used?
        #ids has to be a list
        if ids:
            if len(ids) == 1:
                instance = self.browse(cr, uid, ids, ctx)[0]
                if instance:
                    core_imp_conn = self.external_connection(cr, uid, instance, DEBUG)
                    if core_imp_conn.connect():
                        return core_imp_conn
        return False

    def core_sync(self, cr, uid, ids, ctx={}):
        instances = self.browse(cr, uid, ids, ctx)
        filter = []
        for inst in instances:
            core_imp_conn = self.external_connection(cr, uid, inst, DEBUG)
            if core_imp_conn:
                #New import methods
                self.pool.get('external.shop.group').mage_import_base(cr, uid,core_imp_conn, inst.id)
                self.pool.get('magerp.storeviews').mage_import_base(cr,uid,core_imp_conn, inst.id)
                self.pool.get('sale.shop').mage_import_base(cr, uid, core_imp_conn, inst.id)
            else:
                osv.except_osv(_("Connection Error"), _("Could not connect to server\nCheck location, username & password."))

    def sync_categs(self, cr, uid, ids, ctx):
        instances = self.browse(cr, uid, ids, ctx)
        for inst in instances:
            pro_cat_conn = self.external_connection(cr, uid, inst, DEBUG)
            if pro_cat_conn:
                confirmation = pro_cat_conn.call('catalog_category.currentStore', [0])   #Set browse to root store
                if confirmation:
                    categ_tree = pro_cat_conn.call('catalog_category.tree')             #Get the tree
                    self.pool.get('product.category').record_entire_tree(cr, uid, inst.id, pro_cat_conn, categ_tree, DEBUG)
                    exp_ids = self.pool.get('product.category').search(cr,uid,[('exportable','=',True)])
                    self.pool.get('product.category').ext_export(cr,uid,exp_ids,[inst.id],{},{'conn_obj':pro_cat_conn})
            else:
                osv.except_osv(_("Connection Error"), _("Could not connect to server\nCheck location, username & password."))

    def sync_attribs(self, cr, uid, ids, ctx):
        instances = self.browse(cr, uid, ids, ctx)
        for inst in instances:
            attr_conn = self.external_connection(cr, uid, inst, DEBUG)
            if attr_conn:
                attrib_set_ids = self.pool.get('magerp.product_attribute_set').search(cr, uid, [('instance', '=', inst.id)])
                attrib_sets = self.pool.get('magerp.product_attribute_set').read(cr, uid, attrib_set_ids, ['magento_id'])
                #Get all attribute set ids to get all attributes in one go
                all_attr_set_ids = self.pool.get('magerp.product_attribute_set').get_all_mage_ids(cr, uid, [], inst.id)
                #Call magento for all attributes
                mage_inp = attr_conn.call('ol_catalog_product_attribute.list', all_attr_set_ids)             #Get the tree
                #self.pool.get('magerp.product_attributes').sync_import(cr, uid, mage_inp, inst.id, DEBUG) #Last argument is extra mage2oe filter as same attribute ids
                self.pool.get('magerp.product_attributes').ext_import(cr, uid, mage_inp, inst.id, defaults={'instance':inst.id}, context={'referential_id':inst.id})
                #Relate attribute sets & attributes
                mage_inp = {}
                #Pass in {attribute_set_id:{attributes},attribute_set_id2:{attributes}}
                #print "Attribute sets are:", attrib_sets
                for each in attrib_sets:
                    mage_inp[each['magento_id']] = attr_conn.call('ol_catalog_product_attribute.relations', [each['magento_id']])
                if mage_inp:
                    self.pool.get('magerp.product_attribute_set').relate(cr, uid, mage_inp, inst.id, DEBUG)
            else:
                osv.except_osv(_("Connection Error"), _("Could not connect to server\nCheck location, username & password."))

    def sync_attrib_sets(self, cr, uid, ids, ctx):
        instances = self.browse(cr, uid, ids, ctx)
        for inst in instances:
            attr_conn = self.external_connection(cr, uid, inst, DEBUG)
            if attr_conn:
                filter = []
                #self.pool.get('magerp.product_attribute_set').mage_import(cr, uid, filter, attr_conn, inst.id, DEBUG)
                self.pool.get('magerp.product_attribute_set').mage_import_base(cr, uid, attr_conn, inst.id,{'instance':inst.id},{'ids_or_filter':filter})
            else:
                osv.except_osv(_("Connection Error"), _("Could not connect to server\nCheck location, username & password."))

    def sync_attrib_groups(self, cr, uid, ids, ctx):
        instances = self.browse(cr, uid, ids, ctx)
        for inst in instances:
            attr_conn = self.external_connection(cr, uid, inst, DEBUG)
            attrset_ids = self.pool.get('magerp.product_attribute_set').get_all_mage_ids(cr, uid, [], inst.id)
            filter = [{'attribute_set_id':{'in':attrset_ids}}]
            if attr_conn:
                #self.pool.get('magerp.product_attribute_groups').mage_import(cr, uid, filter, attr_conn, inst.id, DEBUG)
                self.pool.get('magerp.product_attribute_groups').mage_import_base(cr, uid, attr_conn, inst.id, {'instance':inst.id}, {'ids_or_filter':filter})
            else:
                osv.except_osv(_("Connection Error"), _("Could not connect to server\nCheck location, username & password."))

    def sync_customer_groups(self, cr, uid, ids, ctx):
        instances = self.browse(cr, uid, ids, ctx)
        for inst in instances:
            attr_conn = self.external_connection(cr, uid, inst, DEBUG)
            filter = []
            if attr_conn:
                self.pool.get('res.partner.category').mage_import(cr, uid, filter, attr_conn, inst.id, DEBUG)
            else:
                osv.except_osv(_("Connection Error"), _("Could not connect to server\nCheck location, username & password."))

    def sync_customer_addresses(self, cr, uid, ids, ctx):
        instances = self.browse(cr, uid, ids, ctx)
        for inst in instances:
            attr_conn = self.external_connection(cr, uid, inst, DEBUG)
            filter = []
            if attr_conn:
                self.pool.get('res.partner').mage_import(cr, uid, filter, attr_conn, inst.id, DEBUG)
            else:
                osv.except_osv(_("Connection Error"), _("Could not connect to server\nCheck location, username & password."))

    def sync_products(self, cr, uid, ids, ctx):
        instances = self.browse(cr, uid, ids, ctx)
        for inst in instances:
            attr_conn = self.external_connection(cr, uid, inst, DEBUG)
            filter = []
            if attr_conn:
                list_prods = attr_conn.call('catalog_product.list')
                #self.pool.get('product.product').mage_import(cr, uid, filter, attr_conn, inst.id, DEBUG)
                result = []
                for each in list_prods:
                    each_product_info = attr_conn.call('catalog_product.info', [each['product_id']])
                    result.append(each_product_info)
                self.pool.get('product.product').ext_import(cr, uid, result, inst.id, defaults={'instance':inst.id,'categ_id':inst.default_pro_cat.id}, context={})
            else:
                osv.except_osv(_("Connection Error"), _("Could not connect to server\nCheck location, username & password."))
    def redefine_prod_view(self,cr,uid,ids,ctx):
        #This function will rebuild the view for product from instances, attribute groups etc
        #Get all objects needed
        inst_obj = self.pool.get('external.referential')
        attr_set_obj = self.pool.get('magerp.product_attribute_set')
        attr_group_obj = self.pool.get('magerp.product_attribute_group')
        attr_obj = self.pool.get('magerp.product_attributes')
        #Predefined items on top
        #Instance
        #Attribute Set
        #Get all instances
        inst_ids = inst_obj.search(cr,uid,[])#TODO:Search for active instances only
        instances = inst_obj.read(cr,uid,inst_ids,[])
        for each_instance in instances:
            #create a group & a notebook inside, group attr
            attr_set_ids = attr_set_obj.search(cr,uid,[('instance','=',each_instance['id'])])
            attr_sets = attr_set_obj.browse(cr,uid,attr_set_ids)
            for each_set in attr_sets:
                #Create a page with attrs corresponding to the set id
                attr_grp_ids = attr_group_obj.search(cr,uid,[('attribute_set','=',each_set['id'])]) #attribute_set is a function field, may slow down the whole thing
                attr_groups = attr_group_obj.read(cr,uid,attr_grp_ids,[])
                for each_group in attr_groups:
                    #Create a page for the attribute group
                    attribute_ids = each_set.attributes
                    attr

    def export_products(self, cr, uid, ids, ctx):
        instances = self.browse(cr, uid, ids, ctx)
        for inst in instances:
            attr_conn = self.external_connection(cr, uid, inst, DEBUG)
            if attr_conn:
                ids = self.pool.get('product.product').search(cr, uid, [])
                self.pool.get('product.product').mage_export(cr, uid, ids, attr_conn, inst.id, DEBUG)
            else:
                osv.except_osv(_("Connection Error"), _("Could not connect to server\nCheck location, username & password."))

external_referential()

class external_shop_group(magerp_osv.magerp_osv):
    _inherit = "external.shop.group"
    #Return format of API:{'code': 'base', 'name': 'Main', 'website_id': '1', 'is_default': '1', 'sort_order': '0', 'default_group_id': '1'}

    def _get_group(self, cr, uid, ids, prop, unknow_none, context):
        res = self.group_get(cr, uid, ids, context={'field':'default_group_id'})
        return dict(res)

    _order = 'magento_id'

    _columns = {
        'code':fields.char('Code', size=100),
        'magento_id':fields.integer('Website ID'),
        'is_default':fields.boolean('Is Active?'),
        'sort_order':fields.integer('Sort Order'),
        'default_group_id':fields.integer('Default Store Group'), #Many 2 one?
        'default_group':fields.function(_get_group, type="many2one", relation="sale.shop", method=True, string="Default Store (Group)"),
        'instance':fields.many2one('external.referential', 'Instance', ondelete='cascade')
    }

external_shop_group()

class magerp_storeviews(magerp_osv.magerp_osv):
    _name = "magerp.storeviews"
    _description = "The magento store views information"

    def _get_website(self, cr, uid, ids, prop, unknow_none, context):
        res = self.website_get(cr, uid, ids, context={'field':'magento_id'})
        return dict(res)

    def _get_group(self, cr, uid, ids, prop, unknow_none, context):
        res = self.group_get(cr, uid, ids, context={'field':'group_id'})
        return dict(res)

    _order = 'magento_id'
    _LIST_METHOD = 'ol_storeviews.list'
    _MAGE_P_KEY = 'store_id'
    _columns = {
        'name':fields.char('Store View Name', size=100),
        'code':fields.char('Code', size=100),
        'magento_id':fields.integer('Store ID'),
        'website_id':fields.integer('Website'), # Many 2 one ?
        'website':fields.function(_get_website, type="many2one", relation="external.shop.group", method=True, string="Website"),
        'is_active':fields.boolean('Default ?'),
        'sort_order':fields.integer('Sort Order'),
        'group_id':fields.integer('Default Store Group'), #Many 2 one?
        'default_group':fields.function(_get_group, type="many2one", relation="sale.shop", method=True, string="Default Store (Group)"),
        #'instance':fields.many2one('external.referential', 'Instance', ondelete='cascade')
    }

    #Return format of API:{'code': 'default', 'store_id': '1', 'website_id': '1', 'is_active': '1', 'sort_order': '0', 'group_id': '1', 'name': 'Default Store View'}

magerp_storeviews()<commit_msg>[IMP] shop_group referential_id passed as default param using standard API<commit_after>#########################################################################
#This module intergrates Open ERP with the magento core                 #
#Core settings are stored here                                          #
#########################################################################
#                                                                       #
# Copyright (C) 2009  Sharoon Thomas                                    #
#                                                                       #
#This program is free software: you can redistribute it and/or modify   #
#it under the terms of the GNU General Public License as published by   #
#the Free Software Foundation, either version 3 of the License, or      #
#(at your option) any later version.                                    #
#                                                                       #
#This program is distributed in the hope that it will be useful,        #
#but WITHOUT ANY WARRANTY; without even the implied warranty of         #
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the          #
#GNU General Public License for more details.                           #
#                                                                       #
#You should have received a copy of the GNU General Public License      #
#along with this program.  If not, see <http://www.gnu.org/licenses/>.  #
#########################################################################

from osv import osv, fields
import magerp_osv
from base_external_referentials import external_osv

DEBUG = True
TIMEOUT = 2

class external_referential(magerp_osv.magerp_osv):
    #This class stores instances of magento to which the ERP will connect, the concept of multi website, multistore integration?
    _inherit = "external.referential"

    _columns = {
        'attribute_sets':fields.one2many('magerp.product_attribute_set', 'instance', 'Attribute Sets'),
        'default_pro_cat':fields.many2one('product.category','Default Product Category',required=True, help="Products imported from magento may have many categories.\nOpen ERP requires a specific category for a product to facilitate invoicing etc.")
    }

    def connect(self, cr, uid, ids, ctx={}):#TODO used?
        #ids has to be a list
        if ids:
            if len(ids) == 1:
                instance = self.browse(cr, uid, ids, ctx)[0]
                if instance:
                    core_imp_conn = self.external_connection(cr, uid, instance, DEBUG)
                    if core_imp_conn.connect():
                        return core_imp_conn
        return False

    def core_sync(self, cr, uid, ids, ctx={}):
        instances = self.browse(cr, uid, ids, ctx)
        filter = []
        for inst in instances:
            core_imp_conn = self.external_connection(cr, uid, inst, DEBUG)
            if core_imp_conn:
                #New import methods
                self.pool.get('external.shop.group').mage_import_base(cr, uid,core_imp_conn, inst.id, defaults={'referential_id':inst.id})
                self.pool.get('magerp.storeviews').mage_import_base(cr,uid,core_imp_conn, inst.id, defaults={})
                self.pool.get('sale.shop').mage_import_base(cr, uid, core_imp_conn, inst.id, defaults={})
            else:
                osv.except_osv(_("Connection Error"), _("Could not connect to server\nCheck location, username & password."))

    def sync_categs(self, cr, uid, ids, ctx):
        instances = self.browse(cr, uid, ids, ctx)
        for inst in instances:
            pro_cat_conn = self.external_connection(cr, uid, inst, DEBUG)
            if pro_cat_conn:
                confirmation = pro_cat_conn.call('catalog_category.currentStore', [0])   #Set browse to root store
                if confirmation:
                    categ_tree = pro_cat_conn.call('catalog_category.tree')             #Get the tree
                    self.pool.get('product.category').record_entire_tree(cr, uid, inst.id, pro_cat_conn, categ_tree, DEBUG)
                    exp_ids = self.pool.get('product.category').search(cr,uid,[('exportable','=',True)])
                    self.pool.get('product.category').ext_export(cr,uid,exp_ids,[inst.id],{},{'conn_obj':pro_cat_conn})
            else:
                osv.except_osv(_("Connection Error"), _("Could not connect to server\nCheck location, username & password."))

    def sync_attribs(self, cr, uid, ids, ctx):
        instances = self.browse(cr, uid, ids, ctx)
        for inst in instances:
            attr_conn = self.external_connection(cr, uid, inst, DEBUG)
            if attr_conn:
                attrib_set_ids = self.pool.get('magerp.product_attribute_set').search(cr, uid, [('instance', '=', inst.id)])
                attrib_sets = self.pool.get('magerp.product_attribute_set').read(cr, uid, attrib_set_ids, ['magento_id'])
                #Get all attribute set ids to get all attributes in one go
                all_attr_set_ids = self.pool.get('magerp.product_attribute_set').get_all_mage_ids(cr, uid, [], inst.id)
                #Call magento for all attributes
                mage_inp = attr_conn.call('ol_catalog_product_attribute.list', all_attr_set_ids)             #Get the tree
                #self.pool.get('magerp.product_attributes').sync_import(cr, uid, mage_inp, inst.id, DEBUG) #Last argument is extra mage2oe filter as same attribute ids
                self.pool.get('magerp.product_attributes').ext_import(cr, uid, mage_inp, inst.id, defaults={'instance':inst.id}, context={'referential_id':inst.id})
                #Relate attribute sets & attributes
                mage_inp = {}
                #Pass in {attribute_set_id:{attributes},attribute_set_id2:{attributes}}
                #print "Attribute sets are:", attrib_sets
                for each in attrib_sets:
                    mage_inp[each['magento_id']] = attr_conn.call('ol_catalog_product_attribute.relations', [each['magento_id']])
                if mage_inp:
                    self.pool.get('magerp.product_attribute_set').relate(cr, uid, mage_inp, inst.id, DEBUG)
            else:
                osv.except_osv(_("Connection Error"), _("Could not connect to server\nCheck location, username & password."))

    def sync_attrib_sets(self, cr, uid, ids, ctx):
        instances = self.browse(cr, uid, ids, ctx)
        for inst in instances:
            attr_conn = self.external_connection(cr, uid, inst, DEBUG)
            if attr_conn:
                filter = []
                #self.pool.get('magerp.product_attribute_set').mage_import(cr, uid, filter, attr_conn, inst.id, DEBUG)
                self.pool.get('magerp.product_attribute_set').mage_import_base(cr, uid, attr_conn, inst.id,{'instance':inst.id},{'ids_or_filter':filter})
            else:
                osv.except_osv(_("Connection Error"), _("Could not connect to server\nCheck location, username & password."))

    def sync_attrib_groups(self, cr, uid, ids, ctx):
        instances = self.browse(cr, uid, ids, ctx)
        for inst in instances:
            attr_conn = self.external_connection(cr, uid, inst, DEBUG)
            attrset_ids = self.pool.get('magerp.product_attribute_set').get_all_mage_ids(cr, uid, [], inst.id)
            filter = [{'attribute_set_id':{'in':attrset_ids}}]
            if attr_conn:
                #self.pool.get('magerp.product_attribute_groups').mage_import(cr, uid, filter, attr_conn, inst.id, DEBUG)
                self.pool.get('magerp.product_attribute_groups').mage_import_base(cr, uid, attr_conn, inst.id, {'instance':inst.id}, {'ids_or_filter':filter})
            else:
                osv.except_osv(_("Connection Error"), _("Could not connect to server\nCheck location, username & password."))

    def sync_customer_groups(self, cr, uid, ids, ctx):
        instances = self.browse(cr, uid, ids, ctx)
        for inst in instances:
            attr_conn = self.external_connection(cr, uid, inst, DEBUG)
            filter = []
            if attr_conn:
                self.pool.get('res.partner.category').mage_import(cr, uid, filter, attr_conn, inst.id, DEBUG)
            else:
                osv.except_osv(_("Connection Error"), _("Could not connect to server\nCheck location, username & password."))

    def sync_customer_addresses(self, cr, uid, ids, ctx):
        instances = self.browse(cr, uid, ids, ctx)
        for inst in instances:
            attr_conn = self.external_connection(cr, uid, inst, DEBUG)
            filter = []
            if attr_conn:
                self.pool.get('res.partner').mage_import(cr, uid, filter, attr_conn, inst.id, DEBUG)
            else:
                osv.except_osv(_("Connection Error"), _("Could not connect to server\nCheck location, username & password."))

    def sync_products(self, cr, uid, ids, ctx):
        instances = self.browse(cr, uid, ids, ctx)
        for inst in instances:
            attr_conn = self.external_connection(cr, uid, inst, DEBUG)
            filter = []
            if attr_conn:
                list_prods = attr_conn.call('catalog_product.list')
                #self.pool.get('product.product').mage_import(cr, uid, filter, attr_conn, inst.id, DEBUG)
                result = []
                for each in list_prods:
                    each_product_info = attr_conn.call('catalog_product.info', [each['product_id']])
                    result.append(each_product_info)
                self.pool.get('product.product').ext_import(cr, uid, result, inst.id, defaults={'instance':inst.id,'categ_id':inst.default_pro_cat.id}, context={})
            else:
                osv.except_osv(_("Connection Error"), _("Could not connect to server\nCheck location, username & password."))
    def redefine_prod_view(self,cr,uid,ids,ctx):
        #This function will rebuild the view for product from instances, attribute groups etc
        #Get all objects needed
        inst_obj = self.pool.get('external.referential')
        attr_set_obj = self.pool.get('magerp.product_attribute_set')
        attr_group_obj = self.pool.get('magerp.product_attribute_group')
        attr_obj = self.pool.get('magerp.product_attributes')
        #Predefined items on top
        #Instance
        #Attribute Set
        #Get all instances
        inst_ids = inst_obj.search(cr,uid,[])#TODO:Search for active instances only
        instances = inst_obj.read(cr,uid,inst_ids,[])
        for each_instance in instances:
            #create a group & a notebook inside, group attr
            attr_set_ids = attr_set_obj.search(cr,uid,[('instance','=',each_instance['id'])])
            attr_sets = attr_set_obj.browse(cr,uid,attr_set_ids)
            for each_set in attr_sets:
                #Create a page with attrs corresponding to the set id
                attr_grp_ids = attr_group_obj.search(cr,uid,[('attribute_set','=',each_set['id'])]) #attribute_set is a function field, may slow down the whole thing
                attr_groups = attr_group_obj.read(cr,uid,attr_grp_ids,[])
                for each_group in attr_groups:
                    #Create a page for the attribute group
                    attribute_ids = each_set.attributes
                    attr

    def export_products(self, cr, uid, ids, ctx):
        instances = self.browse(cr, uid, ids, ctx)
        for inst in instances:
            attr_conn = self.external_connection(cr, uid, inst, DEBUG)
            if attr_conn:
                ids = self.pool.get('product.product').search(cr, uid, [])
                self.pool.get('product.product').mage_export(cr, uid, ids, attr_conn, inst.id, DEBUG)
            else:
                osv.except_osv(_("Connection Error"), _("Could not connect to server\nCheck location, username & password."))

external_referential()

class external_shop_group(magerp_osv.magerp_osv):
    _inherit = "external.shop.group"
    #Return format of API:{'code': 'base', 'name': 'Main', 'website_id': '1', 'is_default': '1', 'sort_order': '0', 'default_group_id': '1'}

    def _get_group(self, cr, uid, ids, prop, unknow_none, context):
        res = self.group_get(cr, uid, ids, context={'field':'default_group_id'})
        return dict(res)

    _order = 'magento_id'

    _columns = {
        'code':fields.char('Code', size=100),
        'magento_id':fields.integer('Website ID'),
        'is_default':fields.boolean('Is Active?'),
        'sort_order':fields.integer('Sort Order'),
        'default_group_id':fields.integer('Default Store Group'), #Many 2 one?
        'default_group':fields.function(_get_group, type="many2one", relation="sale.shop", method=True, string="Default Store (Group)"),
        'instance':fields.many2one('external.referential', 'Instance', ondelete='cascade')
    }

external_shop_group()

class magerp_storeviews(magerp_osv.magerp_osv):
    _name = "magerp.storeviews"
    _description = "The magento store views information"

    def _get_website(self, cr, uid, ids, prop, unknow_none, context):
        res = self.website_get(cr, uid, ids, context={'field':'magento_id'})
        return dict(res)

    def _get_group(self, cr, uid, ids, prop, unknow_none, context):
        res = self.group_get(cr, uid, ids, context={'field':'group_id'})
        return dict(res)

    _order = 'magento_id'
    _LIST_METHOD = 'ol_storeviews.list'
    _MAGE_P_KEY = 'store_id'
    _columns = {
        'name':fields.char('Store View Name', size=100),
        'code':fields.char('Code', size=100),
        'magento_id':fields.integer('Store ID'),
        'website_id':fields.integer('Website'), # Many 2 one ?
        'website':fields.function(_get_website, type="many2one", relation="external.shop.group", method=True, string="Website"),
        'is_active':fields.boolean('Default ?'),
        'sort_order':fields.integer('Sort Order'),
        'group_id':fields.integer('Default Store Group'), #Many 2 one?
        'default_group':fields.function(_get_group, type="many2one", relation="sale.shop", method=True, string="Default Store (Group)"),
        #'instance':fields.many2one('external.referential', 'Instance', ondelete='cascade')
    }

    #Return format of API:{'code': 'default', 'store_id': '1', 'website_id': '1', 'is_active': '1', 'sort_order': '0', 'group_id': '1', 'name': 'Default Store View'}

magerp_storeviews()<|endoftext|>


<commit_before>from .naive_bayes import GaussianNB
<commit_msg>adds KMeans as import for models<commit_after>from .naive_bayes import GaussianNB
from .k_means import KMeans
<|endoftext|>


<commit_before>import subprocess
import time
import locale
import logging

from pyp2rpm import version
from pyp2rpm import utils

logger = logging.getLogger(__name__)

class PackageData(object):
    credit_line = '# Created by pyp2rpm-{0}'.format(version.version)

    """A simple object that carries data about a package."""

    def __init__(self, local_file, name, pkg_name, version, md5='', url=''):
        object.__setattr__(self, 'data', {})
        self.data['local_file'] = local_file
        self.data['name'] = name
        self.data['pkg_name'] = pkg_name
        self.data['version'] = version
        self.data['python_versions'] = []
        self.data['md5'] = md5
        self.data['url'] = url
        self.data['sphinx_dir'] = None

    def __getattr__(self, name):
        if name == 'underscored_name':
            return self.data['name'].replace('-', '_')
        elif name == 'changelog_date_packager':
            return self.get_changelog_date_packager()
        return self.data.get(name, 'TODO:')

    def __setattr__(self, name, value):
        if name == 'summary' and isinstance(value, utils.str_classes):
            value = value.rstrip('.').replace('\n', ' ')
        if value:
            self.data[name] = value

    def set_from(self, data_dict):
        for k, v in data_dict.items():
            setattr(self, k, v)

    def get_changelog_date_packager(self):
        """Returns part of the changelog entry, containing date and packager.
        """
        try:
            packager = subprocess.Popen(
                'rpmdev-packager', stdout=subprocess.PIPE).communicate()[0].strip()
        except OSError:
            # Hi John Doe, you should install rpmdevtools
            packager = "John Doe <john@doe.com>"
            logger.warn(
                'Package rpmdevtools is missing, using default name: {0}.'.format(packager))
        date_str = time.strftime('%a %b %d %Y', time.gmtime())
        encoding = locale.getpreferredencoding()
        return u'{0} {1}'.format(date_str, packager.decode(encoding))
<commit_msg>Check only for None value in PackageData setattr<commit_after>import subprocess
import time
import locale
import logging

from pyp2rpm import version
from pyp2rpm import utils

logger = logging.getLogger(__name__)

class PackageData(object):
    credit_line = '# Created by pyp2rpm-{0}'.format(version.version)

    """A simple object that carries data about a package."""

    def __init__(self, local_file, name, pkg_name, version, md5='', url=''):
        object.__setattr__(self, 'data', {})
        self.data['local_file'] = local_file
        self.data['name'] = name
        self.data['pkg_name'] = pkg_name
        self.data['version'] = version
        self.data['python_versions'] = []
        self.data['md5'] = md5
        self.data['url'] = url
        self.data['sphinx_dir'] = None

    def __getattr__(self, name):
        if name == 'underscored_name':
            return self.data['name'].replace('-', '_')
        elif name == 'changelog_date_packager':
            return self.get_changelog_date_packager()
        return self.data.get(name, 'TODO:')

    def __setattr__(self, name, value):
        if name == 'summary' and isinstance(value, utils.str_classes):
            value = value.rstrip('.').replace('\n', ' ')
        if value is not None:
            self.data[name] = value

    def set_from(self, data_dict):
        for k, v in data_dict.items():
            setattr(self, k, v)

    def get_changelog_date_packager(self):
        """Returns part of the changelog entry, containing date and packager.
        """
        try:
            packager = subprocess.Popen(
                'rpmdev-packager', stdout=subprocess.PIPE).communicate()[0].strip()
        except OSError:
            # Hi John Doe, you should install rpmdevtools
            packager = "John Doe <john@doe.com>"
            logger.warn(
                'Package rpmdevtools is missing, using default name: {0}.'.format(packager))
        date_str = time.strftime('%a %b %d %Y', time.gmtime())
        encoding = locale.getpreferredencoding()
        return u'{0} {1}'.format(date_str, packager.decode(encoding))
<|endoftext|>


<commit_before>import requests

class Timeline(object):
    api_root = 'https://timeline-api.getpebble.com'

    error_codes = {
        400: 'The pin object submitted was invalid.',
        403: 'The API key submitted was invalid.',
        410: 'The user token submitted was invalid or does not exist.',
        429: 'Server is sending updates too quickly.',
        503: 'Could not save pin due to a temporary server error.',
    }

    def __init__(self, api_key=None, api_root=None):
        self._api_key = api_key

        if api_root is not None:
            self._api_root = api_root
        else:
            self._api_root = type(self).api_root

    def send_shared_pin(self, topics, pin):
        url = "{}/v1/shared/pins/{}".format(self._api_root, pin['id'])

        result = requests.put(
            url,
            headers={
                'X-API-Key': self._api_key,
                'X-PIN-Topics': ','.join(str(t) for t in topics),
            },
            json=dict(pin),
        )
        return result

    def send_user_pin(self, user_token, pin):
        url = "{}/v1/user/pins/{}".format(self._api_root, pin['id'])

        result = requests.put(
            url,
            headers={
                'X-User-Token': user_token,
            },
            json=dict(pin),
        )
        return result

    def delete_user_pin(self, user_token, pin):
        url = "{}/v1/user/pins/{}".format(self._api_root, pin['id'])

        result = requests.delete(
            url,
            headers={
                'X-User-Token': self.user_token,
            },
        )
        return result

    def subscribe(self, user_token, topic):
        url = "{}/v1/user/subscription/{}".format(self._api_root, topic)

        result = requests.post(
            url,
            headers={
                'X-User-Token': self.user_token,
            },
        )
        return result

    def unsubscribe(self, user_token, topic):
        url = "{}/v1/user/subscription/{}".format(self._api_root, topic)

        result = requests.delete(
            url,
            headers={
                'X-User-Token': self.user_token,
            },
        )
        return result
<commit_msg>fix other problems in timeline.py<commit_after>import requests

class Timeline(object):
    api_root = 'https://timeline-api.getpebble.com'

    error_codes = {
        400: 'The pin object submitted was invalid.',
        403: 'The API key submitted was invalid.',
        410: 'The user token submitted was invalid or does not exist.',
        429: 'Server is sending updates too quickly.',
        503: 'Could not save pin due to a temporary server error.',
    }

    def __init__(self, api_key=None, api_root=None):
        self._api_key = api_key

        if api_root is not None:
            self._api_root = api_root
        else:
            self._api_root = type(self).api_root

    def send_shared_pin(self, topics, pin):
        url = "{}/v1/shared/pins/{}".format(self._api_root, pin['id'])

        result = requests.put(
            url,
            headers={
                'X-API-Key': self._api_key,
                'X-PIN-Topics': ','.join(str(t) for t in topics),
            },
            json=dict(pin),
        )
        return result

    def send_user_pin(self, user_token, pin):
        url = "{}/v1/user/pins/{}".format(self._api_root, pin['id'])

        result = requests.put(
            url,
            headers={
                'X-User-Token': user_token,
            },
            json=dict(pin),
        )
        return result

    def delete_user_pin(self, user_token, pin):
        url = "{}/v1/user/pins/{}".format(self._api_root, pin['id'])

        result = requests.delete(
            url,
            headers={
                'X-User-Token': user_token,
            },
        )
        return result

    def subscribe(self, user_token, topic):
        url = "{}/v1/user/subscription/{}".format(self._api_root, topic)

        result = requests.post(
            url,
            headers={
                'X-User-Token': user_token,
            },
        )
        return result

    def unsubscribe(self, user_token, topic):
        url = "{}/v1/user/subscription/{}".format(self._api_root, topic)

        result = requests.delete(
            url,
            headers={
                'X-User-Token': user_token,
            },
        )
        return result
<|endoftext|>